2021-02-17 12:09:50 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
|
|
|
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
|
2021-09-09 13:12:32 +08:00
|
|
|
#include <linux/io-64-nonatomic-lo-hi.h>
|
2022-02-01 07:51:45 +08:00
|
|
|
#include <linux/moduleparam.h>
|
2021-02-17 12:09:50 +08:00
|
|
|
#include <linux/module.h>
|
2022-02-01 07:51:45 +08:00
|
|
|
#include <linux/delay.h>
|
2021-04-17 08:43:30 +08:00
|
|
|
#include <linux/sizes.h>
|
2021-02-17 12:09:52 +08:00
|
|
|
#include <linux/mutex.h>
|
2021-06-04 08:50:36 +08:00
|
|
|
#include <linux/list.h>
|
2021-02-17 12:09:50 +08:00
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/io.h>
|
2021-08-03 01:29:38 +08:00
|
|
|
#include "cxlmem.h"
|
2022-01-24 08:30:25 +08:00
|
|
|
#include "cxlpci.h"
|
2021-02-17 12:09:51 +08:00
|
|
|
#include "cxl.h"
|
|
|
|
|
|
|
|
/**
|
cxl: Rename mem to pci
As the driver has undergone development, it's become clear that the
majority [entirety?] of the current functionality in mem.c is actually a
layer encapsulating functionality exposed through PCI based
interactions. This layer can be used either in isolation or to provide
functionality for higher level functionality.
CXL capabilities exist in a parallel domain to PCIe. CXL devices are
enumerable and controllable via "legacy" PCIe mechanisms; however, their
CXL capabilities are a superset of PCIe. For example, a CXL device may
be connected to a non-CXL capable PCIe root port, and therefore will not
be able to participate in CXL.mem or CXL.cache operations, but can still
be accessed through PCIe mechanisms for CXL.io operations.
To properly represent the PCI nature of this driver, and in preparation for
introducing a new driver for the CXL.mem / HDM decoder (Host-managed Device
Memory) capabilities of a CXL memory expander, rename mem.c to pci.c so that
mem.c is available for this new driver.
The result of the change is that there is a clear layering distinction
in the driver, and a systems administrator may load only the cxl_pci
module and gain access to such operations as, firmware update, offline
provisioning of devices, and error collection. In addition to freeing up
the file name for another purpose, there are two primary reasons this is
useful,
1. Acting upon devices which don't have full CXL capabilities. This
may happen for instance if the CXL device is connected in a CXL
unaware part of the platform topology.
2. Userspace-first provisioning for devices without kernel driver
interference. This may be useful when provisioning a new device
in a specific manner that might otherwise be blocked or prevented
by the real CXL mem driver.
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
Link: https://lore.kernel.org/r/20210526174413.802913-1-ben.widawsky@intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-27 01:44:13 +08:00
|
|
|
* DOC: cxl pci
|
2021-02-17 12:09:51 +08:00
|
|
|
*
|
cxl: Rename mem to pci
As the driver has undergone development, it's become clear that the
majority [entirety?] of the current functionality in mem.c is actually a
layer encapsulating functionality exposed through PCI based
interactions. This layer can be used either in isolation or to provide
functionality for higher level functionality.
CXL capabilities exist in a parallel domain to PCIe. CXL devices are
enumerable and controllable via "legacy" PCIe mechanisms; however, their
CXL capabilities are a superset of PCIe. For example, a CXL device may
be connected to a non-CXL capable PCIe root port, and therefore will not
be able to participate in CXL.mem or CXL.cache operations, but can still
be accessed through PCIe mechanisms for CXL.io operations.
To properly represent the PCI nature of this driver, and in preparation for
introducing a new driver for the CXL.mem / HDM decoder (Host-managed Device
Memory) capabilities of a CXL memory expander, rename mem.c to pci.c so that
mem.c is available for this new driver.
The result of the change is that there is a clear layering distinction
in the driver, and a systems administrator may load only the cxl_pci
module and gain access to such operations as, firmware update, offline
provisioning of devices, and error collection. In addition to freeing up
the file name for another purpose, there are two primary reasons this is
useful,
1. Acting upon devices which don't have full CXL capabilities. This
may happen for instance if the CXL device is connected in a CXL
unaware part of the platform topology.
2. Userspace-first provisioning for devices without kernel driver
interference. This may be useful when provisioning a new device
in a specific manner that might otherwise be blocked or prevented
by the real CXL mem driver.
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
Link: https://lore.kernel.org/r/20210526174413.802913-1-ben.widawsky@intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-27 01:44:13 +08:00
|
|
|
* This implements the PCI exclusive functionality for a CXL device as it is
|
|
|
|
* defined by the Compute Express Link specification. CXL devices may surface
|
2021-09-14 00:33:24 +08:00
|
|
|
* certain functionality even if it isn't CXL enabled. While this driver is
|
|
|
|
* focused around the PCI specific aspects of a CXL device, it binds to the
|
|
|
|
* specific CXL memory device class code, and therefore the implementation of
|
|
|
|
* cxl_pci is focused around CXL memory devices.
|
2021-02-17 12:09:51 +08:00
|
|
|
*
|
|
|
|
* The driver has several responsibilities, mainly:
|
|
|
|
* - Create the memX device and register on the CXL bus.
|
|
|
|
* - Enumerate device's register interface and map them.
|
2021-09-14 00:33:24 +08:00
|
|
|
* - Registers nvdimm bridge device with cxl_core.
|
|
|
|
* - Registers a CXL mailbox with cxl_core.
|
2021-02-17 12:09:51 +08:00
|
|
|
*/
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
#define cxl_doorbell_busy(cxlds) \
|
|
|
|
(readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \
|
2021-02-17 12:09:51 +08:00
|
|
|
CXLDEV_MBOX_CTRL_DOORBELL)
|
|
|
|
|
|
|
|
/* CXL 2.0 - 8.2.8.4 */
|
|
|
|
#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
|
|
|
|
|
2022-02-01 07:51:45 +08:00
|
|
|
/*
|
|
|
|
* CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
|
|
|
|
* dictate how long to wait for the mailbox to become ready. The new
|
|
|
|
* field allows the device to tell software the amount of time to wait
|
|
|
|
* before mailbox ready. This field per the spec theoretically allows
|
|
|
|
* for up to 255 seconds. 255 seconds is unreasonably long, its longer
|
|
|
|
* than the maximum SATA port link recovery wait. Default to 60 seconds
|
|
|
|
* until someone builds a CXL device that needs more time in practice.
|
|
|
|
*/
|
|
|
|
static unsigned short mbox_ready_timeout = 60;
|
|
|
|
module_param(mbox_ready_timeout, ushort, 0644);
|
|
|
|
MODULE_PARM_DESC(mbox_ready_timeout,
|
2022-01-24 08:31:13 +08:00
|
|
|
"seconds to wait for mailbox ready / memory active status");
|
2022-02-01 07:51:45 +08:00
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
|
|
|
const unsigned long start = jiffies;
|
|
|
|
unsigned long end = start;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
while (cxl_doorbell_busy(cxlds)) {
|
2021-02-17 12:09:51 +08:00
|
|
|
end = jiffies;
|
|
|
|
|
|
|
|
if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
|
|
|
|
/* Check again in case preempted before timeout test */
|
2021-11-03 04:29:01 +08:00
|
|
|
if (!cxl_doorbell_busy(cxlds))
|
2021-02-17 12:09:51 +08:00
|
|
|
break;
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
dev_dbg(cxlds->dev, "Doorbell wait took %dms",
|
2021-02-17 12:09:51 +08:00
|
|
|
jiffies_to_msecs(end) - jiffies_to_msecs(start));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-01-24 08:28:54 +08:00
|
|
|
#define cxl_err(dev, status, msg) \
|
|
|
|
dev_err_ratelimited(dev, msg ", device state %s%s\n", \
|
|
|
|
status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
|
|
|
|
status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
|
2021-02-17 12:09:51 +08:00
|
|
|
|
2022-01-24 08:28:54 +08:00
|
|
|
#define cxl_cmd_err(dev, cmd, status, msg) \
|
|
|
|
dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \
|
|
|
|
(cmd)->opcode, \
|
|
|
|
status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
|
|
|
|
status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
/**
|
2021-09-14 00:33:24 +08:00
|
|
|
* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
|
2021-11-03 04:29:01 +08:00
|
|
|
* @cxlds: The device state to communicate with.
|
2021-02-17 12:09:51 +08:00
|
|
|
* @mbox_cmd: Command to send to the memory device.
|
|
|
|
*
|
|
|
|
* Context: Any context. Expects mbox_mutex to be held.
|
|
|
|
* Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
|
|
|
|
* Caller should check the return code in @mbox_cmd to make sure it
|
|
|
|
* succeeded.
|
|
|
|
*
|
|
|
|
* This is a generic form of the CXL mailbox send command thus only using the
|
|
|
|
* registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
|
|
|
|
* devices, and perhaps other types of CXL devices may have further information
|
|
|
|
* available upon error conditions. Driver facilities wishing to send mailbox
|
|
|
|
* commands should use the wrapper command.
|
|
|
|
*
|
|
|
|
* The CXL spec allows for up to two mailboxes. The intention is for the primary
|
|
|
|
* mailbox to be OS controlled and the secondary mailbox to be used by system
|
|
|
|
* firmware. This allows the OS and firmware to communicate with the device and
|
|
|
|
* not need to coordinate with each other. The driver only uses the primary
|
|
|
|
* mailbox.
|
|
|
|
*/
|
2021-11-03 04:29:01 +08:00
|
|
|
static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
|
2021-09-09 13:12:21 +08:00
|
|
|
struct cxl_mbox_cmd *mbox_cmd)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
2021-11-03 04:29:01 +08:00
|
|
|
void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
|
|
|
|
struct device *dev = cxlds->dev;
|
2021-02-17 12:09:51 +08:00
|
|
|
u64 cmd_reg, status_reg;
|
|
|
|
size_t out_len;
|
|
|
|
int rc;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
lockdep_assert_held(&cxlds->mbox_mutex);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
|
|
|
|
* 1. Caller reads MB Control Register to verify doorbell is clear
|
|
|
|
* 2. Caller writes Command Register
|
|
|
|
* 3. Caller writes Command Payload Registers if input payload is non-empty
|
|
|
|
* 4. Caller writes MB Control Register to set doorbell
|
|
|
|
* 5. Caller either polls for doorbell to be clear or waits for interrupt if configured
|
|
|
|
* 6. Caller reads MB Status Register to fetch Return code
|
|
|
|
* 7. If command successful, Caller reads Command Register to get Payload Length
|
|
|
|
* 8. If output payload is non-empty, host reads Command Payload Registers
|
|
|
|
*
|
|
|
|
* Hardware is free to do whatever it wants before the doorbell is rung,
|
|
|
|
* and isn't allowed to change anything after it clears the doorbell. As
|
|
|
|
* such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can
|
|
|
|
* also happen in any order (though some orders might not make sense).
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* #1 */
|
2021-11-03 04:29:01 +08:00
|
|
|
if (cxl_doorbell_busy(cxlds)) {
|
2022-01-24 08:28:54 +08:00
|
|
|
u64 md_status =
|
|
|
|
readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
|
|
|
|
|
|
|
|
cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,
|
|
|
|
"mailbox queue busy");
|
2021-02-17 12:09:51 +08:00
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
|
|
|
cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
|
|
|
|
mbox_cmd->opcode);
|
|
|
|
if (mbox_cmd->size_in) {
|
|
|
|
if (WARN_ON(!mbox_cmd->payload_in))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK,
|
|
|
|
mbox_cmd->size_in);
|
|
|
|
memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* #2, #3 */
|
2021-11-03 04:29:01 +08:00
|
|
|
writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
/* #4 */
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_dbg(dev, "Sending command\n");
|
2021-02-17 12:09:51 +08:00
|
|
|
writel(CXLDEV_MBOX_CTRL_DOORBELL,
|
2021-11-03 04:29:01 +08:00
|
|
|
cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
/* #5 */
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc == -ETIMEDOUT) {
|
2022-01-24 08:28:54 +08:00
|
|
|
u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
|
|
|
|
|
|
|
|
cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");
|
2021-02-17 12:09:51 +08:00
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* #6 */
|
2021-11-03 04:29:01 +08:00
|
|
|
status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
mbox_cmd->return_code =
|
|
|
|
FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
|
|
|
|
|
2022-04-04 10:12:15 +08:00
|
|
|
if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) {
|
2022-04-04 10:12:16 +08:00
|
|
|
dev_dbg(dev, "Mailbox operation had an error: %s\n",
|
|
|
|
cxl_mbox_cmd_rc2str(mbox_cmd));
|
2022-04-04 10:12:14 +08:00
|
|
|
return 0; /* completed but caller must check return_code */
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* #7 */
|
2021-11-03 04:29:01 +08:00
|
|
|
cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);
|
|
|
|
|
|
|
|
/* #8 */
|
|
|
|
if (out_len && mbox_cmd->payload_out) {
|
|
|
|
/*
|
|
|
|
* Sanitize the copy. If hardware misbehaves, out_len per the
|
|
|
|
* spec can actually be greater than the max allowed size (21
|
|
|
|
* bits available but spec defined 1M max). The caller also may
|
|
|
|
* have requested less data than the hardware supplied even
|
|
|
|
* within spec.
|
|
|
|
*/
|
2021-11-03 04:29:01 +08:00
|
|
|
size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
memcpy_fromio(mbox_cmd->payload_out, payload, n);
|
|
|
|
mbox_cmd->size_out = n;
|
|
|
|
} else {
|
|
|
|
mbox_cmd->size_out = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
|
2021-09-09 13:12:21 +08:00
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
2022-01-24 08:28:54 +08:00
|
|
|
mutex_lock_io(&cxlds->mbox_mutex);
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);
|
2022-01-24 08:28:54 +08:00
|
|
|
mutex_unlock(&cxlds->mbox_mutex);
|
2021-09-09 13:12:21 +08:00
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
2021-11-03 04:29:01 +08:00
|
|
|
const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
|
2022-02-01 07:51:45 +08:00
|
|
|
unsigned long timeout;
|
|
|
|
u64 md_status;
|
|
|
|
|
|
|
|
timeout = jiffies + mbox_ready_timeout * HZ;
|
|
|
|
do {
|
|
|
|
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
|
|
|
|
if (md_status & CXLMDEV_MBOX_IF_READY)
|
|
|
|
break;
|
|
|
|
if (msleep_interruptible(100))
|
|
|
|
break;
|
|
|
|
} while (!time_after(jiffies, timeout));
|
|
|
|
|
|
|
|
if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
|
2022-01-24 08:28:54 +08:00
|
|
|
cxl_err(cxlds->dev, md_status,
|
|
|
|
"timeout awaiting mailbox ready");
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A command may be in flight from a previous driver instance,
|
|
|
|
* think kexec, do one doorbell wait so that
|
|
|
|
* __cxl_pci_mbox_send_cmd() can assume that it is the only
|
|
|
|
* source for future doorbell busy events.
|
|
|
|
*/
|
|
|
|
if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
|
|
|
|
cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");
|
|
|
|
return -ETIMEDOUT;
|
2022-02-01 07:51:45 +08:00
|
|
|
}
|
2021-02-17 12:09:51 +08:00
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
cxlds->mbox_send = cxl_pci_mbox_send;
|
|
|
|
cxlds->payload_size =
|
2021-02-17 12:09:51 +08:00
|
|
|
1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register
|
|
|
|
*
|
|
|
|
* If the size is too small, mandatory commands will not work and so
|
|
|
|
* there's no point in going forward. If the size is too large, there's
|
|
|
|
* no harm is soft limiting it.
|
|
|
|
*/
|
2021-11-03 04:29:01 +08:00
|
|
|
cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M);
|
|
|
|
if (cxlds->payload_size < 256) {
|
|
|
|
dev_err(cxlds->dev, "Mailbox is too small (%zub)",
|
|
|
|
cxlds->payload_size);
|
2021-02-17 12:09:51 +08:00
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
dev_dbg(cxlds->dev, "Mailbox payload sized %zu",
|
|
|
|
cxlds->payload_size);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map)
|
2021-04-08 06:26:20 +08:00
|
|
|
{
|
2021-05-28 08:49:19 +08:00
|
|
|
void __iomem *addr;
|
2021-10-14 07:53:29 +08:00
|
|
|
int bar = map->barno;
|
|
|
|
struct device *dev = &pdev->dev;
|
|
|
|
resource_size_t offset = map->block_offset;
|
2021-04-08 06:26:20 +08:00
|
|
|
|
2021-02-17 12:09:51 +08:00
|
|
|
/* Basic sanity check that BAR is big enough */
|
|
|
|
if (pci_resource_len(pdev, bar) < offset) {
|
2021-10-14 07:53:29 +08:00
|
|
|
dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar,
|
|
|
|
&pdev->resource[bar], &offset);
|
2021-10-16 05:57:27 +08:00
|
|
|
return -ENXIO;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
|
|
|
|
2021-06-04 08:50:36 +08:00
|
|
|
addr = pci_iomap(pdev, bar, 0);
|
2021-05-28 08:49:19 +08:00
|
|
|
if (!addr) {
|
2021-02-17 12:09:51 +08:00
|
|
|
dev_err(dev, "failed to map registers\n");
|
2021-10-16 05:57:27 +08:00
|
|
|
return -ENOMEM;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
|
|
|
|
2021-10-14 07:53:29 +08:00
|
|
|
dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n",
|
|
|
|
bar, &offset);
|
2021-05-21 05:29:53 +08:00
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
map->base = addr + map->block_offset;
|
|
|
|
return 0;
|
2021-06-04 08:50:36 +08:00
|
|
|
}
|
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static void cxl_unmap_regblock(struct pci_dev *pdev,
|
|
|
|
struct cxl_register_map *map)
|
2021-06-04 08:50:36 +08:00
|
|
|
{
|
2021-10-16 05:57:27 +08:00
|
|
|
pci_iounmap(pdev, map->base - map->block_offset);
|
|
|
|
map->base = NULL;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
2021-02-17 12:09:50 +08:00
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map)
|
2021-06-04 08:50:36 +08:00
|
|
|
{
|
2021-05-28 08:49:22 +08:00
|
|
|
struct cxl_component_reg_map *comp_map;
|
2021-06-04 08:50:36 +08:00
|
|
|
struct cxl_device_reg_map *dev_map;
|
2021-10-14 07:53:29 +08:00
|
|
|
struct device *dev = &pdev->dev;
|
2021-10-16 05:57:27 +08:00
|
|
|
void __iomem *base = map->base;
|
2021-06-04 08:50:36 +08:00
|
|
|
|
|
|
|
switch (map->reg_type) {
|
2021-05-28 08:49:22 +08:00
|
|
|
case CXL_REGLOC_RBI_COMPONENT:
|
|
|
|
comp_map = &map->component_map;
|
|
|
|
cxl_probe_component_regs(dev, base, comp_map);
|
|
|
|
if (!comp_map->hdm_decoder.valid) {
|
|
|
|
dev_err(dev, "HDM decoder registers not found\n");
|
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev_dbg(dev, "Set up component registers\n");
|
|
|
|
break;
|
2021-06-04 08:50:36 +08:00
|
|
|
case CXL_REGLOC_RBI_MEMDEV:
|
|
|
|
dev_map = &map->device_map;
|
|
|
|
cxl_probe_device_regs(dev, base, dev_map);
|
|
|
|
if (!dev_map->status.valid || !dev_map->mbox.valid ||
|
|
|
|
!dev_map->memdev.valid) {
|
|
|
|
dev_err(dev, "registers not found: %s%s%s\n",
|
|
|
|
!dev_map->status.valid ? "status " : "",
|
2021-09-04 10:20:50 +08:00
|
|
|
!dev_map->mbox.valid ? "mbox " : "",
|
|
|
|
!dev_map->memdev.valid ? "memdev " : "");
|
2021-06-04 08:50:36 +08:00
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev_dbg(dev, "Probing device registers...\n");
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
static int cxl_map_regs(struct cxl_dev_state *cxlds, struct cxl_register_map *map)
|
2021-06-04 08:50:36 +08:00
|
|
|
{
|
2021-11-03 04:29:01 +08:00
|
|
|
struct device *dev = cxlds->dev;
|
2021-09-09 13:12:09 +08:00
|
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
2021-06-04 08:50:36 +08:00
|
|
|
|
|
|
|
switch (map->reg_type) {
|
2021-05-28 08:49:22 +08:00
|
|
|
case CXL_REGLOC_RBI_COMPONENT:
|
2021-11-03 04:29:01 +08:00
|
|
|
cxl_map_component_regs(pdev, &cxlds->regs.component, map);
|
2021-05-28 08:49:22 +08:00
|
|
|
dev_dbg(dev, "Mapping component registers...\n");
|
|
|
|
break;
|
2021-06-04 08:50:36 +08:00
|
|
|
case CXL_REGLOC_RBI_MEMDEV:
|
2021-11-03 04:29:01 +08:00
|
|
|
cxl_map_device_regs(pdev, &cxlds->regs.device_regs, map);
|
2021-06-04 08:50:36 +08:00
|
|
|
dev_dbg(dev, "Probing device registers...\n");
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
|
|
|
|
struct cxl_register_map *map)
|
|
|
|
{
|
|
|
|
int rc;
|
2021-07-17 07:15:47 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_find_regblock(pdev, type, map);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_map_regblock(pdev, map);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
rc = cxl_probe_regs(pdev, map);
|
|
|
|
cxl_unmap_regblock(pdev, map);
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
return rc;
|
2021-04-08 06:26:21 +08:00
|
|
|
}
|
|
|
|
|
2022-02-02 07:48:56 +08:00
|
|
|
static int wait_for_valid(struct cxl_dev_state *cxlds)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
|
|
|
|
int d = cxlds->cxl_dvsec, rc;
|
|
|
|
u32 val;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high
|
|
|
|
* and Size Low registers are valid. Must be set within 1 second of
|
|
|
|
* deassertion of reset to CXL device. Likely it is already set by the
|
|
|
|
* time this runs, but otherwise give a 1.5 second timeout in case of
|
|
|
|
* clock skew.
|
|
|
|
*/
|
|
|
|
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
if (val & CXL_DVSEC_MEM_INFO_VALID)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
msleep(1500);
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
if (val & CXL_DVSEC_MEM_INFO_VALID)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
}
|
|
|
|
|
2022-01-24 08:31:13 +08:00
|
|
|
/*
|
|
|
|
* Wait up to @mbox_ready_timeout for the device to report memory
|
|
|
|
* active.
|
|
|
|
*/
|
|
|
|
static int wait_for_media_ready(struct cxl_dev_state *cxlds)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
|
|
|
|
int d = cxlds->cxl_dvsec;
|
|
|
|
bool active = false;
|
|
|
|
u64 md_status;
|
|
|
|
int rc, i;
|
|
|
|
|
|
|
|
rc = wait_for_valid(cxlds);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
for (i = mbox_ready_timeout; i; i--) {
|
|
|
|
u32 temp;
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(
|
|
|
|
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
|
|
|
|
if (active)
|
|
|
|
break;
|
|
|
|
msleep(1000);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!active) {
|
|
|
|
dev_err(&pdev->dev,
|
|
|
|
"timeout awaiting memory active after %d seconds\n",
|
|
|
|
mbox_ready_timeout);
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
}
|
|
|
|
|
|
|
|
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
|
|
|
|
if (!CXLMDEV_READY(md_status))
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-02-02 07:48:56 +08:00
|
|
|
static int cxl_dvsec_ranges(struct cxl_dev_state *cxlds)
|
|
|
|
{
|
|
|
|
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
|
|
|
|
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
|
|
|
|
int d = cxlds->cxl_dvsec;
|
|
|
|
int hdm_count, rc, i;
|
|
|
|
u16 cap, ctrl;
|
|
|
|
|
|
|
|
if (!d)
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
if (!(cap & CXL_DVSEC_MEM_CAPABLE))
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It is not allowed by spec for MEM.capable to be set and have 0 legacy
|
|
|
|
* HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
|
|
|
|
* driver is for a spec defined class code which must be CXL.mem
|
|
|
|
* capable, there is no point in continuing to enable CXL.mem.
|
|
|
|
*/
|
|
|
|
hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
|
|
|
|
if (!hdm_count || hdm_count > 2)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
rc = wait_for_valid(cxlds);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
|
|
|
|
|
|
|
|
for (i = 0; i < hdm_count; i++) {
|
|
|
|
u64 base, size;
|
|
|
|
u32 temp;
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(
|
|
|
|
pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
size = (u64)temp << 32;
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(
|
|
|
|
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(
|
|
|
|
pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
base = (u64)temp << 32;
|
|
|
|
|
|
|
|
rc = pci_read_config_dword(
|
|
|
|
pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
|
|
|
|
|
|
|
|
info->dvsec_range[i] = (struct range) {
|
|
|
|
.start = base,
|
|
|
|
.end = base + size - 1
|
|
|
|
};
|
|
|
|
|
|
|
|
if (size)
|
|
|
|
info->ranges++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
2021-02-17 12:09:50 +08:00
|
|
|
{
|
2021-10-16 07:30:42 +08:00
|
|
|
struct cxl_register_map map;
|
2021-06-16 07:36:31 +08:00
|
|
|
struct cxl_memdev *cxlmd;
|
2021-11-03 04:29:01 +08:00
|
|
|
struct cxl_dev_state *cxlds;
|
2021-04-08 06:26:21 +08:00
|
|
|
int rc;
|
2021-02-17 12:09:51 +08:00
|
|
|
|
2021-09-09 13:12:38 +08:00
|
|
|
/*
|
|
|
|
* Double check the anonymous union trickery in struct cxl_regs
|
|
|
|
* FIXME switch to struct_group()
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
|
|
|
|
offsetof(struct cxl_regs, device_regs.memdev));
|
|
|
|
|
2021-02-17 12:09:51 +08:00
|
|
|
rc = pcim_enable_device(pdev);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
2021-02-17 12:09:50 +08:00
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
cxlds = cxl_dev_state_create(&pdev->dev);
|
|
|
|
if (IS_ERR(cxlds))
|
|
|
|
return PTR_ERR(cxlds);
|
2021-04-08 06:26:20 +08:00
|
|
|
|
2022-02-01 05:56:11 +08:00
|
|
|
cxlds->serial = pci_get_dsn(pdev);
|
2022-02-02 06:06:32 +08:00
|
|
|
cxlds->cxl_dvsec = pci_find_dvsec_capability(
|
|
|
|
pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
|
|
|
|
if (!cxlds->cxl_dvsec)
|
|
|
|
dev_warn(&pdev->dev,
|
|
|
|
"Device DVSEC not present, skip CXL.mem init\n");
|
|
|
|
|
2022-01-24 08:31:13 +08:00
|
|
|
cxlds->wait_media_ready = wait_for_media_ready;
|
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_map_regs(cxlds, &map);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2022-02-02 05:28:53 +08:00
|
|
|
/*
|
|
|
|
* If the component registers can't be found, the cxl_pci driver may
|
|
|
|
* still be useful for management functions so don't return an error.
|
|
|
|
*/
|
|
|
|
cxlds->component_reg_phys = CXL_RESOURCE_NONE;
|
|
|
|
rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
|
|
|
|
if (rc)
|
|
|
|
dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
|
|
|
|
|
|
|
|
cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_pci_setup_mailbox(cxlds);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_enumerate_cmds(cxlds);
|
2021-02-17 12:09:55 +08:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_dev_state_identify(cxlds);
|
2021-02-17 12:09:52 +08:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
rc = cxl_mem_create_range_info(cxlds);
|
2021-08-11 02:57:59 +08:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2022-02-02 07:48:56 +08:00
|
|
|
rc = cxl_dvsec_ranges(cxlds);
|
|
|
|
if (rc)
|
|
|
|
dev_warn(&pdev->dev,
|
|
|
|
"Failed to get DVSEC range information (%d)\n", rc);
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
cxlmd = devm_cxl_add_memdev(cxlds);
|
2021-06-16 07:36:31 +08:00
|
|
|
if (IS_ERR(cxlmd))
|
|
|
|
return PTR_ERR(cxlmd);
|
|
|
|
|
2021-11-03 04:29:01 +08:00
|
|
|
if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
|
2021-06-16 07:36:31 +08:00
|
|
|
rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);
|
|
|
|
|
|
|
|
return rc;
|
2021-02-17 12:09:50 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct pci_device_id cxl_mem_pci_tbl[] = {
|
|
|
|
/* PCI class code for CXL.mem Type-3 Devices */
|
|
|
|
{ PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)},
|
|
|
|
{ /* terminate list */ },
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static struct pci_driver cxl_pci_driver = {
|
2021-02-17 12:09:50 +08:00
|
|
|
.name = KBUILD_MODNAME,
|
|
|
|
.id_table = cxl_mem_pci_tbl,
|
2021-09-14 00:33:24 +08:00
|
|
|
.probe = cxl_pci_probe,
|
2021-02-17 12:09:50 +08:00
|
|
|
.driver = {
|
|
|
|
.probe_type = PROBE_PREFER_ASYNCHRONOUS,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
MODULE_LICENSE("GPL v2");
|
2021-09-14 00:33:24 +08:00
|
|
|
module_pci_driver(cxl_pci_driver);
|
2021-02-17 12:09:52 +08:00
|
|
|
MODULE_IMPORT_NS(CXL);
|