dmaengine: ptdma: Initial driver for the AMD PTDMA
Add support for AMD PTDMA controller. It performs high-bandwidth memory to memory and IO copy operation. Device commands are managed via a circular queue of 'descriptors', each of which specifies source and destination addresses for copying a single buffer of data. Signed-off-by: Sanjay R Mehta <sanju.mehta@amd.com> Link: https://lore.kernel.org/r/1629208559-51964-2-git-send-email-Sanju.Mehta@amd.com Signed-off-by: Vinod Koul <vkoul@kernel.org>
This commit is contained in:
Sanjay R Mehta
Vinod Koul
parent
64d57d2c64
commit
fa5d823b16
|
|
@ -979,6 +979,12 @@ S: Supported
|
|||
T: git https://gitlab.freedesktop.org/agd5f/linux.git
|
||||
F: drivers/gpu/drm/amd/pm/powerplay/
|
||||
|
||||
+AMD PTDMA DRIVER
|
||||
+M: Sanjay R Mehta <sanju.mehta@amd.com>
|
||||
+L: dmaengine@vger.kernel.org
|
||||
+S: Maintained
|
||||
+F: drivers/dma/ptdma/
|
||||
|
||||
AMD SEATTLE DEVICE TREE SUPPORT
|
||||
M: Brijesh Singh <brijeshkumar.singh@amd.com>
|
||||
M: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
|
||||
|
|
|
|||
|
|
@ -738,6 +738,8 @@ source "drivers/dma/bestcomm/Kconfig"
|
|||
|
||||
source "drivers/dma/mediatek/Kconfig"
|
||||
|
||||
source "drivers/dma/ptdma/Kconfig"
|
||||
|
||||
source "drivers/dma/qcom/Kconfig"
|
||||
|
||||
source "drivers/dma/dw/Kconfig"
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ obj-$(CONFIG_DMATEST) += dmatest.o
|
|||
obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o
|
||||
obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o
|
||||
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
|
||||
obj-$(CONFIG_AMD_PTDMA) += ptdma/
|
||||
obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
|
||||
obj-$(CONFIG_AT_XDMAC) += at_xdmac.o
|
||||
obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
config AMD_PTDMA
|
||||
tristate "AMD PassThru DMA Engine"
|
||||
depends on X86_64 && PCI
|
||||
help
|
||||
Enable support for the AMD PTDMA controller. This controller
|
||||
provides DMA capabilities to perform high bandwidth memory to
|
||||
memory and IO copy operations. It performs DMA transfer through
|
||||
queue-based descriptor management. This DMA controller is intended
|
||||
to be used with AMD Non-Transparent Bridge devices and not for
|
||||
general purpose peripheral DMA.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
# AMD Passthru DMA driver
|
||||
#
|
||||
|
||||
obj-$(CONFIG_AMD_PTDMA) += ptdma.o
|
||||
|
||||
ptdma-objs := ptdma-dev.o
|
||||
|
||||
ptdma-$(CONFIG_PCI) += ptdma-pci.o
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* AMD Passthru DMA device driver
|
||||
* -- Based on the CCP driver
|
||||
*
|
||||
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
|
||||
* Author: Gary R Hook <gary.hook@amd.com>
|
||||
*/
|
||||
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/debugfs.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/pci.h>
|
||||
|
||||
#include "ptdma.h"
|
||||
|
||||
/* Human-readable error strings */
|
||||
static char *pt_error_codes[] = {
|
||||
"",
|
||||
"ERR 01: ILLEGAL_ENGINE",
|
||||
"ERR 03: ILLEGAL_FUNCTION_TYPE",
|
||||
"ERR 04: ILLEGAL_FUNCTION_MODE",
|
||||
"ERR 06: ILLEGAL_FUNCTION_SIZE",
|
||||
"ERR 08: ILLEGAL_FUNCTION_RSVD",
|
||||
"ERR 09: ILLEGAL_BUFFER_LENGTH",
|
||||
"ERR 10: VLSB_FAULT",
|
||||
"ERR 11: ILLEGAL_MEM_ADDR",
|
||||
"ERR 12: ILLEGAL_MEM_SEL",
|
||||
"ERR 13: ILLEGAL_CONTEXT_ID",
|
||||
"ERR 15: 0xF Reserved",
|
||||
"ERR 18: CMD_TIMEOUT",
|
||||
"ERR 19: IDMA0_AXI_SLVERR",
|
||||
"ERR 20: IDMA0_AXI_DECERR",
|
||||
"ERR 21: 0x15 Reserved",
|
||||
"ERR 22: IDMA1_AXI_SLAVE_FAULT",
|
||||
"ERR 23: IDMA1_AIXI_DECERR",
|
||||
"ERR 24: 0x18 Reserved",
|
||||
"ERR 27: 0x1B Reserved",
|
||||
"ERR 38: ODMA0_AXI_SLVERR",
|
||||
"ERR 39: ODMA0_AXI_DECERR",
|
||||
"ERR 40: 0x28 Reserved",
|
||||
"ERR 41: ODMA1_AXI_SLVERR",
|
||||
"ERR 42: ODMA1_AXI_DECERR",
|
||||
"ERR 43: LSB_PARITY_ERR",
|
||||
};
|
||||
|
||||
static void pt_log_error(struct pt_device *d, int e)
|
||||
{
|
||||
dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
|
||||
}
|
||||
|
||||
void pt_start_queue(struct pt_cmd_queue *cmd_q)
|
||||
{
|
||||
/* Turn on the run bit */
|
||||
iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
|
||||
}
|
||||
|
||||
void pt_stop_queue(struct pt_cmd_queue *cmd_q)
|
||||
{
|
||||
/* Turn off the run bit */
|
||||
iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
|
||||
}
|
||||
|
||||
static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
|
||||
{
|
||||
bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
|
||||
u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
|
||||
u32 tail;
|
||||
|
||||
if (soc) {
|
||||
desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
|
||||
desc->dw0 &= ~DWORD0_SOC;
|
||||
}
|
||||
mutex_lock(&cmd_q->q_mutex);
|
||||
|
||||
/* Copy 32-byte command descriptor to hw queue. */
|
||||
memcpy(q_desc, desc, 32);
|
||||
cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
|
||||
|
||||
/* The data used by this command must be flushed to memory */
|
||||
wmb();
|
||||
|
||||
/* Write the new tail address back to the queue register */
|
||||
tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
|
||||
iowrite32(tail, cmd_q->reg_control + 0x0004);
|
||||
|
||||
/* Turn the queue back on using our cached control register */
|
||||
pt_start_queue(cmd_q);
|
||||
mutex_unlock(&cmd_q->q_mutex);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
|
||||
struct pt_passthru_engine *pt_engine)
|
||||
{
|
||||
struct ptdma_desc desc;
|
||||
|
||||
cmd_q->cmd_error = 0;
|
||||
memset(&desc, 0, sizeof(desc));
|
||||
desc.dw0 = CMD_DESC_DW0_VAL;
|
||||
desc.length = pt_engine->src_len;
|
||||
desc.src_lo = lower_32_bits(pt_engine->src_dma);
|
||||
desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
|
||||
desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
|
||||
desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
|
||||
|
||||
return pt_core_execute_cmd(&desc, cmd_q);
|
||||
}
|
||||
|
||||
static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
|
||||
{
|
||||
iowrite32(0, pt->cmd_q.reg_control + 0x000C);
|
||||
}
|
||||
|
||||
static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
|
||||
{
|
||||
iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
|
||||
}
|
||||
|
||||
static irqreturn_t pt_core_irq_handler(int irq, void *data)
|
||||
{
|
||||
struct pt_device *pt = data;
|
||||
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
|
||||
u32 status;
|
||||
|
||||
pt_core_disable_queue_interrupts(pt);
|
||||
status = ioread32(cmd_q->reg_control + 0x0010);
|
||||
if (status) {
|
||||
cmd_q->int_status = status;
|
||||
cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
|
||||
cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
|
||||
|
||||
/* On error, only save the first error value */
|
||||
if ((status & INT_ERROR) && !cmd_q->cmd_error)
|
||||
cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
|
||||
|
||||
/* Acknowledge the interrupt */
|
||||
iowrite32(status, cmd_q->reg_control + 0x0010);
|
||||
pt_core_enable_queue_interrupts(pt);
|
||||
}
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
int pt_core_init(struct pt_device *pt)
|
||||
{
|
||||
char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
|
||||
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
|
||||
u32 dma_addr_lo, dma_addr_hi;
|
||||
struct device *dev = pt->dev;
|
||||
struct dma_pool *dma_pool;
|
||||
int ret;
|
||||
|
||||
/* Allocate a dma pool for the queue */
|
||||
snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
|
||||
|
||||
dma_pool = dma_pool_create(dma_pool_name, dev,
|
||||
PT_DMAPOOL_MAX_SIZE,
|
||||
PT_DMAPOOL_ALIGN, 0);
|
||||
if (!dma_pool)
|
||||
return -ENOMEM;
|
||||
|
||||
/* ptdma core initialisation */
|
||||
iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
|
||||
iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
|
||||
iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
|
||||
iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
|
||||
iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
|
||||
|
||||
cmd_q->pt = pt;
|
||||
cmd_q->dma_pool = dma_pool;
|
||||
mutex_init(&cmd_q->q_mutex);
|
||||
|
||||
/* Page alignment satisfies our needs for N <= 128 */
|
||||
cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
|
||||
cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
|
||||
&cmd_q->qbase_dma,
|
||||
GFP_KERNEL);
|
||||
if (!cmd_q->qbase) {
|
||||
dev_err(dev, "unable to allocate command queue\n");
|
||||
ret = -ENOMEM;
|
||||
goto e_dma_alloc;
|
||||
}
|
||||
|
||||
cmd_q->qidx = 0;
|
||||
|
||||
/* Preset some register values */
|
||||
cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
|
||||
|
||||
/* Turn off the queues and disable interrupts until ready */
|
||||
pt_core_disable_queue_interrupts(pt);
|
||||
|
||||
cmd_q->qcontrol = 0; /* Start with nothing */
|
||||
iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
|
||||
|
||||
ioread32(cmd_q->reg_control + 0x0104);
|
||||
ioread32(cmd_q->reg_control + 0x0100);
|
||||
|
||||
/* Clear the interrupt status */
|
||||
iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
|
||||
|
||||
/* Request an irq */
|
||||
ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
|
||||
if (ret)
|
||||
goto e_pool;
|
||||
|
||||
/* Update the device registers with queue information. */
|
||||
cmd_q->qcontrol &= ~CMD_Q_SIZE;
|
||||
cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
|
||||
|
||||
cmd_q->qdma_tail = cmd_q->qbase_dma;
|
||||
dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
|
||||
iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
|
||||
iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
|
||||
|
||||
dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
|
||||
cmd_q->qcontrol |= (dma_addr_hi << 16);
|
||||
iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
|
||||
|
||||
pt_core_enable_queue_interrupts(pt);
|
||||
|
||||
return 0;
|
||||
|
||||
e_dma_alloc:
|
||||
dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
|
||||
|
||||
e_pool:
|
||||
dev_err(dev, "unable to allocate an IRQ\n");
|
||||
dma_pool_destroy(pt->cmd_q.dma_pool);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void pt_core_destroy(struct pt_device *pt)
|
||||
{
|
||||
struct device *dev = pt->dev;
|
||||
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
|
||||
struct pt_cmd *cmd;
|
||||
|
||||
/* Disable and clear interrupts */
|
||||
pt_core_disable_queue_interrupts(pt);
|
||||
|
||||
/* Turn off the run bit */
|
||||
pt_stop_queue(cmd_q);
|
||||
|
||||
/* Clear the interrupt status */
|
||||
iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
|
||||
ioread32(cmd_q->reg_control + 0x0104);
|
||||
ioread32(cmd_q->reg_control + 0x0100);
|
||||
|
||||
free_irq(pt->pt_irq, pt);
|
||||
|
||||
dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
|
||||
cmd_q->qbase_dma);
|
||||
|
||||
/* Flush the cmd queue */
|
||||
while (!list_empty(&pt->cmd)) {
|
||||
/* Invoke the callback directly with an error code */
|
||||
cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
|
||||
list_del(&cmd->entry);
|
||||
cmd->pt_cmd_callback(cmd->data, -ENODEV);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* AMD Passthru DMA device driver
|
||||
* -- Based on the CCP driver
|
||||
*
|
||||
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
|
||||
* Author: Tom Lendacky <thomas.lendacky@amd.com>
|
||||
* Author: Gary R Hook <gary.hook@amd.com>
|
||||
*/
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/delay.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/pci_ids.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/spinlock.h>
|
||||
|
||||
#include "ptdma.h"
|
||||
|
||||
struct pt_msix {
|
||||
int msix_count;
|
||||
struct msix_entry msix_entry;
|
||||
};
|
||||
|
||||
/*
|
||||
* pt_alloc_struct - allocate and initialize the pt_device struct
|
||||
*
|
||||
* @dev: device struct of the PTDMA
|
||||
*/
|
||||
static struct pt_device *pt_alloc_struct(struct device *dev)
|
||||
{
|
||||
struct pt_device *pt;
|
||||
|
||||
pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL);
|
||||
|
||||
if (!pt)
|
||||
return NULL;
|
||||
pt->dev = dev;
|
||||
|
||||
INIT_LIST_HEAD(&pt->cmd);
|
||||
|
||||
return pt;
|
||||
}
|
||||
|
||||
static int pt_get_msix_irqs(struct pt_device *pt)
|
||||
{
|
||||
struct pt_msix *pt_msix = pt->pt_msix;
|
||||
struct device *dev = pt->dev;
|
||||
struct pci_dev *pdev = to_pci_dev(dev);
|
||||
int ret;
|
||||
|
||||
pt_msix->msix_entry.entry = 0;
|
||||
|
||||
ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
pt_msix->msix_count = ret;
|
||||
|
||||
pt->pt_irq = pt_msix->msix_entry.vector;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int pt_get_msi_irq(struct pt_device *pt)
|
||||
{
|
||||
struct device *dev = pt->dev;
|
||||
struct pci_dev *pdev = to_pci_dev(dev);
|
||||
int ret;
|
||||
|
||||
ret = pci_enable_msi(pdev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
pt->pt_irq = pdev->irq;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int pt_get_irqs(struct pt_device *pt)
|
||||
{
|
||||
struct device *dev = pt->dev;
|
||||
int ret;
|
||||
|
||||
ret = pt_get_msix_irqs(pt);
|
||||
if (!ret)
|
||||
return 0;
|
||||
|
||||
/* Couldn't get MSI-X vectors, try MSI */
|
||||
dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
|
||||
ret = pt_get_msi_irq(pt);
|
||||
if (!ret)
|
||||
return 0;
|
||||
|
||||
/* Couldn't get MSI interrupt */
|
||||
dev_err(dev, "could not enable MSI (%d)\n", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void pt_free_irqs(struct pt_device *pt)
|
||||
{
|
||||
struct pt_msix *pt_msix = pt->pt_msix;
|
||||
struct device *dev = pt->dev;
|
||||
struct pci_dev *pdev = to_pci_dev(dev);
|
||||
|
||||
if (pt_msix->msix_count)
|
||||
pci_disable_msix(pdev);
|
||||
else if (pt->pt_irq)
|
||||
pci_disable_msi(pdev);
|
||||
|
||||
pt->pt_irq = 0;
|
||||
}
|
||||
|
||||
static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
||||
{
|
||||
struct pt_device *pt;
|
||||
struct pt_msix *pt_msix;
|
||||
struct device *dev = &pdev->dev;
|
||||
void __iomem * const *iomap_table;
|
||||
int bar_mask;
|
||||
int ret = -ENOMEM;
|
||||
|
||||
pt = pt_alloc_struct(dev);
|
||||
if (!pt)
|
||||
goto e_err;
|
||||
|
||||
pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL);
|
||||
if (!pt_msix)
|
||||
goto e_err;
|
||||
|
||||
pt->pt_msix = pt_msix;
|
||||
pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data;
|
||||
if (!pt->dev_vdata) {
|
||||
ret = -ENODEV;
|
||||
dev_err(dev, "missing driver data\n");
|
||||
goto e_err;
|
||||
}
|
||||
|
||||
ret = pcim_enable_device(pdev);
|
||||
if (ret) {
|
||||
dev_err(dev, "pcim_enable_device failed (%d)\n", ret);
|
||||
goto e_err;
|
||||
}
|
||||
|
||||
bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
|
||||
ret = pcim_iomap_regions(pdev, bar_mask, "ptdma");
|
||||
if (ret) {
|
||||
dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret);
|
||||
goto e_err;
|
||||
}
|
||||
|
||||
iomap_table = pcim_iomap_table(pdev);
|
||||
if (!iomap_table) {
|
||||
dev_err(dev, "pcim_iomap_table failed\n");
|
||||
ret = -ENOMEM;
|
||||
goto e_err;
|
||||
}
|
||||
|
||||
pt->io_regs = iomap_table[pt->dev_vdata->bar];
|
||||
if (!pt->io_regs) {
|
||||
dev_err(dev, "ioremap failed\n");
|
||||
ret = -ENOMEM;
|
||||
goto e_err;
|
||||
}
|
||||
|
||||
ret = pt_get_irqs(pt);
|
||||
if (ret)
|
||||
goto e_err;
|
||||
|
||||
pci_set_master(pdev);
|
||||
|
||||
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
|
||||
if (ret) {
|
||||
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
|
||||
if (ret) {
|
||||
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
|
||||
ret);
|
||||
goto e_err;
|
||||
}
|
||||
}
|
||||
|
||||
dev_set_drvdata(dev, pt);
|
||||
|
||||
if (pt->dev_vdata)
|
||||
ret = pt_core_init(pt);
|
||||
|
||||
if (ret)
|
||||
goto e_err;
|
||||
|
||||
return 0;
|
||||
|
||||
e_err:
|
||||
dev_err(dev, "initialization failed ret = %d\n", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void pt_pci_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct pt_device *pt = dev_get_drvdata(dev);
|
||||
|
||||
if (!pt)
|
||||
return;
|
||||
|
||||
if (pt->dev_vdata)
|
||||
pt_core_destroy(pt);
|
||||
|
||||
pt_free_irqs(pt);
|
||||
}
|
||||
|
||||
static const struct pt_dev_vdata dev_vdata[] = {
|
||||
{
|
||||
.bar = 2,
|
||||
},
|
||||
};
|
||||
|
||||
static const struct pci_device_id pt_pci_table[] = {
|
||||
{ PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] },
|
||||
/* Last entry must be zero */
|
||||
{ 0, }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, pt_pci_table);
|
||||
|
||||
static struct pci_driver pt_pci_driver = {
|
||||
.name = "ptdma",
|
||||
.id_table = pt_pci_table,
|
||||
.probe = pt_pci_probe,
|
||||
.remove = pt_pci_remove,
|
||||
};
|
||||
|
||||
module_pci_driver(pt_pci_driver);
|
||||
|
||||
MODULE_AUTHOR("Sanjay R Mehta <sanju.mehta@amd.com>");
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_DESCRIPTION("AMD PassThru DMA driver");
|
||||
|
|
@ -0,0 +1,293 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* AMD Passthru DMA device driver
|
||||
* -- Based on the CCP driver
|
||||
*
|
||||
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
|
||||
* Author: Tom Lendacky <thomas.lendacky@amd.com>
|
||||
* Author: Gary R Hook <gary.hook@amd.com>
|
||||
*/
|
||||
|
||||
#ifndef __PT_DEV_H__
|
||||
#define __PT_DEV_H__
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/dmapool.h>
|
||||
|
||||
#define MAX_PT_NAME_LEN 16
|
||||
#define MAX_DMAPOOL_NAME_LEN 32
|
||||
|
||||
#define MAX_HW_QUEUES 1
|
||||
#define MAX_CMD_QLEN 100
|
||||
|
||||
#define PT_ENGINE_PASSTHRU 5
|
||||
#define PT_OFFSET 0x0
|
||||
|
||||
/* Register Mappings */
|
||||
#define IRQ_MASK_REG 0x040
|
||||
#define IRQ_STATUS_REG 0x200
|
||||
|
||||
#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
|
||||
|
||||
#define CMD_QUEUE_PRIO_OFFSET 0x00
|
||||
#define CMD_REQID_CONFIG_OFFSET 0x04
|
||||
#define CMD_TIMEOUT_OFFSET 0x08
|
||||
#define CMD_PT_VERSION 0x10
|
||||
|
||||
#define CMD_Q_CONTROL_BASE 0x0000
|
||||
#define CMD_Q_TAIL_LO_BASE 0x0004
|
||||
#define CMD_Q_HEAD_LO_BASE 0x0008
|
||||
#define CMD_Q_INT_ENABLE_BASE 0x000C
|
||||
#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010
|
||||
|
||||
#define CMD_Q_STATUS_BASE 0x0100
|
||||
#define CMD_Q_INT_STATUS_BASE 0x0104
|
||||
#define CMD_Q_DMA_STATUS_BASE 0x0108
|
||||
#define CMD_Q_DMA_READ_STATUS_BASE 0x010C
|
||||
#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110
|
||||
#define CMD_Q_ABORT_BASE 0x0114
|
||||
#define CMD_Q_AX_CACHE_BASE 0x0118
|
||||
|
||||
#define CMD_CONFIG_OFFSET 0x1120
|
||||
#define CMD_CLK_GATE_CTL_OFFSET 0x6004
|
||||
|
||||
#define CMD_DESC_DW0_VAL 0x500012
|
||||
|
||||
/* Address offset for virtual queue registers */
|
||||
#define CMD_Q_STATUS_INCR 0x1000
|
||||
|
||||
/* Bit masks */
|
||||
#define CMD_CONFIG_REQID 0
|
||||
#define CMD_TIMEOUT_DISABLE 0
|
||||
#define CMD_CLK_DYN_GATING_DIS 0
|
||||
#define CMD_CLK_SW_GATE_MODE 0
|
||||
#define CMD_CLK_GATE_CTL 0
|
||||
#define CMD_QUEUE_PRIO GENMASK(2, 1)
|
||||
#define CMD_CONFIG_VHB_EN BIT(0)
|
||||
#define CMD_CLK_DYN_GATING_EN BIT(0)
|
||||
#define CMD_CLK_HW_GATE_MODE BIT(0)
|
||||
#define CMD_CLK_GATE_ON_DELAY BIT(12)
|
||||
#define CMD_CLK_GATE_OFF_DELAY BIT(12)
|
||||
|
||||
#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \
|
||||
CMD_CLK_HW_GATE_MODE | \
|
||||
CMD_CLK_GATE_ON_DELAY | \
|
||||
CMD_CLK_DYN_GATING_EN | \
|
||||
CMD_CLK_GATE_OFF_DELAY)
|
||||
|
||||
#define CMD_Q_LEN 32
|
||||
#define CMD_Q_RUN BIT(0)
|
||||
#define CMD_Q_HALT BIT(1)
|
||||
#define CMD_Q_MEM_LOCATION BIT(2)
|
||||
#define CMD_Q_SIZE_MASK GENMASK(4, 0)
|
||||
#define CMD_Q_SIZE GENMASK(7, 3)
|
||||
#define CMD_Q_SHIFT GENMASK(1, 0)
|
||||
#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \
|
||||
CMD_Q_SIZE_MASK)
|
||||
#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1)
|
||||
#define Q_DESC_SIZE sizeof(struct ptdma_desc)
|
||||
#define Q_SIZE(n) (CMD_Q_LEN * (n))
|
||||
|
||||
#define INT_COMPLETION BIT(0)
|
||||
#define INT_ERROR BIT(1)
|
||||
#define INT_QUEUE_STOPPED BIT(2)
|
||||
#define INT_EMPTY_QUEUE BIT(3)
|
||||
#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR)
|
||||
|
||||
/****** Local Storage Block ******/
|
||||
#define LSB_START 0
|
||||
#define LSB_END 127
|
||||
#define LSB_COUNT (LSB_END - LSB_START + 1)
|
||||
|
||||
#define PT_DMAPOOL_MAX_SIZE 64
|
||||
#define PT_DMAPOOL_ALIGN BIT(5)
|
||||
|
||||
#define PT_PASSTHRU_BLOCKSIZE 512
|
||||
|
||||
struct pt_device;
|
||||
|
||||
struct pt_tasklet_data {
|
||||
struct completion completion;
|
||||
struct pt_cmd *cmd;
|
||||
};
|
||||
|
||||
/*
|
||||
* struct pt_passthru_engine - pass-through operation
|
||||
* without performing DMA mapping
|
||||
* @mask: mask to be applied to data
|
||||
* @mask_len: length in bytes of mask
|
||||
* @src_dma: data to be used for this operation
|
||||
* @dst_dma: data produced by this operation
|
||||
* @src_len: length in bytes of data used for this operation
|
||||
*
|
||||
* Variables required to be set when calling pt_enqueue_cmd():
|
||||
* - bit_mod, byte_swap, src, dst, src_len
|
||||
* - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP
|
||||
*/
|
||||
struct pt_passthru_engine {
|
||||
dma_addr_t mask;
|
||||
u32 mask_len; /* In bytes */
|
||||
|
||||
dma_addr_t src_dma, dst_dma;
|
||||
u64 src_len; /* In bytes */
|
||||
};
|
||||
|
||||
/*
|
||||
* struct pt_cmd - PTDMA operation request
|
||||
* @entry: list element
|
||||
* @work: work element used for callbacks
|
||||
* @pt: PT device to be run on
|
||||
* @ret: operation return code
|
||||
* @flags: cmd processing flags
|
||||
* @engine: PTDMA operation to perform (passthru)
|
||||
* @engine_error: PT engine return code
|
||||
* @passthru: engine specific structures, refer to specific engine struct below
|
||||
* @callback: operation completion callback function
|
||||
* @data: parameter value to be supplied to the callback function
|
||||
*
|
||||
* Variables required to be set when calling pt_enqueue_cmd():
|
||||
* - engine, callback
|
||||
* - See the operation structures below for what is required for each
|
||||
* operation.
|
||||
*/
|
||||
struct pt_cmd {
|
||||
struct list_head entry;
|
||||
struct work_struct work;
|
||||
struct pt_device *pt;
|
||||
int ret;
|
||||
u32 engine;
|
||||
u32 engine_error;
|
||||
struct pt_passthru_engine passthru;
|
||||
/* Completion callback support */
|
||||
void (*pt_cmd_callback)(void *data, int err);
|
||||
void *data;
|
||||
};
|
||||
|
||||
struct pt_cmd_queue {
|
||||
struct pt_device *pt;
|
||||
|
||||
/* Queue dma pool */
|
||||
struct dma_pool *dma_pool;
|
||||
|
||||
/* Queue base address (not neccessarily aligned)*/
|
||||
struct ptdma_desc *qbase;
|
||||
|
||||
/* Aligned queue start address (per requirement) */
|
||||
struct mutex q_mutex ____cacheline_aligned;
|
||||
unsigned int qidx;
|
||||
|
||||
unsigned int qsize;
|
||||
dma_addr_t qbase_dma;
|
||||
dma_addr_t qdma_tail;
|
||||
|
||||
unsigned int active;
|
||||
unsigned int suspended;
|
||||
|
||||
/* Register addresses for queue */
|
||||
void __iomem *reg_control;
|
||||
u32 qcontrol; /* Cached control register */
|
||||
|
||||
/* Status values from job */
|
||||
u32 int_status;
|
||||
u32 q_status;
|
||||
u32 q_int_status;
|
||||
u32 cmd_error;
|
||||
} ____cacheline_aligned;
|
||||
|
||||
struct pt_device {
|
||||
struct list_head entry;
|
||||
|
||||
unsigned int ord;
|
||||
char name[MAX_PT_NAME_LEN];
|
||||
|
||||
struct device *dev;
|
||||
|
||||
/* Bus specific device information */
|
||||
struct pt_msix *pt_msix;
|
||||
|
||||
struct pt_dev_vdata *dev_vdata;
|
||||
|
||||
unsigned int pt_irq;
|
||||
|
||||
/* I/O area used for device communication */
|
||||
void __iomem *io_regs;
|
||||
|
||||
spinlock_t cmd_lock ____cacheline_aligned;
|
||||
unsigned int cmd_count;
|
||||
struct list_head cmd;
|
||||
|
||||
/*
|
||||
* The command queue. This represent the queue available on the
|
||||
* PTDMA that are available for processing cmds
|
||||
*/
|
||||
struct pt_cmd_queue cmd_q;
|
||||
|
||||
wait_queue_head_t lsb_queue;
|
||||
|
||||
struct pt_tasklet_data tdata;
|
||||
};
|
||||
|
||||
/*
|
||||
* descriptor for PTDMA commands
|
||||
* 8 32-bit words:
|
||||
* word 0: function; engine; control bits
|
||||
* word 1: length of source data
|
||||
* word 2: low 32 bits of source pointer
|
||||
* word 3: upper 16 bits of source pointer; source memory type
|
||||
* word 4: low 32 bits of destination pointer
|
||||
* word 5: upper 16 bits of destination pointer; destination memory type
|
||||
* word 6: reserved 32 bits
|
||||
* word 7: reserved 32 bits
|
||||
*/
|
||||
|
||||
#define DWORD0_SOC BIT(0)
|
||||
#define DWORD0_IOC BIT(1)
|
||||
|
||||
struct dword3 {
|
||||
unsigned int src_hi:16;
|
||||
unsigned int src_mem:2;
|
||||
unsigned int lsb_cxt_id:8;
|
||||
unsigned int rsvd1:5;
|
||||
unsigned int fixed:1;
|
||||
};
|
||||
|
||||
struct dword5 {
|
||||
unsigned int dst_hi:16;
|
||||
unsigned int dst_mem:2;
|
||||
unsigned int rsvd1:13;
|
||||
unsigned int fixed:1;
|
||||
};
|
||||
|
||||
struct ptdma_desc {
|
||||
u32 dw0;
|
||||
u32 length;
|
||||
u32 src_lo;
|
||||
struct dword3 dw3;
|
||||
u32 dst_lo;
|
||||
struct dword5 dw5;
|
||||
__le32 rsvd1;
|
||||
__le32 rsvd2;
|
||||
};
|
||||
|
||||
/* Structure to hold PT device data */
|
||||
struct pt_dev_vdata {
|
||||
const unsigned int bar;
|
||||
};
|
||||
|
||||
int pt_core_init(struct pt_device *pt);
|
||||
void pt_core_destroy(struct pt_device *pt);
|
||||
|
||||
int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
|
||||
struct pt_passthru_engine *pt_engine);
|
||||
|
||||
void pt_start_queue(struct pt_cmd_queue *cmd_q);
|
||||
void pt_stop_queue(struct pt_cmd_queue *cmd_q);
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue