UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity
OpenCloudOS-Kernel/drivers/dma/dw-edma/dw-edma-core.c

1040 lines
25 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
* Synopsys DesignWare eDMA core driver
*
* Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
*/
#include <linux/module.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/dmaengine.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/dma/edma.h>
#include <linux/dma-mapping.h>
#include "dw-edma-core.h"
#include "dw-edma-v0-core.h"
#include "../dmaengine.h"
#include "../virt-dma.h"
static inline
struct device *dchan2dev(struct dma_chan *dchan)
{
return &dchan->dev->device;
}
static inline
struct device *chan2dev(struct dw_edma_chan *chan)
{
return &chan->vc.chan.dev->device;
}
static inline
struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
{
return container_of(vd, struct dw_edma_desc, vd);
}
static inline
u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
{
struct dw_edma_chip *chip = chan->dw->chip;
if (chip->ops->pci_address)
return chip->ops->pci_address(chip->dev, cpu_addr);
return cpu_addr;
}
static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *burst;
burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
if (unlikely(!burst))
return NULL;
INIT_LIST_HEAD(&burst->list);
if (chunk->burst) {
/* Create and add new element into the linked list */
chunk->bursts_alloc++;
list_add_tail(&burst->list, &chunk->burst->list);
} else {
/* List head */
chunk->bursts_alloc = 0;
chunk->burst = burst;
}
return burst;
}
static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
{
struct dw_edma_chip *chip = desc->chan->dw->chip;
struct dw_edma_chan *chan = desc->chan;
struct dw_edma_chunk *chunk;
chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
if (unlikely(!chunk))
return NULL;
INIT_LIST_HEAD(&chunk->list);
chunk->chan = chan;
/* Toggling change bit (CB) in each chunk, this is a mechanism to
* inform the eDMA HW block that this is a new linked list ready
* to be consumed.
* - Odd chunks originate CB equal to 0
* - Even chunks originate CB equal to 1
*/
chunk->cb = !(desc->chunks_alloc % 2);
if (chan->dir == EDMA_DIR_WRITE) {
chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
} else {
chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
}
if (desc->chunk) {
/* Create and add new element into the linked list */
if (!dw_edma_alloc_burst(chunk)) {
kfree(chunk);
return NULL;
}
desc->chunks_alloc++;
list_add_tail(&chunk->list, &desc->chunk->list);
} else {
/* List head */
chunk->burst = NULL;
desc->chunks_alloc = 0;
desc->chunk = chunk;
}
return chunk;
}
static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
{
struct dw_edma_desc *desc;
desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
if (unlikely(!desc))
return NULL;
desc->chan = chan;
if (!dw_edma_alloc_chunk(desc)) {
kfree(desc);
return NULL;
}
return desc;
}
static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *child, *_next;
/* Remove all the list elements */
list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
list_del(&child->list);
kfree(child);
chunk->bursts_alloc--;
}
/* Remove the list head */
kfree(child);
chunk->burst = NULL;
}
static void dw_edma_free_chunk(struct dw_edma_desc *desc)
{
struct dw_edma_chunk *child, *_next;
if (!desc->chunk)
return;
/* Remove all the list elements */
list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
dw_edma_free_burst(child);
list_del(&child->list);
kfree(child);
desc->chunks_alloc--;
}
/* Remove the list head */
kfree(child);
desc->chunk = NULL;
}
static void dw_edma_free_desc(struct dw_edma_desc *desc)
{
dw_edma_free_chunk(desc);
kfree(desc);
}
static void vchan_free_desc(struct virt_dma_desc *vdesc)
{
dw_edma_free_desc(vd2dw_edma_desc(vdesc));
}
static void dw_edma_start_transfer(struct dw_edma_chan *chan)
{
struct dw_edma_chunk *child;
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
if (!vd)
return;
desc = vd2dw_edma_desc(vd);
if (!desc)
return;
child = list_first_entry_or_null(&desc->chunk->list,
struct dw_edma_chunk, list);
if (!child)
return;
dw_edma_v0_core_start(child, !desc->xfer_sz);
desc->xfer_sz += child->ll_region.sz;
dw_edma_free_burst(child);
list_del(&child->list);
kfree(child);
desc->chunks_alloc--;
}
static void dw_edma_device_caps(struct dma_chan *dchan,
struct dma_slave_caps *caps)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
if (chan->dir == EDMA_DIR_READ)
caps->directions = BIT(DMA_DEV_TO_MEM);
else
caps->directions = BIT(DMA_MEM_TO_DEV);
} else {
if (chan->dir == EDMA_DIR_WRITE)
caps->directions = BIT(DMA_DEV_TO_MEM);
else
caps->directions = BIT(DMA_MEM_TO_DEV);
}
}
static int dw_edma_device_config(struct dma_chan *dchan,
struct dma_slave_config *config)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
memcpy(&chan->config, config, sizeof(*config));
chan->configured = true;
return 0;
}
static int dw_edma_device_pause(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
if (!chan->configured)
err = -EPERM;
else if (chan->status != EDMA_ST_BUSY)
err = -EPERM;
else if (chan->request != EDMA_REQ_NONE)
err = -EPERM;
else
chan->request = EDMA_REQ_PAUSE;
return err;
}
static int dw_edma_device_resume(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
if (!chan->configured) {
err = -EPERM;
} else if (chan->status != EDMA_ST_PAUSE) {
err = -EPERM;
} else if (chan->request != EDMA_REQ_NONE) {
err = -EPERM;
} else {
chan->status = EDMA_ST_BUSY;
dw_edma_start_transfer(chan);
}
return err;
}
static int dw_edma_device_terminate_all(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
if (!chan->configured) {
/* Do nothing */
} else if (chan->status == EDMA_ST_PAUSE) {
chan->status = EDMA_ST_IDLE;
chan->configured = false;
} else if (chan->status == EDMA_ST_IDLE) {
chan->configured = false;
} else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) {
/*
* The channel is in a false BUSY state, probably didn't
* receive or lost an interrupt
*/
chan->status = EDMA_ST_IDLE;
chan->configured = false;
} else if (chan->request > EDMA_REQ_PAUSE) {
err = -EPERM;
} else {
chan->request = EDMA_REQ_STOP;
}
return err;
}
static void dw_edma_device_issue_pending(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
if (chan->configured && chan->request == EDMA_REQ_NONE &&
chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) {
chan->status = EDMA_ST_BUSY;
dw_edma_start_transfer(chan);
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
static enum dma_status
dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
struct dma_tx_state *txstate)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
unsigned long flags;
enum dma_status ret;
u32 residue = 0;
ret = dma_cookie_status(dchan, cookie, txstate);
if (ret == DMA_COMPLETE)
return ret;
if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
ret = DMA_PAUSED;
if (!txstate)
goto ret_residue;
spin_lock_irqsave(&chan->vc.lock, flags);
vd = vchan_find_desc(&chan->vc, cookie);
if (vd) {
desc = vd2dw_edma_desc(vd);
if (desc)
residue = desc->alloc_sz - desc->xfer_sz;
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
ret_residue:
dma_set_residue(txstate, residue);
return ret;
}
static struct dma_async_tx_descriptor *
dw_edma_device_transfer(struct dw_edma_transfer *xfer)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
enum dma_transfer_direction dir = xfer->direction;
struct scatterlist *sg = NULL;
struct dw_edma_chunk *chunk;
struct dw_edma_burst *burst;
struct dw_edma_desc *desc;
u64 src_addr, dst_addr;
size_t fsz = 0;
u32 cnt = 0;
int i;
if (!chan->configured)
return NULL;
/*
* Local Root Port/End-point Remote End-point
* +-----------------------+ PCIe bus +----------------------+
* | | +-+ | |
* | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM |
* | | | | | |
* | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV |
* | | +-+ | |
* +-----------------------+ +----------------------+
*
* 1. Normal logic:
* If eDMA is embedded into the DW PCIe RP/EP and controlled from the
* CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used
* for the device read operations (DEV_TO_MEM) and the Tx channel
* (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).
*
* 2. Inverted logic:
* If eDMA is embedded into a Remote PCIe EP and is controlled by the
* MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx
* channel (EDMA_DIR_WRITE) will be used for the device read operations
* (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write
* operations (MEM_TO_DEV).
*
* It is the client driver responsibility to choose a proper channel
* for the DMA transfers.
*/
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||
(chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))
return NULL;
} else {
if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||
(chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))
return NULL;
}
if (xfer->type == EDMA_XFER_CYCLIC) {
if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
return NULL;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
if (xfer->xfer.sg.len < 1)
return NULL;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
return NULL;
if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
return NULL;
} else {
return NULL;
}
desc = dw_edma_alloc_desc(chan);
if (unlikely(!desc))
goto err_alloc;
chunk = dw_edma_alloc_chunk(desc);
if (unlikely(!chunk))
goto err_alloc;
if (xfer->type == EDMA_XFER_INTERLEAVED) {
src_addr = xfer->xfer.il->src_start;
dst_addr = xfer->xfer.il->dst_start;
} else {
src_addr = chan->config.src_addr;
dst_addr = chan->config.dst_addr;
}
if (dir == DMA_DEV_TO_MEM)
src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
else
dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
if (xfer->type == EDMA_XFER_CYCLIC) {
cnt = xfer->xfer.cyclic.cnt;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
cnt = xfer->xfer.sg.len;
sg = xfer->xfer.sg.sgl;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
fsz = xfer->xfer.il->frame_size;
}
for (i = 0; i < cnt; i++) {
if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
break;
if (chunk->bursts_alloc == chan->ll_max) {
chunk = dw_edma_alloc_chunk(desc);
if (unlikely(!chunk))
goto err_alloc;
}
burst = dw_edma_alloc_burst(chunk);
if (unlikely(!burst))
goto err_alloc;
if (xfer->type == EDMA_XFER_CYCLIC)
burst->sz = xfer->xfer.cyclic.len;
else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
burst->sz = sg_dma_len(sg);
else if (xfer->type == EDMA_XFER_INTERLEAVED)
burst->sz = xfer->xfer.il->sgl[i % fsz].size;
chunk->ll_region.sz += burst->sz;
desc->alloc_sz += burst->sz;
if (dir == DMA_DEV_TO_MEM) {
burst->sar = src_addr;
if (xfer->type == EDMA_XFER_CYCLIC) {
burst->dar = xfer->xfer.cyclic.paddr;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
src_addr += sg_dma_len(sg);
burst->dar = sg_dma_address(sg);
/* Unlike the typical assumption by other
* drivers/IPs the peripheral memory isn't
* a FIFO memory, in this case, it's a
* linear memory and that why the source
* and destination addresses are increased
* by the same portion (data length)
*/
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
burst->dar = dst_addr;
}
} else {
burst->dar = dst_addr;
if (xfer->type == EDMA_XFER_CYCLIC) {
burst->sar = xfer->xfer.cyclic.paddr;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
dst_addr += sg_dma_len(sg);
burst->sar = sg_dma_address(sg);
/* Unlike the typical assumption by other
* drivers/IPs the peripheral memory isn't
* a FIFO memory, in this case, it's a
* linear memory and that why the source
* and destination addresses are increased
* by the same portion (data length)
*/
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
burst->sar = src_addr;
}
}
if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
sg = sg_next(sg);
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
struct dma_interleaved_template *il = xfer->xfer.il;
struct data_chunk *dc = &il->sgl[i % fsz];
src_addr += burst->sz;
if (il->src_sgl)
src_addr += dmaengine_get_src_icg(il, dc);
dst_addr += burst->sz;
if (il->dst_sgl)
dst_addr += dmaengine_get_dst_icg(il, dc);
}
}
return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
err_alloc:
if (desc)
dw_edma_free_desc(desc);
return NULL;
}
static struct dma_async_tx_descriptor *
dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
unsigned int len,
enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct dw_edma_transfer xfer;
xfer.dchan = dchan;
xfer.direction = direction;
xfer.xfer.sg.sgl = sgl;
xfer.xfer.sg.len = len;
xfer.flags = flags;
xfer.type = EDMA_XFER_SCATTER_GATHER;
return dw_edma_device_transfer(&xfer);
}
static struct dma_async_tx_descriptor *
dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
size_t len, size_t count,
enum dma_transfer_direction direction,
unsigned long flags)
{
struct dw_edma_transfer xfer;
xfer.dchan = dchan;
xfer.direction = direction;
xfer.xfer.cyclic.paddr = paddr;
xfer.xfer.cyclic.len = len;
xfer.xfer.cyclic.cnt = count;
xfer.flags = flags;
xfer.type = EDMA_XFER_CYCLIC;
return dw_edma_device_transfer(&xfer);
}
static struct dma_async_tx_descriptor *
dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
struct dma_interleaved_template *ilt,
unsigned long flags)
{
struct dw_edma_transfer xfer;
xfer.dchan = dchan;
xfer.direction = ilt->dir;
xfer.xfer.il = ilt;
xfer.flags = flags;
xfer.type = EDMA_XFER_INTERLEAVED;
return dw_edma_device_transfer(&xfer);
}
static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
{
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
unsigned long flags;
dw_edma_v0_core_clear_done_int(chan);
spin_lock_irqsave(&chan->vc.lock, flags);
vd = vchan_next_desc(&chan->vc);
if (vd) {
switch (chan->request) {
case EDMA_REQ_NONE:
desc = vd2dw_edma_desc(vd);
if (desc->chunks_alloc) {
chan->status = EDMA_ST_BUSY;
dw_edma_start_transfer(chan);
} else {
list_del(&vd->node);
vchan_cookie_complete(vd);
chan->status = EDMA_ST_IDLE;
}
break;
case EDMA_REQ_STOP:
list_del(&vd->node);
vchan_cookie_complete(vd);
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
break;
case EDMA_REQ_PAUSE:
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_PAUSE;
break;
default:
break;
}
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
{
struct virt_dma_desc *vd;
unsigned long flags;
dw_edma_v0_core_clear_abort_int(chan);
spin_lock_irqsave(&chan->vc.lock, flags);
vd = vchan_next_desc(&chan->vc);
if (vd) {
list_del(&vd->node);
vchan_cookie_complete(vd);
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
}
static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write)
{
struct dw_edma_irq *dw_irq = data;
struct dw_edma *dw = dw_irq->dw;
unsigned long total, pos, val;
unsigned long off;
u32 mask;
if (write) {
total = dw->wr_ch_cnt;
off = 0;
mask = dw_irq->wr_mask;
} else {
total = dw->rd_ch_cnt;
off = dw->wr_ch_cnt;
mask = dw_irq->rd_mask;
}
val = dw_edma_v0_core_status_done_int(dw, write ?
EDMA_DIR_WRITE :
EDMA_DIR_READ);
val &= mask;
for_each_set_bit(pos, &val, total) {
struct dw_edma_chan *chan = &dw->chan[pos + off];
dw_edma_done_interrupt(chan);
}
val = dw_edma_v0_core_status_abort_int(dw, write ?
EDMA_DIR_WRITE :
EDMA_DIR_READ);
val &= mask;
for_each_set_bit(pos, &val, total) {
struct dw_edma_chan *chan = &dw->chan[pos + off];
dw_edma_abort_interrupt(chan);
}
return IRQ_HANDLED;
}
static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
{
return dw_edma_interrupt(irq, data, true);
}
static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
{
return dw_edma_interrupt(irq, data, false);
}
static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
{
dw_edma_interrupt(irq, data, true);
dw_edma_interrupt(irq, data, false);
return IRQ_HANDLED;
}
static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
if (chan->status != EDMA_ST_IDLE)
return -EBUSY;
return 0;
}
static void dw_edma_free_chan_resources(struct dma_chan *dchan)
{
unsigned long timeout = jiffies + msecs_to_jiffies(5000);
int ret;
while (time_before(jiffies, timeout)) {
ret = dw_edma_device_terminate_all(dchan);
if (!ret)
break;
if (time_after_eq(jiffies, timeout))
return;
cpu_relax();
}
}
static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
{
struct dw_edma_chip *chip = dw->chip;
struct device *dev = chip->dev;
struct dw_edma_chan *chan;
struct dw_edma_irq *irq;
struct dma_device *dma;
u32 i, ch_cnt;
u32 pos;
ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
dma = &dw->dma;
INIT_LIST_HEAD(&dma->channels);
for (i = 0; i < ch_cnt; i++) {
chan = &dw->chan[i];
chan->dw = dw;
if (i < dw->wr_ch_cnt) {
chan->id = i;
chan->dir = EDMA_DIR_WRITE;
} else {
chan->id = i - dw->wr_ch_cnt;
chan->dir = EDMA_DIR_READ;
}
chan->configured = false;
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
if (chan->dir == EDMA_DIR_WRITE)
chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
else
chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
chan->ll_max -= 1;
dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
chan->dir == EDMA_DIR_WRITE ? "write" : "read",
chan->id, chan->ll_max);
if (dw->nr_irqs == 1)
pos = 0;
else if (chan->dir == EDMA_DIR_WRITE)
pos = chan->id % wr_alloc;
else
pos = wr_alloc + chan->id % rd_alloc;
irq = &dw->irq[pos];
if (chan->dir == EDMA_DIR_WRITE)
irq->wr_mask |= BIT(chan->id);
else
irq->rd_mask |= BIT(chan->id);
irq->dw = dw;
memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id,
chan->msi.address_hi, chan->msi.address_lo,
chan->msi.data);
chan->vc.desc_free = vchan_free_desc;
chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
&dw->chip->dt_region_wr[chan->id] :
&dw->chip->dt_region_rd[chan->id];
vchan_init(&chan->vc, dma);
dw_edma_v0_core_device_config(chan);
}
/* Set DMA channel capabilities */
dma_cap_zero(dma->cap_mask);
dma_cap_set(DMA_SLAVE, dma->cap_mask);
dma_cap_set(DMA_CYCLIC, dma->cap_mask);
dma_cap_set(DMA_PRIVATE, dma->cap_mask);
dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
/* Set DMA channel callbacks */
dma->dev = chip->dev;
dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
dma->device_free_chan_resources = dw_edma_free_chan_resources;
dma->device_caps = dw_edma_device_caps;
dma->device_config = dw_edma_device_config;
dma->device_pause = dw_edma_device_pause;
dma->device_resume = dw_edma_device_resume;
dma->device_terminate_all = dw_edma_device_terminate_all;
dma->device_issue_pending = dw_edma_device_issue_pending;
dma->device_tx_status = dw_edma_device_tx_status;
dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
dma_set_max_seg_size(dma->dev, U32_MAX);
/* Register DMA device */
return dma_async_device_register(dma);
}
static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
{
if (*nr_irqs && *alloc < cnt) {
(*alloc)++;
(*nr_irqs)--;
}
}
static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
{
while (*mask * alloc < cnt)
(*mask)++;
}
static int dw_edma_irq_request(struct dw_edma *dw,
u32 *wr_alloc, u32 *rd_alloc)
{
struct dw_edma_chip *chip = dw->chip;
struct device *dev = dw->chip->dev;
u32 wr_mask = 1;
u32 rd_mask = 1;
int i, err = 0;
u32 ch_cnt;
int irq;
ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
if (chip->nr_irqs < 1 || !chip->ops->irq_vector)
return -EINVAL;
dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
if (!dw->irq)
return -ENOMEM;
if (chip->nr_irqs == 1) {
/* Common IRQ shared among all channels */
irq = chip->ops->irq_vector(dev, 0);
err = request_irq(irq, dw_edma_interrupt_common,
IRQF_SHARED, dw->name, &dw->irq[0]);
if (err) {
dw->nr_irqs = 0;
return err;
}
if (irq_get_msi_desc(irq))
get_cached_msi_msg(irq, &dw->irq[0].msi);
dw->nr_irqs = 1;
} else {
/* Distribute IRQs equally among all channels */
int tmp = chip->nr_irqs;
while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
}
dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
irq = chip->ops->irq_vector(dev, i);
err = request_irq(irq,
i < *wr_alloc ?
dw_edma_interrupt_write :
dw_edma_interrupt_read,
IRQF_SHARED, dw->name,
&dw->irq[i]);
if (err)
goto err_irq_free;
if (irq_get_msi_desc(irq))
get_cached_msi_msg(irq, &dw->irq[i].msi);
}
dw->nr_irqs = i;
}
return 0;
err_irq_free:
for (i--; i >= 0; i--) {
irq = chip->ops->irq_vector(dev, i);
free_irq(irq, &dw->irq[i]);
}
return err;
}
int dw_edma_probe(struct dw_edma_chip *chip)
{
struct device *dev;
struct dw_edma *dw;
u32 wr_alloc = 0;
u32 rd_alloc = 0;
int i, err;
if (!chip)
return -EINVAL;
dev = chip->dev;
if (!dev || !chip->ops)
return -EINVAL;
dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
if (!dw)
return -ENOMEM;
dw->chip = chip;
raw_spin_lock_init(&dw->lock);
dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE));
dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ));
dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
return -EINVAL;
dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
dw->wr_ch_cnt, dw->rd_ch_cnt);
/* Allocate channels */
dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
sizeof(*dw->chan), GFP_KERNEL);
if (!dw->chan)
return -ENOMEM;
snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
dev_name(chip->dev));
/* Disable eDMA, only to establish the ideal initial conditions */
dw_edma_v0_core_off(dw);
/* Request IRQs */
err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
if (err)
return err;
/* Setup write/read channels */
err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
if (err)
goto err_irq_free;
/* Turn debugfs on */
dw_edma_v0_core_debugfs_on(dw);
chip->dw = dw;
return 0;
err_irq_free:
for (i = (dw->nr_irqs - 1); i >= 0; i--)
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
return err;
}
EXPORT_SYMBOL_GPL(dw_edma_probe);
int dw_edma_remove(struct dw_edma_chip *chip)
{
struct dw_edma_chan *chan, *_chan;
struct device *dev = chip->dev;
struct dw_edma *dw = chip->dw;
int i;
/* Skip removal if no private data found */
if (!dw)
return -ENODEV;
/* Disable eDMA */
dw_edma_v0_core_off(dw);
/* Free irqs */
for (i = (dw->nr_irqs - 1); i >= 0; i--)
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
/* Deregister eDMA device */
dma_async_device_unregister(&dw->dma);
list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
vc.chan.device_node) {
tasklet_kill(&chan->vc.task);
list_del(&chan->vc.chan.device_node);
}
return 0;
}
EXPORT_SYMBOL_GPL(dw_edma_remove);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
Reference in New Issue View Git Blame Copy Permalink