dmaengine: mediatek: Add MediaTek Command-Queue DMA controller for MT6765 SoC
MediaTek Command-Queue DMA controller (CQDMA) on MT6765 SoC is dedicated to memory-to-memory transfer through queue based descriptor management. There are only 3 physical channels inside CQDMA, while the driver is extended to support 32 virtual channels for multiple dma users to issue dma requests onto the CQDMA simultaneously. Signed-off-by: Shun-Chih Yu <shun-chih.yu@mediatek.com> Signed-off-by: Vinod Koul <vkoul@kernel.org>
This commit is contained in:
Shun-Chih Yu
Vinod Koul
parent
651022382c
commit
b1f01e48df
|
|
@ -11,3 +11,16 @@ config MTK_HSDMA
|
|||
This controller provides the channels which is dedicated to
|
||||
memory-to-memory transfer to offload from CPU through ring-
|
||||
based descriptor management.
|
||||
|
||||
config MTK_CQDMA
|
||||
tristate "MediaTek Command-Queue DMA controller support"
|
||||
depends on ARCH_MEDIATEK || COMPILE_TEST
|
||||
select DMA_ENGINE
|
||||
select DMA_VIRTUAL_CHANNELS
|
||||
select ASYNC_TX_ENABLE_CHANNEL_SWITCH
|
||||
help
|
||||
Enable support for Command-Queue DMA controller on MediaTek
|
||||
SoCs.
|
||||
|
||||
This controller provides the channels which is dedicated to
|
||||
memory-to-memory transfer to offload from CPU.
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o
|
||||
obj-$(CONFIG_MTK_CQDMA) += mtk-cqdma.o
|
||||
|
|
|
|||
|
|
@ -0,0 +1,951 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
// Copyright (c) 2018-2019 MediaTek Inc.
|
||||
|
||||
/*
|
||||
* Driver for MediaTek Command-Queue DMA Controller
|
||||
*
|
||||
* Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
|
||||
*
|
||||
*/
|
||||
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/clk.h>
|
||||
#include <linux/dmaengine.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/iopoll.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/of_device.h>
|
||||
#include <linux/of_dma.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/pm_runtime.h>
|
||||
#include <linux/refcount.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include "../virt-dma.h"
|
||||
|
||||
#define MTK_CQDMA_USEC_POLL 10
|
||||
#define MTK_CQDMA_TIMEOUT_POLL 1000
|
||||
#define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
|
||||
#define MTK_CQDMA_ALIGN_SIZE 1
|
||||
|
||||
/* The default number of virtual channel */
|
||||
#define MTK_CQDMA_NR_VCHANS 32
|
||||
|
||||
/* The default number of physical channel */
|
||||
#define MTK_CQDMA_NR_PCHANS 3
|
||||
|
||||
/* Registers for underlying dma manipulation */
|
||||
#define MTK_CQDMA_INT_FLAG 0x0
|
||||
#define MTK_CQDMA_INT_EN 0x4
|
||||
#define MTK_CQDMA_EN 0x8
|
||||
#define MTK_CQDMA_RESET 0xc
|
||||
#define MTK_CQDMA_FLUSH 0x14
|
||||
#define MTK_CQDMA_SRC 0x1c
|
||||
#define MTK_CQDMA_DST 0x20
|
||||
#define MTK_CQDMA_LEN1 0x24
|
||||
#define MTK_CQDMA_LEN2 0x28
|
||||
#define MTK_CQDMA_SRC2 0x60
|
||||
#define MTK_CQDMA_DST2 0x64
|
||||
|
||||
/* Registers setting */
|
||||
#define MTK_CQDMA_EN_BIT BIT(0)
|
||||
#define MTK_CQDMA_INT_FLAG_BIT BIT(0)
|
||||
#define MTK_CQDMA_INT_EN_BIT BIT(0)
|
||||
#define MTK_CQDMA_FLUSH_BIT BIT(0)
|
||||
|
||||
#define MTK_CQDMA_WARM_RST_BIT BIT(0)
|
||||
#define MTK_CQDMA_HARD_RST_BIT BIT(1)
|
||||
|
||||
#define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
|
||||
#define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
|
||||
#define MTK_CQDMA_ADDR2_SHFIT (32)
|
||||
|
||||
/**
|
||||
* struct mtk_cqdma_vdesc - The struct holding info describing virtual
|
||||
* descriptor (CVD)
|
||||
* @vd: An instance for struct virt_dma_desc
|
||||
* @len: The total data size device wants to move
|
||||
* @residue: The remaining data size device will move
|
||||
* @dest: The destination address device wants to move to
|
||||
* @src: The source address device wants to move from
|
||||
* @ch: The pointer to the corresponding dma channel
|
||||
* @node: The lise_head struct to build link-list for VDs
|
||||
* @parent: The pointer to the parent CVD
|
||||
*/
|
||||
struct mtk_cqdma_vdesc {
|
||||
struct virt_dma_desc vd;
|
||||
size_t len;
|
||||
size_t residue;
|
||||
dma_addr_t dest;
|
||||
dma_addr_t src;
|
||||
struct dma_chan *ch;
|
||||
|
||||
struct list_head node;
|
||||
struct mtk_cqdma_vdesc *parent;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct mtk_cqdma_pchan - The struct holding info describing physical
|
||||
* channel (PC)
|
||||
* @queue: Queue for the PDs issued to this PC
|
||||
* @base: The mapped register I/O base of this PC
|
||||
* @irq: The IRQ that this PC are using
|
||||
* @refcnt: Track how many VCs are using this PC
|
||||
* @tasklet: Tasklet for this PC
|
||||
* @lock: Lock protect agaisting multiple VCs access PC
|
||||
*/
|
||||
struct mtk_cqdma_pchan {
|
||||
struct list_head queue;
|
||||
void __iomem *base;
|
||||
u32 irq;
|
||||
|
||||
refcount_t refcnt;
|
||||
|
||||
struct tasklet_struct tasklet;
|
||||
|
||||
/* lock to protect PC */
|
||||
spinlock_t lock;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct mtk_cqdma_vchan - The struct holding info describing virtual
|
||||
* channel (VC)
|
||||
* @vc: An instance for struct virt_dma_chan
|
||||
* @pc: The pointer to the underlying PC
|
||||
* @issue_completion: The wait for all issued descriptors completited
|
||||
* @issue_synchronize: Bool indicating channel synchronization starts
|
||||
*/
|
||||
struct mtk_cqdma_vchan {
|
||||
struct virt_dma_chan vc;
|
||||
struct mtk_cqdma_pchan *pc;
|
||||
struct completion issue_completion;
|
||||
bool issue_synchronize;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct mtk_cqdma_device - The struct holding info describing CQDMA
|
||||
* device
|
||||
* @ddev: An instance for struct dma_device
|
||||
* @clk: The clock that device internal is using
|
||||
* @dma_requests: The number of VCs the device supports to
|
||||
* @dma_channels: The number of PCs the device supports to
|
||||
* @vc: The pointer to all available VCs
|
||||
* @pc: The pointer to all the underlying PCs
|
||||
*/
|
||||
struct mtk_cqdma_device {
|
||||
struct dma_device ddev;
|
||||
struct clk *clk;
|
||||
|
||||
u32 dma_requests;
|
||||
u32 dma_channels;
|
||||
struct mtk_cqdma_vchan *vc;
|
||||
struct mtk_cqdma_pchan **pc;
|
||||
};
|
||||
|
||||
static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
|
||||
{
|
||||
return container_of(chan->device, struct mtk_cqdma_device, ddev);
|
||||
}
|
||||
|
||||
static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
|
||||
{
|
||||
return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
|
||||
}
|
||||
|
||||
static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
|
||||
{
|
||||
return container_of(vd, struct mtk_cqdma_vdesc, vd);
|
||||
}
|
||||
|
||||
static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
|
||||
{
|
||||
return cqdma->ddev.dev;
|
||||
}
|
||||
|
||||
static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
|
||||
{
|
||||
return readl(pc->base + reg);
|
||||
}
|
||||
|
||||
static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
|
||||
{
|
||||
writel_relaxed(val, pc->base + reg);
|
||||
}
|
||||
|
||||
static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
|
||||
u32 mask, u32 set)
|
||||
{
|
||||
u32 val;
|
||||
|
||||
val = mtk_dma_read(pc, reg);
|
||||
val &= ~mask;
|
||||
val |= set;
|
||||
mtk_dma_write(pc, reg, val);
|
||||
}
|
||||
|
||||
static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
|
||||
{
|
||||
mtk_dma_rmw(pc, reg, 0, val);
|
||||
}
|
||||
|
||||
static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
|
||||
{
|
||||
mtk_dma_rmw(pc, reg, val, 0);
|
||||
}
|
||||
|
||||
static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
|
||||
{
|
||||
kfree(to_cqdma_vdesc(vd));
|
||||
}
|
||||
|
||||
static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
|
||||
{
|
||||
u32 status = 0;
|
||||
|
||||
if (!atomic)
|
||||
return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
|
||||
status,
|
||||
!(status & MTK_CQDMA_EN_BIT),
|
||||
MTK_CQDMA_USEC_POLL,
|
||||
MTK_CQDMA_TIMEOUT_POLL);
|
||||
|
||||
return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
|
||||
status,
|
||||
!(status & MTK_CQDMA_EN_BIT),
|
||||
MTK_CQDMA_USEC_POLL,
|
||||
MTK_CQDMA_TIMEOUT_POLL);
|
||||
}
|
||||
|
||||
static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
|
||||
{
|
||||
mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
|
||||
mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
|
||||
|
||||
return mtk_cqdma_poll_engine_done(pc, false);
|
||||
}
|
||||
|
||||
static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
|
||||
struct mtk_cqdma_vdesc *cvd)
|
||||
{
|
||||
/* wait for the previous transaction done */
|
||||
if (mtk_cqdma_poll_engine_done(pc, true) < 0)
|
||||
dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
|
||||
|
||||
/* warm reset the dma engine for the new transaction */
|
||||
mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
|
||||
if (mtk_cqdma_poll_engine_done(pc, true) < 0)
|
||||
dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
|
||||
|
||||
/* setup the source */
|
||||
mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
|
||||
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
|
||||
mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
|
||||
#else
|
||||
mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
|
||||
#endif
|
||||
|
||||
/* setup the destination */
|
||||
mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
|
||||
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
|
||||
mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
|
||||
#else
|
||||
mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
|
||||
#endif
|
||||
|
||||
/* setup the length */
|
||||
mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
|
||||
|
||||
/* start dma engine */
|
||||
mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
|
||||
}
|
||||
|
||||
static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
|
||||
{
|
||||
struct virt_dma_desc *vd, *vd2;
|
||||
struct mtk_cqdma_pchan *pc = cvc->pc;
|
||||
struct mtk_cqdma_vdesc *cvd;
|
||||
bool trigger_engine = false;
|
||||
|
||||
lockdep_assert_held(&cvc->vc.lock);
|
||||
lockdep_assert_held(&pc->lock);
|
||||
|
||||
list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
|
||||
/* need to trigger dma engine if PC's queue is empty */
|
||||
if (list_empty(&pc->queue))
|
||||
trigger_engine = true;
|
||||
|
||||
cvd = to_cqdma_vdesc(vd);
|
||||
|
||||
/* add VD into PC's queue */
|
||||
list_add_tail(&cvd->node, &pc->queue);
|
||||
|
||||
/* start the dma engine */
|
||||
if (trigger_engine)
|
||||
mtk_cqdma_start(pc, cvd);
|
||||
|
||||
/* remove VD from list desc_issued */
|
||||
list_del(&vd->node);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* return true if this VC is active,
|
||||
* meaning that there are VDs under processing by the PC
|
||||
*/
|
||||
static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
|
||||
{
|
||||
struct mtk_cqdma_vdesc *cvd;
|
||||
|
||||
list_for_each_entry(cvd, &cvc->pc->queue, node)
|
||||
if (cvc == to_cqdma_vchan(cvd->ch))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* return the pointer of the CVD that is just consumed by the PC
|
||||
*/
|
||||
static struct mtk_cqdma_vdesc
|
||||
*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc;
|
||||
struct mtk_cqdma_vdesc *cvd, *ret = NULL;
|
||||
|
||||
/* consume a CVD from PC's queue */
|
||||
cvd = list_first_entry_or_null(&pc->queue,
|
||||
struct mtk_cqdma_vdesc, node);
|
||||
if (unlikely(!cvd || !cvd->parent))
|
||||
return NULL;
|
||||
|
||||
cvc = to_cqdma_vchan(cvd->ch);
|
||||
ret = cvd;
|
||||
|
||||
/* update residue of the parent CVD */
|
||||
cvd->parent->residue -= cvd->len;
|
||||
|
||||
/* delete CVD from PC's queue */
|
||||
list_del(&cvd->node);
|
||||
|
||||
spin_lock(&cvc->vc.lock);
|
||||
|
||||
/* check whether all the child CVDs completed */
|
||||
if (!cvd->parent->residue) {
|
||||
/* add the parent VD into list desc_completed */
|
||||
vchan_cookie_complete(&cvd->parent->vd);
|
||||
|
||||
/* setup completion if this VC is under synchronization */
|
||||
if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
|
||||
complete(&cvc->issue_completion);
|
||||
cvc->issue_synchronize = false;
|
||||
}
|
||||
}
|
||||
|
||||
spin_unlock(&cvc->vc.lock);
|
||||
|
||||
/* start transaction for next CVD in the queue */
|
||||
cvd = list_first_entry_or_null(&pc->queue,
|
||||
struct mtk_cqdma_vdesc, node);
|
||||
if (cvd)
|
||||
mtk_cqdma_start(pc, cvd);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void mtk_cqdma_tasklet_cb(unsigned long data)
|
||||
{
|
||||
struct mtk_cqdma_pchan *pc = (struct mtk_cqdma_pchan *)data;
|
||||
struct mtk_cqdma_vdesc *cvd = NULL;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&pc->lock, flags);
|
||||
/* consume the queue */
|
||||
cvd = mtk_cqdma_consume_work_queue(pc);
|
||||
spin_unlock_irqrestore(&pc->lock, flags);
|
||||
|
||||
/* submit the next CVD */
|
||||
if (cvd) {
|
||||
dma_run_dependencies(&cvd->vd.tx);
|
||||
|
||||
/*
|
||||
* free child CVD after completion.
|
||||
* the parent CVD would be freeed with desc_free by user.
|
||||
*/
|
||||
if (cvd->parent != cvd)
|
||||
kfree(cvd);
|
||||
}
|
||||
|
||||
/* re-enable interrupt before leaving tasklet */
|
||||
enable_irq(pc->irq);
|
||||
}
|
||||
|
||||
static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
|
||||
{
|
||||
struct mtk_cqdma_device *cqdma = devid;
|
||||
irqreturn_t ret = IRQ_NONE;
|
||||
bool schedule_tasklet = false;
|
||||
u32 i;
|
||||
|
||||
/* clear interrupt flags for each PC */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
|
||||
spin_lock(&cqdma->pc[i]->lock);
|
||||
if (mtk_dma_read(cqdma->pc[i],
|
||||
MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
|
||||
/* clear interrupt */
|
||||
mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
|
||||
MTK_CQDMA_INT_FLAG_BIT);
|
||||
|
||||
schedule_tasklet = true;
|
||||
ret = IRQ_HANDLED;
|
||||
}
|
||||
spin_unlock(&cqdma->pc[i]->lock);
|
||||
|
||||
if (schedule_tasklet) {
|
||||
/* disable interrupt */
|
||||
disable_irq_nosync(cqdma->pc[i]->irq);
|
||||
|
||||
/* schedule the tasklet to handle the transactions */
|
||||
tasklet_schedule(&cqdma->pc[i]->tasklet);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
|
||||
dma_cookie_t cookie)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
|
||||
struct virt_dma_desc *vd;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&cvc->pc->lock, flags);
|
||||
list_for_each_entry(vd, &cvc->pc->queue, node)
|
||||
if (vd->tx.cookie == cookie) {
|
||||
spin_unlock_irqrestore(&cvc->pc->lock, flags);
|
||||
return vd;
|
||||
}
|
||||
spin_unlock_irqrestore(&cvc->pc->lock, flags);
|
||||
|
||||
list_for_each_entry(vd, &cvc->vc.desc_issued, node)
|
||||
if (vd->tx.cookie == cookie)
|
||||
return vd;
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
|
||||
dma_cookie_t cookie,
|
||||
struct dma_tx_state *txstate)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
|
||||
struct mtk_cqdma_vdesc *cvd;
|
||||
struct virt_dma_desc *vd;
|
||||
enum dma_status ret;
|
||||
unsigned long flags;
|
||||
size_t bytes = 0;
|
||||
|
||||
ret = dma_cookie_status(c, cookie, txstate);
|
||||
if (ret == DMA_COMPLETE || !txstate)
|
||||
return ret;
|
||||
|
||||
spin_lock_irqsave(&cvc->vc.lock, flags);
|
||||
vd = mtk_cqdma_find_active_desc(c, cookie);
|
||||
spin_unlock_irqrestore(&cvc->vc.lock, flags);
|
||||
|
||||
if (vd) {
|
||||
cvd = to_cqdma_vdesc(vd);
|
||||
bytes = cvd->residue;
|
||||
}
|
||||
|
||||
dma_set_residue(txstate, bytes);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void mtk_cqdma_issue_pending(struct dma_chan *c)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
|
||||
unsigned long pc_flags;
|
||||
unsigned long vc_flags;
|
||||
|
||||
/* acquire PC's lock before VS's lock for lock dependency in tasklet */
|
||||
spin_lock_irqsave(&cvc->pc->lock, pc_flags);
|
||||
spin_lock_irqsave(&cvc->vc.lock, vc_flags);
|
||||
|
||||
if (vchan_issue_pending(&cvc->vc))
|
||||
mtk_cqdma_issue_vchan_pending(cvc);
|
||||
|
||||
spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
|
||||
spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
|
||||
}
|
||||
|
||||
static struct dma_async_tx_descriptor *
|
||||
mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
|
||||
dma_addr_t src, size_t len, unsigned long flags)
|
||||
{
|
||||
struct mtk_cqdma_vdesc **cvd;
|
||||
struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
|
||||
size_t i, tlen, nr_vd;
|
||||
|
||||
/*
|
||||
* In the case that trsanction length is larger than the
|
||||
* DMA engine supports, a single memcpy transaction needs
|
||||
* to be separated into several DMA transactions.
|
||||
* Each DMA transaction would be described by a CVD,
|
||||
* and the first one is referred as the parent CVD,
|
||||
* while the others are child CVDs.
|
||||
* The parent CVD's tx descriptor is the only tx descriptor
|
||||
* returned to the DMA user, and it should not be completed
|
||||
* until all the child CVDs completed.
|
||||
*/
|
||||
nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
|
||||
cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
|
||||
if (!cvd)
|
||||
return NULL;
|
||||
|
||||
for (i = 0; i < nr_vd; ++i) {
|
||||
cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
|
||||
if (!cvd[i]) {
|
||||
for (; i > 0; --i)
|
||||
kfree(cvd[i - 1]);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* setup dma channel */
|
||||
cvd[i]->ch = c;
|
||||
|
||||
/* setup sourece, destination, and length */
|
||||
tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
|
||||
cvd[i]->len = tlen;
|
||||
cvd[i]->src = src;
|
||||
cvd[i]->dest = dest;
|
||||
|
||||
/* setup tx descriptor */
|
||||
tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
|
||||
tx->next = NULL;
|
||||
|
||||
if (!i) {
|
||||
cvd[0]->residue = len;
|
||||
} else {
|
||||
prev_tx->next = tx;
|
||||
cvd[i]->residue = tlen;
|
||||
}
|
||||
|
||||
cvd[i]->parent = cvd[0];
|
||||
|
||||
/* update the src, dest, len, prev_tx for the next CVD */
|
||||
src += tlen;
|
||||
dest += tlen;
|
||||
len -= tlen;
|
||||
prev_tx = tx;
|
||||
}
|
||||
|
||||
return &cvd[0]->vd.tx;
|
||||
}
|
||||
|
||||
static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
|
||||
{
|
||||
struct virt_dma_chan *vc = to_virt_chan(c);
|
||||
unsigned long flags;
|
||||
LIST_HEAD(head);
|
||||
|
||||
/*
|
||||
* set desc_allocated, desc_submitted,
|
||||
* and desc_issued as the candicates to be freed
|
||||
*/
|
||||
spin_lock_irqsave(&vc->lock, flags);
|
||||
list_splice_tail_init(&vc->desc_allocated, &head);
|
||||
list_splice_tail_init(&vc->desc_submitted, &head);
|
||||
list_splice_tail_init(&vc->desc_issued, &head);
|
||||
spin_unlock_irqrestore(&vc->lock, flags);
|
||||
|
||||
/* free descriptor lists */
|
||||
vchan_dma_desc_free_list(vc, &head);
|
||||
}
|
||||
|
||||
static void mtk_cqdma_free_active_desc(struct dma_chan *c)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
|
||||
bool sync_needed = false;
|
||||
unsigned long pc_flags;
|
||||
unsigned long vc_flags;
|
||||
|
||||
/* acquire PC's lock first due to lock dependency in dma ISR */
|
||||
spin_lock_irqsave(&cvc->pc->lock, pc_flags);
|
||||
spin_lock_irqsave(&cvc->vc.lock, vc_flags);
|
||||
|
||||
/* synchronization is required if this VC is active */
|
||||
if (mtk_cqdma_is_vchan_active(cvc)) {
|
||||
cvc->issue_synchronize = true;
|
||||
sync_needed = true;
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
|
||||
spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
|
||||
|
||||
/* waiting for the completion of this VC */
|
||||
if (sync_needed)
|
||||
wait_for_completion(&cvc->issue_completion);
|
||||
|
||||
/* free all descriptors in list desc_completed */
|
||||
vchan_synchronize(&cvc->vc);
|
||||
|
||||
WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
|
||||
"Desc pending still in list desc_completed\n");
|
||||
}
|
||||
|
||||
static int mtk_cqdma_terminate_all(struct dma_chan *c)
|
||||
{
|
||||
/* free descriptors not processed yet by hardware */
|
||||
mtk_cqdma_free_inactive_desc(c);
|
||||
|
||||
/* free descriptors being processed by hardware */
|
||||
mtk_cqdma_free_active_desc(c);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
|
||||
{
|
||||
struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
|
||||
struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
|
||||
struct mtk_cqdma_pchan *pc = NULL;
|
||||
u32 i, min_refcnt = U32_MAX, refcnt;
|
||||
unsigned long flags;
|
||||
|
||||
/* allocate PC with the minimun refcount */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i) {
|
||||
refcnt = refcount_read(&cqdma->pc[i]->refcnt);
|
||||
if (refcnt < min_refcnt) {
|
||||
pc = cqdma->pc[i];
|
||||
min_refcnt = refcnt;
|
||||
}
|
||||
}
|
||||
|
||||
if (!pc)
|
||||
return -ENOSPC;
|
||||
|
||||
spin_lock_irqsave(&pc->lock, flags);
|
||||
|
||||
if (!refcount_read(&pc->refcnt)) {
|
||||
/* allocate PC when the refcount is zero */
|
||||
mtk_cqdma_hard_reset(pc);
|
||||
|
||||
/* enable interrupt for this PC */
|
||||
mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
|
||||
|
||||
/*
|
||||
* refcount_inc would complain increment on 0; use-after-free.
|
||||
* Thus, we need to explicitly set it as 1 initially.
|
||||
*/
|
||||
refcount_set(&pc->refcnt, 1);
|
||||
} else {
|
||||
refcount_inc(&pc->refcnt);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&pc->lock, flags);
|
||||
|
||||
vc->pc = pc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
|
||||
{
|
||||
struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
|
||||
unsigned long flags;
|
||||
|
||||
/* free all descriptors in all lists on the VC */
|
||||
mtk_cqdma_terminate_all(c);
|
||||
|
||||
spin_lock_irqsave(&cvc->pc->lock, flags);
|
||||
|
||||
/* PC is not freed until there is no VC mapped to it */
|
||||
if (refcount_dec_and_test(&cvc->pc->refcnt)) {
|
||||
/* start the flush operation and stop the engine */
|
||||
mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
|
||||
|
||||
/* wait for the completion of flush operation */
|
||||
if (mtk_cqdma_poll_engine_done(cvc->pc, false) < 0)
|
||||
dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
|
||||
|
||||
/* clear the flush bit and interrupt flag */
|
||||
mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
|
||||
mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
|
||||
MTK_CQDMA_INT_FLAG_BIT);
|
||||
|
||||
/* disable interrupt for this PC */
|
||||
mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&cvc->pc->lock, flags);
|
||||
}
|
||||
|
||||
static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
|
||||
{
|
||||
unsigned long flags;
|
||||
int err;
|
||||
u32 i;
|
||||
|
||||
pm_runtime_enable(cqdma2dev(cqdma));
|
||||
pm_runtime_get_sync(cqdma2dev(cqdma));
|
||||
|
||||
err = clk_prepare_enable(cqdma->clk);
|
||||
|
||||
if (err) {
|
||||
pm_runtime_put_sync(cqdma2dev(cqdma));
|
||||
pm_runtime_disable(cqdma2dev(cqdma));
|
||||
return err;
|
||||
}
|
||||
|
||||
/* reset all PCs */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i) {
|
||||
spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
|
||||
if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
|
||||
dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
|
||||
spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
|
||||
|
||||
clk_disable_unprepare(cqdma->clk);
|
||||
pm_runtime_put_sync(cqdma2dev(cqdma));
|
||||
pm_runtime_disable(cqdma2dev(cqdma));
|
||||
return -EINVAL;
|
||||
}
|
||||
spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
|
||||
{
|
||||
unsigned long flags;
|
||||
u32 i;
|
||||
|
||||
/* reset all PCs */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i) {
|
||||
spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
|
||||
if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
|
||||
dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
|
||||
spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
|
||||
}
|
||||
|
||||
clk_disable_unprepare(cqdma->clk);
|
||||
|
||||
pm_runtime_put_sync(cqdma2dev(cqdma));
|
||||
pm_runtime_disable(cqdma2dev(cqdma));
|
||||
}
|
||||
|
||||
static const struct of_device_id mtk_cqdma_match[] = {
|
||||
{ .compatible = "mediatek,mt6765-cqdma" },
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
|
||||
|
||||
static int mtk_cqdma_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct mtk_cqdma_device *cqdma;
|
||||
struct mtk_cqdma_vchan *vc;
|
||||
struct dma_device *dd;
|
||||
struct resource *res;
|
||||
int err;
|
||||
u32 i;
|
||||
|
||||
cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
|
||||
if (!cqdma)
|
||||
return -ENOMEM;
|
||||
|
||||
dd = &cqdma->ddev;
|
||||
|
||||
cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
|
||||
if (IS_ERR(cqdma->clk)) {
|
||||
dev_err(&pdev->dev, "No clock for %s\n",
|
||||
dev_name(&pdev->dev));
|
||||
return PTR_ERR(cqdma->clk);
|
||||
}
|
||||
|
||||
dma_cap_set(DMA_MEMCPY, dd->cap_mask);
|
||||
|
||||
dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
|
||||
dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
|
||||
dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
|
||||
dd->device_tx_status = mtk_cqdma_tx_status;
|
||||
dd->device_issue_pending = mtk_cqdma_issue_pending;
|
||||
dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
|
||||
dd->device_terminate_all = mtk_cqdma_terminate_all;
|
||||
dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
|
||||
dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
|
||||
dd->directions = BIT(DMA_MEM_TO_MEM);
|
||||
dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
|
||||
dd->dev = &pdev->dev;
|
||||
INIT_LIST_HEAD(&dd->channels);
|
||||
|
||||
if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
|
||||
"dma-requests",
|
||||
&cqdma->dma_requests)) {
|
||||
dev_info(&pdev->dev,
|
||||
"Using %u as missing dma-requests property\n",
|
||||
MTK_CQDMA_NR_VCHANS);
|
||||
|
||||
cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
|
||||
}
|
||||
|
||||
if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
|
||||
"dma-channels",
|
||||
&cqdma->dma_channels)) {
|
||||
dev_info(&pdev->dev,
|
||||
"Using %u as missing dma-channels property\n",
|
||||
MTK_CQDMA_NR_PCHANS);
|
||||
|
||||
cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
|
||||
}
|
||||
|
||||
cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
|
||||
sizeof(*cqdma->pc), GFP_KERNEL);
|
||||
if (!cqdma->pc)
|
||||
return -ENOMEM;
|
||||
|
||||
/* initialization for PCs */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i) {
|
||||
cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
|
||||
sizeof(**cqdma->pc), GFP_KERNEL);
|
||||
if (!cqdma->pc[i])
|
||||
return -ENOMEM;
|
||||
|
||||
INIT_LIST_HEAD(&cqdma->pc[i]->queue);
|
||||
spin_lock_init(&cqdma->pc[i]->lock);
|
||||
refcount_set(&cqdma->pc[i]->refcnt, 0);
|
||||
|
||||
res = platform_get_resource(pdev, IORESOURCE_MEM, i);
|
||||
if (!res) {
|
||||
dev_err(&pdev->dev, "No mem resource for %s\n",
|
||||
dev_name(&pdev->dev));
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
cqdma->pc[i]->base = devm_ioremap_resource(&pdev->dev, res);
|
||||
if (IS_ERR(cqdma->pc[i]->base))
|
||||
return PTR_ERR(cqdma->pc[i]->base);
|
||||
|
||||
/* allocate IRQ resource */
|
||||
res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
|
||||
if (!res) {
|
||||
dev_err(&pdev->dev, "No irq resource for %s\n",
|
||||
dev_name(&pdev->dev));
|
||||
return -EINVAL;
|
||||
}
|
||||
cqdma->pc[i]->irq = res->start;
|
||||
|
||||
err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
|
||||
mtk_cqdma_irq, 0, dev_name(&pdev->dev),
|
||||
cqdma);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev,
|
||||
"request_irq failed with err %d\n", err);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
/* allocate resource for VCs */
|
||||
cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
|
||||
sizeof(*cqdma->vc), GFP_KERNEL);
|
||||
if (!cqdma->vc)
|
||||
return -ENOMEM;
|
||||
|
||||
for (i = 0; i < cqdma->dma_requests; i++) {
|
||||
vc = &cqdma->vc[i];
|
||||
vc->vc.desc_free = mtk_cqdma_vdesc_free;
|
||||
vchan_init(&vc->vc, dd);
|
||||
init_completion(&vc->issue_completion);
|
||||
}
|
||||
|
||||
err = dma_async_device_register(dd);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = of_dma_controller_register(pdev->dev.of_node,
|
||||
of_dma_xlate_by_chan_id, cqdma);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev,
|
||||
"MediaTek CQDMA OF registration failed %d\n", err);
|
||||
goto err_unregister;
|
||||
}
|
||||
|
||||
err = mtk_cqdma_hw_init(cqdma);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev,
|
||||
"MediaTek CQDMA HW initialization failed %d\n", err);
|
||||
goto err_unregister;
|
||||
}
|
||||
|
||||
platform_set_drvdata(pdev, cqdma);
|
||||
|
||||
/* initialize tasklet for each PC */
|
||||
for (i = 0; i < cqdma->dma_channels; ++i)
|
||||
tasklet_init(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb,
|
||||
(unsigned long)cqdma->pc[i]);
|
||||
|
||||
dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
|
||||
|
||||
return 0;
|
||||
|
||||
err_unregister:
|
||||
dma_async_device_unregister(dd);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int mtk_cqdma_remove(struct platform_device *pdev)
|
||||
{
|
||||
struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
|
||||
struct mtk_cqdma_vchan *vc;
|
||||
unsigned long flags;
|
||||
int i;
|
||||
|
||||
/* kill VC task */
|
||||
for (i = 0; i < cqdma->dma_requests; i++) {
|
||||
vc = &cqdma->vc[i];
|
||||
|
||||
list_del(&vc->vc.chan.device_node);
|
||||
tasklet_kill(&vc->vc.task);
|
||||
}
|
||||
|
||||
/* disable interrupt */
|
||||
for (i = 0; i < cqdma->dma_channels; i++) {
|
||||
spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
|
||||
mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
|
||||
MTK_CQDMA_INT_EN_BIT);
|
||||
spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
|
||||
|
||||
/* Waits for any pending IRQ handlers to complete */
|
||||
synchronize_irq(cqdma->pc[i]->irq);
|
||||
|
||||
tasklet_kill(&cqdma->pc[i]->tasklet);
|
||||
}
|
||||
|
||||
/* disable hardware */
|
||||
mtk_cqdma_hw_deinit(cqdma);
|
||||
|
||||
dma_async_device_unregister(&cqdma->ddev);
|
||||
of_dma_controller_free(pdev->dev.of_node);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct platform_driver mtk_cqdma_driver = {
|
||||
.probe = mtk_cqdma_probe,
|
||||
.remove = mtk_cqdma_remove,
|
||||
.driver = {
|
||||
.name = KBUILD_MODNAME,
|
||||
.of_match_table = mtk_cqdma_match,
|
||||
},
|
||||
};
|
||||
module_platform_driver(mtk_cqdma_driver);
|
||||
|
||||
MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
|
||||
MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
Loading…
Reference in New Issue