276 lines
7.8 KiB
C
276 lines
7.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Intel MIC Platform Software Stack (MPSS)
|
|
*
|
|
* Copyright(c) 2014 Intel Corporation.
|
|
*
|
|
* Intel MIC X100 DMA Driver.
|
|
*
|
|
* Adapted from IOAT dma driver.
|
|
*/
|
|
#ifndef _MIC_X100_DMA_H_
|
|
#define _MIC_X100_DMA_H_
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/debugfs.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/mic_bus.h>
|
|
|
|
#include "dmaengine.h"
|
|
|
|
/*
|
|
* MIC has a total of 8 dma channels.
|
|
* Four channels are assigned for host SW use & the remaining for MIC SW.
|
|
* MIC DMA transfer size & addresses need to be 64 byte aligned.
|
|
*/
|
|
#define MIC_DMA_MAX_NUM_CHAN 8
|
|
#define MIC_DMA_NUM_CHAN 4
|
|
#define MIC_DMA_ALIGN_SHIFT DMAENGINE_ALIGN_64_BYTES
|
|
#define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT)
|
|
#define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4)
|
|
|
|
/*
|
|
* Register descriptions
|
|
* All the registers are 32 bit registers.
|
|
* DCR is a global register and all others are per-channel.
|
|
* DCR - bits 0, 2, 4, 6, 8, 10, 12, 14 - enable bits for channels 0 to 7
|
|
* bits 1, 3, 5, 7, 9, 11, 13, 15 - owner bits for channels 0 to 7
|
|
* DCAR - bit 24 & 25 interrupt masks for mic owned & host owned channels
|
|
* DHPR - head of the descriptor ring updated by s/w
|
|
* DTPR - tail of the descriptor ring updated by h/w
|
|
* DRAR_LO - lower 32 bits of descriptor ring's mic address
|
|
* DRAR_HI - 3:0 - remaining 4 bits of descriptor ring's mic address
|
|
* 20:4 descriptor ring size
|
|
* 25:21 mic smpt entry number
|
|
* DSTAT - 16:0 h/w completion count; 31:28 dma engine status
|
|
* DCHERR - this register is non-zero on error
|
|
* DCHERRMSK - interrupt mask register
|
|
*/
|
|
#define MIC_DMA_HW_CMP_CNT_MASK 0x1ffff
|
|
#define MIC_DMA_CHAN_QUIESCE 0x20000000
|
|
#define MIC_DMA_SBOX_BASE 0x00010000
|
|
#define MIC_DMA_SBOX_DCR 0x0000A280
|
|
#define MIC_DMA_SBOX_CH_BASE 0x0001A000
|
|
#define MIC_DMA_SBOX_CHAN_OFF 0x40
|
|
#define MIC_DMA_SBOX_DCAR_IM0 (0x1 << 24)
|
|
#define MIC_DMA_SBOX_DCAR_IM1 (0x1 << 25)
|
|
#define MIC_DMA_SBOX_DRARHI_SYS_MASK (0x1 << 26)
|
|
#define MIC_DMA_REG_DCAR 0
|
|
#define MIC_DMA_REG_DHPR 4
|
|
#define MIC_DMA_REG_DTPR 8
|
|
#define MIC_DMA_REG_DRAR_LO 20
|
|
#define MIC_DMA_REG_DRAR_HI 24
|
|
#define MIC_DMA_REG_DSTAT 32
|
|
#define MIC_DMA_REG_DCHERR 44
|
|
#define MIC_DMA_REG_DCHERRMSK 48
|
|
|
|
/* HW dma desc */
|
|
struct mic_dma_desc {
|
|
u64 qw0;
|
|
u64 qw1;
|
|
};
|
|
|
|
enum mic_dma_chan_owner {
|
|
MIC_DMA_CHAN_MIC = 0,
|
|
MIC_DMA_CHAN_HOST
|
|
};
|
|
|
|
/*
|
|
* mic_dma_chan - channel specific information
|
|
* @ch_num: channel number
|
|
* @owner: owner of this channel
|
|
* @last_tail: cached value of descriptor ring tail
|
|
* @head: index of next descriptor in desc_ring
|
|
* @issued: hardware notification point
|
|
* @submitted: index that will be used to submit descriptors to h/w
|
|
* @api_ch: dma engine api channel
|
|
* @desc_ring: dma descriptor ring
|
|
* @desc_ring_micpa: mic physical address of desc_ring
|
|
* @status_dest: destination for status (fence) descriptor
|
|
* @status_dest_micpa: mic address for status_dest,
|
|
* DMA controller uses this address
|
|
* @tx_array: array of async_tx
|
|
* @cleanup_lock: lock held when processing completed tx
|
|
* @prep_lock: lock held in prep_memcpy & released in tx_submit
|
|
* @issue_lock: lock used to synchronize writes to head
|
|
* @cookie: mic_irq cookie used with mic irq request
|
|
*/
|
|
struct mic_dma_chan {
|
|
int ch_num;
|
|
enum mic_dma_chan_owner owner;
|
|
u32 last_tail;
|
|
u32 head;
|
|
u32 issued;
|
|
u32 submitted;
|
|
struct dma_chan api_ch;
|
|
struct mic_dma_desc *desc_ring;
|
|
dma_addr_t desc_ring_micpa;
|
|
u64 *status_dest;
|
|
dma_addr_t status_dest_micpa;
|
|
struct dma_async_tx_descriptor *tx_array;
|
|
spinlock_t cleanup_lock;
|
|
spinlock_t prep_lock;
|
|
spinlock_t issue_lock;
|
|
struct mic_irq *cookie;
|
|
};
|
|
|
|
/*
|
|
* struct mic_dma_device - per mic device
|
|
* @mic_ch: dma channels
|
|
* @dma_dev: underlying dma device
|
|
* @mbdev: mic bus dma device
|
|
* @mmio: virtual address of the mmio space
|
|
* @dbg_dir: debugfs directory
|
|
* @start_ch: first channel number that can be used
|
|
* @max_xfer_size: maximum transfer size per dma descriptor
|
|
*/
|
|
struct mic_dma_device {
|
|
struct mic_dma_chan mic_ch[MIC_DMA_MAX_NUM_CHAN];
|
|
struct dma_device dma_dev;
|
|
struct mbus_device *mbdev;
|
|
void __iomem *mmio;
|
|
struct dentry *dbg_dir;
|
|
int start_ch;
|
|
size_t max_xfer_size;
|
|
};
|
|
|
|
static inline struct mic_dma_chan *to_mic_dma_chan(struct dma_chan *ch)
|
|
{
|
|
return container_of(ch, struct mic_dma_chan, api_ch);
|
|
}
|
|
|
|
static inline struct mic_dma_device *to_mic_dma_dev(struct mic_dma_chan *ch)
|
|
{
|
|
return
|
|
container_of((const typeof(((struct mic_dma_device *)0)->mic_ch)*)
|
|
(ch - ch->ch_num), struct mic_dma_device, mic_ch);
|
|
}
|
|
|
|
static inline struct mbus_device *to_mbus_device(struct mic_dma_chan *ch)
|
|
{
|
|
return to_mic_dma_dev(ch)->mbdev;
|
|
}
|
|
|
|
static inline struct mbus_hw_ops *to_mbus_hw_ops(struct mic_dma_chan *ch)
|
|
{
|
|
return to_mbus_device(ch)->hw_ops;
|
|
}
|
|
|
|
static inline struct device *mic_dma_ch_to_device(struct mic_dma_chan *ch)
|
|
{
|
|
return to_mic_dma_dev(ch)->dma_dev.dev;
|
|
}
|
|
|
|
static inline void __iomem *mic_dma_chan_to_mmio(struct mic_dma_chan *ch)
|
|
{
|
|
return to_mic_dma_dev(ch)->mmio;
|
|
}
|
|
|
|
static inline u32 mic_dma_read_reg(struct mic_dma_chan *ch, u32 reg)
|
|
{
|
|
return ioread32(mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
|
|
ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
|
|
}
|
|
|
|
static inline void mic_dma_write_reg(struct mic_dma_chan *ch, u32 reg, u32 val)
|
|
{
|
|
iowrite32(val, mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
|
|
ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
|
|
}
|
|
|
|
static inline u32 mic_dma_mmio_read(struct mic_dma_chan *ch, u32 offset)
|
|
{
|
|
return ioread32(mic_dma_chan_to_mmio(ch) + offset);
|
|
}
|
|
|
|
static inline void mic_dma_mmio_write(struct mic_dma_chan *ch, u32 val,
|
|
u32 offset)
|
|
{
|
|
iowrite32(val, mic_dma_chan_to_mmio(ch) + offset);
|
|
}
|
|
|
|
static inline u32 mic_dma_read_cmp_cnt(struct mic_dma_chan *ch)
|
|
{
|
|
return mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) &
|
|
MIC_DMA_HW_CMP_CNT_MASK;
|
|
}
|
|
|
|
static inline void mic_dma_chan_set_owner(struct mic_dma_chan *ch)
|
|
{
|
|
u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
u32 chan_num = ch->ch_num;
|
|
|
|
dcr = (dcr & ~(0x1 << (chan_num * 2))) | (ch->owner << (chan_num * 2));
|
|
mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
}
|
|
|
|
static inline void mic_dma_enable_chan(struct mic_dma_chan *ch)
|
|
{
|
|
u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
|
|
dcr |= 2 << (ch->ch_num << 1);
|
|
mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
}
|
|
|
|
static inline void mic_dma_disable_chan(struct mic_dma_chan *ch)
|
|
{
|
|
u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
|
|
dcr &= ~(2 << (ch->ch_num << 1));
|
|
mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
|
|
}
|
|
|
|
static void mic_dma_chan_set_desc_ring(struct mic_dma_chan *ch)
|
|
{
|
|
u32 drar_hi;
|
|
dma_addr_t desc_ring_micpa = ch->desc_ring_micpa;
|
|
|
|
drar_hi = (MIC_DMA_DESC_RX_SIZE & 0x1ffff) << 4;
|
|
if (MIC_DMA_CHAN_MIC == ch->owner) {
|
|
drar_hi |= (desc_ring_micpa >> 32) & 0xf;
|
|
} else {
|
|
drar_hi |= MIC_DMA_SBOX_DRARHI_SYS_MASK;
|
|
drar_hi |= ((desc_ring_micpa >> 34)
|
|
& 0x1f) << 21;
|
|
drar_hi |= (desc_ring_micpa >> 32) & 0x3;
|
|
}
|
|
mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_LO, (u32) desc_ring_micpa);
|
|
mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_HI, drar_hi);
|
|
}
|
|
|
|
static inline void mic_dma_chan_mask_intr(struct mic_dma_chan *ch)
|
|
{
|
|
u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
|
|
|
|
if (MIC_DMA_CHAN_MIC == ch->owner)
|
|
dcar |= MIC_DMA_SBOX_DCAR_IM0;
|
|
else
|
|
dcar |= MIC_DMA_SBOX_DCAR_IM1;
|
|
mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
|
|
}
|
|
|
|
static inline void mic_dma_chan_unmask_intr(struct mic_dma_chan *ch)
|
|
{
|
|
u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
|
|
|
|
if (MIC_DMA_CHAN_MIC == ch->owner)
|
|
dcar &= ~MIC_DMA_SBOX_DCAR_IM0;
|
|
else
|
|
dcar &= ~MIC_DMA_SBOX_DCAR_IM1;
|
|
mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
|
|
}
|
|
|
|
static void mic_dma_ack_interrupt(struct mic_dma_chan *ch)
|
|
{
|
|
if (MIC_DMA_CHAN_MIC == ch->owner) {
|
|
/* HW errata */
|
|
mic_dma_chan_mask_intr(ch);
|
|
mic_dma_chan_unmask_intr(ch);
|
|
}
|
|
to_mbus_hw_ops(ch)->ack_interrupt(to_mbus_device(ch), ch->ch_num);
|
|
}
|
|
#endif
|