hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

crypto: cavium - Add support for CNN55XX adapters. 

Add Physical Function driver support for CNN55XX crypto adapters.
CNN55XX adapters belongs to Cavium NITROX family series,
which accelerate both Symmetric and Asymmetric crypto workloads.

These adapters have crypto engines that need firmware
to become operational.

Signed-off-by: Srikanth Jampala <Jampala.Srikanth@cavium.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Srikanth Jampala 2017-05-30 17:28:01 +05:30 committed by Herbert Xu
parent 3bca64c1b9
commit 14fa93cdcd
13 changed files with 4002 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/crypto/Kconfig
View File

@ -541,6 +541,7 @@ config CRYPTO_DEV_MXS_DCP
source "drivers/crypto/qat/Kconfig"
source "drivers/crypto/cavium/cpt/Kconfig"
source "drivers/crypto/cavium/nitrox/Kconfig"
config CRYPTO_DEV_CAVIUM_ZIP
tristate "Cavium ZIP driver"

1
drivers/crypto/Makefile
View File

@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/
obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o

21
drivers/crypto/cavium/nitrox/Kconfig Normal file
View File

@ -0,0 +1,21 @@
#
# Cavium NITROX Crypto Device configuration
#
config CRYPTO_DEV_NITROX
tristate
select CRYPTO_BLKCIPHER
select CRYPTO_AES
select CRYPTO_DES
select FW_LOADER
config CRYPTO_DEV_NITROX_CNN55XX
tristate "Support for Cavium CNN55XX driver"
depends on PCI_MSI && 64BIT
select CRYPTO_DEV_NITROX
default m
help
Support for Cavium NITROX family CNN55XX driver
for accelerating crypto workloads.
To compile this as a module, choose M here: the module
will be called n5pf.

7
drivers/crypto/cavium/nitrox/Makefile Normal file
View File

@ -0,0 +1,7 @@
obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o
n5pf-objs := nitrox_main.o \
nitrox_isr.o \
nitrox_lib.o \
nitrox_hal.o \
nitrox_reqmgr.o

35
drivers/crypto/cavium/nitrox/nitrox_common.h Normal file
View File

@ -0,0 +1,35 @@
#ifndef __NITROX_COMMON_H
#define __NITROX_COMMON_H
#include "nitrox_dev.h"
#include "nitrox_req.h"
void nitrox_pf_cleanup_isr(struct nitrox_device *ndev);
int nitrox_pf_init_isr(struct nitrox_device *ndev);
int nitrox_common_sw_init(struct nitrox_device *ndev);
void nitrox_common_sw_cleanup(struct nitrox_device *ndev);
void pkt_slc_resp_handler(unsigned long data);
int nitrox_process_se_request(struct nitrox_device *ndev,
struct se_crypto_request *req,
completion_t cb,
struct skcipher_request *skreq);
void backlog_qflush_work(struct work_struct *work);
void nitrox_config_emu_unit(struct nitrox_device *ndev);
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
void nitrox_config_vfmode(struct nitrox_device *ndev, int mode);
void nitrox_config_nps_unit(struct nitrox_device *ndev);
void nitrox_config_pom_unit(struct nitrox_device *ndev);
void nitrox_config_rand_unit(struct nitrox_device *ndev);
void nitrox_config_efl_unit(struct nitrox_device *ndev);
void nitrox_config_bmi_unit(struct nitrox_device *ndev);
void nitrox_config_bmo_unit(struct nitrox_device *ndev);
void nitrox_config_lbc_unit(struct nitrox_device *ndev);
void invalidate_lbc(struct nitrox_device *ndev);
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
#endif /* __NITROX_COMMON_H */

1080
drivers/crypto/cavium/nitrox/nitrox_csr.h Normal file
View File

File diff suppressed because it is too large Load Diff

175
drivers/crypto/cavium/nitrox/nitrox_dev.h Normal file
View File

@ -0,0 +1,175 @@
#ifndef __NITROX_DEV_H
#define __NITROX_DEV_H
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#define VERSION_LEN 32
struct nitrox_cmdq {
/* command queue lock */
spinlock_t cmdq_lock;
/* response list lock */
spinlock_t response_lock;
/* backlog list lock */
spinlock_t backlog_lock;
/* request submitted to chip, in progress */
struct list_head response_head;
/* hw queue full, hold in backlog list */
struct list_head backlog_head;
/* doorbell address */
u8 __iomem *dbell_csr_addr;
/* base address of the queue */
u8 *head;
struct nitrox_device *ndev;
/* flush pending backlog commands */
struct work_struct backlog_qflush;
/* requests posted waiting for completion */
atomic_t pending_count;
/* requests in backlog queues */
atomic_t backlog_count;
/* command size 32B/64B */
u8 instr_size;
u8 qno;
u32 qsize;
/* unaligned addresses */
u8 *head_unaligned;
dma_addr_t dma_unaligned;
/* dma address of the base */
dma_addr_t dma;
};
struct nitrox_hw {
/* firmware version */
char fw_name[VERSION_LEN];
u16 vendor_id;
u16 device_id;
u8 revision_id;
/* CNN55XX cores */
u8 se_cores;
u8 ae_cores;
u8 zip_cores;
};
#define MAX_MSIX_VECTOR_NAME 20
/**
* vectors for queues (64 AE, 64 SE and 64 ZIP) and
* error condition/mailbox.
*/
#define MAX_MSIX_VECTORS 192
struct nitrox_msix {
struct msix_entry *entries;
char **names;
DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS);
u32 nr_entries;
};
struct bh_data {
/* slc port completion count address */
u8 __iomem *completion_cnt_csr_addr;
struct nitrox_cmdq *cmdq;
struct tasklet_struct resp_handler;
};
struct nitrox_bh {
struct bh_data *slc;
};
/* NITROX-5 driver state */
#define NITROX_UCODE_LOADED 0
#define NITROX_READY 1
/* command queue size */
#define DEFAULT_CMD_QLEN 2048
/* command timeout in milliseconds */
#define CMD_TIMEOUT 2000
#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev))
#define PF_MODE 0
#define NITROX_CSR_ADDR(ndev, offset) \
((ndev)->bar_addr + (offset))
/**
* struct nitrox_device - NITROX Device Information.
* @list: pointer to linked list of devices
* @bar_addr: iomap address
* @pdev: PCI device information
* @status: NITROX status
* @timeout: Request timeout in jiffies
* @refcnt: Device usage count
* @idx: device index (0..N)
* @node: NUMA node id attached
* @qlen: Command queue length
* @nr_queues: Number of command queues
* @ctx_pool: DMA pool for crypto context
* @pkt_cmdqs: SE Command queues
* @msix: MSI-X information
* @bh: post processing work
* @hw: hardware information
*/
struct nitrox_device {
struct list_head list;
u8 __iomem *bar_addr;
struct pci_dev *pdev;
unsigned long status;
unsigned long timeout;
refcount_t refcnt;
u8 idx;
int node;
u16 qlen;
u16 nr_queues;
struct dma_pool *ctx_pool;
struct nitrox_cmdq *pkt_cmdqs;
struct nitrox_msix msix;
struct nitrox_bh bh;
struct nitrox_hw hw;
};
/**
* nitrox_read_csr - Read from device register
* @ndev: NITROX device
* @offset: offset of the register to read
*
* Returns: value read
*/
static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset)
{
return readq(ndev->bar_addr + offset);
}
/**
* nitrox_write_csr - Write to device register
* @ndev: NITROX device
* @offset: offset of the register to write
* @value: value to write
*/
static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset,
u64 value)
{
writeq(value, (ndev->bar_addr + offset));
}
static inline int nitrox_ready(struct nitrox_device *ndev)
{
return test_bit(NITROX_READY, &ndev->status);
}
#endif /* __NITROX_DEV_H */

401
drivers/crypto/cavium/nitrox/nitrox_hal.c Normal file
View File

@ -0,0 +1,401 @@
#include <linux/delay.h>
#include "nitrox_dev.h"
#include "nitrox_csr.h"
/**
* emu_enable_cores - Enable EMU cluster cores.
* @ndev: N5 device
*/
static void emu_enable_cores(struct nitrox_device *ndev)
{
union emu_se_enable emu_se;
union emu_ae_enable emu_ae;
int i;
/* AE cores 20 per cluster */
emu_ae.value = 0;
emu_ae.s.enable = 0xfffff;
/* SE cores 16 per cluster */
emu_se.value = 0;
emu_se.s.enable = 0xffff;
/* enable per cluster cores */
for (i = 0; i < NR_CLUSTERS; i++) {
nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value);
nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value);
}
}
/**
* nitrox_config_emu_unit - configure EMU unit.
* @ndev: N5 device
*/
void nitrox_config_emu_unit(struct nitrox_device *ndev)
{
union emu_wd_int_ena_w1s emu_wd_int;
union emu_ge_int_ena_w1s emu_ge_int;
u64 offset;
int i;
/* enable cores */
emu_enable_cores(ndev);
/* enable general error and watch dog interrupts */
emu_ge_int.value = 0;
emu_ge_int.s.se_ge = 0xffff;
emu_ge_int.s.ae_ge = 0xfffff;
emu_wd_int.value = 0;
emu_wd_int.s.se_wd = 1;
for (i = 0; i < NR_CLUSTERS; i++) {
offset = EMU_WD_INT_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, emu_wd_int.value);
offset = EMU_GE_INT_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, emu_ge_int.value);
}
}
static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring)
{
union nps_pkt_in_instr_ctl pkt_in_ctl;
union nps_pkt_in_instr_baoff_dbell pkt_in_dbell;
union nps_pkt_in_done_cnts pkt_in_cnts;
u64 offset;
offset = NPS_PKT_IN_INSTR_CTLX(ring);
/* disable the ring */
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
pkt_in_ctl.s.enb = 0;
nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
usleep_range(100, 150);
/* wait to clear [ENB] */
do {
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
} while (pkt_in_ctl.s.enb);
/* clear off door bell counts */
offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(ring);
pkt_in_dbell.value = 0;
pkt_in_dbell.s.dbell = 0xffffffff;
nitrox_write_csr(ndev, offset, pkt_in_dbell.value);
/* clear done counts */
offset = NPS_PKT_IN_DONE_CNTSX(ring);
pkt_in_cnts.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, pkt_in_cnts.value);
usleep_range(50, 100);
}
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring)
{
union nps_pkt_in_instr_ctl pkt_in_ctl;
u64 offset;
/* 64-byte instruction size */
offset = NPS_PKT_IN_INSTR_CTLX(ring);
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
pkt_in_ctl.s.is64b = 1;
pkt_in_ctl.s.enb = 1;
nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
/* wait for set [ENB] */
do {
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
} while (!pkt_in_ctl.s.enb);
}
/**
* nitrox_config_pkt_input_rings - configure Packet Input Rings
* @ndev: N5 device
*/
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
union nps_pkt_in_instr_rsize pkt_in_rsize;
u64 offset;
reset_pkt_input_ring(ndev, i);
/* configure ring base address 16-byte aligned,
* size and interrupt threshold.
*/
offset = NPS_PKT_IN_INSTR_BADDRX(i);
nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma);
/* configure ring size */
offset = NPS_PKT_IN_INSTR_RSIZEX(i);
pkt_in_rsize.value = 0;
pkt_in_rsize.s.rsize = ndev->qlen;
nitrox_write_csr(ndev, offset, pkt_in_rsize.value);
/* set high threshold for pkt input ring interrupts */
offset = NPS_PKT_IN_INT_LEVELSX(i);
nitrox_write_csr(ndev, offset, 0xffffffff);
enable_pkt_input_ring(ndev, i);
}
}
static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port)
{
union nps_pkt_slc_ctl pkt_slc_ctl;
union nps_pkt_slc_cnts pkt_slc_cnts;
u64 offset;
/* disable slc port */
offset = NPS_PKT_SLC_CTLX(port);
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
pkt_slc_ctl.s.enb = 0;
nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
usleep_range(100, 150);
/* wait to clear [ENB] */
do {
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
} while (pkt_slc_ctl.s.enb);
/* clear slc counters */
offset = NPS_PKT_SLC_CNTSX(port);
pkt_slc_cnts.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, pkt_slc_cnts.value);
usleep_range(50, 100);
}
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port)
{
union nps_pkt_slc_ctl pkt_slc_ctl;
u64 offset;
offset = NPS_PKT_SLC_CTLX(port);
pkt_slc_ctl.value = 0;
pkt_slc_ctl.s.enb = 1;
/*
* 8 trailing 0x00 bytes will be added
* to the end of the outgoing packet.
*/
pkt_slc_ctl.s.z = 1;
/* enable response header */
pkt_slc_ctl.s.rh = 1;
nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
/* wait to set [ENB] */
do {
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
} while (!pkt_slc_ctl.s.enb);
}
static void config_single_pkt_solicit_port(struct nitrox_device *ndev,
int port)
{
union nps_pkt_slc_int_levels pkt_slc_int;
u64 offset;
reset_pkt_solicit_port(ndev, port);
offset = NPS_PKT_SLC_INT_LEVELSX(port);
pkt_slc_int.value = 0;
/* time interrupt threshold */
pkt_slc_int.s.timet = 0x3fffff;
nitrox_write_csr(ndev, offset, pkt_slc_int.value);
enable_pkt_solicit_port(ndev, port);
}
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++)
config_single_pkt_solicit_port(ndev, i);
}
/**
* enable_nps_interrupts - enable NPS interrutps
* @ndev: N5 device.
*
* This includes NPS core, packet in and slc interrupts.
*/
static void enable_nps_interrupts(struct nitrox_device *ndev)
{
union nps_core_int_ena_w1s core_int;
/* NPS core interrutps */
core_int.value = 0;
core_int.s.host_wr_err = 1;
core_int.s.host_wr_timeout = 1;
core_int.s.exec_wr_timeout = 1;
core_int.s.npco_dma_malform = 1;
core_int.s.host_nps_wr_err = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
/* NPS packet in ring interrupts */
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
/* NPS packet slc port interrupts */
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
}
void nitrox_config_nps_unit(struct nitrox_device *ndev)
{
union nps_core_gbl_vfcfg core_gbl_vfcfg;
/* endian control information */
nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL);
/* disable ILK interface */
core_gbl_vfcfg.value = 0;
core_gbl_vfcfg.s.ilk_disable = 1;
core_gbl_vfcfg.s.cfg = PF_MODE;
nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
/* config input and solicit ports */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
/* enable interrupts */
enable_nps_interrupts(ndev);
}
void nitrox_config_pom_unit(struct nitrox_device *ndev)
{
union pom_int_ena_w1s pom_int;
int i;
/* enable pom interrupts */
pom_int.value = 0;
pom_int.s.illegal_dport = 1;
nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value);
/* enable perf counters */
for (i = 0; i < ndev->hw.se_cores; i++)
nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i));
}
/**
* nitrox_config_rand_unit - enable N5 random number unit
* @ndev: N5 device
*/
void nitrox_config_rand_unit(struct nitrox_device *ndev)
{
union efl_rnm_ctl_status efl_rnm_ctl;
u64 offset;
offset = EFL_RNM_CTL_STATUS;
efl_rnm_ctl.value = nitrox_read_csr(ndev, offset);
efl_rnm_ctl.s.ent_en = 1;
efl_rnm_ctl.s.rng_en = 1;
nitrox_write_csr(ndev, offset, efl_rnm_ctl.value);
}
void nitrox_config_efl_unit(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
union efl_core_int_ena_w1s efl_core_int;
u64 offset;
/* EFL core interrupts */
offset = EFL_CORE_INT_ENA_W1SX(i);
efl_core_int.value = 0;
efl_core_int.s.len_ovr = 1;
efl_core_int.s.d_left = 1;
efl_core_int.s.epci_decode_err = 1;
nitrox_write_csr(ndev, offset, efl_core_int.value);
offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, (~0ULL));
offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, (~0ULL));
}
}
void nitrox_config_bmi_unit(struct nitrox_device *ndev)
{
union bmi_ctl bmi_ctl;
union bmi_int_ena_w1s bmi_int_ena;
u64 offset;
/* no threshold limits for PCIe */
offset = BMI_CTL;
bmi_ctl.value = nitrox_read_csr(ndev, offset);
bmi_ctl.s.max_pkt_len = 0xff;
bmi_ctl.s.nps_free_thrsh = 0xff;
bmi_ctl.s.nps_hdrq_thrsh = 0x7a;
nitrox_write_csr(ndev, offset, bmi_ctl.value);
/* enable interrupts */
offset = BMI_INT_ENA_W1S;
bmi_int_ena.value = 0;
bmi_int_ena.s.max_len_err_nps = 1;
bmi_int_ena.s.pkt_rcv_err_nps = 1;
bmi_int_ena.s.fpf_undrrn = 1;
nitrox_write_csr(ndev, offset, bmi_int_ena.value);
}
void nitrox_config_bmo_unit(struct nitrox_device *ndev)
{
union bmo_ctl2 bmo_ctl2;
u64 offset;
/* no threshold limits for PCIe */
offset = BMO_CTL2;
bmo_ctl2.value = nitrox_read_csr(ndev, offset);
bmo_ctl2.s.nps_slc_buf_thrsh = 0xff;
nitrox_write_csr(ndev, offset, bmo_ctl2.value);
}
void invalidate_lbc(struct nitrox_device *ndev)
{
union lbc_inval_ctl lbc_ctl;
union lbc_inval_status lbc_stat;
u64 offset;
/* invalidate LBC */
offset = LBC_INVAL_CTL;
lbc_ctl.value = nitrox_read_csr(ndev, offset);
lbc_ctl.s.cam_inval_start = 1;
nitrox_write_csr(ndev, offset, lbc_ctl.value);
offset = LBC_INVAL_STATUS;
do {
lbc_stat.value = nitrox_read_csr(ndev, offset);
} while (!lbc_stat.s.done);
}
void nitrox_config_lbc_unit(struct nitrox_device *ndev)
{
union lbc_int_ena_w1s lbc_int_ena;
u64 offset;
invalidate_lbc(ndev);
/* enable interrupts */
offset = LBC_INT_ENA_W1S;
lbc_int_ena.value = 0;
lbc_int_ena.s.dma_rd_err = 1;
lbc_int_ena.s.over_fetch_err = 1;
lbc_int_ena.s.cam_inval_abort = 1;
lbc_int_ena.s.cam_hard_err = 1;
nitrox_write_csr(ndev, offset, lbc_int_ena.value);
offset = LBC_PLM_VF1_64_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_PLM_VF65_128_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_ELM_VF1_64_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_ELM_VF65_128_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
}

467
drivers/crypto/cavium/nitrox/nitrox_isr.c Normal file
View File

@ -0,0 +1,467 @@
#include <linux/pci.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include "nitrox_dev.h"
#include "nitrox_csr.h"
#include "nitrox_common.h"
#define NR_RING_VECTORS 3
#define NPS_CORE_INT_ACTIVE_ENTRY 192
/**
* nps_pkt_slc_isr - IRQ handler for NPS solicit port
* @irq: irq number
* @data: argument
*/
static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
{
struct bh_data *slc = data;
union nps_pkt_slc_cnts pkt_slc_cnts;
pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr);
/* New packet on SLC output port */
if (pkt_slc_cnts.s.slc_int)
tasklet_hi_schedule(&slc->resp_handler);
return IRQ_HANDLED;
}
static void clear_nps_core_err_intr(struct nitrox_device *ndev)
{
u64 value;
/* Write 1 to clear */
value = nitrox_read_csr(ndev, NPS_CORE_INT);
nitrox_write_csr(ndev, NPS_CORE_INT, value);
dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value);
}
static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
{
union nps_pkt_int pkt_int;
unsigned long value, offset;
int i;
pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n",
pkt_int.value);
if (pkt_int.s.slc_err) {
offset = NPS_PKT_SLC_ERR_TYPE;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value);
offset = NPS_PKT_SLC_RERR_LO;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
/* enable the solicit ports */
for_each_set_bit(i, &value, BITS_PER_LONG)
enable_pkt_solicit_port(ndev, i);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_RERR_LO 0x%016lx\n", value);
offset = NPS_PKT_SLC_RERR_HI;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_RERR_HI 0x%016lx\n", value);
}
if (pkt_int.s.in_err) {
offset = NPS_PKT_IN_ERR_TYPE;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value);
offset = NPS_PKT_IN_RERR_LO;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
/* enable the input ring */
for_each_set_bit(i, &value, BITS_PER_LONG)
enable_pkt_input_ring(ndev, i);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_RERR_LO 0x%016lx\n", value);
offset = NPS_PKT_IN_RERR_HI;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_RERR_HI 0x%016lx\n", value);
}
}
static void clear_pom_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, POM_INT);
nitrox_write_csr(ndev, POM_INT, value);
dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value);
}
static void clear_pem_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, PEM0_INT);
nitrox_write_csr(ndev, PEM0_INT, value);
dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value);
}
static void clear_lbc_err_intr(struct nitrox_device *ndev)
{
union lbc_int lbc_int;
u64 value, offset;
int i;
lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value);
if (lbc_int.s.dma_rd_err) {
for (i = 0; i < NR_CLUSTERS; i++) {
offset = EFL_CORE_VF_ERR_INT0X(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = EFL_CORE_VF_ERR_INT1X(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
}
if (lbc_int.s.cam_soft_err) {
dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
invalidate_lbc(ndev);
}
if (lbc_int.s.pref_dat_len_mismatch_err) {
offset = LBC_PLM_VF1_64_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = LBC_PLM_VF65_128_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
if (lbc_int.s.rd_dat_len_mismatch_err) {
offset = LBC_ELM_VF1_64_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = LBC_ELM_VF65_128_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
}
static void clear_efl_err_intr(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
union efl_core_int core_int;
u64 value, offset;
offset = EFL_CORE_INTX(i);
core_int.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, core_int.value);
dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n",
i, core_int.value);
if (core_int.s.se_err) {
offset = EFL_CORE_SE_ERR_INTX(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
}
}
static void clear_bmi_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, BMI_INT);
nitrox_write_csr(ndev, BMI_INT, value);
dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value);
}
/**
* clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts
* @ndev: NITROX device
*/
static void clear_nps_core_int_active(struct nitrox_device *ndev)
{
union nps_core_int_active core_int_active;
core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
if (core_int_active.s.nps_core)
clear_nps_core_err_intr(ndev);
if (core_int_active.s.nps_pkt)
clear_nps_pkt_err_intr(ndev);
if (core_int_active.s.pom)
clear_pom_err_intr(ndev);
if (core_int_active.s.pem)
clear_pem_err_intr(ndev);
if (core_int_active.s.lbc)
clear_lbc_err_intr(ndev);
if (core_int_active.s.efl)
clear_efl_err_intr(ndev);
if (core_int_active.s.bmi)
clear_bmi_err_intr(ndev);
/* If more work callback the ISR, set resend */
core_int_active.s.resend = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value);
}
static irqreturn_t nps_core_int_isr(int irq, void *data)
{
struct nitrox_device *ndev = data;
clear_nps_core_int_active(ndev);
return IRQ_HANDLED;
}
static int nitrox_enable_msix(struct nitrox_device *ndev)
{
struct msix_entry *entries;
char **names;
int i, nr_entries, ret;
/*
* PF MSI-X vectors
*
* Entry 0: NPS PKT ring 0
* Entry 1: AQMQ ring 0
* Entry 2: ZQM ring 0
* Entry 3: NPS PKT ring 1
* Entry 4: AQMQ ring 1
* Entry 5: ZQM ring 1
* ....
* Entry 192: NPS_CORE_INT_ACTIVE
*/
nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1;
entries = kzalloc_node(nr_entries * sizeof(struct msix_entry),
GFP_KERNEL, ndev->node);
if (!entries)
return -ENOMEM;
names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL);
if (!names) {
kfree(entries);
return -ENOMEM;
}
/* fill entires */
for (i = 0; i < (nr_entries - 1); i++)
entries[i].entry = i;
entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY;
for (i = 0; i < nr_entries; i++) {
*(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
if (!(*(names + i))) {
ret = -ENOMEM;
goto msix_fail;
}
}
ndev->msix.entries = entries;
ndev->msix.names = names;
ndev->msix.nr_entries = nr_entries;
ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries,
ndev->msix.nr_entries);
if (ret) {
dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n",
ret);
goto msix_fail;
}
return 0;
msix_fail:
for (i = 0; i < nr_entries; i++)
kfree(*(names + i));
kfree(entries);
kfree(names);
return ret;
}
static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev)
{
int i;
if (!ndev->bh.slc)
return;
for (i = 0; i < ndev->nr_queues; i++) {
struct bh_data *bh = &ndev->bh.slc[i];
tasklet_disable(&bh->resp_handler);
tasklet_kill(&bh->resp_handler);
}
kfree(ndev->bh.slc);
ndev->bh.slc = NULL;
}
static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev)
{
u32 size;
int i;
size = ndev->nr_queues * sizeof(struct bh_data);
ndev->bh.slc = kzalloc(size, GFP_KERNEL);
if (!ndev->bh.slc)
return -ENOMEM;
for (i = 0; i < ndev->nr_queues; i++) {
struct bh_data *bh = &ndev->bh.slc[i];
u64 offset;
offset = NPS_PKT_SLC_CNTSX(i);
/* pre calculate completion count address */
bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
bh->cmdq = &ndev->pkt_cmdqs[i];
tasklet_init(&bh->resp_handler, pkt_slc_resp_handler,
(unsigned long)bh);
}
return 0;
}
static int nitrox_request_irqs(struct nitrox_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
struct msix_entry *msix_ent = ndev->msix.entries;
int nr_ring_vectors, i = 0, ring, cpu, ret;
char *name;
/*
* PF MSI-X vectors
*
* Entry 0: NPS PKT ring 0
* Entry 1: AQMQ ring 0
* Entry 2: ZQM ring 0
* Entry 3: NPS PKT ring 1
* ....
* Entry 192: NPS_CORE_INT_ACTIVE
*/
nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS;
/* request irq for pkt ring/ports only */
while (i < nr_ring_vectors) {
name = *(ndev->msix.names + i);
ring = (i / NR_RING_VECTORS);
snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d",
ndev->idx, ring);
ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0,
name, &ndev->bh.slc[ring]);
if (ret) {
dev_err(&pdev->dev, "failed to get irq %d for %s\n",
msix_ent[i].vector, name);
return ret;
}
cpu = ring % num_online_cpus();
irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu));
set_bit(i, ndev->msix.irqs);
i += NR_RING_VECTORS;
}
/* Request IRQ for NPS_CORE_INT_ACTIVE */
name = *(ndev->msix.names + i);
snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx);
ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev);
if (ret) {
dev_err(&pdev->dev, "failed to get irq %d for %s\n",
msix_ent[i].vector, name);
return ret;
}
set_bit(i, ndev->msix.irqs);
return 0;
}
static void nitrox_disable_msix(struct nitrox_device *ndev)
{
struct msix_entry *msix_ent = ndev->msix.entries;
char **names = ndev->msix.names;
int i = 0, ring, nr_ring_vectors;
nr_ring_vectors = ndev->msix.nr_entries - 1;
/* clear pkt ring irqs */
while (i < nr_ring_vectors) {
if (test_and_clear_bit(i, ndev->msix.irqs)) {
ring = (i / NR_RING_VECTORS);
irq_set_affinity_hint(msix_ent[i].vector, NULL);
free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]);
}
i += NR_RING_VECTORS;
}
irq_set_affinity_hint(msix_ent[i].vector, NULL);
free_irq(msix_ent[i].vector, ndev);
clear_bit(i, ndev->msix.irqs);
kfree(ndev->msix.entries);
for (i = 0; i < ndev->msix.nr_entries; i++)
kfree(*(names + i));
kfree(names);
pci_disable_msix(ndev->pdev);
}
/**
* nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ
* @ndev: NITROX device
*/
void nitrox_pf_cleanup_isr(struct nitrox_device *ndev)
{
nitrox_disable_msix(ndev);
nitrox_cleanup_pkt_slc_bh(ndev);
}
/**
* nitrox_init_isr - Initialize PF MSI-X vectors and IRQ
* @ndev: NITROX device
*
* Return: 0 on success, a negative value on failure.
*/
int nitrox_pf_init_isr(struct nitrox_device *ndev)
{
int err;
err = nitrox_setup_pkt_slc_bh(ndev);
if (err)
return err;
err = nitrox_enable_msix(ndev);
if (err)
goto msix_fail;
err = nitrox_request_irqs(ndev);
if (err)
goto irq_fail;
return 0;
irq_fail:
nitrox_disable_msix(ndev);
msix_fail:
nitrox_cleanup_pkt_slc_bh(ndev);
return err;
}

172
drivers/crypto/cavium/nitrox/nitrox_lib.c Normal file
View File

@ -0,0 +1,172 @@
#include <linux/cpumask.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/delay.h>
#include <linux/gfp.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci_regs.h>
#include <linux/vmalloc.h>
#include <linux/pci.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_req.h"
#include "nitrox_csr.h"
#define CRYPTO_CTX_SIZE 256
/* command queue alignments */
#define PKT_IN_ALIGN 16
static int cmdq_common_init(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
u32 qsize;
qsize = (ndev->qlen) * cmdq->instr_size;
cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev),
(qsize + PKT_IN_ALIGN),
&cmdq->dma_unaligned,
GFP_KERNEL);
if (!cmdq->head_unaligned)
return -ENOMEM;
cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN);
cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN);
cmdq->qsize = (qsize + PKT_IN_ALIGN);
spin_lock_init(&cmdq->response_lock);
spin_lock_init(&cmdq->cmdq_lock);
spin_lock_init(&cmdq->backlog_lock);
INIT_LIST_HEAD(&cmdq->response_head);
INIT_LIST_HEAD(&cmdq->backlog_head);
INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
atomic_set(&cmdq->pending_count, 0);
atomic_set(&cmdq->backlog_count, 0);
return 0;
}
static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
cancel_work_sync(&cmdq->backlog_qflush);
dma_free_coherent(DEV(ndev), cmdq->qsize,
cmdq->head_unaligned, cmdq->dma_unaligned);
atomic_set(&cmdq->pending_count, 0);
atomic_set(&cmdq->backlog_count, 0);
cmdq->dbell_csr_addr = NULL;
cmdq->head = NULL;
cmdq->dma = 0;
cmdq->qsize = 0;
cmdq->instr_size = 0;
}
static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
cmdq_common_cleanup(cmdq);
}
kfree(ndev->pkt_cmdqs);
ndev->pkt_cmdqs = NULL;
}
static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev)
{
int i, err, size;
size = ndev->nr_queues * sizeof(struct nitrox_cmdq);
ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL);
if (!ndev->pkt_cmdqs)
return -ENOMEM;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq;
u64 offset;
cmdq = &ndev->pkt_cmdqs[i];
cmdq->ndev = ndev;
cmdq->qno = i;
cmdq->instr_size = sizeof(struct nps_pkt_instr);
offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
/* SE ring doorbell address for this queue */
cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
err = cmdq_common_init(cmdq);
if (err)
goto pkt_cmdq_fail;
}
return 0;
pkt_cmdq_fail:
nitrox_cleanup_pkt_cmdqs(ndev);
return err;
}
static int create_crypto_dma_pool(struct nitrox_device *ndev)
{
size_t size;
/* Crypto context pool, 16 byte aligned */
size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
ndev->ctx_pool = dma_pool_create("crypto-context",
DEV(ndev), size, 16, 0);
if (!ndev->ctx_pool)
return -ENOMEM;
return 0;
}
static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
{
if (!ndev->ctx_pool)
return;
dma_pool_destroy(ndev->ctx_pool);
ndev->ctx_pool = NULL;
}
/**
* nitrox_common_sw_init - allocate software resources.
* @ndev: NITROX device
*
* Allocates crypto context pools and command queues etc.
*
* Return: 0 on success, or a negative error code on error.
*/
int nitrox_common_sw_init(struct nitrox_device *ndev)
{
int err = 0;
/* per device crypto context pool */
err = create_crypto_dma_pool(ndev);
if (err)
return err;
err = nitrox_init_pkt_cmdqs(ndev);
if (err)
destroy_crypto_dma_pool(ndev);
return err;
}
/**
* nitrox_common_sw_cleanup - free software resources.
* @ndev: NITROX device
*/
void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
{
nitrox_cleanup_pkt_cmdqs(ndev);
destroy_crypto_dma_pool(ndev);
}

465
drivers/crypto/cavium/nitrox/nitrox_main.c Normal file
View File

@ -0,0 +1,465 @@
#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"
#define CNN55XX_DEV_ID 0x12
#define MAX_PF_QUEUES 64
#define UCODE_HLEN 48
#define SE_GROUP 0
#define DRIVER_VERSION "1.0"
/* SE microcode */
#define SE_FW "cnn55xx_se.fw"
static const char nitrox_driver_name[] = "CNN55XX";
static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;
/**
* nitrox_pci_tbl - PCI Device ID Table
*/
static const struct pci_device_id nitrox_pci_tbl[] = {
{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
/* required last entry */
{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
/**
* struct ucode - Firmware Header
* @id: microcode ID
* @version: firmware version
* @code_size: code section size
* @raz: alignment
* @code: code section
*/
struct ucode {
u8 id;
char version[VERSION_LEN - 1];
__be32 code_size;
u8 raz[12];
u64 code[0];
};
/**
* write_to_ucd_unit - Write Firmware to NITROX UCD unit
*/
static void write_to_ucd_unit(struct nitrox_device *ndev,
struct ucode *ucode)
{
u32 code_size = be32_to_cpu(ucode->code_size) * 2;
u64 offset, data;
int i = 0;
/*
* UCD structure
*
* -------------
* | BLK 7 |
* -------------
* | BLK 6 |
* -------------
* | ... |
* -------------
* | BLK 0 |
* -------------
* Total of 8 blocks, each size 32KB
*/
/* set the block number */
offset = UCD_UCODE_LOAD_BLOCK_NUM;
nitrox_write_csr(ndev, offset, 0);
code_size = roundup(code_size, 8);
while (code_size) {
data = ucode->code[i];
/* write 8 bytes at a time */
offset = UCD_UCODE_LOAD_IDX_DATAX(i);
nitrox_write_csr(ndev, offset, data);
code_size -= 8;
i++;
}
/* put all SE cores in group 0 */
offset = POM_GRP_EXECMASKX(SE_GROUP);
nitrox_write_csr(ndev, offset, (~0ULL));
for (i = 0; i < ndev->hw.se_cores; i++) {
/*
* write block number and firware length
* bit:<2:0> block number
* bit:3 is set SE uses 32KB microcode
* bit:3 is clear SE uses 64KB microcode
*/
offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, 0x8);
}
usleep_range(300, 400);
}
static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
const struct firmware *fw;
struct ucode *ucode;
int ret;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
if (ret < 0) {
dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
return ret;
}
ucode = (struct ucode *)fw->data;
/* copy the firmware version */
memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
write_to_ucd_unit(ndev, ucode);
release_firmware(fw);
set_bit(NITROX_UCODE_LOADED, &ndev->status);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
return 0;
}
/**
* nitrox_add_to_devlist - add NITROX device to global device list
* @ndev: NITROX device
*/
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
struct nitrox_device *dev;
int ret = 0;
INIT_LIST_HEAD(&ndev->list);
refcount_set(&ndev->refcnt, 1);
mutex_lock(&devlist_lock);
list_for_each_entry(dev, &ndevlist, list) {
if (dev == ndev) {
ret = -EEXIST;
goto unlock;
}
}
ndev->idx = num_devices++;
list_add_tail(&ndev->list, &ndevlist);
unlock:
mutex_unlock(&devlist_lock);
return ret;
}
/**
* nitrox_remove_from_devlist - remove NITROX device from
* global device list
* @ndev: NITROX device
*/
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
mutex_lock(&devlist_lock);
list_del(&ndev->list);
num_devices--;
mutex_unlock(&devlist_lock);
}
static int nitrox_reset_device(struct pci_dev *pdev)
{
int pos = 0;
pos = pci_save_state(pdev);
if (pos) {
dev_err(&pdev->dev, "Failed to save pci state\n");
return -ENOMEM;
}
pos = pci_pcie_cap(pdev);
if (!pos)
return -ENOTTY;
if (!pci_wait_for_pending_transaction(pdev))
dev_err(&pdev->dev, "waiting for pending transaction\n");
pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
msleep(100);
pci_restore_state(pdev);
return 0;
}
static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
int err;
err = nitrox_common_sw_init(ndev);
if (err)
return err;
err = nitrox_pf_init_isr(ndev);
if (err)
nitrox_common_sw_cleanup(ndev);
return err;
}
static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
nitrox_pf_cleanup_isr(ndev);
nitrox_common_sw_cleanup(ndev);
}
/**
* nitrox_bist_check - Check NITORX BIST registers status
* @ndev: NITROX device
*/
static int nitrox_bist_check(struct nitrox_device *ndev)
{
u64 value = 0;
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
}
value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
value += nitrox_read_csr(ndev, POM_BIST_REG);
value += nitrox_read_csr(ndev, BMI_BIST_REG);
value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
value += nitrox_read_csr(ndev, BMO_BIST_REG);
value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
if (value)
return -EIO;
return 0;
}
static void nitrox_get_hwinfo(struct nitrox_device *ndev)
{
union emu_fuse_map emu_fuse;
u64 offset;
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
u8 dead_cores;
offset = EMU_FUSE_MAPX(i);
emu_fuse.value = nitrox_read_csr(ndev, offset);
if (emu_fuse.s.valid) {
dead_cores = hweight32(emu_fuse.s.ae_fuse);
ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
dead_cores = hweight16(emu_fuse.s.se_fuse);
ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
}
}
}
static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
int err;
err = nitrox_bist_check(ndev);
if (err) {
dev_err(&ndev->pdev->dev, "BIST check failed\n");
return err;
}
/* get cores information */
nitrox_get_hwinfo(ndev);
nitrox_config_nps_unit(ndev);
nitrox_config_pom_unit(ndev);
nitrox_config_efl_unit(ndev);
/* configure IO units */
nitrox_config_bmi_unit(ndev);
nitrox_config_bmo_unit(ndev);
/* configure Local Buffer Cache */
nitrox_config_lbc_unit(ndev);
nitrox_config_rand_unit(ndev);
/* load firmware on SE cores */
err = nitrox_load_fw(ndev, SE_FW);
if (err)
return err;
nitrox_config_emu_unit(ndev);
return 0;
}
/**
* nitrox_probe - NITROX Initialization function.
* @pdev: PCI device information struct
* @id: entry in nitrox_pci_tbl
*
* Return: 0, if the driver is bound to the device, or
* a negative error if there is failure.
*/
static int nitrox_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
{
struct nitrox_device *ndev;
int err;
dev_info_once(&pdev->dev, "%s driver version %s\n",
nitrox_driver_name, DRIVER_VERSION);
err = pci_enable_device_mem(pdev);
if (err)
return err;
/* do FLR */
err = nitrox_reset_device(pdev);
if (err) {
dev_err(&pdev->dev, "FLR failed\n");
pci_disable_device(pdev);
return err;
}
if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
} else {
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "DMA configuration failed\n");
pci_disable_device(pdev);
return err;
}
}
err = pci_request_mem_regions(pdev, nitrox_driver_name);
if (err) {
pci_disable_device(pdev);
return err;
}
pci_set_master(pdev);
ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
if (!ndev)
goto ndev_fail;
pci_set_drvdata(pdev, ndev);
ndev->pdev = pdev;
/* add to device list */
nitrox_add_to_devlist(ndev);
ndev->hw.vendor_id = pdev->vendor;
ndev->hw.device_id = pdev->device;
ndev->hw.revision_id = pdev->revision;
/* command timeout in jiffies */
ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
ndev->node = dev_to_node(&pdev->dev);
if (ndev->node == NUMA_NO_NODE)
ndev->node = 0;
ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
pci_resource_len(pdev, 0));
if (!ndev->bar_addr) {
err = -EIO;
goto ioremap_err;
}
/* allocate command queus based on cpus, max queues are 64 */
ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
ndev->qlen = qlen;
err = nitrox_pf_sw_init(ndev);
if (err)
goto ioremap_err;
err = nitrox_pf_hw_init(ndev);
if (err)
goto pf_hw_fail;
set_bit(NITROX_READY, &ndev->status);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
return 0;
pf_hw_fail:
nitrox_pf_sw_cleanup(ndev);
ioremap_err:
nitrox_remove_from_devlist(ndev);
kfree(ndev);
pci_set_drvdata(pdev, NULL);
ndev_fail:
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
return err;
}
/**
* nitrox_remove - Unbind the driver from the device.
* @pdev: PCI device information struct
*/
static void nitrox_remove(struct pci_dev *pdev)
{
struct nitrox_device *ndev = pci_get_drvdata(pdev);
if (!ndev)
return;
if (!refcount_dec_and_test(&ndev->refcnt)) {
dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
refcount_read(&ndev->refcnt));
return;
}
dev_info(DEV(ndev), "Removing Device %x:%x\n",
ndev->hw.vendor_id, ndev->hw.device_id);
clear_bit(NITROX_READY, &ndev->status);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
nitrox_remove_from_devlist(ndev);
nitrox_pf_sw_cleanup(ndev);
iounmap(ndev->bar_addr);
kfree(ndev);
pci_set_drvdata(pdev, NULL);
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
}
static void nitrox_shutdown(struct pci_dev *pdev)
{
pci_set_drvdata(pdev, NULL);
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
}
static struct pci_driver nitrox_driver = {
.name = nitrox_driver_name,
.id_table = nitrox_pci_tbl,
.probe = nitrox_probe,
.remove = nitrox_remove,
.shutdown = nitrox_shutdown,
};
module_pci_driver(nitrox_driver);
MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);

445
drivers/crypto/cavium/nitrox/nitrox_req.h Normal file
View File

@ -0,0 +1,445 @@
#ifndef __NITROX_REQ_H
#define __NITROX_REQ_H
#include <linux/dma-mapping.h>
#include <crypto/aes.h>
#include "nitrox_dev.h"
/**
* struct gphdr - General purpose Header
* @param0: first parameter.
* @param1: second parameter.
* @param2: third parameter.
* @param3: fourth parameter.
*
* Params tell the iv and enc/dec data offsets.
*/
struct gphdr {
__be16 param0;
__be16 param1;
__be16 param2;
__be16 param3;
};
/**
* struct se_req_ctrl - SE request information.
* @arg: Minor number of the opcode
* @ctxc: Context control.
* @unca: Uncertainity enabled.
* @info: Additional information for SE cores.
* @ctxl: Context length in bytes.
* @uddl: User defined data length
*/
union se_req_ctrl {
u64 value;
struct {
u64 raz : 22;
u64 arg : 8;
u64 ctxc : 2;
u64 unca : 1;
u64 info : 3;
u64 unc : 8;
u64 ctxl : 12;
u64 uddl : 8;
} s;
};
struct nitrox_sglist {
u16 len;
u16 raz0;
u32 raz1;
dma_addr_t dma;
};
#define MAX_IV_LEN 16
/**
* struct se_crypto_request - SE crypto request structure.
* @opcode: Request opcode (enc/dec)
* @flags: flags from crypto subsystem
* @ctx_handle: Crypto context handle.
* @gph: GP Header
* @ctrl: Request Information.
* @in: Input sglist
* @out: Output sglist
*/
struct se_crypto_request {
u8 opcode;
gfp_t gfp;
u32 flags;
u64 ctx_handle;
struct gphdr gph;
union se_req_ctrl ctrl;
u8 iv[MAX_IV_LEN];
u16 ivsize;
struct scatterlist *src;
struct scatterlist *dst;
};
/* Crypto opcodes */
#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
#define ENCRYPT 0
#define DECRYPT 1
/* IV from context */
#define IV_FROM_CTX 0
/* IV from Input data */
#define IV_FROM_DPTR 1
/**
* cipher opcodes for firmware
*/
enum flexi_cipher {
CIPHER_NULL = 0,
CIPHER_3DES_CBC,
CIPHER_3DES_ECB,
CIPHER_AES_CBC,
CIPHER_AES_ECB,
CIPHER_AES_CFB,
CIPHER_AES_CTR,
CIPHER_AES_GCM,
CIPHER_AES_XTS,
CIPHER_AES_CCM,
CIPHER_AES_CBC_CTS,
CIPHER_AES_ECB_CTS,
CIPHER_INVALID
};
/**
* struct crypto_keys - Crypto keys
* @key: Encryption key or KEY1 for AES-XTS
* @iv: Encryption IV or Tweak for AES-XTS
*/
struct crypto_keys {
union {
u8 key[AES_MAX_KEY_SIZE];
u8 key1[AES_MAX_KEY_SIZE];
} u;
u8 iv[AES_BLOCK_SIZE];
};
/**
* struct auth_keys - Authentication keys
* @ipad: IPAD or KEY2 for AES-XTS
* @opad: OPAD or AUTH KEY if auth_input_type = 1
*/
struct auth_keys {
union {
u8 ipad[64];
u8 key2[64];
} u;
u8 opad[64];
};
/**
* struct flexi_crypto_context - Crypto context
* @cipher_type: Encryption cipher type
* @aes_keylen: AES key length
* @iv_source: Encryption IV source
* @hash_type: Authentication type
* @auth_input_type: Authentication input type
* 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
* 0 - Authentication OPAD/IPAD
* @mac_len: mac length
* @crypto: Crypto keys
* @auth: Authentication keys
*/
struct flexi_crypto_context {
union {
__be64 flags;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 cipher_type : 4;
u64 reserved_59 : 1;
u64 aes_keylen : 2;
u64 iv_source : 1;
u64 hash_type : 4;
u64 reserved_49_51 : 3;
u64 auth_input_type: 1;
u64 mac_len : 8;
u64 reserved_0_39 : 40;
#else
u64 reserved_0_39 : 40;
u64 mac_len : 8;
u64 auth_input_type: 1;
u64 reserved_49_51 : 3;
u64 hash_type : 4;
u64 iv_source : 1;
u64 aes_keylen : 2;
u64 reserved_59 : 1;
u64 cipher_type : 4;
#endif
} w0;
};
struct crypto_keys crypto;
struct auth_keys auth;
};
struct nitrox_crypto_ctx {
struct nitrox_device *ndev;
union {
u64 ctx_handle;
struct flexi_crypto_context *fctx;
} u;
};
struct nitrox_kcrypt_request {
struct se_crypto_request creq;
struct nitrox_crypto_ctx *nctx;
struct skcipher_request *skreq;
};
/**
* struct pkt_instr_hdr - Packet Instruction Header
* @g: Gather used
* When [G] is set and [GSZ] != 0, the instruction is
* indirect gather instruction.
* When [G] is set and [GSZ] = 0, the instruction is
* direct gather instruction.
* @gsz: Number of pointers in the indirect gather list
* @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
* and adds them to the packet after the pkt_instr_hdr but before any UDD
* @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
* when [IHI] is set.
* @fsz: The number of front data bytes directly included in the
* PCIe instruction.
* @tlen: The length of the input packet in bytes, include:
* - 16B pkt_hdr
* - Inline context bytes if any,
* - UDD if any,
* - packet payload bytes
*/
union pkt_instr_hdr {
u64 value;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 raz_48_63 : 16;
u64 g : 1;
u64 gsz : 7;
u64 ihi : 1;
u64 ssz : 7;
u64 raz_30_31 : 2;
u64 fsz : 6;
u64 raz_16_23 : 8;
u64 tlen : 16;
#else
u64 tlen : 16;
u64 raz_16_23 : 8;
u64 fsz : 6;
u64 raz_30_31 : 2;
u64 ssz : 7;
u64 ihi : 1;
u64 gsz : 7;
u64 g : 1;
u64 raz_48_63 : 16;
#endif
} s;
};
/**
* struct pkt_hdr - Packet Input Header
* @opcode: Request opcode (Major)
* @arg: Request opcode (Minor)
* @ctxc: Context control.
* @unca: When set [UNC] is the uncertainty count for an input packet.
* The hardware uses uncertainty counts to predict
* output buffer use and avoid deadlock.
* @info: Not used by input hardware. Available for use
* during SE processing.
* @destport: The expected destination port/ring/channel for the packet.
* @unc: Uncertainty count for an input packet.
* @grp: SE group that will process the input packet.
* @ctxl: Context Length in 64-bit words.
* @uddl: User-defined data (UDD) length in bytes.
* @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
*/
union pkt_hdr {
u64 value[2];
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 opcode : 8;
u64 arg : 8;
u64 ctxc : 2;
u64 unca : 1;
u64 raz_44 : 1;
u64 info : 3;
u64 destport : 9;
u64 unc : 8;
u64 raz_19_23 : 5;
u64 grp : 3;
u64 raz_15 : 1;
u64 ctxl : 7;
u64 uddl : 8;
#else
u64 uddl : 8;
u64 ctxl : 7;
u64 raz_15 : 1;
u64 grp : 3;
u64 raz_19_23 : 5;
u64 unc : 8;
u64 destport : 9;
u64 info : 3;
u64 raz_44 : 1;
u64 unca : 1;
u64 ctxc : 2;
u64 arg : 8;
u64 opcode : 8;
#endif
__be64 ctxp;
} s;
};
/**
* struct slc_store_info - Solicited Paceket Output Store Information.
* @ssz: The number of scatterlist pointers for the solicited output port
* packet.
* @rptr: The result pointer for the solicited output port packet.
* If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
* host that is large enough to hold the entire output packet.
* If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
* sglist components at [RPTR] on the remote host.
*/
union slc_store_info {
u64 value[2];
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 raz_39_63 : 25;
u64 ssz : 7;
u64 raz_0_31 : 32;
#else
u64 raz_0_31 : 32;
u64 ssz : 7;
u64 raz_39_63 : 25;
#endif
__be64 rptr;
} s;
};
/**
* struct nps_pkt_instr - NPS Packet Instruction of SE cores.
* @dptr0 : Input pointer points to buffer in remote host.
* @ih: Packet Instruction Header (8 bytes)
* @irh: Packet Input Header (16 bytes)
* @slc: Solicited Packet Output Store Information (16 bytes)
* @fdata: Front data
*
* 64-Byte Instruction Format
*/
struct nps_pkt_instr {
__be64 dptr0;
union pkt_instr_hdr ih;
union pkt_hdr irh;
union slc_store_info slc;
u64 fdata[2];
};
/**
* struct ctx_hdr - Book keeping data about the crypto context
* @pool: Pool used to allocate crypto context
* @dma: Base DMA address of the cypto context
* @ctx_dma: Actual usable crypto context for NITROX
*/
struct ctx_hdr {
struct dma_pool *pool;
dma_addr_t dma;
dma_addr_t ctx_dma;
};
/*
* struct sglist_component - SG list component format
* @len0: The number of bytes at [PTR0] on the remote host.
* @len1: The number of bytes at [PTR1] on the remote host.
* @len2: The number of bytes at [PTR2] on the remote host.
* @len3: The number of bytes at [PTR3] on the remote host.
* @dma0: First pointer point to buffer in remote host.
* @dma1: Second pointer point to buffer in remote host.
* @dma2: Third pointer point to buffer in remote host.
* @dma3: Fourth pointer point to buffer in remote host.
*/
struct nitrox_sgcomp {
__be16 len[4];
__be64 dma[4];
};
/*
* strutct nitrox_sgtable - SG list information
* @map_cnt: Number of buffers mapped
* @nr_comp: Number of sglist components
* @total_bytes: Total bytes in sglist.
* @len: Total sglist components length.
* @dma: DMA address of sglist component.
* @dir: DMA direction.
* @buf: crypto request buffer.
* @sglist: SG list of input/output buffers.
* @sgcomp: sglist component for NITROX.
*/
struct nitrox_sgtable {
u8 map_bufs_cnt;
u8 nr_sgcomp;
u16 total_bytes;
u32 len;
dma_addr_t dma;
enum dma_data_direction dir;
struct scatterlist *buf;
struct nitrox_sglist *sglist;
struct nitrox_sgcomp *sgcomp;
};
/* Response Header Length */
#define ORH_HLEN 8
/* Completion bytes Length */
#define COMP_HLEN 8
struct resp_hdr {
u64 orh;
dma_addr_t orh_dma;
u64 completion;
dma_addr_t completion_dma;
};
typedef void (*completion_t)(struct skcipher_request *skreq, int err);
/**
* struct nitrox_softreq - Represents the NIROX Request.
* @response: response list entry
* @backlog: Backlog list entry
* @ndev: Device used to submit the request
* @cmdq: Command queue for submission
* @resp: Response headers
* @instr: 64B instruction
* @in: SG table for input
* @out SG table for output
* @tstamp: Request submitted time in jiffies
* @callback: callback after request completion/timeout
* @cb_arg: callback argument
*/
struct nitrox_softreq {
struct list_head response;
struct list_head backlog;
u32 flags;
gfp_t gfp;
atomic_t status;
bool inplace;
struct nitrox_device *ndev;
struct nitrox_cmdq *cmdq;
struct nps_pkt_instr instr;
struct resp_hdr resp;
struct nitrox_sgtable in;
struct nitrox_sgtable out;
unsigned long tstamp;
completion_t callback;
struct skcipher_request *skreq;
};
#endif /* __NITROX_REQ_H */

732
drivers/crypto/cavium/nitrox/nitrox_reqmgr.c Normal file
View File

@ -0,0 +1,732 @@
#include <linux/gfp.h>
#include <linux/workqueue.h>
#include <crypto/internal/skcipher.h>
#include "nitrox_dev.h"
#include "nitrox_req.h"
#include "nitrox_csr.h"
#include "nitrox_req.h"
/* SLC_STORE_INFO */
#define MIN_UDD_LEN 16
/* PKT_IN_HDR + SLC_STORE_INFO */
#define FDATA_SIZE 32
/* Base destination port for the solicited requests */
#define SOLICIT_BASE_DPORT 256
#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
#define REQ_NOT_POSTED 1
#define REQ_BACKLOG 2
#define REQ_POSTED 3
/**
* Response codes from SE microcode
* 0x00 - Success
* Completion with no error
* 0x43 - ERR_GC_DATA_LEN_INVALID
* Invalid Data length if Encryption Data length is
* less than 16 bytes for AES-XTS and AES-CTS.
* 0x45 - ERR_GC_CTX_LEN_INVALID
* Invalid context length: CTXL != 23 words.
* 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
* DOCSIS support is enabled with other than
* AES/DES-CBC mode encryption.
* 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
* Authentication offset is other than 0 with
* Encryption IV source = 0.
* Authentication offset is other than 8 (DES)/16 (AES)
* with Encryption IV source = 1
* 0x51 - ERR_GC_CRC32_INVALID_SELECTION
* CRC32 is enabled for other than DOCSIS encryption.
* 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
* Invalid flag options in AES-CCM IV.
*/
/**
* dma_free_sglist - unmap and free the sg lists.
* @ndev: N5 device
* @sgtbl: SG table
*/
static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
{
struct nitrox_device *ndev = sr->ndev;
struct device *dev = DEV(ndev);
struct nitrox_sglist *sglist;
/* unmap in sgbuf */
sglist = sr->in.sglist;
if (!sglist)
goto out_unmap;
/* unmap iv */
dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
/* unmpa src sglist */
dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
/* unamp gather component */
dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
kfree(sr->in.sglist);
kfree(sr->in.sgcomp);
sr->in.sglist = NULL;
sr->in.buf = NULL;
sr->in.map_bufs_cnt = 0;
out_unmap:
/* unmap out sgbuf */
sglist = sr->out.sglist;
if (!sglist)
return;
/* unmap orh */
dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
/* unmap dst sglist */
if (!sr->inplace) {
dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
sr->out.dir);
}
/* unmap completion */
dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
/* unmap scatter component */
dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
kfree(sr->out.sglist);
kfree(sr->out.sgcomp);
sr->out.sglist = NULL;
sr->out.buf = NULL;
sr->out.map_bufs_cnt = 0;
}
static void softreq_destroy(struct nitrox_softreq *sr)
{
softreq_unmap_sgbufs(sr);
kfree(sr);
}
/**
* create_sg_component - create SG componets for N5 device.
* @sr: Request structure
* @sgtbl: SG table
* @nr_comp: total number of components required
*
* Component structure
*
* 63 48 47 32 31 16 15 0
* --------------------------------------
* | LEN0 | LEN1 | LEN2 | LEN3 |
* |-------------------------------------
* | PTR0 |
* --------------------------------------
* | PTR1 |
* --------------------------------------
* | PTR2 |
* --------------------------------------
* | PTR3 |
* --------------------------------------
*
* Returns 0 if success or a negative errno code on error.
*/
static int create_sg_component(struct nitrox_softreq *sr,
struct nitrox_sgtable *sgtbl, int map_nents)
{
struct nitrox_device *ndev = sr->ndev;
struct nitrox_sgcomp *sgcomp;
struct nitrox_sglist *sglist;
dma_addr_t dma;
size_t sz_comp;
int i, j, nr_sgcomp;
nr_sgcomp = roundup(map_nents, 4) / 4;
/* each component holds 4 dma pointers */
sz_comp = nr_sgcomp * sizeof(*sgcomp);
sgcomp = kzalloc(sz_comp, sr->gfp);
if (!sgcomp)
return -ENOMEM;
sgtbl->sgcomp = sgcomp;
sgtbl->nr_sgcomp = nr_sgcomp;
sglist = sgtbl->sglist;
/* populate device sg component */
for (i = 0; i < nr_sgcomp; i++) {
for (j = 0; j < 4; j++) {
sgcomp->len[j] = cpu_to_be16(sglist->len);
sgcomp->dma[j] = cpu_to_be64(sglist->dma);
sglist++;
}
sgcomp++;
}
/* map the device sg component */
dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
if (dma_mapping_error(DEV(ndev), dma)) {
kfree(sgtbl->sgcomp);
sgtbl->sgcomp = NULL;
return -ENOMEM;
}
sgtbl->dma = dma;
sgtbl->len = sz_comp;
return 0;
}
/**
* dma_map_inbufs - DMA map input sglist and creates sglist component
* for N5 device.
* @sr: Request structure
* @req: Crypto request structre
*
* Returns 0 if successful or a negative errno code on error.
*/
static int dma_map_inbufs(struct nitrox_softreq *sr,
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
struct scatterlist *sg = req->src;
struct nitrox_sglist *glist;
int i, nents, ret = 0;
dma_addr_t dma;
size_t sz;
nents = sg_nents(req->src);
/* creater gather list IV and src entries */
sz = roundup((1 + nents), 4) * sizeof(*glist);
glist = kzalloc(sz, sr->gfp);
if (!glist)
return -ENOMEM;
sr->in.sglist = glist;
/* map IV */
dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
ret = dma_mapping_error(dev, dma);
if (ret)
goto iv_map_err;
sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
/* map src entries */
nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
if (!nents) {
ret = -EINVAL;
goto src_map_err;
}
sr->in.buf = req->src;
/* store the mappings */
glist->len = req->ivsize;
glist->dma = dma;
glist++;
sr->in.total_bytes += req->ivsize;
for_each_sg(req->src, sg, nents, i) {
glist->len = sg_dma_len(sg);
glist->dma = sg_dma_address(sg);
sr->in.total_bytes += glist->len;
glist++;
}
/* roundup map count to align with entires in sg component */
sr->in.map_bufs_cnt = (1 + nents);
/* create NITROX gather component */
ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
if (ret)
goto incomp_err;
return 0;
incomp_err:
dma_unmap_sg(dev, req->src, nents, sr->in.dir);
sr->in.map_bufs_cnt = 0;
src_map_err:
dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
iv_map_err:
kfree(sr->in.sglist);
sr->in.sglist = NULL;
return ret;
}
static int dma_map_outbufs(struct nitrox_softreq *sr,
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
struct nitrox_sglist *glist = sr->in.sglist;
struct nitrox_sglist *slist;
struct scatterlist *sg;
int i, nents, map_bufs_cnt, ret = 0;
size_t sz;
nents = sg_nents(req->dst);
/* create scatter list ORH, IV, dst entries and Completion header */
sz = roundup((3 + nents), 4) * sizeof(*slist);
slist = kzalloc(sz, sr->gfp);
if (!slist)
return -ENOMEM;
sr->out.sglist = slist;
sr->out.dir = DMA_BIDIRECTIONAL;
/* map ORH */
sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
sr->out.dir);
ret = dma_mapping_error(dev, sr->resp.orh_dma);
if (ret)
goto orh_map_err;
/* map completion */
sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
COMP_HLEN, sr->out.dir);
ret = dma_mapping_error(dev, sr->resp.completion_dma);
if (ret)
goto compl_map_err;
sr->inplace = (req->src == req->dst) ? true : false;
/* out place */
if (!sr->inplace) {
nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
if (!nents) {
ret = -EINVAL;
goto dst_map_err;
}
}
sr->out.buf = req->dst;
/* store the mappings */
/* orh */
slist->len = ORH_HLEN;
slist->dma = sr->resp.orh_dma;
slist++;
/* copy the glist mappings */
if (sr->inplace) {
nents = sr->in.map_bufs_cnt - 1;
map_bufs_cnt = sr->in.map_bufs_cnt;
while (map_bufs_cnt--) {
slist->len = glist->len;
slist->dma = glist->dma;
slist++;
glist++;
}
} else {
/* copy iv mapping */
slist->len = glist->len;
slist->dma = glist->dma;
slist++;
/* copy remaining maps */
for_each_sg(req->dst, sg, nents, i) {
slist->len = sg_dma_len(sg);
slist->dma = sg_dma_address(sg);
slist++;
}
}
/* completion */
slist->len = COMP_HLEN;
slist->dma = sr->resp.completion_dma;
sr->out.map_bufs_cnt = (3 + nents);
ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
if (ret)
goto outcomp_map_err;
return 0;
outcomp_map_err:
if (!sr->inplace)
dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
sr->out.map_bufs_cnt = 0;
sr->out.buf = NULL;
dst_map_err:
dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
sr->resp.completion_dma = 0;
compl_map_err:
dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
sr->resp.orh_dma = 0;
orh_map_err:
kfree(sr->out.sglist);
sr->out.sglist = NULL;
return ret;
}
static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
struct se_crypto_request *creq)
{
int ret;
ret = dma_map_inbufs(sr, creq);
if (ret)
return ret;
ret = dma_map_outbufs(sr, creq);
if (ret)
softreq_unmap_sgbufs(sr);
return ret;
}
static inline void backlog_list_add(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
INIT_LIST_HEAD(&sr->backlog);
spin_lock_bh(&cmdq->backlog_lock);
list_add_tail(&sr->backlog, &cmdq->backlog_head);
atomic_inc(&cmdq->backlog_count);
atomic_set(&sr->status, REQ_BACKLOG);
spin_unlock_bh(&cmdq->backlog_lock);
}
static inline void response_list_add(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
INIT_LIST_HEAD(&sr->response);
spin_lock_bh(&cmdq->response_lock);
list_add_tail(&sr->response, &cmdq->response_head);
spin_unlock_bh(&cmdq->response_lock);
}
static inline void response_list_del(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
spin_lock_bh(&cmdq->response_lock);
list_del(&sr->response);
spin_unlock_bh(&cmdq->response_lock);
}
static struct nitrox_softreq *
get_first_response_entry(struct nitrox_cmdq *cmdq)
{
return list_first_entry_or_null(&cmdq->response_head,
struct nitrox_softreq, response);
}
static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
{
if (atomic_inc_return(&cmdq->pending_count) > qlen) {
atomic_dec(&cmdq->pending_count);
/* sync with other cpus */
smp_mb__after_atomic();
return true;
}
return false;
}
/**
* post_se_instr - Post SE instruction to Packet Input ring
* @sr: Request structure
*
* Returns 0 if successful or a negative error code,
* if no space in ring.
*/
static void post_se_instr(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = sr->ndev;
union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell;
u64 offset;
u8 *ent;
spin_lock_bh(&cmdq->cmdq_lock);
/* get the next write offset */
offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno);
pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset);
/* copy the instruction */
ent = cmdq->head + pkt_in_baoff_dbell.s.aoff;
memcpy(ent, &sr->instr, cmdq->instr_size);
/* flush the command queue updates */
dma_wmb();
sr->tstamp = jiffies;
atomic_set(&sr->status, REQ_POSTED);
response_list_add(sr, cmdq);
/* Ring doorbell with count 1 */
writeq(1, cmdq->dbell_csr_addr);
/* orders the doorbell rings */
mmiowb();
spin_unlock_bh(&cmdq->cmdq_lock);
}
static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
struct nitrox_softreq *sr, *tmp;
int ret = 0;
spin_lock_bh(&cmdq->backlog_lock);
list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
struct skcipher_request *skreq;
/* submit until space available */
if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
ret = -EBUSY;
break;
}
/* delete from backlog list */
list_del(&sr->backlog);
atomic_dec(&cmdq->backlog_count);
/* sync with other cpus */
smp_mb__after_atomic();
skreq = sr->skreq;
/* post the command */
post_se_instr(sr, cmdq);
/* backlog requests are posted, wakeup with -EINPROGRESS */
skcipher_request_complete(skreq, -EINPROGRESS);
}
spin_unlock_bh(&cmdq->backlog_lock);
return ret;
}
static int nitrox_enqueue_request(struct nitrox_softreq *sr)
{
struct nitrox_cmdq *cmdq = sr->cmdq;
struct nitrox_device *ndev = sr->ndev;
int ret = -EBUSY;
if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EAGAIN;
backlog_list_add(sr, cmdq);
} else {
ret = post_backlog_cmds(cmdq);
if (ret) {
backlog_list_add(sr, cmdq);
return ret;
}
post_se_instr(sr, cmdq);
ret = -EINPROGRESS;
}
return ret;
}
/**
* nitrox_se_request - Send request to SE core
* @ndev: NITROX device
* @req: Crypto request
*
* Returns 0 on success, or a negative error code.
*/
int nitrox_process_se_request(struct nitrox_device *ndev,
struct se_crypto_request *req,
completion_t callback,
struct skcipher_request *skreq)
{
struct nitrox_softreq *sr;
dma_addr_t ctx_handle = 0;
int qno, ret = 0;
if (!nitrox_ready(ndev))
return -ENODEV;
sr = kzalloc(sizeof(*sr), req->gfp);
if (!sr)
return -ENOMEM;
sr->ndev = ndev;
sr->flags = req->flags;
sr->gfp = req->gfp;
sr->callback = callback;
sr->skreq = skreq;
atomic_set(&sr->status, REQ_NOT_POSTED);
WRITE_ONCE(sr->resp.orh, PENDING_SIG);
WRITE_ONCE(sr->resp.completion, PENDING_SIG);
ret = softreq_map_iobuf(sr, req);
if (ret) {
kfree(sr);
return ret;
}
/* get the context handle */
if (req->ctx_handle) {
struct ctx_hdr *hdr;
u8 *ctx_ptr;
ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
ctx_handle = hdr->ctx_dma;
}
/* select the queue */
qno = smp_processor_id() % ndev->nr_queues;
sr->cmdq = &ndev->pkt_cmdqs[qno];
/*
* 64-Byte Instruction Format
*
* ----------------------
* | DPTR0 | 8 bytes
* ----------------------
* | PKT_IN_INSTR_HDR | 8 bytes
* ----------------------
* | PKT_IN_HDR | 16 bytes
* ----------------------
* | SLC_INFO | 16 bytes
* ----------------------
* | Front data | 16 bytes
* ----------------------
*/
/* fill the packet instruction */
/* word 0 */
sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
/* word 1 */
sr->instr.ih.value = 0;
sr->instr.ih.s.g = 1;
sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
/* word 2 */
sr->instr.irh.value[0] = 0;
sr->instr.irh.s.uddl = MIN_UDD_LEN;
/* context length in 64-bit words */
sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
/* offset from solicit base port 256 */
sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
sr->instr.irh.s.arg = req->ctrl.s.arg;
sr->instr.irh.s.opcode = req->opcode;
sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
/* word 3 */
sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
/* word 4 */
sr->instr.slc.value[0] = 0;
sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
/* word 5 */
sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
/*
* No conversion for front data,
* It goes into payload
* put GP Header in front data
*/
sr->instr.fdata[0] = *((u64 *)&req->gph);
sr->instr.fdata[1] = 0;
/* flush the soft_req changes before posting the cmd */
wmb();
ret = nitrox_enqueue_request(sr);
if (ret == -EAGAIN)
goto send_fail;
return ret;
send_fail:
softreq_destroy(sr);
return ret;
}
static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
{
return time_after_eq(jiffies, (tstamp + timeout));
}
void backlog_qflush_work(struct work_struct *work)
{
struct nitrox_cmdq *cmdq;
cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
post_backlog_cmds(cmdq);
}
/**
* process_request_list - process completed requests
* @ndev: N5 device
* @qno: queue to operate
*
* Returns the number of responses processed.
*/
static void process_response_list(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
struct nitrox_softreq *sr;
struct skcipher_request *skreq;
completion_t callback;
int req_completed = 0, err = 0, budget;
/* check all pending requests */
budget = atomic_read(&cmdq->pending_count);
while (req_completed < budget) {
sr = get_first_response_entry(cmdq);
if (!sr)
break;
if (atomic_read(&sr->status) != REQ_POSTED)
break;
/* check orh and completion bytes updates */
if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
/* request not completed, check for timeout */
if (!cmd_timeout(sr->tstamp, ndev->timeout))
break;
dev_err_ratelimited(DEV(ndev),
"Request timeout, orh 0x%016llx\n",
READ_ONCE(sr->resp.orh));
}
atomic_dec(&cmdq->pending_count);
/* sync with other cpus */
smp_mb__after_atomic();
/* remove from response list */
response_list_del(sr, cmdq);
callback = sr->callback;
skreq = sr->skreq;
/* ORH error code */
err = READ_ONCE(sr->resp.orh) & 0xff;
softreq_destroy(sr);
if (callback)
callback(skreq, err);
req_completed++;
}
}
/**
* pkt_slc_resp_handler - post processing of SE responses
*/
void pkt_slc_resp_handler(unsigned long data)
{
struct bh_data *bh = (void *)(uintptr_t)(data);
struct nitrox_cmdq *cmdq = bh->cmdq;
union nps_pkt_slc_cnts pkt_slc_cnts;
/* read completion count */
pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
/* resend the interrupt if more work to do */
pkt_slc_cnts.s.resend = 1;
process_response_list(cmdq);
/*
* clear the interrupt with resend bit enabled,
* MSI-X interrupt generates if Completion count > Threshold
*/
writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
/* order the writes */
mmiowb();
if (atomic_read(&cmdq->backlog_count))
schedule_work(&cmdq->backlog_qflush);
}