hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chcr: Add support for Inline IPSec 

register xfrmdev_ops callbacks, Send IPsec tunneled data
to HW for inline processing.
The driver use hardware crypto accelerator to encrypt and
generate ICV for the transmitted packet in Inline mode.

Signed-off-by: Atul Gupta <atul.gupta@chelsio.com>
Signed-off-by: Harsh Jain <harsh@chelsio.com>
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Atul Gupta 2017-11-16 16:57:08 +05:30 committed by Herbert Xu
parent a6ec572bfa
commit 6dad4e8ab3
8 changed files with 860 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
drivers/crypto/chelsio/Kconfig
View File

@ -18,3 +18,13 @@ config CRYPTO_DEV_CHELSIO
To compile this driver as a module, choose M here: the module
will be called chcr.
config CHELSIO_IPSEC_INLINE
bool "Chelsio IPSec XFRM Tx crypto offload"
depends on CHELSIO_T4
depends on CRYPTO_DEV_CHELSIO
depends on XFRM_OFFLOAD
depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
default n
---help---
Enable support for IPSec Tx Inline.

1
drivers/crypto/chelsio/Makefile
View File

@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
chcr-objs := chcr_core.o chcr_algo.o
chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o

221
drivers/crypto/chelsio/chcr_algo.c
View File

@ -73,6 +73,25 @@
#define IV AES_BLOCK_SIZE
unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
96, 112, 120, 136, 144, 160, 168, 184,
192, 208, 216, 232, 240, 256, 264, 280,
288, 304, 312, 328, 336, 352, 360, 376};
unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
112, 112, 128, 128, 144, 144, 160, 160,
192, 192, 208, 208, 224, 224, 240, 240,
272, 272, 288, 288, 304, 304, 320, 320};
static u32 round_constant[11] = {
0x01000000, 0x02000000, 0x04000000, 0x08000000,
0x10000000, 0x20000000, 0x40000000, 0x80000000,
0x1B000000, 0x36000000, 0x6C000000
};
static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err);
static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
{
return ctx->crypto_ctx->aeadctx;
@ -108,18 +127,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb)
return (skb->len <= SGE_MAX_WR_LEN);
}
/*
* sgl_len - calculates the size of an SGL of the given capacity
* @n: the number of SGL entries
* Calculates the number of flits needed for a scatter/gather list that
* can hold the given number of entries.
*/
static inline unsigned int sgl_len(unsigned int n)
{
n--;
return (3 * n) / 2 + (n & 1) + 2;
}
static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
unsigned int entlen,
unsigned int skip)
@ -182,30 +189,17 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req,
}
out:
req->base.complete(&req->base, err);
}
}
static inline void chcr_handle_aead_resp(struct aead_request *req,
unsigned char *input,
int err)
static inline int get_aead_subtype(struct crypto_aead *aead)
{
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
if (reqctx->b0_dma)
dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
reqctx->b0_len, DMA_BIDIRECTIONAL);
if (reqctx->verify == VERIFY_SW) {
chcr_verify_tag(req, input, &err);
reqctx->verify = VERIFY_HW;
struct aead_alg *alg = crypto_aead_alg(aead);
struct chcr_alg_template *chcr_crypto_alg =
container_of(alg, struct chcr_alg_template, alg.aead);
return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
}
req->base.complete(&req->base, err);
}
static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
{
u8 temp[SHA512_DIGEST_SIZE];
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@ -230,6 +224,25 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
*err = 0;
}
static inline void chcr_handle_aead_resp(struct aead_request *req,
unsigned char *input,
int err)
{
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
if (reqctx->b0_dma)
dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
reqctx->b0_len, DMA_BIDIRECTIONAL);
if (reqctx->verify == VERIFY_SW) {
chcr_verify_tag(req, input, &err);
reqctx->verify = VERIFY_HW;
}
req->base.complete(&req->base, err);
}
/*
* chcr_handle_resp - Unmap the DMA buffers associated with the request
* @req: crypto request
@ -594,14 +607,6 @@ static void ulptx_walk_add_sg(struct ulptx_walk *walk,
}
}
static inline int get_aead_subtype(struct crypto_aead *aead)
{
struct aead_alg *alg = crypto_aead_alg(aead);
struct chcr_alg_template *chcr_crypto_alg =
container_of(alg, struct chcr_alg_template, alg.aead);
return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
}
static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
{
struct crypto_alg *alg = tfm->__crt_alg;
@ -1100,7 +1105,6 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req,
}
static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err)
{
@ -2198,9 +2202,9 @@ err:
return ERR_PTR(error);
}
static int chcr_aead_dma_map(struct device *dev,
struct aead_request *req,
unsigned short op_type)
int chcr_aead_dma_map(struct device *dev,
struct aead_request *req,
unsigned short op_type)
{
int error;
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
@ -2242,9 +2246,9 @@ err:
return -ENOMEM;
}
static void chcr_aead_dma_unmap(struct device *dev,
struct aead_request *req,
unsigned short op_type)
void chcr_aead_dma_unmap(struct device *dev,
struct aead_request *req,
unsigned short op_type)
{
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@ -2269,10 +2273,10 @@ static void chcr_aead_dma_unmap(struct device *dev,
}
}
static inline void chcr_add_aead_src_ent(struct aead_request *req,
struct ulptx_sgl *ulptx,
unsigned int assoclen,
unsigned short op_type)
void chcr_add_aead_src_ent(struct aead_request *req,
struct ulptx_sgl *ulptx,
unsigned int assoclen,
unsigned short op_type)
{
struct ulptx_walk ulp_walk;
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
@ -2304,11 +2308,11 @@ static inline void chcr_add_aead_src_ent(struct aead_request *req,
}
}
static inline void chcr_add_aead_dst_ent(struct aead_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
unsigned int assoclen,
unsigned short op_type,
unsigned short qid)
void chcr_add_aead_dst_ent(struct aead_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
unsigned int assoclen,
unsigned short op_type,
unsigned short qid)
{
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@ -2326,9 +2330,9 @@ static inline void chcr_add_aead_dst_ent(struct aead_request *req,
dsgl_walk_end(&dsgl_walk, qid);
}
static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
struct ulptx_sgl *ulptx,
struct cipher_wr_param *wrparam)
void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
struct ulptx_sgl *ulptx,
struct cipher_wr_param *wrparam)
{
struct ulptx_walk ulp_walk;
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
@ -2351,10 +2355,10 @@ static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
}
}
static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
struct cipher_wr_param *wrparam,
unsigned short qid)
void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
struct cipher_wr_param *wrparam,
unsigned short qid)
{
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
struct dsgl_walk dsgl_walk;
@ -2369,9 +2373,9 @@ static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
dsgl_walk_end(&dsgl_walk, qid);
}
static inline void chcr_add_hash_src_ent(struct ahash_request *req,
struct ulptx_sgl *ulptx,
struct hash_wr_param *param)
void chcr_add_hash_src_ent(struct ahash_request *req,
struct ulptx_sgl *ulptx,
struct hash_wr_param *param)
{
struct ulptx_walk ulp_walk;
struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
@ -2398,9 +2402,8 @@ static inline void chcr_add_hash_src_ent(struct ahash_request *req,
}
}
static inline int chcr_hash_dma_map(struct device *dev,
struct ahash_request *req)
int chcr_hash_dma_map(struct device *dev,
struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
int error = 0;
@ -2415,8 +2418,8 @@ static inline int chcr_hash_dma_map(struct device *dev,
return 0;
}
static inline void chcr_hash_dma_unmap(struct device *dev,
struct ahash_request *req)
void chcr_hash_dma_unmap(struct device *dev,
struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
@ -2429,9 +2432,8 @@ static inline void chcr_hash_dma_unmap(struct device *dev,
}
static int chcr_cipher_dma_map(struct device *dev,
struct ablkcipher_request *req)
int chcr_cipher_dma_map(struct device *dev,
struct ablkcipher_request *req)
{
int error;
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
@ -2465,8 +2467,9 @@ err:
dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
return -ENOMEM;
}
static void chcr_cipher_dma_unmap(struct device *dev,
struct ablkcipher_request *req)
void chcr_cipher_dma_unmap(struct device *dev,
struct ablkcipher_request *req)
{
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
@ -3371,6 +3374,40 @@ out:
aeadctx->enckey_len = 0;
return -EINVAL;
}
static int chcr_aead_op(struct aead_request *req,
unsigned short op_type,
int size,
create_wr_t create_wr_fn)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct uld_ctx *u_ctx;
struct sk_buff *skb;
if (!a_ctx(tfm)->dev) {
pr_err("chcr : %s : No crypto device.\n", __func__);
return -ENXIO;
}
u_ctx = ULD_CTX(a_ctx(tfm));
if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
a_ctx(tfm)->tx_qidx)) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
/* Form a WR from req */
skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
op_type);
if (IS_ERR(skb) || !skb)
return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
static int chcr_aead_encrypt(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@ -3423,38 +3460,6 @@ static int chcr_aead_decrypt(struct aead_request *req)
}
}
static int chcr_aead_op(struct aead_request *req,
unsigned short op_type,
int size,
create_wr_t create_wr_fn)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct uld_ctx *u_ctx;
struct sk_buff *skb;
if (!a_ctx(tfm)->dev) {
pr_err("chcr : %s : No crypto device.\n", __func__);
return -ENXIO;
}
u_ctx = ULD_CTX(a_ctx(tfm));
if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
a_ctx(tfm)->tx_qidx)) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
/* Form a WR from req */
skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
op_type);
if (IS_ERR(skb) || !skb)
return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
static struct chcr_alg_template driver_algs[] = {
/* AES-CBC */
{

15
drivers/crypto/chelsio/chcr_algo.h
View File

@ -226,15 +226,6 @@
#define SPACE_LEFT(len) \
((SGE_MAX_WR_LEN - WR_MIN_LEN - (len)))
unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
96, 112, 120, 136, 144, 160, 168, 184,
192, 208, 216, 232, 240, 256, 264, 280,
288, 304, 312, 328, 336, 352, 360, 376};
unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
112, 112, 128, 128, 144, 144, 160, 160,
192, 192, 208, 208, 224, 224, 240, 240,
272, 272, 288, 288, 304, 304, 320, 320};
struct algo_param {
unsigned int auth_mode;
unsigned int mk_size;
@ -404,10 +395,4 @@ static inline u32 aes_ks_subword(const u32 w)
return *(u32 *)(&bytes[0]);
}
static u32 round_constant[11] = {
0x01000000, 0x02000000, 0x04000000, 0x08000000,
0x10000000, 0x20000000, 0x40000000, 0x80000000,
0x1B000000, 0x36000000, 0x6C000000
};
#endif /* __CHCR_ALGO_H__ */

14
drivers/crypto/chelsio/chcr_core.c
View File

@ -48,6 +48,9 @@ static struct cxgb4_uld_info chcr_uld_info = {
.add = chcr_uld_add,
.state_change = chcr_uld_state_change,
.rx_handler = chcr_uld_rx_handler,
#ifdef CONFIG_CHELSIO_IPSEC_INLINE
.tx_handler = chcr_uld_tx_handler,
#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
};
struct uld_ctx *assign_chcr_device(void)
@ -164,6 +167,10 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
goto out;
}
u_ctx->lldi = *lld;
#ifdef CONFIG_CHELSIO_IPSEC_INLINE
if (lld->crypto & ULP_CRYPTO_IPSEC_INLINE)
chcr_add_xfrmops(lld);
#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
out:
return u_ctx;
}
@ -187,6 +194,13 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
return 0;
}
#ifdef CONFIG_CHELSIO_IPSEC_INLINE
int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev)
{
return chcr_ipsec_xmit(skb, dev);
}
#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
{
struct uld_ctx *u_ctx = handle;

38
drivers/crypto/chelsio/chcr_core.h
View File

@ -39,6 +39,7 @@
#include <crypto/algapi.h>
#include "t4_hw.h"
#include "cxgb4.h"
#include "t4_msg.h"
#include "cxgb4_uld.h"
#define DRV_MODULE_NAME "chcr"
@ -89,12 +90,49 @@ struct uld_ctx {
struct chcr_dev *dev;
};
struct chcr_ipsec_req {
struct ulp_txpkt ulptx;
struct ulptx_idata sc_imm;
struct cpl_tx_sec_pdu sec_cpl;
struct _key_ctx key_ctx;
};
struct chcr_ipsec_wr {
struct fw_ulptx_wr wreq;
struct chcr_ipsec_req req;
};
struct ipsec_sa_entry {
int hmac_ctrl;
unsigned int enckey_len;
unsigned int kctx_len;
unsigned int authsize;
__be32 key_ctx_hdr;
char salt[MAX_SALT];
char key[2 * AES_MAX_KEY_SIZE];
};
/*
* sgl_len - calculates the size of an SGL of the given capacity
* @n: the number of SGL entries
* Calculates the number of flits needed for a scatter/gather list that
* can hold the given number of entries.
*/
static inline unsigned int sgl_len(unsigned int n)
{
n--;
return (3 * n) / 2 + (n & 1) + 2;
}
struct uld_ctx *assign_chcr_device(void);
int chcr_send_wr(struct sk_buff *skb);
int start_crypto(void);
int stop_crypto(void);
int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
const struct pkt_gl *pgl);
int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev);
int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
int err);
int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
void chcr_add_xfrmops(const struct cxgb4_lld_info *lld);
#endif /* __CHCR_CORE_H__ */

69
drivers/crypto/chelsio/chcr_crypto.h
View File

@ -210,8 +210,6 @@ struct dsgl_walk {
struct phys_sge_pairs *to;
};
struct chcr_gcm_ctx {
u8 ghash_h[AEAD_H_SIZE];
};
@ -227,8 +225,6 @@ struct __aead_ctx {
struct chcr_authenc_ctx authenc[0];
};
struct chcr_aead_ctx {
__be32 key_ctx_hdr;
unsigned int enckey_len;
@ -240,8 +236,6 @@ struct chcr_aead_ctx {
struct __aead_ctx ctx[0];
};
struct hmac_ctx {
struct crypto_shash *base_hash;
u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
@ -307,44 +301,29 @@ typedef struct sk_buff *(*create_wr_t)(struct aead_request *req,
int size,
unsigned short op_type);
static int chcr_aead_op(struct aead_request *req_base,
unsigned short op_type,
int size,
create_wr_t create_wr_fn);
static inline int get_aead_subtype(struct crypto_aead *aead);
static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err);
static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
static int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
unsigned short op_type);
static void chcr_aead_dma_unmap(struct device *dev, struct aead_request
*req, unsigned short op_type);
static inline void chcr_add_aead_dst_ent(struct aead_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
unsigned int assoclen,
unsigned short op_type,
unsigned short qid);
static inline void chcr_add_aead_src_ent(struct aead_request *req,
struct ulptx_sgl *ulptx,
unsigned int assoclen,
unsigned short op_type);
static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
struct ulptx_sgl *ulptx,
struct cipher_wr_param *wrparam);
static int chcr_cipher_dma_map(struct device *dev,
struct ablkcipher_request *req);
static void chcr_cipher_dma_unmap(struct device *dev,
struct ablkcipher_request *req);
static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
struct cipher_wr_param *wrparam,
unsigned short qid);
void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
unsigned short op_type);
void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req,
unsigned short op_type);
void chcr_add_aead_dst_ent(struct aead_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
unsigned int assoclen, unsigned short op_type,
unsigned short qid);
void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx,
unsigned int assoclen, unsigned short op_type);
void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
struct ulptx_sgl *ulptx,
struct cipher_wr_param *wrparam);
int chcr_cipher_dma_map(struct device *dev, struct ablkcipher_request *req);
void chcr_cipher_dma_unmap(struct device *dev, struct ablkcipher_request *req);
void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
struct cpl_rx_phys_dsgl *phys_cpl,
struct cipher_wr_param *wrparam,
unsigned short qid);
int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip);
static inline void chcr_add_hash_src_ent(struct ahash_request *req,
struct ulptx_sgl *ulptx,
struct hash_wr_param *param);
static inline int chcr_hash_dma_map(struct device *dev,
struct ahash_request *req);
static inline void chcr_hash_dma_unmap(struct device *dev,
struct ahash_request *req);
void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx,
struct hash_wr_param *param);
int chcr_hash_dma_map(struct device *dev, struct ahash_request *req);
void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req);
#endif /* __CHCR_CRYPTO_H__ */

660
drivers/crypto/chelsio/chcr_ipsec.c Normal file
View File

@ -0,0 +1,660 @@
/*
* This file is part of the Chelsio T6 Crypto driver for Linux.
*
* Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* Written and Maintained by:
* Atul Gupta (atul.gupta@chelsio.com)
*/
#define pr_fmt(fmt) "chcr:" fmt
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/highmem.h>
#include <linux/if_vlan.h>
#include <linux/ip.h>
#include <linux/netdevice.h>
#include <net/esp.h>
#include <net/xfrm.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <crypto/authenc.h>
#include <crypto/internal/aead.h>
#include <crypto/null.h>
#include <crypto/internal/skcipher.h>
#include <crypto/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/hash.h>
#include "chcr_core.h"
#include "chcr_algo.h"
#include "chcr_crypto.h"
/*
* Max Tx descriptor space we allow for an Ethernet packet to be inlined
* into a WR.
*/
#define MAX_IMM_TX_PKT_LEN 256
#define GCM_ESP_IV_SIZE 8
static int chcr_xfrm_add_state(struct xfrm_state *x);
static void chcr_xfrm_del_state(struct xfrm_state *x);
static void chcr_xfrm_free_state(struct xfrm_state *x);
static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
static const struct xfrmdev_ops chcr_xfrmdev_ops = {
.xdo_dev_state_add = chcr_xfrm_add_state,
.xdo_dev_state_delete = chcr_xfrm_del_state,
.xdo_dev_state_free = chcr_xfrm_free_state,
.xdo_dev_offload_ok = chcr_ipsec_offload_ok,
};
/* Add offload xfrms to Chelsio Interface */
void chcr_add_xfrmops(const struct cxgb4_lld_info *lld)
{
struct net_device *netdev = NULL;
int i;
for (i = 0; i < lld->nports; i++) {
netdev = lld->ports[i];
if (!netdev)
continue;
netdev->xfrmdev_ops = &chcr_xfrmdev_ops;
netdev->hw_enc_features |= NETIF_F_HW_ESP;
netdev->features |= NETIF_F_HW_ESP;
rtnl_lock();
netdev_change_features(netdev);
rtnl_unlock();
}
}
static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
struct ipsec_sa_entry *sa_entry)
{
int hmac_ctrl;
int authsize = x->aead->alg_icv_len / 8;
sa_entry->authsize = authsize;
switch (authsize) {
case ICV_8:
hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
break;
case ICV_12:
hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
break;
case ICV_16:
hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
break;
default:
return -EINVAL;
}
return hmac_ctrl;
}
static inline int chcr_ipsec_setkey(struct xfrm_state *x,
struct ipsec_sa_entry *sa_entry)
{
struct crypto_cipher *cipher;
int keylen = (x->aead->alg_key_len + 7) / 8;
unsigned char *key = x->aead->alg_key;
int ck_size, key_ctx_size = 0;
unsigned char ghash_h[AEAD_H_SIZE];
int ret = 0;
if (keylen > 3) {
keylen -= 4; /* nonce/salt is present in the last 4 bytes */
memcpy(sa_entry->salt, key + keylen, 4);
}
if (keylen == AES_KEYSIZE_128) {
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
} else if (keylen == AES_KEYSIZE_192) {
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
} else if (keylen == AES_KEYSIZE_256) {
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
} else {
pr_err("GCM: Invalid key length %d\n", keylen);
ret = -EINVAL;
goto out;
}
memcpy(sa_entry->key, key, keylen);
sa_entry->enckey_len = keylen;
key_ctx_size = sizeof(struct _key_ctx) +
((DIV_ROUND_UP(keylen, 16)) << 4) +
AEAD_H_SIZE;
sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
CHCR_KEYCTX_MAC_KEY_SIZE_128,
0, 0,
key_ctx_size >> 4);
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
cipher = crypto_alloc_cipher("aes-generic", 0, 0);
if (IS_ERR(cipher)) {
sa_entry->enckey_len = 0;
ret = -ENOMEM;
goto out;
}
ret = crypto_cipher_setkey(cipher, key, keylen);
if (ret) {
sa_entry->enckey_len = 0;
goto out1;
}
memset(ghash_h, 0, AEAD_H_SIZE);
crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
16), ghash_h, AEAD_H_SIZE);
sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
AEAD_H_SIZE;
out1:
crypto_free_cipher(cipher);
out:
return ret;
}
/*
* chcr_xfrm_add_state
* returns 0 on success, negative error if failed to send message to FPGA
* positive error if FPGA returned a bad response
*/
static int chcr_xfrm_add_state(struct xfrm_state *x)
{
struct net_device *netdev = x->xso.dev;
struct port_info *pi = netdev_priv(netdev);
struct ipsec_sa_entry *sa_entry;
struct adapter *adap;
int res = 0;
adap = pi->adapter;
if (x->props.aalgo != SADB_AALG_NONE) {
pr_debug("CHCR: Cannot offload authenticated xfrm states\n");
return -EINVAL;
}
if (x->props.calgo != SADB_X_CALG_NONE) {
pr_debug("CHCR: Cannot offload compressed xfrm states\n");
return -EINVAL;
}
if (x->props.flags & XFRM_STATE_ESN) {
pr_debug("CHCR: Cannot offload ESN xfrm states\n");
return -EINVAL;
}
if (x->props.family != AF_INET &&
x->props.family != AF_INET6) {
pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n");
return -EINVAL;
}
if (x->props.mode != XFRM_MODE_TRANSPORT &&
x->props.mode != XFRM_MODE_TUNNEL) {
pr_debug("CHCR: Only transport and tunnel xfrm offload\n");
return -EINVAL;
}
if (x->id.proto != IPPROTO_ESP) {
pr_debug("CHCR: Only ESP xfrm state offloaded\n");
return -EINVAL;
}
if (x->encap) {
pr_debug("CHCR: Encapsulated xfrm state not offloaded\n");
return -EINVAL;
}
if (!x->aead) {
pr_debug("CHCR: Cannot offload xfrm states without aead\n");
return -EINVAL;
}
if (x->aead->alg_icv_len != 128 &&
x->aead->alg_icv_len != 96) {
pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n");
return -EINVAL;
}
if ((x->aead->alg_key_len != 128 + 32) &&
(x->aead->alg_key_len != 256 + 32)) {
pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
return -EINVAL;
}
if (x->tfcpad) {
pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n");
return -EINVAL;
}
if (!x->geniv) {
pr_debug("CHCR: Cannot offload xfrm states without geniv\n");
return -EINVAL;
}
if (strcmp(x->geniv, "seqiv")) {
pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n");
return -EINVAL;
}
sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
if (!sa_entry) {
res = -ENOMEM;
goto out;
}
sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry);
chcr_ipsec_setkey(x, sa_entry);
x->xso.offload_handle = (unsigned long)sa_entry;
try_module_get(THIS_MODULE);
out:
return res;
}
static void chcr_xfrm_del_state(struct xfrm_state *x)
{
/* do nothing */
if (!x->xso.offload_handle)
return;
}
static void chcr_xfrm_free_state(struct xfrm_state *x)
{
struct ipsec_sa_entry *sa_entry;
if (!x->xso.offload_handle)
return;
sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
kfree(sa_entry);
module_put(THIS_MODULE);
}
static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
/* Offload with IP options is not supported yet */
if (ip_hdr(skb)->ihl > 5)
return false;
return true;
}
static inline int is_eth_imm(const struct sk_buff *skb, unsigned int kctx_len)
{
int hdrlen = sizeof(struct chcr_ipsec_req) + kctx_len;
hdrlen += sizeof(struct cpl_tx_pkt);
if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
return hdrlen;
return 0;
}
static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
unsigned int kctx_len)
{
unsigned int flits;
int hdrlen = is_eth_imm(skb, kctx_len);
/* If the skb is small enough, we can pump it out as a work request
* with only immediate data. In that case we just have to have the
* TX Packet header plus the skb data in the Work Request.
*/
if (hdrlen)
return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
/* Otherwise, we're going to have to construct a Scatter gather list
* of the skb body and fragments. We also include the flits necessary
* for the TX Packet Work Request and CPL. We always have a firmware
* Write Header (incorporated as part of the cpl_tx_pkt_lso and
* cpl_tx_pkt structures), followed by either a TX Packet Write CPL
* message or, if we're doing a Large Send Offload, an LSO CPL message
* with an embedded TX Packet Write CPL message.
*/
flits += (sizeof(struct fw_ulptx_wr) +
sizeof(struct chcr_ipsec_req) +
kctx_len +
sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
return flits;
}
inline void *copy_cpltx_pktxt(struct sk_buff *skb,
struct net_device *dev,
void *pos)
{
struct adapter *adap;
struct port_info *pi;
struct sge_eth_txq *q;
struct cpl_tx_pkt_core *cpl;
u64 cntrl;
u32 ctrl0, qidx;
pi = netdev_priv(dev);
adap = pi->adapter;
qidx = skb->queue_mapping;
q = &adap->sge.ethtxq[qidx + pi->first_qset];
cpl = (struct cpl_tx_pkt_core *)pos;
if (skb->ip_summed == CHECKSUM_PARTIAL)
cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
TXPKT_PF_V(adap->pf);
if (skb_vlan_tag_present(skb)) {
q->vlan_ins++;
cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
}
cpl->ctrl0 = htonl(ctrl0);
cpl->pack = htons(0);
cpl->len = htons(skb->len);
cpl->ctrl1 = cpu_to_be64(cntrl);
pos += sizeof(struct cpl_tx_pkt_core);
return pos;
}
inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
struct net_device *dev,
void *pos,
struct ipsec_sa_entry *sa_entry)
{
struct adapter *adap;
struct port_info *pi;
struct sge_eth_txq *q;
unsigned int len, qidx;
struct _key_ctx *key_ctx;
int left, eoq, key_len;
pi = netdev_priv(dev);
adap = pi->adapter;
qidx = skb->queue_mapping;
q = &adap->sge.ethtxq[qidx + pi->first_qset];
len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core);
key_len = sa_entry->kctx_len;
/* end of queue, reset pos to start of queue */
eoq = (void *)q->q.stat - pos;
left = eoq;
if (!eoq) {
pos = q->q.desc;
left = 64 * q->q.size;
}
/* Copy the Key context header */
key_ctx = (struct _key_ctx *)pos;
key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
pos += sizeof(struct _key_ctx);
left -= sizeof(struct _key_ctx);
if (likely(len <= left)) {
memcpy(key_ctx->key, sa_entry->key, key_len);
pos += key_len;
} else {
if (key_len <= left) {
memcpy(pos, sa_entry->key, key_len);
pos += key_len;
} else {
memcpy(pos, sa_entry->key, left);
memcpy(q->q.desc, sa_entry->key + left,
key_len - left);
pos = q->q.desc + (key_len - left);
}
}
/* Copy CPL TX PKT XT */
pos = copy_cpltx_pktxt(skb, dev, pos);
return pos;
}
inline void *chcr_crypto_wreq(struct sk_buff *skb,
struct net_device *dev,
void *pos,
int credits,
struct ipsec_sa_entry *sa_entry)
{
struct port_info *pi = netdev_priv(dev);
struct adapter *adap = pi->adapter;
unsigned int immdatalen = 0;
unsigned int ivsize = GCM_ESP_IV_SIZE;
struct chcr_ipsec_wr *wr;
unsigned int flits;
u32 wr_mid;
int qidx = skb_get_queue_mapping(skb);
struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
unsigned int kctx_len = sa_entry->kctx_len;
int qid = q->q.cntxt_id;
atomic_inc(&adap->chcr_stats.ipsec_cnt);
flits = calc_tx_sec_flits(skb, kctx_len);
if (is_eth_imm(skb, kctx_len))
immdatalen = skb->len;
/* WR Header */
wr = (struct chcr_ipsec_wr *)pos;
wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
if (unlikely(credits < ETHTXQ_STOP_THRES)) {
netif_tx_stop_queue(q->txq);
q->q.stops++;
wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
}
wr_mid |= FW_ULPTX_WR_DATA_F;
wr->wreq.flowid_len16 = htonl(wr_mid);
/* ULPTX */
wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2) - 1);
/* Sub-command */
wr->req.sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
sizeof(wr->req.key_ctx) +
kctx_len +
sizeof(struct cpl_tx_pkt_core) +
immdatalen);
/* CPL_SEC_PDU */
wr->req.sec_cpl.op_ivinsrtofst = htonl(
CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
CPL_TX_SEC_PDU_CPLLEN_V(2) |
CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
CPL_TX_SEC_PDU_IVINSRTOFST_V(
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) + 1)));
wr->req.sec_cpl.pldlen = htonl(skb->len);
wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
(skb_transport_offset(skb) + 1),
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr)),
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) +
GCM_ESP_IV_SIZE + 1), 0);
wr->req.sec_cpl.cipherstop_lo_authinsert =
FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) +
GCM_ESP_IV_SIZE + 1,
sa_entry->authsize,
sa_entry->authsize);
wr->req.sec_cpl.seqno_numivs =
FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
CHCR_SCMD_CIPHER_MODE_AES_GCM,
CHCR_SCMD_AUTH_MODE_GHASH,
sa_entry->hmac_ctrl,
ivsize >> 1);
wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
0, 0, 0);
pos += sizeof(struct fw_ulptx_wr) +
sizeof(struct ulp_txpkt) +
sizeof(struct ulptx_idata) +
sizeof(struct cpl_tx_sec_pdu);
pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
return pos;
}
/**
* flits_to_desc - returns the num of Tx descriptors for the given flits
* @n: the number of flits
*
* Returns the number of Tx descriptors needed for the supplied number
* of flits.
*/
static inline unsigned int flits_to_desc(unsigned int n)
{
WARN_ON(n > SGE_MAX_WR_LEN / 8);
return DIV_ROUND_UP(n, 8);
}
static inline unsigned int txq_avail(const struct sge_txq *q)
{
return q->size - 1 - q->in_use;
}
static void eth_txq_stop(struct sge_eth_txq *q)
{
netif_tx_stop_queue(q->txq);
q->q.stops++;
}
static inline void txq_advance(struct sge_txq *q, unsigned int n)
{
q->in_use += n;
q->pidx += n;
if (q->pidx >= q->size)
q->pidx -= q->size;
}
/*
* chcr_ipsec_xmit called from ULD Tx handler
*/
int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct xfrm_state *x = xfrm_input_state(skb);
struct ipsec_sa_entry *sa_entry;
u64 *pos, *end, *before, cntrl, *sgl;
int qidx, left, credits;
unsigned int flits = 0, ndesc, kctx_len;
struct adapter *adap;
struct sge_eth_txq *q;
struct port_info *pi;
dma_addr_t addr[MAX_SKB_FRAGS + 1];
bool immediate = false;
if (!x->xso.offload_handle)
return NETDEV_TX_BUSY;
sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
kctx_len = sa_entry->kctx_len;
if (skb->sp->len != 1) {
out_free: dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
pi = netdev_priv(dev);
adap = pi->adapter;
qidx = skb->queue_mapping;
q = &adap->sge.ethtxq[qidx + pi->first_qset];
cxgb4_reclaim_completed_tx(adap, &q->q, true);
cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
flits = calc_tx_sec_flits(skb, sa_entry->kctx_len);
ndesc = flits_to_desc(flits);
credits = txq_avail(&q->q) - ndesc;
if (unlikely(credits < 0)) {
eth_txq_stop(q);
dev_err(adap->pdev_dev,
"%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
dev->name, qidx, credits, ndesc, txq_avail(&q->q),
flits);
return NETDEV_TX_BUSY;
}
if (is_eth_imm(skb, kctx_len))
immediate = true;
if (!immediate &&
unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
q->mapping_err++;
goto out_free;
}
pos = (u64 *)&q->q.desc[q->q.pidx];
before = (u64 *)pos;
end = (u64 *)pos + flits;
/* Setup IPSec CPL */
pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos,
credits, sa_entry);
if (before > (u64 *)pos) {
left = (u8 *)end - (u8 *)q->q.stat;
end = (void *)q->q.desc + left;
}
if (pos == (u64 *)q->q.stat) {
left = (u8 *)end - (u8 *)q->q.stat;
end = (void *)q->q.desc + left;
pos = (void *)q->q.desc;
}
sgl = (void *)pos;
if (immediate) {
cxgb4_inline_tx_skb(skb, &q->q, sgl);
dev_consume_skb_any(skb);
} else {
int last_desc;
cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
0, addr);
skb_orphan(skb);
last_desc = q->q.pidx + ndesc - 1;
if (last_desc >= q->q.size)
last_desc -= q->q.size;
q->q.sdesc[last_desc].skb = skb;
q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
}
txq_advance(&q->q, ndesc);
cxgb4_ring_tx_db(adap, &q->q, ndesc);
return NETDEV_TX_OK;
}