hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

net/mlx5e: kTLS, Improve TLS feature modularity 

Better separate the code into c/h files, so that kTLS internals
are exposed to the corresponding non-accel flow as follows:
- Necessary datapath functions are exposed via ktls_txrx.h.
- Necessary caps and configuration functions are exposed via ktls.h,
  which became very small.

In addition, kTLS internal code sharing is done via ktls_utils.h,
which is not exposed to any non-accel file.

Add explicit WQE structures for the TLS static and progress
params, breaking the union of the static with UMR, and the progress
with PSV.

Generalize the API as a preparation for TLS RX offload support.

Move kTLS TX-specific code to the proper file.
Remove the inline tag for function in C files, let the compiler decide.
Use kzalloc/kfree for the priv_tx context.

Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
Reviewed-by: Maxim Mikityanskiy <maximmi@mellanox.com>
This commit is contained in:
Tariq Toukan 2020-05-26 13:58:09 +03:00 committed by Saeed Mahameed
parent 5229a96e59
commit 7d0d0d86ec
13 changed files with 378 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/net/ethernet/mellanox/mlx5/core/Makefile
View File

@ -74,7 +74,8 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
en_accel/ipsec_stats.o
mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \
en_accel/ktls.o en_accel/ktls_tx.o en_accel/fs_tcp.o
en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \
en_accel/ktls_tx.o
mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \
steering/dr_matcher.o steering/dr_rule.o \

14
drivers/net/ethernet/mellanox/mlx5/core/en.h
View File

@ -191,13 +191,8 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
struct mlx5e_tx_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
union {
struct {
struct mlx5_wqe_eth_seg eth;
struct mlx5_wqe_data_seg data[0];
};
u8 tls_progress_params_ctx[0];
};
struct mlx5_wqe_eth_seg eth;
struct mlx5_wqe_data_seg data[0];
};
struct mlx5e_rx_wqe_ll {
@ -213,10 +208,7 @@ struct mlx5e_umr_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_umr_ctrl_seg uctrl;
struct mlx5_mkey_seg mkc;
union {
struct mlx5_mtt inline_mtts[0];
u8 tls_static_params_ctx[0];
};
struct mlx5_mtt inline_mtts[0];
};
extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];

62
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
View File

@ -3,31 +3,7 @@
#include "en.h"
#include "en_accel/ktls.h"
u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq)
{
u16 num_dumps, stop_room = 0;
num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_STATIC_WQEBBS);
stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_PROGRESS_WQEBBS);
stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS);
return stop_room;
}
static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
{
u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
void *tisc;
tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
MLX5_SET(tisc, tisc, tls_en, 1);
return mlx5e_create_tis(mdev, in, tisn);
}
#include "en_accel/ktls_utils.h"
static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
enum tls_offload_ctx_dir direction,
@ -35,8 +11,6 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
u32 start_offload_tcp_sn)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5e_ktls_offload_context_tx *tx_priv;
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct mlx5_core_dev *mdev = priv->mdev;
int err;
@ -46,31 +20,8 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
if (WARN_ON(!mlx5e_ktls_type_check(mdev, crypto_info)))
return -EOPNOTSUPP;
tx_priv = kvzalloc(sizeof(*tx_priv), GFP_KERNEL);
if (!tx_priv)
return -ENOMEM;
err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn);
tx_priv->expected_seq = start_offload_tcp_sn;
tx_priv->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
mlx5e_set_ktls_tx_priv_ctx(tls_ctx, tx_priv);
/* tc and underlay_qpn values are not in use for tls tis */
err = mlx5e_ktls_create_tis(mdev, &tx_priv->tisn);
if (err)
goto create_tis_fail;
err = mlx5_ktls_create_key(mdev, crypto_info, &tx_priv->key_id);
if (err)
goto encryption_key_create_fail;
mlx5e_ktls_tx_offload_set_pending(tx_priv);
return 0;
encryption_key_create_fail:
mlx5e_destroy_tis(priv->mdev, tx_priv->tisn);
create_tis_fail:
kvfree(tx_priv);
return err;
}
@ -78,13 +29,10 @@ static void mlx5e_ktls_del(struct net_device *netdev,
struct tls_context *tls_ctx,
enum tls_offload_ctx_dir direction)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5e_ktls_offload_context_tx *tx_priv =
mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
if (direction != TLS_OFFLOAD_CTX_DIR_TX)
return;
mlx5e_destroy_tis(priv->mdev, tx_priv->tisn);
mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id);
kvfree(tx_priv);
mlx5e_ktls_del_tx(netdev, tls_ctx);
}
static const struct tlsdev_ops mlx5e_ktls_ops = {

107
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
View File

@ -7,122 +7,15 @@
#include "en.h"
#ifdef CONFIG_MLX5_EN_TLS
#include <net/tls.h>
#include "accel/tls.h"
#include "en_accel/tls_rxtx.h"
#define MLX5E_KTLS_STATIC_UMR_WQE_SZ \
(offsetof(struct mlx5e_umr_wqe, tls_static_params_ctx) + \
MLX5_ST_SZ_BYTES(tls_static_params))
#define MLX5E_KTLS_STATIC_WQEBBS \
(DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_BB))
#define MLX5E_KTLS_PROGRESS_WQE_SZ \
(offsetof(struct mlx5e_tx_wqe, tls_progress_params_ctx) + \
sizeof(struct mlx5_wqe_tls_progress_params_seg))
#define MLX5E_KTLS_PROGRESS_WQEBBS \
(DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB))
struct mlx5e_dump_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_data_seg data;
};
#define MLX5E_TLS_FETCH_UMR_WQE(sq, pi) \
((struct mlx5e_umr_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_STATIC_UMR_WQE_SZ))
#define MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi) \
((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_PROGRESS_WQE_SZ))
#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \
((struct mlx5e_dump_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, \
sizeof(struct mlx5e_dump_wqe)))
#define MLX5E_KTLS_DUMP_WQEBBS \
(DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
enum {
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
};
enum {
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
};
struct mlx5e_ktls_offload_context_tx {
struct tls_offload_context_tx *tx_ctx;
struct tls12_crypto_info_aes_gcm_128 crypto_info;
u32 expected_seq;
u32 tisn;
u32 key_id;
bool ctx_post_pending;
};
struct mlx5e_ktls_offload_context_tx_shadow {
struct tls_offload_context_tx tx_ctx;
struct mlx5e_ktls_offload_context_tx *priv_tx;
};
static inline void
mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
struct mlx5e_ktls_offload_context_tx *priv_tx)
{
struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
struct mlx5e_ktls_offload_context_tx_shadow *shadow;
BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
shadow->priv_tx = priv_tx;
priv_tx->tx_ctx = tx_ctx;
}
static inline struct mlx5e_ktls_offload_context_tx *
mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
{
struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
struct mlx5e_ktls_offload_context_tx_shadow *shadow;
BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
return shadow->priv_tx;
}
void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv);
void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx);
bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq,
struct sk_buff *skb, int datalen,
struct mlx5e_accel_tx_tls_state *state);
void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc);
u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq);
static inline u8
mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags,
unsigned int sync_len)
{
/* Given the MTU and sync_len, calculates an upper bound for the
* number of DUMP WQEs needed for the TX resync of a record.
*/
return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu);
}
#else
static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv)
{
}
static inline void
mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc) {}
#endif
#endif /* __MLX5E_TLS_H__ */

218
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
View File

@ -1,109 +1,149 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2019 Mellanox Technologies.
#include <linux/tls.h>
#include "en.h"
#include "en/txrx.h"
#include "en_accel/ktls.h"
#include "en_accel/ktls_txrx.h"
#include "en_accel/ktls_utils.h"
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
struct mlx5e_dump_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_data_seg data;
};
enum {
MLX5E_ENCRYPTION_STANDARD_TLS = 0x1,
#define MLX5E_KTLS_DUMP_WQEBBS \
(DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
static u8
mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags,
unsigned int sync_len)
{
/* Given the MTU and sync_len, calculates an upper bound for the
* number of DUMP WQEs needed for the TX resync of a record.
*/
return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu);
}
u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq)
{
u16 num_dumps, stop_room = 0;
num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS);
stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS);
stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS);
return stop_room;
}
static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
{
u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
void *tisc;
tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
MLX5_SET(tisc, tisc, tls_en, 1);
return mlx5e_create_tis(mdev, in, tisn);
}
struct mlx5e_ktls_offload_context_tx {
struct tls_offload_context_tx *tx_ctx;
struct tls12_crypto_info_aes_gcm_128 crypto_info;
u32 expected_seq;
u32 tisn;
u32 key_id;
bool ctx_post_pending;
};
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
} while (0)
struct mlx5e_ktls_offload_context_tx_shadow {
struct tls_offload_context_tx tx_ctx;
struct mlx5e_ktls_offload_context_tx *priv_tx;
};
static void
fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
struct mlx5e_ktls_offload_context_tx *priv_tx)
{
struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
char *initial_rn, *gcm_iv;
u16 salt_sz, rec_seq_sz;
char *salt, *rec_seq;
u8 tls_version;
struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
struct mlx5e_ktls_offload_context_tx_shadow *shadow;
EXTRACT_INFO_FIELDS;
BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
MLX5_SET(tls_static_params, ctx, const_2, 2);
MLX5_SET(tls_static_params, ctx, encryption_standard,
MLX5E_ENCRYPTION_STANDARD_TLS);
MLX5_SET(tls_static_params, ctx, dek_index, priv_tx->key_id);
shadow->priv_tx = priv_tx;
priv_tx->tx_ctx = tx_ctx;
}
static void
build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn,
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
static struct mlx5e_ktls_offload_context_tx *
mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
{
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
struct mlx5e_ktls_offload_context_tx_shadow *shadow;
#define STATIC_PARAMS_DS_CNT \
DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_DS)
BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR |
(MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24));
cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
STATIC_PARAMS_DS_CNT);
cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
cseg->tis_tir_num = cpu_to_be32(priv_tx->tisn << 8);
shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
ucseg->flags = MLX5_UMR_INLINE;
ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx);
return shadow->priv_tx;
}
static void
fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn)
{
struct mlx5_wqe_tls_progress_params_seg *params;
struct mlx5e_ktls_offload_context_tx *priv_tx;
struct tls_context *tls_ctx;
struct mlx5_core_dev *mdev;
struct mlx5e_priv *priv;
int err;
params = ctx;
tls_ctx = tls_get_ctx(sk);
priv = netdev_priv(netdev);
mdev = priv->mdev;
params->tis_tir_num = cpu_to_be32(priv_tx->tisn);
MLX5_SET(tls_progress_params, params->ctx, record_tracker_state,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
MLX5_SET(tls_progress_params, params->ctx, auth_state,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL);
if (!priv_tx)
return -ENOMEM;
err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id);
if (err)
goto err_create_key;
priv_tx->expected_seq = start_offload_tcp_sn;
priv_tx->crypto_info =
*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);
err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn);
if (err)
goto err_create_tis;
priv_tx->ctx_post_pending = true;
return 0;
err_create_tis:
mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
err_create_key:
kfree(priv_tx);
return err;
}
static void
build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn,
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
{
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
struct mlx5e_ktls_offload_context_tx *priv_tx;
struct mlx5_core_dev *mdev;
struct mlx5e_priv *priv;
#define PROGRESS_PARAMS_DS_CNT \
DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_DS)
priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
priv = netdev_priv(netdev);
mdev = priv->mdev;
cseg->opmod_idx_opcode =
cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV |
(MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24));
cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
PROGRESS_PARAMS_DS_CNT);
cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx);
mlx5e_destroy_tis(mdev, priv_tx->tisn);
mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
kfree(priv_tx);
}
static void tx_fill_wi(struct mlx5e_txqsq *sq,
@ -119,11 +159,6 @@ static void tx_fill_wi(struct mlx5e_txqsq *sq,
};
}
void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
{
priv_tx->ctx_post_pending = true;
}
static bool
mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
{
@ -139,12 +174,15 @@ post_static_params(struct mlx5e_txqsq *sq,
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
{
u16 pi, num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS;
struct mlx5e_umr_wqe *umr_wqe;
struct mlx5e_set_tls_static_params_wqe *wqe;
u16 pi, num_wqebbs;
num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi);
build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence);
wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
priv_tx->tisn, priv_tx->key_id, fence,
TLS_OFFLOAD_CTX_DIR_TX);
tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
sq->pc += num_wqebbs;
}
@ -154,12 +192,14 @@ post_progress_params(struct mlx5e_txqsq *sq,
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
{
u16 pi, num_wqebbs = MLX5E_KTLS_PROGRESS_WQEBBS;
struct mlx5e_tx_wqe *wqe;
struct mlx5e_set_tls_progress_params_wqe *wqe;
u16 pi, num_wqebbs;
num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi);
build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence);
wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence,
TLS_OFFLOAD_CTX_DIR_TX);
tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
sq->pc += num_wqebbs;
}

110
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c Normal file
View File

@ -0,0 +1,110 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
#include "en_accel/ktls_txrx.h"
#include "en_accel/ktls_utils.h"
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
};
enum {
MLX5E_ENCRYPTION_STANDARD_TLS = 0x1,
};
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
} while (0)
static void
fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
struct tls12_crypto_info_aes_gcm_128 *info,
u32 key_id)
{
char *initial_rn, *gcm_iv;
u16 salt_sz, rec_seq_sz;
char *salt, *rec_seq;
u8 tls_version;
u8 *ctx;
ctx = params->ctx;
EXTRACT_INFO_FIELDS;
gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
MLX5_SET(tls_static_params, ctx, const_2, 2);
MLX5_SET(tls_static_params, ctx, encryption_standard,
MLX5E_ENCRYPTION_STANDARD_TLS);
MLX5_SET(tls_static_params, ctx, dek_index, key_id);
}
void
mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
u16 pc, u32 sqn,
struct tls12_crypto_info_aes_gcm_128 *info,
u32 tis_tir_num, u32 key_id,
bool fence, enum tls_offload_ctx_dir direction)
{
struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
#define STATIC_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)
cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR |
(MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24));
cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
STATIC_PARAMS_DS_CNT);
cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
cseg->tis_tir_num = cpu_to_be32(tis_tir_num << 8);
ucseg->flags = MLX5_UMR_INLINE;
ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
fill_static_params(&wqe->params, info, key_id);
}
static void
fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num)
{
u8 *ctx = params->ctx;
params->tis_tir_num = cpu_to_be32(tis_tir_num);
MLX5_SET(tls_progress_params, ctx, record_tracker_state,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
MLX5_SET(tls_progress_params, ctx, auth_state,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
}
void
mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe,
u16 pc, u32 sqn,
u32 tis_tir_num, bool fence,
enum tls_offload_ctx_dir direction)
{
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
#define PROGRESS_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)
cseg->opmod_idx_opcode =
cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV |
(MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24));
cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
PROGRESS_PARAMS_DS_CNT);
cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
fill_progress_params(&wqe->params, tis_tir_num);
}

36
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
#ifndef __MLX5E_KTLS_TXRX_H__
#define __MLX5E_KTLS_TXRX_H__
#ifdef CONFIG_MLX5_EN_TLS
#include <net/tls.h>
#include "en.h"
#include "en/txrx.h"
struct mlx5e_accel_tx_tls_state {
u32 tls_tisn;
};
u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq);
bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq,
struct sk_buff *skb, int datalen,
struct mlx5e_accel_tx_tls_state *state);
void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc);
#else
static inline void
mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc)
{
}
#endif /* CONFIG_MLX5_EN_TLS */
#endif /* __MLX5E_TLS_TXRX_H__ */

69
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h Normal file
View File

@ -0,0 +1,69 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
#ifndef __MLX5E_KTLS_UTILS_H__
#define __MLX5E_KTLS_UTILS_H__
#include <net/tls.h>
#include "en.h"
#include "accel/tls.h"
enum {
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
};
enum {
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
};
int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn);
void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx);
struct mlx5e_set_tls_static_params_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_umr_ctrl_seg uctrl;
struct mlx5_mkey_seg mkc;
struct mlx5_wqe_tls_static_params_seg params;
};
struct mlx5e_set_tls_progress_params_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_tls_progress_params_seg params;
};
#define MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS \
(DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_static_params_wqe), MLX5_SEND_WQE_BB))
#define MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS \
(DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_progress_params_wqe), MLX5_SEND_WQE_BB))
#define MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi) \
((struct mlx5e_set_tls_static_params_wqe *)\
mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_static_params_wqe)))
#define MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi) \
((struct mlx5e_set_tls_progress_params_wqe *)\
mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_progress_params_wqe)))
#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \
((struct mlx5e_dump_wqe *)\
mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_dump_wqe)))
void
mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
u16 pc, u32 sqn,
struct tls12_crypto_info_aes_gcm_128 *info,
u32 tis_tir_num, u32 key_id,
bool fence, enum tls_offload_ctx_dir direction);
void
mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe,
u16 pc, u32 sqn,
u32 tis_tir_num, bool fence,
enum tls_offload_ctx_dir direction);
#endif /* __MLX5E_TLS_UTILS_H__ */

14
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
View File

@ -240,17 +240,3 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *priv)
kfree(tls);
priv->tls = NULL;
}
u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
{
struct mlx5_core_dev *mdev = sq->channel->mdev;
if (!mlx5_accel_is_tls_device(mdev))
return 0;
if (MLX5_CAP_GEN(mdev, tls_tx))
return mlx5e_ktls_get_stop_room(sq);
/* Resync SKB. */
return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
}

7
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
View File

@ -94,8 +94,6 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv);
int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data);
int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data);
u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq);
#else
static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv)
@ -110,11 +108,6 @@ static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; }
static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; }
static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; }
static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
{
return 0;
}
#endif
#endif /* __MLX5E_TLS_H__ */

14
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
View File

@ -387,3 +387,17 @@ void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb,
remove_metadata_hdr(skb);
*cqe_bcnt -= MLX5E_METADATA_ETHER_LEN;
}
u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
{
struct mlx5_core_dev *mdev = sq->channel->mdev;
if (!mlx5_accel_is_tls_device(mdev))
return 0;
if (MLX5_CAP_GEN(mdev, tls_tx))
return mlx5e_ktls_get_stop_room(sq);
/* Resync SKB. */
return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
}

13
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
View File

@ -34,15 +34,15 @@
#ifndef __MLX5E_TLS_RXTX_H__
#define __MLX5E_TLS_RXTX_H__
#include "en_accel/ktls_txrx.h"
#ifdef CONFIG_MLX5_EN_TLS
#include <linux/skbuff.h>
#include "en.h"
#include "en/txrx.h"
struct mlx5e_accel_tx_tls_state {
u32 tls_tisn;
};
u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq);
bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq,
struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state);
@ -52,6 +52,13 @@ void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *c
void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb,
u32 *cqe_bcnt);
#else
static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
{
return 0;
}
#endif /* CONFIG_MLX5_EN_TLS */
#endif /* __MLX5E_TLS_RXTX_H__ */

1
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
View File

@ -38,7 +38,6 @@
#include "en/txrx.h"
#include "ipoib/ipoib.h"
#include "en_accel/en_accel.h"
#include "en_accel/ktls.h"
#include "lib/clock.h"
static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)