net/mlx4_core: Dynamic VST to VST vlan/qos changes

Within VST mode, enable modifying the vlan and/or qos
for a VF without requiring unbind/rebind.

This requires firmware which supports the UPDATE_QP command.
(If the command is not available, we fall back to requiring
unbind/bind to activate these changes).

To avoid race conditions with modify-qp on QPs that are affected
by update-qp, this operation is performed on the comm_wq.

If the update operation succeeds for all the necessary QPs, a
vlan_unregister is performed for the abandoned vlan id.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Jack Morgenstein 2013-06-27 19:05:21 +03:00 committed by David S. Miller
parent 4e144d3a80
commit b01978cacf
7 changed files with 319 additions and 11 deletions

View File

@ -112,6 +112,14 @@ enum {
GO_BIT_TIMEOUT_MSECS = 10000
};
enum mlx4_vlan_transition {
MLX4_VLAN_TRANSITION_VST_VST = 0,
MLX4_VLAN_TRANSITION_VST_VGT = 1,
MLX4_VLAN_TRANSITION_VGT_VST = 2,
MLX4_VLAN_TRANSITION_VGT_VGT = 3,
};
struct mlx4_cmd_context {
struct completion done;
int result;
@ -792,6 +800,15 @@ static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave,
vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
}
int MLX4_CMD_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
return -EPERM;
}
int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
@ -1225,6 +1242,15 @@ static struct mlx4_cmd_info cmd_info[] = {
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_UPDATE_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = MLX4_CMD_UPDATE_QP_wrapper
},
{
.opcode = MLX4_CMD_CONF_SPECIAL_QP,
.has_inbox = false,
@ -1495,6 +1521,72 @@ out:
return ret;
}
int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv,
int slave, int port)
{
struct mlx4_vport_oper_state *vp_oper;
struct mlx4_vport_state *vp_admin;
struct mlx4_vf_immed_vlan_work *work;
int err;
int admin_vlan_ix = NO_INDX;
vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
if (vp_oper->state.default_vlan == vp_admin->default_vlan &&
vp_oper->state.default_qos == vp_admin->default_qos)
return 0;
work = kzalloc(sizeof(*work), GFP_KERNEL);
if (!work)
return -ENOMEM;
if (vp_oper->state.default_vlan != vp_admin->default_vlan) {
err = __mlx4_register_vlan(&priv->dev, port,
vp_admin->default_vlan,
&admin_vlan_ix);
if (err) {
mlx4_warn((&priv->dev),
"No vlan resources slave %d, port %d\n",
slave, port);
return err;
}
work->flags |= MLX4_VF_IMMED_VLAN_FLAG_VLAN;
mlx4_dbg((&(priv->dev)),
"alloc vlan %d idx %d slave %d port %d\n",
(int)(vp_admin->default_vlan),
admin_vlan_ix, slave, port);
}
/* save original vlan ix and vlan id */
work->orig_vlan_id = vp_oper->state.default_vlan;
work->orig_vlan_ix = vp_oper->vlan_idx;
/* handle new qos */
if (vp_oper->state.default_qos != vp_admin->default_qos)
work->flags |= MLX4_VF_IMMED_VLAN_FLAG_QOS;
if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN)
vp_oper->vlan_idx = admin_vlan_ix;
vp_oper->state.default_vlan = vp_admin->default_vlan;
vp_oper->state.default_qos = vp_admin->default_qos;
/* iterate over QPs owned by this slave, using UPDATE_QP */
work->port = port;
work->slave = slave;
work->qos = vp_oper->state.default_qos;
work->vlan_id = vp_oper->state.default_vlan;
work->vlan_ix = vp_oper->vlan_idx;
work->priv = priv;
INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler);
queue_work(priv->mfunc.master.comm_wq, &work->work);
return 0;
}
static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave)
{
int port, err;
@ -2109,11 +2201,18 @@ int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u64 mac)
}
EXPORT_SYMBOL_GPL(mlx4_set_vf_mac);
static int calculate_transition(u16 oper_vlan, u16 admin_vlan)
{
return (2 * (oper_vlan == MLX4_VGT) + (admin_vlan == MLX4_VGT));
}
int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_vport_state *s_info;
struct mlx4_vport_oper_state *vf_oper;
struct mlx4_vport_state *vf_admin;
int slave;
enum mlx4_vlan_transition vlan_trans;
if ((!mlx4_is_master(dev)) ||
!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VLAN_CONTROL))
@ -2126,12 +2225,25 @@ int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
if (slave < 0)
return -EINVAL;
s_info = &priv->mfunc.master.vf_admin[slave].vport[port];
vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
if ((0 == vlan) && (0 == qos))
s_info->default_vlan = MLX4_VGT;
vf_admin->default_vlan = MLX4_VGT;
else
s_info->default_vlan = vlan;
s_info->default_qos = qos;
vf_admin->default_vlan = vlan;
vf_admin->default_qos = qos;
vlan_trans = calculate_transition(vf_oper->state.default_vlan,
vf_admin->default_vlan);
if (priv->mfunc.master.slave_state[slave].active &&
dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP &&
vlan_trans == MLX4_VLAN_TRANSITION_VST_VST) {
mlx4_info(dev, "updating vf %d port %d config params immediately\n",
vf, port);
mlx4_master_immediate_activate_vlan_qos(priv, slave, port);
}
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_set_vf_vlan);

View File

@ -133,7 +133,8 @@ static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags)
[4] = "Automatic MAC reassignment support",
[5] = "Time stamping support",
[6] = "VST (control vlan insertion/stripping) support",
[7] = "FSM (MAC anti-spoofing) support"
[7] = "FSM (MAC anti-spoofing) support",
[8] = "Dynamic QP updates support"
};
int i;
@ -659,6 +660,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
QUERY_DEV_CAP_MAX_COUNTERS_OFFSET);
MLX4_GET(field32, outbox, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET);
if (field32 & (1 << 16))
dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_UPDATE_QP;
if (field32 & (1 << 26))
dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_VLAN_CONTROL;
if (field32 & (1 << 20))

View File

@ -571,6 +571,24 @@ struct mlx4_cmd {
u8 comm_toggle;
};
enum {
MLX4_VF_IMMED_VLAN_FLAG_VLAN = 1 << 0,
MLX4_VF_IMMED_VLAN_FLAG_QOS = 1 << 1,
};
struct mlx4_vf_immed_vlan_work {
struct work_struct work;
struct mlx4_priv *priv;
int flags;
int slave;
int vlan_ix;
int orig_vlan_ix;
u8 port;
u8 qos;
u16 vlan_id;
u16 orig_vlan_id;
};
struct mlx4_uar_table {
struct mlx4_bitmap bitmap;
};
@ -1218,4 +1236,6 @@ static inline spinlock_t *mlx4_tlock(struct mlx4_dev *dev)
#define NOT_MASKED_PD_BITS 17
void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work);
#endif /* MLX4_H */

View File

@ -101,6 +101,8 @@ struct res_qp {
spinlock_t mcg_spl;
int local_qpn;
atomic_t ref_count;
u32 qpc_flags;
u8 sched_queue;
};
enum res_mtt_states {
@ -355,7 +357,7 @@ static void update_gid(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *inbox,
static int update_vport_qp_param(struct mlx4_dev *dev,
struct mlx4_cmd_mailbox *inbox,
u8 slave)
u8 slave, u32 qpn)
{
struct mlx4_qp_context *qpc = inbox->buf + 8;
struct mlx4_vport_oper_state *vp_oper;
@ -369,9 +371,17 @@ static int update_vport_qp_param(struct mlx4_dev *dev,
if (MLX4_VGT != vp_oper->state.default_vlan) {
qp_type = (be32_to_cpu(qpc->flags) >> 16) & 0xff;
if (MLX4_QP_ST_RC == qp_type)
if (MLX4_QP_ST_RC == qp_type ||
(MLX4_QP_ST_UD == qp_type &&
!mlx4_is_qp_reserved(dev, qpn)))
return -EINVAL;
/* the reserved QPs (special, proxy, tunnel)
* do not operate over vlans
*/
if (mlx4_is_qp_reserved(dev, qpn))
return 0;
/* force strip vlan by clear vsd */
qpc->param3 &= ~cpu_to_be32(MLX4_STRIP_VLAN);
if (0 != vp_oper->state.default_vlan) {
@ -2114,6 +2124,8 @@ int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave,
if (err)
return err;
qp->local_qpn = local_qpn;
qp->sched_queue = 0;
qp->qpc_flags = be32_to_cpu(qpc->flags);
err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
if (err)
@ -2836,6 +2848,9 @@ int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave,
{
int err;
struct mlx4_qp_context *qpc = inbox->buf + 8;
int qpn = vhcr->in_modifier & 0x7fffff;
struct res_qp *qp;
u8 orig_sched_queue;
err = verify_qp_parameters(dev, inbox, QP_TRANS_INIT2RTR, slave);
if (err)
@ -2844,11 +2859,30 @@ int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave,
update_pkey_index(dev, slave, inbox);
update_gid(dev, inbox, (u8)slave);
adjust_proxy_tun_qkey(dev, vhcr, qpc);
err = update_vport_qp_param(dev, inbox, slave);
orig_sched_queue = qpc->pri_path.sched_queue;
err = update_vport_qp_param(dev, inbox, slave, qpn);
if (err)
return err;
return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
err = get_res(dev, slave, qpn, RES_QP, &qp);
if (err)
return err;
if (qp->com.from_state != RES_QP_HW) {
err = -EBUSY;
goto out;
}
err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
out:
/* if no error, save sched queue value passed in by VF. This is
* essentially the QOS value provided by the VF. This will be useful
* if we allow dynamic changes from VST back to VGT
*/
if (!err)
qp->sched_queue = orig_sched_queue;
put_res(dev, slave, qpn, RES_QP);
return err;
}
int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave,
@ -3932,3 +3966,106 @@ void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave)
rem_slave_xrcdns(dev, slave);
mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex);
}
void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work)
{
struct mlx4_vf_immed_vlan_work *work =
container_of(_work, struct mlx4_vf_immed_vlan_work, work);
struct mlx4_cmd_mailbox *mailbox;
struct mlx4_update_qp_context *upd_context;
struct mlx4_dev *dev = &work->priv->dev;
struct mlx4_resource_tracker *tracker =
&work->priv->mfunc.master.res_tracker;
struct list_head *qp_list =
&tracker->slave_list[work->slave].res_list[RES_QP];
struct res_qp *qp;
struct res_qp *tmp;
u64 qp_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED) |
(1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P) |
(1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED) |
(1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED) |
(1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P) |
(1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED) |
(1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) |
(1ULL << MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE));
int err;
int port, errors = 0;
u8 vlan_control;
if (mlx4_is_slave(dev)) {
mlx4_warn(dev, "Trying to update-qp in slave %d\n",
work->slave);
goto out;
}
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
goto out;
if (!work->vlan_id)
vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
else
vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
upd_context = mailbox->buf;
upd_context->primary_addr_path_mask = cpu_to_be64(qp_mask);
upd_context->qp_context.pri_path.vlan_control = vlan_control;
upd_context->qp_context.pri_path.vlan_index = work->vlan_ix;
spin_lock_irq(mlx4_tlock(dev));
list_for_each_entry_safe(qp, tmp, qp_list, com.list) {
spin_unlock_irq(mlx4_tlock(dev));
if (qp->com.owner == work->slave) {
if (qp->com.from_state != RES_QP_HW ||
!qp->sched_queue || /* no INIT2RTR trans yet */
mlx4_is_qp_reserved(dev, qp->local_qpn) ||
qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) {
spin_lock_irq(mlx4_tlock(dev));
continue;
}
port = (qp->sched_queue >> 6 & 1) + 1;
if (port != work->port) {
spin_lock_irq(mlx4_tlock(dev));
continue;
}
upd_context->qp_context.pri_path.sched_queue =
qp->sched_queue & 0xC7;
upd_context->qp_context.pri_path.sched_queue |=
((work->qos & 0x7) << 3);
err = mlx4_cmd(dev, mailbox->dma,
qp->local_qpn & 0xffffff,
0, MLX4_CMD_UPDATE_QP,
MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
if (err) {
mlx4_info(dev, "UPDATE_QP failed for slave %d, "
"port %d, qpn %d (%d)\n",
work->slave, port, qp->local_qpn,
err);
errors++;
}
}
spin_lock_irq(mlx4_tlock(dev));
}
spin_unlock_irq(mlx4_tlock(dev));
mlx4_free_cmd_mailbox(dev, mailbox);
if (errors)
mlx4_err(dev, "%d UPDATE_QP failures for slave %d, port %d\n",
errors, work->slave, work->port);
/* unregister previous vlan_id if needed and we had no errors
* while updating the QPs
*/
if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN && !errors &&
NO_INDX != work->orig_vlan_ix)
__mlx4_unregister_vlan(&work->priv->dev, work->port,
work->orig_vlan_ix);
out:
kfree(work);
return;
}

View File

@ -111,6 +111,7 @@ enum {
MLX4_CMD_INIT2INIT_QP = 0x2d,
MLX4_CMD_SUSPEND_QP = 0x32,
MLX4_CMD_UNSUSPEND_QP = 0x33,
MLX4_CMD_UPDATE_QP = 0x61,
/* special QP and management commands */
MLX4_CMD_CONF_SPECIAL_QP = 0x23,
MLX4_CMD_MAD_IFC = 0x24,

View File

@ -157,7 +157,8 @@ enum {
MLX4_DEV_CAP_FLAGS2_REASSIGN_MAC_EN = 1LL << 4,
MLX4_DEV_CAP_FLAG2_TS = 1LL << 5,
MLX4_DEV_CAP_FLAG2_VLAN_CONTROL = 1LL << 6,
MLX4_DEV_CAP_FLAG2_FSM = 1LL << 7
MLX4_DEV_CAP_FLAG2_FSM = 1LL << 7,
MLX4_DEV_CAP_FLAG2_UPDATE_QP = 1LL << 8
};
enum {

View File

@ -206,6 +206,40 @@ struct mlx4_qp_context {
u32 reserved5[10];
};
struct mlx4_update_qp_context {
__be64 qp_mask;
__be64 primary_addr_path_mask;
__be64 secondary_addr_path_mask;
u64 reserved1;
struct mlx4_qp_context qp_context;
u64 reserved2[58];
};
enum {
MLX4_UPD_QP_MASK_PM_STATE = 32,
MLX4_UPD_QP_MASK_VSD = 33,
};
enum {
MLX4_UPD_QP_PATH_MASK_PKEY_INDEX = 0 + 32,
MLX4_UPD_QP_PATH_MASK_FSM = 1 + 32,
MLX4_UPD_QP_PATH_MASK_MAC_INDEX = 2 + 32,
MLX4_UPD_QP_PATH_MASK_FVL = 3 + 32,
MLX4_UPD_QP_PATH_MASK_CV = 4 + 32,
MLX4_UPD_QP_PATH_MASK_VLAN_INDEX = 5 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN = 6 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED = 7 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P = 8 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED = 9 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED = 10 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P = 11 + 32,
MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED = 12 + 32,
MLX4_UPD_QP_PATH_MASK_FEUP = 13 + 32,
MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE = 14 + 32,
MLX4_UPD_QP_PATH_MASK_IF_COUNTER_INDEX = 15 + 32,
MLX4_UPD_QP_PATH_MASK_FVL_RX = 16 + 32,
};
enum { /* param3 */
MLX4_STRIP_VLAN = 1 << 30
};