RDMA 5.7 pull request

The majority of the patches are cleanups, refactorings and clarity
 improvements
 
 - Various driver updates for siw, bnxt_re, rxe, efa, mlx5, hfi1
 
 - Lots of cleanup patches for hns
 
 - Convert more places to use refcount
 
 - Aggressively lock the RDMA CM code that syzkaller says isn't working
 
 - Work to clarify ib_cm
 
 - Use the new ib_device lifecycle model in bnxt_re
 
 - Fix mlx5's MR cache which seems to be failing more often with the new
   ODP code
 
 - mlx5 'dynamic uar' and 'tx steering' user interfaces
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEfB7FMLh+8QxL+6i3OG33FX4gmxoFAl6CSr0ACgkQOG33FX4g
 mxrtKg//XovbOfYAO7nC05FtGz9iEkIUBiwQOjUojgNSi6RMNDqRW1bmqKUugm1o
 9nXA6tw+fueEvUNSD541SCcxkUZJzWvubO9wHB6N3Fgy68N3Vf2rKV3EBBTh99rK
 Cb7rnmTTN6izRyI1wdyP2sjDJGyF8zvsgIrG2sibzLnqlyScrnD98YS0FdPZUfOa
 a1mmXBN/T7eaQ4TbE3lLLzGnifRlGmZ5vxEvOQmAlOdqAfIKQdbbW7oCRLVIleso
 gfQlOOvIgzHRwQ3VrFa3i6ETYtywXq7EgmQxCjqPVJQjCA79n5TLBkP1iRhvn8xi
 3+LO4YCkiSJ/NjTA2d9KwT6K4djj3cYfcueuqo2MoXXr0YLiY6TLv1OffKcUIq7c
 LM3d4CSwIAG+C2FZwaQrdSGa2h/CNfLAEeKxv430zggeDNKlwHJPV5w3rUJ8lT56
 wlyT7Lzosl0O9Z/1BSLYckTvbBCtYcmanVyCfHG8EJKAM1/tXy5LS8btJ3e51rPu
 XekR9ELrTTA2CTuoSCQGP6J0dBD2U7qO4XRCQ9N5BYLrI6RdP7Z4xYzzey49Z3Cx
 JaF86eurM7nS5biUszTtwww8AJMyYicB+0VyjBfk+mhv90w8tS1vZ1aZKzaQ1L6Z
 jWn8WgIN4rWY0YGQs6PiovT1FplyGs3p1wNmjn92WO0wZZ3WsmQ=
 =ae+a
 -----END PGP SIGNATURE-----

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe:
 "The majority of the patches are cleanups, refactorings and clarity
  improvements.

  This cycle saw some more activity from Syzkaller, I think we are now
  clean on all but one of those bugs, including the long standing and
  obnoxious rdma_cm locking design defect. Continue to see many drivers
  getting cleanups, with a few new user visible features.

  Summary:

   - Various driver updates for siw, bnxt_re, rxe, efa, mlx5, hfi1

   - Lots of cleanup patches for hns

   - Convert more places to use refcount

   - Aggressively lock the RDMA CM code that syzkaller says isn't
     working

   - Work to clarify ib_cm

   - Use the new ib_device lifecycle model in bnxt_re

   - Fix mlx5's MR cache which seems to be failing more often with the
     new ODP code

   - mlx5 'dynamic uar' and 'tx steering' user interfaces"

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (144 commits)
  RDMA/bnxt_re: make bnxt_re_ib_init static
  IB/qib: Delete struct qib_ivdev.qp_rnd
  RDMA/hns: Fix uninitialized variable bug
  RDMA/hns: Modify the mask of QP number for CQE of hip08
  RDMA/hns: Reduce the maximum number of extend SGE per WQE
  RDMA/hns: Reduce PFC frames in congestion scenarios
  RDMA/mlx5: Add support for RDMA TX flow table
  net/mlx5: Add support for RDMA TX steering
  IB/hfi1: Call kobject_put() when kobject_init_and_add() fails
  IB/hfi1: Fix memory leaks in sysfs registration and unregistration
  IB/mlx5: Move to fully dynamic UAR mode once user space supports it
  IB/mlx5: Limit the scope of struct mlx5_bfreg_info to mlx5_ib
  IB/mlx5: Extend QP creation to get uar page index from user space
  IB/mlx5: Extend CQ creation to get uar page index from user space
  IB/mlx5: Expose UAR object and its alloc/destroy commands
  IB/hfi1: Get rid of a warning
  RDMA/hns: Remove redundant judgment of qp_type
  RDMA/hns: Remove redundant assignment of wc->smac when polling cq
  RDMA/hns: Remove redundant qpc setup operations
  RDMA/hns: Remove meaningless prints
  ...
This commit is contained in:
Linus Torvalds 2020-04-01 18:18:18 -07:00
commit 919dce2470
120 changed files with 5467 additions and 4561 deletions

View File

@ -46,7 +46,7 @@
struct ib_pkey_cache {
int table_len;
u16 table[0];
u16 table[];
};
struct ib_update_work {
@ -972,6 +972,23 @@ done:
}
EXPORT_SYMBOL(rdma_query_gid);
/**
* rdma_read_gid_hw_context - Read the HW GID context from GID attribute
* @attr: Potinter to the GID attribute
*
* rdma_read_gid_hw_context() reads the drivers GID HW context corresponding
* to the SGID attr. Callers are required to already be holding the reference
* to an existing GID entry.
*
* Returns the HW GID context
*
*/
void *rdma_read_gid_hw_context(const struct ib_gid_attr *attr)
{
return container_of(attr, struct ib_gid_table_entry, attr)->context;
}
EXPORT_SYMBOL(rdma_read_gid_hw_context);
/**
* rdma_find_gid - Returns SGID attributes if the matching GID is found.
* @device: The device to query.

File diff suppressed because it is too large Load Diff

View File

@ -199,7 +199,7 @@ struct cma_device {
struct list_head list;
struct ib_device *device;
struct completion comp;
atomic_t refcount;
refcount_t refcount;
struct list_head id_list;
enum ib_gid_type *default_gid_type;
u8 *default_roce_tos;
@ -247,9 +247,15 @@ enum {
CMA_OPTION_AFONLY,
};
void cma_ref_dev(struct cma_device *cma_dev)
void cma_dev_get(struct cma_device *cma_dev)
{
atomic_inc(&cma_dev->refcount);
refcount_inc(&cma_dev->refcount);
}
void cma_dev_put(struct cma_device *cma_dev)
{
if (refcount_dec_and_test(&cma_dev->refcount))
complete(&cma_dev->comp);
}
struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,
@ -267,7 +273,7 @@ struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,
}
if (found_cma_dev)
cma_ref_dev(found_cma_dev);
cma_dev_get(found_cma_dev);
mutex_unlock(&lock);
return found_cma_dev;
}
@ -463,7 +469,7 @@ static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
struct cma_device *cma_dev)
{
cma_ref_dev(cma_dev);
cma_dev_get(cma_dev);
id_priv->cma_dev = cma_dev;
id_priv->id.device = cma_dev->device;
id_priv->id.route.addr.dev_addr.transport =
@ -484,12 +490,6 @@ static void cma_attach_to_dev(struct rdma_id_private *id_priv,
rdma_start_port(cma_dev->device)];
}
void cma_deref_dev(struct cma_device *cma_dev)
{
if (atomic_dec_and_test(&cma_dev->refcount))
complete(&cma_dev->comp);
}
static inline void release_mc(struct kref *kref)
{
struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
@ -502,7 +502,7 @@ static void cma_release_dev(struct rdma_id_private *id_priv)
{
mutex_lock(&lock);
list_del(&id_priv->list);
cma_deref_dev(id_priv->cma_dev);
cma_dev_put(id_priv->cma_dev);
id_priv->cma_dev = NULL;
mutex_unlock(&lock);
}
@ -728,8 +728,8 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
struct cma_device *cma_dev;
enum ib_gid_type gid_type;
int ret = -ENODEV;
unsigned int port;
union ib_gid gid;
u8 port;
if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
id_priv->id.ps == RDMA_PS_IPOIB)
@ -753,7 +753,7 @@ static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
}
list_for_each_entry(cma_dev, &dev_list, list) {
for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
rdma_for_each_port (cma_dev->device, port) {
if (listen_id_priv->cma_dev == cma_dev &&
listen_id_priv->id.port_num == port)
continue;
@ -786,8 +786,8 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
struct cma_device *cma_dev, *cur_dev;
struct sockaddr_ib *addr;
union ib_gid gid, sgid, *dgid;
unsigned int p;
u16 pkey, index;
u8 p;
enum ib_port_state port_state;
int i;
@ -798,7 +798,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
mutex_lock(&lock);
list_for_each_entry(cur_dev, &dev_list, list) {
for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
rdma_for_each_port (cur_dev->device, p) {
if (!rdma_cap_af_ib(cur_dev->device, p))
continue;
@ -840,9 +840,14 @@ found:
return 0;
}
static void cma_deref_id(struct rdma_id_private *id_priv)
static void cma_id_get(struct rdma_id_private *id_priv)
{
if (atomic_dec_and_test(&id_priv->refcount))
refcount_inc(&id_priv->refcount);
}
static void cma_id_put(struct rdma_id_private *id_priv)
{
if (refcount_dec_and_test(&id_priv->refcount))
complete(&id_priv->comp);
}
@ -870,7 +875,7 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,
spin_lock_init(&id_priv->lock);
mutex_init(&id_priv->qp_mutex);
init_completion(&id_priv->comp);
atomic_set(&id_priv->refcount, 1);
refcount_set(&id_priv->refcount, 1);
mutex_init(&id_priv->handler_mutex);
INIT_LIST_HEAD(&id_priv->listen_list);
INIT_LIST_HEAD(&id_priv->mc_list);
@ -1846,11 +1851,11 @@ void rdma_destroy_id(struct rdma_cm_id *id)
}
cma_release_port(id_priv);
cma_deref_id(id_priv);
cma_id_put(id_priv);
wait_for_completion(&id_priv->comp);
if (id_priv->internal_id)
cma_deref_id(id_priv->id.context);
cma_id_put(id_priv->id.context);
kfree(id_priv->id.route.path_rec);
@ -2187,7 +2192,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id,
* Protect against the user destroying conn_id from another thread
* until we're done accessing it.
*/
atomic_inc(&conn_id->refcount);
cma_id_get(conn_id);
ret = cma_cm_event_handler(conn_id, &event);
if (ret)
goto err3;
@ -2204,13 +2209,13 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id,
mutex_unlock(&lock);
mutex_unlock(&conn_id->handler_mutex);
mutex_unlock(&listen_id->handler_mutex);
cma_deref_id(conn_id);
cma_id_put(conn_id);
if (net_dev)
dev_put(net_dev);
return 0;
err3:
cma_deref_id(conn_id);
cma_id_put(conn_id);
/* Destroy the CM ID by returning a non-zero value. */
conn_id->cm_id.ib = NULL;
err2:
@ -2391,7 +2396,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
* Protect against the user destroying conn_id from another thread
* until we're done accessing it.
*/
atomic_inc(&conn_id->refcount);
cma_id_get(conn_id);
ret = cma_cm_event_handler(conn_id, &event);
if (ret) {
/* User wants to destroy the CM ID */
@ -2399,13 +2404,13 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
cma_exch(conn_id, RDMA_CM_DESTROYING);
mutex_unlock(&conn_id->handler_mutex);
mutex_unlock(&listen_id->handler_mutex);
cma_deref_id(conn_id);
cma_id_put(conn_id);
rdma_destroy_id(&conn_id->id);
return ret;
}
mutex_unlock(&conn_id->handler_mutex);
cma_deref_id(conn_id);
cma_id_put(conn_id);
out:
mutex_unlock(&listen_id->handler_mutex);
@ -2492,7 +2497,7 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv,
_cma_attach_to_dev(dev_id_priv, cma_dev);
list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
atomic_inc(&id_priv->refcount);
cma_id_get(id_priv);
dev_id_priv->internal_id = 1;
dev_id_priv->afonly = id_priv->afonly;
dev_id_priv->tos_set = id_priv->tos_set;
@ -2647,7 +2652,7 @@ static void cma_work_handler(struct work_struct *_work)
}
out:
mutex_unlock(&id_priv->handler_mutex);
cma_deref_id(id_priv);
cma_id_put(id_priv);
if (destroy)
rdma_destroy_id(&id_priv->id);
kfree(work);
@ -2671,7 +2676,7 @@ static void cma_ndev_work_handler(struct work_struct *_work)
out:
mutex_unlock(&id_priv->handler_mutex);
cma_deref_id(id_priv);
cma_id_put(id_priv);
if (destroy)
rdma_destroy_id(&id_priv->id);
kfree(work);
@ -2687,14 +2692,19 @@ static void cma_init_resolve_route_work(struct cma_work *work,
work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
}
static void cma_init_resolve_addr_work(struct cma_work *work,
struct rdma_id_private *id_priv)
static void enqueue_resolve_addr_work(struct cma_work *work,
struct rdma_id_private *id_priv)
{
/* Balances with cma_id_put() in cma_work_handler */
cma_id_get(id_priv);
work->id = id_priv;
INIT_WORK(&work->work, cma_work_handler);
work->old_state = RDMA_CM_ADDR_QUERY;
work->new_state = RDMA_CM_ADDR_RESOLVED;
work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
queue_work(cma_wq, &work->work);
}
static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
@ -2968,6 +2978,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
err2:
kfree(route->path_rec);
route->path_rec = NULL;
route->num_paths = 0;
err1:
kfree(work);
return ret;
@ -2982,7 +2993,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
return -EINVAL;
atomic_inc(&id_priv->refcount);
cma_id_get(id_priv);
if (rdma_cap_ib_sa(id->device, id->port_num))
ret = cma_resolve_ib_route(id_priv, timeout_ms);
else if (rdma_protocol_roce(id->device, id->port_num))
@ -2998,7 +3009,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
return 0;
err:
cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
cma_deref_id(id_priv);
cma_id_put(id_priv);
return ret;
}
EXPORT_SYMBOL(rdma_resolve_route);
@ -3025,9 +3036,9 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv)
struct cma_device *cma_dev, *cur_dev;
union ib_gid gid;
enum ib_port_state port_state;
unsigned int p;
u16 pkey;
int ret;
u8 p;
cma_dev = NULL;
mutex_lock(&lock);
@ -3039,7 +3050,7 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv)
if (!cma_dev)
cma_dev = cur_dev;
for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
rdma_for_each_port (cur_dev->device, p) {
if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
port_state == IB_PORT_ACTIVE) {
cma_dev = cur_dev;
@ -3148,9 +3159,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv)
rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
atomic_inc(&id_priv->refcount);
cma_init_resolve_addr_work(work, id_priv);
queue_work(cma_wq, &work->work);
enqueue_resolve_addr_work(work, id_priv);
return 0;
err:
kfree(work);
@ -3175,9 +3184,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
&(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
atomic_inc(&id_priv->refcount);
cma_init_resolve_addr_work(work, id_priv);
queue_work(cma_wq, &work->work);
enqueue_resolve_addr_work(work, id_priv);
return 0;
err:
kfree(work);
@ -4588,7 +4595,7 @@ static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id
INIT_WORK(&work->work, cma_ndev_work_handler);
work->id = id_priv;
work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
atomic_inc(&id_priv->refcount);
cma_id_get(id_priv);
queue_work(cma_wq, &work->work);
}
@ -4663,7 +4670,7 @@ static void cma_add_one(struct ib_device *device)
}
init_completion(&cma_dev->comp);
atomic_set(&cma_dev->refcount, 1);
refcount_set(&cma_dev->refcount, 1);
INIT_LIST_HEAD(&cma_dev->id_list);
ib_set_client_data(device, &cma_client, cma_dev);
@ -4722,11 +4729,11 @@ static void cma_process_remove(struct cma_device *cma_dev)
list_del(&id_priv->listen_list);
list_del_init(&id_priv->list);
atomic_inc(&id_priv->refcount);
cma_id_get(id_priv);
mutex_unlock(&lock);
ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
cma_deref_id(id_priv);
cma_id_put(id_priv);
if (ret)
rdma_destroy_id(&id_priv->id);
@ -4734,7 +4741,7 @@ static void cma_process_remove(struct cma_device *cma_dev)
}
mutex_unlock(&lock);
cma_deref_dev(cma_dev);
cma_dev_put(cma_dev);
wait_for_completion(&cma_dev->comp);
}
@ -4790,6 +4797,19 @@ static int __init cma_init(void)
{
int ret;
/*
* There is a rare lock ordering dependency in cma_netdev_callback()
* that only happens when bonding is enabled. Teach lockdep that rtnl
* must never be nested under lock so it can find these without having
* to test with bonding.
*/
if (IS_ENABLED(CONFIG_LOCKDEP)) {
rtnl_lock();
mutex_lock(&lock);
mutex_unlock(&lock);
rtnl_unlock();
}
cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
if (!cma_wq)
return -ENOMEM;

View File

@ -94,7 +94,7 @@ static int cma_configfs_params_get(struct config_item *item,
static void cma_configfs_params_put(struct cma_device *cma_dev)
{
cma_deref_dev(cma_dev);
cma_dev_put(cma_dev);
}
static ssize_t default_roce_mode_show(struct config_item *item,
@ -312,12 +312,12 @@ static struct config_group *make_cma_dev(struct config_group *group,
configfs_add_default_group(&cma_dev_group->ports_group,
&cma_dev_group->device_group);
cma_deref_dev(cma_dev);
cma_dev_put(cma_dev);
return &cma_dev_group->device_group;
fail:
if (cma_dev)
cma_deref_dev(cma_dev);
cma_dev_put(cma_dev);
kfree(cma_dev_group);
return ERR_PTR(err);
}

View File

@ -66,7 +66,7 @@ struct rdma_id_private {
struct mutex qp_mutex;
struct completion comp;
atomic_t refcount;
refcount_t refcount;
struct mutex handler_mutex;
int backlog;
@ -111,8 +111,8 @@ static inline void cma_configfs_exit(void)
}
#endif
void cma_ref_dev(struct cma_device *dev);
void cma_deref_dev(struct cma_device *dev);
void cma_dev_get(struct cma_device *dev);
void cma_dev_put(struct cma_device *dev);
typedef bool (*cma_device_filter)(struct ib_device *, void *);
struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,
void *cookie);

View File

@ -79,13 +79,13 @@ struct ib_mad_private {
struct ib_mad_private_header header;
size_t mad_size;
struct ib_grh grh;
u8 mad[0];
u8 mad[];
} __packed;
struct ib_rmpp_segment {
struct list_head list;
u32 num;
u8 data[0];
u8 data[];
};
struct ib_mad_agent_private {

View File

@ -71,7 +71,7 @@ struct mcast_device {
struct ib_event_handler event_handler;
int start_port;
int end_port;
struct mcast_port port[0];
struct mcast_port port[];
};
enum mcast_state {

View File

@ -391,13 +391,13 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
return -EINVAL;
}
ret = ib_dma_map_sg(dev, sg, sg_cnt, dir);
ret = rdma_rw_map_sg(dev, sg, sg_cnt, dir);
if (!ret)
return -ENOMEM;
sg_cnt = ret;
if (prot_sg_cnt) {
ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir);
ret = rdma_rw_map_sg(dev, prot_sg, prot_sg_cnt, dir);
if (!ret) {
ret = -ENOMEM;
goto out_unmap_sg;
@ -466,9 +466,9 @@ out_free_ctx:
kfree(ctx->reg);
out_unmap_prot_sg:
if (prot_sg_cnt)
ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
rdma_rw_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
out_unmap_sg:
ib_dma_unmap_sg(dev, sg, sg_cnt, dir);
rdma_rw_unmap_sg(dev, sg, sg_cnt, dir);
return ret;
}
EXPORT_SYMBOL(rdma_rw_ctx_signature_init);
@ -628,9 +628,9 @@ void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr);
kfree(ctx->reg);
ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
if (prot_sg_cnt)
ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir);
rdma_rw_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir);
rdma_rw_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
}
EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);

View File

@ -101,7 +101,7 @@ struct ib_sa_port {
struct ib_sa_device {
int start_port, end_port;
struct ib_event_handler event_handler;
struct ib_sa_port port[0];
struct ib_sa_port port[];
};
struct ib_sa_query {

View File

@ -85,12 +85,13 @@ struct ucma_file {
struct ucma_context {
u32 id;
struct completion comp;
atomic_t ref;
refcount_t ref;
int events_reported;
int backlog;
struct ucma_file *file;
struct rdma_cm_id *cm_id;
struct mutex mutex;
u64 uid;
struct list_head list;
@ -152,7 +153,7 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)
if (ctx->closing)
ctx = ERR_PTR(-EIO);
else
atomic_inc(&ctx->ref);
refcount_inc(&ctx->ref);
}
xa_unlock(&ctx_table);
return ctx;
@ -160,7 +161,7 @@ static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)
static void ucma_put_ctx(struct ucma_context *ctx)
{
if (atomic_dec_and_test(&ctx->ref))
if (refcount_dec_and_test(&ctx->ref))
complete(&ctx->comp);
}
@ -212,10 +213,11 @@ static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)
return NULL;
INIT_WORK(&ctx->close_work, ucma_close_id);
atomic_set(&ctx->ref, 1);
refcount_set(&ctx->ref, 1);
init_completion(&ctx->comp);
INIT_LIST_HEAD(&ctx->mc_list);
ctx->file = file;
mutex_init(&ctx->mutex);
if (xa_alloc(&ctx_table, &ctx->id, ctx, xa_limit_32b, GFP_KERNEL))
goto error;
@ -589,6 +591,7 @@ static int ucma_free_ctx(struct ucma_context *ctx)
}
events_reported = ctx->events_reported;
mutex_destroy(&ctx->mutex);
kfree(ctx);
return events_reported;
}
@ -658,7 +661,10 @@ static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -681,7 +687,9 @@ static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -705,8 +713,10 @@ static ssize_t ucma_resolve_ip(struct ucma_file *file,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr,
(struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -731,8 +741,10 @@ static ssize_t ucma_resolve_addr(struct ucma_file *file,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr,
(struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -752,7 +764,9 @@ static ssize_t ucma_resolve_route(struct ucma_file *file,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_resolve_route(ctx->cm_id, cmd.timeout_ms);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -841,6 +855,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
memset(&resp, 0, sizeof resp);
addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr;
memcpy(&resp.src_addr, addr, addr->sa_family == AF_INET ?
@ -864,6 +879,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
ucma_copy_iw_route(&resp, &ctx->cm_id->route);
out:
mutex_unlock(&ctx->mutex);
if (copy_to_user(u64_to_user_ptr(cmd.response),
&resp, sizeof(resp)))
ret = -EFAULT;
@ -1014,6 +1030,7 @@ static ssize_t ucma_query(struct ucma_file *file,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
switch (cmd.option) {
case RDMA_USER_CM_QUERY_ADDR:
ret = ucma_query_addr(ctx, response, out_len);
@ -1028,6 +1045,7 @@ static ssize_t ucma_query(struct ucma_file *file,
ret = -ENOSYS;
break;
}
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
@ -1045,7 +1063,7 @@ static void ucma_copy_conn_param(struct rdma_cm_id *id,
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
dst->srq = src->srq;
dst->qp_num = src->qp_num;
dst->qp_num = src->qp_num & 0xFFFFFF;
dst->qkey = (id->route.addr.src_addr.ss_family == AF_IB) ? src->qkey : 0;
}
@ -1068,7 +1086,9 @@ static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
return PTR_ERR(ctx);
ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
mutex_lock(&ctx->mutex);
ret = rdma_connect(ctx->cm_id, &conn_param);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -1089,7 +1109,9 @@ static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf,
ctx->backlog = cmd.backlog > 0 && cmd.backlog < max_backlog ?
cmd.backlog : max_backlog;
mutex_lock(&ctx->mutex);
ret = rdma_listen(ctx->cm_id, ctx->backlog);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -1112,13 +1134,17 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
if (cmd.conn_param.valid) {
ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
mutex_lock(&file->mut);
mutex_lock(&ctx->mutex);
ret = __rdma_accept(ctx->cm_id, &conn_param, NULL);
mutex_unlock(&ctx->mutex);
if (!ret)
ctx->uid = cmd.uid;
mutex_unlock(&file->mut);
} else
} else {
mutex_lock(&ctx->mutex);
ret = __rdma_accept(ctx->cm_id, NULL, NULL);
mutex_unlock(&ctx->mutex);
}
ucma_put_ctx(ctx);
return ret;
}
@ -1137,7 +1163,9 @@ static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_reject(ctx->cm_id, cmd.private_data, cmd.private_data_len);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -1156,7 +1184,9 @@ static ssize_t ucma_disconnect(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
ret = rdma_disconnect(ctx->cm_id);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
}
@ -1187,7 +1217,9 @@ static ssize_t ucma_init_qp_attr(struct ucma_file *file,
resp.qp_attr_mask = 0;
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.qp_state = cmd.qp_state;
mutex_lock(&ctx->mutex);
ret = rdma_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask);
mutex_unlock(&ctx->mutex);
if (ret)
goto out;
@ -1273,9 +1305,13 @@ static int ucma_set_ib_path(struct ucma_context *ctx,
struct sa_path_rec opa;
sa_convert_path_ib_to_opa(&opa, &sa_path);
mutex_lock(&ctx->mutex);
ret = rdma_set_ib_path(ctx->cm_id, &opa);
mutex_unlock(&ctx->mutex);
} else {
mutex_lock(&ctx->mutex);
ret = rdma_set_ib_path(ctx->cm_id, &sa_path);
mutex_unlock(&ctx->mutex);
}
if (ret)
return ret;
@ -1308,7 +1344,9 @@ static int ucma_set_option_level(struct ucma_context *ctx, int level,
switch (level) {
case RDMA_OPTION_ID:
mutex_lock(&ctx->mutex);
ret = ucma_set_option_id(ctx, optname, optval, optlen);
mutex_unlock(&ctx->mutex);
break;
case RDMA_OPTION_IB:
ret = ucma_set_option_ib(ctx, optname, optval, optlen);
@ -1368,8 +1406,10 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,
if (IS_ERR(ctx))
return PTR_ERR(ctx);
mutex_lock(&ctx->mutex);
if (ctx->cm_id->device)
ret = rdma_notify(ctx->cm_id, (enum ib_event_type)cmd.event);
mutex_unlock(&ctx->mutex);
ucma_put_ctx(ctx);
return ret;
@ -1412,8 +1452,10 @@ static ssize_t ucma_process_join(struct ucma_file *file,
mc->join_state = join_state;
mc->uid = cmd->uid;
memcpy(&mc->addr, addr, cmd->addr_size);
mutex_lock(&ctx->mutex);
ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr,
join_state, mc);
mutex_unlock(&ctx->mutex);
if (ret)
goto err2;
@ -1502,7 +1544,7 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,
mc = ERR_PTR(-ENOENT);
else if (mc->ctx->file != file)
mc = ERR_PTR(-EINVAL);
else if (!atomic_inc_not_zero(&mc->ctx->ref))
else if (!refcount_inc_not_zero(&mc->ctx->ref))
mc = ERR_PTR(-ENXIO);
else
__xa_erase(&multicast_table, mc->id);
@ -1513,7 +1555,10 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,
goto out;
}
mutex_lock(&mc->ctx->mutex);
rdma_leave_multicast(mc->ctx->cm_id, (struct sockaddr *) &mc->addr);
mutex_unlock(&mc->ctx->mutex);
mutex_lock(&mc->ctx->file->mut);
ucma_cleanup_mc_events(mc);
list_del(&mc->list);

View File

@ -197,6 +197,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
unsigned long lock_limit;
unsigned long new_pinned;
unsigned long cur_base;
unsigned long dma_attr = 0;
struct mm_struct *mm;
unsigned long npages;
int ret;
@ -278,10 +279,12 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
sg_mark_end(sg);
umem->nmap = ib_dma_map_sg(device,
umem->sg_head.sgl,
umem->sg_nents,
DMA_BIDIRECTIONAL);
if (access & IB_ACCESS_RELAXED_ORDERING)
dma_attr |= DMA_ATTR_WEAK_ORDERING;
umem->nmap =
ib_dma_map_sg_attrs(device, umem->sg_head.sgl, umem->sg_nents,
DMA_BIDIRECTIONAL, dma_attr);
if (!umem->nmap) {
ret = -ENOMEM;

View File

@ -54,8 +54,6 @@
#include "core_priv.h"
#include <trace/events/rdma_core.h>
#include <trace/events/rdma_core.h>
static int ib_resolve_eth_dmac(struct ib_device *device,
struct rdma_ah_attr *ah_attr);
@ -1127,8 +1125,7 @@ struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
EXPORT_SYMBOL(ib_open_qp);
static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
struct ib_qp_init_attr *qp_init_attr,
struct ib_udata *udata)
struct ib_qp_init_attr *qp_init_attr)
{
struct ib_qp *real_qp = qp;
@ -1150,9 +1147,18 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
return qp;
}
struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
struct ib_qp_init_attr *qp_init_attr,
struct ib_udata *udata)
/**
* ib_create_qp - Creates a kernel QP associated with the specified protection
* domain.
* @pd: The protection domain associated with the QP.
* @qp_init_attr: A list of initial attributes required to create the
* QP. If QP creation succeeds, then the attributes are updated to
* the actual capabilities of the created QP.
*
* NOTE: for user qp use ib_create_qp_user with valid udata!
*/
struct ib_qp *ib_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *qp_init_attr)
{
struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device;
struct ib_qp *qp;
@ -1187,7 +1193,7 @@ struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
struct ib_qp *xrc_qp =
create_xrc_qp_user(qp, qp_init_attr, udata);
create_xrc_qp_user(qp, qp_init_attr);
if (IS_ERR(xrc_qp)) {
ret = PTR_ERR(xrc_qp);
@ -1243,7 +1249,7 @@ err:
return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_create_qp_user);
EXPORT_SYMBOL(ib_create_qp);
static const struct {
int valid;

View File

@ -89,6 +89,15 @@
#define BNXT_RE_DEFAULT_ACK_DELAY 16
struct bnxt_re_ring_attr {
dma_addr_t *dma_arr;
int pages;
int type;
u32 depth;
u32 lrid; /* Logical ring id */
u8 mode;
};
struct bnxt_re_work {
struct work_struct work;
unsigned long event;
@ -104,6 +113,14 @@ struct bnxt_re_sqp_entries {
struct bnxt_re_qp *qp1_qp;
};
#define BNXT_RE_MAX_GSI_SQP_ENTRIES 1024
struct bnxt_re_gsi_context {
struct bnxt_re_qp *gsi_qp;
struct bnxt_re_qp *gsi_sqp;
struct bnxt_re_ah *gsi_sah;
struct bnxt_re_sqp_entries *sqp_tbl;
};
#define BNXT_RE_MIN_MSIX 2
#define BNXT_RE_MAX_MSIX 9
#define BNXT_RE_AEQ_IDX 0
@ -115,7 +132,6 @@ struct bnxt_re_dev {
struct list_head list;
unsigned long flags;
#define BNXT_RE_FLAG_NETDEV_REGISTERED 0
#define BNXT_RE_FLAG_IBDEV_REGISTERED 1
#define BNXT_RE_FLAG_GOT_MSIX 2
#define BNXT_RE_FLAG_HAVE_L2_REF 3
#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 4
@ -125,7 +141,7 @@ struct bnxt_re_dev {
#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29
struct net_device *netdev;
unsigned int version, major, minor;
struct bnxt_qplib_chip_ctx chip_ctx;
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
int num_msix;
@ -160,15 +176,11 @@ struct bnxt_re_dev {
atomic_t srq_count;
atomic_t mr_count;
atomic_t mw_count;
atomic_t sched_count;
/* Max of 2 lossless traffic class supported per port */
u16 cosq[2];
/* QP for for handling QP1 packets */
u32 sqp_id;
struct bnxt_re_qp *qp1_sqp;
struct bnxt_re_ah *sqp_ah;
struct bnxt_re_sqp_entries sqp_tbl[1024];
struct bnxt_re_gsi_context gsi_ctx;
atomic_t nq_alloc_cnt;
u32 is_virtfn;
u32 num_vfs;

File diff suppressed because it is too large Load Diff

View File

@ -78,26 +78,43 @@ static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
/* Mutex to protect the list of bnxt_re devices added */
static DEFINE_MUTEX(bnxt_re_dev_lock);
static struct workqueue_struct *bnxt_re_wq;
static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev);
static void bnxt_re_remove_device(struct bnxt_re_dev *rdev);
static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
static void bnxt_re_stop_irq(void *handle);
static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
if (!rdev->chip_ctx)
return;
chip_ctx = rdev->chip_ctx;
rdev->chip_ctx = NULL;
rdev->rcfw.res = NULL;
rdev->qplib_res.cctx = NULL;
rdev->qplib_res.pdev = NULL;
rdev->qplib_res.netdev = NULL;
kfree(chip_ctx);
}
static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
struct bnxt *bp;
en_dev = rdev->en_dev;
bp = netdev_priv(en_dev->net);
rdev->chip_ctx.chip_num = bp->chip_num;
chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
if (!chip_ctx)
return -ENOMEM;
chip_ctx->chip_num = bp->chip_num;
rdev->chip_ctx = chip_ctx;
/* rest members to follow eventually */
rdev->qplib_res.cctx = &rdev->chip_ctx;
rdev->qplib_res.cctx = rdev->chip_ctx;
rdev->rcfw.res = &rdev->qplib_res;
return 0;
@ -136,9 +153,9 @@ static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
ctx->srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
attr->max_srq);
ctx->cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr->max_cq);
if (!bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx))
if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))
for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
rdev->qplib_ctx.tqm_count[i] =
rdev->qplib_ctx.tqm_ctx.qcount[i] =
rdev->dev_attr.tqm_alloc_reqs[i];
}
@ -185,7 +202,7 @@ static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
memset(&rdev->qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
bnxt_re_limit_pf_res(rdev);
num_vfs = bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ?
num_vfs = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
BNXT_RE_GEN_P5_MAX_VF : rdev->num_vfs;
if (num_vfs)
bnxt_re_limit_vf_res(&rdev->qplib_ctx, num_vfs);
@ -208,7 +225,7 @@ static void bnxt_re_sriov_config(void *p, int num_vfs)
return;
rdev->num_vfs = num_vfs;
if (!bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx)) {
if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
bnxt_re_set_resource_limits(rdev);
bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
&rdev->qplib_ctx);
@ -221,8 +238,10 @@ static void bnxt_re_shutdown(void *p)
if (!rdev)
return;
bnxt_re_ib_unreg(rdev);
ASSERT_RTNL();
/* Release the MSIx vectors before queuing unregister */
bnxt_re_stop_irq(rdev);
ib_unregister_device_queued(&rdev->ibdev);
}
static void bnxt_re_stop_irq(void *handle)
@ -254,7 +273,7 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
* to f/w will timeout and that will set the
* timeout bit.
*/
dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
ibdev_err(&rdev->ibdev, "Failed to re-start IRQs\n");
return;
}
@ -271,8 +290,8 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
msix_ent[indx].vector, false);
if (rc)
dev_warn(rdev_to_dev(rdev),
"Failed to reinit NQ index %d\n", indx - 1);
ibdev_warn(&rdev->ibdev, "Failed to reinit NQ index %d\n",
indx - 1);
}
}
@ -358,9 +377,9 @@ static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
goto done;
}
if (num_msix_got != num_msix_want) {
dev_warn(rdev_to_dev(rdev),
"Requested %d MSI-X vectors, got %d\n",
num_msix_want, num_msix_got);
ibdev_warn(&rdev->ibdev,
"Requested %d MSI-X vectors, got %d\n",
num_msix_want, num_msix_got);
}
rdev->num_msix = num_msix_got;
done:
@ -407,14 +426,14 @@ static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
if (rc)
dev_err(rdev_to_dev(rdev),
"Failed to free HW ring:%d :%#x", req.ring_id, rc);
ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
req.ring_id, rc);
return rc;
}
static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr,
int pages, int type, u32 ring_mask,
u32 map_index, u16 *fw_ring_id)
static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
struct bnxt_re_ring_attr *ring_attr,
u16 *fw_ring_id)
{
struct bnxt_en_dev *en_dev = rdev->en_dev;
struct hwrm_ring_alloc_input req = {0};
@ -428,18 +447,18 @@ static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr,
memset(&fw_msg, 0, sizeof(fw_msg));
bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
req.enables = 0;
req.page_tbl_addr = cpu_to_le64(dma_arr[0]);
if (pages > 1) {
req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
if (ring_attr->pages > 1) {
/* Page size is in log2 units */
req.page_size = BNXT_PAGE_SHIFT;
req.page_tbl_depth = 1;
}
req.fbo = 0;
/* Association of ring index with doorbell index and MSIX number */
req.logical_id = cpu_to_le16(map_index);
req.length = cpu_to_le32(ring_mask + 1);
req.ring_type = type;
req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
req.logical_id = cpu_to_le16(ring_attr->lrid);
req.length = cpu_to_le32(ring_attr->depth + 1);
req.ring_type = ring_attr->type;
req.int_mode = ring_attr->mode;
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
@ -468,8 +487,8 @@ static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
sizeof(req), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
if (rc)
dev_err(rdev_to_dev(rdev),
"Failed to free HW stats context %#x", rc);
ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
rc);
return rc;
}
@ -524,17 +543,12 @@ static bool is_bnxt_re_dev(struct net_device *netdev)
static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
{
struct bnxt_re_dev *rdev;
struct ib_device *ibdev =
ib_device_get_by_netdev(netdev, RDMA_DRIVER_BNXT_RE);
if (!ibdev)
return NULL;
rcu_read_lock();
list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
if (rdev->netdev == netdev) {
rcu_read_unlock();
return rdev;
}
}
rcu_read_unlock();
return NULL;
return container_of(ibdev, struct bnxt_re_dev, ibdev);
}
static void bnxt_re_dev_unprobe(struct net_device *netdev,
@ -608,11 +622,6 @@ static const struct attribute_group bnxt_re_dev_attr_group = {
.attrs = bnxt_re_attributes,
};
static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)
{
ib_unregister_device(&rdev->ibdev);
}
static const struct ib_device_ops bnxt_re_dev_ops = {
.owner = THIS_MODULE,
.driver_id = RDMA_DRIVER_BNXT_RE,
@ -627,6 +636,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
.create_cq = bnxt_re_create_cq,
.create_qp = bnxt_re_create_qp,
.create_srq = bnxt_re_create_srq,
.dealloc_driver = bnxt_re_dealloc_driver,
.dealloc_pd = bnxt_re_dealloc_pd,
.dealloc_ucontext = bnxt_re_dealloc_ucontext,
.del_gid = bnxt_re_del_gid,
@ -723,15 +733,11 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
{
dev_put(rdev->netdev);
rdev->netdev = NULL;
mutex_lock(&bnxt_re_dev_lock);
list_del_rcu(&rdev->list);
mutex_unlock(&bnxt_re_dev_lock);
synchronize_rcu();
ib_dealloc_device(&rdev->ibdev);
/* rdev is gone */
}
static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
@ -742,8 +748,8 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
/* Allocate bnxt_re_dev instance here */
rdev = ib_alloc_device(bnxt_re_dev, ibdev);
if (!rdev) {
dev_err(NULL, "%s: bnxt_re_dev allocation failure!",
ROCE_DRV_MODULE_NAME);
ibdev_err(NULL, "%s: bnxt_re_dev allocation failure!",
ROCE_DRV_MODULE_NAME);
return NULL;
}
/* Default values */
@ -872,8 +878,8 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
int rc = 0;
if (!srq) {
dev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
ROCE_DRV_MODULE_NAME);
ibdev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
ROCE_DRV_MODULE_NAME);
rc = -EINVAL;
goto done;
}
@ -900,8 +906,8 @@ static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
qplib_cq);
if (!cq) {
dev_err(NULL, "%s: CQ is NULL, CQN not handled",
ROCE_DRV_MODULE_NAME);
ibdev_err(NULL, "%s: CQ is NULL, CQN not handled",
ROCE_DRV_MODULE_NAME);
return -EINVAL;
}
if (cq->ib_cq.comp_handler) {
@ -916,7 +922,7 @@ static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
#define BNXT_RE_GEN_P5_VF_NQ_DB 0x4000
static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
{
return bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ?
return bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
(rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
BNXT_RE_GEN_P5_PF_NQ_DB) :
rdev->msix_entries[indx].db_offset;
@ -948,8 +954,8 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
db_offt, &bnxt_re_cqn_handler,
&bnxt_re_srqn_handler);
if (rc) {
dev_err(rdev_to_dev(rdev),
"Failed to enable NQ with rc = 0x%x", rc);
ibdev_err(&rdev->ibdev,
"Failed to enable NQ with rc = 0x%x", rc);
goto fail;
}
num_vec_enabled++;
@ -967,10 +973,10 @@ static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)
int i;
for (i = 0; i < rdev->num_msix - 1; i++) {
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
rdev->nq[i].res = NULL;
bnxt_qplib_free_nq(&rdev->nq[i]);
rdev->nq[i].res = NULL;
}
}
@ -991,10 +997,10 @@ static void bnxt_re_free_res(struct bnxt_re_dev *rdev)
static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
{
struct bnxt_re_ring_attr rattr = {};
struct bnxt_qplib_ctx *qplib_ctx;
int num_vec_created = 0;
dma_addr_t *pg_map;
int rc = 0, i;
int pages;
u8 type;
/* Configure and allocate resources for qplib */
@ -1015,27 +1021,31 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
if (rc)
goto dealloc_res;
qplib_ctx = &rdev->qplib_ctx;
for (i = 0; i < rdev->num_msix - 1; i++) {
rdev->nq[i].res = &rdev->qplib_res;
rdev->nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
BNXT_RE_MAX_SRQC_COUNT + 2;
rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq[i]);
struct bnxt_qplib_nq *nq;
nq = &rdev->nq[i];
nq->hwq.max_elements = (qplib_ctx->cq_count +
qplib_ctx->srqc_count + 2);
rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, &rdev->nq[i]);
if (rc) {
dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x",
i, rc);
ibdev_err(&rdev->ibdev, "Alloc Failed NQ%d rc:%#x",
i, rc);
goto free_nq;
}
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
pg_map = rdev->nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr;
pages = rdev->nq[i].hwq.pbl[rdev->nq[i].hwq.level].pg_count;
rc = bnxt_re_net_ring_alloc(rdev, pg_map, pages, type,
BNXT_QPLIB_NQE_MAX_CNT - 1,
rdev->msix_entries[i + 1].ring_idx,
&rdev->nq[i].ring_id);
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;
rattr.type = type;
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
rattr.lrid = rdev->msix_entries[i + 1].ring_idx;
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
if (rc) {
dev_err(rdev_to_dev(rdev),
"Failed to allocate NQ fw id with rc = 0x%x",
rc);
ibdev_err(&rdev->ibdev,
"Failed to allocate NQ fw id with rc = 0x%x",
rc);
bnxt_qplib_free_nq(&rdev->nq[i]);
goto free_nq;
}
@ -1043,8 +1053,8 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
}
return 0;
free_nq:
for (i = num_vec_created; i >= 0; i--) {
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
for (i = num_vec_created - 1; i >= 0; i--) {
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
bnxt_qplib_free_nq(&rdev->nq[i]);
}
@ -1109,10 +1119,10 @@ static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
return rc;
if (resp.queue_cfg_info) {
dev_warn(rdev_to_dev(rdev),
"Asymmetric cos queue configuration detected");
dev_warn(rdev_to_dev(rdev),
" on device, QoS may not be fully functional\n");
ibdev_warn(&rdev->ibdev,
"Asymmetric cos queue configuration detected");
ibdev_warn(&rdev->ibdev,
" on device, QoS may not be fully functional\n");
}
qcfgmap = &resp.pri0_cos_queue_id;
tmp_map = (u8 *)cid_map;
@ -1125,7 +1135,8 @@ static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
struct bnxt_re_qp *qp)
{
return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp);
return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
(qp == rdev->gsi_ctx.gsi_sqp);
}
static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
@ -1160,12 +1171,13 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
u16 gid_idx, index;
int rc = 0;
if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
if (!ib_device_try_get(&rdev->ibdev))
return 0;
if (!sgid_tbl) {
dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
return -EINVAL;
ibdev_err(&rdev->ibdev, "QPLIB: SGID table not allocated");
rc = -EINVAL;
goto out;
}
for (index = 0; index < sgid_tbl->active; index++) {
@ -1185,7 +1197,8 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
rdev->qplib_res.netdev->dev_addr);
}
out:
ib_device_put(&rdev->ibdev);
return rc;
}
@ -1241,7 +1254,7 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
/* Get cosq id for this priority */
rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map);
if (rc) {
dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map);
ibdev_warn(&rdev->ibdev, "no cos for p_mask %x\n", prio_map);
return rc;
}
/* Parse CoS IDs for app priority */
@ -1250,8 +1263,8 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
/* Config BONO. */
rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq);
if (rc) {
dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n",
rdev->cosq[0], rdev->cosq[1]);
ibdev_warn(&rdev->ibdev, "no tc for cos{%x, %x}\n",
rdev->cosq[0], rdev->cosq[1]);
return rc;
}
@ -1286,8 +1299,8 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
if (rc) {
dev_err(rdev_to_dev(rdev),
"Failed to query HW version, rc = 0x%x", rc);
ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
rc);
return;
}
rdev->qplib_ctx.hwrm_intf_ver =
@ -1297,15 +1310,35 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
le16_to_cpu(resp.hwrm_intf_patch);
}
static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev)
static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
{
int rc = 0;
u32 event;
/* Register ib dev */
rc = bnxt_re_register_ib(rdev);
if (rc) {
pr_err("Failed to register with IB: %#x\n", rc);
return rc;
}
dev_info(rdev_to_dev(rdev), "Device registered successfully");
ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
&rdev->active_width);
set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
event = netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev) ?
IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, event);
return rc;
}
static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
{
u8 type;
int rc;
if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
/* Cleanup ib dev */
bnxt_re_unregister_ib(rdev);
}
if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags))
cancel_delayed_work_sync(&rdev->worker);
@ -1318,28 +1351,28 @@ static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev)
if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
if (rc)
dev_warn(rdev_to_dev(rdev),
"Failed to deinitialize RCFW: %#x", rc);
ibdev_warn(&rdev->ibdev,
"Failed to deinitialize RCFW: %#x", rc);
bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, type);
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
}
if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) {
rc = bnxt_re_free_msix(rdev);
if (rc)
dev_warn(rdev_to_dev(rdev),
"Failed to free MSI-X vectors: %#x", rc);
ibdev_warn(&rdev->ibdev,
"Failed to free MSI-X vectors: %#x", rc);
}
bnxt_re_destroy_chip_ctx(rdev);
if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
rc = bnxt_re_unregister_netdev(rdev);
if (rc)
dev_warn(rdev_to_dev(rdev),
"Failed to unregister with netdev: %#x", rc);
ibdev_warn(&rdev->ibdev,
"Failed to unregister with netdev: %#x", rc);
}
}
@ -1353,31 +1386,29 @@ static void bnxt_re_worker(struct work_struct *work)
schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
}
static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
static int bnxt_re_dev_init(struct bnxt_re_dev *rdev)
{
dma_addr_t *pg_map;
u32 db_offt, ridx;
int pages, vid;
bool locked;
struct bnxt_qplib_creq_ctx *creq;
struct bnxt_re_ring_attr rattr;
u32 db_offt;
int vid;
u8 type;
int rc;
/* Acquire rtnl lock through out this function */
rtnl_lock();
locked = true;
/* Registered a new RoCE device instance to netdev */
memset(&rattr, 0, sizeof(rattr));
rc = bnxt_re_register_netdev(rdev);
if (rc) {
rtnl_unlock();
pr_err("Failed to register with netedev: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to register with netedev: %#x\n", rc);
return -EINVAL;
}
set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
rc = bnxt_re_setup_chip_ctx(rdev);
if (rc) {
dev_err(rdev_to_dev(rdev), "Failed to get chip context\n");
ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
return -EINVAL;
}
@ -1386,7 +1417,8 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
rc = bnxt_re_request_msix(rdev);
if (rc) {
pr_err("Failed to get MSI-X vectors: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to get MSI-X vectors: %#x\n", rc);
rc = -EINVAL;
goto fail;
}
@ -1397,31 +1429,36 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
/* Establish RCFW Communication Channel to initialize the context
* memory for the function and all child VFs
*/
rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw,
rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res, &rdev->rcfw,
&rdev->qplib_ctx,
BNXT_RE_MAX_QPC_COUNT);
if (rc) {
pr_err("Failed to allocate RCFW Channel: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to allocate RCFW Channel: %#x\n", rc);
goto fail;
}
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
pg_map = rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr;
pages = rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count;
ridx = rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
rc = bnxt_re_net_ring_alloc(rdev, pg_map, pages, type,
BNXT_QPLIB_CREQE_MAX_CNT - 1,
ridx, &rdev->rcfw.creq_ring_id);
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
creq = &rdev->rcfw.creq;
rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
rattr.type = type;
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
rattr.lrid = rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
if (rc) {
pr_err("Failed to allocate CREQ: %#x\n", rc);
ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
goto free_rcfw;
}
db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
vid = rdev->msix_entries[BNXT_RE_AEQ_IDX].vector;
rc = bnxt_qplib_enable_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw,
rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw,
vid, db_offt, rdev->is_virtfn,
&bnxt_re_aeq_handler);
if (rc) {
pr_err("Failed to enable RCFW channel: %#x\n", rc);
ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
rc);
goto free_ring;
}
@ -1432,24 +1469,27 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
bnxt_re_set_resource_limits(rdev);
rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0,
bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx));
rc = bnxt_qplib_alloc_ctx(&rdev->qplib_res, &rdev->qplib_ctx, 0,
bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx));
if (rc) {
pr_err("Failed to allocate QPLIB context: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to allocate QPLIB context: %#x\n", rc);
goto disable_rcfw;
}
rc = bnxt_re_net_stats_ctx_alloc(rdev,
rdev->qplib_ctx.stats.dma_map,
&rdev->qplib_ctx.stats.fw_id);
if (rc) {
pr_err("Failed to allocate stats context: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to allocate stats context: %#x\n", rc);
goto free_ctx;
}
rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx,
rdev->is_virtfn);
if (rc) {
pr_err("Failed to initialize RCFW: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to initialize RCFW: %#x\n", rc);
goto free_sctx;
}
set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
@ -1457,13 +1497,15 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
/* Resources based on the 'new' device caps */
rc = bnxt_re_alloc_res(rdev);
if (rc) {
pr_err("Failed to allocate resources: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to allocate resources: %#x\n", rc);
goto fail;
}
set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags);
rc = bnxt_re_init_res(rdev);
if (rc) {
pr_err("Failed to initialize resources: %#x\n", rc);
ibdev_err(&rdev->ibdev,
"Failed to initialize resources: %#x\n", rc);
goto fail;
}
@ -1472,46 +1514,28 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
if (!rdev->is_virtfn) {
rc = bnxt_re_setup_qos(rdev);
if (rc)
pr_info("RoCE priority not yet configured\n");
ibdev_info(&rdev->ibdev,
"RoCE priority not yet configured\n");
INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
}
rtnl_unlock();
locked = false;
/* Register ib dev */
rc = bnxt_re_register_ib(rdev);
if (rc) {
pr_err("Failed to register with IB: %#x\n", rc);
goto fail;
}
set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
dev_info(rdev_to_dev(rdev), "Device registered successfully");
ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
&rdev->active_width);
set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
return 0;
free_sctx:
bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
free_ctx:
bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
disable_rcfw:
bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
free_ring:
type = bnxt_qplib_get_ring_type(&rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, type);
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
free_rcfw:
bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
fail:
if (!locked)
rtnl_lock();
bnxt_re_ib_unreg(rdev);
rtnl_unlock();
bnxt_re_dev_uninit(rdev);
return rc;
}
@ -1538,7 +1562,8 @@ static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)
en_dev = bnxt_re_dev_probe(netdev);
if (IS_ERR(en_dev)) {
if (en_dev != ERR_PTR(-ENODEV))
pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME);
ibdev_err(&(*rdev)->ibdev, "%s: Failed to probe\n",
ROCE_DRV_MODULE_NAME);
rc = PTR_ERR(en_dev);
goto exit;
}
@ -1552,9 +1577,47 @@ exit:
return rc;
}
static void bnxt_re_remove_one(struct bnxt_re_dev *rdev)
static void bnxt_re_remove_device(struct bnxt_re_dev *rdev)
{
bnxt_re_dev_uninit(rdev);
pci_dev_put(rdev->en_dev->pdev);
bnxt_re_dev_unreg(rdev);
}
static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
struct net_device *netdev)
{
int rc;
rc = bnxt_re_dev_reg(rdev, netdev);
if (rc == -ENODEV)
return rc;
if (rc) {
pr_err("Failed to register with the device %s: %#x\n",
netdev->name, rc);
return rc;
}
pci_dev_get((*rdev)->en_dev->pdev);
rc = bnxt_re_dev_init(*rdev);
if (rc) {
pci_dev_put((*rdev)->en_dev->pdev);
bnxt_re_dev_unreg(*rdev);
}
return rc;
}
static void bnxt_re_dealloc_driver(struct ib_device *ib_dev)
{
struct bnxt_re_dev *rdev =
container_of(ib_dev, struct bnxt_re_dev, ibdev);
dev_info(rdev_to_dev(rdev), "Unregistering Device");
rtnl_lock();
bnxt_re_remove_device(rdev);
rtnl_unlock();
}
/* Handle all deferred netevents tasks */
@ -1567,21 +1630,23 @@ static void bnxt_re_task(struct work_struct *work)
re_work = container_of(work, struct bnxt_re_work, work);
rdev = re_work->rdev;
if (re_work->event != NETDEV_REGISTER &&
!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
return;
switch (re_work->event) {
case NETDEV_REGISTER:
rc = bnxt_re_ib_reg(rdev);
if (re_work->event == NETDEV_REGISTER) {
rc = bnxt_re_ib_init(rdev);
if (rc) {
dev_err(rdev_to_dev(rdev),
"Failed to register with IB: %#x", rc);
bnxt_re_remove_one(rdev);
bnxt_re_dev_unreg(rdev);
ibdev_err(&rdev->ibdev,
"Failed to register with IB: %#x", rc);
rtnl_lock();
bnxt_re_remove_device(rdev);
rtnl_unlock();
goto exit;
}
break;
goto exit;
}
if (!ib_device_try_get(&rdev->ibdev))
goto exit;
switch (re_work->event) {
case NETDEV_UP:
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_PORT_ACTIVE);
@ -1601,17 +1666,12 @@ static void bnxt_re_task(struct work_struct *work)
default:
break;
}
smp_mb__before_atomic();
atomic_dec(&rdev->sched_count);
ib_device_put(&rdev->ibdev);
exit:
put_device(&rdev->ibdev.dev);
kfree(re_work);
}
static void bnxt_re_init_one(struct bnxt_re_dev *rdev)
{
pci_dev_get(rdev->en_dev->pdev);
}
/*
* "Notifier chain callback can be invoked for the same chain from
* different CPUs at the same time".
@ -1634,6 +1694,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
struct bnxt_re_dev *rdev;
int rc = 0;
bool sch_work = false;
bool release = true;
real_dev = rdma_vlan_dev_real_dev(netdev);
if (!real_dev)
@ -1641,7 +1702,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
rdev = bnxt_re_from_netdev(real_dev);
if (!rdev && event != NETDEV_REGISTER)
goto exit;
return NOTIFY_OK;
if (real_dev != netdev)
goto exit;
@ -1649,27 +1711,14 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
case NETDEV_REGISTER:
if (rdev)
break;
rc = bnxt_re_dev_reg(&rdev, real_dev);
if (rc == -ENODEV)
break;
if (rc) {
pr_err("Failed to register with the device %s: %#x\n",
real_dev->name, rc);
break;
}
bnxt_re_init_one(rdev);
sch_work = true;
rc = bnxt_re_add_device(&rdev, real_dev);
if (!rc)
sch_work = true;
release = false;
break;
case NETDEV_UNREGISTER:
/* netdev notifier will call NETDEV_UNREGISTER again later since
* we are still holding the reference to the netdev
*/
if (atomic_read(&rdev->sched_count) > 0)
goto exit;
bnxt_re_ib_unreg(rdev);
bnxt_re_remove_one(rdev);
bnxt_re_dev_unreg(rdev);
ib_unregister_device_queued(&rdev->ibdev);
break;
default:
@ -1680,17 +1729,19 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
/* Allocate for the deferred task */
re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC);
if (re_work) {
get_device(&rdev->ibdev.dev);
re_work->rdev = rdev;
re_work->event = event;
re_work->vlan_dev = (real_dev == netdev ?
NULL : netdev);
INIT_WORK(&re_work->work, bnxt_re_task);
atomic_inc(&rdev->sched_count);
queue_work(bnxt_re_wq, &re_work->work);
}
}
exit:
if (rdev && release)
ib_device_put(&rdev->ibdev);
return NOTIFY_DONE;
}
@ -1726,36 +1777,21 @@ err_netdev:
static void __exit bnxt_re_mod_exit(void)
{
struct bnxt_re_dev *rdev, *next;
LIST_HEAD(to_be_deleted);
struct bnxt_re_dev *rdev;
mutex_lock(&bnxt_re_dev_lock);
/* Free all adapter allocated resources */
if (!list_empty(&bnxt_re_dev_list))
list_splice_init(&bnxt_re_dev_list, &to_be_deleted);
mutex_unlock(&bnxt_re_dev_lock);
/*
* Cleanup the devices in reverse order so that the VF device
* cleanup is done before PF cleanup
*/
list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) {
dev_info(rdev_to_dev(rdev), "Unregistering Device");
/*
* Flush out any scheduled tasks before destroying the
* resources
*/
flush_workqueue(bnxt_re_wq);
bnxt_re_dev_stop(rdev);
/* Acquire the rtnl_lock as the L2 resources are freed here */
rtnl_lock();
bnxt_re_ib_unreg(rdev);
rtnl_unlock();
bnxt_re_remove_one(rdev);
bnxt_re_dev_unreg(rdev);
}
unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
if (bnxt_re_wq)
destroy_workqueue(bnxt_re_wq);
list_for_each_entry(rdev, &bnxt_re_dev_list, list) {
/* VF device removal should be called before the removal
* of PF device. Queue VFs unregister first, so that VFs
* shall be removed before the PF during the call of
* ib_unregister_driver.
*/
if (rdev->is_virtfn)
ib_unregister_device(&rdev->ibdev);
}
ib_unregister_driver(RDMA_DRIVER_BNXT_RE);
}
module_init(bnxt_re_mod_init);

View File

@ -43,6 +43,7 @@
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/prefetch.h>
#include <linux/if_ether.h>
@ -53,9 +54,7 @@
#include "qplib_sp.h"
#include "qplib_fp.h"
static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
{
@ -233,6 +232,70 @@ fail:
return rc;
}
static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq)
{
struct bnxt_qplib_hwq *hwq = &nq->hwq;
struct nq_base *nqe, **nq_ptr;
int budget = nq->budget;
u32 sw_cons, raw_cons;
uintptr_t q_handle;
u16 type;
spin_lock_bh(&hwq->lock);
/* Service the NQ until empty */
raw_cons = hwq->cons;
while (budget--) {
sw_cons = HWQ_CMP(raw_cons, hwq);
nq_ptr = (struct nq_base **)hwq->pbl_ptr;
nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
break;
/*
* The valid test of the entry must be done first before
* reading any further.
*/
dma_rmb();
type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
switch (type) {
case NQ_BASE_TYPE_CQ_NOTIFICATION:
{
struct nq_cn *nqcne = (struct nq_cn *)nqe;
q_handle = le32_to_cpu(nqcne->cq_handle_low);
q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
<< 32;
if ((unsigned long)cq == q_handle) {
nqcne->cq_handle_low = 0;
nqcne->cq_handle_high = 0;
cq->cnq_events++;
}
break;
}
default:
break;
}
raw_cons++;
}
spin_unlock_bh(&hwq->lock);
}
/* Wait for receiving all NQEs for this CQ and clean the NQEs associated with
* this CQ.
*/
static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
{
u32 retry_cnt = 100;
while (retry_cnt--) {
if (cnq_events == cq->cnq_events)
return;
usleep_range(50, 100);
clean_nq(cq->nq, cq);
}
}
static void bnxt_qplib_service_nq(unsigned long data)
{
struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
@ -241,12 +304,12 @@ static void bnxt_qplib_service_nq(unsigned long data)
struct bnxt_qplib_cq *cq;
int num_cqne_processed = 0;
int num_srqne_processed = 0;
u32 sw_cons, raw_cons;
u16 type;
int budget = nq->budget;
u32 sw_cons, raw_cons;
uintptr_t q_handle;
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx);
u16 type;
spin_lock_bh(&hwq->lock);
/* Service the NQ until empty */
raw_cons = hwq->cons;
while (budget--) {
@ -272,7 +335,10 @@ static void bnxt_qplib_service_nq(unsigned long data)
q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
<< 32;
cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
bnxt_qplib_arm_cq_enable(cq);
if (!cq)
break;
bnxt_qplib_armen_db(&cq->dbinfo,
DBC_DBC_TYPE_CQ_ARMENA);
spin_lock_bh(&cq->compl_lock);
atomic_set(&cq->arm_state, 0);
if (!nq->cqn_handler(nq, (cq)))
@ -280,19 +346,22 @@ static void bnxt_qplib_service_nq(unsigned long data)
else
dev_warn(&nq->pdev->dev,
"cqn - type 0x%x not handled\n", type);
cq->cnq_events++;
spin_unlock_bh(&cq->compl_lock);
break;
}
case NQ_BASE_TYPE_SRQ_EVENT:
{
struct bnxt_qplib_srq *srq;
struct nq_srq_event *nqsrqe =
(struct nq_srq_event *)nqe;
q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
<< 32;
bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
DBC_DBC_TYPE_SRQ_ARMENA);
srq = (struct bnxt_qplib_srq *)q_handle;
bnxt_qplib_armen_db(&srq->dbinfo,
DBC_DBC_TYPE_SRQ_ARMENA);
if (!nq->srqn_handler(nq,
(struct bnxt_qplib_srq *)q_handle,
nqsrqe->event))
@ -314,10 +383,9 @@ static void bnxt_qplib_service_nq(unsigned long data)
}
if (hwq->cons != raw_cons) {
hwq->cons = raw_cons;
bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, hwq->cons,
hwq->max_elements, nq->ring_id,
gen_p5);
bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
}
spin_unlock_bh(&hwq->lock);
}
static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
@ -333,25 +401,23 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
/* Fan out to CPU affinitized kthreads? */
tasklet_schedule(&nq->worker);
tasklet_schedule(&nq->nq_tasklet);
return IRQ_HANDLED;
}
void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
{
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx);
tasklet_disable(&nq->worker);
tasklet_disable(&nq->nq_tasklet);
/* Mask h/w interrupt */
bnxt_qplib_ring_nq_db(nq->bar_reg_iomem, nq->hwq.cons,
nq->hwq.max_elements, nq->ring_id, gen_p5);
bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false);
/* Sync with last running IRQ handler */
synchronize_irq(nq->vector);
synchronize_irq(nq->msix_vec);
if (kill)
tasklet_kill(&nq->worker);
tasklet_kill(&nq->nq_tasklet);
if (nq->requested) {
irq_set_affinity_hint(nq->vector, NULL);
free_irq(nq->vector, nq);
irq_set_affinity_hint(nq->msix_vec, NULL);
free_irq(nq->msix_vec, nq);
nq->requested = false;
}
}
@ -364,89 +430,108 @@ void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
}
/* Make sure the HW is stopped! */
if (nq->requested)
bnxt_qplib_nq_stop_irq(nq, true);
bnxt_qplib_nq_stop_irq(nq, true);
if (nq->bar_reg_iomem)
iounmap(nq->bar_reg_iomem);
nq->bar_reg_iomem = NULL;
if (nq->nq_db.reg.bar_reg) {
iounmap(nq->nq_db.reg.bar_reg);
nq->nq_db.reg.bar_reg = NULL;
}
nq->cqn_handler = NULL;
nq->srqn_handler = NULL;
nq->vector = 0;
nq->msix_vec = 0;
}
int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
int msix_vector, bool need_init)
{
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx);
int rc;
if (nq->requested)
return -EFAULT;
nq->vector = msix_vector;
nq->msix_vec = msix_vector;
if (need_init)
tasklet_init(&nq->worker, bnxt_qplib_service_nq,
tasklet_init(&nq->nq_tasklet, bnxt_qplib_service_nq,
(unsigned long)nq);
else
tasklet_enable(&nq->worker);
tasklet_enable(&nq->nq_tasklet);
snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
if (rc)
return rc;
cpumask_clear(&nq->mask);
cpumask_set_cpu(nq_indx, &nq->mask);
rc = irq_set_affinity_hint(nq->vector, &nq->mask);
rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask);
if (rc) {
dev_warn(&nq->pdev->dev,
"set affinity failed; vector: %d nq_idx: %d\n",
nq->vector, nq_indx);
nq->msix_vec, nq_indx);
}
nq->requested = true;
bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, nq->hwq.cons,
nq->hwq.max_elements, nq->ring_id, gen_p5);
bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
return rc;
}
static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt)
{
resource_size_t reg_base;
struct bnxt_qplib_nq_db *nq_db;
struct pci_dev *pdev;
int rc = 0;
pdev = nq->pdev;
nq_db = &nq->nq_db;
nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION;
nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id);
if (!nq_db->reg.bar_base) {
dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!",
nq_db->reg.bar_id);
rc = -ENOMEM;
goto fail;
}
reg_base = nq_db->reg.bar_base + reg_offt;
/* Unconditionally map 8 bytes to support 57500 series */
nq_db->reg.len = 8;
nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len);
if (!nq_db->reg.bar_reg) {
dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed",
nq_db->reg.bar_id);
rc = -ENOMEM;
goto fail;
}
nq_db->dbinfo.db = nq_db->reg.bar_reg;
nq_db->dbinfo.hwq = &nq->hwq;
nq_db->dbinfo.xid = nq->ring_id;
fail:
return rc;
}
int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
int nq_idx, int msix_vector, int bar_reg_offset,
int (*cqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *),
int (*srqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_srq *,
u8 event))
cqn_handler_t cqn_handler,
srqn_handler_t srqn_handler)
{
resource_size_t nq_base;
int rc = -1;
if (cqn_handler)
nq->cqn_handler = cqn_handler;
if (srqn_handler)
nq->srqn_handler = srqn_handler;
nq->pdev = pdev;
nq->cqn_handler = cqn_handler;
nq->srqn_handler = srqn_handler;
/* Have a task to schedule CQ notifiers in post send case */
nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
if (!nq->cqn_wq)
return -ENOMEM;
nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
nq->bar_reg_off = bar_reg_offset;
nq_base = pci_resource_start(pdev, nq->bar_reg);
if (!nq_base) {
rc = -ENOMEM;
rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset);
if (rc)
goto fail;
}
/* Unconditionally map 8 bytes to support 57500 series */
nq->bar_reg_iomem = ioremap(nq_base + nq->bar_reg_off, 8);
if (!nq->bar_reg_iomem) {
rc = -ENOMEM;
goto fail;
}
rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
if (rc) {
@ -464,49 +549,38 @@ fail:
void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
{
if (nq->hwq.max_elements) {
bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
bnxt_qplib_free_hwq(nq->res, &nq->hwq);
nq->hwq.max_elements = 0;
}
}
int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq)
{
u8 hwq_type;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
nq->pdev = pdev;
nq->pdev = res->pdev;
nq->res = res;
if (!nq->hwq.max_elements ||
nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
hwq_type = bnxt_qplib_get_hwq_type(nq->res);
if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL,
&nq->hwq.max_elements,
BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
PAGE_SIZE, hwq_type))
return -ENOMEM;
sginfo.pgsize = PAGE_SIZE;
sginfo.pgshft = PAGE_SHIFT;
hwq_attr.res = res;
hwq_attr.sginfo = &sginfo;
hwq_attr.depth = nq->hwq.max_elements;
hwq_attr.stride = sizeof(struct nq_base);
hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res);
if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
dev_err(&nq->pdev->dev, "FP NQ allocation failed");
return -ENOMEM;
}
nq->budget = 8;
return 0;
}
/* SRQ */
static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
{
struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
void __iomem *db;
u32 sw_prod;
u64 val = 0;
/* Ring DB */
sw_prod = (arm_type == DBC_DBC_TYPE_SRQ_ARM) ?
srq->threshold : HWQ_CMP(srq_hwq->prod, srq_hwq);
db = (arm_type == DBC_DBC_TYPE_SRQ_ARMENA) ? srq->dbr_base :
srq->dpi->dbr;
val = ((srq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | arm_type;
val <<= 32;
val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK;
writeq(val, db);
}
void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
@ -526,24 +600,26 @@ void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
kfree(srq->swq);
if (rc)
return;
bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
bnxt_qplib_free_hwq(res, &srq->hwq);
}
int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_create_srq req;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct creq_create_srq_resp resp;
struct cmdq_create_srq req;
struct bnxt_qplib_pbl *pbl;
u16 cmd_flags = 0;
int rc, idx;
srq->hwq.max_elements = srq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, &srq->sg_info,
&srq->hwq.max_elements,
BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.sginfo = &srq->sg_info;
hwq_attr.depth = srq->max_wqe;
hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
if (rc)
goto exit;
@ -595,14 +671,17 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
srq->swq[srq->last_idx].next_idx = -1;
srq->id = le32_to_cpu(resp.xid);
srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
srq->dbinfo.hwq = &srq->hwq;
srq->dbinfo.xid = srq->id;
srq->dbinfo.db = srq->dpi->dbr;
srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
if (srq->threshold)
bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARMENA);
bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
srq->arm_req = false;
return 0;
fail:
bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
bnxt_qplib_free_hwq(res, &srq->hwq);
kfree(srq->swq);
exit:
return rc;
@ -621,7 +700,7 @@ int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
srq_hwq->max_elements - sw_cons + sw_prod;
if (count > srq->threshold) {
srq->arm_req = false;
bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM);
bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
} else {
/* Deferred arming */
srq->arm_req = true;
@ -709,10 +788,10 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
srq_hwq->max_elements - sw_cons + sw_prod;
spin_unlock(&srq_hwq->lock);
/* Ring DB */
bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ);
bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
if (srq->arm_req == true && count > srq->threshold) {
srq->arm_req = false;
bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM);
bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
}
done:
return rc;
@ -721,15 +800,16 @@ done:
/* QP */
int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_create_qp1 req;
struct creq_create_qp1_resp resp;
struct bnxt_qplib_pbl *pbl;
struct bnxt_qplib_q *sq = &qp->sq;
struct bnxt_qplib_q *rq = &qp->rq;
int rc;
struct creq_create_qp1_resp resp;
struct cmdq_create_qp1 req;
struct bnxt_qplib_pbl *pbl;
u16 cmd_flags = 0;
u32 qp_flags = 0;
int rc;
RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
@ -739,11 +819,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req.qp_handle = cpu_to_le64(qp->qp_handle);
/* SQ */
sq->hwq.max_elements = sq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL,
&sq->hwq.max_elements,
BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sg_info;
hwq_attr.depth = sq->max_wqe;
hwq_attr.stride = BNXT_QPLIB_MAX_SQE_ENTRY_SIZE;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
goto exit;
@ -778,11 +859,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
/* RQ */
if (rq->max_wqe) {
rq->hwq.max_elements = qp->rq.max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL,
&rq->hwq.max_elements,
BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sg_info;
hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
hwq_attr.depth = qp->rq.max_wqe;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
if (rc)
goto fail_sq;
@ -840,6 +922,15 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
qp->id = le32_to_cpu(resp.xid);
qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
qp->cctx = res->cctx;
sq->dbinfo.hwq = &sq->hwq;
sq->dbinfo.xid = qp->id;
sq->dbinfo.db = qp->dpi->dbr;
if (rq->max_wqe) {
rq->dbinfo.hwq = &rq->hwq;
rq->dbinfo.xid = qp->id;
rq->dbinfo.db = qp->dpi->dbr;
}
rcfw->qp_tbl[qp->id].qp_id = qp->id;
rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
@ -848,10 +939,10 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
fail:
bnxt_qplib_free_qp_hdr_buf(res, qp);
fail_rq:
bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
bnxt_qplib_free_hwq(res, &rq->hwq);
kfree(rq->swq);
fail_sq:
bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
bnxt_qplib_free_hwq(res, &sq->hwq);
kfree(sq->swq);
exit:
return rc;
@ -860,7 +951,9 @@ exit:
int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_hwq_attr hwq_attr = {};
unsigned long int psn_search, poff = 0;
struct bnxt_qplib_sg_info sginfo = {};
struct sq_psn_search **psn_search_ptr;
struct bnxt_qplib_q *sq = &qp->sq;
struct bnxt_qplib_q *rq = &qp->rq;
@ -887,12 +980,15 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
sizeof(struct sq_psn_search_ext) :
sizeof(struct sq_psn_search);
}
sq->hwq.max_elements = sq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, &sq->sg_info,
&sq->hwq.max_elements,
BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
psn_sz,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sg_info;
hwq_attr.stride = BNXT_QPLIB_MAX_SQE_ENTRY_SIZE;
hwq_attr.depth = sq->max_wqe;
hwq_attr.aux_stride = psn_sz;
hwq_attr.aux_depth = hwq_attr.depth;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
goto exit;
@ -956,12 +1052,14 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
/* RQ */
if (rq->max_wqe) {
rq->hwq.max_elements = rq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq,
&rq->sg_info,
&rq->hwq.max_elements,
BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sg_info;
hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
hwq_attr.depth = rq->max_wqe;
hwq_attr.aux_stride = 0;
hwq_attr.aux_depth = 0;
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
if (rc)
goto fail_sq;
@ -1029,10 +1127,17 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req_size = xrrq->max_elements *
BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
req_size &= ~(PAGE_SIZE - 1);
rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL,
&xrrq->max_elements,
BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
0, req_size, HWQ_TYPE_CTX);
sginfo.pgsize = req_size;
sginfo.pgshft = PAGE_SHIFT;
hwq_attr.res = res;
hwq_attr.sginfo = &sginfo;
hwq_attr.depth = xrrq->max_elements;
hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
hwq_attr.aux_stride = 0;
hwq_attr.aux_depth = 0;
hwq_attr.type = HWQ_TYPE_CTX;
rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
if (rc)
goto fail_buf_free;
pbl = &xrrq->pbl[PBL_LVL_0];
@ -1044,11 +1149,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req_size = xrrq->max_elements *
BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
req_size &= ~(PAGE_SIZE - 1);
rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL,
&xrrq->max_elements,
BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
0, req_size, HWQ_TYPE_CTX);
sginfo.pgsize = req_size;
hwq_attr.depth = xrrq->max_elements;
hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
if (rc)
goto fail_orrq;
@ -1064,9 +1168,17 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
qp->id = le32_to_cpu(resp.xid);
qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
qp->cctx = res->cctx;
INIT_LIST_HEAD(&qp->sq_flush);
INIT_LIST_HEAD(&qp->rq_flush);
qp->cctx = res->cctx;
sq->dbinfo.hwq = &sq->hwq;
sq->dbinfo.xid = qp->id;
sq->dbinfo.db = qp->dpi->dbr;
if (rq->max_wqe) {
rq->dbinfo.hwq = &rq->hwq;
rq->dbinfo.xid = qp->id;
rq->dbinfo.db = qp->dpi->dbr;
}
rcfw->qp_tbl[qp->id].qp_id = qp->id;
rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
@ -1074,17 +1186,17 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
fail:
if (qp->irrq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
bnxt_qplib_free_hwq(res, &qp->irrq);
fail_orrq:
if (qp->orrq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
bnxt_qplib_free_hwq(res, &qp->orrq);
fail_buf_free:
bnxt_qplib_free_qp_hdr_buf(res, qp);
fail_rq:
bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
bnxt_qplib_free_hwq(res, &rq->hwq);
kfree(rq->swq);
fail_sq:
bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
bnxt_qplib_free_hwq(res, &sq->hwq);
kfree(sq->swq);
exit:
return rc;
@ -1440,16 +1552,16 @@ void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
struct bnxt_qplib_qp *qp)
{
bnxt_qplib_free_qp_hdr_buf(res, qp);
bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
bnxt_qplib_free_hwq(res, &qp->sq.hwq);
kfree(qp->sq.swq);
bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
bnxt_qplib_free_hwq(res, &qp->rq.hwq);
kfree(qp->rq.swq);
if (qp->irrq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
bnxt_qplib_free_hwq(res, &qp->irrq);
if (qp->orrq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
bnxt_qplib_free_hwq(res, &qp->orrq);
}
@ -1506,16 +1618,8 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_q *sq = &qp->sq;
u32 sw_prod;
u64 val = 0;
val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) |
DBC_DBC_TYPE_SQ);
val <<= 32;
sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK;
/* Flush all the WQE writes to HW */
writeq(val, qp->dpi->dbr);
bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
}
int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
@ -1807,16 +1911,8 @@ done:
void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_q *rq = &qp->rq;
u32 sw_prod;
u64 val = 0;
val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) |
DBC_DBC_TYPE_RQ);
val <<= 32;
sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK;
/* Flush the writes to HW Rx WQE before the ringing Rx DB */
writeq(val, qp->dpi->dbr);
bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
}
int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
@ -1896,48 +1992,22 @@ done:
}
/* CQ */
/* Spinlock must be held */
static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
{
u64 val = 0;
val = ((cq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) |
DBC_DBC_TYPE_CQ_ARMENA;
val <<= 32;
/* Flush memory writes before enabling the CQ */
writeq(val, cq->dbr_base);
}
static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
{
struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
u32 sw_cons;
u64 val = 0;
/* Ring DB */
val = ((cq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | arm_type;
val <<= 32;
sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
val |= (sw_cons << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK;
/* flush memory writes before arming the CQ */
writeq(val, cq->dpi->dbr);
}
int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_create_cq req;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct creq_create_cq_resp resp;
struct cmdq_create_cq req;
struct bnxt_qplib_pbl *pbl;
u16 cmd_flags = 0;
int rc;
cq->hwq.max_elements = cq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, &cq->sg_info,
&cq->hwq.max_elements,
BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
hwq_attr.res = res;
hwq_attr.depth = cq->max_wqe;
hwq_attr.stride = sizeof(struct cq_base);
hwq_attr.type = HWQ_TYPE_QUEUE;
hwq_attr.sginfo = &cq->sg_info;
rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
if (rc)
goto exit;
@ -1976,7 +2046,6 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
goto fail;
cq->id = le32_to_cpu(resp.xid);
cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
init_waitqueue_head(&cq->waitq);
INIT_LIST_HEAD(&cq->sqf_head);
@ -1984,11 +2053,17 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
spin_lock_init(&cq->compl_lock);
spin_lock_init(&cq->flush_lock);
bnxt_qplib_arm_cq_enable(cq);
cq->dbinfo.hwq = &cq->hwq;
cq->dbinfo.xid = cq->id;
cq->dbinfo.db = cq->dpi->dbr;
cq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
return 0;
fail:
bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
bnxt_qplib_free_hwq(res, &cq->hwq);
exit:
return rc;
}
@ -1998,6 +2073,7 @@ int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_destroy_cq req;
struct creq_destroy_cq_resp resp;
u16 total_cnq_events;
u16 cmd_flags = 0;
int rc;
@ -2008,7 +2084,9 @@ int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
(void *)&resp, NULL, 0);
if (rc)
return rc;
bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
total_cnq_events = le16_to_cpu(resp.total_cnq_events);
__wait_for_all_nqes(cq, total_cnq_events);
bnxt_qplib_free_hwq(res, &cq->hwq);
return 0;
}
@ -2141,8 +2219,7 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
sq->send_phantom = true;
/* TODO: Only ARM if the previous SQE is ARMALL */
bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ_ARMALL);
bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
rc = -EAGAIN;
goto out;
}
@ -2426,7 +2503,7 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
}
cqe = *pcqe;
cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
cqe->length = (u32)le16_to_cpu(hwcqe->length);
cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK;
cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata);
cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
cqe->flags = le16_to_cpu(hwcqe->flags);
@ -2812,7 +2889,7 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
}
if (cq->hwq.cons != raw_cons) {
cq->hwq.cons = raw_cons;
bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ);
bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
}
exit:
return num_cqes - budget;
@ -2821,7 +2898,7 @@ exit:
void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
{
if (arm_type)
bnxt_qplib_arm_cq(cq, arm_type);
bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
/* Using cq->arm_state variable to track whether to issue cq handler */
atomic_set(&cq->arm_state, 1);
}

View File

@ -42,7 +42,7 @@
struct bnxt_qplib_srq {
struct bnxt_qplib_pd *pd;
struct bnxt_qplib_dpi *dpi;
void __iomem *dbr_base;
struct bnxt_qplib_db_info dbinfo;
u64 srq_handle;
u32 id;
u32 max_wqe;
@ -236,6 +236,7 @@ struct bnxt_qplib_swqe {
struct bnxt_qplib_q {
struct bnxt_qplib_hwq hwq;
struct bnxt_qplib_swq *swq;
struct bnxt_qplib_db_info dbinfo;
struct bnxt_qplib_sg_info sg_info;
u32 max_wqe;
u16 q_full_delta;
@ -370,7 +371,7 @@ struct bnxt_qplib_cqe {
#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01
struct bnxt_qplib_cq {
struct bnxt_qplib_dpi *dpi;
void __iomem *dbr_base;
struct bnxt_qplib_db_info dbinfo;
u32 max_wqe;
u32 id;
u16 count;
@ -401,6 +402,7 @@ struct bnxt_qplib_cq {
* of the same QP while manipulating the flush list.
*/
spinlock_t flush_lock; /* QP flush management */
u16 cnq_events;
};
#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
@ -433,66 +435,32 @@ struct bnxt_qplib_cq {
NQ_DB_IDX_VALID | \
NQ_DB_IRQ_DIS)
static inline void bnxt_qplib_ring_nq_db64(void __iomem *db, u32 index,
u32 xid, bool arm)
{
u64 val;
struct bnxt_qplib_nq_db {
struct bnxt_qplib_reg_desc reg;
struct bnxt_qplib_db_info dbinfo;
};
val = xid & DBC_DBC_XID_MASK;
val |= DBC_DBC_PATH_ROCE;
val |= arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;
val <<= 32;
val |= index & DBC_DBC_INDEX_MASK;
writeq(val, db);
}
static inline void bnxt_qplib_ring_nq_db_rearm(void __iomem *db, u32 raw_cons,
u32 max_elements, u32 xid,
bool gen_p5)
{
u32 index = raw_cons & (max_elements - 1);
if (gen_p5)
bnxt_qplib_ring_nq_db64(db, index, xid, true);
else
writel(NQ_DB_CP_FLAGS_REARM | (index & DBC_DBC32_XID_MASK), db);
}
static inline void bnxt_qplib_ring_nq_db(void __iomem *db, u32 raw_cons,
u32 max_elements, u32 xid,
bool gen_p5)
{
u32 index = raw_cons & (max_elements - 1);
if (gen_p5)
bnxt_qplib_ring_nq_db64(db, index, xid, false);
else
writel(NQ_DB_CP_FLAGS | (index & DBC_DBC32_XID_MASK), db);
}
typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *cq);
typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_srq *srq, u8 event);
struct bnxt_qplib_nq {
struct pci_dev *pdev;
struct bnxt_qplib_res *res;
struct pci_dev *pdev;
struct bnxt_qplib_res *res;
char name[32];
struct bnxt_qplib_hwq hwq;
struct bnxt_qplib_nq_db nq_db;
u16 ring_id;
int msix_vec;
cpumask_t mask;
struct tasklet_struct nq_tasklet;
bool requested;
int budget;
int vector;
cpumask_t mask;
int budget;
bool requested;
struct tasklet_struct worker;
struct bnxt_qplib_hwq hwq;
u16 bar_reg;
u32 bar_reg_off;
u16 ring_id;
void __iomem *bar_reg_iomem;
int (*cqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *cq);
int (*srqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_srq *srq,
u8 event);
struct workqueue_struct *cqn_wq;
char name[32];
cqn_handler_t cqn_handler;
srqn_handler_t srqn_handler;
struct workqueue_struct *cqn_wq;
};
struct bnxt_qplib_nq_work {
@ -507,11 +475,8 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
int msix_vector, bool need_init);
int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
int nq_idx, int msix_vector, int bar_reg_offset,
int (*cqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *cq),
int (*srqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_srq *srq,
u8 event));
cqn_handler_t cqn_handler,
srqn_handler_t srq_handler);
int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq);
int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
@ -550,7 +515,7 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);
void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq);
void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
unsigned long *flags);

View File

@ -55,12 +55,14 @@ static void bnxt_qplib_service_creq(unsigned long data);
/* Hardware communication channel */
static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
{
struct bnxt_qplib_cmdq_ctx *cmdq;
u16 cbit;
int rc;
cmdq = &rcfw->cmdq;
cbit = cookie % rcfw->cmdq_depth;
rc = wait_event_timeout(rcfw->waitq,
!test_bit(cbit, rcfw->cmdq_bitmap),
rc = wait_event_timeout(cmdq->waitq,
!test_bit(cbit, cmdq->cmdq_bitmap),
msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS));
return rc ? 0 : -ETIMEDOUT;
};
@ -68,15 +70,17 @@ static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
{
u32 count = RCFW_BLOCKED_CMD_WAIT_COUNT;
struct bnxt_qplib_cmdq_ctx *cmdq;
u16 cbit;
cmdq = &rcfw->cmdq;
cbit = cookie % rcfw->cmdq_depth;
if (!test_bit(cbit, rcfw->cmdq_bitmap))
if (!test_bit(cbit, cmdq->cmdq_bitmap))
goto done;
do {
mdelay(1); /* 1m sec */
bnxt_qplib_service_creq((unsigned long)rcfw);
} while (test_bit(cbit, rcfw->cmdq_bitmap) && --count);
} while (test_bit(cbit, cmdq->cmdq_bitmap) && --count);
done:
return count ? 0 : -ETIMEDOUT;
};
@ -84,56 +88,60 @@ done:
static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
struct creq_base *resp, void *sb, u8 is_block)
{
struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr;
struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
struct bnxt_qplib_cmdqe *cmdqe, **hwq_ptr;
struct bnxt_qplib_hwq *hwq = &cmdq->hwq;
struct bnxt_qplib_crsqe *crsqe;
u32 cmdq_depth = rcfw->cmdq_depth;
struct bnxt_qplib_crsq *crsqe;
u32 sw_prod, cmdq_prod;
struct pci_dev *pdev;
unsigned long flags;
u32 size, opcode;
u16 cookie, cbit;
u8 *preq;
pdev = rcfw->pdev;
opcode = req->opcode;
if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
(opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {
dev_err(&rcfw->pdev->dev,
dev_err(&pdev->dev,
"RCFW not initialized, reject opcode 0x%x\n", opcode);
return -EINVAL;
}
if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
dev_err(&rcfw->pdev->dev, "RCFW already initialized!\n");
dev_err(&pdev->dev, "RCFW already initialized!\n");
return -EINVAL;
}
if (test_bit(FIRMWARE_TIMED_OUT, &rcfw->flags))
if (test_bit(FIRMWARE_TIMED_OUT, &cmdq->flags))
return -ETIMEDOUT;
/* Cmdq are in 16-byte units, each request can consume 1 or more
* cmdqe
*/
spin_lock_irqsave(&cmdq->lock, flags);
if (req->cmd_size >= HWQ_FREE_SLOTS(cmdq)) {
dev_err(&rcfw->pdev->dev, "RCFW: CMDQ is full!\n");
spin_unlock_irqrestore(&cmdq->lock, flags);
spin_lock_irqsave(&hwq->lock, flags);
if (req->cmd_size >= HWQ_FREE_SLOTS(hwq)) {
dev_err(&pdev->dev, "RCFW: CMDQ is full!\n");
spin_unlock_irqrestore(&hwq->lock, flags);
return -EAGAIN;
}
cookie = rcfw->seq_num & RCFW_MAX_COOKIE_VALUE;
cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
cbit = cookie % rcfw->cmdq_depth;
if (is_block)
cookie |= RCFW_CMD_IS_BLOCKING;
set_bit(cbit, rcfw->cmdq_bitmap);
set_bit(cbit, cmdq->cmdq_bitmap);
req->cookie = cpu_to_le16(cookie);
crsqe = &rcfw->crsqe_tbl[cbit];
if (crsqe->resp) {
spin_unlock_irqrestore(&cmdq->lock, flags);
spin_unlock_irqrestore(&hwq->lock, flags);
return -EBUSY;
}
@ -155,15 +163,15 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
BNXT_QPLIB_CMDQE_UNITS;
}
cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr;
hwq_ptr = (struct bnxt_qplib_cmdqe **)hwq->pbl_ptr;
preq = (u8 *)req;
do {
/* Locate the next cmdq slot */
sw_prod = HWQ_CMP(cmdq->prod, cmdq);
cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod, cmdq_depth)]
sw_prod = HWQ_CMP(hwq->prod, hwq);
cmdqe = &hwq_ptr[get_cmdq_pg(sw_prod, cmdq_depth)]
[get_cmdq_idx(sw_prod, cmdq_depth)];
if (!cmdqe) {
dev_err(&rcfw->pdev->dev,
dev_err(&pdev->dev,
"RCFW request failed with no cmdqe!\n");
goto done;
}
@ -172,31 +180,27 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe)));
preq += min_t(u32, size, sizeof(*cmdqe));
size -= min_t(u32, size, sizeof(*cmdqe));
cmdq->prod++;
rcfw->seq_num++;
hwq->prod++;
} while (size > 0);
cmdq->seq_num++;
rcfw->seq_num++;
cmdq_prod = cmdq->prod;
if (test_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags)) {
cmdq_prod = hwq->prod;
if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) {
/* The very first doorbell write
* is required to set this flag
* which prompts the FW to reset
* its internal pointers
*/
cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);
clear_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags);
clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
}
/* ring CMDQ DB */
wmb();
writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem +
rcfw->cmdq_bar_reg_prod_off);
writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem +
rcfw->cmdq_bar_reg_trig_off);
writel(cmdq_prod, cmdq->cmdq_mbox.prod);
writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
done:
spin_unlock_irqrestore(&cmdq->lock, flags);
spin_unlock_irqrestore(&hwq->lock, flags);
/* Return the CREQ response pointer */
return 0;
}
@ -236,7 +240,7 @@ int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
/* timed out */
dev_err(&rcfw->pdev->dev, "cmdq[%#x]=%#x timedout (%d)msec\n",
cookie, opcode, RCFW_CMD_WAIT_TIME_MS);
set_bit(FIRMWARE_TIMED_OUT, &rcfw->flags);
set_bit(FIRMWARE_TIMED_OUT, &rcfw->cmdq.flags);
return rc;
}
@ -253,6 +257,8 @@ int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,
struct creq_func_event *func_event)
{
int rc;
switch (func_event->event) {
case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
break;
@ -286,37 +292,41 @@ static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,
default:
return -EINVAL;
}
return 0;
rc = rcfw->creq.aeq_handler(rcfw, (void *)func_event, NULL);
return rc;
}
static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
struct creq_qp_event *qp_event)
{
struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
struct creq_qp_error_notification *err_event;
struct bnxt_qplib_crsq *crsqe;
unsigned long flags;
struct bnxt_qplib_hwq *hwq = &rcfw->cmdq.hwq;
struct bnxt_qplib_crsqe *crsqe;
struct bnxt_qplib_qp *qp;
u16 cbit, blocked = 0;
u16 cookie;
struct pci_dev *pdev;
unsigned long flags;
__le16 mcookie;
u16 cookie;
int rc = 0;
u32 qp_id;
pdev = rcfw->pdev;
switch (qp_event->event) {
case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
err_event = (struct creq_qp_error_notification *)qp_event;
qp_id = le32_to_cpu(err_event->xid);
qp = rcfw->qp_tbl[qp_id].qp_handle;
dev_dbg(&rcfw->pdev->dev,
"Received QP error notification\n");
dev_dbg(&rcfw->pdev->dev,
dev_dbg(&pdev->dev, "Received QP error notification\n");
dev_dbg(&pdev->dev,
"qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",
qp_id, err_event->req_err_state_reason,
err_event->res_err_state_reason);
if (!qp)
break;
bnxt_qplib_mark_qp_error(qp);
rcfw->aeq_handler(rcfw, qp_event, qp);
rc = rcfw->creq.aeq_handler(rcfw, qp_event, qp);
break;
default:
/*
@ -328,7 +338,7 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
*
*/
spin_lock_irqsave_nested(&cmdq->lock, flags,
spin_lock_irqsave_nested(&hwq->lock, flags,
SINGLE_DEPTH_NESTING);
cookie = le16_to_cpu(qp_event->cookie);
mcookie = qp_event->cookie;
@ -342,44 +352,44 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
crsqe->resp = NULL;
} else {
if (crsqe->resp && crsqe->resp->cookie)
dev_err(&rcfw->pdev->dev,
dev_err(&pdev->dev,
"CMD %s cookie sent=%#x, recd=%#x\n",
crsqe->resp ? "mismatch" : "collision",
crsqe->resp ? crsqe->resp->cookie : 0,
mcookie);
}
if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap))
dev_warn(&rcfw->pdev->dev,
if (!test_and_clear_bit(cbit, rcfw->cmdq.cmdq_bitmap))
dev_warn(&pdev->dev,
"CMD bit %d was not requested\n", cbit);
cmdq->cons += crsqe->req_size;
hwq->cons += crsqe->req_size;
crsqe->req_size = 0;
if (!blocked)
wake_up(&rcfw->waitq);
spin_unlock_irqrestore(&cmdq->lock, flags);
wake_up(&rcfw->cmdq.waitq);
spin_unlock_irqrestore(&hwq->lock, flags);
}
return 0;
return rc;
}
/* SP - CREQ Completion handlers */
static void bnxt_qplib_service_creq(unsigned long data)
{
struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx);
struct bnxt_qplib_hwq *creq = &rcfw->creq;
struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;
u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
struct creq_base *creqe, **creq_ptr;
struct bnxt_qplib_hwq *hwq = &creq->hwq;
struct creq_base *creqe, **hwq_ptr;
u32 sw_cons, raw_cons;
unsigned long flags;
/* Service the CREQ until budget is over */
spin_lock_irqsave(&creq->lock, flags);
raw_cons = creq->cons;
spin_lock_irqsave(&hwq->lock, flags);
raw_cons = hwq->cons;
while (budget > 0) {
sw_cons = HWQ_CMP(raw_cons, creq);
creq_ptr = (struct creq_base **)creq->pbl_ptr;
creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];
if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements))
sw_cons = HWQ_CMP(raw_cons, hwq);
hwq_ptr = (struct creq_base **)hwq->pbl_ptr;
creqe = &hwq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];
if (!CREQ_CMP_VALID(creqe, raw_cons, hwq->max_elements))
break;
/* The valid test of the entry must be done first before
* reading any further.
@ -391,12 +401,12 @@ static void bnxt_qplib_service_creq(unsigned long data)
case CREQ_BASE_TYPE_QP_EVENT:
bnxt_qplib_process_qp_event
(rcfw, (struct creq_qp_event *)creqe);
rcfw->creq_qp_event_processed++;
creq->stats.creq_qp_event_processed++;
break;
case CREQ_BASE_TYPE_FUNC_EVENT:
if (!bnxt_qplib_process_func_event
(rcfw, (struct creq_func_event *)creqe))
rcfw->creq_func_event_processed++;
creq->stats.creq_func_event_processed++;
else
dev_warn(&rcfw->pdev->dev,
"aeqe:%#x Not handled\n", type);
@ -412,28 +422,30 @@ static void bnxt_qplib_service_creq(unsigned long data)
budget--;
}
if (creq->cons != raw_cons) {
creq->cons = raw_cons;
bnxt_qplib_ring_creq_db_rearm(rcfw->creq_bar_reg_iomem,
raw_cons, creq->max_elements,
rcfw->creq_ring_id, gen_p5);
if (hwq->cons != raw_cons) {
hwq->cons = raw_cons;
bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo,
rcfw->res->cctx, true);
}
spin_unlock_irqrestore(&creq->lock, flags);
spin_unlock_irqrestore(&hwq->lock, flags);
}
static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
{
struct bnxt_qplib_rcfw *rcfw = dev_instance;
struct bnxt_qplib_hwq *creq = &rcfw->creq;
struct bnxt_qplib_creq_ctx *creq;
struct creq_base **creq_ptr;
struct bnxt_qplib_hwq *hwq;
u32 sw_cons;
creq = &rcfw->creq;
hwq = &creq->hwq;
/* Prefetch the CREQ element */
sw_cons = HWQ_CMP(creq->cons, creq);
creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr;
sw_cons = HWQ_CMP(hwq->cons, hwq);
creq_ptr = (struct creq_base **)creq->hwq.pbl_ptr;
prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]);
tasklet_schedule(&rcfw->worker);
tasklet_schedule(&creq->creq_tasklet);
return IRQ_HANDLED;
}
@ -452,7 +464,7 @@ int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
if (rc)
return rc;
clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
return 0;
}
@ -520,9 +532,10 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
level = ctx->tim_tbl.level;
req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) |
__get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]);
level = ctx->tqm_pde_level;
req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |
__get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]);
level = ctx->tqm_ctx.pde.level;
req.tqm_pg_size_tqm_lvl =
(level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |
__get_pbl_pg_idx(&ctx->tqm_ctx.pde.pbl[level]);
req.qpc_page_dir =
cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
@ -535,7 +548,7 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
req.tim_page_dir =
cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
req.tqm_page_dir =
cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]);
cpu_to_le64(ctx->tqm_ctx.pde.pbl[PBL_LVL_0].pg_map_arr[0]);
req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements);
req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements);
@ -555,33 +568,46 @@ skip_ctx_setup:
NULL, 0);
if (rc)
return rc;
set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
return 0;
}
void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
{
kfree(rcfw->cmdq.cmdq_bitmap);
kfree(rcfw->qp_tbl);
kfree(rcfw->crsqe_tbl);
bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq);
bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq);
bnxt_qplib_free_hwq(rcfw->res, &rcfw->cmdq.hwq);
bnxt_qplib_free_hwq(rcfw->res, &rcfw->creq.hwq);
rcfw->pdev = NULL;
}
int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,
struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_ctx *ctx,
int qp_tbl_sz)
{
u8 hwq_type;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
struct bnxt_qplib_cmdq_ctx *cmdq;
struct bnxt_qplib_creq_ctx *creq;
u32 bmap_size = 0;
rcfw->pdev = pdev;
rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;
hwq_type = bnxt_qplib_get_hwq_type(rcfw->res);
if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL,
&rcfw->creq.max_elements,
BNXT_QPLIB_CREQE_UNITS,
0, PAGE_SIZE, hwq_type)) {
rcfw->pdev = res->pdev;
cmdq = &rcfw->cmdq;
creq = &rcfw->creq;
rcfw->res = res;
sginfo.pgsize = PAGE_SIZE;
sginfo.pgshft = PAGE_SHIFT;
hwq_attr.sginfo = &sginfo;
hwq_attr.res = rcfw->res;
hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT;
hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS;
hwq_attr.type = bnxt_qplib_get_hwq_type(res);
if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) {
dev_err(&rcfw->pdev->dev,
"HW channel CREQ allocation failed\n");
goto fail;
@ -591,23 +617,28 @@ int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
else
rcfw->cmdq_depth = BNXT_QPLIB_CMDQE_MAX_CNT_8192;
rcfw->cmdq.max_elements = rcfw->cmdq_depth;
if (bnxt_qplib_alloc_init_hwq
(rcfw->pdev, &rcfw->cmdq, NULL,
&rcfw->cmdq.max_elements,
BNXT_QPLIB_CMDQE_UNITS, 0,
bnxt_qplib_cmdqe_page_size(rcfw->cmdq_depth),
HWQ_TYPE_CTX)) {
sginfo.pgsize = bnxt_qplib_cmdqe_page_size(rcfw->cmdq_depth);
hwq_attr.depth = rcfw->cmdq_depth;
hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS;
hwq_attr.type = HWQ_TYPE_CTX;
if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) {
dev_err(&rcfw->pdev->dev,
"HW channel CMDQ allocation failed\n");
goto fail;
}
rcfw->crsqe_tbl = kcalloc(rcfw->cmdq.max_elements,
rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements,
sizeof(*rcfw->crsqe_tbl), GFP_KERNEL);
if (!rcfw->crsqe_tbl)
goto fail;
bmap_size = BITS_TO_LONGS(rcfw->cmdq_depth) * sizeof(unsigned long);
cmdq->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);
if (!cmdq->cmdq_bitmap)
goto fail;
cmdq->bmap_size = bmap_size;
rcfw->qp_tbl_size = qp_tbl_sz;
rcfw->qp_tbl = kcalloc(qp_tbl_sz, sizeof(struct bnxt_qplib_qp_node),
GFP_KERNEL);
@ -623,137 +654,199 @@ fail:
void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)
{
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx);
struct bnxt_qplib_creq_ctx *creq;
tasklet_disable(&rcfw->worker);
creq = &rcfw->creq;
tasklet_disable(&creq->creq_tasklet);
/* Mask h/w interrupts */
bnxt_qplib_ring_creq_db(rcfw->creq_bar_reg_iomem, rcfw->creq.cons,
rcfw->creq.max_elements, rcfw->creq_ring_id,
gen_p5);
bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, rcfw->res->cctx, false);
/* Sync with last running IRQ-handler */
synchronize_irq(rcfw->vector);
synchronize_irq(creq->msix_vec);
if (kill)
tasklet_kill(&rcfw->worker);
tasklet_kill(&creq->creq_tasklet);
if (rcfw->requested) {
free_irq(rcfw->vector, rcfw);
rcfw->requested = false;
if (creq->requested) {
free_irq(creq->msix_vec, rcfw);
creq->requested = false;
}
}
void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
{
struct bnxt_qplib_creq_ctx *creq;
struct bnxt_qplib_cmdq_ctx *cmdq;
unsigned long indx;
creq = &rcfw->creq;
cmdq = &rcfw->cmdq;
/* Make sure the HW channel is stopped! */
bnxt_qplib_rcfw_stop_irq(rcfw, true);
iounmap(rcfw->cmdq_bar_reg_iomem);
iounmap(rcfw->creq_bar_reg_iomem);
iounmap(cmdq->cmdq_mbox.reg.bar_reg);
iounmap(creq->creq_db.reg.bar_reg);
indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size);
if (indx != rcfw->bmap_size)
indx = find_first_bit(cmdq->cmdq_bitmap, cmdq->bmap_size);
if (indx != cmdq->bmap_size)
dev_err(&rcfw->pdev->dev,
"disabling RCFW with pending cmd-bit %lx\n", indx);
kfree(rcfw->cmdq_bitmap);
rcfw->bmap_size = 0;
rcfw->cmdq_bar_reg_iomem = NULL;
rcfw->creq_bar_reg_iomem = NULL;
rcfw->aeq_handler = NULL;
rcfw->vector = 0;
cmdq->cmdq_mbox.reg.bar_reg = NULL;
creq->creq_db.reg.bar_reg = NULL;
creq->aeq_handler = NULL;
creq->msix_vec = 0;
}
int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
bool need_init)
{
bool gen_p5 = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx);
struct bnxt_qplib_creq_ctx *creq;
int rc;
if (rcfw->requested)
creq = &rcfw->creq;
if (creq->requested)
return -EFAULT;
rcfw->vector = msix_vector;
creq->msix_vec = msix_vector;
if (need_init)
tasklet_init(&rcfw->worker,
tasklet_init(&creq->creq_tasklet,
bnxt_qplib_service_creq, (unsigned long)rcfw);
else
tasklet_enable(&rcfw->worker);
rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0,
tasklet_enable(&creq->creq_tasklet);
rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0,
"bnxt_qplib_creq", rcfw);
if (rc)
return rc;
rcfw->requested = true;
bnxt_qplib_ring_creq_db_rearm(rcfw->creq_bar_reg_iomem,
rcfw->creq.cons, rcfw->creq.max_elements,
rcfw->creq_ring_id, gen_p5);
creq->requested = true;
bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, rcfw->res->cctx, true);
return 0;
}
int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
struct bnxt_qplib_rcfw *rcfw,
static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw, bool is_vf)
{
struct bnxt_qplib_cmdq_mbox *mbox;
resource_size_t bar_reg;
struct pci_dev *pdev;
u16 prod_offt;
int rc = 0;
pdev = rcfw->pdev;
mbox = &rcfw->cmdq.cmdq_mbox;
mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION;
mbox->reg.len = RCFW_COMM_SIZE;
mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id);
if (!mbox->reg.bar_base) {
dev_err(&pdev->dev,
"QPLIB: CMDQ BAR region %d resc start is 0!\n",
mbox->reg.bar_id);
return -ENOMEM;
}
bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET;
mbox->reg.len = RCFW_COMM_SIZE;
mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len);
if (!mbox->reg.bar_reg) {
dev_err(&pdev->dev,
"QPLIB: CMDQ BAR region %d mapping failed\n",
mbox->reg.bar_id);
return -ENOMEM;
}
prod_offt = is_vf ? RCFW_VF_COMM_PROD_OFFSET :
RCFW_PF_COMM_PROD_OFFSET;
mbox->prod = (void __iomem *)(mbox->reg.bar_reg + prod_offt);
mbox->db = (void __iomem *)(mbox->reg.bar_reg + RCFW_COMM_TRIG_OFFSET);
return rc;
}
static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)
{
struct bnxt_qplib_creq_db *creq_db;
resource_size_t bar_reg;
struct pci_dev *pdev;
pdev = rcfw->pdev;
creq_db = &rcfw->creq.creq_db;
creq_db->reg.bar_id = RCFW_COMM_CONS_PCI_BAR_REGION;
creq_db->reg.bar_base = pci_resource_start(pdev, creq_db->reg.bar_id);
if (!creq_db->reg.bar_id)
dev_err(&pdev->dev,
"QPLIB: CREQ BAR region %d resc start is 0!",
creq_db->reg.bar_id);
bar_reg = creq_db->reg.bar_base + reg_offt;
/* Unconditionally map 8 bytes to support 57500 series */
creq_db->reg.len = 8;
creq_db->reg.bar_reg = ioremap(bar_reg, creq_db->reg.len);
if (!creq_db->reg.bar_reg) {
dev_err(&pdev->dev,
"QPLIB: CREQ BAR region %d mapping failed",
creq_db->reg.bar_id);
return -ENOMEM;
}
creq_db->dbinfo.db = creq_db->reg.bar_reg;
creq_db->dbinfo.hwq = &rcfw->creq.hwq;
creq_db->dbinfo.xid = rcfw->creq.ring_id;
return 0;
}
static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)
{
struct bnxt_qplib_cmdq_ctx *cmdq;
struct bnxt_qplib_creq_ctx *creq;
struct bnxt_qplib_cmdq_mbox *mbox;
struct cmdq_init init = {0};
cmdq = &rcfw->cmdq;
creq = &rcfw->creq;
mbox = &cmdq->cmdq_mbox;
init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
init.cmdq_size_cmdq_lvl =
cpu_to_le16(((rcfw->cmdq_depth <<
CMDQ_INIT_CMDQ_SIZE_SFT) &
CMDQ_INIT_CMDQ_SIZE_MASK) |
((cmdq->hwq.level <<
CMDQ_INIT_CMDQ_LVL_SFT) &
CMDQ_INIT_CMDQ_LVL_MASK));
init.creq_ring_id = cpu_to_le16(creq->ring_id);
/* Write to the Bono mailbox register */
__iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4);
}
int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
int msix_vector,
int cp_bar_reg_off, int virt_fn,
int (*aeq_handler)(struct bnxt_qplib_rcfw *,
void *, void *))
aeq_handler_t aeq_handler)
{
resource_size_t res_base;
struct cmdq_init init;
u16 bmap_size;
struct bnxt_qplib_cmdq_ctx *cmdq;
struct bnxt_qplib_creq_ctx *creq;
int rc;
/* General */
rcfw->seq_num = 0;
set_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags);
bmap_size = BITS_TO_LONGS(rcfw->cmdq_depth) * sizeof(unsigned long);
rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);
if (!rcfw->cmdq_bitmap)
return -ENOMEM;
rcfw->bmap_size = bmap_size;
cmdq = &rcfw->cmdq;
creq = &rcfw->creq;
/* CMDQ */
rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION;
res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg);
if (!res_base)
return -ENOMEM;
/* Clear to defaults */
rcfw->cmdq_bar_reg_iomem = ioremap(res_base +
RCFW_COMM_BASE_OFFSET,
RCFW_COMM_SIZE);
if (!rcfw->cmdq_bar_reg_iomem) {
dev_err(&rcfw->pdev->dev, "CMDQ BAR region %d mapping failed\n",
rcfw->cmdq_bar_reg);
return -ENOMEM;
}
cmdq->seq_num = 0;
set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
init_waitqueue_head(&cmdq->waitq);
rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET :
RCFW_PF_COMM_PROD_OFFSET;
creq->stats.creq_qp_event_processed = 0;
creq->stats.creq_func_event_processed = 0;
creq->aeq_handler = aeq_handler;
rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET;
rc = bnxt_qplib_map_cmdq_mbox(rcfw, virt_fn);
if (rc)
return rc;
/* CREQ */
rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION;
res_base = pci_resource_start(pdev, rcfw->creq_bar_reg);
if (!res_base)
dev_err(&rcfw->pdev->dev,
"CREQ BAR region %d resc start is 0!\n",
rcfw->creq_bar_reg);
/* Unconditionally map 8 bytes to support 57500 series */
rcfw->creq_bar_reg_iomem = ioremap(res_base + cp_bar_reg_off,
8);
if (!rcfw->creq_bar_reg_iomem) {
dev_err(&rcfw->pdev->dev, "CREQ BAR region %d mapping failed\n",
rcfw->creq_bar_reg);
iounmap(rcfw->cmdq_bar_reg_iomem);
rcfw->cmdq_bar_reg_iomem = NULL;
return -ENOMEM;
}
rcfw->creq_qp_event_processed = 0;
rcfw->creq_func_event_processed = 0;
if (aeq_handler)
rcfw->aeq_handler = aeq_handler;
init_waitqueue_head(&rcfw->waitq);
rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);
if (rc)
return rc;
rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);
if (rc) {
@ -763,16 +856,8 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
return rc;
}
init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]);
init.cmdq_size_cmdq_lvl = cpu_to_le16(
((rcfw->cmdq_depth << CMDQ_INIT_CMDQ_SIZE_SFT) &
CMDQ_INIT_CMDQ_SIZE_MASK) |
((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
CMDQ_INIT_CMDQ_LVL_MASK));
init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id);
bnxt_qplib_start_rcfw(rcfw);
/* Write to the Bono mailbox register */
__iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4);
return 0;
}

View File

@ -206,8 +206,9 @@ static inline void bnxt_qplib_ring_creq_db(void __iomem *db, u32 raw_cons,
#define CREQ_ENTRY_POLL_BUDGET 0x100
/* HWQ */
typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *);
struct bnxt_qplib_crsq {
struct bnxt_qplib_crsqe {
struct creq_qp_event *resp;
u32 req_size;
};
@ -225,41 +226,53 @@ struct bnxt_qplib_qp_node {
#define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF
#define FIRMWARE_INITIALIZED_FLAG (0)
#define FIRMWARE_FIRST_FLAG (31)
#define FIRMWARE_TIMED_OUT (3)
struct bnxt_qplib_cmdq_mbox {
struct bnxt_qplib_reg_desc reg;
void __iomem *prod;
void __iomem *db;
};
struct bnxt_qplib_cmdq_ctx {
struct bnxt_qplib_hwq hwq;
struct bnxt_qplib_cmdq_mbox cmdq_mbox;
wait_queue_head_t waitq;
unsigned long flags;
unsigned long *cmdq_bitmap;
u32 bmap_size;
u32 seq_num;
};
struct bnxt_qplib_creq_db {
struct bnxt_qplib_reg_desc reg;
struct bnxt_qplib_db_info dbinfo;
};
struct bnxt_qplib_creq_stat {
u64 creq_qp_event_processed;
u64 creq_func_event_processed;
};
struct bnxt_qplib_creq_ctx {
struct bnxt_qplib_hwq hwq;
struct bnxt_qplib_creq_db creq_db;
struct bnxt_qplib_creq_stat stats;
struct tasklet_struct creq_tasklet;
aeq_handler_t aeq_handler;
u16 ring_id;
int msix_vec;
bool requested; /*irq handler installed */
};
/* RCFW Communication Channels */
struct bnxt_qplib_rcfw {
struct pci_dev *pdev;
struct bnxt_qplib_res *res;
int vector;
struct tasklet_struct worker;
bool requested;
unsigned long *cmdq_bitmap;
u32 bmap_size;
unsigned long flags;
#define FIRMWARE_INITIALIZED_FLAG 0
#define FIRMWARE_FIRST_FLAG 31
#define FIRMWARE_TIMED_OUT 3
wait_queue_head_t waitq;
int (*aeq_handler)(struct bnxt_qplib_rcfw *,
void *, void *);
u32 seq_num;
/* Bar region info */
void __iomem *cmdq_bar_reg_iomem;
u16 cmdq_bar_reg;
u16 cmdq_bar_reg_prod_off;
u16 cmdq_bar_reg_trig_off;
u16 creq_ring_id;
u16 creq_bar_reg;
void __iomem *creq_bar_reg_iomem;
/* Cmd-Resp and Async Event notification queue */
struct bnxt_qplib_hwq creq;
u64 creq_qp_event_processed;
u64 creq_func_event_processed;
/* Actual Cmd and Resp Queues */
struct bnxt_qplib_hwq cmdq;
struct bnxt_qplib_crsq *crsqe_tbl;
struct bnxt_qplib_cmdq_ctx cmdq;
struct bnxt_qplib_creq_ctx creq;
struct bnxt_qplib_crsqe *crsqe_tbl;
int qp_tbl_size;
struct bnxt_qplib_qp_node *qp_tbl;
u64 oos_prev;
@ -268,7 +281,7 @@ struct bnxt_qplib_rcfw {
};
void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,
struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_ctx *ctx,
int qp_tbl_sz);
@ -276,12 +289,10 @@ void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);
void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
bool need_init);
int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
struct bnxt_qplib_rcfw *rcfw,
int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
int msix_vector,
int cp_bar_reg_off, int virt_fn,
int (*aeq_handler)(struct bnxt_qplib_rcfw *,
void *aeqe, void *obj));
aeq_handler_t aeq_handler);
struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
struct bnxt_qplib_rcfw *rcfw,

View File

@ -44,6 +44,7 @@
#include <linux/inetdevice.h>
#include <linux/dma-mapping.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
#include "roce_hsi.h"
#include "qplib_res.h"
#include "qplib_sp.h"
@ -55,9 +56,10 @@ static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
struct bnxt_qplib_stats *stats);
/* PBL */
static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
bool is_umem)
{
struct pci_dev *pdev = res->pdev;
int i;
if (!is_umem) {
@ -74,35 +76,56 @@ static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
pbl->pg_arr[i] = NULL;
}
}
kfree(pbl->pg_arr);
vfree(pbl->pg_arr);
pbl->pg_arr = NULL;
kfree(pbl->pg_map_arr);
vfree(pbl->pg_map_arr);
pbl->pg_map_arr = NULL;
pbl->pg_count = 0;
pbl->pg_size = 0;
}
static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
struct scatterlist *sghead, u32 pages,
u32 nmaps, u32 pg_size)
static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
struct bnxt_qplib_sg_info *sginfo)
{
struct scatterlist *sghead = sginfo->sghead;
struct sg_dma_page_iter sg_iter;
int i = 0;
for_each_sg_dma_page(sghead, &sg_iter, sginfo->nmap, 0) {
pbl->pg_map_arr[i] = sg_page_iter_dma_address(&sg_iter);
pbl->pg_arr[i] = NULL;
pbl->pg_count++;
i++;
}
}
static int __alloc_pbl(struct bnxt_qplib_res *res,
struct bnxt_qplib_pbl *pbl,
struct bnxt_qplib_sg_info *sginfo)
{
struct pci_dev *pdev = res->pdev;
struct scatterlist *sghead;
bool is_umem = false;
u32 pages;
int i;
if (sginfo->nopte)
return 0;
pages = sginfo->npages;
sghead = sginfo->sghead;
/* page ptr arrays */
pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL);
pbl->pg_arr = vmalloc(pages * sizeof(void *));
if (!pbl->pg_arr)
return -ENOMEM;
pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL);
pbl->pg_map_arr = vmalloc(pages * sizeof(dma_addr_t));
if (!pbl->pg_map_arr) {
kfree(pbl->pg_arr);
vfree(pbl->pg_arr);
pbl->pg_arr = NULL;
return -ENOMEM;
}
pbl->pg_count = 0;
pbl->pg_size = pg_size;
pbl->pg_size = sginfo->pgsize;
if (!sghead) {
for (i = 0; i < pages; i++) {
@ -115,25 +138,19 @@ static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
pbl->pg_count++;
}
} else {
i = 0;
is_umem = true;
for_each_sg_dma_page(sghead, &sg_iter, nmaps, 0) {
pbl->pg_map_arr[i] = sg_page_iter_dma_address(&sg_iter);
pbl->pg_arr[i] = NULL;
pbl->pg_count++;
i++;
}
bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
}
return 0;
fail:
__free_pbl(pdev, pbl, is_umem);
__free_pbl(res, pbl, is_umem);
return -ENOMEM;
}
/* HWQ */
void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
struct bnxt_qplib_hwq *hwq)
{
int i;
@ -144,9 +161,9 @@ void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
for (i = 0; i < hwq->level + 1; i++) {
if (i == hwq->level)
__free_pbl(pdev, &hwq->pbl[i], hwq->is_user);
__free_pbl(res, &hwq->pbl[i], hwq->is_user);
else
__free_pbl(pdev, &hwq->pbl[i], false);
__free_pbl(res, &hwq->pbl[i], false);
}
hwq->level = PBL_LVL_MAX;
@ -158,79 +175,113 @@ void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
}
/* All HWQs are power of 2 in size */
int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_sg_info *sg_info,
u32 *elements, u32 element_size, u32 aux,
u32 pg_size, enum bnxt_qplib_hwq_type hwq_type)
int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_hwq_attr *hwq_attr)
{
u32 pages, maps, slots, size, aux_pages = 0, aux_size = 0;
u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
struct bnxt_qplib_sg_info sginfo = {};
u32 depth, stride, npbl, npde;
dma_addr_t *src_phys_ptr, **dst_virt_ptr;
struct scatterlist *sghead = NULL;
int i, rc;
struct bnxt_qplib_res *res;
struct pci_dev *pdev;
int i, rc, lvl;
res = hwq_attr->res;
pdev = res->pdev;
sghead = hwq_attr->sginfo->sghead;
pg_size = hwq_attr->sginfo->pgsize;
hwq->level = PBL_LVL_MAX;
slots = roundup_pow_of_two(*elements);
if (aux) {
aux_size = roundup_pow_of_two(aux);
aux_pages = (slots * aux_size) / pg_size;
if ((slots * aux_size) % pg_size)
depth = roundup_pow_of_two(hwq_attr->depth);
stride = roundup_pow_of_two(hwq_attr->stride);
if (hwq_attr->aux_depth) {
aux_slots = hwq_attr->aux_depth;
aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
aux_pages = (aux_slots * aux_size) / pg_size;
if ((aux_slots * aux_size) % pg_size)
aux_pages++;
}
size = roundup_pow_of_two(element_size);
if (sg_info)
sghead = sg_info->sglist;
if (!sghead) {
hwq->is_user = false;
pages = (slots * size) / pg_size + aux_pages;
if ((slots * size) % pg_size)
pages++;
if (!pages)
npages = (depth * stride) / pg_size + aux_pages;
if ((depth * stride) % pg_size)
npages++;
if (!npages)
return -EINVAL;
maps = 0;
hwq_attr->sginfo->npages = npages;
} else {
hwq->is_user = true;
pages = sg_info->npages;
maps = sg_info->nmap;
npages = hwq_attr->sginfo->npages;
npages = (npages * PAGE_SIZE) /
BIT_ULL(hwq_attr->sginfo->pgshft);
if ((hwq_attr->sginfo->npages * PAGE_SIZE) %
BIT_ULL(hwq_attr->sginfo->pgshft))
if (!npages)
npages++;
}
/* Alloc the 1st memory block; can be a PDL/PTL/PBL */
if (sghead && (pages == MAX_PBL_LVL_0_PGS))
rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead,
pages, maps, pg_size);
else
rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL,
1, 0, pg_size);
if (rc)
goto fail;
if (npages == MAX_PBL_LVL_0_PGS) {
/* This request is Level 0, map PTE */
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
if (rc)
goto fail;
hwq->level = PBL_LVL_0;
}
hwq->level = PBL_LVL_0;
if (pages > MAX_PBL_LVL_0_PGS) {
if (pages > MAX_PBL_LVL_1_PGS) {
if (npages > MAX_PBL_LVL_0_PGS) {
if (npages > MAX_PBL_LVL_1_PGS) {
u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
0 : PTU_PTE_VALID;
/* 2 levels of indirection */
rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL,
MAX_PBL_LVL_1_PGS_FOR_LVL_2,
0, pg_size);
npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
npbl++;
npde = npbl >> MAX_PDL_LVL_SHIFT;
if (npbl % BIT(MAX_PDL_LVL_SHIFT))
npde++;
/* Alloc PDE pages */
sginfo.pgsize = npde * pg_size;
sginfo.npages = 1;
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
/* Alloc PBL pages */
sginfo.npages = npbl;
sginfo.pgsize = PAGE_SIZE;
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
if (rc)
goto fail;
/* Fill in lvl0 PBL */
/* Fill PDL with PBL page pointers */
dst_virt_ptr =
(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
src_phys_ptr[i] | PTU_PDE_VALID;
hwq->level = PBL_LVL_1;
rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead,
pages, maps, pg_size);
if (hwq_attr->type == HWQ_TYPE_MR) {
/* For MR it is expected that we supply only 1 contigous
* page i.e only 1 entry in the PDL that will contain
* all the PBLs for the user supplied memory region
*/
for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
i++)
dst_virt_ptr[0][i] = src_phys_ptr[i] |
flag;
} else {
for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
i++)
dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
src_phys_ptr[i] |
PTU_PDE_VALID;
}
/* Alloc or init PTEs */
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
hwq_attr->sginfo);
if (rc)
goto fail;
/* Fill in lvl1 PBL */
hwq->level = PBL_LVL_2;
if (hwq_attr->sginfo->nopte)
goto done;
/* Fill PBLs with PTE pointers */
dst_virt_ptr =
(dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
@ -238,7 +289,7 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
src_phys_ptr[i] | PTU_PTE_VALID;
}
if (hwq_type == HWQ_TYPE_QUEUE) {
if (hwq_attr->type == HWQ_TYPE_QUEUE) {
/* Find the last pg of the size */
i = hwq->pbl[PBL_LVL_2].pg_count;
dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
@ -248,25 +299,36 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
[PTR_IDX(i - 2)] |=
PTU_PTE_NEXT_TO_LAST;
}
hwq->level = PBL_LVL_2;
} else {
u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 :
PTU_PTE_VALID;
} else { /* pages < 512 npbl = 1, npde = 0 */
u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
0 : PTU_PTE_VALID;
/* 1 level of indirection */
rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead,
pages, maps, pg_size);
npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
npbl++;
sginfo.npages = npbl;
sginfo.pgsize = PAGE_SIZE;
/* Alloc PBL page */
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
if (rc)
goto fail;
/* Fill in lvl0 PBL */
/* Alloc or init PTEs */
rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
hwq_attr->sginfo);
if (rc)
goto fail;
hwq->level = PBL_LVL_1;
if (hwq_attr->sginfo->nopte)
goto done;
/* Fill PBL with PTE pointers */
dst_virt_ptr =
(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) {
for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
src_phys_ptr[i] | flag;
}
if (hwq_type == HWQ_TYPE_QUEUE) {
if (hwq_attr->type == HWQ_TYPE_QUEUE) {
/* Find the last pg of the size */
i = hwq->pbl[PBL_LVL_1].pg_count;
dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
@ -276,42 +338,141 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
[PTR_IDX(i - 2)] |=
PTU_PTE_NEXT_TO_LAST;
}
hwq->level = PBL_LVL_1;
}
}
hwq->pdev = pdev;
spin_lock_init(&hwq->lock);
done:
hwq->prod = 0;
hwq->cons = 0;
*elements = hwq->max_elements = slots;
hwq->element_size = size;
hwq->pdev = pdev;
hwq->depth = hwq_attr->depth;
hwq->max_elements = depth;
hwq->element_size = stride;
/* For direct access to the elements */
hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr;
hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr;
lvl = hwq->level;
if (hwq_attr->sginfo->nopte && hwq->level)
lvl = hwq->level - 1;
hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
spin_lock_init(&hwq->lock);
return 0;
fail:
bnxt_qplib_free_hwq(pdev, hwq);
bnxt_qplib_free_hwq(res, hwq);
return -ENOMEM;
}
/* Context Tables */
void bnxt_qplib_free_ctx(struct pci_dev *pdev,
void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx)
{
int i;
bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl);
bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl);
bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl);
bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl);
bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl);
bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]);
bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde);
bnxt_qplib_free_stats_ctx(pdev, &ctx->stats);
bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
/* restore original pde level before destroy */
ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
bnxt_qplib_free_stats_ctx(res->pdev, &ctx->stats);
}
static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx)
{
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
struct bnxt_qplib_tqm_ctx *tqmctx;
int rc = 0;
int i;
tqmctx = &ctx->tqm_ctx;
sginfo.pgsize = PAGE_SIZE;
sginfo.pgshft = PAGE_SHIFT;
hwq_attr.sginfo = &sginfo;
hwq_attr.res = res;
hwq_attr.type = HWQ_TYPE_CTX;
hwq_attr.depth = 512;
hwq_attr.stride = sizeof(u64);
/* Alloc pdl buffer */
rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
if (rc)
goto out;
/* Save original pdl level */
tqmctx->pde_level = tqmctx->pde.level;
hwq_attr.stride = 1;
for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
if (!tqmctx->qcount[i])
continue;
hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
if (rc)
goto out;
}
out:
return rc;
}
static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
{
struct bnxt_qplib_hwq *tbl;
dma_addr_t *dma_ptr;
__le64 **pbl_ptr, *ptr;
int i, j, k;
int fnz_idx = -1;
int pg_count;
pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
tbl = &ctx->qtbl[i];
if (!tbl->max_elements)
continue;
if (fnz_idx == -1)
fnz_idx = i; /* first non-zero index */
switch (tbl->level) {
case PBL_LVL_2:
pg_count = tbl->pbl[PBL_LVL_1].pg_count;
for (k = 0; k < pg_count; k++) {
ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
*ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
}
break;
case PBL_LVL_1:
case PBL_LVL_0:
default:
ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
*ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
PTU_PTE_VALID);
break;
}
}
if (fnz_idx == -1)
fnz_idx = 0;
/* update pde level as per page table programming */
ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
ctx->qtbl[fnz_idx].level + 1;
}
static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx)
{
int rc = 0;
rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
if (rc)
goto fail;
bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
fail:
return rc;
}
/*
@ -335,120 +496,72 @@ void bnxt_qplib_free_ctx(struct pci_dev *pdev,
* Returns:
* 0 if success, else -ERRORS
*/
int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx,
bool virt_fn, bool is_p5)
{
int i, j, k, rc = 0;
int fnz_idx = -1;
__le64 **pbl_ptr;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
int rc = 0;
if (virt_fn || is_p5)
goto stats_alloc;
/* QPC Tables */
ctx->qpc_tbl.max_elements = ctx->qpc_count;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL,
&ctx->qpc_tbl.max_elements,
BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_CTX);
sginfo.pgsize = PAGE_SIZE;
sginfo.pgshft = PAGE_SHIFT;
hwq_attr.sginfo = &sginfo;
hwq_attr.res = res;
hwq_attr.depth = ctx->qpc_count;
hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
hwq_attr.type = HWQ_TYPE_CTX;
rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
if (rc)
goto fail;
/* MRW Tables */
ctx->mrw_tbl.max_elements = ctx->mrw_count;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL,
&ctx->mrw_tbl.max_elements,
BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_CTX);
hwq_attr.depth = ctx->mrw_count;
hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
if (rc)
goto fail;
/* SRQ Tables */
ctx->srqc_tbl.max_elements = ctx->srqc_count;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL,
&ctx->srqc_tbl.max_elements,
BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_CTX);
hwq_attr.depth = ctx->srqc_count;
hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
if (rc)
goto fail;
/* CQ Tables */
ctx->cq_tbl.max_elements = ctx->cq_count;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL,
&ctx->cq_tbl.max_elements,
BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_CTX);
hwq_attr.depth = ctx->cq_count;
hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
if (rc)
goto fail;
/* TQM Buffer */
ctx->tqm_pde.max_elements = 512;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL,
&ctx->tqm_pde.max_elements, sizeof(u64),
0, PAGE_SIZE, HWQ_TYPE_CTX);
rc = bnxt_qplib_setup_tqm_rings(res, ctx);
if (rc)
goto fail;
for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
if (!ctx->tqm_count[i])
continue;
ctx->tqm_tbl[i].max_elements = ctx->qpc_count *
ctx->tqm_count[i];
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL,
&ctx->tqm_tbl[i].max_elements, 1,
0, PAGE_SIZE, HWQ_TYPE_CTX);
if (rc)
goto fail;
}
pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr;
for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
if (!ctx->tqm_tbl[i].max_elements)
continue;
if (fnz_idx == -1)
fnz_idx = i;
switch (ctx->tqm_tbl[i].level) {
case PBL_LVL_2:
for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count;
k++)
pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] =
cpu_to_le64(
ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k]
| PTU_PTE_VALID);
break;
case PBL_LVL_1:
case PBL_LVL_0:
default:
pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64(
ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] |
PTU_PTE_VALID);
break;
}
}
if (fnz_idx == -1)
fnz_idx = 0;
ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ?
PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1;
/* TIM Buffer */
ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL,
&ctx->tim_tbl.max_elements, 1,
0, PAGE_SIZE, HWQ_TYPE_CTX);
hwq_attr.depth = ctx->qpc_count * 16;
hwq_attr.stride = 1;
rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
if (rc)
goto fail;
stats_alloc:
/* Stats */
rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats);
rc = bnxt_qplib_alloc_stats_ctx(res->pdev, &ctx->stats);
if (rc)
goto fail;
return 0;
fail:
bnxt_qplib_free_ctx(pdev, ctx);
bnxt_qplib_free_ctx(res, ctx);
return rc;
}
@ -808,9 +921,6 @@ void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
bnxt_qplib_free_pd_tbl(&res->pd_tbl);
bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
res->netdev = NULL;
res->pdev = NULL;
}
int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,

View File

@ -55,7 +55,8 @@ extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
enum bnxt_qplib_hwq_type {
HWQ_TYPE_CTX,
HWQ_TYPE_QUEUE,
HWQ_TYPE_L2_CMPL
HWQ_TYPE_L2_CMPL,
HWQ_TYPE_MR
};
#define MAX_PBL_LVL_0_PGS 1
@ -63,6 +64,7 @@ enum bnxt_qplib_hwq_type {
#define MAX_PBL_LVL_1_PGS_SHIFT 9
#define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256
#define MAX_PBL_LVL_2_PGS (256 * 512)
#define MAX_PDL_LVL_SHIFT 9
enum bnxt_qplib_pbl_lvl {
PBL_LVL_0,
@ -78,6 +80,13 @@ enum bnxt_qplib_pbl_lvl {
#define ROCE_PG_SIZE_8M (8 * 1024 * 1024)
#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024)
struct bnxt_qplib_reg_desc {
u8 bar_id;
resource_size_t bar_base;
void __iomem *bar_reg;
size_t len;
};
struct bnxt_qplib_pbl {
u32 pg_count;
u32 pg_size;
@ -85,17 +94,37 @@ struct bnxt_qplib_pbl {
dma_addr_t *pg_map_arr;
};
struct bnxt_qplib_sg_info {
struct scatterlist *sghead;
u32 nmap;
u32 npages;
u32 pgshft;
u32 pgsize;
bool nopte;
};
struct bnxt_qplib_hwq_attr {
struct bnxt_qplib_res *res;
struct bnxt_qplib_sg_info *sginfo;
enum bnxt_qplib_hwq_type type;
u32 depth;
u32 stride;
u32 aux_stride;
u32 aux_depth;
};
struct bnxt_qplib_hwq {
struct pci_dev *pdev;
/* lock to protect qplib_hwq */
spinlock_t lock;
struct bnxt_qplib_pbl pbl[PBL_LVL_MAX];
struct bnxt_qplib_pbl pbl[PBL_LVL_MAX + 1];
enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */
/* ptr for easy access to the PBL entries */
void **pbl_ptr;
/* ptr for easy access to the dma_addr */
dma_addr_t *pbl_dma_ptr;
u32 max_elements;
u32 depth;
u16 element_size; /* Size of each entry */
u32 prod; /* raw */
@ -104,6 +133,13 @@ struct bnxt_qplib_hwq {
u8 is_user;
};
struct bnxt_qplib_db_info {
void __iomem *db;
void __iomem *priv_db;
struct bnxt_qplib_hwq *hwq;
u32 xid;
};
/* Tables */
struct bnxt_qplib_pd_tbl {
unsigned long *tbl;
@ -159,6 +195,15 @@ struct bnxt_qplib_vf_res {
#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64
#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128
#define MAX_TQM_ALLOC_REQ 48
#define MAX_TQM_ALLOC_BLK_SIZE 8
struct bnxt_qplib_tqm_ctx {
struct bnxt_qplib_hwq pde;
u8 pde_level; /* Original level */
struct bnxt_qplib_hwq qtbl[MAX_TQM_ALLOC_REQ];
u8 qcount[MAX_TQM_ALLOC_REQ];
};
struct bnxt_qplib_ctx {
u32 qpc_count;
struct bnxt_qplib_hwq qpc_tbl;
@ -169,12 +214,7 @@ struct bnxt_qplib_ctx {
u32 cq_count;
struct bnxt_qplib_hwq cq_tbl;
struct bnxt_qplib_hwq tim_tbl;
#define MAX_TQM_ALLOC_REQ 48
#define MAX_TQM_ALLOC_BLK_SIZE 8
u8 tqm_count[MAX_TQM_ALLOC_REQ];
struct bnxt_qplib_hwq tqm_pde;
u32 tqm_pde_level;
struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ];
struct bnxt_qplib_tqm_ctx tqm_ctx;
struct bnxt_qplib_stats stats;
struct bnxt_qplib_vf_res vf_res;
u64 hwrm_intf_ver;
@ -223,11 +263,6 @@ static inline u8 bnxt_qplib_get_ring_type(struct bnxt_qplib_chip_ctx *cctx)
RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL;
}
struct bnxt_qplib_sg_info {
struct scatterlist *sglist;
u32 nmap;
u32 npages;
};
#define to_bnxt_qplib(ptr, type, member) \
container_of(ptr, type, member)
@ -235,11 +270,10 @@ struct bnxt_qplib_sg_info {
struct bnxt_qplib_pd;
struct bnxt_qplib_dev_attr;
void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq);
int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_sg_info *sg_info, u32 *elements,
u32 elements_per_page, u32 aux, u32 pg_size,
enum bnxt_qplib_hwq_type hwq_type);
void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
struct bnxt_qplib_hwq *hwq);
int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_hwq_attr *hwq_attr);
void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid);
int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl,
struct bnxt_qplib_pd *pd);
@ -258,9 +292,80 @@ void bnxt_qplib_free_res(struct bnxt_qplib_res *res);
int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
struct net_device *netdev,
struct bnxt_qplib_dev_attr *dev_attr);
void bnxt_qplib_free_ctx(struct pci_dev *pdev,
void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx);
int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
struct bnxt_qplib_ctx *ctx,
bool virt_fn, bool is_p5);
static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
bool arm)
{
u32 key;
key = info->hwq->cons & (info->hwq->max_elements - 1);
key |= (CMPL_DOORBELL_IDX_VALID |
(CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK));
if (!arm)
key |= CMPL_DOORBELL_MASK;
writel(key, info->db);
}
static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info,
u32 type)
{
u64 key = 0;
key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
key <<= 32;
key |= (info->hwq->cons & (info->hwq->max_elements - 1)) &
DBC_DBC_INDEX_MASK;
writeq(key, info->db);
}
static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info,
u32 type)
{
u64 key = 0;
key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
key <<= 32;
key |= (info->hwq->prod & (info->hwq->max_elements - 1)) &
DBC_DBC_INDEX_MASK;
writeq(key, info->db);
}
static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info,
u32 type)
{
u64 key = 0;
key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
key <<= 32;
writeq(key, info->priv_db);
}
static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info,
u32 th)
{
u64 key = 0;
key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | th;
key <<= 32;
key |= th & DBC_DBC_INDEX_MASK;
writeq(key, info->priv_db);
}
static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info,
struct bnxt_qplib_chip_ctx *cctx,
bool arm)
{
u32 type;
type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;
if (bnxt_qplib_is_chip_gen_p5(cctx))
bnxt_qplib_ring_db(info, type);
else
bnxt_qplib_ring_db32(info, arm);
}
#endif /* __BNXT_QPLIB_RES_H__ */

View File

@ -585,7 +585,7 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
/* Free the qplib's MRW memory */
if (mrw->hwq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
bnxt_qplib_free_hwq(res, &mrw->hwq);
return 0;
}
@ -646,7 +646,7 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
if (mrw->hwq.max_elements) {
mrw->va = 0;
mrw->total_size = 0;
bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
bnxt_qplib_free_hwq(res, &mrw->hwq);
}
return 0;
@ -656,10 +656,12 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_register_mr req;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
struct creq_register_mr_resp resp;
u16 cmd_flags = 0, level;
struct cmdq_register_mr req;
int pg_ptrs, pages, i, rc;
u16 cmd_flags = 0, level;
dma_addr_t **pbl_ptr;
u32 pg_size;
@ -674,20 +676,23 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
if (pages > MAX_PBL_LVL_1_PGS) {
dev_err(&res->pdev->dev,
"SP: Reg MR pages requested (0x%x) exceeded max (0x%x)\n",
"SP: Reg MR: pages requested (0x%x) exceeded max (0x%x)\n",
pages, MAX_PBL_LVL_1_PGS);
return -ENOMEM;
}
/* Free the hwq if it already exist, must be a rereg */
if (mr->hwq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
mr->hwq.max_elements = pages;
bnxt_qplib_free_hwq(res, &mr->hwq);
/* Use system PAGE_SIZE */
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL,
&mr->hwq.max_elements,
PAGE_SIZE, 0, PAGE_SIZE,
HWQ_TYPE_CTX);
hwq_attr.res = res;
hwq_attr.depth = pages;
hwq_attr.stride = PAGE_SIZE;
hwq_attr.type = HWQ_TYPE_MR;
hwq_attr.sginfo = &sginfo;
hwq_attr.sginfo->npages = pages;
hwq_attr.sginfo->pgsize = PAGE_SIZE;
hwq_attr.sginfo->pgshft = PAGE_SHIFT;
rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr);
if (rc) {
dev_err(&res->pdev->dev,
"SP: Reg MR memory allocation failed\n");
@ -734,7 +739,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
fail:
if (mr->hwq.max_elements)
bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
bnxt_qplib_free_hwq(res, &mr->hwq);
return rc;
}
@ -742,6 +747,8 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
struct bnxt_qplib_frpl *frpl,
int max_pg_ptrs)
{
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
int pg_ptrs, pages, rc;
/* Re-calculate the max to fit the HWQ allocation model */
@ -753,10 +760,15 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
if (pages > MAX_PBL_LVL_1_PGS)
return -ENOMEM;
frpl->hwq.max_elements = pages;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL,
&frpl->hwq.max_elements, PAGE_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_CTX);
sginfo.pgsize = PAGE_SIZE;
sginfo.nopte = true;
hwq_attr.res = res;
hwq_attr.depth = pg_ptrs;
hwq_attr.stride = PAGE_SIZE;
hwq_attr.sginfo = &sginfo;
hwq_attr.type = HWQ_TYPE_CTX;
rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr);
if (!rc)
frpl->max_pg_ptrs = pg_ptrs;
@ -766,7 +778,7 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
struct bnxt_qplib_frpl *frpl)
{
bnxt_qplib_free_hwq(res->pdev, &frpl->hwq);
bnxt_qplib_free_hwq(res, &frpl->hwq);
return 0;
}

View File

@ -707,7 +707,7 @@ struct mpa_message {
u8 flags;
u8 revision;
__be16 private_data_size;
u8 private_data[0];
u8 private_data[];
};
struct mpa_v2_conn_params {
@ -719,7 +719,7 @@ struct terminate_message {
u8 layer_etype;
u8 ecode;
__be16 hdrct_rsvd;
u8 len_hdrs[0];
u8 len_hdrs[];
};
#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)

View File

@ -2127,7 +2127,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
pr_debug("ib_pd %p\n", pd);
if (attrs->qp_type != IB_QPT_RC)
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
php = to_c4iw_pd(pd);
rhp = php->rhp;

View File

@ -123,7 +123,7 @@ struct fw_ri_dsgl {
__be32 len0;
__be64 addr0;
#ifndef C99_NOT_SUPPORTED
struct fw_ri_dsge_pair sge[0];
struct fw_ri_dsge_pair sge[];
#endif
};
@ -139,7 +139,7 @@ struct fw_ri_isgl {
__be16 nsge;
__be32 r2;
#ifndef C99_NOT_SUPPORTED
struct fw_ri_sge sge[0];
struct fw_ri_sge sge[];
#endif
};
@ -149,7 +149,7 @@ struct fw_ri_immd {
__be16 r2;
__be32 immdlen;
#ifndef C99_NOT_SUPPORTED
__u8 data[0];
__u8 data[];
#endif
};
@ -321,7 +321,7 @@ struct fw_ri_res_wr {
__be32 len16_pkd;
__u64 cookie;
#ifndef C99_NOT_SUPPORTED
struct fw_ri_res res[0];
struct fw_ri_res res[];
#endif
};

View File

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_ADMIN_CMDS_H_
@ -801,21 +801,16 @@ struct efa_admin_mmio_req_read_less_resp {
/* create_qp_cmd */
#define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK BIT(0)
#define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_SHIFT 1
#define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK BIT(1)
/* reg_mr_cmd */
#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0)
#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_SHIFT 7
#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7)
#define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0)
#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_SHIFT 2
#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK BIT(2)
/* create_cq_cmd */
#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT 5
#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)
#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_SHIFT 6
#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK BIT(6)
#define EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0)

View File

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_ADMIN_H_
@ -121,9 +121,7 @@ struct efa_admin_aenq_entry {
/* aq_common_desc */
#define EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0)
#define EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0)
#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT 1
#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK BIT(1)
#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT 2
#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK BIT(2)
/* acq_common_desc */

View File

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include "efa_com.h"
@ -16,26 +16,13 @@
#define EFA_ASYNC_QUEUE_DEPTH 16
#define EFA_ADMIN_QUEUE_DEPTH 32
#define MIN_EFA_VER\
((EFA_ADMIN_API_VERSION_MAJOR << EFA_REGS_VERSION_MAJOR_VERSION_SHIFT) | \
(EFA_ADMIN_API_VERSION_MINOR & EFA_REGS_VERSION_MINOR_VERSION_MASK))
#define EFA_CTRL_MAJOR 0
#define EFA_CTRL_MINOR 0
#define EFA_CTRL_SUB_MINOR 1
#define MIN_EFA_CTRL_VER \
(((EFA_CTRL_MAJOR) << \
(EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
((EFA_CTRL_MINOR) << \
(EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
(EFA_CTRL_SUB_MINOR))
#define EFA_DMA_ADDR_TO_UINT32_LOW(x) ((u32)((u64)(x)))
#define EFA_DMA_ADDR_TO_UINT32_HIGH(x) ((u32)(((u64)(x)) >> 32))
#define EFA_REGS_ADMIN_INTR_MASK 1
enum efa_cmd_status {
EFA_CMD_SUBMITTED,
EFA_CMD_COMPLETED,
@ -84,7 +71,7 @@ static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)
struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
struct efa_admin_mmio_req_read_less_resp *read_resp;
unsigned long exp_time;
u32 mmio_read_reg;
u32 mmio_read_reg = 0;
u32 err;
read_resp = mmio_read->read_resp;
@ -94,10 +81,9 @@ static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)
/* trash DMA req_id to identify when hardware is done */
read_resp->req_id = mmio_read->seq_num + 0x9aL;
mmio_read_reg = (offset << EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
EFA_REGS_MMIO_REG_READ_REG_OFF_MASK;
mmio_read_reg |= mmio_read->seq_num &
EFA_REGS_MMIO_REG_READ_REQ_ID_MASK;
EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset);
EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID,
mmio_read->seq_num);
writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF);
@ -137,9 +123,9 @@ static int efa_com_admin_init_sq(struct efa_com_dev *edev)
struct efa_com_admin_queue *aq = &edev->aq;
struct efa_com_admin_sq *sq = &aq->sq;
u16 size = aq->depth * sizeof(*sq->entries);
u32 aq_caps = 0;
u32 addr_high;
u32 addr_low;
u32 aq_caps;
sq->entries =
dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);
@ -160,10 +146,9 @@ static int efa_com_admin_init_sq(struct efa_com_dev *edev)
writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF);
writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF);
aq_caps = aq->depth & EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
aq_caps |= (sizeof(struct efa_admin_aq_entry) <<
EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth);
EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE,
sizeof(struct efa_admin_aq_entry));
writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF);
@ -175,9 +160,9 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
struct efa_com_admin_queue *aq = &edev->aq;
struct efa_com_admin_cq *cq = &aq->cq;
u16 size = aq->depth * sizeof(*cq->entries);
u32 acq_caps = 0;
u32 addr_high;
u32 addr_low;
u32 acq_caps;
cq->entries =
dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
@ -195,13 +180,11 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF);
writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF);
acq_caps = aq->depth & EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
acq_caps |= (sizeof(struct efa_admin_acq_entry) <<
EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
acq_caps |= (aq->msix_vector_idx <<
EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT) &
EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK;
EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth);
EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE,
sizeof(struct efa_admin_acq_entry));
EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR,
aq->msix_vector_idx);
writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF);
@ -212,7 +195,8 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev,
struct efa_aenq_handlers *aenq_handlers)
{
struct efa_com_aenq *aenq = &edev->aenq;
u32 addr_low, addr_high, aenq_caps;
u32 addr_low, addr_high;
u32 aenq_caps = 0;
u16 size;
if (!aenq_handlers) {
@ -237,13 +221,11 @@ static int efa_com_admin_init_aenq(struct efa_com_dev *edev,
writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF);
writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF);
aenq_caps = aenq->depth & EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
aenq_caps |= (sizeof(struct efa_admin_aenq_entry) <<
EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
aenq_caps |= (aenq->msix_vector_idx
<< EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT) &
EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK;
EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth);
EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE,
sizeof(struct efa_admin_aenq_entry));
EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR,
aenq->msix_vector_idx);
writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF);
/*
@ -280,8 +262,8 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,
static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,
struct efa_comp_ctx *comp_ctx)
{
u16 cmd_id = comp_ctx->user_cqe->acq_common_descriptor.command &
EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command,
EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
u16 ctx_id = cmd_id & (aq->depth - 1);
ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id);
@ -335,8 +317,8 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu
cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
cmd->aq_common_descriptor.command_id = cmd_id;
cmd->aq_common_descriptor.flags |= aq->sq.phase &
EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
EFA_SET(&cmd->aq_common_descriptor.flags,
EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase);
comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);
if (!comp_ctx) {
@ -427,8 +409,8 @@ static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *a
struct efa_comp_ctx *comp_ctx;
u16 cmd_id;
cmd_id = cqe->acq_common_descriptor.command &
EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
cmd_id = EFA_GET(&cqe->acq_common_descriptor.command,
EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);
if (!comp_ctx) {
@ -705,7 +687,7 @@ void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling)
u32 mask_value = 0;
if (polling)
mask_value = EFA_REGS_ADMIN_INTR_MASK;
EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1);
writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF);
if (polling)
@ -743,7 +725,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
int err;
dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
if (!(dev_sts & EFA_REGS_DEV_STS_READY_MASK)) {
if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) {
ibdev_err(edev->efa_dev,
"Device isn't ready, abort com init %#x\n", dev_sts);
return -ENODEV;
@ -778,8 +760,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
goto err_destroy_cq;
cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
timeout = (cap & EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
if (timeout)
/* the resolution of timeout reg is 100ms */
aq->completion_timeout = timeout * 100000;
@ -940,7 +921,9 @@ void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev)
int efa_com_validate_version(struct efa_com_dev *edev)
{
u32 min_ctrl_ver = 0;
u32 ctrl_ver_masked;
u32 min_ver = 0;
u32 ctrl_ver;
u32 ver;
@ -953,33 +936,42 @@ int efa_com_validate_version(struct efa_com_dev *edev)
EFA_REGS_CONTROLLER_VERSION_OFF);
ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n",
(ver & EFA_REGS_VERSION_MAJOR_VERSION_MASK) >>
EFA_REGS_VERSION_MAJOR_VERSION_SHIFT,
ver & EFA_REGS_VERSION_MINOR_VERSION_MASK);
EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION),
EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION));
if (ver < MIN_EFA_VER) {
EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION,
EFA_ADMIN_API_VERSION_MAJOR);
EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION,
EFA_ADMIN_API_VERSION_MINOR);
if (ver < min_ver) {
ibdev_err(edev->efa_dev,
"EFA version is lower than the minimal version the driver supports\n");
return -EOPNOTSUPP;
}
ibdev_dbg(edev->efa_dev,
"efa controller version: %d.%d.%d implementation version %d\n",
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
ibdev_dbg(
edev->efa_dev,
"efa controller version: %d.%d.%d implementation version %d\n",
EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION),
EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION),
EFA_GET(&ctrl_ver,
EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION),
EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID));
ctrl_ver_masked =
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) |
EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) |
EFA_GET(&ctrl_ver,
EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION);
EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION,
EFA_CTRL_MAJOR);
EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION,
EFA_CTRL_MINOR);
EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION,
EFA_CTRL_SUB_MINOR);
/* Validate the ctrl version without the implementation ID */
if (ctrl_ver_masked < MIN_EFA_CTRL_VER) {
if (ctrl_ver_masked < min_ctrl_ver) {
ibdev_err(edev->efa_dev,
"EFA ctrl version is lower than the minimal ctrl version the driver supports\n");
return -EOPNOTSUPP;
@ -1002,8 +994,7 @@ int efa_com_get_dma_width(struct efa_com_dev *edev)
u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
int width;
width = (caps & EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH);
ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width);
@ -1017,16 +1008,14 @@ int efa_com_get_dma_width(struct efa_com_dev *edev)
return width;
}
static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout,
u16 exp_state)
static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on)
{
u32 val, i;
for (i = 0; i < timeout; i++) {
val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
if ((val & EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
exp_state)
if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on)
return 0;
ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val);
@ -1046,36 +1035,34 @@ static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout,
int efa_com_dev_reset(struct efa_com_dev *edev,
enum efa_regs_reset_reason_types reset_reason)
{
u32 stat, timeout, cap, reset_val;
u32 stat, timeout, cap;
u32 reset_val = 0;
int err;
stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
if (!(stat & EFA_REGS_DEV_STS_READY_MASK)) {
if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) {
ibdev_err(edev->efa_dev,
"Device isn't ready, can't reset device\n");
return -EINVAL;
}
timeout = (cap & EFA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT);
if (!timeout) {
ibdev_err(edev->efa_dev, "Invalid timeout value\n");
return -EINVAL;
}
/* start reset */
reset_val = EFA_REGS_DEV_CTL_DEV_RESET_MASK;
reset_val |= (reset_reason << EFA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
EFA_REGS_DEV_CTL_RESET_REASON_MASK;
EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1);
EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason);
writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
/* reset clears the mmio readless address, restore it */
efa_com_mmio_reg_read_resp_addr_init(edev);
err = wait_for_reset_state(edev, timeout,
EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
err = wait_for_reset_state(edev, timeout, 1);
if (err) {
ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n");
return err;
@ -1089,8 +1076,7 @@ int efa_com_dev_reset(struct efa_com_dev *edev,
return err;
}
timeout = (cap & EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
if (timeout)
/* the resolution of timeout reg is 100ms */
edev->aq.completion_timeout = timeout * 100000;

View File

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include "efa_com.h"
@ -161,8 +161,9 @@ int efa_com_create_cq(struct efa_com_dev *edev,
int err;
create_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_CQ;
create_cmd.cq_caps_2 = (params->entry_size_in_bytes / 4) &
EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
EFA_SET(&create_cmd.cq_caps_2,
EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS,
params->entry_size_in_bytes / 4);
create_cmd.cq_depth = params->cq_depth;
create_cmd.num_sub_cqs = params->num_sub_cqs;
create_cmd.uar = params->uarn;
@ -227,8 +228,8 @@ int efa_com_register_mr(struct efa_com_dev *edev,
mr_cmd.aq_common_desc.opcode = EFA_ADMIN_REG_MR;
mr_cmd.pd = params->pd;
mr_cmd.mr_length = params->mr_length_in_bytes;
mr_cmd.flags |= params->page_shift &
EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK;
EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
params->page_shift);
mr_cmd.iova = params->iova;
mr_cmd.permissions = params->permissions;
@ -242,11 +243,11 @@ int efa_com_register_mr(struct efa_com_dev *edev,
params->pbl.pbl.address.mem_addr_low;
mr_cmd.pbl.pbl.address.mem_addr_high =
params->pbl.pbl.address.mem_addr_high;
mr_cmd.aq_common_desc.flags |=
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK;
EFA_SET(&mr_cmd.aq_common_desc.flags,
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);
if (params->indirect)
mr_cmd.aq_common_desc.flags |=
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
EFA_SET(&mr_cmd.aq_common_desc.flags,
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);
}
err = efa_com_cmd_exec(aq,
@ -386,9 +387,8 @@ static int efa_com_get_feature_ex(struct efa_com_dev *edev,
get_cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_FEATURE;
if (control_buff_size)
get_cmd.aq_common_descriptor.flags =
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
EFA_SET(&get_cmd.aq_common_descriptor.flags,
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);
efa_com_set_dma_addr(control_buf_dma_addr,
&get_cmd.control_buffer.address.mem_addr_high,
@ -538,8 +538,9 @@ static int efa_com_set_feature_ex(struct efa_com_dev *edev,
set_cmd->aq_common_descriptor.opcode = EFA_ADMIN_SET_FEATURE;
if (control_buff_size) {
set_cmd->aq_common_descriptor.flags =
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
set_cmd->aq_common_descriptor.flags = 0;
EFA_SET(&set_cmd->aq_common_descriptor.flags,
EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);
efa_com_set_dma_addr(control_buf_dma_addr,
&set_cmd->control_buffer.address.mem_addr_high,
&set_cmd->control_buffer.address.mem_addr_low);

View File

@ -1,14 +1,25 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_COMMON_H_
#define _EFA_COMMON_H_
#include <linux/bitfield.h>
#define EFA_COMMON_SPEC_VERSION_MAJOR 2
#define EFA_COMMON_SPEC_VERSION_MINOR 0
#define EFA_GET(ptr, mask) FIELD_GET(mask##_MASK, *(ptr))
#define EFA_SET(ptr, mask, value) \
({ \
typeof(ptr) _ptr = ptr; \
*_ptr = (*_ptr & ~(mask##_MASK)) | \
FIELD_PREP(mask##_MASK, value); \
})
struct efa_common_mem_addr {
u32 mem_addr_low;

View File

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_REGS_H_
@ -45,69 +45,52 @@ enum efa_regs_reset_reason_types {
/* version register */
#define EFA_REGS_VERSION_MINOR_VERSION_MASK 0xff
#define EFA_REGS_VERSION_MAJOR_VERSION_SHIFT 8
#define EFA_REGS_VERSION_MAJOR_VERSION_MASK 0xff00
/* controller_version register */
#define EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK 0xff
#define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT 8
#define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK 0xff00
#define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT 16
#define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK 0xff0000
#define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT 24
#define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK 0xff000000
/* caps register */
#define EFA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK 0x1
#define EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT 1
#define EFA_REGS_CAPS_RESET_TIMEOUT_MASK 0x3e
#define EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT 8
#define EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK 0xff00
#define EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT 16
#define EFA_REGS_CAPS_ADMIN_CMD_TO_MASK 0xf0000
/* aq_caps register */
#define EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK 0xffff
#define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT 16
#define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK 0xffff0000
/* acq_caps register */
#define EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK 0xffff
#define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT 16
#define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK 0xff0000
#define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT 24
#define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK 0xff000000
/* aenq_caps register */
#define EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK 0xffff
#define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT 16
#define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK 0xff0000
#define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT 24
#define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK 0xff000000
/* intr_mask register */
#define EFA_REGS_INTR_MASK_EN_MASK 0x1
/* dev_ctl register */
#define EFA_REGS_DEV_CTL_DEV_RESET_MASK 0x1
#define EFA_REGS_DEV_CTL_AQ_RESTART_SHIFT 1
#define EFA_REGS_DEV_CTL_AQ_RESTART_MASK 0x2
#define EFA_REGS_DEV_CTL_RESET_REASON_SHIFT 28
#define EFA_REGS_DEV_CTL_RESET_REASON_MASK 0xf0000000
/* dev_sts register */
#define EFA_REGS_DEV_STS_READY_MASK 0x1
#define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT 1
#define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK 0x2
#define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT 2
#define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK 0x4
#define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT 3
#define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK 0x8
#define EFA_REGS_DEV_STS_RESET_FINISHED_SHIFT 4
#define EFA_REGS_DEV_STS_RESET_FINISHED_MASK 0x10
#define EFA_REGS_DEV_STS_FATAL_ERROR_SHIFT 5
#define EFA_REGS_DEV_STS_FATAL_ERROR_MASK 0x20
/* mmio_reg_read register */
#define EFA_REGS_MMIO_REG_READ_REQ_ID_MASK 0xffff
#define EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT 16
#define EFA_REGS_MMIO_REG_READ_REG_OFF_MASK 0xffff0000
#endif /* _EFA_REGS_H_ */

View File

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include <linux/vmalloc.h>
@ -144,9 +144,6 @@ static inline bool is_rdma_read_cap(struct efa_dev *dev)
return dev->dev_attr.device_caps & EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK;
}
#define field_avail(x, fld, sz) (offsetof(typeof(x), fld) + \
sizeof_field(typeof(x), fld) <= (sz))
#define is_reserved_cleared(reserved) \
!memchr_inv(reserved, 0, sizeof(reserved))
@ -169,6 +166,14 @@ static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr,
return addr;
}
static void efa_free_mapped(struct efa_dev *dev, void *cpu_addr,
dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir)
{
dma_unmap_single(&dev->pdev->dev, dma_addr, size, dir);
free_pages_exact(cpu_addr, size);
}
int efa_query_device(struct ib_device *ibdev,
struct ib_device_attr *props,
struct ib_udata *udata)
@ -402,6 +407,9 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
int err;
ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num);
efa_qp_user_mmap_entries_remove(qp);
err = efa_destroy_qp_handle(dev, qp->qp_handle);
if (err)
return err;
@ -411,11 +419,10 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
"qp->cpu_addr[0x%p] freed: size[%lu], dma[%pad]\n",
qp->rq_cpu_addr, qp->rq_size,
&qp->rq_dma_addr);
dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size,
DMA_TO_DEVICE);
efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr,
qp->rq_size, DMA_TO_DEVICE);
}
efa_qp_user_mmap_entries_remove(qp);
kfree(qp);
return 0;
}
@ -599,7 +606,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
if (err)
goto err_out;
if (!field_avail(cmd, driver_qp_type, udata->inlen)) {
if (offsetofend(typeof(cmd), driver_qp_type) > udata->inlen) {
ibdev_dbg(&dev->ibdev,
"Incompatible ABI params, no input udata\n");
err = -EINVAL;
@ -720,13 +727,9 @@ err_remove_mmap_entries:
err_destroy_qp:
efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);
err_free_mapped:
if (qp->rq_size) {
dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size,
DMA_TO_DEVICE);
if (!qp->rq_mmap_entry)
free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
}
if (qp->rq_size)
efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr,
qp->rq_size, DMA_TO_DEVICE);
err_free_qp:
kfree(qp);
err_out:
@ -845,10 +848,10 @@ void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
"Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",
cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
efa_destroy_cq_idx(dev, cq->cq_idx);
dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
rdma_user_mmap_entry_remove(cq->mmap_entry);
efa_destroy_cq_idx(dev, cq->cq_idx);
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
}
static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
@ -890,7 +893,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
goto err_out;
}
if (!field_avail(cmd, num_sub_cqs, udata->inlen)) {
if (offsetofend(typeof(cmd), num_sub_cqs) > udata->inlen) {
ibdev_dbg(ibdev,
"Incompatible ABI params, no input udata\n");
err = -EINVAL;
@ -985,10 +988,8 @@ err_remove_mmap:
err_destroy_cq:
efa_destroy_cq_idx(dev, cq->cq_idx);
err_free_mapped:
dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (!cq->mmap_entry)
free_pages_exact(cq->cpu_addr, cq->size);
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
err_out:
atomic64_inc(&dev->stats.sw_stats.create_cq_err);
@ -1550,10 +1551,6 @@ void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
{
struct efa_user_mmap_entry *entry = to_emmap(rdma_entry);
/* DMA mapping is already gone, now free the pages */
if (entry->mmap_flag == EFA_MMAP_DMA_PAGE)
free_pages_exact(phys_to_virt(entry->address),
entry->rdma_entry.npages * PAGE_SIZE);
kfree(entry);
}

View File

@ -222,11 +222,11 @@ static ssize_t fault_opcodes_read(struct file *file, char __user *buf,
while (bit < bitsize) {
zero = find_next_zero_bit(fault->opcodes, bitsize, bit);
if (zero - 1 != bit)
size += snprintf(data + size,
size += scnprintf(data + size,
datalen - size - 1,
"0x%lx-0x%lx,", bit, zero - 1);
else
size += snprintf(data + size,
size += scnprintf(data + size,
datalen - size - 1, "0x%lx,",
bit);
bit = find_next_bit(fault->opcodes, bitsize, zero);

View File

@ -209,7 +209,6 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)
fd->mm = current->mm;
mmgrab(fd->mm);
fd->dd = dd;
kobject_get(&fd->dd->kobj);
fp->private_data = fd;
return 0;
nomem:
@ -713,7 +712,6 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
deallocate_ctxt(uctxt);
done:
mmdrop(fdata->mm);
kobject_put(&dd->kobj);
if (atomic_dec_and_test(&dd->user_refcount))
complete(&dd->user_comp);
@ -1696,7 +1694,7 @@ static int user_add(struct hfi1_devdata *dd)
snprintf(name, sizeof(name), "%s_%d", class_name(), dd->unit);
ret = hfi1_cdev_init(dd->unit, name, &hfi1_file_ops,
&dd->user_cdev, &dd->user_device,
true, &dd->kobj);
true, &dd->verbs_dev.rdi.ibdev.dev.kobj);
if (ret)
user_remove(dd);

View File

@ -1413,8 +1413,6 @@ struct hfi1_devdata {
bool aspm_enabled; /* ASPM state: enabled/disabled */
struct rhashtable *sdma_rht;
struct kobject kobj;
/* vnic data */
struct hfi1_vnic_data vnic;
/* Lock to protect IRQ SRC register access */

View File

@ -1198,13 +1198,13 @@ static void finalize_asic_data(struct hfi1_devdata *dd,
}
/**
* hfi1_clean_devdata - cleans up per-unit data structure
* hfi1_free_devdata - cleans up and frees per-unit data structure
* @dd: pointer to a valid devdata structure
*
* It cleans up all data structures set up by
* It cleans up and frees all data structures set up by
* by hfi1_alloc_devdata().
*/
static void hfi1_clean_devdata(struct hfi1_devdata *dd)
void hfi1_free_devdata(struct hfi1_devdata *dd)
{
struct hfi1_asic_data *ad;
unsigned long flags;
@ -1231,23 +1231,6 @@ static void hfi1_clean_devdata(struct hfi1_devdata *dd)
rvt_dealloc_device(&dd->verbs_dev.rdi);
}
static void __hfi1_free_devdata(struct kobject *kobj)
{
struct hfi1_devdata *dd =
container_of(kobj, struct hfi1_devdata, kobj);
hfi1_clean_devdata(dd);
}
static struct kobj_type hfi1_devdata_type = {
.release = __hfi1_free_devdata,
};
void hfi1_free_devdata(struct hfi1_devdata *dd)
{
kobject_put(&dd->kobj);
}
/**
* hfi1_alloc_devdata - Allocate our primary per-unit data structure.
* @pdev: Valid PCI device
@ -1333,11 +1316,10 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
goto bail;
}
kobject_init(&dd->kobj, &hfi1_devdata_type);
return dd;
bail:
hfi1_clean_devdata(dd);
hfi1_free_devdata(dd);
return ERR_PTR(ret);
}

View File

@ -2381,7 +2381,7 @@ struct opa_port_status_rsp {
__be64 port_vl_rcv_bubble;
__be64 port_vl_mark_fecn;
__be64 port_vl_xmit_discards;
} vls[0]; /* real array size defined by # bits set in vl_select_mask */
} vls[]; /* real array size defined by # bits set in vl_select_mask */
};
enum counter_selects {
@ -2423,7 +2423,7 @@ struct opa_aggregate {
__be16 attr_id;
__be16 err_reqlength; /* 1 bit, 8 res, 7 bit */
__be32 attr_mod;
u8 data[0];
u8 data[];
};
#define MSK_LLI 0x000000f0

View File

@ -165,7 +165,7 @@ struct opa_mad_notice_attr {
} __packed ntc_2048;
};
u8 class_data[0];
u8 class_data[];
};
#define IB_VLARB_LOWPRI_0_31 1

View File

@ -243,7 +243,7 @@ struct sc_config_sizes {
*/
struct pio_map_elem {
u32 mask;
struct send_context *ksc[0];
struct send_context *ksc[];
};
/*
@ -263,7 +263,7 @@ struct pio_vl_map {
u32 mask;
u8 actual_vls;
u8 vls;
struct pio_map_elem *map[0];
struct pio_map_elem *map[];
};
int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls,

View File

@ -833,7 +833,7 @@ struct sdma_engine *sdma_select_engine_sc(
struct sdma_rht_map_elem {
u32 mask;
u8 ctr;
struct sdma_engine *sde[0];
struct sdma_engine *sde[];
};
struct sdma_rht_node {

View File

@ -1002,7 +1002,7 @@ void sdma_engine_interrupt(struct sdma_engine *sde, u64 status);
*/
struct sdma_map_elem {
u32 mask;
struct sdma_engine *sde[0];
struct sdma_engine *sde[];
};
/**
@ -1024,7 +1024,7 @@ struct sdma_vl_map {
u32 mask;
u8 actual_vls;
u8 vls;
struct sdma_map_elem *map[0];
struct sdma_map_elem *map[];
};
int sdma_map_init(

View File

@ -674,7 +674,11 @@ int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,
dd_dev_err(dd,
"Skipping sc2vl sysfs info, (err %d) port %u\n",
ret, port_num);
goto bail;
/*
* Based on the documentation for kobject_init_and_add(), the
* caller should call kobject_put even if this call fails.
*/
goto bail_sc2vl;
}
kobject_uevent(&ppd->sc2vl_kobj, KOBJ_ADD);
@ -684,7 +688,7 @@ int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,
dd_dev_err(dd,
"Skipping sl2sc sysfs info, (err %d) port %u\n",
ret, port_num);
goto bail_sc2vl;
goto bail_sl2sc;
}
kobject_uevent(&ppd->sl2sc_kobj, KOBJ_ADD);
@ -694,7 +698,7 @@ int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,
dd_dev_err(dd,
"Skipping vl2mtu sysfs info, (err %d) port %u\n",
ret, port_num);
goto bail_sl2sc;
goto bail_vl2mtu;
}
kobject_uevent(&ppd->vl2mtu_kobj, KOBJ_ADD);
@ -704,7 +708,7 @@ int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,
dd_dev_err(dd,
"Skipping Congestion Control sysfs info, (err %d) port %u\n",
ret, port_num);
goto bail_vl2mtu;
goto bail_cc;
}
kobject_uevent(&ppd->pport_cc_kobj, KOBJ_ADD);
@ -742,7 +746,6 @@ bail_sl2sc:
kobject_put(&ppd->sl2sc_kobj);
bail_sc2vl:
kobject_put(&ppd->sc2vl_kobj);
bail:
return ret;
}
@ -853,8 +856,13 @@ int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd)
return 0;
bail:
for (i = 0; i < dd->num_sdma; i++)
kobject_del(&dd->per_sdma[i].kobj);
/*
* The function kobject_put() will call kobject_del() if the kobject
* has been added successfully. The sysfs files created under the
* kobject directory will also be removed during the process.
*/
for (; i >= 0; i--)
kobject_put(&dd->per_sdma[i].kobj);
return ret;
}
@ -867,6 +875,10 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd)
struct hfi1_pportdata *ppd;
int i;
/* Unwind operations in hfi1_verbs_register_sysfs() */
for (i = 0; i < dd->num_sdma; i++)
kobject_put(&dd->per_sdma[i].kobj);
for (i = 0; i < dd->num_pports; i++) {
ppd = &dd->pport[i];

View File

@ -73,7 +73,7 @@ struct tid_rb_node {
dma_addr_t dma_addr;
bool freed;
unsigned int npages;
struct page *pages[0];
struct page *pages[];
};
static inline int num_user_pages(unsigned long addr,

View File

@ -257,8 +257,8 @@ static int create_user_cq(struct hns_roce_dev *hr_dev,
return ret;
}
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
(udata->outlen >= sizeof(*resp))) {
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB &&
udata->outlen >= offsetofend(typeof(*resp), cap_flags)) {
ret = hns_roce_db_map_user(context, udata, ucmd.db_addr,
&hr_cq->db);
if (ret) {
@ -321,8 +321,8 @@ static void destroy_user_cq(struct hns_roce_dev *hr_dev,
struct hns_roce_ucontext *context = rdma_udata_to_drv_context(
udata, struct hns_roce_ucontext, ibucontext);
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
(udata->outlen >= sizeof(*resp)))
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB &&
udata->outlen >= offsetofend(typeof(*resp), cap_flags))
hns_roce_db_unmap_user(context, &hr_cq->db);
hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);

View File

@ -641,6 +641,19 @@ struct hns_roce_rinl_buf {
u32 wqe_cnt;
};
enum {
HNS_ROCE_FLUSH_FLAG = 0,
};
struct hns_roce_work {
struct hns_roce_dev *hr_dev;
struct work_struct work;
u32 qpn;
u32 cqn;
int event_type;
int sub_type;
};
struct hns_roce_qp {
struct ib_qp ibqp;
struct hns_roce_buf hr_buf;
@ -656,11 +669,6 @@ struct hns_roce_qp {
struct ib_umem *umem;
struct hns_roce_mtt mtt;
struct hns_roce_mtr mtr;
/* this define must less than HNS_ROCE_MAX_BT_REGION */
#define HNS_ROCE_WQE_REGION_MAX 3
struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX];
int region_cnt;
int wqe_bt_pg_shift;
u32 buff_size;
@ -684,6 +692,9 @@ struct hns_roce_qp {
struct hns_roce_sge sge;
u32 next_sge;
/* 0: flush needed, 1: unneeded */
unsigned long flush_flag;
struct hns_roce_work flush_work;
struct hns_roce_rinl_buf rq_inl_buf;
struct list_head node; /* all qps are on a list */
struct list_head rq_node; /* all recv qps are on a list */
@ -762,14 +773,8 @@ struct hns_roce_eq {
int eqe_ba_pg_sz;
int eqe_buf_pg_sz;
int hop_num;
u64 *bt_l0; /* Base address table for L0 */
u64 **bt_l1; /* Base address table for L1 */
u64 **buf;
dma_addr_t l0_dma;
dma_addr_t *l1_dma;
dma_addr_t *buf_dma;
u32 l0_last_num; /* L0 last chunk num */
u32 l1_last_num; /* L1 last chunk num */
struct hns_roce_mtr mtr;
struct hns_roce_buf buf;
int eq_max_cnt;
int eq_period;
int shift;
@ -881,7 +886,7 @@ struct hns_roce_caps {
u32 cqc_timer_ba_pg_sz;
u32 cqc_timer_buf_pg_sz;
u32 cqc_timer_hop_num;
u32 cqe_ba_pg_sz;
u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */
u32 cqe_buf_pg_sz;
u32 cqe_hop_num;
u32 srqwqe_ba_pg_sz;
@ -906,15 +911,6 @@ struct hns_roce_caps {
u16 default_ceq_arm_st;
};
struct hns_roce_work {
struct hns_roce_dev *hr_dev;
struct work_struct work;
u32 qpn;
u32 cqn;
int event_type;
int sub_type;
};
struct hns_roce_dfx_hw {
int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
int *buffer);
@ -1237,9 +1233,10 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
struct ib_udata *udata);
int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata);
void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n);
void *get_send_wqe(struct hns_roce_qp *hr_qp, int n);
void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n);
void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n);
void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n);
void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n);
bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
struct ib_cq *ib_cq);
enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
@ -1248,9 +1245,8 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,
void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
struct hns_roce_cq *recv_cq);
void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
int cnt);
void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
struct ib_udata *udata);
__be32 send_ieth(const struct ib_send_wr *wr);
int to_hr_qp_type(int qp_type);

View File

@ -39,6 +39,16 @@
#define DMA_ADDR_T_SHIFT 12
#define BT_BA_SHIFT 32
#define HEM_INDEX_BUF BIT(0)
#define HEM_INDEX_L0 BIT(1)
#define HEM_INDEX_L1 BIT(2)
struct hns_roce_hem_index {
u64 buf;
u64 l0;
u64 l1;
u32 inited; /* indicate which index is available */
};
bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
{
int hop_num = 0;
@ -84,25 +94,27 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
return hop_num ? true : false;
}
static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
u32 bt_chunk_num, u64 hem_max_num)
static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
u32 bt_chunk_num, u64 hem_max_num)
{
u64 start_idx = round_down(hem_idx, bt_chunk_num);
u64 check_max_num = start_idx + bt_chunk_num;
u64 i;
for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
if (hem[i])
if (i != hem_idx && hem[i])
return false;
return true;
}
static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
{
u64 start_idx = round_down(ba_idx, bt_chunk_num);
int i;
for (i = 0; i < bt_chunk_num; i++)
if (bt[start_idx + i])
if (i != ba_idx && bt[start_idx + i])
return false;
return true;
@ -434,178 +446,235 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
return ret;
}
static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
unsigned long obj)
static int calc_hem_config(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, unsigned long obj,
struct hns_roce_hem_mhop *mhop,
struct hns_roce_hem_index *index)
{
struct device *dev = hr_dev->dev;
struct hns_roce_hem_mhop mhop;
struct hns_roce_hem_iter iter;
u32 buf_chunk_size;
u32 bt_chunk_size;
u32 chunk_ba_num;
u32 hop_num;
u32 size;
u32 bt_num;
u64 hem_idx;
u64 bt_l1_idx = 0;
u64 bt_l0_idx = 0;
u64 bt_ba;
struct ib_device *ibdev = &hr_dev->ib_dev;
unsigned long mhop_obj = obj;
int bt_l1_allocated = 0;
int bt_l0_allocated = 0;
int step_idx;
u32 l0_idx, l1_idx, l2_idx;
u32 chunk_ba_num;
u32 bt_num;
int ret;
ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
if (ret)
return ret;
buf_chunk_size = mhop.buf_chunk_size;
bt_chunk_size = mhop.bt_chunk_size;
hop_num = mhop.hop_num;
chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
bt_num = hns_roce_get_bt_num(table->type, hop_num);
l0_idx = mhop->l0_idx;
l1_idx = mhop->l1_idx;
l2_idx = mhop->l2_idx;
chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
switch (bt_num) {
case 3:
hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
bt_l0_idx = mhop.l0_idx;
index->l1 = l0_idx * chunk_ba_num + l1_idx;
index->l0 = l0_idx;
index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
l1_idx * chunk_ba_num + l2_idx;
break;
case 2:
hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
bt_l0_idx = mhop.l0_idx;
index->l0 = l0_idx;
index->buf = l0_idx * chunk_ba_num + l1_idx;
break;
case 1:
hem_idx = mhop.l0_idx;
index->buf = l0_idx;
break;
default:
dev_err(dev, "Table %d not support hop_num = %d!\n",
table->type, hop_num);
ibdev_err(ibdev, "Table %d not support mhop.hop_num = %d!\n",
table->type, mhop->hop_num);
return -EINVAL;
}
if (unlikely(hem_idx >= table->num_hem)) {
dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
table->type, hem_idx, table->num_hem);
if (unlikely(index->buf >= table->num_hem)) {
ibdev_err(ibdev, "Table %d exceed hem limt idx %llu,max %lu!\n",
table->type, index->buf, table->num_hem);
return -EINVAL;
}
mutex_lock(&table->mutex);
return 0;
}
if (table->hem[hem_idx]) {
++table->hem[hem_idx]->refcount;
goto out;
static void free_mhop_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
struct hns_roce_hem_mhop *mhop,
struct hns_roce_hem_index *index)
{
u32 bt_size = mhop->bt_chunk_size;
struct device *dev = hr_dev->dev;
if (index->inited & HEM_INDEX_BUF) {
hns_roce_free_hem(hr_dev, table->hem[index->buf]);
table->hem[index->buf] = NULL;
}
if (index->inited & HEM_INDEX_L1) {
dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
table->bt_l1_dma_addr[index->l1]);
table->bt_l1[index->l1] = NULL;
}
if (index->inited & HEM_INDEX_L0) {
dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
table->bt_l0_dma_addr[index->l0]);
table->bt_l0[index->l0] = NULL;
}
}
static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
struct hns_roce_hem_mhop *mhop,
struct hns_roce_hem_index *index)
{
u32 bt_size = mhop->bt_chunk_size;
struct device *dev = hr_dev->dev;
struct hns_roce_hem_iter iter;
gfp_t flag;
u64 bt_ba;
u32 size;
int ret;
/* alloc L1 BA's chunk */
if ((check_whether_bt_num_3(table->type, hop_num) ||
check_whether_bt_num_2(table->type, hop_num)) &&
!table->bt_l0[bt_l0_idx]) {
table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
&(table->bt_l0_dma_addr[bt_l0_idx]),
if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
check_whether_bt_num_2(table->type, mhop->hop_num)) &&
!table->bt_l0[index->l0]) {
table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
&table->bt_l0_dma_addr[index->l0],
GFP_KERNEL);
if (!table->bt_l0[bt_l0_idx]) {
if (!table->bt_l0[index->l0]) {
ret = -ENOMEM;
goto out;
}
bt_l0_allocated = 1;
/* set base address to hardware */
if (table->type < HEM_TYPE_MTT) {
step_idx = 0;
if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
ret = -ENODEV;
dev_err(dev, "set HEM base address to HW failed!\n");
goto err_dma_alloc_l1;
}
}
index->inited |= HEM_INDEX_L0;
}
/* alloc L2 BA's chunk */
if (check_whether_bt_num_3(table->type, hop_num) &&
!table->bt_l1[bt_l1_idx]) {
table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
&(table->bt_l1_dma_addr[bt_l1_idx]),
if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
!table->bt_l1[index->l1]) {
table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
&table->bt_l1_dma_addr[index->l1],
GFP_KERNEL);
if (!table->bt_l1[bt_l1_idx]) {
if (!table->bt_l1[index->l1]) {
ret = -ENOMEM;
goto err_dma_alloc_l1;
}
bt_l1_allocated = 1;
*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
table->bt_l1_dma_addr[bt_l1_idx];
/* set base address to hardware */
step_idx = 1;
if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
ret = -ENODEV;
dev_err(dev, "set HEM base address to HW failed!\n");
goto err_alloc_hem_buf;
goto err_alloc_hem;
}
index->inited |= HEM_INDEX_L1;
*(table->bt_l0[index->l0] + mhop->l1_idx) =
table->bt_l1_dma_addr[index->l1];
}
/*
* alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
* alloc bt space chunk for MTT/CQE.
*/
size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
size >> PAGE_SHIFT,
size,
(table->lowmem ? GFP_KERNEL :
GFP_HIGHUSER) | __GFP_NOWARN);
if (!table->hem[hem_idx]) {
size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
size, flag);
if (!table->hem[index->buf]) {
ret = -ENOMEM;
goto err_alloc_hem_buf;
goto err_alloc_hem;
}
hns_roce_hem_first(table->hem[hem_idx], &iter);
index->inited |= HEM_INDEX_BUF;
hns_roce_hem_first(table->hem[index->buf], &iter);
bt_ba = hns_roce_hem_addr(&iter);
if (table->type < HEM_TYPE_MTT) {
if (hop_num == 2) {
*(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
step_idx = 2;
} else if (hop_num == 1) {
*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
step_idx = 1;
} else if (hop_num == HNS_ROCE_HOP_NUM_0) {
step_idx = 0;
} else {
ret = -EINVAL;
goto err_dma_alloc_l1;
}
/* set HEM base address to hardware */
if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
ret = -ENODEV;
dev_err(dev, "set HEM base address to HW failed!\n");
goto err_alloc_hem_buf;
}
} else if (hop_num == 2) {
*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
if (mhop->hop_num == 2)
*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
else if (mhop->hop_num == 1)
*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
} else if (mhop->hop_num == 2) {
*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
}
++table->hem[hem_idx]->refcount;
return 0;
err_alloc_hem:
free_mhop_hem(hr_dev, table, mhop, index);
out:
return ret;
}
static int set_mhop_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, unsigned long obj,
struct hns_roce_hem_mhop *mhop,
struct hns_roce_hem_index *index)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
int step_idx;
int ret = 0;
if (index->inited & HEM_INDEX_L0) {
ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
if (ret) {
ibdev_err(ibdev, "set HEM step 0 failed!\n");
goto out;
}
}
if (index->inited & HEM_INDEX_L1) {
ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
if (ret) {
ibdev_err(ibdev, "set HEM step 1 failed!\n");
goto out;
}
}
if (index->inited & HEM_INDEX_BUF) {
if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
step_idx = 0;
else
step_idx = mhop->hop_num;
ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
if (ret)
ibdev_err(ibdev, "set HEM step last failed!\n");
}
out:
return ret;
}
static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
unsigned long obj)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_hem_index index = {};
struct hns_roce_hem_mhop mhop = {};
int ret;
ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
if (ret) {
ibdev_err(ibdev, "calc hem config failed!\n");
return ret;
}
mutex_lock(&table->mutex);
if (table->hem[index.buf]) {
++table->hem[index.buf]->refcount;
goto out;
}
ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
if (ret) {
ibdev_err(ibdev, "alloc mhop hem failed!\n");
goto out;
}
/* set HEM base address to hardware */
if (table->type < HEM_TYPE_MTT) {
ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
if (ret) {
ibdev_err(ibdev, "set HEM address to HW failed!\n");
goto err_alloc;
}
}
++table->hem[index.buf]->refcount;
goto out;
err_alloc_hem_buf:
if (bt_l1_allocated) {
dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
table->bt_l1_dma_addr[bt_l1_idx]);
table->bt_l1[bt_l1_idx] = NULL;
}
err_dma_alloc_l1:
if (bt_l0_allocated) {
dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
table->bt_l0_dma_addr[bt_l0_idx]);
table->bt_l0[bt_l0_idx] = NULL;
}
err_alloc:
free_mhop_hem(hr_dev, table, &mhop, &index);
out:
mutex_unlock(&table->mutex);
return ret;
@ -656,116 +725,75 @@ out:
return ret;
}
static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, unsigned long obj,
struct hns_roce_hem_mhop *mhop,
struct hns_roce_hem_index *index)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
u32 hop_num = mhop->hop_num;
u32 chunk_ba_num;
int step_idx;
index->inited = HEM_INDEX_BUF;
chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
if (check_whether_bt_num_2(table->type, hop_num)) {
if (hns_roce_check_hem_null(table->hem, index->buf,
chunk_ba_num, table->num_hem))
index->inited |= HEM_INDEX_L0;
} else if (check_whether_bt_num_3(table->type, hop_num)) {
if (hns_roce_check_hem_null(table->hem, index->buf,
chunk_ba_num, table->num_hem)) {
index->inited |= HEM_INDEX_L1;
if (hns_roce_check_bt_null(table->bt_l1, index->l1,
chunk_ba_num))
index->inited |= HEM_INDEX_L0;
}
}
if (table->type < HEM_TYPE_MTT) {
if (hop_num == HNS_ROCE_HOP_NUM_0)
step_idx = 0;
else
step_idx = hop_num;
if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
ibdev_warn(ibdev, "Clear hop%d HEM failed.\n", hop_num);
if (index->inited & HEM_INDEX_L1)
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
ibdev_warn(ibdev, "Clear HEM step 1 failed.\n");
if (index->inited & HEM_INDEX_L0)
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
ibdev_warn(ibdev, "Clear HEM step 0 failed.\n");
}
}
static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table,
unsigned long obj,
int check_refcount)
{
struct device *dev = hr_dev->dev;
struct hns_roce_hem_mhop mhop;
unsigned long mhop_obj = obj;
u32 bt_chunk_size;
u32 chunk_ba_num;
u32 hop_num;
u32 start_idx;
u32 bt_num;
u64 hem_idx;
u64 bt_l1_idx = 0;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_hem_index index = {};
struct hns_roce_hem_mhop mhop = {};
int ret;
ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
if (ret)
return;
bt_chunk_size = mhop.bt_chunk_size;
hop_num = mhop.hop_num;
chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
bt_num = hns_roce_get_bt_num(table->type, hop_num);
switch (bt_num) {
case 3:
hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
break;
case 2:
hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
break;
case 1:
hem_idx = mhop.l0_idx;
break;
default:
dev_err(dev, "Table %d not support hop_num = %d!\n",
table->type, hop_num);
ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
if (ret) {
ibdev_err(ibdev, "calc hem config failed!\n");
return;
}
mutex_lock(&table->mutex);
if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
if (check_refcount && (--table->hem[index.buf]->refcount > 0)) {
mutex_unlock(&table->mutex);
return;
}
if (table->type < HEM_TYPE_MTT && hop_num == 1) {
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
dev_warn(dev, "Clear HEM base address failed.\n");
} else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
dev_warn(dev, "Clear HEM base address failed.\n");
} else if (table->type < HEM_TYPE_MTT &&
hop_num == HNS_ROCE_HOP_NUM_0) {
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
dev_warn(dev, "Clear HEM base address failed.\n");
}
/*
* free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
* free bt space chunk for MTT/CQE.
*/
hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
table->hem[hem_idx] = NULL;
if (check_whether_bt_num_2(table->type, hop_num)) {
start_idx = mhop.l0_idx * chunk_ba_num;
if (hns_roce_check_hem_null(table->hem, start_idx,
chunk_ba_num, table->num_hem)) {
if (table->type < HEM_TYPE_MTT &&
hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
dev_warn(dev, "Clear HEM base address failed.\n");
dma_free_coherent(dev, bt_chunk_size,
table->bt_l0[mhop.l0_idx],
table->bt_l0_dma_addr[mhop.l0_idx]);
table->bt_l0[mhop.l0_idx] = NULL;
}
} else if (check_whether_bt_num_3(table->type, hop_num)) {
start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
mhop.l1_idx * chunk_ba_num;
if (hns_roce_check_hem_null(table->hem, start_idx,
chunk_ba_num, table->num_hem)) {
if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
dev_warn(dev, "Clear HEM base address failed.\n");
dma_free_coherent(dev, bt_chunk_size,
table->bt_l1[bt_l1_idx],
table->bt_l1_dma_addr[bt_l1_idx]);
table->bt_l1[bt_l1_idx] = NULL;
start_idx = mhop.l0_idx * chunk_ba_num;
if (hns_roce_check_bt_null(table->bt_l1, start_idx,
chunk_ba_num)) {
if (hr_dev->hw->clear_hem(hr_dev, table, obj,
0))
dev_warn(dev, "Clear HEM base address failed.\n");
dma_free_coherent(dev, bt_chunk_size,
table->bt_l0[mhop.l0_idx],
table->bt_l0_dma_addr[mhop.l0_idx]);
table->bt_l0[mhop.l0_idx] = NULL;
}
}
}
clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
free_mhop_hem(hr_dev, table, &mhop, &index);
mutex_unlock(&table->mutex);
}
@ -1383,6 +1411,7 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
void *cpu_base;
u64 phy_base;
int ret = 0;
int ba_num;
int offset;
int total;
int step;
@ -1393,12 +1422,16 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
if (root_hem)
return 0;
ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
if (ba_num < 1)
return -ENOMEM;
INIT_LIST_HEAD(&temp_root);
total = r->offset;
offset = r->offset;
/* indicate to last region */
r = &regions[region_cnt - 1];
root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1,
unit, true, 0);
root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
ba_num, true, 0);
if (!root_hem)
return -ENOMEM;
list_add(&root_hem->list, &temp_root);
@ -1410,7 +1443,7 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
INIT_LIST_HEAD(&temp_list[i]);
total = 0;
for (i = 0; i < region_cnt && total < unit; i++) {
for (i = 0; i < region_cnt && total < ba_num; i++) {
r = &regions[i];
if (!r->count)
continue;
@ -1443,7 +1476,8 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
/* if exist mid bt, link L1 to L0 */
list_for_each_entry_safe(hem, temp_hem,
&hem_list->mid_bt[i][1], list) {
offset = hem->start / step * BA_BYTE_LEN;
offset = (hem->start - r->offset) / step *
BA_BYTE_LEN;
hem_list_link_bt(hr_dev, cpu_base + offset,
hem->dma_addr);
total++;

View File

@ -69,7 +69,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp,
struct hns_roce_wqe_data_seg *dseg = NULL;
struct hns_roce_qp *qp = to_hr_qp(ibqp);
struct device *dev = &hr_dev->pdev->dev;
struct hns_roce_sq_db sq_db;
struct hns_roce_sq_db sq_db = {};
int ps_opcode = 0, i = 0;
unsigned long flags = 0;
void *wqe = NULL;
@ -106,7 +106,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp,
goto out;
}
wqe = get_send_wqe(qp, wqe_idx);
wqe = hns_roce_get_send_wqe(qp, wqe_idx);
qp->sq.wrid[wqe_idx] = wr->wr_id;
/* Corresponding to the RC and RD type wqe process separately */
@ -318,8 +318,6 @@ out:
/* Memory barrier */
wmb();
sq_db.u32_4 = 0;
sq_db.u32_8 = 0;
roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
SQ_DOORBELL_U32_4_SQ_HEAD_S,
(qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
@ -351,7 +349,7 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp,
struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
struct device *dev = &hr_dev->pdev->dev;
struct hns_roce_rq_db rq_db;
struct hns_roce_rq_db rq_db = {};
__le32 doorbell[2] = {0};
unsigned long flags = 0;
unsigned int wqe_idx;
@ -380,7 +378,7 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp,
goto out;
}
ctrl = get_recv_wqe(hr_qp, wqe_idx);
ctrl = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
roce_set_field(ctrl->rwqe_byte_12,
RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
@ -418,9 +416,6 @@ out:
ROCEE_QP1C_CFG3_0_REG +
QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
} else {
rq_db.u32_4 = 0;
rq_db.u32_8 = 0;
roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
RQ_DOORBELL_U32_4_RQ_HEAD_S,
hr_qp->rq.head);
@ -2289,9 +2284,10 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
if (is_send) {
/* SQ conrespond to CQE */
sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
sq_wqe = hns_roce_get_send_wqe(*cur_qp,
roce_get_field(cqe->cqe_byte_4,
CQE_BYTE_4_WQE_INDEX_M,
CQE_BYTE_4_WQE_INDEX_S)&
CQE_BYTE_4_WQE_INDEX_S) &
((*cur_qp)->sq.wqe_cnt-1));
switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) {
case HNS_ROCE_WQE_OPCODE_SEND:
@ -3623,26 +3619,11 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
if (send_cq && send_cq != recv_cq)
__hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
}
hns_roce_qp_remove(hr_dev, hr_qp);
hns_roce_unlock_cqs(send_cq, recv_cq);
hns_roce_qp_remove(hr_dev, hr_qp);
hns_roce_qp_free(hr_dev, hr_qp);
hns_roce_qp_destroy(hr_dev, hr_qp, udata);
/* RC QP, release QPN */
if (hr_qp->ibqp.qp_type == IB_QPT_RC)
hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
ib_umem_release(hr_qp->umem);
if (!udata) {
kfree(hr_qp->sq.wrid);
kfree(hr_qp->rq.wrid);
hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
}
kfree(hr_qp);
return 0;
}
@ -3954,10 +3935,8 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev,
eq->cons_index++;
aeqes_found = 1;
if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
dev_warn(dev, "cons_index overflow, set back to 0.\n");
if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1)
eq->cons_index = 0;
}
}
set_eq_cons_index_v1(eq, 0);
@ -4007,11 +3986,8 @@ static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev,
ceqes_found = 1;
if (eq->cons_index >
EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1) {
dev_warn(&eq->hr_dev->pdev->dev,
"cons_index overflow, set back to 0.\n");
EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1)
eq->cons_index = 0;
}
}
set_eq_cons_index_v1(eq, 0);

File diff suppressed because it is too large Load Diff

View File

@ -50,15 +50,14 @@
#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000
#define HNS_ROCE_V2_MAX_SRQ 0x100000
#define HNS_ROCE_V2_MAX_SRQ_WR 0x8000
#define HNS_ROCE_V2_MAX_SRQ_SGE 0x100
#define HNS_ROCE_V2_MAX_SRQ_SGE 64
#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000
#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100
#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000
#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000
#define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000
#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 0x100
#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 0xff
#define HNS_ROCE_V2_MAX_SRQ_SGE_NUM 0x100
#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64
#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64
#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000
#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20
#define HNS_ROCE_V2_UAR_NUM 256
@ -163,7 +162,7 @@ enum {
#define GID_LEN_V2 16
#define HNS_ROCE_V2_CQE_QPN_MASK 0x3ffff
#define HNS_ROCE_V2_CQE_QPN_MASK 0xfffff
enum {
HNS_ROCE_V2_WQE_OP_SEND = 0x0,
@ -460,8 +459,8 @@ enum hns_roce_v2_qp_state {
HNS_ROCE_QP_ST_INIT,
HNS_ROCE_QP_ST_RTR,
HNS_ROCE_QP_ST_RTS,
HNS_ROCE_QP_ST_SQER,
HNS_ROCE_QP_ST_SQD,
HNS_ROCE_QP_ST_SQER,
HNS_ROCE_QP_ST_ERR,
HNS_ROCE_QP_ST_SQ_DRAINING,
HNS_ROCE_QP_NUM_ST
@ -1056,11 +1055,6 @@ struct hns_roce_v2_mpt_entry {
#define V2_DB_PARAMETER_SL_S 16
#define V2_DB_PARAMETER_SL_M GENMASK(18, 16)
struct hns_roce_v2_cq_db {
__le32 byte_4;
__le32 parameter;
};
#define V2_CQ_DB_BYTE_4_TAG_S 0
#define V2_CQ_DB_BYTE_4_TAG_M GENMASK(23, 0)

View File

@ -243,7 +243,7 @@ int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
/* Allocate MTT entry */
ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
mtt->mtt_type);
if (ret == -1)
if (ret)
return -ENOMEM;
return 0;

View File

@ -60,14 +60,12 @@ void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev)
int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct ib_device *ib_dev = ibpd->device;
struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
struct device *dev = hr_dev->dev;
struct hns_roce_pd *pd = to_hr_pd(ibpd);
int ret;
ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &pd->pdn);
if (ret) {
dev_err(dev, "[alloc_pd]hns_roce_pd_alloc failed!\n");
ibdev_err(ib_dev, "failed to alloc pd, ret = %d\n", ret);
return ret;
}
@ -76,7 +74,7 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
hns_roce_pd_free(to_hr_dev(ib_dev), pd->pdn);
dev_err(dev, "[alloc_pd]ib_copy_to_udata failed!\n");
ibdev_err(ib_dev, "failed to copy to udata\n");
return -EFAULT;
}
}

File diff suppressed because it is too large Load Diff

View File

@ -381,7 +381,8 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1);
srq->max_gs = init_attr->attr.max_sge;
srq_desc_size = roundup_pow_of_two(max(16, 16 * srq->max_gs));
srq_desc_size = roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,
HNS_ROCE_SGE_SIZE * srq->max_gs));
srq->wqe_shift = ilog2(srq_desc_size);

View File

@ -67,7 +67,7 @@
#include "i40iw_user.h"
#include "i40iw_puda.h"
#define I40IW_FW_VERSION 2
#define I40IW_FW_VER_DEFAULT 2
#define I40IW_HW_VERSION 2
#define I40IW_ARP_ADD 1
@ -325,6 +325,26 @@ struct i40iw_handler {
struct i40e_info ldev;
};
/**
* i40iw_fw_major_ver - get firmware major version
* @dev: iwarp device
**/
static inline u64 i40iw_fw_major_ver(struct i40iw_sc_dev *dev)
{
return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO],
I40IW_FW_VER_MAJOR);
}
/**
* i40iw_fw_minor_ver - get firmware minor version
* @dev: iwarp device
**/
static inline u64 i40iw_fw_minor_ver(struct i40iw_sc_dev *dev)
{
return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO],
I40IW_FW_VER_MINOR);
}
/**
* to_iwdev - get device
* @ibdev: ib device

View File

@ -85,7 +85,7 @@ struct ietf_mpa_v1 {
u8 flags;
u8 rev;
__be16 priv_data_len;
u8 priv_data[0];
u8 priv_data[];
};
#define ietf_mpa_req_resp_frame ietf_mpa_frame
@ -101,7 +101,7 @@ struct ietf_mpa_v2 {
u8 rev;
__be16 priv_data_len;
struct ietf_rtr_msg rtr_msg;
u8 priv_data[0];
u8 priv_data[];
};
struct i40iw_cm_node;

View File

@ -1021,6 +1021,95 @@ static enum i40iw_status_code i40iw_sc_commit_fpm_values(
return ret_code;
}
/**
* i40iw_sc_query_rdma_features_done - poll cqp for query features done
* @cqp: struct for cqp hw
*/
static enum i40iw_status_code
i40iw_sc_query_rdma_features_done(struct i40iw_sc_cqp *cqp)
{
return i40iw_sc_poll_for_cqp_op_done(
cqp, I40IW_CQP_OP_QUERY_RDMA_FEATURES, NULL);
}
/**
* i40iw_sc_query_rdma_features - query rdma features
* @cqp: struct for cqp hw
* @feat_mem: holds PA for HW to use
* @scratch: u64 saved to be used during cqp completion
*/
static enum i40iw_status_code
i40iw_sc_query_rdma_features(struct i40iw_sc_cqp *cqp,
struct i40iw_dma_mem *feat_mem, u64 scratch)
{
u64 *wqe;
u64 header;
wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
if (wqe)
return I40IW_ERR_RING_FULL;
set_64bit_val(wqe, 32, feat_mem->pa);
header = LS_64(I40IW_CQP_OP_QUERY_RDMA_FEATURES, I40IW_CQPSQ_OPCODE) |
LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | feat_mem->size;
i40iw_insert_wqe_hdr(wqe, header);
i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY RDMA FEATURES WQE",
wqe, I40IW_CQP_WQE_SIZE * 8);
i40iw_sc_cqp_post_sq(cqp);
return 0;
}
/**
* i40iw_get_rdma_features - get RDMA features
* @dev - sc device struct
*/
enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev)
{
enum i40iw_status_code ret_code;
struct i40iw_dma_mem feat_buf;
u64 temp;
u16 byte_idx, feat_type, feat_cnt;
ret_code = i40iw_allocate_dma_mem(dev->hw, &feat_buf,
I40IW_FEATURE_BUF_SIZE,
I40IW_FEATURE_BUF_ALIGNMENT);
if (ret_code)
return I40IW_ERR_NO_MEMORY;
ret_code = i40iw_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
if (!ret_code)
ret_code = i40iw_sc_query_rdma_features_done(dev->cqp);
if (ret_code)
goto exit;
get_64bit_val(feat_buf.va, 0, &temp);
feat_cnt = RS_64(temp, I40IW_FEATURE_CNT);
if (feat_cnt < I40IW_MAX_FEATURES) {
ret_code = I40IW_ERR_INVALID_FEAT_CNT;
goto exit;
} else if (feat_cnt > I40IW_MAX_FEATURES) {
i40iw_debug(dev, I40IW_DEBUG_CQP,
"features buf size insufficient\n");
}
for (byte_idx = 0, feat_type = 0; feat_type < I40IW_MAX_FEATURES;
feat_type++, byte_idx += 8) {
get_64bit_val((u64 *)feat_buf.va, byte_idx, &temp);
dev->feature_info[feat_type] = RS_64(temp, I40IW_FEATURE_INFO);
}
exit:
i40iw_free_dma_mem(dev->hw, &feat_buf);
return ret_code;
}
/**
* i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
* @cqp: struct for cqp hw
@ -4265,6 +4354,13 @@ static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
true,
I40IW_CQP_WAIT_EVENT);
break;
case OP_QUERY_RDMA_FEATURES:
values_mem.pa = pcmdinfo->in.u.query_rdma_features.cap_pa;
values_mem.va = pcmdinfo->in.u.query_rdma_features.cap_va;
status = i40iw_sc_query_rdma_features(
pcmdinfo->in.u.query_rdma_features.cqp, &values_mem,
pcmdinfo->in.u.query_rdma_features.scratch);
break;
default:
status = I40IW_NOT_SUPPORTED;
break;

View File

@ -403,7 +403,7 @@
#define I40IW_CQP_OP_MANAGE_ARP 0x0f
#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10
#define I40IW_CQP_OP_MANAGE_PUSH_PAGES 0x11
#define I40IW_CQP_OP_MANAGE_PE_TEAM 0x12
#define I40IW_CQP_OP_QUERY_RDMA_FEATURES 0x12
#define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13
#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14
#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
@ -431,6 +431,24 @@
#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b
#define I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE 0x2d
#define I40IW_FEATURE_BUF_SIZE (8 * I40IW_MAX_FEATURES)
#define I40IW_FW_VER_MINOR_SHIFT 0
#define I40IW_FW_VER_MINOR_MASK \
(0xffffULL << I40IW_FW_VER_MINOR_SHIFT)
#define I40IW_FW_VER_MAJOR_SHIFT 16
#define I40IW_FW_VER_MAJOR_MASK \
(0xffffULL << I40IW_FW_VER_MAJOR_SHIFT)
#define I40IW_FEATURE_INFO_SHIFT 0
#define I40IW_FEATURE_INFO_MASK \
(0xffffULL << I40IW_FEATURE_INFO_SHIFT)
#define I40IW_FEATURE_CNT_SHIFT 32
#define I40IW_FEATURE_CNT_MASK \
(0xffffULL << I40IW_FEATURE_CNT_SHIFT)
#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16
#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff << I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT)
@ -1529,7 +1547,8 @@ enum i40iw_alignment {
I40IW_AEQ_ALIGNMENT = 0x100,
I40IW_CEQ_ALIGNMENT = 0x100,
I40IW_CQ0_ALIGNMENT = 0x100,
I40IW_SD_BUF_ALIGNMENT = 0x80
I40IW_SD_BUF_ALIGNMENT = 0x80,
I40IW_FEATURE_BUF_ALIGNMENT = 0x8
};
#define I40IW_WQE_SIZE_64 64
@ -1732,6 +1751,7 @@ enum i40iw_alignment {
#define OP_REQUESTED_COMMANDS 31
#define OP_COMPLETED_COMMANDS 32
#define OP_GEN_AE 33
#define OP_SIZE_CQP_STAT_ARRAY 34
#define OP_QUERY_RDMA_FEATURES 34
#define OP_SIZE_CQP_STAT_ARRAY 35
#endif

View File

@ -1212,22 +1212,19 @@ static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
{
struct net_device *dev;
struct in_device *idev;
bool got_lock = true;
u32 ip_addr;
if (!rtnl_trylock())
got_lock = false;
for_each_netdev(&init_net, dev) {
rcu_read_lock();
for_each_netdev_rcu(&init_net, dev) {
if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
(rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
(dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
(dev == iwdev->netdev)) && (READ_ONCE(dev->flags) & IFF_UP)) {
const struct in_ifaddr *ifa;
idev = in_dev_get(dev);
idev = __in_dev_get_rcu(dev);
if (!idev)
continue;
in_dev_for_each_ifa_rtnl(ifa, idev) {
in_dev_for_each_ifa_rcu(ifa, idev) {
i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
"IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
@ -1239,12 +1236,9 @@ static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
true,
I40IW_ARP_ADD);
}
in_dev_put(idev);
}
}
if (got_lock)
rtnl_unlock();
rcu_read_unlock();
}
/**
@ -1689,6 +1683,12 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
status = i40iw_setup_ceqs(iwdev, ldev);
if (status)
break;
status = i40iw_get_rdma_features(dev);
if (status)
dev->feature_info[I40IW_FEATURE_FW_INFO] =
I40IW_FW_VER_DEFAULT;
iwdev->init_state = CEQ_CREATED;
status = i40iw_initialize_hw_resources(iwdev);
if (status)

View File

@ -105,6 +105,7 @@ enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(struct i40iw_sc_cqp *
bool poll_registers);
enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count);
enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev);
void free_sd_mem(struct i40iw_sc_dev *dev);

View File

@ -95,7 +95,8 @@ enum i40iw_status_code {
I40IW_ERR_INVALID_MAC_ADDR = -65,
I40IW_ERR_BAD_STAG = -66,
I40IW_ERR_CQ_COMPL_ERROR = -67,
I40IW_ERR_QUEUE_DESTROYED = -68
I40IW_ERR_QUEUE_DESTROYED = -68,
I40IW_ERR_INVALID_FEAT_CNT = -69
};
#endif

View File

@ -234,6 +234,11 @@ enum i40iw_hw_stats_index_64b {
I40IW_HW_STAT_INDEX_MAX_64
};
enum i40iw_feature_type {
I40IW_FEATURE_FW_INFO = 0,
I40IW_MAX_FEATURES
};
struct i40iw_dev_hw_stats_offsets {
u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32];
u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64];
@ -501,6 +506,7 @@ struct i40iw_sc_dev {
const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops;
struct i40iw_hmc_fpm_misc hmc_fpm_misc;
u64 feature_info[I40IW_MAX_FEATURES];
u32 debug_mask;
u8 hmc_fn_id;
bool is_pf;
@ -1340,6 +1346,12 @@ struct cqp_info {
struct i40iw_sc_qp *qp;
u64 scratch;
} suspend_resume;
struct {
struct i40iw_sc_cqp *cqp;
void *cap_va;
u64 cap_pa;
u64 scratch;
} query_rdma_features;
} u;
};

View File

@ -64,7 +64,8 @@ static int i40iw_query_device(struct ib_device *ibdev,
return -EINVAL;
memset(props, 0, sizeof(*props));
ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);
props->fw_ver = I40IW_FW_VERSION;
props->fw_ver = i40iw_fw_major_ver(&iwdev->sc_dev) << 32 |
i40iw_fw_minor_ver(&iwdev->sc_dev);
props->device_cap_flags = iwdev->device_cap_flags;
props->vendor_id = iwdev->ldev->pcidev->vendor;
props->vendor_part_id = iwdev->ldev->pcidev->device;
@ -617,7 +618,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
if (init_attr->qp_type != IB_QPT_RC) {
err_code = -EINVAL;
err_code = -EOPNOTSUPP;
goto error;
}
if (iwdev->push_mode)
@ -2534,10 +2535,11 @@ static const char * const i40iw_hw_stat_names[] = {
static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str)
{
u32 firmware_version = I40IW_FW_VERSION;
struct i40iw_device *iwdev = to_iwdev(dev);
snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", firmware_version,
(firmware_version & 0x000000ff));
snprintf(str, IB_FW_VERSION_NAME_MAX, "%llu.%llu",
i40iw_fw_major_ver(&iwdev->sc_dev),
i40iw_fw_minor_ver(&iwdev->sc_dev));
}
/**

View File

@ -434,9 +434,6 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
return real_index;
}
#define field_avail(type, fld, sz) (offsetof(type, fld) + \
sizeof(((type *)0)->fld) <= (sz))
static int mlx4_ib_query_device(struct ib_device *ibdev,
struct ib_device_attr *props,
struct ib_udata *uhw)
@ -447,7 +444,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
int err;
int have_ib_ports;
struct mlx4_uverbs_ex_query_device cmd;
struct mlx4_uverbs_ex_query_device_resp resp = {.comp_mask = 0};
struct mlx4_uverbs_ex_query_device_resp resp = {};
struct mlx4_clock_params clock_params;
if (uhw->inlen) {
@ -602,7 +599,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
sizeof(struct mlx4_wqe_data_seg);
}
if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
if (offsetofend(typeof(resp), rss_caps) <= uhw->outlen) {
if (props->rss_caps.supported_qpts) {
resp.rss_caps.rx_hash_function =
MLX4_IB_RX_HASH_FUNC_TOEPLITZ;
@ -626,7 +623,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
sizeof(resp.rss_caps);
}
if (field_avail(typeof(resp), tso_caps, uhw->outlen)) {
if (offsetofend(typeof(resp), tso_caps) <= uhw->outlen) {
if (dev->dev->caps.max_gso_sz &&
((mlx4_ib_port_link_layer(ibdev, 1) ==
IB_LINK_LAYER_ETHERNET) ||

View File

@ -1636,7 +1636,7 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
}
default:
/* Don't support raw QPs */
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
}
return &qp->ibqp;

View File

@ -8,3 +8,4 @@ mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o
mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += flow.o
mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += qos.o

View File

@ -47,6 +47,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {
"rp_byte_reset",
"rp_threshold",
"rp_ai_rate",
"rp_max_rate",
"rp_hai_rate",
"rp_min_dec_fac",
"rp_min_rate",
@ -56,6 +57,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {
"rp_rate_reduce_monitor_period",
"rp_initial_alpha_value",
"rp_gd",
"np_min_time_between_cnps",
"np_cnp_dscp",
"np_cnp_prio_mode",
"np_cnp_prio",
@ -66,6 +68,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {
#define MLX5_IB_RP_TIME_RESET_ATTR BIT(3)
#define MLX5_IB_RP_BYTE_RESET_ATTR BIT(4)
#define MLX5_IB_RP_THRESHOLD_ATTR BIT(5)
#define MLX5_IB_RP_MAX_RATE_ATTR BIT(6)
#define MLX5_IB_RP_AI_RATE_ATTR BIT(7)
#define MLX5_IB_RP_HAI_RATE_ATTR BIT(8)
#define MLX5_IB_RP_MIN_DEC_FAC_ATTR BIT(9)
@ -77,6 +80,7 @@ static const char * const mlx5_ib_dbg_cc_name[] = {
#define MLX5_IB_RP_INITIAL_ALPHA_VALUE_ATTR BIT(15)
#define MLX5_IB_RP_GD_ATTR BIT(16)
#define MLX5_IB_NP_MIN_TIME_BETWEEN_CNPS_ATTR BIT(2)
#define MLX5_IB_NP_CNP_DSCP_ATTR BIT(3)
#define MLX5_IB_NP_CNP_PRIO_MODE_ATTR BIT(4)
@ -111,6 +115,9 @@ static u32 mlx5_get_cc_param_val(void *field, int offset)
case MLX5_IB_DBG_CC_RP_AI_RATE:
return MLX5_GET(cong_control_r_roce_ecn_rp, field,
rpg_ai_rate);
case MLX5_IB_DBG_CC_RP_MAX_RATE:
return MLX5_GET(cong_control_r_roce_ecn_rp, field,
rpg_max_rate);
case MLX5_IB_DBG_CC_RP_HAI_RATE:
return MLX5_GET(cong_control_r_roce_ecn_rp, field,
rpg_hai_rate);
@ -138,6 +145,9 @@ static u32 mlx5_get_cc_param_val(void *field, int offset)
case MLX5_IB_DBG_CC_RP_GD:
return MLX5_GET(cong_control_r_roce_ecn_rp, field,
rpg_gd);
case MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS:
return MLX5_GET(cong_control_r_roce_ecn_np, field,
min_time_between_cnps);
case MLX5_IB_DBG_CC_NP_CNP_DSCP:
return MLX5_GET(cong_control_r_roce_ecn_np, field,
cnp_dscp);
@ -186,6 +196,11 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,
MLX5_SET(cong_control_r_roce_ecn_rp, field,
rpg_ai_rate, var);
break;
case MLX5_IB_DBG_CC_RP_MAX_RATE:
*attr_mask |= MLX5_IB_RP_MAX_RATE_ATTR;
MLX5_SET(cong_control_r_roce_ecn_rp, field,
rpg_max_rate, var);
break;
case MLX5_IB_DBG_CC_RP_HAI_RATE:
*attr_mask |= MLX5_IB_RP_HAI_RATE_ATTR;
MLX5_SET(cong_control_r_roce_ecn_rp, field,
@ -231,6 +246,11 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,
MLX5_SET(cong_control_r_roce_ecn_rp, field,
rpg_gd, var);
break;
case MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS:
*attr_mask |= MLX5_IB_NP_MIN_TIME_BETWEEN_CNPS_ATTR;
MLX5_SET(cong_control_r_roce_ecn_np, field,
min_time_between_cnps, var);
break;
case MLX5_IB_DBG_CC_NP_CNP_DSCP:
*attr_mask |= MLX5_IB_NP_CNP_DSCP_ATTR;
MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_dscp, var);

View File

@ -715,17 +715,19 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
ucmdlen = udata->inlen < sizeof(ucmd) ?
(sizeof(ucmd) - sizeof(ucmd.flags)) : sizeof(ucmd);
ucmdlen = min(udata->inlen, sizeof(ucmd));
if (ucmdlen < offsetof(struct mlx5_ib_create_cq, flags))
return -EINVAL;
if (ib_copy_from_udata(&ucmd, udata, ucmdlen))
return -EFAULT;
if (ucmdlen == sizeof(ucmd) &&
(ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD)))
if ((ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD |
MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX)))
return -EINVAL;
if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128)
if ((ucmd.cqe_size != 64 && ucmd.cqe_size != 128) ||
ucmd.reserved0 || ucmd.reserved1)
return -EINVAL;
*cqe_size = ucmd.cqe_size;
@ -762,7 +764,14 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
MLX5_SET(cqc, cqc, log_page_size,
page_shift - MLX5_ADAPTER_PAGE_SHIFT);
*index = context->bfregi.sys_pages[0];
if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {
*index = ucmd.uar_page_index;
} else if (context->bfregi.lib_uar_dyn) {
err = -EINVAL;
goto err_cqb;
} else {
*index = context->bfregi.sys_pages[0];
}
if (ucmd.cqe_comp_en == 1) {
int mini_cqe_format;

View File

@ -35,6 +35,9 @@ mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
break;
case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
break;
default:
return -EINVAL;
}

View File

@ -39,9 +39,6 @@
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/bitmap.h>
#if defined(CONFIG_X86)
#include <asm/memtype.h>
#endif
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
@ -898,7 +895,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
props->raw_packet_caps |=
IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
if (field_avail(typeof(resp), tso_caps, uhw_outlen)) {
if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
if (max_tso) {
resp.tso_caps.max_tso = 1 << max_tso;
@ -908,7 +905,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
}
if (field_avail(typeof(resp), rss_caps, uhw_outlen)) {
if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
resp.rss_caps.rx_hash_function =
MLX5_RX_HASH_FUNC_TOEPLITZ;
resp.rss_caps.rx_hash_fields_mask =
@ -928,9 +925,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
resp.response_length += sizeof(resp.rss_caps);
}
} else {
if (field_avail(typeof(resp), tso_caps, uhw_outlen))
if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
resp.response_length += sizeof(resp.tso_caps);
if (field_avail(typeof(resp), rss_caps, uhw_outlen))
if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
resp.response_length += sizeof(resp.rss_caps);
}
@ -1072,7 +1069,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
MLX5_MAX_CQ_PERIOD;
}
if (field_avail(typeof(resp), cqe_comp_caps, uhw_outlen)) {
if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
resp.response_length += sizeof(resp.cqe_comp_caps);
if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
@ -1090,7 +1087,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
}
if (field_avail(typeof(resp), packet_pacing_caps, uhw_outlen) &&
if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
raw_support) {
if (MLX5_CAP_QOS(mdev, packet_pacing) &&
MLX5_CAP_GEN(mdev, qos)) {
@ -1108,8 +1105,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
resp.response_length += sizeof(resp.packet_pacing_caps);
}
if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
uhw_outlen)) {
if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
uhw_outlen) {
if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
resp.mlx5_ib_support_multi_pkt_send_wqes =
MLX5_IB_ALLOW_MPW;
@ -1122,7 +1119,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
}
if (field_avail(typeof(resp), flags, uhw_outlen)) {
if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
resp.response_length += sizeof(resp.flags);
if (MLX5_CAP_GEN(mdev, cqe_compression_128))
@ -1138,7 +1135,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
}
if (field_avail(typeof(resp), sw_parsing_caps, uhw_outlen)) {
if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
resp.response_length += sizeof(resp.sw_parsing_caps);
if (MLX5_CAP_ETH(mdev, swp)) {
resp.sw_parsing_caps.sw_parsing_offloads |=
@ -1158,7 +1155,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
}
if (field_avail(typeof(resp), striding_rq_caps, uhw_outlen) &&
if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
raw_support) {
resp.response_length += sizeof(resp.striding_rq_caps);
if (MLX5_CAP_GEN(mdev, striding_rq)) {
@ -1181,7 +1178,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
}
if (field_avail(typeof(resp), tunnel_offloads_caps, uhw_outlen)) {
if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
resp.response_length += sizeof(resp.tunnel_offloads_caps);
if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
resp.tunnel_offloads_caps |=
@ -1192,12 +1189,10 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
resp.tunnel_offloads_caps |=
MLX5_IB_TUNNELED_OFFLOADS_GRE;
if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
MLX5_FLEX_PROTO_CW_MPLS_GRE)
if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
resp.tunnel_offloads_caps |=
MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
MLX5_FLEX_PROTO_CW_MPLS_UDP)
if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
resp.tunnel_offloads_caps |=
MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
}
@ -1791,6 +1786,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
max_cqe_version);
u32 dump_fill_mkey;
bool lib_uar_4k;
bool lib_uar_dyn;
if (!dev->ib_active)
return -EAGAIN;
@ -1849,8 +1845,14 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
}
lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
bfregi = &context->bfregi;
if (lib_uar_dyn) {
bfregi->lib_uar_dyn = lib_uar_dyn;
goto uar_done;
}
/* updates req->total_num_bfregs */
err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
if (err)
@ -1877,6 +1879,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
if (err)
goto out_sys_pages;
uar_done:
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
err = mlx5_ib_devx_create(dev, true);
if (err < 0)
@ -1898,19 +1901,19 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex);
resp.tot_bfregs = req.total_num_bfregs;
resp.tot_bfregs = lib_uar_dyn ? 0 : req.total_num_bfregs;
resp.num_ports = dev->num_ports;
if (field_avail(typeof(resp), cqe_version, udata->outlen))
if (offsetofend(typeof(resp), cqe_version) <= udata->outlen)
resp.response_length += sizeof(resp.cqe_version);
if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
if (offsetofend(typeof(resp), cmds_supp_uhw) <= udata->outlen) {
resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
resp.response_length += sizeof(resp.cmds_supp_uhw);
}
if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
if (offsetofend(typeof(resp), eth_min_inline) <= udata->outlen) {
if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
resp.eth_min_inline++;
@ -1918,7 +1921,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
resp.response_length += sizeof(resp.eth_min_inline);
}
if (field_avail(typeof(resp), clock_info_versions, udata->outlen)) {
if (offsetofend(typeof(resp), clock_info_versions) <= udata->outlen) {
if (mdev->clock_info)
resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
resp.response_length += sizeof(resp.clock_info_versions);
@ -1930,7 +1933,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
* pretend we don't support reading the HCA's core clock. This is also
* forced by mmap function.
*/
if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
if (offsetofend(typeof(resp), hca_core_clock_offset) <= udata->outlen) {
if (PAGE_SIZE <= 4096) {
resp.comp_mask |=
MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
@ -1940,18 +1943,18 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
resp.response_length += sizeof(resp.hca_core_clock_offset);
}
if (field_avail(typeof(resp), log_uar_size, udata->outlen))
if (offsetofend(typeof(resp), log_uar_size) <= udata->outlen)
resp.response_length += sizeof(resp.log_uar_size);
if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
if (offsetofend(typeof(resp), num_uars_per_page) <= udata->outlen)
resp.response_length += sizeof(resp.num_uars_per_page);
if (field_avail(typeof(resp), num_dyn_bfregs, udata->outlen)) {
if (offsetofend(typeof(resp), num_dyn_bfregs) <= udata->outlen) {
resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
resp.response_length += sizeof(resp.num_dyn_bfregs);
}
if (field_avail(typeof(resp), dump_fill_mkey, udata->outlen)) {
if (offsetofend(typeof(resp), dump_fill_mkey) <= udata->outlen) {
if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
resp.dump_fill_mkey = dump_fill_mkey;
resp.comp_mask |=
@ -2026,6 +2029,17 @@ static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
}
static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
int uar_idx)
{
unsigned int fw_uars_per_page;
fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
MLX5_UARS_IN_PAGE : 1;
return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
}
static int get_command(unsigned long offset)
{
return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
@ -2110,6 +2124,11 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
mutex_unlock(&var_table->bitmap_lock);
kfree(mentry);
break;
case MLX5_IB_MMAP_TYPE_UAR_WC:
case MLX5_IB_MMAP_TYPE_UAR_NC:
mlx5_cmd_free_uar(dev->mdev, mentry->page_idx);
kfree(mentry);
break;
default:
WARN_ON(true);
}
@ -2130,6 +2149,9 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
bfregi->num_static_sys_pages;
if (bfregi->lib_uar_dyn)
return -EINVAL;
if (vma->vm_end - vma->vm_start != PAGE_SIZE)
return -EINVAL;
@ -2147,14 +2169,6 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
switch (cmd) {
case MLX5_IB_MMAP_WC_PAGE:
case MLX5_IB_MMAP_ALLOC_WC:
/* Some architectures don't support WC memory */
#if defined(CONFIG_X86)
if (!pat_enabled())
return -EPERM;
#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
return -EPERM;
#endif
/* fall through */
case MLX5_IB_MMAP_REGULAR_PAGE:
/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
prot = pgprot_writecombine(vma->vm_page_prot);
@ -2269,7 +2283,8 @@ static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
mentry = to_mmmap(entry);
pfn = (mentry->address >> PAGE_SHIFT);
if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR)
if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
prot = pgprot_noncached(vma->vm_page_prot);
else
prot = pgprot_writecombine(vma->vm_page_prot);
@ -2300,9 +2315,12 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
command = get_command(vma->vm_pgoff);
switch (command) {
case MLX5_IB_MMAP_WC_PAGE:
case MLX5_IB_MMAP_ALLOC_WC:
if (!dev->wc_support)
return -EPERM;
fallthrough;
case MLX5_IB_MMAP_NC_PAGE:
case MLX5_IB_MMAP_REGULAR_PAGE:
case MLX5_IB_MMAP_ALLOC_WC:
return uar_mmap(dev, command, vma, context);
case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
@ -4046,6 +4064,11 @@ _get_flow_table(struct mlx5_ib_dev *dev,
BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
log_max_ft_size));
priority = fs_matcher->priority;
} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
max_table_size =
BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
log_max_ft_size));
priority = fs_matcher->priority;
}
max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
@ -4062,6 +4085,8 @@ _get_flow_table(struct mlx5_ib_dev *dev,
prio = &dev->flow_db->fdb;
else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
prio = &dev->flow_db->rdma_rx[priority];
else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)
prio = &dev->flow_db->rdma_tx[priority];
if (!prio)
return ERR_PTR(-EINVAL);
@ -6090,9 +6115,9 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
mlx5_nic_vport_disable_roce(dev->mdev);
}
static int var_obj_cleanup(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
static int mmap_obj_cleanup(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct mlx5_user_mmap_entry *obj = uobject->object;
@ -6100,6 +6125,16 @@ static int var_obj_cleanup(struct ib_uobject *uobject,
return 0;
}
static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
struct mlx5_user_mmap_entry *entry,
size_t length)
{
return rdma_user_mmap_entry_insert_range(
&c->ibucontext, &entry->rdma_entry, length,
(MLX5_IB_MMAP_OFFSET_START << 16),
((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
}
static struct mlx5_user_mmap_entry *
alloc_var_entry(struct mlx5_ib_ucontext *c)
{
@ -6130,10 +6165,8 @@ alloc_var_entry(struct mlx5_ib_ucontext *c)
entry->page_idx = page_idx;
entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
err = rdma_user_mmap_entry_insert_range(
&c->ibucontext, &entry->rdma_entry, var_table->stride_size,
MLX5_IB_MMAP_OFFSET_START << 16,
(MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1);
err = mlx5_rdma_user_mmap_entry_insert(c, entry,
var_table->stride_size);
if (err)
goto err_insert;
@ -6217,7 +6250,7 @@ DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
UVERBS_TYPE_ALLOC_IDR(var_obj_cleanup),
UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
&UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
&UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
@ -6229,6 +6262,134 @@ static bool var_is_supported(struct ib_device *device)
MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
}
static struct mlx5_user_mmap_entry *
alloc_uar_entry(struct mlx5_ib_ucontext *c,
enum mlx5_ib_uapi_uar_alloc_type alloc_type)
{
struct mlx5_user_mmap_entry *entry;
struct mlx5_ib_dev *dev;
u32 uar_index;
int err;
entry = kzalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return ERR_PTR(-ENOMEM);
dev = to_mdev(c->ibucontext.device);
err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
if (err)
goto end;
entry->page_idx = uar_index;
entry->address = uar_index2paddress(dev, uar_index);
if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
else
entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
if (err)
goto err_insert;
return entry;
err_insert:
mlx5_cmd_free_uar(dev->mdev, uar_index);
end:
kfree(entry);
return ERR_PTR(err);
}
static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
enum mlx5_ib_uapi_uar_alloc_type alloc_type;
struct mlx5_ib_ucontext *c;
struct mlx5_user_mmap_entry *entry;
u64 mmap_offset;
u32 length;
int err;
c = to_mucontext(ib_uverbs_get_ucontext(attrs));
if (IS_ERR(c))
return PTR_ERR(c);
err = uverbs_get_const(&alloc_type, attrs,
MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
if (err)
return err;
if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
return -EOPNOTSUPP;
if (!to_mdev(c->ibucontext.device)->wc_support &&
alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
return -EOPNOTSUPP;
entry = alloc_uar_entry(c, alloc_type);
if (IS_ERR(entry))
return PTR_ERR(entry);
mmap_offset = mlx5_entry_to_mmap_offset(entry);
length = entry->rdma_entry.npages * PAGE_SIZE;
uobj->object = entry;
err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
&mmap_offset, sizeof(mmap_offset));
if (err)
goto err;
err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
&entry->page_idx, sizeof(entry->page_idx));
if (err)
goto err;
err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
&length, sizeof(length));
if (err)
goto err;
return 0;
err:
rdma_user_mmap_entry_remove(&entry->rdma_entry);
return err;
}
DECLARE_UVERBS_NAMED_METHOD(
MLX5_IB_METHOD_UAR_OBJ_ALLOC,
UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
MLX5_IB_OBJECT_UAR,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
enum mlx5_ib_uapi_uar_alloc_type,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
MLX5_IB_METHOD_UAR_OBJ_DESTROY,
UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
MLX5_IB_OBJECT_UAR,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
&UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
&UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
ADD_UVERBS_ATTRIBUTES_SIMPLE(
mlx5_ib_dm,
UVERBS_OBJECT_DM,
@ -6253,12 +6414,14 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE(
static const struct uapi_definition mlx5_ib_defs[] = {
UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
&mlx5_ib_flow_action),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
{}
};
@ -6392,7 +6555,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
spin_lock_init(&dev->reset_flow_resource_lock);
xa_init(&dev->odp_mkeys);
xa_init(&dev->sig_mrs);
spin_lock_init(&dev->mkey_lock);
atomic_set(&dev->mkey_var, 0);
spin_lock_init(&dev->dm.lock);
dev->dm.dev = mdev;
@ -6548,7 +6711,8 @@ static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
doorbell_bar_offset);
bar_size = (1ULL << log_doorbell_bar_size) * 4096;
var_table->stride_size = 1ULL << log_doorbell_stride;
var_table->num_var_hw_entries = div64_u64(bar_size, var_table->stride_size);
var_table->num_var_hw_entries = div_u64(bar_size,
var_table->stride_size);
mutex_init(&var_table->bitmap_lock);
var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
GFP_KERNEL);
@ -7080,6 +7244,9 @@ const struct mlx5_ib_profile raw_eth_profile = {
STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
mlx5_ib_stage_counters_init,
mlx5_ib_stage_counters_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
mlx5_ib_stage_cong_debugfs_init,
mlx5_ib_stage_cong_debugfs_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_UAR,
mlx5_ib_stage_uar_init,
mlx5_ib_stage_uar_cleanup),

View File

@ -316,7 +316,7 @@ int mlx5_ib_test_wc(struct mlx5_ib_dev *dev)
if (!dev->mdev->roce.roce_en &&
port_type_cap == MLX5_CAP_PORT_TYPE_ETH) {
if (mlx5_core_is_pf(dev->mdev))
dev->wc_support = true;
dev->wc_support = arch_can_pci_mmap_wc();
return 0;
}

View File

@ -64,8 +64,6 @@
dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
__LINE__, current->pid, ##arg)
#define field_avail(type, fld, sz) (offsetof(type, fld) + \
sizeof(((type *)0)->fld) <= (sz))
#define MLX5_IB_DEFAULT_UIDX 0xffffff
#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
@ -126,11 +124,27 @@ enum {
enum mlx5_ib_mmap_type {
MLX5_IB_MMAP_TYPE_MEMIC = 1,
MLX5_IB_MMAP_TYPE_VAR = 2,
MLX5_IB_MMAP_TYPE_UAR_WC = 3,
MLX5_IB_MMAP_TYPE_UAR_NC = 4,
};
#define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) \
(MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))
#define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
struct mlx5_bfreg_info {
u32 *sys_pages;
int num_low_latency_bfregs;
unsigned int *count;
/*
* protect bfreg allocation data structs
*/
struct mutex lock;
u32 ver;
u8 lib_uar_4k : 1;
u8 lib_uar_dyn : 1;
u32 num_sys_pages;
u32 num_static_sys_pages;
u32 total_num_bfregs;
u32 num_dyn_bfregs;
};
struct mlx5_ib_ucontext {
struct ib_ucontext ibucontext;
@ -203,6 +217,11 @@ struct mlx5_ib_flow_matcher {
u8 match_criteria_enable;
};
struct mlx5_ib_pp {
u16 index;
struct mlx5_core_dev *mdev;
};
struct mlx5_ib_flow_db {
struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
struct mlx5_ib_flow_prio egress_prios[MLX5_IB_NUM_FLOW_FT];
@ -210,6 +229,7 @@ struct mlx5_ib_flow_db {
struct mlx5_ib_flow_prio egress[MLX5_IB_NUM_EGRESS_FTS];
struct mlx5_ib_flow_prio fdb;
struct mlx5_ib_flow_prio rdma_rx[MLX5_IB_NUM_FLOW_FT];
struct mlx5_ib_flow_prio rdma_tx[MLX5_IB_NUM_FLOW_FT];
struct mlx5_flow_table *lag_demux_ft;
/* Protect flow steering bypass flow tables
* when add/del flow rules.
@ -618,8 +638,8 @@ struct mlx5_ib_mr {
struct ib_umem *umem;
struct mlx5_shared_mr_info *smr_info;
struct list_head list;
int order;
bool allocated_from_cache;
unsigned int order;
struct mlx5_cache_ent *cache_ent;
int npages;
struct mlx5_ib_dev *dev;
u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
@ -701,22 +721,34 @@ struct mlx5_cache_ent {
u32 access_mode;
u32 page;
u32 size;
u32 cur;
u8 disabled:1;
u8 fill_to_high_water:1;
/*
* - available_mrs is the length of list head, ie the number of MRs
* available for immediate allocation.
* - total_mrs is available_mrs plus all in use MRs that could be
* returned to the cache.
* - limit is the low water mark for available_mrs, 2* limit is the
* upper water mark.
* - pending is the number of MRs currently being created
*/
u32 total_mrs;
u32 available_mrs;
u32 limit;
u32 pending;
/* Statistics */
u32 miss;
u32 limit;
struct mlx5_ib_dev *dev;
struct work_struct work;
struct delayed_work dwork;
int pending;
struct completion compl;
};
struct mlx5_mr_cache {
struct workqueue_struct *wq;
struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
int stopped;
struct dentry *root;
unsigned long last_add;
};
@ -794,6 +826,7 @@ enum mlx5_ib_dbg_cc_types {
MLX5_IB_DBG_CC_RP_BYTE_RESET,
MLX5_IB_DBG_CC_RP_THRESHOLD,
MLX5_IB_DBG_CC_RP_AI_RATE,
MLX5_IB_DBG_CC_RP_MAX_RATE,
MLX5_IB_DBG_CC_RP_HAI_RATE,
MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,
MLX5_IB_DBG_CC_RP_MIN_RATE,
@ -803,6 +836,7 @@ enum mlx5_ib_dbg_cc_types {
MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,
MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,
MLX5_IB_DBG_CC_RP_GD,
MLX5_IB_DBG_CC_NP_MIN_TIME_BETWEEN_CNPS,
MLX5_IB_DBG_CC_NP_CNP_DSCP,
MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
MLX5_IB_DBG_CC_NP_CNP_PRIO,
@ -986,19 +1020,16 @@ struct mlx5_ib_dev {
*/
struct mutex cap_mask_mutex;
u8 ib_active:1;
u8 fill_delay:1;
u8 is_rep:1;
u8 lag_active:1;
u8 wc_support:1;
u8 fill_delay;
struct umr_common umrc;
/* sync used page count stats
*/
struct mlx5_ib_resources devr;
/* protect mkey key part */
spinlock_t mkey_lock;
u8 mkey_key;
atomic_t mkey_var;
struct mlx5_mr_cache cache;
struct timer_list delay_timer;
/* Prevents soft lock on massive reg MRs */
@ -1268,7 +1299,8 @@ int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
unsigned int entry);
void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr);
@ -1388,6 +1420,7 @@ int mlx5_ib_fill_stat_entry(struct sk_buff *msg,
extern const struct uapi_definition mlx5_ib_devx_defs[];
extern const struct uapi_definition mlx5_ib_flow_defs[];
extern const struct uapi_definition mlx5_ib_qos_defs[];
#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
@ -1477,12 +1510,11 @@ static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
{
u8 cqe_version = ucontext->cqe_version;
if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
!cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
if ((offsetofend(typeof(*ucmd), uidx) <= inlen) && !cqe_version &&
(ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
return 0;
if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
!!cqe_version))
if ((offsetofend(typeof(*ucmd), uidx) <= inlen) != !!cqe_version)
return -EINVAL;
return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
@ -1495,12 +1527,11 @@ static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
{
u8 cqe_version = ucontext->cqe_version;
if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
!cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
if ((offsetofend(typeof(*ucmd), uidx) <= inlen) && !cqe_version &&
(ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
return 0;
if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
!!cqe_version))
if ((offsetofend(typeof(*ucmd), uidx) <= inlen) != !!cqe_version)
return -EINVAL;
return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
@ -1539,7 +1570,9 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
return false;
if (access_flags & IB_ACCESS_RELAXED_ORDERING)
if (access_flags & IB_ACCESS_RELAXED_ORDERING &&
(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) ||
MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)))
return false;
return true;

View File

@ -54,12 +54,8 @@ static void
assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
u32 *in)
{
u8 key = atomic_inc_return(&dev->mkey_var);
void *mkc;
u8 key;
spin_lock_irq(&dev->mkey_lock);
key = dev->mkey_key++;
spin_unlock_irq(&dev->mkey_lock);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, mkey_7_0, key);
@ -90,6 +86,7 @@ mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
static int mr_cache_max_order(struct mlx5_ib_dev *dev);
static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
{
@ -103,16 +100,6 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
}
static int order2idx(struct mlx5_ib_dev *dev, int order)
{
struct mlx5_mr_cache *cache = &dev->cache;
if (order < cache->ent[0].order)
return 0;
else
return order - cache->ent[0].order;
}
static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
{
return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
@ -124,18 +111,16 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)
struct mlx5_ib_mr *mr =
container_of(context, struct mlx5_ib_mr, cb_work);
struct mlx5_ib_dev *dev = mr->dev;
struct mlx5_mr_cache *cache = &dev->cache;
int c = order2idx(dev, mr->order);
struct mlx5_cache_ent *ent = &cache->ent[c];
struct mlx5_cache_ent *ent = mr->cache_ent;
unsigned long flags;
spin_lock_irqsave(&ent->lock, flags);
ent->pending--;
spin_unlock_irqrestore(&ent->lock, flags);
if (status) {
mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
kfree(mr);
dev->fill_delay = 1;
spin_lock_irqsave(&ent->lock, flags);
ent->pending--;
WRITE_ONCE(dev->fill_delay, 1);
spin_unlock_irqrestore(&ent->lock, flags);
mod_timer(&dev->delay_timer, jiffies + HZ);
return;
}
@ -144,23 +129,44 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)
mr->mmkey.key |= mlx5_idx_to_mkey(
MLX5_GET(create_mkey_out, mr->out, mkey_index));
cache->last_add = jiffies;
WRITE_ONCE(dev->cache.last_add, jiffies);
spin_lock_irqsave(&ent->lock, flags);
list_add_tail(&mr->list, &ent->head);
ent->cur++;
ent->size++;
ent->available_mrs++;
ent->total_mrs++;
/* If we are doing fill_to_high_water then keep going. */
queue_adjust_cache_locked(ent);
ent->pending--;
spin_unlock_irqrestore(&ent->lock, flags);
if (!completion_done(&ent->compl))
complete(&ent->compl);
}
static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent = &cache->ent[c];
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_ib_mr *mr;
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
if (!mr)
return NULL;
mr->order = ent->order;
mr->cache_ent = ent;
mr->dev = ent->dev;
MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
MLX5_SET(mkc, mkc, log_page_size, ent->page);
return mr;
}
/* Asynchronously schedule new MRs to be populated in the cache. */
static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
{
size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_ib_mr *mr;
void *mkc;
u32 *in;
@ -173,42 +179,29 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
for (i = 0; i < num; i++) {
if (ent->pending >= MAX_PENDING_REG_MR) {
err = -EAGAIN;
break;
}
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
mr = alloc_cache_mr(ent, mkc);
if (!mr) {
err = -ENOMEM;
break;
}
mr->order = ent->order;
mr->allocated_from_cache = true;
mr->dev = dev;
MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
MLX5_SET(mkc, mkc, access_mode_4_2,
(ent->access_mode >> 2) & 0x7);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
MLX5_SET(mkc, mkc, log_page_size, ent->page);
spin_lock_irq(&ent->lock);
if (ent->pending >= MAX_PENDING_REG_MR) {
err = -EAGAIN;
spin_unlock_irq(&ent->lock);
kfree(mr);
break;
}
ent->pending++;
spin_unlock_irq(&ent->lock);
err = mlx5_ib_create_mkey_cb(dev, &mr->mmkey,
&dev->async_ctx, in, inlen,
mr->out, sizeof(mr->out),
&mr->cb_work);
err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey,
&ent->dev->async_ctx, in, inlen,
mr->out, sizeof(mr->out),
&mr->cb_work);
if (err) {
spin_lock_irq(&ent->lock);
ent->pending--;
spin_unlock_irq(&ent->lock);
mlx5_ib_warn(dev, "create mkey failed %d\n", err);
mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
kfree(mr);
break;
}
@ -218,32 +211,89 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
return err;
}
static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
/* Synchronously create a MR in the cache */
static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent = &cache->ent[c];
struct mlx5_ib_mr *tmp_mr;
size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_ib_mr *mr;
LIST_HEAD(del_list);
int i;
void *mkc;
u32 *in;
int err;
for (i = 0; i < num; i++) {
spin_lock_irq(&ent->lock);
if (list_empty(&ent->head)) {
spin_unlock_irq(&ent->lock);
break;
}
mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
list_move(&mr->list, &del_list);
ent->cur--;
ent->size--;
spin_unlock_irq(&ent->lock);
mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
in = kzalloc(inlen, GFP_KERNEL);
if (!in)
return ERR_PTR(-ENOMEM);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
mr = alloc_cache_mr(ent, mkc);
if (!mr) {
err = -ENOMEM;
goto free_in;
}
list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
list_del(&mr->list);
kfree(mr);
err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen);
if (err)
goto free_mr;
mr->mmkey.type = MLX5_MKEY_MR;
WRITE_ONCE(ent->dev->cache.last_add, jiffies);
spin_lock_irq(&ent->lock);
ent->total_mrs++;
spin_unlock_irq(&ent->lock);
kfree(in);
return mr;
free_mr:
kfree(mr);
free_in:
kfree(in);
return ERR_PTR(err);
}
static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
{
struct mlx5_ib_mr *mr;
lockdep_assert_held(&ent->lock);
if (list_empty(&ent->head))
return;
mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
list_del(&mr->list);
ent->available_mrs--;
ent->total_mrs--;
spin_unlock_irq(&ent->lock);
mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey);
kfree(mr);
spin_lock_irq(&ent->lock);
}
static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
bool limit_fill)
{
int err;
lockdep_assert_held(&ent->lock);
while (true) {
if (limit_fill)
target = ent->limit * 2;
if (target == ent->available_mrs + ent->pending)
return 0;
if (target > ent->available_mrs + ent->pending) {
u32 todo = target - (ent->available_mrs + ent->pending);
spin_unlock_irq(&ent->lock);
err = add_keys(ent, todo);
if (err == -EAGAIN)
usleep_range(3000, 5000);
spin_lock_irq(&ent->lock);
if (err) {
if (err != -EAGAIN)
return err;
} else
return 0;
} else {
remove_cache_mr_locked(ent);
}
}
}
@ -251,37 +301,38 @@ static ssize_t size_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos)
{
struct mlx5_cache_ent *ent = filp->private_data;
struct mlx5_ib_dev *dev = ent->dev;
char lbuf[20] = {0};
u32 var;
u32 target;
int err;
int c;
count = min(count, sizeof(lbuf) - 1);
if (copy_from_user(lbuf, buf, count))
return -EFAULT;
err = kstrtou32_from_user(buf, count, 0, &target);
if (err)
return err;
c = order2idx(dev, ent->order);
if (sscanf(lbuf, "%u", &var) != 1)
return -EINVAL;
if (var < ent->limit)
return -EINVAL;
if (var > ent->size) {
do {
err = add_keys(dev, c, var - ent->size);
if (err && err != -EAGAIN)
return err;
usleep_range(3000, 5000);
} while (err);
} else if (var < ent->size) {
remove_keys(dev, c, ent->size - var);
/*
* Target is the new value of total_mrs the user requests, however we
* cannot free MRs that are in use. Compute the target value for
* available_mrs.
*/
spin_lock_irq(&ent->lock);
if (target < ent->total_mrs - ent->available_mrs) {
err = -EINVAL;
goto err_unlock;
}
target = target - (ent->total_mrs - ent->available_mrs);
if (target < ent->limit || target > ent->limit*2) {
err = -EINVAL;
goto err_unlock;
}
err = resize_available_mrs(ent, target, false);
if (err)
goto err_unlock;
spin_unlock_irq(&ent->lock);
return count;
err_unlock:
spin_unlock_irq(&ent->lock);
return err;
}
static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
@ -291,7 +342,7 @@ static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
char lbuf[20];
int err;
err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs);
if (err < 0)
return err;
@ -309,32 +360,23 @@ static ssize_t limit_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos)
{
struct mlx5_cache_ent *ent = filp->private_data;
struct mlx5_ib_dev *dev = ent->dev;
char lbuf[20] = {0};
u32 var;
int err;
int c;
count = min(count, sizeof(lbuf) - 1);
if (copy_from_user(lbuf, buf, count))
return -EFAULT;
c = order2idx(dev, ent->order);
if (sscanf(lbuf, "%u", &var) != 1)
return -EINVAL;
if (var > ent->size)
return -EINVAL;
err = kstrtou32_from_user(buf, count, 0, &var);
if (err)
return err;
/*
* Upon set we immediately fill the cache to high water mark implied by
* the limit.
*/
spin_lock_irq(&ent->lock);
ent->limit = var;
if (ent->cur < ent->limit) {
err = add_keys(dev, c, 2 * ent->limit - ent->cur);
if (err)
return err;
}
err = resize_available_mrs(ent, 0, true);
spin_unlock_irq(&ent->lock);
if (err)
return err;
return count;
}
@ -359,68 +401,119 @@ static const struct file_operations limit_fops = {
.read = limit_read,
};
static int someone_adding(struct mlx5_mr_cache *cache)
static bool someone_adding(struct mlx5_mr_cache *cache)
{
int i;
unsigned int i;
for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
if (cache->ent[i].cur < cache->ent[i].limit)
return 1;
}
struct mlx5_cache_ent *ent = &cache->ent[i];
bool ret;
return 0;
spin_lock_irq(&ent->lock);
ret = ent->available_mrs < ent->limit;
spin_unlock_irq(&ent->lock);
if (ret)
return true;
}
return false;
}
/*
* Check if the bucket is outside the high/low water mark and schedule an async
* update. The cache refill has hysteresis, once the low water mark is hit it is
* refilled up to the high mark.
*/
static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
{
lockdep_assert_held(&ent->lock);
if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
return;
if (ent->available_mrs < ent->limit) {
ent->fill_to_high_water = true;
queue_work(ent->dev->cache.wq, &ent->work);
} else if (ent->fill_to_high_water &&
ent->available_mrs + ent->pending < 2 * ent->limit) {
/*
* Once we start populating due to hitting a low water mark
* continue until we pass the high water mark.
*/
queue_work(ent->dev->cache.wq, &ent->work);
} else if (ent->available_mrs == 2 * ent->limit) {
ent->fill_to_high_water = false;
} else if (ent->available_mrs > 2 * ent->limit) {
/* Queue deletion of excess entries */
ent->fill_to_high_water = false;
if (ent->pending)
queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
msecs_to_jiffies(1000));
else
queue_work(ent->dev->cache.wq, &ent->work);
}
}
static void __cache_work_func(struct mlx5_cache_ent *ent)
{
struct mlx5_ib_dev *dev = ent->dev;
struct mlx5_mr_cache *cache = &dev->cache;
int i = order2idx(dev, ent->order);
int err;
if (cache->stopped)
return;
spin_lock_irq(&ent->lock);
if (ent->disabled)
goto out;
ent = &dev->cache.ent[i];
if (ent->cur < 2 * ent->limit && !dev->fill_delay) {
err = add_keys(dev, i, 1);
if (ent->cur < 2 * ent->limit) {
if (err == -EAGAIN) {
mlx5_ib_dbg(dev, "returned eagain, order %d\n",
i + 2);
queue_delayed_work(cache->wq, &ent->dwork,
msecs_to_jiffies(3));
} else if (err) {
mlx5_ib_warn(dev, "command failed order %d, err %d\n",
i + 2, err);
if (ent->fill_to_high_water &&
ent->available_mrs + ent->pending < 2 * ent->limit &&
!READ_ONCE(dev->fill_delay)) {
spin_unlock_irq(&ent->lock);
err = add_keys(ent, 1);
spin_lock_irq(&ent->lock);
if (ent->disabled)
goto out;
if (err) {
/*
* EAGAIN only happens if pending is positive, so we
* will be rescheduled from reg_mr_callback(). The only
* failure path here is ENOMEM.
*/
if (err != -EAGAIN) {
mlx5_ib_warn(
dev,
"command failed order %d, err %d\n",
ent->order, err);
queue_delayed_work(cache->wq, &ent->dwork,
msecs_to_jiffies(1000));
} else {
queue_work(cache->wq, &ent->work);
}
}
} else if (ent->cur > 2 * ent->limit) {
} else if (ent->available_mrs > 2 * ent->limit) {
bool need_delay;
/*
* The remove_keys() logic is performed as garbage collection
* task. Such task is intended to be run when no other active
* processes are running.
* The remove_cache_mr() logic is performed as garbage
* collection task. Such task is intended to be run when no
* other active processes are running.
*
* The need_resched() will return TRUE if there are user tasks
* to be activated in near future.
*
* In such case, we don't execute remove_keys() and postpone
* the garbage collection work to try to run in next cycle,
* in order to free CPU resources to other tasks.
* In such case, we don't execute remove_cache_mr() and postpone
* the garbage collection work to try to run in next cycle, in
* order to free CPU resources to other tasks.
*/
if (!need_resched() && !someone_adding(cache) &&
time_after(jiffies, cache->last_add + 300 * HZ)) {
remove_keys(dev, i, 1);
if (ent->cur > ent->limit)
queue_work(cache->wq, &ent->work);
} else {
spin_unlock_irq(&ent->lock);
need_delay = need_resched() || someone_adding(cache) ||
time_after(jiffies,
READ_ONCE(cache->last_add) + 300 * HZ);
spin_lock_irq(&ent->lock);
if (ent->disabled)
goto out;
if (need_delay)
queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
}
remove_cache_mr_locked(ent);
queue_adjust_cache_locked(ent);
}
out:
spin_unlock_irq(&ent->lock);
}
static void delayed_cache_work_func(struct work_struct *work)
@ -439,117 +532,95 @@ static void cache_work_func(struct work_struct *work)
__cache_work_func(ent);
}
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)
/* Allocate a special entry from the cache */
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
unsigned int entry)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent;
struct mlx5_ib_mr *mr;
int err;
if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) {
mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);
if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
entry >= ARRAY_SIZE(cache->ent)))
return ERR_PTR(-EINVAL);
}
ent = &cache->ent[entry];
while (1) {
spin_lock_irq(&ent->lock);
if (list_empty(&ent->head)) {
spin_unlock_irq(&ent->lock);
err = add_keys(dev, entry, 1);
if (err && err != -EAGAIN)
return ERR_PTR(err);
wait_for_completion(&ent->compl);
} else {
mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
list);
list_del(&mr->list);
ent->cur--;
spin_unlock_irq(&ent->lock);
if (ent->cur < ent->limit)
queue_work(cache->wq, &ent->work);
spin_lock_irq(&ent->lock);
if (list_empty(&ent->head)) {
spin_unlock_irq(&ent->lock);
mr = create_cache_mr(ent);
if (IS_ERR(mr))
return mr;
}
} else {
mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
list_del(&mr->list);
ent->available_mrs--;
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
}
return mr;
}
static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
/* Return a MR already available in the cache */
static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_ib_dev *dev = req_ent->dev;
struct mlx5_ib_mr *mr = NULL;
struct mlx5_cache_ent *ent;
int last_umr_cache_entry;
int c;
int i;
struct mlx5_cache_ent *ent = req_ent;
c = order2idx(dev, order);
last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev));
if (c < 0 || c > last_umr_cache_entry) {
mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
return NULL;
}
for (i = c; i <= last_umr_cache_entry; i++) {
ent = &cache->ent[i];
mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
/* Try larger MR pools from the cache to satisfy the allocation */
for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) {
mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order,
ent - dev->cache.ent);
spin_lock_irq(&ent->lock);
if (!list_empty(&ent->head)) {
mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
list);
list_del(&mr->list);
ent->cur--;
ent->available_mrs--;
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
if (ent->cur < ent->limit)
queue_work(cache->wq, &ent->work);
break;
}
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
queue_work(cache->wq, &ent->work);
}
if (!mr)
cache->ent[c].miss++;
req_ent->miss++;
return mr;
}
static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
{
struct mlx5_cache_ent *ent = mr->cache_ent;
mr->cache_ent = NULL;
spin_lock_irq(&ent->lock);
ent->total_mrs--;
spin_unlock_irq(&ent->lock);
}
void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent;
int shrink = 0;
int c;
struct mlx5_cache_ent *ent = mr->cache_ent;
if (!mr->allocated_from_cache)
if (!ent)
return;
c = order2idx(dev, mr->order);
WARN_ON(c < 0 || c >= MAX_MR_CACHE_ENTRIES);
if (mlx5_mr_cache_invalidate(mr)) {
mr->allocated_from_cache = false;
detach_mr_from_cache(mr);
destroy_mkey(dev, mr);
ent = &cache->ent[c];
if (ent->cur < ent->limit)
queue_work(cache->wq, &ent->work);
return;
}
ent = &cache->ent[c];
spin_lock_irq(&ent->lock);
list_add_tail(&mr->list, &ent->head);
ent->cur++;
if (ent->cur > 2 * ent->limit)
shrink = 1;
ent->available_mrs++;
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
if (shrink)
queue_work(cache->wq, &ent->work);
}
static void clean_keys(struct mlx5_ib_dev *dev, int c)
@ -569,8 +640,8 @@ static void clean_keys(struct mlx5_ib_dev *dev, int c)
}
mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
list_move(&mr->list, &del_list);
ent->cur--;
ent->size--;
ent->available_mrs--;
ent->total_mrs--;
spin_unlock_irq(&ent->lock);
mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
}
@ -608,7 +679,7 @@ static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
dir = debugfs_create_dir(ent->name, cache->root);
debugfs_create_file("size", 0600, dir, ent, &size_fops);
debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
debugfs_create_u32("cur", 0400, dir, &ent->cur);
debugfs_create_u32("cur", 0400, dir, &ent->available_mrs);
debugfs_create_u32("miss", 0600, dir, &ent->miss);
}
}
@ -617,7 +688,7 @@ static void delay_time_func(struct timer_list *t)
{
struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
dev->fill_delay = 0;
WRITE_ONCE(dev->fill_delay, 0);
}
int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
@ -643,7 +714,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
ent->dev = dev;
ent->limit = 0;
init_completion(&ent->compl);
INIT_WORK(&ent->work, cache_work_func);
INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
@ -665,7 +735,9 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
ent->limit = dev->mdev->profile->mr_cache[i].limit;
else
ent->limit = 0;
queue_work(cache->wq, &ent->work);
spin_lock_irq(&ent->lock);
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
}
mlx5_mr_cache_debugfs_init(dev);
@ -675,13 +747,20 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
{
int i;
unsigned int i;
if (!dev->cache.wq)
return 0;
dev->cache.stopped = 1;
flush_workqueue(dev->cache.wq);
for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
struct mlx5_cache_ent *ent = &dev->cache.ent[i];
spin_lock_irq(&ent->lock);
ent->disabled = true;
spin_unlock_irq(&ent->lock);
cancel_work_sync(&ent->work);
cancel_delayed_work_sync(&ent->dwork);
}
mlx5_mr_cache_debugfs_cleanup(dev);
mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
@ -876,31 +955,37 @@ static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
return err;
}
static struct mlx5_ib_mr *alloc_mr_from_cache(
struct ib_pd *pd, struct ib_umem *umem,
u64 virt_addr, u64 len, int npages,
int page_shift, int order, int access_flags)
static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
unsigned int order)
{
struct mlx5_mr_cache *cache = &dev->cache;
if (order < cache->ent[0].order)
return &cache->ent[0];
order = order - cache->ent[0].order;
if (order > MR_CACHE_LAST_STD_ENTRY)
return NULL;
return &cache->ent[order];
}
static struct mlx5_ib_mr *
alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
u64 len, int npages, int page_shift, unsigned int order,
int access_flags)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
struct mlx5_ib_mr *mr;
int err = 0;
int i;
for (i = 0; i < 1; i++) {
mr = alloc_cached_mr(dev, order);
if (mr)
break;
err = add_keys(dev, order2idx(dev, order), 1);
if (err && err != -EAGAIN) {
mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);
break;
}
if (!ent)
return ERR_PTR(-E2BIG);
mr = get_cache_mr(ent);
if (!mr) {
mr = create_cache_mr(ent);
if (IS_ERR(mr))
return mr;
}
if (!mr)
return ERR_PTR(-EAGAIN);
mr->ibmr.pd = pd;
mr->umem = umem;
mr->access_flags = access_flags;
@ -1474,10 +1559,9 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
/*
* UMR can't be used - MKey needs to be replaced.
*/
if (mr->allocated_from_cache)
err = mlx5_mr_cache_invalidate(mr);
else
err = destroy_mkey(dev, mr);
if (mr->cache_ent)
detach_mr_from_cache(mr);
err = destroy_mkey(dev, mr);
if (err)
goto err;
@ -1489,8 +1573,6 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
mr = to_mmr(ib_mr);
goto err;
}
mr->allocated_from_cache = false;
} else {
/*
* Send a UMR WQE
@ -1577,8 +1659,6 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{
int allocated_from_cache = mr->allocated_from_cache;
if (mr->sig) {
if (mlx5_core_destroy_psv(dev->mdev,
mr->sig->psv_memory.psv_idx))
@ -1593,7 +1673,7 @@ static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
mr->sig = NULL;
}
if (!allocated_from_cache) {
if (!mr->cache_ent) {
destroy_mkey(dev, mr);
mlx5_free_priv_descs(mr);
}
@ -1610,7 +1690,7 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
else
clean_mr(dev, mr);
if (mr->allocated_from_cache)
if (mr->cache_ent)
mlx5_mr_cache_free(dev, mr);
else
kfree(mr);

View File

@ -197,7 +197,7 @@ static void dma_fence_odp_mr(struct mlx5_ib_mr *mr)
odp->private = NULL;
mutex_unlock(&odp->umem_mutex);
if (!mr->allocated_from_cache) {
if (!mr->cache_ent) {
mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey);
WARN_ON(mr->descs);
}

View File

@ -0,0 +1,136 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
*/
#include <rdma/uverbs_ioctl.h>
#include <rdma/mlx5_user_ioctl_cmds.h>
#include <rdma/mlx5_user_ioctl_verbs.h>
#include <linux/mlx5/driver.h>
#include "mlx5_ib.h"
#define UVERBS_MODULE_NAME mlx5_ib
#include <rdma/uverbs_named_ioctl.h>
static bool pp_is_supported(struct ib_device *device)
{
struct mlx5_ib_dev *dev = to_mdev(device);
return (MLX5_CAP_GEN(dev->mdev, qos) &&
MLX5_CAP_QOS(dev->mdev, packet_pacing) &&
MLX5_CAP_QOS(dev->mdev, packet_pacing_uid));
}
static int UVERBS_HANDLER(MLX5_IB_METHOD_PP_OBJ_ALLOC)(
struct uverbs_attr_bundle *attrs)
{
u8 rl_raw[MLX5_ST_SZ_BYTES(set_pp_rate_limit_context)] = {};
struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE);
struct mlx5_ib_dev *dev;
struct mlx5_ib_ucontext *c;
struct mlx5_ib_pp *pp_entry;
void *in_ctx;
u16 uid;
int inlen;
u32 flags;
int err;
c = to_mucontext(ib_uverbs_get_ucontext(attrs));
if (IS_ERR(c))
return PTR_ERR(c);
/* The allocated entry can be used only by a DEVX context */
if (!c->devx_uid)
return -EINVAL;
dev = to_mdev(c->ibucontext.device);
pp_entry = kzalloc(sizeof(*pp_entry), GFP_KERNEL);
if (!pp_entry)
return -ENOMEM;
in_ctx = uverbs_attr_get_alloced_ptr(attrs,
MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX);
inlen = uverbs_attr_get_len(attrs,
MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX);
memcpy(rl_raw, in_ctx, inlen);
err = uverbs_get_flags32(&flags, attrs,
MLX5_IB_ATTR_PP_OBJ_ALLOC_FLAGS,
MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX);
if (err)
goto err;
uid = (flags & MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX) ?
c->devx_uid : MLX5_SHARED_RESOURCE_UID;
err = mlx5_rl_add_rate_raw(dev->mdev, rl_raw, uid,
(flags & MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX),
&pp_entry->index);
if (err)
goto err;
err = uverbs_copy_to(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,
&pp_entry->index, sizeof(pp_entry->index));
if (err)
goto clean;
pp_entry->mdev = dev->mdev;
uobj->object = pp_entry;
return 0;
clean:
mlx5_rl_remove_rate_raw(dev->mdev, pp_entry->index);
err:
kfree(pp_entry);
return err;
}
static int pp_obj_cleanup(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct mlx5_ib_pp *pp_entry = uobject->object;
mlx5_rl_remove_rate_raw(pp_entry->mdev, pp_entry->index);
kfree(pp_entry);
return 0;
}
DECLARE_UVERBS_NAMED_METHOD(
MLX5_IB_METHOD_PP_OBJ_ALLOC,
UVERBS_ATTR_IDR(MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE,
MLX5_IB_OBJECT_PP,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(
MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX,
UVERBS_ATTR_SIZE(1,
MLX5_ST_SZ_BYTES(set_pp_rate_limit_context)),
UA_MANDATORY,
UA_ALLOC_AND_COPY),
UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_PP_OBJ_ALLOC_FLAGS,
enum mlx5_ib_uapi_pp_alloc_flags,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,
UVERBS_ATTR_TYPE(u16),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
MLX5_IB_METHOD_PP_OBJ_DESTROY,
UVERBS_ATTR_IDR(MLX5_IB_ATTR_PP_OBJ_DESTROY_HANDLE,
MLX5_IB_OBJECT_PP,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_PP,
UVERBS_TYPE_ALLOC_IDR(pp_obj_cleanup),
&UVERBS_METHOD(MLX5_IB_METHOD_PP_OBJ_ALLOC),
&UVERBS_METHOD(MLX5_IB_METHOD_PP_OBJ_DESTROY));
const struct uapi_definition mlx5_ib_qos_defs[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
MLX5_IB_OBJECT_PP,
UAPI_DEF_IS_OBJ_SUPPORTED(pp_is_supported)),
{},
};

View File

@ -697,6 +697,9 @@ static int alloc_bfreg(struct mlx5_ib_dev *dev,
{
int bfregn = -ENOMEM;
if (bfregi->lib_uar_dyn)
return -EINVAL;
mutex_lock(&bfregi->lock);
if (bfregi->ver >= 2) {
bfregn = alloc_high_class_bfreg(dev, bfregi);
@ -768,6 +771,9 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
u32 index_of_sys_page;
u32 offset;
if (bfregi->lib_uar_dyn)
return -EINVAL;
bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
MLX5_NON_FP_BFREGS_PER_UAR;
index_of_sys_page = bfregn / bfregs_per_sys_page;
@ -919,6 +925,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
void *qpc;
int err;
u16 uid;
u32 uar_flags;
err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
if (err) {
@ -928,24 +935,29 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
ibucontext);
if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {
uar_flags = ucmd.flags & (MLX5_QP_FLAG_UAR_PAGE_INDEX |
MLX5_QP_FLAG_BFREG_INDEX);
switch (uar_flags) {
case MLX5_QP_FLAG_UAR_PAGE_INDEX:
uar_index = ucmd.bfreg_index;
bfregn = MLX5_IB_INVALID_BFREG;
break;
case MLX5_QP_FLAG_BFREG_INDEX:
uar_index = bfregn_to_uar_index(dev, &context->bfregi,
ucmd.bfreg_index, true);
if (uar_index < 0)
return uar_index;
bfregn = MLX5_IB_INVALID_BFREG;
} else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) {
/*
* TBD: should come from the verbs when we have the API
*/
/* In CROSS_CHANNEL CQ and QP must use the same UAR */
bfregn = MLX5_CROSS_CHANNEL_BFREG;
}
else {
break;
case 0:
if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
return -EINVAL;
bfregn = alloc_bfreg(dev, &context->bfregi);
if (bfregn < 0)
return bfregn;
break;
default:
return -EINVAL;
}
mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
@ -2100,6 +2112,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC |
MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
MLX5_QP_FLAG_TUNNEL_OFFLOADS |
MLX5_QP_FLAG_UAR_PAGE_INDEX |
MLX5_QP_FLAG_TYPE_DCI |
MLX5_QP_FLAG_TYPE_DCT))
return -EINVAL;
@ -2789,7 +2802,7 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
mlx5_ib_dbg(dev, "unsupported qp type %d\n",
init_attr->qp_type);
/* Don't support raw QPs */
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
}
if (verbs_init_attr->qp_type == IB_QPT_DRIVER)

View File

@ -58,7 +58,7 @@ struct mthca_user_db_table {
u64 uvirt;
struct scatterlist mem;
int refcount;
} page[0];
} page[];
};
static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk)

View File

@ -68,7 +68,7 @@ struct mthca_icm_table {
int lowmem;
int coherent;
struct mutex mutex;
struct mthca_icm *icm[0];
struct mthca_icm *icm[];
};
struct mthca_icm_iter {

View File

@ -561,7 +561,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
}
default:
/* Don't support raw QPs */
return ERR_PTR(-ENOSYS);
return ERR_PTR(-EOPNOTSUPP);
}
if (err) {

View File

@ -1111,7 +1111,7 @@ static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
(attrs->qp_type != IB_QPT_UD)) {
pr_err("%s(%d) unsupported qp type=0x%x requested\n",
__func__, dev->id, attrs->qp_type);
return -EINVAL;
return -EOPNOTSUPP;
}
/* Skip the check for QP1 to support CM size of 128 */
if ((attrs->qp_type != IB_QPT_GSI) &&

View File

@ -1186,7 +1186,7 @@ static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
DP_DEBUG(dev, QEDR_MSG_QP,
"create qp: unsupported qp type=0x%x requested\n",
attrs->qp_type);
return -EINVAL;
return -EOPNOTSUPP;
}
if (attrs->cap.max_send_wr > qattr->max_sqe) {

View File

@ -39,7 +39,6 @@
#include <linux/utsname.h>
#include <linux/rculist.h>
#include <linux/mm.h>
#include <linux/random.h>
#include <linux/vmalloc.h>
#include <rdma/rdma_vt.h>
@ -1503,7 +1502,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
unsigned i, ctxt;
int ret;
get_random_bytes(&dev->qp_rnd, sizeof(dev->qp_rnd));
for (i = 0; i < dd->num_pports; i++)
init_ibport(ppd + i);

View File

@ -177,7 +177,6 @@ struct qib_ibdev {
struct timer_list mem_timer;
struct qib_pio_header *pio_hdrs;
dma_addr_t pio_hdrs_phys;
u32 qp_rnd; /* random bytes for hash */
u32 n_piowait;
u32 n_txwait;

View File

@ -504,7 +504,7 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd,
if (init_attr->qp_type != IB_QPT_UD) {
usnic_err("%s asked to make a non-UD QP: %d\n",
dev_name(&us_ibdev->ib_dev.dev), init_attr->qp_type);
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
}
trans_spec = cmd.spec;

View File

@ -77,7 +77,7 @@ struct usnic_uiom_reg {
struct usnic_uiom_chunk {
struct list_head list;
int nents;
struct scatterlist page_list[0];
struct scatterlist page_list[];
};
struct usnic_uiom_pd *usnic_uiom_alloc_pd(void);

View File

@ -217,7 +217,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
init_attr->qp_type != IB_QPT_GSI) {
dev_warn(&dev->pdev->dev, "queuepair type %d not supported\n",
init_attr->qp_type);
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
}
if (is_srq && !dev->dsr->caps.max_srq) {

View File

@ -1220,7 +1220,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
default:
/* Don't support raw QPs */
return ERR_PTR(-EINVAL);
return ERR_PTR(-EOPNOTSUPP);
}
init_attr->cap.max_inline_data = 0;

View File

@ -284,12 +284,6 @@ static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
&gid->global.interface_id);
}
static inline struct rvt_ucontext *to_iucontext(struct ib_ucontext
*ibucontext)
{
return container_of(ibucontext, struct rvt_ucontext, ibucontext);
}
/**
* rvt_alloc_ucontext - Allocate a user context
* @uctx: Verbs context

View File

@ -103,6 +103,8 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
rxe->attr.max_fast_reg_page_list_len = RXE_MAX_FMR_PAGE_LIST_LEN;
rxe->attr.max_pkeys = RXE_MAX_PKEYS;
rxe->attr.local_ca_ack_delay = RXE_LOCAL_CA_ACK_DELAY;
addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
rxe->ndev->dev_addr);
rxe->max_ucontext = RXE_MAX_UCONTEXT;
}

View File

@ -590,15 +590,16 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
int err;
if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
int max_rd_atomic = __roundup_pow_of_two(attr->max_rd_atomic);
int max_rd_atomic = attr->max_rd_atomic ?
roundup_pow_of_two(attr->max_rd_atomic) : 0;
qp->attr.max_rd_atomic = max_rd_atomic;
atomic_set(&qp->req.rd_atomic, max_rd_atomic);
}
if (mask & IB_QP_MAX_DEST_RD_ATOMIC) {
int max_dest_rd_atomic =
__roundup_pow_of_two(attr->max_dest_rd_atomic);
int max_dest_rd_atomic = attr->max_dest_rd_atomic ?
roundup_pow_of_two(attr->max_dest_rd_atomic) : 0;
qp->attr.max_dest_rd_atomic = max_dest_rd_atomic;

View File

@ -63,7 +63,7 @@ struct rxe_queue_buf {
__u32 pad_2[31];
__u32 consumer_index;
__u32 pad_3[31];
__u8 data[0];
__u8 data[];
};
struct rxe_queue {

View File

@ -1769,14 +1769,23 @@ int siw_reject(struct iw_cm_id *id, const void *pdata, u8 pd_len)
return 0;
}
static int siw_listen_address(struct iw_cm_id *id, int backlog,
struct sockaddr *laddr, int addr_family)
/*
* siw_create_listen - Create resources for a listener's IWCM ID @id
*
* Starts listen on the socket address id->local_addr.
*
*/
int siw_create_listen(struct iw_cm_id *id, int backlog)
{
struct socket *s;
struct siw_cep *cep = NULL;
struct siw_device *sdev = to_siw_dev(id->device);
int addr_family = id->local_addr.ss_family;
int rv = 0, s_val;
if (addr_family != AF_INET && addr_family != AF_INET6)
return -EAFNOSUPPORT;
rv = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &s);
if (rv < 0)
return rv;
@ -1791,9 +1800,25 @@ static int siw_listen_address(struct iw_cm_id *id, int backlog,
siw_dbg(id->device, "setsockopt error: %d\n", rv);
goto error;
}
rv = s->ops->bind(s, laddr, addr_family == AF_INET ?
sizeof(struct sockaddr_in) :
sizeof(struct sockaddr_in6));
if (addr_family == AF_INET) {
struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr);
/* For wildcard addr, limit binding to current device only */
if (ipv4_is_zeronet(laddr->sin_addr.s_addr))
s->sk->sk_bound_dev_if = sdev->netdev->ifindex;
rv = s->ops->bind(s, (struct sockaddr *)laddr,
sizeof(struct sockaddr_in));
} else {
struct sockaddr_in6 *laddr = &to_sockaddr_in6(id->local_addr);
/* For wildcard addr, limit binding to current device only */
if (ipv6_addr_any(&laddr->sin6_addr))
s->sk->sk_bound_dev_if = sdev->netdev->ifindex;
rv = s->ops->bind(s, (struct sockaddr *)laddr,
sizeof(struct sockaddr_in6));
}
if (rv) {
siw_dbg(id->device, "socket bind error: %d\n", rv);
goto error;
@ -1852,7 +1877,7 @@ static int siw_listen_address(struct iw_cm_id *id, int backlog,
list_add_tail(&cep->listenq, (struct list_head *)id->provider_data);
cep->state = SIW_EPSTATE_LISTENING;
siw_dbg(id->device, "Listen at laddr %pISp\n", laddr);
siw_dbg(id->device, "Listen at laddr %pISp\n", &id->local_addr);
return 0;
@ -1910,106 +1935,6 @@ static void siw_drop_listeners(struct iw_cm_id *id)
}
}
/*
* siw_create_listen - Create resources for a listener's IWCM ID @id
*
* Listens on the socket address id->local_addr.
*
* If the listener's @id provides a specific local IP address, at most one
* listening socket is created and associated with @id.
*
* If the listener's @id provides the wildcard (zero) local IP address,
* a separate listen is performed for each local IP address of the device
* by creating a listening socket and binding to that local IP address.
*
*/
int siw_create_listen(struct iw_cm_id *id, int backlog)
{
struct net_device *dev = to_siw_dev(id->device)->netdev;
int rv = 0, listeners = 0;
siw_dbg(id->device, "backlog %d\n", backlog);
/*
* For each attached address of the interface, create a
* listening socket, if id->local_addr is the wildcard
* IP address or matches the IP address.
*/
if (id->local_addr.ss_family == AF_INET) {
struct in_device *in_dev = in_dev_get(dev);
struct sockaddr_in s_laddr;
const struct in_ifaddr *ifa;
if (!in_dev) {
rv = -ENODEV;
goto out;
}
memcpy(&s_laddr, &id->local_addr, sizeof(s_laddr));
siw_dbg(id->device, "laddr %pISp\n", &s_laddr);
rtnl_lock();
in_dev_for_each_ifa_rtnl(ifa, in_dev) {
if (ipv4_is_zeronet(s_laddr.sin_addr.s_addr) ||
s_laddr.sin_addr.s_addr == ifa->ifa_address) {
s_laddr.sin_addr.s_addr = ifa->ifa_address;
rv = siw_listen_address(id, backlog,
(struct sockaddr *)&s_laddr,
AF_INET);
if (!rv)
listeners++;
}
}
rtnl_unlock();
in_dev_put(in_dev);
} else if (id->local_addr.ss_family == AF_INET6) {
struct inet6_dev *in6_dev = in6_dev_get(dev);
struct inet6_ifaddr *ifp;
struct sockaddr_in6 *s_laddr = &to_sockaddr_in6(id->local_addr);
if (!in6_dev) {
rv = -ENODEV;
goto out;
}
siw_dbg(id->device, "laddr %pISp\n", &s_laddr);
rtnl_lock();
list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
if (ifp->flags & (IFA_F_TENTATIVE | IFA_F_DEPRECATED))
continue;
if (ipv6_addr_any(&s_laddr->sin6_addr) ||
ipv6_addr_equal(&s_laddr->sin6_addr, &ifp->addr)) {
struct sockaddr_in6 bind_addr = {
.sin6_family = AF_INET6,
.sin6_port = s_laddr->sin6_port,
.sin6_flowinfo = 0,
.sin6_addr = ifp->addr,
.sin6_scope_id = dev->ifindex };
rv = siw_listen_address(id, backlog,
(struct sockaddr *)&bind_addr,
AF_INET6);
if (!rv)
listeners++;
}
}
rtnl_unlock();
in6_dev_put(in6_dev);
} else {
rv = -EAFNOSUPPORT;
}
out:
if (listeners)
rv = 0;
else if (!rv)
rv = -EINVAL;
siw_dbg(id->device, "%s\n", rv ? "FAIL" : "OK");
return rv;
}
int siw_destroy_listen(struct iw_cm_id *id)
{
if (!id->provider_data) {

View File

@ -332,7 +332,7 @@ static struct siw_wqe *siw_rqe_get(struct siw_qp *qp)
struct siw_srq *srq;
struct siw_wqe *wqe = NULL;
bool srq_event = false;
unsigned long flags;
unsigned long uninitialized_var(flags);
srq = qp->srq;
if (srq) {

View File

@ -165,15 +165,16 @@ int siw_query_port(struct ib_device *base_dev, u8 port,
struct ib_port_attr *attr)
{
struct siw_device *sdev = to_siw_dev(base_dev);
int rv;
memset(attr, 0, sizeof(*attr));
attr->active_mtu = attr->max_mtu;
attr->active_speed = 2;
attr->active_width = 2;
rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
&attr->active_width);
attr->gid_tbl_len = 1;
attr->max_msg_sz = -1;
attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
attr->pkey_tbl_len = 1;
@ -192,7 +193,7 @@ int siw_query_port(struct ib_device *base_dev, u8 port,
* attr->subnet_timeout = 0;
* attr->init_type_repy = 0;
*/
return 0;
return rv;
}
int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
@ -322,7 +323,7 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd,
}
if (attrs->qp_type != IB_QPT_RC) {
siw_dbg(base_dev, "only RC QP's supported\n");
rv = -EINVAL;
rv = -EOPNOTSUPP;
goto err_out;
}
if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||

View File

@ -838,6 +838,4 @@ extern int ipoib_debug_level;
#define IPOIB_QPN(ha) (be32_to_cpup((__be32 *) ha) & 0xffffff)
extern const char ipoib_driver_version[];
#endif /* _IPOIB_H */

View File

@ -68,9 +68,6 @@ static void ipoib_get_drvinfo(struct net_device *netdev,
strlcpy(drvinfo->bus_info, dev_name(priv->ca->dev.parent),
sizeof(drvinfo->bus_info));
strlcpy(drvinfo->version, ipoib_driver_version,
sizeof(drvinfo->version));
strlcpy(drvinfo->driver, "ib_ipoib", sizeof(drvinfo->driver));
}

View File

@ -52,10 +52,6 @@
#include <linux/inetdevice.h>
#include <rdma/ib_cache.h>
#define DRV_VERSION "1.0.0"
const char ipoib_driver_version[] = DRV_VERSION;
MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
MODULE_LICENSE("Dual BSD/GPL");

View File

@ -292,12 +292,27 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
{
struct iser_device *device = iser_task->iser_conn->ib_conn.device;
struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
struct iser_fr_desc *desc;
struct ib_mr_status mr_status;
if (!reg->mem_h)
desc = reg->mem_h;
if (!desc)
return;
device->reg_ops->reg_desc_put(&iser_task->iser_conn->ib_conn,
reg->mem_h);
/*
* The signature MR cannot be invalidated and reused without checking.
* libiscsi calls the check_protection transport handler only if
* SCSI-Response is received. And the signature MR is not checked if
* the task is completed for some other reason like a timeout or error
* handling. That's why we must check the signature MR here before
* putting it to the free pool.
*/
if (unlikely(desc->sig_protected)) {
desc->sig_protected = false;
ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS,
&mr_status);
}
device->reg_ops->reg_desc_put(&iser_task->iser_conn->ib_conn, desc);
reg->mem_h = NULL;
}

View File

@ -239,7 +239,7 @@ struct opa_veswport_mactable_entry {
* @offset: mac table starting offset
* @num_entries: Number of entries to get or set
* @mac_tbl_digest: mac table digest
* @tbl_entries[]: Array of table entries
* @tbl_entries: Array of table entries
*
* The EM sends down this structure in a MAD indicating
* the starting offset in the forwarding table that this
@ -258,7 +258,7 @@ struct opa_veswport_mactable {
__be16 offset;
__be16 num_entries;
__be32 mac_tbl_digest;
struct opa_veswport_mactable_entry tbl_entries[0];
struct opa_veswport_mactable_entry tbl_entries[];
} __packed;
/**
@ -440,7 +440,7 @@ struct opa_veswport_iface_macs {
__be16 num_macs_in_msg;
__be16 tot_macs_in_lst;
__be16 gen_count;
struct opa_vnic_iface_mac_entry entry[0];
struct opa_vnic_iface_mac_entry entry[];
} __packed;
/**

View File

@ -125,8 +125,6 @@ static void vnic_get_drvinfo(struct net_device *netdev,
struct ethtool_drvinfo *drvinfo)
{
strlcpy(drvinfo->driver, opa_vnic_driver_name, sizeof(drvinfo->driver));
strlcpy(drvinfo->version, opa_vnic_driver_version,
sizeof(drvinfo->version));
strlcpy(drvinfo->bus_info, dev_name(netdev->dev.parent),
sizeof(drvinfo->bus_info));
}

View File

@ -292,7 +292,6 @@ struct opa_vnic_mac_tbl_node {
hlist_for_each_entry(obj, &name[bkt], member)
extern char opa_vnic_driver_name[];
extern const char opa_vnic_driver_version[];
struct opa_vnic_adapter *opa_vnic_add_netdev(struct ib_device *ibdev,
u8 port_num, u8 vport_num);

Some files were not shown because too many files have changed in this diff Show More