UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

genirq/msi: Convert storage to xarray 

The current linked list storage for MSI descriptors is suboptimal in
several ways:

  1) Looking up a MSI desciptor requires a O(n) list walk in the worst case

  2) The upcoming support of runtime expansion of MSI-X vectors would need
     to do a full list walk to figure out whether a particular index is
     already associated.

  3) Runtime expansion of sparse allocations is even more complex as the
     current implementation assumes an ordered list (increasing MSI index).

Use an xarray which solves all of the above problems nicely.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Michael Kelley <mikelley@microsoft.com>
Tested-by: Nishanth Menon <nm@ti.com>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20211206210749.280627070@linutronix.de
This commit is contained in:
Thomas Gleixner 2021-12-06 23:51:52 +01:00
parent bf5e758f02
commit cd6cf06590
2 changed files with 83 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/linux/msi.h
View File

@ -17,6 +17,7 @@
*/ */
#include <linux/cpumask.h> #include <linux/cpumask.h>
#include <linux/xarray.h>
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/list.h> #include <linux/list.h>
#include <asm/msi.h> #include <asm/msi.h>
@ -123,7 +124,6 @@ struct pci_msi_desc {
/** /**
* struct msi_desc - Descriptor structure for MSI based interrupts * struct msi_desc - Descriptor structure for MSI based interrupts
* @list: List head for management
* @irq: The base interrupt number * @irq: The base interrupt number
* @nvec_used: The number of vectors used * @nvec_used: The number of vectors used
* @dev: Pointer to the device which uses this descriptor * @dev: Pointer to the device which uses this descriptor
@ -140,7 +140,6 @@ struct pci_msi_desc {
*/ */
struct msi_desc { struct msi_desc {
/* Shared device/bus type independent data */ /* Shared device/bus type independent data */
struct list_head list;
unsigned int irq; unsigned int irq;
unsigned int nvec_used; unsigned int nvec_used;
struct device *dev; struct device *dev;
@ -176,16 +175,16 @@ enum msi_desc_filter {
* msi_device_data - MSI per device data * msi_device_data - MSI per device data
* @properties: MSI properties which are interesting to drivers * @properties: MSI properties which are interesting to drivers
* @platform_data: Platform-MSI specific data * @platform_data: Platform-MSI specific data
* @list: List of MSI descriptors associated to the device * @mutex: Mutex protecting the MSI descriptor store
* @mutex: Mutex protecting the MSI list * @__store: Xarray for storing MSI descriptor pointers
* @__next: Cached pointer to the next entry for iterators * @__iter_idx: Index to search the next entry for iterators
*/ */
struct msi_device_data { struct msi_device_data {
unsigned long properties; unsigned long properties;
struct platform_msi_priv_data *platform_data; struct platform_msi_priv_data *platform_data;
struct list_head list;
struct mutex mutex; struct mutex mutex;
struct msi_desc *__next; struct xarray __store;
unsigned long __iter_idx;
}; };
int msi_setup_device_data(struct device *dev); int msi_setup_device_data(struct device *dev);

169
kernel/irq/msi.c
View File

@ -20,7 +20,6 @@
#include "internals.h" #include "internals.h"
static inline int msi_sysfs_create_group(struct device *dev); static inline int msi_sysfs_create_group(struct device *dev);
#define dev_to_msi_list(dev) (&(dev)->msi.data->list)
/** /**
* msi_alloc_desc - Allocate an initialized msi_desc * msi_alloc_desc - Allocate an initialized msi_desc
@ -41,7 +40,6 @@ static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec,
if (!desc) if (!desc)
return NULL; return NULL;
INIT_LIST_HEAD(&desc->list);
desc->dev = dev; desc->dev = dev;
desc->nvec_used = nvec; desc->nvec_used = nvec;
if (affinity) { if (affinity) {
@ -60,6 +58,17 @@ static void msi_free_desc(struct msi_desc *desc)
kfree(desc); kfree(desc);
} }
static int msi_insert_desc(struct msi_device_data *md, struct msi_desc *desc, unsigned int index)
{
int ret;
desc->msi_index = index;
ret = xa_insert(&md->__store, index, desc, GFP_KERNEL);
if (ret)
msi_free_desc(desc);
return ret;
}
/** /**
* msi_add_msi_desc - Allocate and initialize a MSI descriptor * msi_add_msi_desc - Allocate and initialize a MSI descriptor
* @dev: Pointer to the device for which the descriptor is allocated * @dev: Pointer to the device for which the descriptor is allocated
@ -77,12 +86,9 @@ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc)
if (!desc) if (!desc)
return -ENOMEM; return -ENOMEM;
/* Copy the MSI index and type specific data to the new descriptor. */ /* Copy type specific data to the new descriptor. */
desc->msi_index = init_desc->msi_index;
desc->pci = init_desc->pci; desc->pci = init_desc->pci;
return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index);
list_add_tail(&desc->list, &dev->msi.data->list);
return 0;
} }
/** /**
@ -95,28 +101,41 @@ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc)
*/ */
static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc) static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc)
{ {
struct msi_desc *desc, *tmp; unsigned int idx, last = index + ndesc - 1;
LIST_HEAD(list); struct msi_desc *desc;
unsigned int i; int ret;
lockdep_assert_held(&dev->msi.data->mutex); lockdep_assert_held(&dev->msi.data->mutex);
for (i = 0; i < ndesc; i++) { for (idx = index; idx <= last; idx++) {
desc = msi_alloc_desc(dev, 1, NULL); desc = msi_alloc_desc(dev, 1, NULL);
if (!desc) if (!desc)
goto fail_mem;
ret = msi_insert_desc(dev->msi.data, desc, idx);
if (ret)
goto fail; goto fail;
desc->msi_index = index + i;
list_add_tail(&desc->list, &list);
} }
list_splice_tail(&list, &dev->msi.data->list);
return 0; return 0;
fail_mem:
ret = -ENOMEM;
fail: fail:
list_for_each_entry_safe(desc, tmp, &list, list) { msi_free_msi_descs_range(dev, MSI_DESC_NOTASSOCIATED, index, last);
list_del(&desc->list); return ret;
msi_free_desc(desc); }
static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter)
{
switch (filter) {
case MSI_DESC_ALL:
return true;
case MSI_DESC_NOTASSOCIATED:
return !desc->irq;
case MSI_DESC_ASSOCIATED:
return !!desc->irq;
} }
return -ENOMEM; WARN_ON_ONCE(1);
return false;
} }
/** /**
@ -129,19 +148,17 @@ fail:
void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter,
unsigned int first_index, unsigned int last_index) unsigned int first_index, unsigned int last_index)
{ {
struct xarray *xa = &dev->msi.data->__store;
struct msi_desc *desc; struct msi_desc *desc;
unsigned long idx;
lockdep_assert_held(&dev->msi.data->mutex); lockdep_assert_held(&dev->msi.data->mutex);
msi_for_each_desc(desc, dev, filter) { xa_for_each_range(xa, idx, desc, first_index, last_index) {
/* if (msi_desc_match(desc, filter)) {
* Stupid for now to handle MSI device domain until the xa_erase(xa, idx);
* storage is switched over to an xarray. msi_free_desc(desc);
*/ }
if (desc->msi_index < first_index || desc->msi_index > last_index)
continue;
list_del(&desc->list);
msi_free_desc(desc);
} }
} }
@ -162,7 +179,8 @@ static void msi_device_data_release(struct device *dev, void *res)
{ {
struct msi_device_data *md = res; struct msi_device_data *md = res;
WARN_ON_ONCE(!list_empty(&md->list)); WARN_ON_ONCE(!xa_empty(&md->__store));
xa_destroy(&md->__store);
dev->msi.data = NULL; dev->msi.data = NULL;
} }
@ -194,7 +212,7 @@ int msi_setup_device_data(struct device *dev)
return ret; return ret;
} }
INIT_LIST_HEAD(&md->list); xa_init(&md->__store);
mutex_init(&md->mutex); mutex_init(&md->mutex);
dev->msi.data = md; dev->msi.data = md;
devres_add(dev, md); devres_add(dev, md);
@ -217,34 +235,21 @@ EXPORT_SYMBOL_GPL(msi_lock_descs);
*/ */
void msi_unlock_descs(struct device *dev) void msi_unlock_descs(struct device *dev)
{ {
/* Clear the next pointer which was cached by the iterator */ /* Invalidate the index wich was cached by the iterator */
dev->msi.data->__next = NULL; dev->msi.data->__iter_idx = MSI_MAX_INDEX;
mutex_unlock(&dev->msi.data->mutex); mutex_unlock(&dev->msi.data->mutex);
} }
EXPORT_SYMBOL_GPL(msi_unlock_descs); EXPORT_SYMBOL_GPL(msi_unlock_descs);
static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) static struct msi_desc *msi_find_desc(struct msi_device_data *md, enum msi_desc_filter filter)
{
switch (filter) {
case MSI_DESC_ALL:
return true;
case MSI_DESC_NOTASSOCIATED:
return !desc->irq;
case MSI_DESC_ASSOCIATED:
return !!desc->irq;
}
WARN_ON_ONCE(1);
return false;
}
static struct msi_desc *msi_find_first_desc(struct device *dev, enum msi_desc_filter filter)
{ {
struct msi_desc *desc; struct msi_desc *desc;
list_for_each_entry(desc, dev_to_msi_list(dev), list) { xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) {
if (msi_desc_match(desc, filter)) if (msi_desc_match(desc, filter))
return desc; return desc;
} }
md->__iter_idx = MSI_MAX_INDEX;
return NULL; return NULL;
} }
@ -261,37 +266,24 @@ static struct msi_desc *msi_find_first_desc(struct device *dev, enum msi_desc_fi
*/ */
struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter) struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter)
{ {
struct msi_desc *desc; struct msi_device_data *md = dev->msi.data;
if (WARN_ON_ONCE(!dev->msi.data)) if (WARN_ON_ONCE(!md))
return NULL; return NULL;
lockdep_assert_held(&dev->msi.data->mutex); lockdep_assert_held(&md->mutex);
desc = msi_find_first_desc(dev, filter); md->__iter_idx = 0;
dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; return msi_find_desc(md, filter);
return desc;
} }
EXPORT_SYMBOL_GPL(msi_first_desc); EXPORT_SYMBOL_GPL(msi_first_desc);
static struct msi_desc *__msi_next_desc(struct device *dev, enum msi_desc_filter filter,
struct msi_desc *from)
{
struct msi_desc *desc = from;
list_for_each_entry_from(desc, dev_to_msi_list(dev), list) {
if (msi_desc_match(desc, filter))
return desc;
}
return NULL;
}
/** /**
* msi_next_desc - Get the next MSI descriptor of a device * msi_next_desc - Get the next MSI descriptor of a device
* @dev: Device to operate on * @dev: Device to operate on
* *
* The first invocation of msi_next_desc() has to be preceeded by a * The first invocation of msi_next_desc() has to be preceeded by a
* successful incovation of __msi_first_desc(). Consecutive invocations are * successful invocation of __msi_first_desc(). Consecutive invocations are
* only valid if the previous one was successful. All these operations have * only valid if the previous one was successful. All these operations have
* to be done within the same MSI mutex held region. * to be done within the same MSI mutex held region.
* *
@ -300,20 +292,18 @@ static struct msi_desc *__msi_next_desc(struct device *dev, enum msi_desc_filter
*/ */
struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter) struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter)
{ {
struct msi_device_data *data = dev->msi.data; struct msi_device_data *md = dev->msi.data;
struct msi_desc *desc;
if (WARN_ON_ONCE(!data)) if (WARN_ON_ONCE(!md))
return NULL; return NULL;
lockdep_assert_held(&data->mutex); lockdep_assert_held(&md->mutex);
if (!data->__next) if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX)
return NULL; return NULL;
desc = __msi_next_desc(dev, filter, data->__next); md->__iter_idx++;
dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; return msi_find_desc(md, filter);
return desc;
} }
EXPORT_SYMBOL_GPL(msi_next_desc); EXPORT_SYMBOL_GPL(msi_next_desc);
@ -336,21 +326,18 @@ unsigned int msi_get_virq(struct device *dev, unsigned int index)
pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false; pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false;
msi_lock_descs(dev); msi_lock_descs(dev);
msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index);
/* PCI-MSI has only one descriptor for multiple interrupts. */ if (desc && desc->irq) {
if (pcimsi) {
if (index < desc->nvec_used)
ret = desc->irq + index;
break;
}
/* /*
* PCI-MSI has only one descriptor for multiple interrupts.
* PCI-MSIX and platform MSI use a descriptor per * PCI-MSIX and platform MSI use a descriptor per
* interrupt. * interrupt.
*/ */
if (desc->msi_index == index) { if (pcimsi) {
if (index < desc->nvec_used)
ret = desc->irq + index;
} else {
ret = desc->irq; ret = desc->irq;
break;
} }
} }
msi_unlock_descs(dev); msi_unlock_descs(dev);
@ -731,16 +718,13 @@ int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
int ret, virq; int ret, virq;
msi_lock_descs(dev); msi_lock_descs(dev);
for (virq = virq_base; virq < virq_base + nvec; virq++) { ret = msi_add_simple_msi_descs(dev, virq_base, nvec);
desc = msi_alloc_desc(dev, 1, NULL); if (ret)
if (!desc) { goto unlock;
ret = -ENOMEM;
goto fail;
}
desc->msi_index = virq; for (virq = virq_base; virq < virq_base + nvec; virq++) {
desc = xa_load(&dev->msi.data->__store, virq);
desc->irq = virq; desc->irq = virq;
list_add_tail(&desc->list, &dev->msi.data->list);
ops->set_desc(arg, desc); ops->set_desc(arg, desc);
ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg);
@ -756,6 +740,7 @@ fail:
for (--virq; virq >= virq_base; virq--) for (--virq; virq >= virq_base; virq--)
irq_domain_free_irqs_common(domain, virq, 1); irq_domain_free_irqs_common(domain, virq, 1);
msi_free_msi_descs_range(dev, MSI_DESC_ALL, virq_base, virq_base + nvec - 1); msi_free_msi_descs_range(dev, MSI_DESC_ALL, virq_base, virq_base + nvec - 1);
unlock:
msi_unlock_descs(dev); msi_unlock_descs(dev);
return ret; return ret;
} }