UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

vfio/type1: Prepare for batched pinning with struct vfio_batch 

Get ready to pin more pages at once with struct vfio_batch, which
represents a batch of pinned pages.

The struct has a fallback page pointer to avoid two unlikely scenarios:
pointlessly allocating a page if disable_hugepages is enabled or failing
the whole pinning operation if the kernel can't allocate memory.

vaddr_get_pfn() becomes vaddr_get_pfns() to prepare for handling
multiple pages, though for now only one page is stored in the pages
array.

Signed-off-by: Daniel Jordan <daniel.m.jordan@oracle.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
This commit is contained in:
Daniel Jordan 2021-02-19 11:13:04 -05:00 committed by Alex Williamson
parent be16c1fd99
commit 4b6c33b322
1 changed files with 58 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
drivers/vfio/vfio_iommu_type1.c
View File

@ -103,6 +103,12 @@ struct vfio_dma {
unsigned long *bitmap; unsigned long *bitmap;
}; };
struct vfio_batch {
struct page **pages; /* for pin_user_pages_remote */
struct page *fallback_page; /* if pages alloc fails */
int capacity; /* length of pages array */
};
struct vfio_group { struct vfio_group {
struct iommu_group *iommu_group; struct iommu_group *iommu_group;
struct list_head next; struct list_head next;
@ -459,6 +465,31 @@ static int put_pfn(unsigned long pfn, int prot)
return 0; return 0;
} }
#define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *))
static void vfio_batch_init(struct vfio_batch *batch)
{
if (unlikely(disable_hugepages))
goto fallback;
batch->pages = (struct page **) __get_free_page(GFP_KERNEL);
if (!batch->pages)
goto fallback;
batch->capacity = VFIO_BATCH_MAX_CAPACITY;
return;
fallback:
batch->pages = &batch->fallback_page;
batch->capacity = 1;
}
static void vfio_batch_fini(struct vfio_batch *batch)
{
if (batch->capacity == VFIO_BATCH_MAX_CAPACITY)
free_page((unsigned long)batch->pages);
}
static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
unsigned long vaddr, unsigned long *pfn, unsigned long vaddr, unsigned long *pfn,
bool write_fault) bool write_fault)
@ -499,10 +530,10 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
* Returns the positive number of pfns successfully obtained or a negative * Returns the positive number of pfns successfully obtained or a negative
* error code. * error code.
*/ */
static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
int prot, unsigned long *pfn) long npages, int prot, unsigned long *pfn,
struct page **pages)
{ {
struct page *page[1];
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned int flags = 0; unsigned int flags = 0;
int ret; int ret;
@ -511,10 +542,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
flags |= FOLL_WRITE; flags |= FOLL_WRITE;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM, ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM,
page, NULL, NULL); pages, NULL, NULL);
if (ret == 1) { if (ret > 0) {
*pfn = page_to_pfn(page[0]); *pfn = page_to_pfn(pages[0]);
goto done; goto done;
} }
@ -602,7 +633,7 @@ static int vfio_wait_all_valid(struct vfio_iommu *iommu)
*/ */
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
long npage, unsigned long *pfn_base, long npage, unsigned long *pfn_base,
unsigned long limit) unsigned long limit, struct vfio_batch *batch)
{ {
unsigned long pfn = 0; unsigned long pfn = 0;
long ret, pinned = 0, lock_acct = 0; long ret, pinned = 0, lock_acct = 0;
@ -613,7 +644,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
if (!current->mm) if (!current->mm)
return -ENODEV; return -ENODEV;
ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, pfn_base,
batch->pages);
if (ret < 0) if (ret < 0)
return ret; return ret;
@ -640,7 +672,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
/* Lock all the consecutive pages from pfn_base */ /* Lock all the consecutive pages from pfn_base */
for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;
pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, &pfn,
batch->pages);
if (ret < 0) if (ret < 0)
break; break;
@ -703,6 +736,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
unsigned long *pfn_base, bool do_accounting) unsigned long *pfn_base, bool do_accounting)
{ {
struct page *pages[1];
struct mm_struct *mm; struct mm_struct *mm;
int ret; int ret;
@ -710,7 +744,7 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
if (!mm) if (!mm)
return -ENODEV; return -ENODEV;
ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages);
if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
ret = vfio_lock_acct(dma, 1, true); ret = vfio_lock_acct(dma, 1, true);
if (ret) { if (ret) {
@ -1404,15 +1438,19 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
{ {
dma_addr_t iova = dma->iova; dma_addr_t iova = dma->iova;
unsigned long vaddr = dma->vaddr; unsigned long vaddr = dma->vaddr;
struct vfio_batch batch;
size_t size = map_size; size_t size = map_size;
long npage; long npage;
unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
int ret = 0; int ret = 0;
vfio_batch_init(&batch);
while (size) { while (size) {
/* Pin a contiguous chunk of memory */ /* Pin a contiguous chunk of memory */
npage = vfio_pin_pages_remote(dma, vaddr + dma->size, npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
size >> PAGE_SHIFT, &pfn, limit); size >> PAGE_SHIFT, &pfn, limit,
&batch);
if (npage <= 0) { if (npage <= 0) {
WARN_ON(!npage); WARN_ON(!npage);
ret = (int)npage; ret = (int)npage;
@ -1432,6 +1470,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
dma->size += npage << PAGE_SHIFT; dma->size += npage << PAGE_SHIFT;
} }
vfio_batch_fini(&batch);
dma->iommu_mapped = true; dma->iommu_mapped = true;
if (ret) if (ret)
@ -1608,6 +1647,7 @@ static int vfio_bus_type(struct device *dev, void *data)
static int vfio_iommu_replay(struct vfio_iommu *iommu, static int vfio_iommu_replay(struct vfio_iommu *iommu,
struct vfio_domain *domain) struct vfio_domain *domain)
{ {
struct vfio_batch batch;
struct vfio_domain *d = NULL; struct vfio_domain *d = NULL;
struct rb_node *n; struct rb_node *n;
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
@ -1622,6 +1662,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
d = list_first_entry(&iommu->domain_list, d = list_first_entry(&iommu->domain_list,
struct vfio_domain, next); struct vfio_domain, next);
vfio_batch_init(&batch);
n = rb_first(&iommu->dma_list); n = rb_first(&iommu->dma_list);
for (; n; n = rb_next(n)) { for (; n; n = rb_next(n)) {
@ -1669,7 +1711,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
npage = vfio_pin_pages_remote(dma, vaddr, npage = vfio_pin_pages_remote(dma, vaddr,
n >> PAGE_SHIFT, n >> PAGE_SHIFT,
&pfn, limit); &pfn, limit,
&batch);
if (npage <= 0) { if (npage <= 0) {
WARN_ON(!npage); WARN_ON(!npage);
ret = (int)npage; ret = (int)npage;
@ -1702,6 +1745,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
dma->iommu_mapped = true; dma->iommu_mapped = true;
} }
vfio_batch_fini(&batch);
return 0; return 0;
unwind: unwind:
@ -1742,6 +1786,7 @@ unwind:
} }
} }
vfio_batch_fini(&batch);
return ret; return ret;
} }