149 lines
3.1 KiB
C
149 lines
3.1 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* Copyright (C) Gao Xiang <xiang@kernel.org>
|
|
*
|
|
* For low-latency decompression algorithms (e.g. lz4), reserve consecutive
|
|
* per-CPU virtual memory (in pages) in advance to store such inplace I/O
|
|
* data if inplace decompression is failed (due to unmet inplace margin for
|
|
* example).
|
|
*/
|
|
#include "internal.h"
|
|
|
|
struct erofs_pcpubuf {
|
|
raw_spinlock_t lock;
|
|
void *ptr;
|
|
struct page **pages;
|
|
unsigned int nrpages;
|
|
};
|
|
|
|
static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
|
|
|
|
void *erofs_get_pcpubuf(unsigned int requiredpages)
|
|
__acquires(pcb->lock)
|
|
{
|
|
struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
|
|
|
|
raw_spin_lock(&pcb->lock);
|
|
/* check if the per-CPU buffer is too small */
|
|
if (requiredpages > pcb->nrpages) {
|
|
raw_spin_unlock(&pcb->lock);
|
|
put_cpu_var(erofs_pcb);
|
|
/* (for sparse checker) pretend pcb->lock is still taken */
|
|
__acquire(pcb->lock);
|
|
return NULL;
|
|
}
|
|
return pcb->ptr;
|
|
}
|
|
|
|
void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
|
|
{
|
|
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
|
|
|
|
DBG_BUGON(pcb->ptr != ptr);
|
|
raw_spin_unlock(&pcb->lock);
|
|
put_cpu_var(erofs_pcb);
|
|
}
|
|
|
|
/* the next step: support per-CPU page buffers hotplug */
|
|
int erofs_pcpubuf_growsize(unsigned int nrpages)
|
|
{
|
|
static DEFINE_MUTEX(pcb_resize_mutex);
|
|
static unsigned int pcb_nrpages;
|
|
struct page *pagepool = NULL;
|
|
int delta, cpu, ret, i;
|
|
|
|
mutex_lock(&pcb_resize_mutex);
|
|
delta = nrpages - pcb_nrpages;
|
|
ret = 0;
|
|
/* avoid shrinking pcpubuf, since no idea how many fses rely on */
|
|
if (delta <= 0)
|
|
goto out;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
|
|
struct page **pages, **oldpages;
|
|
void *ptr, *old_ptr;
|
|
|
|
pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
|
|
if (!pages) {
|
|
ret = -ENOMEM;
|
|
break;
|
|
}
|
|
|
|
for (i = 0; i < nrpages; ++i) {
|
|
pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
|
|
if (!pages[i]) {
|
|
ret = -ENOMEM;
|
|
oldpages = pages;
|
|
goto free_pagearray;
|
|
}
|
|
}
|
|
ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
|
|
if (!ptr) {
|
|
ret = -ENOMEM;
|
|
oldpages = pages;
|
|
goto free_pagearray;
|
|
}
|
|
raw_spin_lock(&pcb->lock);
|
|
old_ptr = pcb->ptr;
|
|
pcb->ptr = ptr;
|
|
oldpages = pcb->pages;
|
|
pcb->pages = pages;
|
|
i = pcb->nrpages;
|
|
pcb->nrpages = nrpages;
|
|
raw_spin_unlock(&pcb->lock);
|
|
|
|
if (!oldpages) {
|
|
DBG_BUGON(old_ptr);
|
|
continue;
|
|
}
|
|
|
|
if (old_ptr)
|
|
vunmap(old_ptr);
|
|
free_pagearray:
|
|
while (i)
|
|
erofs_pagepool_add(&pagepool, oldpages[--i]);
|
|
kfree(oldpages);
|
|
if (ret)
|
|
break;
|
|
}
|
|
pcb_nrpages = nrpages;
|
|
erofs_release_pages(&pagepool);
|
|
out:
|
|
mutex_unlock(&pcb_resize_mutex);
|
|
return ret;
|
|
}
|
|
|
|
void __init erofs_pcpubuf_init(void)
|
|
{
|
|
int cpu;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
|
|
|
|
raw_spin_lock_init(&pcb->lock);
|
|
}
|
|
}
|
|
|
|
void erofs_pcpubuf_exit(void)
|
|
{
|
|
int cpu, i;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
|
|
|
|
if (pcb->ptr) {
|
|
vunmap(pcb->ptr);
|
|
pcb->ptr = NULL;
|
|
}
|
|
if (!pcb->pages)
|
|
continue;
|
|
|
|
for (i = 0; i < pcb->nrpages; ++i)
|
|
if (pcb->pages[i])
|
|
put_page(pcb->pages[i]);
|
|
kfree(pcb->pages);
|
|
pcb->pages = NULL;
|
|
}
|
|
}
|