Merge branch 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping
Pull DMA-mapping fixes from Marek Szyprowski:
"Another set of fixes for ARM dma-mapping subsystem.
Commit e9da6e9905
replaced custom consistent buffer remapping code
with generic vmalloc areas. It however introduced some regressions
caused by limited support for allocations in atomic context. This
series contains fixes for those regressions.
For some subplatforms the default, pre-allocated pool for atomic
allocations turned out to be too small, so a function for setting its
size has been added.
Another set of patches adds support for atomic allocations to
IOMMU-aware DMA-mapping implementation.
The last part of this pull request contains two fixes for Contiguous
Memory Allocator, which relax too strict requirements."
* 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping:
ARM: dma-mapping: IOMMU allocates pages from atomic_pool with GFP_ATOMIC
ARM: dma-mapping: Introduce __atomic_get_pages() for __iommu_get_pages()
ARM: dma-mapping: Refactor out to introduce __in_atomic_pool
ARM: dma-mapping: atomic_pool with struct page **pages
ARM: Kirkwood: increase atomic coherent pool size
ARM: DMA-Mapping: print warning when atomic coherent allocation fails
ARM: DMA-Mapping: add function for setting coherent pool size from platform code
ARM: relax conditions required for enabling Contiguous Memory Allocator
mm: cma: fix alignment requirements for contiguous regions
This commit is contained in:
commit
32d687cad3
|
@ -6,7 +6,7 @@ config ARM
|
|||
select HAVE_DMA_API_DEBUG
|
||||
select HAVE_IDE if PCI || ISA || PCMCIA
|
||||
select HAVE_DMA_ATTRS
|
||||
select HAVE_DMA_CONTIGUOUS if (CPU_V6 || CPU_V6K || CPU_V7)
|
||||
select HAVE_DMA_CONTIGUOUS if MMU
|
||||
select HAVE_MEMBLOCK
|
||||
select RTC_LIB
|
||||
select SYS_SUPPORTS_APM_EMULATION
|
||||
|
|
|
@ -202,6 +202,13 @@ static inline void dma_free_writecombine(struct device *dev, size_t size,
|
|||
return dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs);
|
||||
}
|
||||
|
||||
/*
|
||||
* This can be called during early boot to increase the size of the atomic
|
||||
* coherent DMA pool above the default value of 256KiB. It must be called
|
||||
* before postcore_initcall.
|
||||
*/
|
||||
extern void __init init_dma_coherent_pool_size(unsigned long size);
|
||||
|
||||
/*
|
||||
* This can be called during boot to increase the size of the consistent
|
||||
* DMA region above it's default value of 2MB. It must be called before the
|
||||
|
|
|
@ -517,6 +517,13 @@ void __init kirkwood_wdt_init(void)
|
|||
void __init kirkwood_init_early(void)
|
||||
{
|
||||
orion_time_set_base(TIMER_VIRT_BASE);
|
||||
|
||||
/*
|
||||
* Some Kirkwood devices allocate their coherent buffers from atomic
|
||||
* context. Increase size of atomic coherent pool to make sure such
|
||||
* the allocations won't fail.
|
||||
*/
|
||||
init_dma_coherent_pool_size(SZ_1M);
|
||||
}
|
||||
|
||||
int kirkwood_tclk;
|
||||
|
|
|
@ -267,17 +267,19 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
|
|||
vunmap(cpu_addr);
|
||||
}
|
||||
|
||||
#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
|
||||
|
||||
struct dma_pool {
|
||||
size_t size;
|
||||
spinlock_t lock;
|
||||
unsigned long *bitmap;
|
||||
unsigned long nr_pages;
|
||||
void *vaddr;
|
||||
struct page *page;
|
||||
struct page **pages;
|
||||
};
|
||||
|
||||
static struct dma_pool atomic_pool = {
|
||||
.size = SZ_256K,
|
||||
.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
|
||||
};
|
||||
|
||||
static int __init early_coherent_pool(char *p)
|
||||
|
@ -287,6 +289,21 @@ static int __init early_coherent_pool(char *p)
|
|||
}
|
||||
early_param("coherent_pool", early_coherent_pool);
|
||||
|
||||
void __init init_dma_coherent_pool_size(unsigned long size)
|
||||
{
|
||||
/*
|
||||
* Catch any attempt to set the pool size too late.
|
||||
*/
|
||||
BUG_ON(atomic_pool.vaddr);
|
||||
|
||||
/*
|
||||
* Set architecture specific coherent pool size only if
|
||||
* it has not been changed by kernel command line parameter.
|
||||
*/
|
||||
if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
|
||||
atomic_pool.size = size;
|
||||
}
|
||||
|
||||
/*
|
||||
* Initialise the coherent pool for atomic allocations.
|
||||
*/
|
||||
|
@ -297,6 +314,7 @@ static int __init atomic_pool_init(void)
|
|||
unsigned long nr_pages = pool->size >> PAGE_SHIFT;
|
||||
unsigned long *bitmap;
|
||||
struct page *page;
|
||||
struct page **pages;
|
||||
void *ptr;
|
||||
int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
|
||||
|
||||
|
@ -304,21 +322,31 @@ static int __init atomic_pool_init(void)
|
|||
if (!bitmap)
|
||||
goto no_bitmap;
|
||||
|
||||
pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
|
||||
if (!pages)
|
||||
goto no_pages;
|
||||
|
||||
if (IS_ENABLED(CONFIG_CMA))
|
||||
ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
|
||||
else
|
||||
ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot,
|
||||
&page, NULL);
|
||||
if (ptr) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < nr_pages; i++)
|
||||
pages[i] = page + i;
|
||||
|
||||
spin_lock_init(&pool->lock);
|
||||
pool->vaddr = ptr;
|
||||
pool->page = page;
|
||||
pool->pages = pages;
|
||||
pool->bitmap = bitmap;
|
||||
pool->nr_pages = nr_pages;
|
||||
pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
|
||||
(unsigned)pool->size / 1024);
|
||||
return 0;
|
||||
}
|
||||
no_pages:
|
||||
kfree(bitmap);
|
||||
no_bitmap:
|
||||
pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
|
||||
|
@ -443,27 +471,45 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page)
|
|||
if (pageno < pool->nr_pages) {
|
||||
bitmap_set(pool->bitmap, pageno, count);
|
||||
ptr = pool->vaddr + PAGE_SIZE * pageno;
|
||||
*ret_page = pool->page + pageno;
|
||||
*ret_page = pool->pages[pageno];
|
||||
} else {
|
||||
pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
|
||||
"Please increase it with coherent_pool= kernel parameter!\n",
|
||||
(unsigned)pool->size / 1024);
|
||||
}
|
||||
spin_unlock_irqrestore(&pool->lock, flags);
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static bool __in_atomic_pool(void *start, size_t size)
|
||||
{
|
||||
struct dma_pool *pool = &atomic_pool;
|
||||
void *end = start + size;
|
||||
void *pool_start = pool->vaddr;
|
||||
void *pool_end = pool->vaddr + pool->size;
|
||||
|
||||
if (start < pool_start || start > pool_end)
|
||||
return false;
|
||||
|
||||
if (end <= pool_end)
|
||||
return true;
|
||||
|
||||
WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
|
||||
start, end - 1, pool_start, pool_end - 1);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static int __free_from_pool(void *start, size_t size)
|
||||
{
|
||||
struct dma_pool *pool = &atomic_pool;
|
||||
unsigned long pageno, count;
|
||||
unsigned long flags;
|
||||
|
||||
if (start < pool->vaddr || start > pool->vaddr + pool->size)
|
||||
if (!__in_atomic_pool(start, size))
|
||||
return 0;
|
||||
|
||||
if (start + size > pool->vaddr + pool->size) {
|
||||
WARN(1, "freeing wrong coherent size from pool\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
pageno = (start - pool->vaddr) >> PAGE_SHIFT;
|
||||
count = size >> PAGE_SHIFT;
|
||||
|
||||
|
@ -1090,10 +1136,22 @@ static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t si
|
|||
return 0;
|
||||
}
|
||||
|
||||
static struct page **__atomic_get_pages(void *addr)
|
||||
{
|
||||
struct dma_pool *pool = &atomic_pool;
|
||||
struct page **pages = pool->pages;
|
||||
int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
|
||||
|
||||
return pages + offs;
|
||||
}
|
||||
|
||||
static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
|
||||
{
|
||||
struct vm_struct *area;
|
||||
|
||||
if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
|
||||
return __atomic_get_pages(cpu_addr);
|
||||
|
||||
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
|
||||
return cpu_addr;
|
||||
|
||||
|
@ -1103,6 +1161,34 @@ static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static void *__iommu_alloc_atomic(struct device *dev, size_t size,
|
||||
dma_addr_t *handle)
|
||||
{
|
||||
struct page *page;
|
||||
void *addr;
|
||||
|
||||
addr = __alloc_from_pool(size, &page);
|
||||
if (!addr)
|
||||
return NULL;
|
||||
|
||||
*handle = __iommu_create_mapping(dev, &page, size);
|
||||
if (*handle == DMA_ERROR_CODE)
|
||||
goto err_mapping;
|
||||
|
||||
return addr;
|
||||
|
||||
err_mapping:
|
||||
__free_from_pool(addr, size);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void __iommu_free_atomic(struct device *dev, struct page **pages,
|
||||
dma_addr_t handle, size_t size)
|
||||
{
|
||||
__iommu_remove_mapping(dev, handle, size);
|
||||
__free_from_pool(page_address(pages[0]), size);
|
||||
}
|
||||
|
||||
static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
|
||||
dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
|
||||
{
|
||||
|
@ -1113,6 +1199,9 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
|
|||
*handle = DMA_ERROR_CODE;
|
||||
size = PAGE_ALIGN(size);
|
||||
|
||||
if (gfp & GFP_ATOMIC)
|
||||
return __iommu_alloc_atomic(dev, size, handle);
|
||||
|
||||
pages = __iommu_alloc_buffer(dev, size, gfp);
|
||||
if (!pages)
|
||||
return NULL;
|
||||
|
@ -1179,6 +1268,11 @@ void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
|
|||
return;
|
||||
}
|
||||
|
||||
if (__in_atomic_pool(cpu_addr, size)) {
|
||||
__iommu_free_atomic(dev, pages, handle, size);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
|
||||
unmap_kernel_range((unsigned long)cpu_addr, size);
|
||||
vunmap(cpu_addr);
|
||||
|
|
|
@ -250,7 +250,7 @@ int __init dma_declare_contiguous(struct device *dev, unsigned long size,
|
|||
return -EINVAL;
|
||||
|
||||
/* Sanitise input arguments */
|
||||
alignment = PAGE_SIZE << max(MAX_ORDER, pageblock_order);
|
||||
alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order);
|
||||
base = ALIGN(base, alignment);
|
||||
size = ALIGN(size, alignment);
|
||||
limit &= ~(alignment - 1);
|
||||
|
|
Loading…
Reference in New Issue