From 7ce9bf1f4785dab0598a19a7fcb0733a18193e4e Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 27 Aug 2012 20:27:19 +0200 Subject: [PATCH 1/9] mm: cma: fix alignment requirements for contiguous regions Contiguous Memory Allocator requires each of its regions to be aligned in such a way that it is possible to change migration type for all pageblocks holding it and then isolate page of largest possible order from the buddy allocator (which is MAX_ORDER-1). This patch relaxes alignment requirements by one order, because MAX_ORDER alignment is not really needed. Signed-off-by: Marek Szyprowski CC: Michal Nazarewicz Acked-by: Michal Nazarewicz --- drivers/base/dma-contiguous.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/base/dma-contiguous.c b/drivers/base/dma-contiguous.c index 78efb0306a44..34d94c762a1e 100644 --- a/drivers/base/dma-contiguous.c +++ b/drivers/base/dma-contiguous.c @@ -250,7 +250,7 @@ int __init dma_declare_contiguous(struct device *dev, unsigned long size, return -EINVAL; /* Sanitise input arguments */ - alignment = PAGE_SIZE << max(MAX_ORDER, pageblock_order); + alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order); base = ALIGN(base, alignment); size = ALIGN(size, alignment); limit &= ~(alignment - 1); From e092705bcd53de3bafc3053b0b55bf83e5d6711f Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 20 Aug 2012 14:39:39 +0200 Subject: [PATCH 2/9] ARM: relax conditions required for enabling Contiguous Memory Allocator Contiguous Memory Allocator requires only paging and MMU enabled not particular CPU architectures, so there is no need for strict dependency on CPU type. This enables to use CMA on some older ARM v5 systems which also might need large contiguous blocks for the multimedia processing hw modules. Reported-by: Prabhakar Lad Signed-off-by: Marek Szyprowski Tested-by: Prabhakar Lad --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 6d6e18fee9fe..0824a91ebc26 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -6,7 +6,7 @@ config ARM select HAVE_DMA_API_DEBUG select HAVE_IDE if PCI || ISA || PCMCIA select HAVE_DMA_ATTRS - select HAVE_DMA_CONTIGUOUS if (CPU_V6 || CPU_V6K || CPU_V7) + select HAVE_DMA_CONTIGUOUS if MMU select HAVE_MEMBLOCK select RTC_LIB select SYS_SUPPORTS_APM_EMULATION From 6e5267aa543817015edb4a65c66e15f9809f92bd Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 20 Aug 2012 11:19:25 +0200 Subject: [PATCH 3/9] ARM: DMA-Mapping: add function for setting coherent pool size from platform code Some platforms might require to increase atomic coherent pool to make sure that their device will be able to allocate all their buffers from atomic context. This function can be also used to decrease atomic coherent pool size if coherent allocations are not used for the given sub-platform. Suggested-by: Josh Coombs Signed-off-by: Marek Szyprowski --- arch/arm/include/asm/dma-mapping.h | 7 +++++++ arch/arm/mm/dma-mapping.c | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 2ae842df4551..5c44dcb0987b 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -202,6 +202,13 @@ static inline void dma_free_writecombine(struct device *dev, size_t size, return dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs); } +/* + * This can be called during early boot to increase the size of the atomic + * coherent DMA pool above the default value of 256KiB. It must be called + * before postcore_initcall. + */ +extern void __init init_dma_coherent_pool_size(unsigned long size); + /* * This can be called during boot to increase the size of the consistent * DMA region above it's default value of 2MB. It must be called before the diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 4e7d1182e8a3..d1cc9c1d3566 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -267,6 +267,8 @@ static void __dma_free_remap(void *cpu_addr, size_t size) vunmap(cpu_addr); } +#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K + struct dma_pool { size_t size; spinlock_t lock; @@ -277,7 +279,7 @@ struct dma_pool { }; static struct dma_pool atomic_pool = { - .size = SZ_256K, + .size = DEFAULT_DMA_COHERENT_POOL_SIZE, }; static int __init early_coherent_pool(char *p) @@ -287,6 +289,21 @@ static int __init early_coherent_pool(char *p) } early_param("coherent_pool", early_coherent_pool); +void __init init_dma_coherent_pool_size(unsigned long size) +{ + /* + * Catch any attempt to set the pool size too late. + */ + BUG_ON(atomic_pool.vaddr); + + /* + * Set architecture specific coherent pool size only if + * it has not been changed by kernel command line parameter. + */ + if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE) + atomic_pool.size = size; +} + /* * Initialise the coherent pool for atomic allocations. */ From fb71285f0c1633a85544784aae7577502274b77a Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Wed, 22 Aug 2012 14:50:42 +0200 Subject: [PATCH 4/9] ARM: DMA-Mapping: print warning when atomic coherent allocation fails Print a loud warning when system runs out of memory from atomic DMA coherent pool to let users notice the potential problem. Reported-by: Aaro Koskinen Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index d1cc9c1d3566..acced9332109 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -461,6 +461,10 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) bitmap_set(pool->bitmap, pageno, count); ptr = pool->vaddr + PAGE_SIZE * pageno; *ret_page = pool->page + pageno; + } else { + pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n" + "Please increase it with coherent_pool= kernel parameter!\n", + (unsigned)pool->size / 1024); } spin_unlock_irqrestore(&pool->lock, flags); From cb01b633eeb77ae7128cab0a3b5d3de56da6e913 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 28 Aug 2012 20:57:41 +0200 Subject: [PATCH 5/9] ARM: Kirkwood: increase atomic coherent pool size The default 256 KiB coherent pool may be too small for some of the Kirkwood devices, so increase it to make sure that devices will be able to allocate their buffers with GFP_ATOMIC flag. Suggested-by: Josh Coombs Signed-off-by: Marek Szyprowski Acked-by: Jason Cooper --- arch/arm/mach-kirkwood/common.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm/mach-kirkwood/common.c b/arch/arm/mach-kirkwood/common.c index c4b64adcbfce..d748f5033861 100644 --- a/arch/arm/mach-kirkwood/common.c +++ b/arch/arm/mach-kirkwood/common.c @@ -517,6 +517,13 @@ void __init kirkwood_wdt_init(void) void __init kirkwood_init_early(void) { orion_time_set_base(TIMER_VIRT_BASE); + + /* + * Some Kirkwood devices allocate their coherent buffers from atomic + * context. Increase size of atomic coherent pool to make sure such + * the allocations won't fail. + */ + init_dma_coherent_pool_size(SZ_1M); } int kirkwood_tclk; From 6b3fe47264262fa082897ebe8ae01041eae65e14 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Tue, 28 Aug 2012 08:13:01 +0300 Subject: [PATCH 6/9] ARM: dma-mapping: atomic_pool with struct page **pages struct page **pages is necessary to align with non atomic path in __iommu_get_pages(). atomic_pool() has the intialized **pages instead of just *page. Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index acced9332109..9a21284a6ac4 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -275,7 +275,7 @@ struct dma_pool { unsigned long *bitmap; unsigned long nr_pages; void *vaddr; - struct page *page; + struct page **pages; }; static struct dma_pool atomic_pool = { @@ -314,6 +314,7 @@ static int __init atomic_pool_init(void) unsigned long nr_pages = pool->size >> PAGE_SHIFT; unsigned long *bitmap; struct page *page; + struct page **pages; void *ptr; int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long); @@ -321,21 +322,31 @@ static int __init atomic_pool_init(void) if (!bitmap) goto no_bitmap; + pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL); + if (!pages) + goto no_pages; + if (IS_ENABLED(CONFIG_CMA)) ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page); else ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot, &page, NULL); if (ptr) { + int i; + + for (i = 0; i < nr_pages; i++) + pages[i] = page + i; + spin_lock_init(&pool->lock); pool->vaddr = ptr; - pool->page = page; + pool->pages = pages; pool->bitmap = bitmap; pool->nr_pages = nr_pages; pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n", (unsigned)pool->size / 1024); return 0; } +no_pages: kfree(bitmap); no_bitmap: pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", @@ -460,7 +471,7 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) if (pageno < pool->nr_pages) { bitmap_set(pool->bitmap, pageno, count); ptr = pool->vaddr + PAGE_SIZE * pageno; - *ret_page = pool->page + pageno; + *ret_page = pool->pages[pageno]; } else { pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n" "Please increase it with coherent_pool= kernel parameter!\n", From 21d0a75951ccf71f671eb24b61a8ad2b497be4b4 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Tue, 28 Aug 2012 08:13:02 +0300 Subject: [PATCH 7/9] ARM: dma-mapping: Refactor out to introduce __in_atomic_pool Check the given range("start", "size") is included in "atomic_pool" or not. Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 9a21284a6ac4..882eacc6ebc1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -482,20 +482,34 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) return ptr; } +static bool __in_atomic_pool(void *start, size_t size) +{ + struct dma_pool *pool = &atomic_pool; + void *end = start + size; + void *pool_start = pool->vaddr; + void *pool_end = pool->vaddr + pool->size; + + if (start < pool_start || start > pool_end) + return false; + + if (end <= pool_end) + return true; + + WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n", + start, end - 1, pool_start, pool_end - 1); + + return false; +} + static int __free_from_pool(void *start, size_t size) { struct dma_pool *pool = &atomic_pool; unsigned long pageno, count; unsigned long flags; - if (start < pool->vaddr || start > pool->vaddr + pool->size) + if (!__in_atomic_pool(start, size)) return 0; - if (start + size > pool->vaddr + pool->size) { - WARN(1, "freeing wrong coherent size from pool\n"); - return 0; - } - pageno = (start - pool->vaddr) >> PAGE_SHIFT; count = size >> PAGE_SHIFT; From 665bad7bb911d392000fa69bc6b599c0df992504 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Tue, 28 Aug 2012 08:13:03 +0300 Subject: [PATCH 8/9] ARM: dma-mapping: Introduce __atomic_get_pages() for __iommu_get_pages() Support atomic allocation in __iommu_get_pages(). Signed-off-by: Hiroshi Doyu [moved __atomic_get_pages() under #ifdef CONFIG_ARM_DMA_USE_IOMMU to avoid unused fuction warning for no-IOMMU case] Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 882eacc6ebc1..54b158df5afa 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1136,10 +1136,22 @@ static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t si return 0; } +static struct page **__atomic_get_pages(void *addr) +{ + struct dma_pool *pool = &atomic_pool; + struct page **pages = pool->pages; + int offs = (addr - pool->vaddr) >> PAGE_SHIFT; + + return pages + offs; +} + static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) { struct vm_struct *area; + if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) + return __atomic_get_pages(cpu_addr); + if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) return cpu_addr; From 479ed93a4b98eef03fd8260f7ddc00019221c450 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Tue, 28 Aug 2012 08:13:04 +0300 Subject: [PATCH 9/9] ARM: dma-mapping: IOMMU allocates pages from atomic_pool with GFP_ATOMIC Make use of the same atomic pool as DMA does, and skip a kernel page mapping which can involve sleep'able operations at allocating a kernel page table. Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 54b158df5afa..051204fc4617 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1161,6 +1161,34 @@ static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) return NULL; } +static void *__iommu_alloc_atomic(struct device *dev, size_t size, + dma_addr_t *handle) +{ + struct page *page; + void *addr; + + addr = __alloc_from_pool(size, &page); + if (!addr) + return NULL; + + *handle = __iommu_create_mapping(dev, &page, size); + if (*handle == DMA_ERROR_CODE) + goto err_mapping; + + return addr; + +err_mapping: + __free_from_pool(addr, size); + return NULL; +} + +static void __iommu_free_atomic(struct device *dev, struct page **pages, + dma_addr_t handle, size_t size) +{ + __iommu_remove_mapping(dev, handle, size); + __free_from_pool(page_address(pages[0]), size); +} + static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) { @@ -1171,6 +1199,9 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, *handle = DMA_ERROR_CODE; size = PAGE_ALIGN(size); + if (gfp & GFP_ATOMIC) + return __iommu_alloc_atomic(dev, size, handle); + pages = __iommu_alloc_buffer(dev, size, gfp); if (!pages) return NULL; @@ -1237,6 +1268,11 @@ void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, return; } + if (__in_atomic_pool(cpu_addr, size)) { + __iommu_free_atomic(dev, pages, handle, size); + return; + } + if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { unmap_kernel_range((unsigned long)cpu_addr, size); vunmap(cpu_addr);