hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branches 's390', 'arm/renesas', 'arm/msm', 'arm/shmobile', 'arm/smmu', 'x86/amd' and 'x86/vt-d' into next

This commit is contained in:
Joerg Roedel 2016-01-19 15:30:43 +01:00
parent afd2ff9b7e 7cd7578782 89ae54baab e7479a1907 ae50dc4874 6d6c7e56a2 1fb260bc00 bc8474549e
commit 32704253dc
17 changed files with 414 additions and 1053 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
View File

@ -7,7 +7,15 @@ connected to the IPMMU through a port called micro-TLB.
Required Properties:
- compatible: Must contain "renesas,ipmmu-vmsa".
- compatible: Must contain SoC-specific and generic entries from below.
- "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU.
- "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU.
- "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU.
- "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU.
- "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU.
- "renesas,ipmmu-vmsa" for generic R-Car Gen2 VMSA-compatible IPMMU.
- reg: Base address and size of the IPMMU registers.
- interrupts: Specifiers for the MMU fault interrupts. For instances that
support secure mode two interrupts must be specified, for non-secure and
@ -27,7 +35,7 @@ node with the following property:
Example: R8A7791 IPMMU-MX and VSP1-D0 bus master
ipmmu_mx: mmu@fe951000 {
compatible = "renasas,ipmmu-vmsa";
compatible = "renasas,ipmmu-r8a7791", "renasas,ipmmu-vmsa";
reg = <0 0xfe951000 0 0x1000>;
interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>,
<0 221 IRQ_TYPE_LEVEL_HIGH>;

75
drivers/iommu/Kconfig
View File

@ -263,81 +263,6 @@ config EXYNOS_IOMMU_DEBUG
Say N unless you need kernel log message for IOMMU debugging.
config SHMOBILE_IPMMU
bool
config SHMOBILE_IPMMU_TLB
bool
config SHMOBILE_IOMMU
bool "IOMMU for Renesas IPMMU/IPMMUI"
default n
depends on ARM && MMU
depends on ARCH_SHMOBILE || COMPILE_TEST
select IOMMU_API
select ARM_DMA_USE_IOMMU
select SHMOBILE_IPMMU
select SHMOBILE_IPMMU_TLB
help
Support for Renesas IPMMU/IPMMUI. This option enables
remapping of DMA memory accesses from all of the IP blocks
on the ICB.
Warning: Drivers (including userspace drivers of UIO
devices) of the IP blocks on the ICB *must* use addresses
allocated from the IPMMU (iova) for DMA with this option
enabled.
If unsure, say N.
choice
prompt "IPMMU/IPMMUI address space size"
default SHMOBILE_IOMMU_ADDRSIZE_2048MB
depends on SHMOBILE_IOMMU
help
This option sets IPMMU/IPMMUI address space size by
adjusting the 1st level page table size. The page table size
is calculated as follows:
page table size = number of page table entries * 4 bytes
number of page table entries = address space size / 1 MiB
For example, when the address space size is 2048 MiB, the
1st level page table size is 8192 bytes.
config SHMOBILE_IOMMU_ADDRSIZE_2048MB
bool "2 GiB"
config SHMOBILE_IOMMU_ADDRSIZE_1024MB
bool "1 GiB"
config SHMOBILE_IOMMU_ADDRSIZE_512MB
bool "512 MiB"
config SHMOBILE_IOMMU_ADDRSIZE_256MB
bool "256 MiB"
config SHMOBILE_IOMMU_ADDRSIZE_128MB
bool "128 MiB"
config SHMOBILE_IOMMU_ADDRSIZE_64MB
bool "64 MiB"
config SHMOBILE_IOMMU_ADDRSIZE_32MB
bool "32 MiB"
endchoice
config SHMOBILE_IOMMU_L1SIZE
int
default 8192 if SHMOBILE_IOMMU_ADDRSIZE_2048MB
default 4096 if SHMOBILE_IOMMU_ADDRSIZE_1024MB
default 2048 if SHMOBILE_IOMMU_ADDRSIZE_512MB
default 1024 if SHMOBILE_IOMMU_ADDRSIZE_256MB
default 512 if SHMOBILE_IOMMU_ADDRSIZE_128MB
default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB
default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB
config IPMMU_VMSA
bool "Renesas VMSA-compatible IPMMU"
depends on ARM_LPAE

2
drivers/iommu/Makefile
View File

@ -22,7 +22,5 @@ obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
obj-$(CONFIG_SHMOBILE_IOMMU) += shmobile-iommu.o
obj-$(CONFIG_SHMOBILE_IPMMU) += shmobile-ipmmu.o
obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o

400
drivers/iommu/amd_iommu.c
View File

@ -35,6 +35,7 @@
#include <linux/msi.h>
#include <linux/dma-contiguous.h>
#include <linux/irqdomain.h>
#include <linux/percpu.h>
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
@ -114,6 +115,45 @@ struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static int protection_domain_init(struct protection_domain *domain);
/*
* For dynamic growth the aperture size is split into ranges of 128MB of
* DMA address space each. This struct represents one such range.
*/
struct aperture_range {
spinlock_t bitmap_lock;
/* address allocation bitmap */
unsigned long *bitmap;
unsigned long offset;
unsigned long next_bit;
/*
* Array of PTE pages for the aperture. In this array we save all the
* leaf pages of the domain page table used for the aperture. This way
* we don't need to walk the page table to find a specific PTE. We can
* just calculate its address in constant time.
*/
u64 *pte_pages[64];
};
/*
* Data container for a dma_ops specific protection domain
*/
struct dma_ops_domain {
/* generic protection domain information */
struct protection_domain domain;
/* size of the aperture for the mappings */
unsigned long aperture_size;
/* aperture index we start searching for free addresses */
u32 __percpu *next_index;
/* address space relevant data */
struct aperture_range *aperture[APERTURE_MAX_RANGES];
};
/****************************************************************************
*
* Helper functions
@ -1167,11 +1207,21 @@ static u64 *alloc_pte(struct protection_domain *domain,
end_lvl = PAGE_SIZE_LEVEL(page_size);
while (level > end_lvl) {
if (!IOMMU_PTE_PRESENT(*pte)) {
u64 __pte, __npte;
__pte = *pte;
if (!IOMMU_PTE_PRESENT(__pte)) {
page = (u64 *)get_zeroed_page(gfp);
if (!page)
return NULL;
*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
__npte = PM_LEVEL_PDE(level, virt_to_phys(page));
if (cmpxchg64(pte, __pte, __npte)) {
free_page((unsigned long)page);
continue;
}
}
/* No level skipping support yet */
@ -1376,8 +1426,10 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
bool populate, gfp_t gfp)
{
int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
struct amd_iommu *iommu;
unsigned long i, old_size, pte_pgsize;
struct aperture_range *range;
struct amd_iommu *iommu;
unsigned long flags;
#ifdef CONFIG_IOMMU_STRESS
populate = false;
@ -1386,15 +1438,17 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
if (index >= APERTURE_MAX_RANGES)
return -ENOMEM;
dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
if (!dma_dom->aperture[index])
range = kzalloc(sizeof(struct aperture_range), gfp);
if (!range)
return -ENOMEM;
dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
if (!dma_dom->aperture[index]->bitmap)
range->bitmap = (void *)get_zeroed_page(gfp);
if (!range->bitmap)
goto out_free;
dma_dom->aperture[index]->offset = dma_dom->aperture_size;
range->offset = dma_dom->aperture_size;
spin_lock_init(&range->bitmap_lock);
if (populate) {
unsigned long address = dma_dom->aperture_size;
@ -1407,14 +1461,20 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
if (!pte)
goto out_free;
dma_dom->aperture[index]->pte_pages[i] = pte_page;
range->pte_pages[i] = pte_page;
address += APERTURE_RANGE_SIZE / 64;
}
}
old_size = dma_dom->aperture_size;
dma_dom->aperture_size += APERTURE_RANGE_SIZE;
spin_lock_irqsave(&dma_dom->domain.lock, flags);
/* First take the bitmap_lock and then publish the range */
spin_lock(&range->bitmap_lock);
old_size = dma_dom->aperture_size;
dma_dom->aperture[index] = range;
dma_dom->aperture_size += APERTURE_RANGE_SIZE;
/* Reserve address range used for MSI messages */
if (old_size < MSI_ADDR_BASE_LO &&
@ -1461,62 +1521,123 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
update_domain(&dma_dom->domain);
spin_unlock(&range->bitmap_lock);
spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
return 0;
out_free:
update_domain(&dma_dom->domain);
free_page((unsigned long)dma_dom->aperture[index]->bitmap);
free_page((unsigned long)range->bitmap);
kfree(dma_dom->aperture[index]);
dma_dom->aperture[index] = NULL;
kfree(range);
return -ENOMEM;
}
static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom,
struct aperture_range *range,
unsigned long pages,
unsigned long dma_mask,
unsigned long boundary_size,
unsigned long align_mask,
bool trylock)
{
unsigned long offset, limit, flags;
dma_addr_t address;
bool flush = false;
offset = range->offset >> PAGE_SHIFT;
limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
dma_mask >> PAGE_SHIFT);
if (trylock) {
if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
return -1;
} else {
spin_lock_irqsave(&range->bitmap_lock, flags);
}
address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
pages, offset, boundary_size, align_mask);
if (address == -1) {
/* Nothing found, retry one time */
address = iommu_area_alloc(range->bitmap, limit,
0, pages, offset, boundary_size,
align_mask);
flush = true;
}
if (address != -1)
range->next_bit = address + pages;
spin_unlock_irqrestore(&range->bitmap_lock, flags);
if (flush) {
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
}
return address;
}
static unsigned long dma_ops_area_alloc(struct device *dev,
struct dma_ops_domain *dom,
unsigned int pages,
unsigned long align_mask,
u64 dma_mask,
unsigned long start)
u64 dma_mask)
{
unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
int i = start >> APERTURE_RANGE_SHIFT;
unsigned long boundary_size, mask;
unsigned long address = -1;
unsigned long limit;
bool first = true;
u32 start, i;
next_bit >>= PAGE_SHIFT;
preempt_disable();
mask = dma_get_seg_boundary(dev);
again:
start = this_cpu_read(*dom->next_index);
/* Sanity check - is it really necessary? */
if (unlikely(start > APERTURE_MAX_RANGES)) {
start = 0;
this_cpu_write(*dom->next_index, 0);
}
boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
1UL << (BITS_PER_LONG - PAGE_SHIFT);
for (;i < max_index; ++i) {
unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
struct aperture_range *range;
int index;
if (dom->aperture[i]->offset >= dma_mask)
break;
index = (start + i) % APERTURE_MAX_RANGES;
limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
dma_mask >> PAGE_SHIFT);
range = dom->aperture[index];
address = iommu_area_alloc(dom->aperture[i]->bitmap,
limit, next_bit, pages, 0,
boundary_size, align_mask);
if (!range || range->offset >= dma_mask)
continue;
address = dma_ops_aperture_alloc(dom, range, pages,
dma_mask, boundary_size,
align_mask, first);
if (address != -1) {
address = dom->aperture[i]->offset +
(address << PAGE_SHIFT);
dom->next_address = address + (pages << PAGE_SHIFT);
address = range->offset + (address << PAGE_SHIFT);
this_cpu_write(*dom->next_index, index);
break;
}
next_bit = 0;
}
if (address == -1 && first) {
first = false;
goto again;
}
preempt_enable();
return address;
}
@ -1526,21 +1647,14 @@ static unsigned long dma_ops_alloc_addresses(struct device *dev,
unsigned long align_mask,
u64 dma_mask)
{
unsigned long address;
unsigned long address = -1;
#ifdef CONFIG_IOMMU_STRESS
dom->next_address = 0;
dom->need_flush = true;
#endif
while (address == -1) {
address = dma_ops_area_alloc(dev, dom, pages,
align_mask, dma_mask);
address = dma_ops_area_alloc(dev, dom, pages, align_mask,
dma_mask, dom->next_address);
if (address == -1) {
dom->next_address = 0;
address = dma_ops_area_alloc(dev, dom, pages, align_mask,
dma_mask, 0);
dom->need_flush = true;
if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
break;
}
if (unlikely(address == -1))
@ -1562,6 +1676,7 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
{
unsigned i = address >> APERTURE_RANGE_SHIFT;
struct aperture_range *range = dom->aperture[i];
unsigned long flags;
BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
@ -1570,12 +1685,18 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
return;
#endif
if (address >= dom->next_address)
dom->need_flush = true;
if (amd_iommu_unmap_flush) {
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
}
address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
spin_lock_irqsave(&range->bitmap_lock, flags);
if (address + pages > range->next_bit)
range->next_bit = address + pages;
bitmap_clear(range->bitmap, address, pages);
spin_unlock_irqrestore(&range->bitmap_lock, flags);
}
@ -1755,6 +1876,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
if (!dom)
return;
free_percpu(dom->next_index);
del_domain_from_list(&dom->domain);
free_pagetable(&dom->domain);
@ -1769,6 +1892,23 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
kfree(dom);
}
static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
int max_apertures)
{
int ret, i, apertures;
apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
ret = 0;
for (i = apertures; i < max_apertures; ++i) {
ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
if (ret)
break;
}
return ret;
}
/*
* Allocates a new protection domain usable for the dma_ops functions.
* It also initializes the page table and the address allocator data
@ -1777,6 +1917,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
static struct dma_ops_domain *dma_ops_domain_alloc(void)
{
struct dma_ops_domain *dma_dom;
int cpu;
dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
if (!dma_dom)
@ -1785,6 +1926,10 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (protection_domain_init(&dma_dom->domain))
goto free_dma_dom;
dma_dom->next_index = alloc_percpu(u32);
if (!dma_dom->next_index)
goto free_dma_dom;
dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
dma_dom->domain.flags = PD_DMA_OPS_MASK;
@ -1792,8 +1937,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (!dma_dom->domain.pt_root)
goto free_dma_dom;
dma_dom->need_flush = false;
add_domain_to_list(&dma_dom->domain);
if (alloc_new_range(dma_dom, true, GFP_KERNEL))
@ -1804,8 +1947,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
* a valid dma-address. So we can use 0 as error value
*/
dma_dom->aperture[0]->bitmap[0] = 1;
dma_dom->next_address = 0;
for_each_possible_cpu(cpu)
*per_cpu_ptr(dma_dom->next_index, cpu) = 0;
return dma_dom;
@ -2328,7 +2472,7 @@ static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
else if (direction == DMA_BIDIRECTIONAL)
__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
WARN_ON(*pte);
WARN_ON_ONCE(*pte);
*pte = __pte;
@ -2357,7 +2501,7 @@ static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
pte += PM_LEVEL_INDEX(0, address);
WARN_ON(!*pte);
WARN_ON_ONCE(!*pte);
*pte = 0ULL;
}
@ -2393,26 +2537,11 @@ static dma_addr_t __map_single(struct device *dev,
if (align)
align_mask = (1UL << get_order(size)) - 1;
retry:
address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
dma_mask);
if (unlikely(address == DMA_ERROR_CODE)) {
/*
* setting next_address here will let the address
* allocator only scan the new allocated range in the
* first run. This is a small optimization.
*/
dma_dom->next_address = dma_dom->aperture_size;
if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
goto out;
/*
* aperture was successfully enlarged by 128 MB, try
* allocation again
*/
goto retry;
}
if (address == DMA_ERROR_CODE)
goto out;
start = address;
for (i = 0; i < pages; ++i) {
@ -2427,11 +2556,10 @@ retry:
ADD_STATS_COUNTER(alloced_io_mem, size);
if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
domain_flush_tlb(&dma_dom->domain);
dma_dom->need_flush = false;
} else if (unlikely(amd_iommu_np_cache))
if (unlikely(amd_iommu_np_cache)) {
domain_flush_pages(&dma_dom->domain, address, size);
domain_flush_complete(&dma_dom->domain);
}
out:
return address;
@ -2478,11 +2606,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
SUB_STATS_COUNTER(alloced_io_mem, size);
dma_ops_free_addresses(dma_dom, dma_addr, pages);
if (amd_iommu_unmap_flush || dma_dom->need_flush) {
domain_flush_pages(&dma_dom->domain, flush_addr, size);
dma_dom->need_flush = false;
}
}
/*
@ -2493,11 +2616,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
enum dma_data_direction dir,
struct dma_attrs *attrs)
{
unsigned long flags;
struct protection_domain *domain;
dma_addr_t addr;
u64 dma_mask;
phys_addr_t paddr = page_to_phys(page) + offset;
struct protection_domain *domain;
u64 dma_mask;
INC_STATS_COUNTER(cnt_map_single);
@ -2509,19 +2630,8 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
dma_mask = *dev->dma_mask;
spin_lock_irqsave(&domain->lock, flags);
addr = __map_single(dev, domain->priv, paddr, size, dir, false,
return __map_single(dev, domain->priv, paddr, size, dir, false,
dma_mask);
if (addr == DMA_ERROR_CODE)
goto out;
domain_flush_complete(domain);
out:
spin_unlock_irqrestore(&domain->lock, flags);
return addr;
}
/*
@ -2530,7 +2640,6 @@ out:
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
unsigned long flags;
struct protection_domain *domain;
INC_STATS_COUNTER(cnt_unmap_single);
@ -2539,13 +2648,7 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
if (IS_ERR(domain))
return;
spin_lock_irqsave(&domain->lock, flags);
__unmap_single(domain->priv, dma_addr, size, dir);
domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
/*
@ -2556,7 +2659,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
unsigned long flags;
struct protection_domain *domain;
int i;
struct scatterlist *s;
@ -2572,8 +2674,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
dma_mask = *dev->dma_mask;
spin_lock_irqsave(&domain->lock, flags);
for_each_sg(sglist, s, nelems, i) {
paddr = sg_phys(s);
@ -2588,12 +2688,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
goto unmap;
}
domain_flush_complete(domain);
out:
spin_unlock_irqrestore(&domain->lock, flags);
return mapped_elems;
unmap:
for_each_sg(sglist, s, mapped_elems, i) {
if (s->dma_address)
@ -2602,9 +2698,7 @@ unmap:
s->dma_address = s->dma_length = 0;
}
mapped_elems = 0;
goto out;
return 0;
}
/*
@ -2615,7 +2709,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
unsigned long flags;
struct protection_domain *domain;
struct scatterlist *s;
int i;
@ -2626,17 +2719,11 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
if (IS_ERR(domain))
return;
spin_lock_irqsave(&domain->lock, flags);
for_each_sg(sglist, s, nelems, i) {
__unmap_single(domain->priv, s->dma_address,
s->dma_length, dir);
s->dma_address = s->dma_length = 0;
}
domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
/*
@ -2648,7 +2735,6 @@ static void *alloc_coherent(struct device *dev, size_t size,
{
u64 dma_mask = dev->coherent_dma_mask;
struct protection_domain *domain;
unsigned long flags;
struct page *page;
INC_STATS_COUNTER(cnt_alloc_coherent);
@ -2680,19 +2766,11 @@ static void *alloc_coherent(struct device *dev, size_t size,
if (!dma_mask)
dma_mask = *dev->dma_mask;
spin_lock_irqsave(&domain->lock, flags);
*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
size, DMA_BIDIRECTIONAL, true, dma_mask);
if (*dma_addr == DMA_ERROR_CODE) {
spin_unlock_irqrestore(&domain->lock, flags);
if (*dma_addr == DMA_ERROR_CODE)
goto out_free;
}
domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
return page_address(page);
@ -2712,7 +2790,6 @@ static void free_coherent(struct device *dev, size_t size,
struct dma_attrs *attrs)
{
struct protection_domain *domain;
unsigned long flags;
struct page *page;
INC_STATS_COUNTER(cnt_free_coherent);
@ -2724,14 +2801,8 @@ static void free_coherent(struct device *dev, size_t size,
if (IS_ERR(domain))
goto free_mem;
spin_lock_irqsave(&domain->lock, flags);
__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
free_mem:
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, get_order(size));
@ -2746,14 +2817,43 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
return check_device(dev);
}
static int set_dma_mask(struct device *dev, u64 mask)
{
struct protection_domain *domain;
int max_apertures = 1;
domain = get_domain(dev);
if (IS_ERR(domain))
return PTR_ERR(domain);
if (mask == DMA_BIT_MASK(64))
max_apertures = 8;
else if (mask > DMA_BIT_MASK(32))
max_apertures = 4;
/*
* To prevent lock contention it doesn't make sense to allocate more
* apertures than online cpus
*/
if (max_apertures > num_online_cpus())
max_apertures = num_online_cpus();
if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures))
dev_err(dev, "Can't allocate %d iommu apertures\n",
max_apertures);
return 0;
}
static struct dma_map_ops amd_iommu_dma_ops = {
.alloc = alloc_coherent,
.free = free_coherent,
.map_page = map_page,
.unmap_page = unmap_page,
.map_sg = map_sg,
.unmap_sg = unmap_sg,
.dma_supported = amd_iommu_dma_supported,
.alloc = alloc_coherent,
.free = free_coherent,
.map_page = map_page,
.unmap_page = unmap_page,
.map_sg = map_sg,
.unmap_sg = unmap_sg,
.dma_supported = amd_iommu_dma_supported,
.set_dma_mask = set_dma_mask,
};
int __init amd_iommu_init_api(void)
@ -3757,11 +3857,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
case X86_IRQ_ALLOC_TYPE_MSI:
case X86_IRQ_ALLOC_TYPE_MSIX:
devid = get_device_id(&info->msi_dev->dev);
if (devid >= 0) {
iommu = amd_iommu_rlookup_table[devid];
if (iommu)
return iommu->msi_domain;
}
iommu = amd_iommu_rlookup_table[devid];
if (iommu)
return iommu->msi_domain;
break;
default:
break;

40
drivers/iommu/amd_iommu_types.h
View File

@ -424,46 +424,6 @@ struct protection_domain {
void *priv; /* private data */
};
/*
* For dynamic growth the aperture size is split into ranges of 128MB of
* DMA address space each. This struct represents one such range.
*/
struct aperture_range {
/* address allocation bitmap */
unsigned long *bitmap;
/*
* Array of PTE pages for the aperture. In this array we save all the
* leaf pages of the domain page table used for the aperture. This way
* we don't need to walk the page table to find a specific PTE. We can
* just calculate its address in constant time.
*/
u64 *pte_pages[64];
unsigned long offset;
};
/*
* Data container for a dma_ops specific protection domain
*/
struct dma_ops_domain {
/* generic protection domain information */
struct protection_domain domain;
/* size of the aperture for the mappings */
unsigned long aperture_size;
/* address we start to search for free addresses */
unsigned long next_address;
/* address space relevant data */
struct aperture_range *aperture[APERTURE_MAX_RANGES];
/* This will be set to true when TLB needs to be flushed */
bool need_flush;
};
/*
* Structure where we save information about one hardware AMD IOMMU in the
* system.

40
drivers/iommu/amd_iommu_v2.c
View File

@ -432,7 +432,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
unbind_pasid(pasid_state);
}
static struct mmu_notifier_ops iommu_mn = {
static const struct mmu_notifier_ops iommu_mn = {
.release = mn_release,
.clear_flush_young = mn_clear_flush_young,
.invalidate_page = mn_invalidate_page,
@ -513,43 +513,39 @@ static bool access_error(struct vm_area_struct *vma, struct fault *fault)
static void do_fault(struct work_struct *work)
{
struct fault *fault = container_of(work, struct fault, work);
struct mm_struct *mm;
struct vm_area_struct *vma;
int ret = VM_FAULT_ERROR;
unsigned int flags = 0;
struct mm_struct *mm;
u64 address;
int ret, write;
write = !!(fault->flags & PPR_FAULT_WRITE);
mm = fault->state->mm;
address = fault->address;
if (fault->flags & PPR_FAULT_USER)
flags |= FAULT_FLAG_USER;
if (fault->flags & PPR_FAULT_WRITE)
flags |= FAULT_FLAG_WRITE;
down_read(&mm->mmap_sem);
vma = find_extend_vma(mm, address);
if (!vma || address < vma->vm_start) {
if (!vma || address < vma->vm_start)
/* failed to get a vma in the right range */
up_read(&mm->mmap_sem);
handle_fault_error(fault);
goto out;
}
/* Check if we have the right permissions on the vma */
if (access_error(vma, fault)) {
up_read(&mm->mmap_sem);
handle_fault_error(fault);
if (access_error(vma, fault))
goto out;
}
ret = handle_mm_fault(mm, vma, address, write);
if (ret & VM_FAULT_ERROR) {
/* failed to service fault */
up_read(&mm->mmap_sem);
handle_fault_error(fault);
goto out;
}
up_read(&mm->mmap_sem);
ret = handle_mm_fault(mm, vma, address, flags);
out:
up_read(&mm->mmap_sem);
if (ret & VM_FAULT_ERROR)
/* failed to service fault */
handle_fault_error(fault);
finish_pri_tag(fault->dev_state, fault->state, fault->tag);
put_pasid_state(fault->state);

210
drivers/iommu/arm-smmu-v3.c
View File

@ -40,7 +40,10 @@
#define IDR0_ST_LVL_SHIFT 27
#define IDR0_ST_LVL_MASK 0x3
#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
#define IDR0_STALL_MODEL (3 << 24)
#define IDR0_STALL_MODEL_SHIFT 24
#define IDR0_STALL_MODEL_MASK 0x3
#define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT)
#define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT)
#define IDR0_TTENDIAN_SHIFT 21
#define IDR0_TTENDIAN_MASK 0x3
#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
@ -253,6 +256,9 @@
#define STRTAB_STE_1_STRW_EL2 2UL
#define STRTAB_STE_1_STRW_SHIFT 30
#define STRTAB_STE_1_SHCFG_INCOMING 1UL
#define STRTAB_STE_1_SHCFG_SHIFT 44
#define STRTAB_STE_2_S2VMID_SHIFT 0
#define STRTAB_STE_2_S2VMID_MASK 0xffffUL
#define STRTAB_STE_2_VTCR_SHIFT 32
@ -378,7 +384,6 @@
#define PRIQ_0_SID_MASK 0xffffffffUL
#define PRIQ_0_SSID_SHIFT 32
#define PRIQ_0_SSID_MASK 0xfffffUL
#define PRIQ_0_OF (1UL << 57)
#define PRIQ_0_PERM_PRIV (1UL << 58)
#define PRIQ_0_PERM_EXEC (1UL << 59)
#define PRIQ_0_PERM_READ (1UL << 60)
@ -855,15 +860,17 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
};
dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
cerror_str[idx]);
idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
switch (idx) {
case CMDQ_ERR_CERROR_ILL_IDX:
break;
case CMDQ_ERR_CERROR_ABT_IDX:
dev_err(smmu->dev, "retrying command fetch\n");
case CMDQ_ERR_CERROR_NONE_IDX:
return;
case CMDQ_ERR_CERROR_ILL_IDX:
/* Fallthrough */
default:
break;
}
/*
@ -1042,6 +1049,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
: STRTAB_STE_0_CFG_BYPASS;
dst[0] = cpu_to_le64(val);
dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
<< STRTAB_STE_1_SHCFG_SHIFT);
dst[2] = 0; /* Nuke the VMID */
if (ste_live)
arm_smmu_sync_ste_for_sid(smmu, sid);
@ -1056,12 +1065,14 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
STRTAB_STE_1_S1C_CACHE_WBRA
<< STRTAB_STE_1_S1COR_SHIFT |
STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
STRTAB_STE_1_S1STALLD |
#ifdef CONFIG_PCI_ATS
STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
#endif
STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
if (smmu->features & ARM_SMMU_FEAT_STALLS)
dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
<< STRTAB_STE_0_S1CTXPTR_SHIFT) |
STRTAB_STE_0_CFG_S1_TRANS;
@ -1123,8 +1134,8 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
desc->span = STRTAB_SPLIT + 1;
desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
GFP_KERNEL);
desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
GFP_KERNEL | __GFP_ZERO);
if (!desc->l2ptr) {
dev_err(smmu->dev,
"failed to allocate l2 stream table for SID %u\n",
@ -1250,50 +1261,50 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
{
u32 gerror, gerrorn;
u32 gerror, gerrorn, active;
struct arm_smmu_device *smmu = dev;
gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
gerror ^= gerrorn;
if (!(gerror & GERROR_ERR_MASK))
active = gerror ^ gerrorn;
if (!(active & GERROR_ERR_MASK))
return IRQ_NONE; /* No errors pending */
dev_warn(smmu->dev,
"unexpected global error reported (0x%08x), this could be serious\n",
gerror);
active);
if (gerror & GERROR_SFM_ERR) {
if (active & GERROR_SFM_ERR) {
dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
arm_smmu_device_disable(smmu);
}
if (gerror & GERROR_MSI_GERROR_ABT_ERR)
if (active & GERROR_MSI_GERROR_ABT_ERR)
dev_warn(smmu->dev, "GERROR MSI write aborted\n");
if (gerror & GERROR_MSI_PRIQ_ABT_ERR) {
if (active & GERROR_MSI_PRIQ_ABT_ERR) {
dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
arm_smmu_priq_handler(irq, smmu->dev);
}
if (gerror & GERROR_MSI_EVTQ_ABT_ERR) {
if (active & GERROR_MSI_EVTQ_ABT_ERR) {
dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
arm_smmu_evtq_handler(irq, smmu->dev);
}
if (gerror & GERROR_MSI_CMDQ_ABT_ERR) {
if (active & GERROR_MSI_CMDQ_ABT_ERR) {
dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
arm_smmu_cmdq_sync_handler(irq, smmu->dev);
}
if (gerror & GERROR_PRIQ_ABT_ERR)
if (active & GERROR_PRIQ_ABT_ERR)
dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
if (gerror & GERROR_EVTQ_ABT_ERR)
if (active & GERROR_EVTQ_ABT_ERR)
dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
if (gerror & GERROR_CMDQ_ERR)
if (active & GERROR_CMDQ_ERR)
arm_smmu_cmdq_skip_err(smmu);
writel(gerror, smmu->base + ARM_SMMU_GERRORN);
@ -1335,7 +1346,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
}
static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
bool leaf, void *cookie)
size_t granule, bool leaf, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_device *smmu = smmu_domain->smmu;
@ -1354,7 +1365,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
}
arm_smmu_cmdq_issue_cmd(smmu, &cmd);
do {
arm_smmu_cmdq_issue_cmd(smmu, &cmd);
cmd.tlbi.addr += granule;
} while (size -= granule);
}
static struct iommu_gather_ops arm_smmu_gather_ops = {
@ -1429,10 +1443,10 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
if (cfg->cdptr) {
dma_free_coherent(smmu_domain->smmu->dev,
CTXDESC_CD_DWORDS << 3,
cfg->cdptr,
cfg->cdptr_dma);
dmam_free_coherent(smmu_domain->smmu->dev,
CTXDESC_CD_DWORDS << 3,
cfg->cdptr,
cfg->cdptr_dma);
arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
}
@ -1457,8 +1471,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
if (IS_ERR_VALUE(asid))
return asid;
cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
&cfg->cdptr_dma, GFP_KERNEL);
cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
&cfg->cdptr_dma,
GFP_KERNEL | __GFP_ZERO);
if (!cfg->cdptr) {
dev_warn(smmu->dev, "failed to allocate context descriptor\n");
ret = -ENOMEM;
@ -1804,13 +1819,13 @@ static int arm_smmu_add_device(struct device *dev)
smmu = arm_smmu_get_for_pci_dev(pdev);
if (!smmu) {
ret = -ENOENT;
goto out_put_group;
goto out_remove_dev;
}
smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
if (!smmu_group) {
ret = -ENOMEM;
goto out_put_group;
goto out_remove_dev;
}
smmu_group->ste.valid = true;
@ -1826,20 +1841,20 @@ static int arm_smmu_add_device(struct device *dev)
for (i = 0; i < smmu_group->num_sids; ++i) {
/* If we already know about this SID, then we're done */
if (smmu_group->sids[i] == sid)
return 0;
goto out_put_group;
}
/* Check the SID is in range of the SMMU and our stream table */
if (!arm_smmu_sid_in_range(smmu, sid)) {
ret = -ERANGE;
goto out_put_group;
goto out_remove_dev;
}
/* Ensure l2 strtab is initialised */
if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
ret = arm_smmu_init_l2_strtab(smmu, sid);
if (ret)
goto out_put_group;
goto out_remove_dev;
}
/* Resize the SID array for the group */
@ -1849,15 +1864,19 @@ static int arm_smmu_add_device(struct device *dev)
if (!sids) {
smmu_group->num_sids--;
ret = -ENOMEM;
goto out_put_group;
goto out_remove_dev;
}
/* Add the new SID */
sids[smmu_group->num_sids - 1] = sid;
smmu_group->sids = sids;
return 0;
out_put_group:
iommu_group_put(group);
return 0;
out_remove_dev:
iommu_group_remove_device(dev);
iommu_group_put(group);
return ret;
}
@ -1937,7 +1956,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
{
size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
if (!q->base) {
dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
qsz);
@ -1957,23 +1976,6 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
return 0;
}
static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu,
struct arm_smmu_queue *q)
{
size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;
dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);
}
static void arm_smmu_free_queues(struct arm_smmu_device *smmu)
{
arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
if (smmu->features & ARM_SMMU_FEAT_PRI)
arm_smmu_free_one_queue(smmu, &smmu->priq.q);
}
static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
{
int ret;
@ -1983,49 +1985,20 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
if (ret)
goto out;
return ret;
/* evtq */
ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
if (ret)
goto out_free_cmdq;
return ret;
/* priq */
if (!(smmu->features & ARM_SMMU_FEAT_PRI))
return 0;
ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
if (ret)
goto out_free_evtq;
return 0;
out_free_evtq:
arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
out_free_cmdq:
arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
out:
return ret;
}
static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu)
{
int i;
size_t size;
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
for (i = 0; i < cfg->num_l1_ents; ++i) {
struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i];
if (!desc->l2ptr)
continue;
dma_free_coherent(smmu->dev, size, desc->l2ptr,
desc->l2ptr_dma);
}
return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
}
static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
@ -2054,7 +2027,6 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
void *strtab;
u64 reg;
u32 size, l1size;
int ret;
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
/*
@ -2077,8 +2049,8 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
size, smmu->sid_bits);
l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
GFP_KERNEL);
strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
GFP_KERNEL | __GFP_ZERO);
if (!strtab) {
dev_err(smmu->dev,
"failed to allocate l1 stream table (%u bytes)\n",
@ -2095,13 +2067,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
<< STRTAB_BASE_CFG_SPLIT_SHIFT;
cfg->strtab_base_cfg = reg;
ret = arm_smmu_init_l1_strtab(smmu);
if (ret)
dma_free_coherent(smmu->dev,
l1size,
strtab,
cfg->strtab_dma);
return ret;
return arm_smmu_init_l1_strtab(smmu);
}
static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
@ -2112,8 +2078,8 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma,
GFP_KERNEL);
strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
GFP_KERNEL | __GFP_ZERO);
if (!strtab) {
dev_err(smmu->dev,
"failed to allocate linear stream table (%u bytes)\n",
@ -2157,21 +2123,6 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
return 0;
}
static void arm_smmu_free_strtab(struct arm_smmu_device *smmu)
{
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
u32 size = cfg->num_l1_ents;
if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
arm_smmu_free_l2_strtab(smmu);
size *= STRTAB_L1_DESC_DWORDS << 3;
} else {
size *= STRTAB_STE_DWORDS * 3;
}
dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma);
}
static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
{
int ret;
@ -2180,21 +2131,7 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
if (ret)
return ret;
ret = arm_smmu_init_strtab(smmu);
if (ret)
goto out_free_queues;
return 0;
out_free_queues:
arm_smmu_free_queues(smmu);
return ret;
}
static void arm_smmu_free_structures(struct arm_smmu_device *smmu)
{
arm_smmu_free_strtab(smmu);
arm_smmu_free_queues(smmu);
return arm_smmu_init_strtab(smmu);
}
static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
@ -2532,8 +2469,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
coherent ? "true" : "false");
if (reg & IDR0_STALL_MODEL)
switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
case IDR0_STALL_MODEL_STALL:
/* Fallthrough */
case IDR0_STALL_MODEL_FORCE:
smmu->features |= ARM_SMMU_FEAT_STALLS;
}
if (reg & IDR0_S1P)
smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
@ -2699,15 +2640,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, smmu);
/* Reset the device */
ret = arm_smmu_device_reset(smmu);
if (ret)
goto out_free_structures;
return 0;
out_free_structures:
arm_smmu_free_structures(smmu);
return ret;
return arm_smmu_device_reset(smmu);
}
static int arm_smmu_device_remove(struct platform_device *pdev)
@ -2715,7 +2648,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
arm_smmu_device_disable(smmu);
arm_smmu_free_structures(smmu);
return 0;
}

23
drivers/iommu/arm-smmu.c
View File

@ -582,7 +582,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
}
static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
bool leaf, void *cookie)
size_t granule, bool leaf, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
@ -597,12 +597,18 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {
iova &= ~12UL;
iova |= ARM_SMMU_CB_ASID(cfg);
writel_relaxed(iova, reg);
do {
writel_relaxed(iova, reg);
iova += granule;
} while (size -= granule);
#ifdef CONFIG_64BIT
} else {
iova >>= 12;
iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48;
writeq_relaxed(iova, reg);
do {
writeq_relaxed(iova, reg);
iova += granule >> 12;
} while (size -= granule);
#endif
}
#ifdef CONFIG_64BIT
@ -610,7 +616,11 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
ARM_SMMU_CB_S2_TLBIIPAS2;
writeq_relaxed(iova >> 12, reg);
iova >>= 12;
do {
writeq_relaxed(iova, reg);
iova += granule >> 12;
} while (size -= granule);
#endif
} else {
reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
@ -945,9 +955,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
free_irq(irq, domain);
}
if (smmu_domain->pgtbl_ops)
free_io_pgtable_ops(smmu_domain->pgtbl_ops);
free_io_pgtable_ops(smmu_domain->pgtbl_ops);
__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
}
@ -1357,6 +1365,7 @@ static int arm_smmu_add_device(struct device *dev)
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
return 0;
}

12
drivers/iommu/dmar.c
View File

@ -1063,13 +1063,19 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
raw_spin_lock_init(&iommu->register_lock);
drhd->iommu = iommu;
if (intel_iommu_enabled)
if (intel_iommu_enabled) {
iommu->iommu_dev = iommu_device_create(NULL, iommu,
intel_iommu_groups,
"%s", iommu->name);
if (IS_ERR(iommu->iommu_dev)) {
err = PTR_ERR(iommu->iommu_dev);
goto err_unmap;
}
}
drhd->iommu = iommu;
return 0;
err_unmap:

49
drivers/iommu/io-pgtable-arm.c
View File

@ -38,9 +38,6 @@
#define io_pgtable_to_data(x) \
container_of((x), struct arm_lpae_io_pgtable, iop)
#define io_pgtable_ops_to_pgtable(x) \
container_of((x), struct io_pgtable, ops)
#define io_pgtable_ops_to_data(x) \
io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
@ -58,8 +55,10 @@
((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \
* (d)->bits_per_level) + (d)->pg_shift)
#define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift)
#define ARM_LPAE_PAGES_PER_PGD(d) \
DIV_ROUND_UP((d)->pgd_size, 1UL << (d)->pg_shift)
DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d))
/*
* Calculate the index at level l used to map virtual address a using the
@ -169,7 +168,7 @@
/* IOPTE accessors */
#define iopte_deref(pte,d) \
(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
& ~((1ULL << (d)->pg_shift) - 1)))
& ~(ARM_LPAE_GRANULE(d) - 1ULL)))
#define iopte_type(pte,l) \
(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
@ -326,7 +325,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
/* Grab a pointer to the next level */
pte = *ptep;
if (!pte) {
cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data),
GFP_ATOMIC, cfg);
if (!cptep)
return -ENOMEM;
@ -405,17 +404,18 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
arm_lpae_iopte *start, *end;
unsigned long table_size;
/* Only leaf entries at the last level */
if (lvl == ARM_LPAE_MAX_LEVELS - 1)
return;
if (lvl == ARM_LPAE_START_LVL(data))
table_size = data->pgd_size;
else
table_size = 1UL << data->pg_shift;
table_size = ARM_LPAE_GRANULE(data);
start = ptep;
end = (void *)ptep + table_size;
/* Only leaf entries at the last level */
if (lvl == ARM_LPAE_MAX_LEVELS - 1)
end = ptep;
else
end = (void *)ptep + table_size;
while (ptep != end) {
arm_lpae_iopte pte = *ptep++;
@ -473,7 +473,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
__arm_lpae_set_pte(ptep, table, cfg);
iova &= ~(blk_size - 1);
cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie);
return size;
}
@ -486,11 +486,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
void *cookie = data->iop.cookie;
size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
/* Something went horribly wrong and we ran out of page table */
if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
return 0;
ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
pte = *ptep;
/* Something went horribly wrong and we ran out of page table */
if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS)))
if (WARN_ON(!pte))
return 0;
/* If the size matches this level, we're in the right place */
@ -499,12 +501,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
if (!iopte_leaf(pte, lvl)) {
/* Also flush any partial walks */
tlb->tlb_add_flush(iova, size, false, cookie);
tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data),
false, cookie);
tlb->tlb_sync(cookie);
ptep = iopte_deref(pte, data);
__arm_lpae_free_pgtable(data, lvl + 1, ptep);
} else {
tlb->tlb_add_flush(iova, size, true, cookie);
tlb->tlb_add_flush(iova, size, size, true, cookie);
}
return size;
@ -570,7 +573,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
return 0;
found_translation:
iova &= ((1 << data->pg_shift) - 1);
iova &= (ARM_LPAE_GRANULE(data) - 1);
return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
}
@ -668,7 +671,7 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
switch (1 << data->pg_shift) {
switch (ARM_LPAE_GRANULE(data)) {
case SZ_4K:
reg |= ARM_LPAE_TCR_TG0_4K;
break;
@ -769,7 +772,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
sl = ARM_LPAE_START_LVL(data);
switch (1 << data->pg_shift) {
switch (ARM_LPAE_GRANULE(data)) {
case SZ_4K:
reg |= ARM_LPAE_TCR_TG0_4K;
sl++; /* SL0 format is different for 4K granule size */
@ -889,8 +892,8 @@ static void dummy_tlb_flush_all(void *cookie)
WARN_ON(cookie != cfg_cookie);
}
static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
void *cookie)
static void dummy_tlb_add_flush(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
{
WARN_ON(cookie != cfg_cookie);
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));

6
drivers/iommu/io-pgtable.h
View File

@ -26,8 +26,8 @@ enum io_pgtable_fmt {
*/
struct iommu_gather_ops {
void (*tlb_flush_all)(void *cookie);
void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf,
void *cookie);
void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,
bool leaf, void *cookie);
void (*tlb_sync)(void *cookie);
};
@ -131,6 +131,8 @@ struct io_pgtable {
struct io_pgtable_ops ops;
};
#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
/**
* struct io_pgtable_init_fns - Alloc/free a set of page tables for a
* particular format.

4
drivers/iommu/ipmmu-vmsa.c
View File

@ -277,8 +277,8 @@ static void ipmmu_tlb_flush_all(void *cookie)
ipmmu_tlb_invalidate(domain);
}
static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
void *cookie)
static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
{
/* The hardware doesn't support selective TLB flush. */
}

25
drivers/iommu/msm_iommu_dev.c
View File

@ -359,30 +359,19 @@ static struct platform_driver msm_iommu_ctx_driver = {
.remove = msm_iommu_ctx_remove,
};
static struct platform_driver * const drivers[] = {
&msm_iommu_driver,
&msm_iommu_ctx_driver,
};
static int __init msm_iommu_driver_init(void)
{
int ret;
ret = platform_driver_register(&msm_iommu_driver);
if (ret != 0) {
pr_err("Failed to register IOMMU driver\n");
goto error;
}
ret = platform_driver_register(&msm_iommu_ctx_driver);
if (ret != 0) {
platform_driver_unregister(&msm_iommu_driver);
pr_err("Failed to register IOMMU context driver\n");
goto error;
}
error:
return ret;
return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
}
static void __exit msm_iommu_driver_exit(void)
{
platform_driver_unregister(&msm_iommu_ctx_driver);
platform_driver_unregister(&msm_iommu_driver);
platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
}
subsys_initcall(msm_iommu_driver_init);

4
drivers/iommu/s390-iommu.c
View File

@ -49,7 +49,7 @@ static bool s390_iommu_capable(enum iommu_cap cap)
}
}
struct iommu_domain *s390_domain_alloc(unsigned domain_type)
static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
{
struct s390_domain *s390_domain;
@ -73,7 +73,7 @@ struct iommu_domain *s390_domain_alloc(unsigned domain_type)
return &s390_domain->domain;
}
void s390_domain_free(struct iommu_domain *domain)
static void s390_domain_free(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);

402
drivers/iommu/shmobile-iommu.c
View File

@ -1,402 +0,0 @@
/*
* IOMMU for IPMMU/IPMMUI
* Copyright (C) 2012 Hideki EIRAKU
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License.
*/
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/platform_device.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <asm/dma-iommu.h>
#include "shmobile-ipmmu.h"
#define L1_SIZE CONFIG_SHMOBILE_IOMMU_L1SIZE
#define L1_LEN (L1_SIZE / 4)
#define L1_ALIGN L1_SIZE
#define L2_SIZE SZ_1K
#define L2_LEN (L2_SIZE / 4)
#define L2_ALIGN L2_SIZE
struct shmobile_iommu_domain_pgtable {
uint32_t *pgtable;
dma_addr_t handle;
};
struct shmobile_iommu_archdata {
struct list_head attached_list;
struct dma_iommu_mapping *iommu_mapping;
spinlock_t attach_lock;
struct shmobile_iommu_domain *attached;
int num_attached_devices;
struct shmobile_ipmmu *ipmmu;
};
struct shmobile_iommu_domain {
struct shmobile_iommu_domain_pgtable l1, l2[L1_LEN];
spinlock_t map_lock;
spinlock_t attached_list_lock;
struct list_head attached_list;
struct iommu_domain domain;
};
static struct shmobile_iommu_archdata *ipmmu_archdata;
static struct kmem_cache *l1cache, *l2cache;
static struct shmobile_iommu_domain *to_sh_domain(struct iommu_domain *dom)
{
return container_of(dom, struct shmobile_iommu_domain, domain);
}
static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable,
struct kmem_cache *cache, size_t size)
{
pgtable->pgtable = kmem_cache_zalloc(cache, GFP_ATOMIC);
if (!pgtable->pgtable)
return -ENOMEM;
pgtable->handle = dma_map_single(NULL, pgtable->pgtable, size,
DMA_TO_DEVICE);
return 0;
}
static void pgtable_free(struct shmobile_iommu_domain_pgtable *pgtable,
struct kmem_cache *cache, size_t size)
{
dma_unmap_single(NULL, pgtable->handle, size, DMA_TO_DEVICE);
kmem_cache_free(cache, pgtable->pgtable);
}
static uint32_t pgtable_read(struct shmobile_iommu_domain_pgtable *pgtable,
unsigned int index)
{
return pgtable->pgtable[index];
}
static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable,
unsigned int index, unsigned int count, uint32_t val)
{
unsigned int i;
for (i = 0; i < count; i++)
pgtable->pgtable[index + i] = val;
dma_sync_single_for_device(NULL, pgtable->handle + index * sizeof(val),
sizeof(val) * count, DMA_TO_DEVICE);
}
static struct iommu_domain *shmobile_iommu_domain_alloc(unsigned type)
{
struct shmobile_iommu_domain *sh_domain;
int i, ret;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
sh_domain = kzalloc(sizeof(*sh_domain), GFP_KERNEL);
if (!sh_domain)
return NULL;
ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE);
if (ret < 0) {
kfree(sh_domain);
return NULL;
}
for (i = 0; i < L1_LEN; i++)
sh_domain->l2[i].pgtable = NULL;
spin_lock_init(&sh_domain->map_lock);
spin_lock_init(&sh_domain->attached_list_lock);
INIT_LIST_HEAD(&sh_domain->attached_list);
return &sh_domain->domain;
}
static void shmobile_iommu_domain_free(struct iommu_domain *domain)
{
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
int i;
for (i = 0; i < L1_LEN; i++) {
if (sh_domain->l2[i].pgtable)
pgtable_free(&sh_domain->l2[i], l2cache, L2_SIZE);
}
pgtable_free(&sh_domain->l1, l1cache, L1_SIZE);
kfree(sh_domain);
}
static int shmobile_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
int ret = -EBUSY;
if (!archdata)
return -ENODEV;
spin_lock(&sh_domain->attached_list_lock);
spin_lock(&archdata->attach_lock);
if (archdata->attached != sh_domain) {
if (archdata->attached)
goto err;
ipmmu_tlb_set(archdata->ipmmu, sh_domain->l1.handle, L1_SIZE,
0);
ipmmu_tlb_flush(archdata->ipmmu);
archdata->attached = sh_domain;
archdata->num_attached_devices = 0;
list_add(&archdata->attached_list, &sh_domain->attached_list);
}
archdata->num_attached_devices++;
ret = 0;
err:
spin_unlock(&archdata->attach_lock);
spin_unlock(&sh_domain->attached_list_lock);
return ret;
}
static void shmobile_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
if (!archdata)
return;
spin_lock(&sh_domain->attached_list_lock);
spin_lock(&archdata->attach_lock);
archdata->num_attached_devices--;
if (!archdata->num_attached_devices) {
ipmmu_tlb_set(archdata->ipmmu, 0, 0, 0);
ipmmu_tlb_flush(archdata->ipmmu);
archdata->attached = NULL;
list_del(&archdata->attached_list);
}
spin_unlock(&archdata->attach_lock);
spin_unlock(&sh_domain->attached_list_lock);
}
static void domain_tlb_flush(struct shmobile_iommu_domain *sh_domain)
{
struct shmobile_iommu_archdata *archdata;
spin_lock(&sh_domain->attached_list_lock);
list_for_each_entry(archdata, &sh_domain->attached_list, attached_list)
ipmmu_tlb_flush(archdata->ipmmu);
spin_unlock(&sh_domain->attached_list_lock);
}
static int l2alloc(struct shmobile_iommu_domain *sh_domain,
unsigned int l1index)
{
int ret;
if (!sh_domain->l2[l1index].pgtable) {
ret = pgtable_alloc(&sh_domain->l2[l1index], l2cache, L2_SIZE);
if (ret < 0)
return ret;
}
pgtable_write(&sh_domain->l1, l1index, 1,
sh_domain->l2[l1index].handle | 0x1);
return 0;
}
static void l2realfree(struct shmobile_iommu_domain_pgtable *l2)
{
if (l2->pgtable)
pgtable_free(l2, l2cache, L2_SIZE);
}
static void l2free(struct shmobile_iommu_domain *sh_domain,
unsigned int l1index,
struct shmobile_iommu_domain_pgtable *l2)
{
pgtable_write(&sh_domain->l1, l1index, 1, 0);
if (sh_domain->l2[l1index].pgtable) {
*l2 = sh_domain->l2[l1index];
sh_domain->l2[l1index].pgtable = NULL;
}
}
static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
unsigned int l1index, l2index;
int ret;
l1index = iova >> 20;
switch (size) {
case SZ_4K:
l2index = (iova >> 12) & 0xff;
spin_lock(&sh_domain->map_lock);
ret = l2alloc(sh_domain, l1index);
if (!ret)
pgtable_write(&sh_domain->l2[l1index], l2index, 1,
paddr | 0xff2);
spin_unlock(&sh_domain->map_lock);
break;
case SZ_64K:
l2index = (iova >> 12) & 0xf0;
spin_lock(&sh_domain->map_lock);
ret = l2alloc(sh_domain, l1index);
if (!ret)
pgtable_write(&sh_domain->l2[l1index], l2index, 0x10,
paddr | 0xff1);
spin_unlock(&sh_domain->map_lock);
break;
case SZ_1M:
spin_lock(&sh_domain->map_lock);
l2free(sh_domain, l1index, &l2);
pgtable_write(&sh_domain->l1, l1index, 1, paddr | 0xc02);
spin_unlock(&sh_domain->map_lock);
ret = 0;
break;
default:
ret = -EINVAL;
}
if (!ret)
domain_tlb_flush(sh_domain);
l2realfree(&l2);
return ret;
}
static size_t shmobile_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
unsigned int l1index, l2index;
uint32_t l2entry = 0;
size_t ret = 0;
l1index = iova >> 20;
if (!(iova & 0xfffff) && size >= SZ_1M) {
spin_lock(&sh_domain->map_lock);
l2free(sh_domain, l1index, &l2);
spin_unlock(&sh_domain->map_lock);
ret = SZ_1M;
goto done;
}
l2index = (iova >> 12) & 0xff;
spin_lock(&sh_domain->map_lock);
if (sh_domain->l2[l1index].pgtable)
l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
switch (l2entry & 3) {
case 1:
if (l2index & 0xf)
break;
pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, 0);
ret = SZ_64K;
break;
case 2:
pgtable_write(&sh_domain->l2[l1index], l2index, 1, 0);
ret = SZ_4K;
break;
}
spin_unlock(&sh_domain->map_lock);
done:
if (ret)
domain_tlb_flush(sh_domain);
l2realfree(&l2);
return ret;
}
static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
uint32_t l1entry = 0, l2entry = 0;
unsigned int l1index, l2index;
l1index = iova >> 20;
l2index = (iova >> 12) & 0xff;
spin_lock(&sh_domain->map_lock);
if (sh_domain->l2[l1index].pgtable)
l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
else
l1entry = pgtable_read(&sh_domain->l1, l1index);
spin_unlock(&sh_domain->map_lock);
switch (l2entry & 3) {
case 1:
return (l2entry & ~0xffff) | (iova & 0xffff);
case 2:
return (l2entry & ~0xfff) | (iova & 0xfff);
default:
if ((l1entry & 3) == 2)
return (l1entry & ~0xfffff) | (iova & 0xfffff);
return 0;
}
}
static int find_dev_name(struct shmobile_ipmmu *ipmmu, const char *dev_name)
{
unsigned int i, n = ipmmu->num_dev_names;
for (i = 0; i < n; i++) {
if (strcmp(ipmmu->dev_names[i], dev_name) == 0)
return 1;
}
return 0;
}
static int shmobile_iommu_add_device(struct device *dev)
{
struct shmobile_iommu_archdata *archdata = ipmmu_archdata;
struct dma_iommu_mapping *mapping;
if (!find_dev_name(archdata->ipmmu, dev_name(dev)))
return 0;
mapping = archdata->iommu_mapping;
if (!mapping) {
mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
L1_LEN << 20);
if (IS_ERR(mapping))
return PTR_ERR(mapping);
archdata->iommu_mapping = mapping;
}
dev->archdata.iommu = archdata;
if (arm_iommu_attach_device(dev, mapping))
pr_err("arm_iommu_attach_device failed\n");
return 0;
}
static const struct iommu_ops shmobile_iommu_ops = {
.domain_alloc = shmobile_iommu_domain_alloc,
.domain_free = shmobile_iommu_domain_free,
.attach_dev = shmobile_iommu_attach_device,
.detach_dev = shmobile_iommu_detach_device,
.map = shmobile_iommu_map,
.unmap = shmobile_iommu_unmap,
.map_sg = default_iommu_map_sg,
.iova_to_phys = shmobile_iommu_iova_to_phys,
.add_device = shmobile_iommu_add_device,
.pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K,
};
int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
{
static struct shmobile_iommu_archdata *archdata;
l1cache = kmem_cache_create("shmobile-iommu-pgtable1", L1_SIZE,
L1_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
if (!l1cache)
return -ENOMEM;
l2cache = kmem_cache_create("shmobile-iommu-pgtable2", L2_SIZE,
L2_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
if (!l2cache) {
kmem_cache_destroy(l1cache);
return -ENOMEM;
}
archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
if (!archdata) {
kmem_cache_destroy(l1cache);
kmem_cache_destroy(l2cache);
return -ENOMEM;
}
spin_lock_init(&archdata->attach_lock);
archdata->ipmmu = ipmmu;
ipmmu_archdata = archdata;
bus_set_iommu(&platform_bus_type, &shmobile_iommu_ops);
return 0;
}

129
drivers/iommu/shmobile-ipmmu.c
View File

@ -1,129 +0,0 @@
/*
* IPMMU/IPMMUI
* Copyright (C) 2012 Hideki EIRAKU
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License.
*/
#include <linux/err.h>
#include <linux/export.h>
#include <linux/io.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/platform_data/sh_ipmmu.h>
#include "shmobile-ipmmu.h"
#define IMCTR1 0x000
#define IMCTR2 0x004
#define IMASID 0x010
#define IMTTBR 0x014
#define IMTTBCR 0x018
#define IMCTR1_TLBEN (1 << 0)
#define IMCTR1_FLUSH (1 << 1)
static void ipmmu_reg_write(struct shmobile_ipmmu *ipmmu, unsigned long reg_off,
unsigned long data)
{
iowrite32(data, ipmmu->ipmmu_base + reg_off);
}
void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu)
{
if (!ipmmu)
return;
spin_lock(&ipmmu->flush_lock);
if (ipmmu->tlb_enabled)
ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH | IMCTR1_TLBEN);
else
ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH);
spin_unlock(&ipmmu->flush_lock);
}
void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
int asid)
{
if (!ipmmu)
return;
spin_lock(&ipmmu->flush_lock);
switch (size) {
default:
ipmmu->tlb_enabled = 0;
break;
case 0x2000:
ipmmu_reg_write(ipmmu, IMTTBCR, 1);
ipmmu->tlb_enabled = 1;
break;
case 0x1000:
ipmmu_reg_write(ipmmu, IMTTBCR, 2);
ipmmu->tlb_enabled = 1;
break;
case 0x800:
ipmmu_reg_write(ipmmu, IMTTBCR, 3);
ipmmu->tlb_enabled = 1;
break;
case 0x400:
ipmmu_reg_write(ipmmu, IMTTBCR, 4);
ipmmu->tlb_enabled = 1;
break;
case 0x200:
ipmmu_reg_write(ipmmu, IMTTBCR, 5);
ipmmu->tlb_enabled = 1;
break;
case 0x100:
ipmmu_reg_write(ipmmu, IMTTBCR, 6);
ipmmu->tlb_enabled = 1;
break;
case 0x80:
ipmmu_reg_write(ipmmu, IMTTBCR, 7);
ipmmu->tlb_enabled = 1;
break;
}
ipmmu_reg_write(ipmmu, IMTTBR, phys);
ipmmu_reg_write(ipmmu, IMASID, asid);
spin_unlock(&ipmmu->flush_lock);
}
static int ipmmu_probe(struct platform_device *pdev)
{
struct shmobile_ipmmu *ipmmu;
struct resource *res;
struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data;
ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL);
if (!ipmmu) {
dev_err(&pdev->dev, "cannot allocate device data\n");
return -ENOMEM;
}
spin_lock_init(&ipmmu->flush_lock);
ipmmu->dev = &pdev->dev;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res);
if (IS_ERR(ipmmu->ipmmu_base))
return PTR_ERR(ipmmu->ipmmu_base);
ipmmu->dev_names = pdata->dev_names;
ipmmu->num_dev_names = pdata->num_dev_names;
platform_set_drvdata(pdev, ipmmu);
ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */
ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */
return ipmmu_iommu_init(ipmmu);
}
static struct platform_driver ipmmu_driver = {
.probe = ipmmu_probe,
.driver = {
.name = "ipmmu",
},
};
static int __init ipmmu_init(void)
{
return platform_driver_register(&ipmmu_driver);
}
subsys_initcall(ipmmu_init);

34
drivers/iommu/shmobile-ipmmu.h
View File

@ -1,34 +0,0 @@
/* shmobile-ipmmu.h
*
* Copyright (C) 2012 Hideki EIRAKU
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License.
*/
#ifndef __SHMOBILE_IPMMU_H__
#define __SHMOBILE_IPMMU_H__
struct shmobile_ipmmu {
struct device *dev;
void __iomem *ipmmu_base;
int tlb_enabled;
spinlock_t flush_lock;
const char * const *dev_names;
unsigned int num_dev_names;
};
#ifdef CONFIG_SHMOBILE_IPMMU_TLB
void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu);
void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
int asid);
int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu);
#else
static inline int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
{
return -EINVAL;
}
#endif
#endif /* __SHMOBILE_IPMMU_H__ */