From 579b2a65d245c093d3e63845c320b9321f112b75 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Fri, 6 Jan 2017 18:58:08 +0530 Subject: [PATCH 1/9] iommu: add IOMMU_PRIV attribute Add the IOMMU_PRIV attribute, which is used to indicate privileged mappings. Reviewed-by: Robin Murphy Tested-by: Robin Murphy Signed-off-by: Mitchel Humpherys Acked-by: Will Deacon Signed-off-by: Will Deacon --- include/linux/iommu.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0ff5111f6959..69e2417a2965 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -31,6 +31,13 @@ #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ #define IOMMU_NOEXEC (1 << 3) #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ +/* + * This is to make the IOMMU API setup privileged + * mapppings accessible by the master only at higher + * privileged execution level and inaccessible at + * less privileged levels. + */ +#define IOMMU_PRIV (1 << 5) struct iommu_ops; struct iommu_group; From e7468a23daec1a7737a004c659fd041ddff8b918 Mon Sep 17 00:00:00 2001 From: Jeremy Gebben Date: Fri, 6 Jan 2017 18:58:09 +0530 Subject: [PATCH 2/9] iommu/io-pgtable-arm: add support for the IOMMU_PRIV flag Allow the creation of privileged mode mappings, for stage 1 only. Reviewed-by: Robin Murphy Tested-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Jeremy Gebben Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index a40ce3406fef..feacc54bec68 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -350,11 +350,14 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, if (data->iop.fmt == ARM_64_LPAE_S1 || data->iop.fmt == ARM_32_LPAE_S1) { - pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG; + pte = ARM_LPAE_PTE_nG; if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) pte |= ARM_LPAE_PTE_AP_RDONLY; + if (!(prot & IOMMU_PRIV)) + pte |= ARM_LPAE_PTE_AP_UNPRIV; + if (prot & IOMMU_MMIO) pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV << ARM_LPAE_PTE_ATTRINDX_SHIFT); From 5baf1e9d0bb223b086c422b880359c9fe2ee1476 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 6 Jan 2017 18:58:10 +0530 Subject: [PATCH 3/9] iommu/io-pgtable-arm-v7s: Add support for the IOMMU_PRIV flag The short-descriptor format also allows privileged-only mappings, so let's wire it up. Signed-off-by: Robin Murphy Tested-by: Sricharan R Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 0769276c0537..1c049e2e12bf 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -265,7 +265,9 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, if (!(prot & IOMMU_MMIO)) pte |= ARM_V7S_ATTR_TEX(1); if (ap) { - pte |= ARM_V7S_PTE_AF | ARM_V7S_PTE_AP_UNPRIV; + pte |= ARM_V7S_PTE_AF; + if (!(prot & IOMMU_PRIV)) + pte |= ARM_V7S_PTE_AP_UNPRIV; if (!(prot & IOMMU_WRITE)) pte |= ARM_V7S_PTE_AP_RDONLY; } @@ -288,6 +290,8 @@ static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) if (!(attr & ARM_V7S_PTE_AP_RDONLY)) prot |= IOMMU_WRITE; + if (!(attr & ARM_V7S_PTE_AP_UNPRIV)) + prot |= IOMMU_PRIV; if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0) prot |= IOMMU_MMIO; else if (pte & ARM_V7S_ATTR_C) From b2fb366425ceb85dca56afa538257ec5a2c4f6d1 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Fri, 6 Jan 2017 18:58:11 +0530 Subject: [PATCH 4/9] common: DMA-mapping: add DMA_ATTR_PRIVILEGED attribute This patch adds the DMA_ATTR_PRIVILEGED attribute to the DMA-mapping subsystem. Some advanced peripherals such as remote processors and GPUs perform accesses to DMA buffers in both privileged "supervisor" and unprivileged "user" modes. This attribute is used to indicate to the DMA-mapping subsystem that the buffer is fully accessible at the elevated privilege level (and ideally inaccessible or at least read-only at the lesser-privileged levels). Cc: linux-doc@vger.kernel.org Reviewed-by: Robin Murphy Tested-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Mitchel Humpherys Signed-off-by: Will Deacon --- Documentation/DMA-attributes.txt | 10 ++++++++++ include/linux/dma-mapping.h | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/Documentation/DMA-attributes.txt b/Documentation/DMA-attributes.txt index 98bf7ac29aad..44c6bc496eee 100644 --- a/Documentation/DMA-attributes.txt +++ b/Documentation/DMA-attributes.txt @@ -143,3 +143,13 @@ So, this provides a way for drivers to avoid those error messages on calls where allocation failures are not a problem, and shouldn't bother the logs. NOTE: At the moment DMA_ATTR_NO_WARN is only implemented on PowerPC. + +DMA_ATTR_PRIVILEGED +------------------------------ + +Some advanced peripherals such as remote processors and GPUs perform +accesses to DMA buffers in both privileged "supervisor" and unprivileged +"user" modes. This attribute is used to indicate to the DMA-mapping +subsystem that the buffer is fully accessible at the elevated privilege +level (and ideally inaccessible or at least read-only at the +lesser-privileged levels). diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 10c5a17b1f51..c24721a33b4c 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -62,6 +62,13 @@ */ #define DMA_ATTR_NO_WARN (1UL << 8) +/* + * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully + * accessible at an elevated privilege level (and ideally inaccessible or + * at least read-only at lesser-privileged levels). + */ +#define DMA_ATTR_PRIVILEGED (1UL << 9) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. * It can be given to a device to use as a DMA source or target. A CPU cannot From 737c85ca1c3af4f97acb61cd53415ec039b31111 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Fri, 6 Jan 2017 18:58:12 +0530 Subject: [PATCH 5/9] arm64/dma-mapping: Implement DMA_ATTR_PRIVILEGED The newly added DMA_ATTR_PRIVILEGED is useful for creating mappings that are only accessible to privileged DMA engines. Implement it in dma-iommu.c so that the ARM64 DMA IOMMU mapper can make use of it. Reviewed-by: Robin Murphy Tested-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Mitchel Humpherys Signed-off-by: Will Deacon --- arch/arm64/mm/dma-mapping.c | 6 +++--- drivers/iommu/dma-iommu.c | 12 +++++++++--- include/linux/dma-iommu.h | 3 ++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index e04082700bb1..ae9f817eadf2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -558,7 +558,7 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, unsigned long attrs) { bool coherent = is_device_dma_coherent(dev); - int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); size_t iosize = size; void *addr; @@ -712,7 +712,7 @@ static dma_addr_t __iommu_map_page(struct device *dev, struct page *page, unsigned long attrs) { bool coherent = is_device_dma_coherent(dev); - int prot = dma_direction_to_prot(dir, coherent); + int prot = dma_info_to_prot(dir, coherent, attrs); dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); if (!iommu_dma_mapping_error(dev, dev_addr) && @@ -770,7 +770,7 @@ static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl, __iommu_sync_sg_for_device(dev, sgl, nelems, dir); return iommu_dma_map_sg(dev, sgl, nelems, - dma_direction_to_prot(dir, coherent)); + dma_info_to_prot(dir, coherent, attrs)); } static void __iommu_unmap_sg_attrs(struct device *dev, diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 2db0d641cf45..3006eeebf521 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -181,16 +181,22 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, EXPORT_SYMBOL(iommu_dma_init_domain); /** - * dma_direction_to_prot - Translate DMA API directions to IOMMU API page flags + * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API + * page flags. * @dir: Direction of DMA transfer * @coherent: Is the DMA master cache-coherent? + * @attrs: DMA attributes for the mapping * * Return: corresponding IOMMU API page protection flags */ -int dma_direction_to_prot(enum dma_data_direction dir, bool coherent) +int dma_info_to_prot(enum dma_data_direction dir, bool coherent, + unsigned long attrs) { int prot = coherent ? IOMMU_CACHE : 0; + if (attrs & DMA_ATTR_PRIVILEGED) + prot |= IOMMU_PRIV; + switch (dir) { case DMA_BIDIRECTIONAL: return prot | IOMMU_READ | IOMMU_WRITE; @@ -633,7 +639,7 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { return __iommu_dma_map(dev, phys, size, - dma_direction_to_prot(dir, false) | IOMMU_MMIO); + dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO); } void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 7f7e9a7e3839..c5511e1d5560 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -34,7 +34,8 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size, struct device *dev); /* General helpers for DMA-API <-> IOMMU-API interaction */ -int dma_direction_to_prot(enum dma_data_direction dir, bool coherent); +int dma_info_to_prot(enum dma_data_direction dir, bool coherent, + unsigned long attrs); /* * These implement the bulk of the relevant DMA mapping callbacks, but require From 7d2822dfea122acd498a8ec4481af5d51777fec1 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Fri, 6 Jan 2017 18:58:13 +0530 Subject: [PATCH 6/9] arm/dma-mapping: Implement DMA_ATTR_PRIVILEGED The newly added DMA_ATTR_PRIVILEGED is useful for creating mappings that are only accessible to privileged DMA engines. Adding it to the arm dma-mapping.c so that the ARM32 DMA IOMMU mapper can make use of it. Signed-off-by: Sricharan R Reviewed-by: Will Deacon Signed-off-by: Will Deacon --- arch/arm/mm/dma-mapping.c | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ab7710002ba6..82d3e79ec82b 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1171,6 +1171,25 @@ core_initcall(dma_debug_do_init); #ifdef CONFIG_ARM_DMA_USE_IOMMU +static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs) +{ + int prot = 0; + + if (attrs & DMA_ATTR_PRIVILEGED) + prot |= IOMMU_PRIV; + + switch (dir) { + case DMA_BIDIRECTIONAL: + return prot | IOMMU_READ | IOMMU_WRITE; + case DMA_TO_DEVICE: + return prot | IOMMU_READ; + case DMA_FROM_DEVICE: + return prot | IOMMU_WRITE; + default: + return prot; + } +} + /* IOMMU */ static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); @@ -1394,7 +1413,8 @@ __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, * Create a mapping in device IO address space for specified pages */ static dma_addr_t -__iommu_create_mapping(struct device *dev, struct page **pages, size_t size) +__iommu_create_mapping(struct device *dev, struct page **pages, size_t size, + unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -1419,7 +1439,7 @@ __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) len = (j - i) << PAGE_SHIFT; ret = iommu_map(mapping->domain, iova, phys, len, - IOMMU_READ|IOMMU_WRITE); + __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs)); if (ret < 0) goto fail; iova += len; @@ -1476,7 +1496,8 @@ static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs) } static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp, - dma_addr_t *handle, int coherent_flag) + dma_addr_t *handle, int coherent_flag, + unsigned long attrs) { struct page *page; void *addr; @@ -1488,7 +1509,7 @@ static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp, if (!addr) return NULL; - *handle = __iommu_create_mapping(dev, &page, size); + *handle = __iommu_create_mapping(dev, &page, size, attrs); if (*handle == DMA_ERROR_CODE) goto err_mapping; @@ -1522,7 +1543,7 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp)) return __iommu_alloc_simple(dev, size, gfp, handle, - coherent_flag); + coherent_flag, attrs); /* * Following is a work-around (a.k.a. hack) to prevent pages @@ -1537,7 +1558,7 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, if (!pages) return NULL; - *handle = __iommu_create_mapping(dev, pages, size); + *handle = __iommu_create_mapping(dev, pages, size, attrs); if (*handle == DMA_ERROR_CODE) goto err_buffer; @@ -1672,27 +1693,6 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, GFP_KERNEL); } -static int __dma_direction_to_prot(enum dma_data_direction dir) -{ - int prot; - - switch (dir) { - case DMA_BIDIRECTIONAL: - prot = IOMMU_READ | IOMMU_WRITE; - break; - case DMA_TO_DEVICE: - prot = IOMMU_READ; - break; - case DMA_FROM_DEVICE: - prot = IOMMU_WRITE; - break; - default: - prot = 0; - } - - return prot; -} - /* * Map a part of the scatter-gather list into contiguous io address space */ @@ -1722,7 +1722,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); - prot = __dma_direction_to_prot(dir); + prot = __dma_info_to_prot(dir, attrs); ret = iommu_map(mapping->domain, iova, phys, len, prot); if (ret < 0) @@ -1930,7 +1930,7 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p if (dma_addr == DMA_ERROR_CODE) return dma_addr; - prot = __dma_direction_to_prot(dir); + prot = __dma_info_to_prot(dir, attrs); ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); if (ret < 0) @@ -2036,7 +2036,7 @@ static dma_addr_t arm_iommu_map_resource(struct device *dev, if (dma_addr == DMA_ERROR_CODE) return dma_addr; - prot = __dma_direction_to_prot(dir) | IOMMU_MMIO; + prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; ret = iommu_map(mapping->domain, dma_addr, addr, len, prot); if (ret < 0) From 1b2354dbcc7b232b50f26dfd47fd4c529edfc20a Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Fri, 6 Jan 2017 18:58:14 +0530 Subject: [PATCH 7/9] dmaengine: pl330: Make sure microcode is privileged The PL330 is hard-wired such that instruction fetches on both the manager and channel threads go out onto the bus with the "privileged" bit set. This can become troublesome once there is an IOMMU or other form of memory protection downstream, since those will typically be programmed by the DMA mapping subsystem in the expectation of normal unprivileged transactions (such as the PL330 channel threads' own data accesses as currently configured by this driver). To avoid the case of, say, an IOMMU blocking an unexpected privileged transaction with a permission fault, use the newly-introduced DMA_ATTR_PRIVILEGED attribute for the mapping of our microcode buffer. That way the DMA layer can do whatever it needs to do to make things continue to work as expected on more complex systems. Cc: Dan Williams Cc: Vinod Koul Reviewed-by: Robin Murphy Tested-by: Robin Murphy Acked-by: Will Deacon Acked-by: Vinod Koul Signed-off-by: Mitchel Humpherys [rm: remove now-redundant local variable, clarify commit message] Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/dma/pl330.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 87fd01539fcb..5a90d0c2528c 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1864,9 +1864,10 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330) * Alloc MicroCode buffer for 'chans' Channel threads. * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN) */ - pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev, + pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev, chans * pl330->mcbufsz, - &pl330->mcode_bus, GFP_KERNEL); + &pl330->mcode_bus, GFP_KERNEL, + DMA_ATTR_PRIVILEGED); if (!pl330->mcode_cpu) { dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n", __func__, __LINE__); From e19898077cfb642fe151ba22981e795c74d9e114 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Fri, 6 Jan 2017 18:58:15 +0530 Subject: [PATCH 8/9] iommu/arm-smmu: Set privileged attribute to 'default' instead of 'unprivileged' Currently the driver sets all the device transactions privileges to UNPRIVILEGED, but there are cases where the iommu masters wants to isolate privileged supervisor and unprivileged user. So don't override the privileged setting to unprivileged, instead set it to default as incoming and let it be controlled by the pagetable settings. Acked-by: Will Deacon Signed-off-by: Sricharan R Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index a60cded8a6ed..73a0a250db25 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1214,7 +1214,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, continue; s2cr[idx].type = type; - s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV; + s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; s2cr[idx].cbndx = cbndx; arm_smmu_write_s2cr(smmu, idx); } From 14b4dbafa7e7e13323e402efd7723aafa391d69a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 6 Jan 2017 18:58:16 +0530 Subject: [PATCH 9/9] Revert "iommu/arm-smmu: Set PRIVCFG in stage 1 STEs" This reverts commit df5e1a0f2a2d779ad467a691203bcbc74d75690e. Now that proper privileged mappings can be requested via IOMMU_PRIV, unconditionally overriding the incoming PRIVCFG becomes the wrong thing to do, so stop it. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4d6ec444a9d6..7d45d8bb91a5 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -269,9 +269,6 @@ #define STRTAB_STE_1_SHCFG_INCOMING 1UL #define STRTAB_STE_1_SHCFG_SHIFT 44 -#define STRTAB_STE_1_PRIVCFG_UNPRIV 2UL -#define STRTAB_STE_1_PRIVCFG_SHIFT 48 - #define STRTAB_STE_2_S2VMID_SHIFT 0 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL #define STRTAB_STE_2_VTCR_SHIFT 32 @@ -1073,9 +1070,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, #ifdef CONFIG_PCI_ATS STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT | #endif - STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT | - STRTAB_STE_1_PRIVCFG_UNPRIV << - STRTAB_STE_1_PRIVCFG_SHIFT); + STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT); if (smmu->features & ARM_SMMU_FEAT_STALLS) dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);