2019-05-19 20:08:55 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Dynamic DMA mapping support.
|
|
|
|
*
|
2007-02-06 10:51:25 +08:00
|
|
|
* This implementation is a fallback for platforms that do not support
|
2005-04-17 06:20:36 +08:00
|
|
|
* I/O TLBs (aka DMA address translation hardware).
|
|
|
|
* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
|
|
|
|
* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
|
|
|
|
* Copyright (C) 2000, 2003 Hewlett-Packard Co
|
|
|
|
* David Mosberger-Tang <davidm@hpl.hp.com>
|
|
|
|
*
|
|
|
|
* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
|
|
|
|
* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
|
|
|
|
* unnecessary i-cache flushing.
|
2005-09-30 05:45:24 +08:00
|
|
|
* 04/07/.. ak Better overflow handling. Assorted fixes.
|
|
|
|
* 05/09/10 linville Add support for syncing ranges, support syncing for
|
|
|
|
* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
|
2008-12-23 02:26:09 +08:00
|
|
|
* 08/12/11 beckyb Add highmem support
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
2018-07-11 07:22:22 +08:00
|
|
|
#define pr_fmt(fmt) "software IO TLB: " fmt
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/cache.h>
|
2022-01-25 00:40:18 +08:00
|
|
|
#include <linux/cc_platform.h>
|
|
|
|
#include <linux/ctype.h>
|
|
|
|
#include <linux/debugfs.h>
|
2018-01-10 23:21:13 +08:00
|
|
|
#include <linux/dma-direct.h>
|
2020-09-22 21:36:11 +08:00
|
|
|
#include <linux/dma-map-ops.h>
|
2011-11-17 10:29:17 +08:00
|
|
|
#include <linux/export.h>
|
2022-01-25 00:40:18 +08:00
|
|
|
#include <linux/gfp.h>
|
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/io.h>
|
|
|
|
#include <linux/iommu-helper.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/memblock.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/pfn.h>
|
|
|
|
#include <linux/scatterlist.h>
|
|
|
|
#include <linux/set_memory.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/string.h>
|
2008-12-17 04:17:27 +08:00
|
|
|
#include <linux/swiotlb.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/types.h>
|
2021-06-19 11:40:41 +08:00
|
|
|
#ifdef CONFIG_DMA_RESTRICTED_POOL
|
|
|
|
#include <linux/of.h>
|
|
|
|
#include <linux/of_fdt.h>
|
|
|
|
#include <linux/of_reserved_mem.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-10-23 19:32:04 +08:00
|
|
|
#define CREATE_TRACE_POINTS
|
2013-09-05 04:11:05 +08:00
|
|
|
#include <trace/events/swiotlb.h>
|
|
|
|
|
2005-09-07 01:20:49 +08:00
|
|
|
#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Minimum IO TLB size to bother booting with. Systems with mainly
|
|
|
|
* 64bit capable cards will only lightly use the swiotlb. If we can't
|
|
|
|
* allocate a contiguous 1MB, we're probably in trouble anyway.
|
|
|
|
*/
|
|
|
|
#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
|
|
|
|
|
2021-03-19 00:14:22 +08:00
|
|
|
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-07-12 14:43:07 +08:00
|
|
|
struct io_tlb_slot {
|
|
|
|
phys_addr_t orig_addr;
|
|
|
|
size_t alloc_size;
|
|
|
|
unsigned int list;
|
|
|
|
};
|
|
|
|
|
2022-03-29 23:27:33 +08:00
|
|
|
static bool swiotlb_force_bounce;
|
|
|
|
static bool swiotlb_force_disable;
|
2019-01-18 15:10:27 +08:00
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
struct io_tlb_mem io_tlb_default_mem;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2021-03-19 00:14:23 +08:00
|
|
|
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
|
2022-07-09 00:15:44 +08:00
|
|
|
static unsigned long default_nareas;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct io_tlb_area - IO TLB memory area descriptor
|
|
|
|
*
|
|
|
|
* This is a single area with a single lock.
|
|
|
|
*
|
|
|
|
* @used: The number of used IO TLB block.
|
|
|
|
* @index: The slot index to start searching in this area for next round.
|
|
|
|
* @lock: The lock to protect the above data structures in the map and
|
|
|
|
* unmap calls.
|
|
|
|
*/
|
|
|
|
struct io_tlb_area {
|
|
|
|
unsigned long used;
|
|
|
|
unsigned int index;
|
|
|
|
spinlock_t lock;
|
|
|
|
};
|
|
|
|
|
2022-07-15 18:45:34 +08:00
|
|
|
/*
|
|
|
|
* Round up number of slabs to the next power of 2. The last area is going
|
|
|
|
* be smaller than the rest if default_nslabs is not power of two.
|
2022-07-15 18:45:35 +08:00
|
|
|
* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
|
|
|
|
* otherwise a segment may span two or more areas. It conflicts with free
|
|
|
|
* contiguous slots tracking: free slots are treated contiguous no matter
|
|
|
|
* whether they cross an area boundary.
|
2022-07-15 18:45:34 +08:00
|
|
|
*
|
|
|
|
* Return true if default_nslabs is rounded up.
|
|
|
|
*/
|
|
|
|
static bool round_up_default_nslabs(void)
|
|
|
|
{
|
2022-07-15 18:45:35 +08:00
|
|
|
if (!default_nareas)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
|
|
|
|
default_nslabs = IO_TLB_SEGSIZE * default_nareas;
|
|
|
|
else if (is_power_of_2(default_nslabs))
|
2022-07-15 18:45:34 +08:00
|
|
|
return false;
|
|
|
|
default_nslabs = roundup_pow_of_two(default_nslabs);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
static void swiotlb_adjust_nareas(unsigned int nareas)
|
|
|
|
{
|
2022-07-22 11:38:46 +08:00
|
|
|
/* use a single area when non is specified */
|
|
|
|
if (!nareas)
|
|
|
|
nareas = 1;
|
|
|
|
else if (!is_power_of_2(nareas))
|
2022-07-09 00:15:44 +08:00
|
|
|
nareas = roundup_pow_of_two(nareas);
|
|
|
|
|
|
|
|
default_nareas = nareas;
|
|
|
|
|
|
|
|
pr_info("area num %d.\n", nareas);
|
2022-07-15 18:45:34 +08:00
|
|
|
if (round_up_default_nslabs())
|
2022-07-09 00:15:44 +08:00
|
|
|
pr_info("SWIOTLB bounce buffer size roundup to %luMB",
|
|
|
|
(default_nslabs << IO_TLB_SHIFT) >> 20);
|
|
|
|
}
|
2021-03-19 00:14:23 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static int __init
|
|
|
|
setup_io_tlb_npages(char *str)
|
|
|
|
{
|
|
|
|
if (isdigit(*str)) {
|
|
|
|
/* avoid tail segment of size < IO_TLB_SEGSIZE */
|
2021-03-19 00:14:23 +08:00
|
|
|
default_nslabs =
|
|
|
|
ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2022-07-09 00:15:44 +08:00
|
|
|
if (*str == ',')
|
|
|
|
++str;
|
|
|
|
if (isdigit(*str))
|
|
|
|
swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
|
2005-04-17 06:20:36 +08:00
|
|
|
if (*str == ',')
|
|
|
|
++str;
|
2021-03-23 09:53:49 +08:00
|
|
|
if (!strcmp(str, "force"))
|
2022-03-29 23:27:33 +08:00
|
|
|
swiotlb_force_bounce = true;
|
2021-03-23 09:53:49 +08:00
|
|
|
else if (!strcmp(str, "noforce"))
|
2022-03-29 23:27:33 +08:00
|
|
|
swiotlb_force_disable = true;
|
2009-11-11 23:03:28 +08:00
|
|
|
|
2013-04-16 13:23:45 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2013-04-16 13:23:45 +08:00
|
|
|
early_param("swiotlb", setup_io_tlb_npages);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-04-16 13:23:45 +08:00
|
|
|
unsigned long swiotlb_size_or_default(void)
|
|
|
|
{
|
2021-03-19 00:14:23 +08:00
|
|
|
return default_nslabs << IO_TLB_SHIFT;
|
2013-04-16 13:23:45 +08:00
|
|
|
}
|
|
|
|
|
2021-03-19 00:14:23 +08:00
|
|
|
void __init swiotlb_adjust_size(unsigned long size)
|
2020-12-10 09:25:15 +08:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If swiotlb parameter has not been specified, give a chance to
|
|
|
|
* architectures such as those supporting memory encryption to
|
|
|
|
* adjust/expand SWIOTLB size for their use.
|
|
|
|
*/
|
2021-04-29 14:28:59 +08:00
|
|
|
if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
|
|
|
|
return;
|
2022-07-09 00:15:44 +08:00
|
|
|
|
2021-03-19 00:14:23 +08:00
|
|
|
size = ALIGN(size, IO_TLB_SIZE);
|
|
|
|
default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
|
2022-07-15 18:45:34 +08:00
|
|
|
if (round_up_default_nslabs())
|
2022-07-09 00:15:44 +08:00
|
|
|
size = default_nslabs << IO_TLB_SHIFT;
|
2021-03-19 00:14:23 +08:00
|
|
|
pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
|
2020-12-10 09:25:15 +08:00
|
|
|
}
|
|
|
|
|
2009-11-10 18:46:19 +08:00
|
|
|
void swiotlb_print_info(void)
|
2008-12-17 04:17:34 +08:00
|
|
|
{
|
2021-07-20 21:38:24 +08:00
|
|
|
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
2008-12-17 04:17:34 +08:00
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
if (!mem->nslabs) {
|
2018-07-11 07:22:22 +08:00
|
|
|
pr_warn("No low mem\n");
|
x86: Don't panic if can not alloc buffer for swiotlb
Normal boot path on system with iommu support:
swiotlb buffer will be allocated early at first and then try to initialize
iommu, if iommu for intel or AMD could setup properly, swiotlb buffer
will be freed.
The early allocating is with bootmem, and could panic when we try to use
kdump with buffer above 4G only, or with memmap to limit mem under 4G.
for example: memmap=4095M$1M to remove memory under 4G.
According to Eric, add _nopanic version and no_iotlb_memory to fail
map single later if swiotlb is still needed.
-v2: don't pass nopanic, and use -ENOMEM return value according to Eric.
panic early instead of using swiotlb_full to panic...according to Eric/Konrad.
-v3: make swiotlb_init to be notpanic, but will affect:
arm64, ia64, powerpc, tile, unicore32, x86.
-v4: cleanup swiotlb_init by removing swiotlb_init_with_default_size.
Suggested-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Link: http://lkml.kernel.org/r/1359058816-7615-36-git-send-email-yinghai@kernel.org
Reviewed-and-tested-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Kyungmin Park <kyungmin.park@samsung.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Andrzej Pietrasiewicz <andrzej.p@samsung.com>
Cc: linux-mips@linux-mips.org
Cc: xen-devel@lists.xensource.com
Cc: virtualization@lists.linux-foundation.org
Cc: Shuah Khan <shuahkhan@gmail.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2013-01-25 04:20:16 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2021-03-19 00:14:22 +08:00
|
|
|
pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
|
2021-03-19 00:14:23 +08:00
|
|
|
(mem->nslabs << IO_TLB_SHIFT) >> 20);
|
2008-12-17 04:17:34 +08:00
|
|
|
}
|
|
|
|
|
2021-02-04 17:11:20 +08:00
|
|
|
static inline unsigned long io_tlb_offset(unsigned long val)
|
|
|
|
{
|
|
|
|
return val & (IO_TLB_SEGSIZE - 1);
|
|
|
|
}
|
|
|
|
|
2021-02-05 18:19:34 +08:00
|
|
|
static inline unsigned long nr_slots(u64 val)
|
|
|
|
{
|
|
|
|
return DIV_ROUND_UP(val, IO_TLB_SIZE);
|
|
|
|
}
|
|
|
|
|
2017-07-18 05:10:21 +08:00
|
|
|
/*
|
|
|
|
* Early SWIOTLB allocation may be too early to allow an architecture to
|
|
|
|
* perform the desired operations. This function allows the architecture to
|
|
|
|
* call SWIOTLB when the operations are possible. It needs to be called
|
|
|
|
* before the SWIOTLB memory is used.
|
|
|
|
*/
|
|
|
|
void __init swiotlb_update_mem_attributes(void)
|
|
|
|
{
|
2021-07-20 21:38:24 +08:00
|
|
|
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
2017-07-18 05:10:21 +08:00
|
|
|
unsigned long bytes;
|
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
if (!mem->nslabs || mem->late_alloc)
|
2017-07-18 05:10:21 +08:00
|
|
|
return;
|
2021-03-19 00:14:22 +08:00
|
|
|
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
|
2023-03-26 21:52:02 +08:00
|
|
|
set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
|
2017-07-18 05:10:21 +08:00
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:32 +08:00
|
|
|
static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
|
2022-07-09 00:15:44 +08:00
|
|
|
unsigned long nslabs, unsigned int flags,
|
|
|
|
bool late_alloc, unsigned int nareas)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2021-06-19 11:40:32 +08:00
|
|
|
void *vaddr = phys_to_virt(start);
|
2021-03-19 00:14:23 +08:00
|
|
|
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
|
2021-06-19 11:40:32 +08:00
|
|
|
|
|
|
|
mem->nslabs = nslabs;
|
|
|
|
mem->start = start;
|
|
|
|
mem->end = mem->start + bytes;
|
|
|
|
mem->late_alloc = late_alloc;
|
2022-07-09 00:15:44 +08:00
|
|
|
mem->nareas = nareas;
|
|
|
|
mem->area_nslabs = nslabs / mem->nareas;
|
2021-06-24 23:55:20 +08:00
|
|
|
|
2022-06-02 02:49:39 +08:00
|
|
|
mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
|
2021-06-24 23:55:20 +08:00
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
for (i = 0; i < mem->nareas; i++) {
|
|
|
|
spin_lock_init(&mem->areas[i].lock);
|
|
|
|
mem->areas[i].index = 0;
|
2022-07-22 11:38:46 +08:00
|
|
|
mem->areas[i].used = 0;
|
2022-07-09 00:15:44 +08:00
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:32 +08:00
|
|
|
for (i = 0; i < mem->nslabs; i++) {
|
|
|
|
mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i);
|
|
|
|
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
|
|
|
|
mem->slots[i].alloc_size = 0;
|
|
|
|
}
|
2021-12-13 15:14:02 +08:00
|
|
|
|
2021-06-19 11:40:32 +08:00
|
|
|
memset(vaddr, 0, bytes);
|
2021-12-13 15:14:02 +08:00
|
|
|
mem->vaddr = vaddr;
|
|
|
|
return;
|
2021-06-19 11:40:32 +08:00
|
|
|
}
|
|
|
|
|
2023-02-22 15:04:11 +08:00
|
|
|
static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
|
|
|
|
unsigned int flags,
|
2022-10-31 16:13:27 +08:00
|
|
|
int (*remap)(void *tlb, unsigned long nslabs))
|
|
|
|
{
|
|
|
|
size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
|
|
|
|
void *tlb;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* By default allocate the bounce buffer memory from low memory, but
|
|
|
|
* allow to pick a location everywhere for hypervisors with guest
|
|
|
|
* memory encryption.
|
|
|
|
*/
|
|
|
|
if (flags & SWIOTLB_ANY)
|
|
|
|
tlb = memblock_alloc(bytes, PAGE_SIZE);
|
|
|
|
else
|
|
|
|
tlb = memblock_alloc_low(bytes, PAGE_SIZE);
|
|
|
|
|
|
|
|
if (!tlb) {
|
|
|
|
pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
|
|
|
|
__func__, bytes);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (remap && remap(tlb, nslabs) < 0) {
|
|
|
|
memblock_free(tlb, PAGE_ALIGN(bytes));
|
|
|
|
pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return tlb;
|
|
|
|
}
|
|
|
|
|
2010-05-11 03:15:12 +08:00
|
|
|
/*
|
|
|
|
* Statically reserve bounce buffer space and initialize bounce buffer data
|
|
|
|
* structures for the software IO TLB used to implement the DMA API.
|
|
|
|
*/
|
2022-03-14 15:02:57 +08:00
|
|
|
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
|
|
|
int (*remap)(void *tlb, unsigned long nslabs))
|
2010-05-11 03:15:12 +08:00
|
|
|
{
|
2022-03-15 14:41:04 +08:00
|
|
|
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
2022-07-09 00:15:44 +08:00
|
|
|
unsigned long nslabs;
|
2022-05-11 14:13:57 +08:00
|
|
|
size_t alloc_size;
|
2021-03-19 00:14:23 +08:00
|
|
|
void *tlb;
|
2010-05-11 03:15:12 +08:00
|
|
|
|
2022-03-29 23:27:33 +08:00
|
|
|
if (!addressing_limit && !swiotlb_force_bounce)
|
|
|
|
return;
|
|
|
|
if (swiotlb_force_disable)
|
2021-03-23 09:53:49 +08:00
|
|
|
return;
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
/*
|
|
|
|
* default_nslabs maybe changed when adjust area number.
|
|
|
|
* So allocate bounce buffer after adjusting area number.
|
|
|
|
*/
|
|
|
|
if (!default_nareas)
|
|
|
|
swiotlb_adjust_nareas(num_possible_cpus());
|
|
|
|
|
|
|
|
nslabs = default_nslabs;
|
2022-10-31 16:13:27 +08:00
|
|
|
while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
|
|
|
|
if (nslabs <= IO_TLB_MIN_SLABS)
|
|
|
|
return;
|
2022-03-14 15:02:57 +08:00
|
|
|
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
|
2022-10-31 16:13:27 +08:00
|
|
|
}
|
2022-09-07 21:38:33 +08:00
|
|
|
|
2022-10-31 16:13:27 +08:00
|
|
|
if (default_nslabs != nslabs) {
|
|
|
|
pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
|
|
|
|
default_nslabs, nslabs);
|
|
|
|
default_nslabs = nslabs;
|
2022-03-14 15:02:57 +08:00
|
|
|
}
|
2021-03-19 00:14:23 +08:00
|
|
|
|
2022-05-11 14:13:57 +08:00
|
|
|
alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
|
2022-03-15 14:41:04 +08:00
|
|
|
mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
|
2022-09-07 21:38:33 +08:00
|
|
|
if (!mem->slots) {
|
|
|
|
pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
|
|
|
|
__func__, alloc_size, PAGE_SIZE);
|
|
|
|
return;
|
|
|
|
}
|
2022-03-15 14:41:04 +08:00
|
|
|
|
2022-07-22 11:38:46 +08:00
|
|
|
mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
|
|
|
|
default_nareas), SMP_CACHE_BYTES);
|
2022-09-07 21:38:33 +08:00
|
|
|
if (!mem->areas) {
|
|
|
|
pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
|
|
|
|
return;
|
|
|
|
}
|
2022-07-09 00:15:44 +08:00
|
|
|
|
|
|
|
swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false,
|
|
|
|
default_nareas);
|
2022-03-15 14:41:04 +08:00
|
|
|
|
|
|
|
if (flags & SWIOTLB_VERBOSE)
|
|
|
|
swiotlb_print_info();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2022-03-14 15:02:57 +08:00
|
|
|
void __init swiotlb_init(bool addressing_limit, unsigned int flags)
|
|
|
|
{
|
2022-06-11 16:25:12 +08:00
|
|
|
swiotlb_init_remap(addressing_limit, flags, NULL);
|
2022-03-14 15:02:57 +08:00
|
|
|
}
|
|
|
|
|
2005-09-07 01:20:49 +08:00
|
|
|
/*
|
|
|
|
* Systems with larger DMA zones (those that don't support ISA) can
|
|
|
|
* initialize the swiotlb later using the slab allocator if needed.
|
|
|
|
* This should be just like above, but with some error catching.
|
|
|
|
*/
|
2022-03-14 15:02:57 +08:00
|
|
|
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
|
|
|
|
int (*remap)(void *tlb, unsigned long nslabs))
|
2005-09-07 01:20:49 +08:00
|
|
|
{
|
2022-03-15 14:41:04 +08:00
|
|
|
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
2022-02-14 18:07:28 +08:00
|
|
|
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
|
2012-10-16 01:19:28 +08:00
|
|
|
unsigned char *vstart = NULL;
|
2022-07-09 00:15:44 +08:00
|
|
|
unsigned int order, area_order;
|
2022-05-11 14:24:10 +08:00
|
|
|
bool retried = false;
|
2012-07-28 08:55:27 +08:00
|
|
|
int rc = 0;
|
2005-09-07 01:20:49 +08:00
|
|
|
|
2022-03-29 23:27:33 +08:00
|
|
|
if (swiotlb_force_disable)
|
2021-03-23 09:53:49 +08:00
|
|
|
return 0;
|
|
|
|
|
2022-03-14 15:02:57 +08:00
|
|
|
retry:
|
2021-03-19 00:14:23 +08:00
|
|
|
order = get_order(nslabs << IO_TLB_SHIFT);
|
|
|
|
nslabs = SLABS_PER_PAGE << order;
|
2005-09-07 01:20:49 +08:00
|
|
|
|
|
|
|
while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
|
2022-02-14 18:12:59 +08:00
|
|
|
vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
|
2012-10-16 01:19:28 +08:00
|
|
|
order);
|
|
|
|
if (vstart)
|
2005-09-07 01:20:49 +08:00
|
|
|
break;
|
|
|
|
order--;
|
2022-05-11 14:24:10 +08:00
|
|
|
nslabs = SLABS_PER_PAGE << order;
|
|
|
|
retried = true;
|
2005-09-07 01:20:49 +08:00
|
|
|
}
|
|
|
|
|
2021-03-19 00:14:23 +08:00
|
|
|
if (!vstart)
|
2012-07-28 08:55:27 +08:00
|
|
|
return -ENOMEM;
|
2021-03-19 00:14:23 +08:00
|
|
|
|
2022-03-14 15:02:57 +08:00
|
|
|
if (remap)
|
|
|
|
rc = remap(vstart, nslabs);
|
|
|
|
if (rc) {
|
|
|
|
free_pages((unsigned long)vstart, order);
|
|
|
|
|
|
|
|
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
|
|
|
|
if (nslabs < IO_TLB_MIN_SLABS)
|
|
|
|
return rc;
|
2022-05-11 14:24:10 +08:00
|
|
|
retried = true;
|
2022-03-14 15:02:57 +08:00
|
|
|
goto retry;
|
|
|
|
}
|
2021-03-01 15:44:31 +08:00
|
|
|
|
2022-05-11 14:24:10 +08:00
|
|
|
if (retried) {
|
|
|
|
pr_warn("only able to allocate %ld MB\n",
|
|
|
|
(PAGE_SIZE << order) >> 20);
|
|
|
|
}
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
if (!default_nareas)
|
|
|
|
swiotlb_adjust_nareas(num_possible_cpus());
|
|
|
|
|
|
|
|
area_order = get_order(array_size(sizeof(*mem->areas),
|
|
|
|
default_nareas));
|
|
|
|
mem->areas = (struct io_tlb_area *)
|
|
|
|
__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
|
|
|
|
if (!mem->areas)
|
|
|
|
goto error_area;
|
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
|
|
|
|
get_order(array_size(sizeof(*mem->slots), nslabs)));
|
2022-07-09 00:15:44 +08:00
|
|
|
if (!mem->slots)
|
|
|
|
goto error_slots;
|
2012-07-28 08:55:27 +08:00
|
|
|
|
2022-05-11 14:24:10 +08:00
|
|
|
set_memory_decrypted((unsigned long)vstart,
|
|
|
|
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
|
2022-07-09 00:15:44 +08:00
|
|
|
swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true,
|
|
|
|
default_nareas);
|
2005-09-07 01:20:49 +08:00
|
|
|
|
2009-11-10 18:46:19 +08:00
|
|
|
swiotlb_print_info();
|
2005-09-07 01:20:49 +08:00
|
|
|
return 0;
|
2022-07-09 00:15:44 +08:00
|
|
|
|
|
|
|
error_slots:
|
|
|
|
free_pages((unsigned long)mem->areas, area_order);
|
|
|
|
error_area:
|
|
|
|
free_pages((unsigned long)vstart, order);
|
|
|
|
return -ENOMEM;
|
2005-09-07 01:20:49 +08:00
|
|
|
}
|
|
|
|
|
2017-12-23 21:14:54 +08:00
|
|
|
void __init swiotlb_exit(void)
|
2009-11-10 18:46:18 +08:00
|
|
|
{
|
2021-07-20 21:38:24 +08:00
|
|
|
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
2021-07-20 21:38:26 +08:00
|
|
|
unsigned long tbl_vaddr;
|
|
|
|
size_t tbl_size, slots_size;
|
2022-07-09 00:15:44 +08:00
|
|
|
unsigned int area_order;
|
2021-03-19 00:14:22 +08:00
|
|
|
|
2022-03-29 23:27:33 +08:00
|
|
|
if (swiotlb_force_bounce)
|
2022-02-14 17:01:26 +08:00
|
|
|
return;
|
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
if (!mem->nslabs)
|
2009-11-10 18:46:18 +08:00
|
|
|
return;
|
|
|
|
|
2021-07-20 21:38:25 +08:00
|
|
|
pr_info("tearing down default memory pool\n");
|
2021-07-20 21:38:26 +08:00
|
|
|
tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
|
|
|
|
tbl_size = PAGE_ALIGN(mem->end - mem->start);
|
|
|
|
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
|
|
|
|
|
|
|
|
set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
|
|
|
|
if (mem->late_alloc) {
|
2022-07-09 00:15:44 +08:00
|
|
|
area_order = get_order(array_size(sizeof(*mem->areas),
|
|
|
|
mem->nareas));
|
|
|
|
free_pages((unsigned long)mem->areas, area_order);
|
2021-07-20 21:38:26 +08:00
|
|
|
free_pages(tbl_vaddr, get_order(tbl_size));
|
|
|
|
free_pages((unsigned long)mem->slots, get_order(slots_size));
|
|
|
|
} else {
|
2022-07-09 00:15:44 +08:00
|
|
|
memblock_free_late(__pa(mem->areas),
|
2022-07-22 11:38:46 +08:00
|
|
|
array_size(sizeof(*mem->areas), mem->nareas));
|
2021-07-20 21:38:26 +08:00
|
|
|
memblock_free_late(mem->start, tbl_size);
|
|
|
|
memblock_free_late(__pa(mem->slots), slots_size);
|
|
|
|
}
|
|
|
|
|
2021-07-20 21:38:24 +08:00
|
|
|
memset(mem, 0, sizeof(*mem));
|
2009-11-10 18:46:18 +08:00
|
|
|
}
|
|
|
|
|
swiotlb: manipulate orig_addr when tlb_addr has offset
in case of driver wants to sync part of ranges with offset,
swiotlb_tbl_sync_single() copies from orig_addr base to tlb_addr with
offset and ends up with data mismatch.
It was removed from
"swiotlb: don't modify orig_addr in swiotlb_tbl_sync_single",
but said logic has to be added back in.
From Linus's email:
"That commit which the removed the offset calculation entirely, because the old
(unsigned long)tlb_addr & (IO_TLB_SIZE - 1)
was wrong, but instead of removing it, I think it should have just
fixed it to be
(tlb_addr - mem->start) & (IO_TLB_SIZE - 1);
instead. That way the slot offset always matches the slot index calculation."
(Unfortunatly that broke NVMe).
The use-case that drivers are hitting is as follow:
1. Get dma_addr_t from dma_map_single()
dma_addr_t tlb_addr = dma_map_single(dev, vaddr, vsize, DMA_TO_DEVICE);
|<---------------vsize------------->|
+-----------------------------------+
| | original buffer
+-----------------------------------+
vaddr
swiotlb_align_offset
|<----->|<---------------vsize------------->|
+-------+-----------------------------------+
| | | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
2. Do something
3. Sync dma_addr_t through dma_sync_single_for_device(..)
dma_sync_single_for_device(dev, tlb_addr + offset, size, DMA_TO_DEVICE);
Error case.
Copy data to original buffer but it is from base addr (instead of
base addr + offset) in original buffer:
swiotlb_align_offset
|<----->|<- offset ->|<- size ->|
+-------+-----------------------------------+
| | |##########| | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
|<- size ->|
+-----------------------------------+
|##########| | original buffer
+-----------------------------------+
vaddr
The fix is to copy the data to the original buffer and take into
account the offset, like so:
swiotlb_align_offset
|<----->|<- offset ->|<- size ->|
+-------+-----------------------------------+
| | |##########| | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
|<- offset ->|<- size ->|
+-----------------------------------+
| |##########| | original buffer
+-----------------------------------+
vaddr
[One fix which was Linus's that made more sense to as it created a
symmetry would break NVMe. The reason for that is the:
unsigned int offset = (tlb_addr - mem->start) & (IO_TLB_SIZE - 1);
would come up with the proper offset, but it would lose the
alignment (which this patch contains).]
Fixes: 16fc3cef33a0 ("swiotlb: don't modify orig_addr in swiotlb_tbl_sync_single")
Signed-off-by: Bumyong Lee <bumyong.lee@samsung.com>
Signed-off-by: Chanho Park <chanho61.park@samsung.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reported-by: Dominique MARTINET <dominique.martinet@atmark-techno.com>
Reported-by: Horia Geantă <horia.geanta@nxp.com>
Tested-by: Horia Geantă <horia.geanta@nxp.com>
CC: stable@vger.kernel.org
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
2021-05-10 17:10:04 +08:00
|
|
|
/*
|
|
|
|
* Return the offset into a iotlb slot required to keep the device happy.
|
|
|
|
*/
|
|
|
|
static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
|
|
|
|
{
|
|
|
|
return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1);
|
|
|
|
}
|
|
|
|
|
2008-12-23 02:26:09 +08:00
|
|
|
/*
|
2019-01-18 15:10:26 +08:00
|
|
|
* Bounce: copy the swiotlb buffer from or back to the original dma location
|
2008-12-23 02:26:09 +08:00
|
|
|
*/
|
2021-03-01 15:44:25 +08:00
|
|
|
static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
|
|
|
|
enum dma_data_direction dir)
|
2008-12-23 02:26:09 +08:00
|
|
|
{
|
2021-06-19 11:40:34 +08:00
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
2021-03-19 00:14:22 +08:00
|
|
|
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
|
2021-03-19 00:14:23 +08:00
|
|
|
phys_addr_t orig_addr = mem->slots[index].orig_addr;
|
|
|
|
size_t alloc_size = mem->slots[index].alloc_size;
|
2012-10-16 01:19:55 +08:00
|
|
|
unsigned long pfn = PFN_DOWN(orig_addr);
|
2021-12-13 15:14:02 +08:00
|
|
|
unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
|
2021-07-07 13:12:54 +08:00
|
|
|
unsigned int tlb_offset, orig_addr_offset;
|
2008-12-23 02:26:09 +08:00
|
|
|
|
2021-03-01 15:44:25 +08:00
|
|
|
if (orig_addr == INVALID_PHYS_ADDR)
|
|
|
|
return;
|
|
|
|
|
2021-07-07 13:12:54 +08:00
|
|
|
tlb_offset = tlb_addr & (IO_TLB_SIZE - 1);
|
|
|
|
orig_addr_offset = swiotlb_align_offset(dev, orig_addr);
|
|
|
|
if (tlb_offset < orig_addr_offset) {
|
|
|
|
dev_WARN_ONCE(dev, 1,
|
|
|
|
"Access before mapping start detected. orig offset %u, requested offset %u.\n",
|
|
|
|
orig_addr_offset, tlb_offset);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
tlb_offset -= orig_addr_offset;
|
|
|
|
if (tlb_offset > alloc_size) {
|
|
|
|
dev_WARN_ONCE(dev, 1,
|
|
|
|
"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu+%u.\n",
|
|
|
|
alloc_size, size, tlb_offset);
|
|
|
|
return;
|
|
|
|
}
|
swiotlb: manipulate orig_addr when tlb_addr has offset
in case of driver wants to sync part of ranges with offset,
swiotlb_tbl_sync_single() copies from orig_addr base to tlb_addr with
offset and ends up with data mismatch.
It was removed from
"swiotlb: don't modify orig_addr in swiotlb_tbl_sync_single",
but said logic has to be added back in.
From Linus's email:
"That commit which the removed the offset calculation entirely, because the old
(unsigned long)tlb_addr & (IO_TLB_SIZE - 1)
was wrong, but instead of removing it, I think it should have just
fixed it to be
(tlb_addr - mem->start) & (IO_TLB_SIZE - 1);
instead. That way the slot offset always matches the slot index calculation."
(Unfortunatly that broke NVMe).
The use-case that drivers are hitting is as follow:
1. Get dma_addr_t from dma_map_single()
dma_addr_t tlb_addr = dma_map_single(dev, vaddr, vsize, DMA_TO_DEVICE);
|<---------------vsize------------->|
+-----------------------------------+
| | original buffer
+-----------------------------------+
vaddr
swiotlb_align_offset
|<----->|<---------------vsize------------->|
+-------+-----------------------------------+
| | | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
2. Do something
3. Sync dma_addr_t through dma_sync_single_for_device(..)
dma_sync_single_for_device(dev, tlb_addr + offset, size, DMA_TO_DEVICE);
Error case.
Copy data to original buffer but it is from base addr (instead of
base addr + offset) in original buffer:
swiotlb_align_offset
|<----->|<- offset ->|<- size ->|
+-------+-----------------------------------+
| | |##########| | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
|<- size ->|
+-----------------------------------+
|##########| | original buffer
+-----------------------------------+
vaddr
The fix is to copy the data to the original buffer and take into
account the offset, like so:
swiotlb_align_offset
|<----->|<- offset ->|<- size ->|
+-------+-----------------------------------+
| | |##########| | swiotlb buffer
+-------+-----------------------------------+
tlb_addr
|<- offset ->|<- size ->|
+-----------------------------------+
| |##########| | original buffer
+-----------------------------------+
vaddr
[One fix which was Linus's that made more sense to as it created a
symmetry would break NVMe. The reason for that is the:
unsigned int offset = (tlb_addr - mem->start) & (IO_TLB_SIZE - 1);
would come up with the proper offset, but it would lose the
alignment (which this patch contains).]
Fixes: 16fc3cef33a0 ("swiotlb: don't modify orig_addr in swiotlb_tbl_sync_single")
Signed-off-by: Bumyong Lee <bumyong.lee@samsung.com>
Signed-off-by: Chanho Park <chanho61.park@samsung.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reported-by: Dominique MARTINET <dominique.martinet@atmark-techno.com>
Reported-by: Horia Geantă <horia.geanta@nxp.com>
Tested-by: Horia Geantă <horia.geanta@nxp.com>
CC: stable@vger.kernel.org
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
2021-05-10 17:10:04 +08:00
|
|
|
|
|
|
|
orig_addr += tlb_offset;
|
|
|
|
alloc_size -= tlb_offset;
|
|
|
|
|
2021-03-01 15:44:25 +08:00
|
|
|
if (size > alloc_size) {
|
|
|
|
dev_WARN_ONCE(dev, 1,
|
|
|
|
"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
|
|
|
|
alloc_size, size);
|
|
|
|
size = alloc_size;
|
|
|
|
}
|
|
|
|
|
2008-12-23 02:26:09 +08:00
|
|
|
if (PageHighMem(pfn_to_page(pfn))) {
|
2012-10-16 01:19:55 +08:00
|
|
|
unsigned int offset = orig_addr & ~PAGE_MASK;
|
2022-09-01 21:29:06 +08:00
|
|
|
struct page *page;
|
2008-12-23 02:26:09 +08:00
|
|
|
unsigned int sz = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
while (size) {
|
2009-04-08 22:09:16 +08:00
|
|
|
sz = min_t(size_t, PAGE_SIZE - offset, size);
|
2008-12-23 02:26:09 +08:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
2022-09-01 21:29:06 +08:00
|
|
|
page = pfn_to_page(pfn);
|
2008-12-23 02:26:09 +08:00
|
|
|
if (dir == DMA_TO_DEVICE)
|
2022-09-01 21:29:06 +08:00
|
|
|
memcpy_from_page(vaddr, page, offset, sz);
|
2008-12-17 04:17:33 +08:00
|
|
|
else
|
2022-09-01 21:29:06 +08:00
|
|
|
memcpy_to_page(page, offset, vaddr, sz);
|
2008-12-17 04:17:33 +08:00
|
|
|
local_irq_restore(flags);
|
2008-12-23 02:26:09 +08:00
|
|
|
|
|
|
|
size -= sz;
|
|
|
|
pfn++;
|
2012-10-16 01:19:55 +08:00
|
|
|
vaddr += sz;
|
2008-12-23 02:26:09 +08:00
|
|
|
offset = 0;
|
2008-12-17 04:17:33 +08:00
|
|
|
}
|
2012-10-16 01:19:55 +08:00
|
|
|
} else if (dir == DMA_TO_DEVICE) {
|
|
|
|
memcpy(vaddr, phys_to_virt(orig_addr), size);
|
2008-12-17 04:17:33 +08:00
|
|
|
} else {
|
2012-10-16 01:19:55 +08:00
|
|
|
memcpy(phys_to_virt(orig_addr), vaddr, size);
|
2008-12-17 04:17:33 +08:00
|
|
|
}
|
2008-12-17 04:17:32 +08:00
|
|
|
}
|
|
|
|
|
2022-08-19 16:45:37 +08:00
|
|
|
static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
|
|
|
|
{
|
|
|
|
return start + (idx << IO_TLB_SHIFT);
|
|
|
|
}
|
2017-07-18 05:10:22 +08:00
|
|
|
|
2021-02-04 18:08:35 +08:00
|
|
|
/*
|
|
|
|
* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
|
|
|
|
*/
|
|
|
|
static inline unsigned long get_max_slots(unsigned long boundary_mask)
|
|
|
|
{
|
|
|
|
if (boundary_mask == ~0UL)
|
|
|
|
return 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
|
|
|
|
return nr_slots(boundary_mask + 1);
|
|
|
|
}
|
2008-02-05 14:28:16 +08:00
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index)
|
2021-02-04 18:08:35 +08:00
|
|
|
{
|
2022-07-09 00:15:44 +08:00
|
|
|
if (index >= mem->area_nslabs)
|
2021-02-04 18:08:35 +08:00
|
|
|
return 0;
|
|
|
|
return index;
|
|
|
|
}
|
swiotlb: add swiotlb_tbl_map_single library function
swiotlb_tbl_map_single() takes the dma address of iotlb instead of
using swiotlb_virt_to_bus().
[v2: changed swiotlb_tlb to swiotlb_tbl]
[v3: changed u64 to dma_addr_t]
This patch:
This is a set of patches that separate the address translation
(virt_to_phys, virt_to_bus, etc) and allocation of the SWIOTLB buffer
from the SWIOTLB library.
The idea behind this set of patches is to make it possible to have separate
mechanisms for translating virtual to physical or virtual to DMA addresses
on platforms which need an SWIOTLB, and where physical != PCI bus address
and also to allocate the core IOTLB memory outside SWIOTLB.
One customers of this is the pv-ops project, which can switch between
different modes of operation depending on the environment it is running in:
bare-metal or virtualized (Xen for now). Another is the Wii DMA - used to
implement the MEM2 DMA facility needed by its EHCI controller (for details:
http://lkml.org/lkml/2010/5/18/303)
On bare-metal SWIOTLB is used when there are no hardware IOMMU. In virtualized
environment it used when PCI pass-through is enabled for the guest. The problems
with PCI pass-through is that the guest's idea of PFN's is not the real thing.
To fix that, there is translation layer for PFN->machine frame number and vice-versa.
To bubble that up to the SWIOTLB layer there are two possible solutions.
One solution has been to wholesale copy the SWIOTLB, stick it in
arch/x86/xen/swiotlb.c and modify the virt_to_phys, phys_to_virt and others
to use the Xen address translation functions. Unfortunately, since the kernel can
run on bare-metal, there would be big code overlap with the real SWIOTLB.
(git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen.git xen/dom0/swiotlb-new)
Another approach, which this set of patches explores, is to abstract the
address translation and address determination functions away from the
SWIOTLB book-keeping functions. This way the core SWIOTLB library functions
are present in one place, while the address related functions are in
a separate library that can be loaded when running under non-bare-metal platform.
Changelog:
Since the last posting [v8.2] Konrad has done:
- Added this changelog in the patch and referenced in the other patches
this description.
- 'enum dma_data_direction direction' to 'enum dma.. dir' so to be
unified.
[v8-v8.2 changes:]
- Rolled-up the last two patches in one.
- Rebased against linus latest. That meant dealing with swiotlb_sync_single_range_* changes.
- added Acked-by: Fujita Tomonori and Tested-by: Albert Herranz
[v7-v8 changes:]
- Minimized the list of exported functions.
- Integrated Fujita's patches and changed "swiotlb_tlb" to "swiotlb_tbl" in them.
[v6-v7 changes:]
- Minimized the amount of exported functions/variable with a prefix of: "swiotbl_tbl".
- Made the usage of 'int dir' to be 'enum dma_data_direction'.
[v5-v6 changes:]
- Made the exported functions/variables have the 'swiotlb_bk' prefix.
- dropped the checkpatches/other reworks
Signed-off-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp>
Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Tested-by: Albert Herranz <albert_herranz@yahoo.es>
2010-05-11 03:14:54 +08:00
|
|
|
|
2023-04-14 01:57:37 +08:00
|
|
|
/*
|
|
|
|
* Track the total used slots with a global atomic value in order to have
|
|
|
|
* correct information to determine the high water mark. The mem_used()
|
|
|
|
* function gives imprecise results because there's no locking across
|
|
|
|
* multiple areas.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
|
|
static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
|
|
|
|
{
|
|
|
|
unsigned long old_hiwater, new_used;
|
|
|
|
|
|
|
|
new_used = atomic_long_add_return(nslots, &mem->total_used);
|
|
|
|
old_hiwater = atomic_long_read(&mem->used_hiwater);
|
|
|
|
do {
|
|
|
|
if (new_used <= old_hiwater)
|
|
|
|
break;
|
|
|
|
} while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
|
|
|
|
&old_hiwater, new_used));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
|
|
|
|
{
|
|
|
|
atomic_long_sub(nslots, &mem->total_used);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* !CONFIG_DEBUG_FS */
|
|
|
|
static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_DEBUG_FS */
|
|
|
|
|
2021-02-04 18:08:35 +08:00
|
|
|
/*
|
|
|
|
* Find a suitable number of IO TLB entries size that will fit this request and
|
|
|
|
* allocate a buffer from that IO TLB pool.
|
|
|
|
*/
|
2022-07-22 11:38:46 +08:00
|
|
|
static int swiotlb_do_find_slots(struct device *dev, int area_index,
|
|
|
|
phys_addr_t orig_addr, size_t alloc_size,
|
|
|
|
unsigned int alloc_align_mask)
|
2021-02-04 18:08:35 +08:00
|
|
|
{
|
2022-07-22 11:38:46 +08:00
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
|
|
|
struct io_tlb_area *area = mem->areas + area_index;
|
2021-02-04 18:08:35 +08:00
|
|
|
unsigned long boundary_mask = dma_get_seg_boundary(dev);
|
|
|
|
dma_addr_t tbl_dma_addr =
|
2021-03-19 00:14:22 +08:00
|
|
|
phys_to_dma_unencrypted(dev, mem->start) & boundary_mask;
|
2021-02-04 18:08:35 +08:00
|
|
|
unsigned long max_slots = get_max_slots(boundary_mask);
|
2021-02-23 03:39:44 +08:00
|
|
|
unsigned int iotlb_align_mask =
|
2023-04-06 22:35:39 +08:00
|
|
|
dma_get_min_align_mask(dev) | alloc_align_mask;
|
2021-02-23 03:39:44 +08:00
|
|
|
unsigned int nslots = nr_slots(alloc_size), stride;
|
2021-06-24 23:55:21 +08:00
|
|
|
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
|
2023-02-23 00:53:15 +08:00
|
|
|
unsigned int index, slots_checked, count = 0, i;
|
2021-02-04 18:08:35 +08:00
|
|
|
unsigned long flags;
|
2022-07-09 00:15:44 +08:00
|
|
|
unsigned int slot_base;
|
|
|
|
unsigned int slot_index;
|
2008-12-17 04:17:29 +08:00
|
|
|
|
2021-02-04 18:08:35 +08:00
|
|
|
BUG_ON(!nslots);
|
2022-07-09 00:15:44 +08:00
|
|
|
BUG_ON(area_index >= mem->nareas);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
swiotlb: fix slot alignment checks
Explicit alignment and page alignment are used only to calculate
the stride, not when checking actual slot physical address.
Originally, only page alignment was implemented, and that worked,
because the whole SWIOTLB is allocated on a page boundary, so
aligning the start index was sufficient to ensure a page-aligned
slot.
When commit 1f221a0d0dbf ("swiotlb: respect min_align_mask") added
support for min_align_mask, the index could be incremented in the
search loop, potentially finding an unaligned slot if minimum device
alignment is between IO_TLB_SIZE and PAGE_SIZE. The bug could go
unnoticed, because the slot size is 2 KiB, and the most common page
size is 4 KiB, so there is no alignment value in between.
IIUC the intention has been to find a slot that conforms to all
alignment constraints: device minimum alignment, an explicit
alignment (given as function parameter) and optionally page
alignment (if allocation size is >= PAGE_SIZE). The most
restrictive mask can be trivially computed with logical AND. The
rest can stay.
Fixes: 1f221a0d0dbf ("swiotlb: respect min_align_mask")
Fixes: e81e99bacc9f ("swiotlb: Support aligned swiotlb buffers")
Signed-off-by: Petr Tesarik <petr.tesarik.ext@huawei.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
2023-03-21 16:31:27 +08:00
|
|
|
/*
|
|
|
|
* For allocations of PAGE_SIZE or larger only look for page aligned
|
|
|
|
* allocations.
|
|
|
|
*/
|
|
|
|
if (alloc_size >= PAGE_SIZE)
|
2023-04-06 22:35:39 +08:00
|
|
|
iotlb_align_mask |= ~PAGE_MASK;
|
|
|
|
iotlb_align_mask &= ~(IO_TLB_SIZE - 1);
|
swiotlb: fix slot alignment checks
Explicit alignment and page alignment are used only to calculate
the stride, not when checking actual slot physical address.
Originally, only page alignment was implemented, and that worked,
because the whole SWIOTLB is allocated on a page boundary, so
aligning the start index was sufficient to ensure a page-aligned
slot.
When commit 1f221a0d0dbf ("swiotlb: respect min_align_mask") added
support for min_align_mask, the index could be incremented in the
search loop, potentially finding an unaligned slot if minimum device
alignment is between IO_TLB_SIZE and PAGE_SIZE. The bug could go
unnoticed, because the slot size is 2 KiB, and the most common page
size is 4 KiB, so there is no alignment value in between.
IIUC the intention has been to find a slot that conforms to all
alignment constraints: device minimum alignment, an explicit
alignment (given as function parameter) and optionally page
alignment (if allocation size is >= PAGE_SIZE). The most
restrictive mask can be trivially computed with logical AND. The
rest can stay.
Fixes: 1f221a0d0dbf ("swiotlb: respect min_align_mask")
Fixes: e81e99bacc9f ("swiotlb: Support aligned swiotlb buffers")
Signed-off-by: Petr Tesarik <petr.tesarik.ext@huawei.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
2023-03-21 16:31:27 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2021-02-23 03:39:44 +08:00
|
|
|
* For mappings with an alignment requirement don't bother looping to
|
swiotlb: fix slot alignment checks
Explicit alignment and page alignment are used only to calculate
the stride, not when checking actual slot physical address.
Originally, only page alignment was implemented, and that worked,
because the whole SWIOTLB is allocated on a page boundary, so
aligning the start index was sufficient to ensure a page-aligned
slot.
When commit 1f221a0d0dbf ("swiotlb: respect min_align_mask") added
support for min_align_mask, the index could be incremented in the
search loop, potentially finding an unaligned slot if minimum device
alignment is between IO_TLB_SIZE and PAGE_SIZE. The bug could go
unnoticed, because the slot size is 2 KiB, and the most common page
size is 4 KiB, so there is no alignment value in between.
IIUC the intention has been to find a slot that conforms to all
alignment constraints: device minimum alignment, an explicit
alignment (given as function parameter) and optionally page
alignment (if allocation size is >= PAGE_SIZE). The most
restrictive mask can be trivially computed with logical AND. The
rest can stay.
Fixes: 1f221a0d0dbf ("swiotlb: respect min_align_mask")
Fixes: e81e99bacc9f ("swiotlb: Support aligned swiotlb buffers")
Signed-off-by: Petr Tesarik <petr.tesarik.ext@huawei.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
2023-03-21 16:31:27 +08:00
|
|
|
* unaligned slots once we found an aligned one.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2021-02-23 03:39:44 +08:00
|
|
|
stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
spin_lock_irqsave(&area->lock, flags);
|
|
|
|
if (unlikely(nslots > mem->area_nslabs - area->used))
|
2019-01-18 15:10:28 +08:00
|
|
|
goto not_found;
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
slot_base = area_index * mem->area_nslabs;
|
swiotlb: fix slot alignment checks
Explicit alignment and page alignment are used only to calculate
the stride, not when checking actual slot physical address.
Originally, only page alignment was implemented, and that worked,
because the whole SWIOTLB is allocated on a page boundary, so
aligning the start index was sufficient to ensure a page-aligned
slot.
When commit 1f221a0d0dbf ("swiotlb: respect min_align_mask") added
support for min_align_mask, the index could be incremented in the
search loop, potentially finding an unaligned slot if minimum device
alignment is between IO_TLB_SIZE and PAGE_SIZE. The bug could go
unnoticed, because the slot size is 2 KiB, and the most common page
size is 4 KiB, so there is no alignment value in between.
IIUC the intention has been to find a slot that conforms to all
alignment constraints: device minimum alignment, an explicit
alignment (given as function parameter) and optionally page
alignment (if allocation size is >= PAGE_SIZE). The most
restrictive mask can be trivially computed with logical AND. The
rest can stay.
Fixes: 1f221a0d0dbf ("swiotlb: respect min_align_mask")
Fixes: e81e99bacc9f ("swiotlb: Support aligned swiotlb buffers")
Signed-off-by: Petr Tesarik <petr.tesarik.ext@huawei.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
2023-03-21 16:31:27 +08:00
|
|
|
index = area->index;
|
2022-07-09 00:15:44 +08:00
|
|
|
|
2023-02-23 00:53:15 +08:00
|
|
|
for (slots_checked = 0; slots_checked < mem->area_nslabs; ) {
|
2022-07-09 00:15:44 +08:00
|
|
|
slot_index = slot_base + index;
|
|
|
|
|
2021-06-19 11:40:40 +08:00
|
|
|
if (orig_addr &&
|
2022-07-09 00:15:44 +08:00
|
|
|
(slot_addr(tbl_dma_addr, slot_index) &
|
|
|
|
iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
|
|
|
|
index = wrap_area_index(mem, index + 1);
|
2023-02-23 00:53:15 +08:00
|
|
|
slots_checked++;
|
2021-02-23 03:39:44 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2008-04-29 15:59:36 +08:00
|
|
|
/*
|
|
|
|
* If we find a slot that indicates we have 'nslots' number of
|
|
|
|
* contiguous buffers, we allocate the buffers from that slot
|
|
|
|
* and mark the entries as '0' indicating unavailable.
|
|
|
|
*/
|
2022-07-09 00:15:44 +08:00
|
|
|
if (!iommu_is_span_boundary(slot_index, nslots,
|
2021-02-04 18:08:35 +08:00
|
|
|
nr_slots(tbl_dma_addr),
|
|
|
|
max_slots)) {
|
2022-07-09 00:15:44 +08:00
|
|
|
if (mem->slots[slot_index].list >= nslots)
|
2021-02-04 18:08:35 +08:00
|
|
|
goto found;
|
2008-04-29 15:59:36 +08:00
|
|
|
}
|
2022-07-09 00:15:44 +08:00
|
|
|
index = wrap_area_index(mem, index + stride);
|
2023-02-23 00:53:15 +08:00
|
|
|
slots_checked += stride;
|
|
|
|
}
|
2008-04-29 15:59:36 +08:00
|
|
|
|
|
|
|
not_found:
|
2022-07-09 00:15:44 +08:00
|
|
|
spin_unlock_irqrestore(&area->lock, flags);
|
2021-02-04 18:08:35 +08:00
|
|
|
return -1;
|
|
|
|
|
2008-04-29 15:59:36 +08:00
|
|
|
found:
|
2022-07-09 00:15:44 +08:00
|
|
|
for (i = slot_index; i < slot_index + nslots; i++) {
|
2021-03-19 00:14:23 +08:00
|
|
|
mem->slots[i].list = 0;
|
2022-07-09 00:15:44 +08:00
|
|
|
mem->slots[i].alloc_size = alloc_size - (offset +
|
|
|
|
((i - slot_index) << IO_TLB_SHIFT));
|
2021-06-24 23:55:21 +08:00
|
|
|
}
|
2022-07-09 00:15:44 +08:00
|
|
|
for (i = slot_index - 1;
|
2021-02-04 18:08:35 +08:00
|
|
|
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
|
2021-03-19 00:14:23 +08:00
|
|
|
mem->slots[i].list; i--)
|
|
|
|
mem->slots[i].list = ++count;
|
2021-02-04 18:08:35 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the indices to avoid searching in the next round.
|
|
|
|
*/
|
2023-03-21 16:31:26 +08:00
|
|
|
area->index = wrap_area_index(mem, index + nslots);
|
2022-07-09 00:15:44 +08:00
|
|
|
area->used += nslots;
|
|
|
|
spin_unlock_irqrestore(&area->lock, flags);
|
2023-04-14 01:57:37 +08:00
|
|
|
|
|
|
|
inc_used_and_hiwater(mem, nslots);
|
2022-07-09 00:15:44 +08:00
|
|
|
return slot_index;
|
|
|
|
}
|
2021-02-04 18:08:35 +08:00
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
|
|
|
|
size_t alloc_size, unsigned int alloc_align_mask)
|
|
|
|
{
|
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
2022-07-22 11:38:46 +08:00
|
|
|
int start = raw_smp_processor_id() & (mem->nareas - 1);
|
2022-07-09 00:15:44 +08:00
|
|
|
int i = start, index;
|
|
|
|
|
|
|
|
do {
|
2022-07-22 11:38:46 +08:00
|
|
|
index = swiotlb_do_find_slots(dev, i, orig_addr, alloc_size,
|
2022-07-09 00:15:44 +08:00
|
|
|
alloc_align_mask);
|
|
|
|
if (index >= 0)
|
|
|
|
return index;
|
|
|
|
if (++i >= mem->nareas)
|
|
|
|
i = 0;
|
|
|
|
} while (i != start);
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long mem_used(struct io_tlb_mem *mem)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned long used = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < mem->nareas; i++)
|
|
|
|
used += mem->areas[i].used;
|
|
|
|
return used;
|
2021-02-04 18:08:35 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
|
|
|
size_t mapping_size, size_t alloc_size,
|
2021-09-29 10:32:59 +08:00
|
|
|
unsigned int alloc_align_mask, enum dma_data_direction dir,
|
|
|
|
unsigned long attrs)
|
2021-02-04 18:08:35 +08:00
|
|
|
{
|
2021-06-19 11:40:34 +08:00
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
2021-02-23 03:39:44 +08:00
|
|
|
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
|
2021-04-22 16:14:53 +08:00
|
|
|
unsigned int i;
|
|
|
|
int index;
|
2021-02-04 18:08:35 +08:00
|
|
|
phys_addr_t tlb_addr;
|
|
|
|
|
2022-09-07 21:38:33 +08:00
|
|
|
if (!mem || !mem->nslabs) {
|
|
|
|
dev_warn_ratelimited(dev,
|
|
|
|
"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
|
|
|
|
return (phys_addr_t)DMA_MAPPING_ERROR;
|
|
|
|
}
|
2021-02-04 18:08:35 +08:00
|
|
|
|
2021-09-09 06:58:39 +08:00
|
|
|
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
|
2021-02-04 18:08:35 +08:00
|
|
|
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
|
|
|
|
|
|
|
|
if (mapping_size > alloc_size) {
|
|
|
|
dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
|
|
|
|
mapping_size, alloc_size);
|
|
|
|
return (phys_addr_t)DMA_MAPPING_ERROR;
|
|
|
|
}
|
|
|
|
|
2021-09-29 10:32:59 +08:00
|
|
|
index = swiotlb_find_slots(dev, orig_addr,
|
|
|
|
alloc_size + offset, alloc_align_mask);
|
2021-02-04 18:08:35 +08:00
|
|
|
if (index == -1) {
|
|
|
|
if (!(attrs & DMA_ATTR_NO_WARN))
|
|
|
|
dev_warn_ratelimited(dev,
|
|
|
|
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
|
2022-07-09 00:15:44 +08:00
|
|
|
alloc_size, mem->nslabs, mem_used(mem));
|
2021-02-04 18:08:35 +08:00
|
|
|
return (phys_addr_t)DMA_MAPPING_ERROR;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Save away the mapping from the original address to the DMA address.
|
|
|
|
* This is needed when we sync the memory. Then we sync the buffer if
|
|
|
|
* needed.
|
|
|
|
*/
|
2021-06-24 23:55:21 +08:00
|
|
|
for (i = 0; i < nr_slots(alloc_size + offset); i++)
|
2021-03-19 00:14:23 +08:00
|
|
|
mem->slots[index + i].orig_addr = slot_addr(orig_addr, i);
|
2021-03-19 00:14:22 +08:00
|
|
|
tlb_addr = slot_addr(mem->start, index) + offset;
|
2022-03-29 02:37:05 +08:00
|
|
|
/*
|
|
|
|
* When dir == DMA_FROM_DEVICE we could omit the copy from the orig
|
|
|
|
* to the tlb buffer, if we knew for sure the device will
|
2022-08-26 17:50:46 +08:00
|
|
|
* overwrite the entire current content. But we don't. Thus
|
2022-03-29 02:37:05 +08:00
|
|
|
* unconditional bounce may prevent leaking swiotlb content (i.e.
|
|
|
|
* kernel memory) to user-space.
|
|
|
|
*/
|
|
|
|
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
|
2012-10-16 01:19:39 +08:00
|
|
|
return tlb_addr;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:39 +08:00
|
|
|
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2021-06-19 11:40:39 +08:00
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long flags;
|
2021-06-19 11:40:39 +08:00
|
|
|
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
|
2021-03-19 00:14:22 +08:00
|
|
|
int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
|
2021-03-19 00:14:23 +08:00
|
|
|
int nslots = nr_slots(mem->slots[index].alloc_size + offset);
|
2022-07-09 00:15:44 +08:00
|
|
|
int aindex = index / mem->area_nslabs;
|
|
|
|
struct io_tlb_area *area = &mem->areas[aindex];
|
2021-03-01 15:44:25 +08:00
|
|
|
int count, i;
|
2021-01-12 23:07:29 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Return the buffer to the free list by setting the corresponding
|
tree-wide: fix assorted typos all over the place
That is "success", "unknown", "through", "performance", "[re|un]mapping"
, "access", "default", "reasonable", "[con]currently", "temperature"
, "channel", "[un]used", "application", "example","hierarchy", "therefore"
, "[over|under]flow", "contiguous", "threshold", "enough" and others.
Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2009-11-14 23:09:05 +08:00
|
|
|
* entries to indicate the number of contiguous entries available.
|
2005-04-17 06:20:36 +08:00
|
|
|
* While returning the entries to the free list, we merge the entries
|
|
|
|
* with slots below and above the pool being returned.
|
|
|
|
*/
|
2022-07-09 00:15:44 +08:00
|
|
|
BUG_ON(aindex >= mem->nareas);
|
|
|
|
|
|
|
|
spin_lock_irqsave(&area->lock, flags);
|
2021-02-04 17:13:40 +08:00
|
|
|
if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
|
2021-03-19 00:14:23 +08:00
|
|
|
count = mem->slots[index + nslots].list;
|
2021-02-04 17:13:40 +08:00
|
|
|
else
|
|
|
|
count = 0;
|
2019-01-18 15:10:27 +08:00
|
|
|
|
2021-02-04 17:13:40 +08:00
|
|
|
/*
|
|
|
|
* Step 1: return the slots to the free list, merging the slots with
|
|
|
|
* superceeding slots
|
|
|
|
*/
|
|
|
|
for (i = index + nslots - 1; i >= index; i--) {
|
2021-03-19 00:14:23 +08:00
|
|
|
mem->slots[i].list = ++count;
|
|
|
|
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
|
|
|
|
mem->slots[i].alloc_size = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2021-02-04 17:13:40 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Step 2: merge the returned slots with the preceding slots, if
|
|
|
|
* available (non zero)
|
|
|
|
*/
|
|
|
|
for (i = index - 1;
|
2021-03-19 00:14:23 +08:00
|
|
|
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
|
2021-02-04 17:13:40 +08:00
|
|
|
i--)
|
2021-03-19 00:14:23 +08:00
|
|
|
mem->slots[i].list = ++count;
|
2022-07-09 00:15:44 +08:00
|
|
|
area->used -= nslots;
|
|
|
|
spin_unlock_irqrestore(&area->lock, flags);
|
2023-04-14 01:57:37 +08:00
|
|
|
|
|
|
|
dec_used(mem, nslots);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:39 +08:00
|
|
|
/*
|
|
|
|
* tlb_addr is the physical address of the bounce buffer to unmap.
|
|
|
|
*/
|
|
|
|
void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
|
|
|
|
size_t mapping_size, enum dma_data_direction dir,
|
|
|
|
unsigned long attrs)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* First, sync the memory before unmapping the entry
|
|
|
|
*/
|
|
|
|
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
|
|
|
|
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
|
|
|
|
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
|
|
|
|
|
|
|
|
swiotlb_release_slots(dev, tlb_addr);
|
|
|
|
}
|
|
|
|
|
2021-03-01 15:44:26 +08:00
|
|
|
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
|
|
|
|
size_t size, enum dma_data_direction dir)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
Revert "swiotlb: rework "fix info leak with DMA_FROM_DEVICE""
This reverts commit aa6f8dcbab473f3a3c7454b74caa46d36cdc5d13.
It turns out this breaks at least the ath9k wireless driver, and
possibly others.
What the ath9k driver does on packet receive is to set up the DMA
transfer with:
int ath_rx_init(..)
..
bf->bf_buf_addr = dma_map_single(sc->dev, skb->data,
common->rx_bufsize,
DMA_FROM_DEVICE);
and then the receive logic (through ath_rx_tasklet()) will fetch
incoming packets
static bool ath_edma_get_buffers(..)
..
dma_sync_single_for_cpu(sc->dev, bf->bf_buf_addr,
common->rx_bufsize, DMA_FROM_DEVICE);
ret = ath9k_hw_process_rxdesc_edma(ah, rs, skb->data);
if (ret == -EINPROGRESS) {
/*let device gain the buffer again*/
dma_sync_single_for_device(sc->dev, bf->bf_buf_addr,
common->rx_bufsize, DMA_FROM_DEVICE);
return false;
}
and it's worth noting how that first DMA sync:
dma_sync_single_for_cpu(..DMA_FROM_DEVICE);
is there to make sure the CPU can read the DMA buffer (possibly by
copying it from the bounce buffer area, or by doing some cache flush).
The iommu correctly turns that into a "copy from bounce bufer" so that
the driver can look at the state of the packets.
In the meantime, the device may continue to write to the DMA buffer, but
we at least have a snapshot of the state due to that first DMA sync.
But that _second_ DMA sync:
dma_sync_single_for_device(..DMA_FROM_DEVICE);
is telling the DMA mapping that the CPU wasn't interested in the area
because the packet wasn't there. In the case of a DMA bounce buffer,
that is a no-op.
Note how it's not a sync for the CPU (the "for_device()" part), and it's
not a sync for data written by the CPU (the "DMA_FROM_DEVICE" part).
Or rather, it _should_ be a no-op. That's what commit aa6f8dcbab47
broke: it made the code bounce the buffer unconditionally, and changed
the DMA_FROM_DEVICE to just unconditionally and illogically be
DMA_TO_DEVICE.
[ Side note: purely within the confines of the swiotlb driver it wasn't
entirely illogical: The reason it did that odd DMA_FROM_DEVICE ->
DMA_TO_DEVICE conversion thing is because inside the swiotlb driver,
it uses just a swiotlb_bounce() helper that doesn't care about the
whole distinction of who the sync is for - only which direction to
bounce.
So it took the "sync for device" to mean that the CPU must have been
the one writing, and thought it meant DMA_TO_DEVICE. ]
Also note how the commentary in that commit was wrong, probably due to
that whole confusion, claiming that the commit makes the swiotlb code
"bounce unconditionally (that is, also
when dir == DMA_TO_DEVICE) in order do avoid synchronising back stale
data from the swiotlb buffer"
which is nonsensical for two reasons:
- that "also when dir == DMA_TO_DEVICE" is nonsensical, as that was
exactly when it always did - and should do - the bounce.
- since this is a sync for the device (not for the CPU), we're clearly
fundamentally not coping back stale data from the bounce buffers at
all, because we'd be copying *to* the bounce buffers.
So that commit was just very confused. It confused the direction of the
synchronization (to the device, not the cpu) with the direction of the
DMA (from the device).
Reported-and-bisected-by: Oleksandr Natalenko <oleksandr@natalenko.name>
Reported-by: Olha Cherevyk <olha.cherevyk@gmail.com>
Cc: Halil Pasic <pasic@linux.ibm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Kalle Valo <kvalo@kernel.org>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Toke Høiland-Jørgensen <toke@toke.dk>
Cc: Maxime Bizon <mbizon@freebox.fr>
Cc: Johannes Berg <johannes@sipsolutions.net>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2022-03-27 01:42:04 +08:00
|
|
|
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
|
|
|
|
swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE);
|
|
|
|
else
|
|
|
|
BUG_ON(dir != DMA_FROM_DEVICE);
|
2021-03-01 15:44:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
|
|
|
|
size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
|
|
|
|
swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE);
|
|
|
|
else
|
|
|
|
BUG_ON(dir != DMA_TO_DEVICE);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2018-12-03 18:43:54 +08:00
|
|
|
/*
|
2020-02-03 21:44:38 +08:00
|
|
|
* Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
|
2018-12-03 18:43:54 +08:00
|
|
|
* to the device copy the data into it as well.
|
|
|
|
*/
|
2020-02-03 21:44:38 +08:00
|
|
|
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
|
|
|
|
enum dma_data_direction dir, unsigned long attrs)
|
2018-08-20 22:21:10 +08:00
|
|
|
{
|
2020-02-03 21:44:38 +08:00
|
|
|
phys_addr_t swiotlb_addr;
|
|
|
|
dma_addr_t dma_addr;
|
2018-08-20 22:21:10 +08:00
|
|
|
|
2022-03-29 23:27:33 +08:00
|
|
|
trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
|
2018-08-20 22:21:10 +08:00
|
|
|
|
2021-09-29 10:32:59 +08:00
|
|
|
swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir,
|
2020-10-23 14:33:09 +08:00
|
|
|
attrs);
|
2020-02-03 21:44:38 +08:00
|
|
|
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
|
|
|
|
return DMA_MAPPING_ERROR;
|
2018-08-20 22:21:10 +08:00
|
|
|
|
|
|
|
/* Ensure that the address returned is DMA'ble */
|
2020-08-17 23:34:03 +08:00
|
|
|
dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
|
2020-02-03 21:44:38 +08:00
|
|
|
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
|
2021-03-01 15:44:24 +08:00
|
|
|
swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
|
2018-08-20 22:21:10 +08:00
|
|
|
attrs | DMA_ATTR_SKIP_CPU_SYNC);
|
2020-02-03 21:44:38 +08:00
|
|
|
dev_WARN_ONCE(dev, 1,
|
|
|
|
"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
|
|
|
|
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
|
|
|
|
return DMA_MAPPING_ERROR;
|
2018-10-19 14:51:53 +08:00
|
|
|
}
|
|
|
|
|
2020-02-03 21:44:38 +08:00
|
|
|
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
|
|
|
arch_sync_dma_for_device(swiotlb_addr, size, dir);
|
|
|
|
return dma_addr;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2019-02-07 19:59:13 +08:00
|
|
|
size_t swiotlb_max_mapping_size(struct device *dev)
|
|
|
|
{
|
2022-05-10 22:21:09 +08:00
|
|
|
int min_align_mask = dma_get_min_align_mask(dev);
|
|
|
|
int min_align = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* swiotlb_find_slots() skips slots according to
|
|
|
|
* min align mask. This affects max mapping size.
|
|
|
|
* Take it into acount here.
|
|
|
|
*/
|
|
|
|
if (min_align_mask)
|
|
|
|
min_align = roundup(min_align_mask, IO_TLB_SIZE);
|
|
|
|
|
|
|
|
return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
|
2019-02-07 19:59:13 +08:00
|
|
|
}
|
2019-02-07 19:59:14 +08:00
|
|
|
|
2021-06-19 11:40:36 +08:00
|
|
|
bool is_swiotlb_active(struct device *dev)
|
2019-02-07 19:59:14 +08:00
|
|
|
{
|
2021-07-20 21:38:24 +08:00
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
|
|
|
|
|
|
|
return mem && mem->nslabs;
|
2019-02-07 19:59:14 +08:00
|
|
|
}
|
2021-03-19 00:14:24 +08:00
|
|
|
EXPORT_SYMBOL_GPL(is_swiotlb_active);
|
2019-03-11 03:47:57 +08:00
|
|
|
|
2023-04-20 17:58:58 +08:00
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
|
|
|
2022-07-28 15:24:20 +08:00
|
|
|
static int io_tlb_used_get(void *data, u64 *val)
|
|
|
|
{
|
2023-04-13 23:37:30 +08:00
|
|
|
struct io_tlb_mem *mem = data;
|
|
|
|
|
|
|
|
*val = mem_used(mem);
|
2022-07-28 15:24:20 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2023-04-14 01:57:37 +08:00
|
|
|
|
|
|
|
static int io_tlb_hiwater_get(void *data, u64 *val)
|
|
|
|
{
|
|
|
|
struct io_tlb_mem *mem = data;
|
|
|
|
|
|
|
|
*val = atomic_long_read(&mem->used_hiwater);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int io_tlb_hiwater_set(void *data, u64 val)
|
|
|
|
{
|
|
|
|
struct io_tlb_mem *mem = data;
|
|
|
|
|
|
|
|
/* Only allow setting to zero */
|
|
|
|
if (val != 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
atomic_long_set(&mem->used_hiwater, val);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-07-28 15:24:20 +08:00
|
|
|
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
|
2023-04-14 01:57:37 +08:00
|
|
|
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
|
|
|
|
io_tlb_hiwater_set, "%llu\n");
|
2022-07-28 15:24:20 +08:00
|
|
|
|
2022-01-25 00:40:17 +08:00
|
|
|
static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
|
|
|
|
const char *dirname)
|
2019-01-18 15:10:27 +08:00
|
|
|
{
|
2023-04-14 01:57:37 +08:00
|
|
|
atomic_long_set(&mem->total_used, 0);
|
|
|
|
atomic_long_set(&mem->used_hiwater, 0);
|
|
|
|
|
2022-01-25 00:40:17 +08:00
|
|
|
mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
|
|
|
|
if (!mem->nslabs)
|
|
|
|
return;
|
|
|
|
|
2021-03-19 00:14:22 +08:00
|
|
|
debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
|
2023-04-13 23:37:30 +08:00
|
|
|
debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
|
2022-07-28 15:24:20 +08:00
|
|
|
&fops_io_tlb_used);
|
2023-04-14 01:57:37 +08:00
|
|
|
debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
|
|
|
|
&fops_io_tlb_hiwater);
|
2021-06-19 11:40:33 +08:00
|
|
|
}
|
|
|
|
|
2023-04-20 17:58:58 +08:00
|
|
|
static int __init swiotlb_create_default_debugfs(void)
|
2021-06-19 11:40:33 +08:00
|
|
|
{
|
2022-01-25 00:40:17 +08:00
|
|
|
swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
|
2019-01-18 15:10:27 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:33 +08:00
|
|
|
late_initcall(swiotlb_create_default_debugfs);
|
2023-04-20 17:58:58 +08:00
|
|
|
|
|
|
|
#else /* !CONFIG_DEBUG_FS */
|
|
|
|
|
|
|
|
static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
|
|
|
|
const char *dirname)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_DEBUG_FS */
|
2021-06-19 11:40:40 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_DMA_RESTRICTED_POOL
|
2021-07-01 11:31:30 +08:00
|
|
|
|
2021-06-19 11:40:40 +08:00
|
|
|
struct page *swiotlb_alloc(struct device *dev, size_t size)
|
|
|
|
{
|
|
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
|
|
|
phys_addr_t tlb_addr;
|
|
|
|
int index;
|
|
|
|
|
|
|
|
if (!mem)
|
|
|
|
return NULL;
|
|
|
|
|
2021-09-29 10:32:59 +08:00
|
|
|
index = swiotlb_find_slots(dev, 0, size, 0);
|
2021-06-19 11:40:40 +08:00
|
|
|
if (index == -1)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
tlb_addr = slot_addr(mem->start, index);
|
|
|
|
|
|
|
|
return pfn_to_page(PFN_DOWN(tlb_addr));
|
|
|
|
}
|
|
|
|
|
|
|
|
bool swiotlb_free(struct device *dev, struct page *page, size_t size)
|
|
|
|
{
|
|
|
|
phys_addr_t tlb_addr = page_to_phys(page);
|
|
|
|
|
|
|
|
if (!is_swiotlb_buffer(dev, tlb_addr))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
swiotlb_release_slots(dev, tlb_addr);
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:41 +08:00
|
|
|
static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
|
|
|
|
struct device *dev)
|
|
|
|
{
|
|
|
|
struct io_tlb_mem *mem = rmem->priv;
|
|
|
|
unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
/* Set Per-device io tlb area to one */
|
|
|
|
unsigned int nareas = 1;
|
|
|
|
|
2023-04-15 05:29:25 +08:00
|
|
|
if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
|
|
|
|
dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:41 +08:00
|
|
|
/*
|
|
|
|
* Since multiple devices can share the same pool, the private data,
|
|
|
|
* io_tlb_mem struct, will be initialized by the first device attached
|
|
|
|
* to it.
|
|
|
|
*/
|
|
|
|
if (!mem) {
|
2021-07-20 21:38:24 +08:00
|
|
|
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
|
2021-06-19 11:40:41 +08:00
|
|
|
if (!mem)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2022-01-25 00:40:19 +08:00
|
|
|
mem->slots = kcalloc(nslabs, sizeof(*mem->slots), GFP_KERNEL);
|
2021-07-20 21:38:24 +08:00
|
|
|
if (!mem->slots) {
|
|
|
|
kfree(mem);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2022-07-09 00:15:44 +08:00
|
|
|
mem->areas = kcalloc(nareas, sizeof(*mem->areas),
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!mem->areas) {
|
|
|
|
kfree(mem->slots);
|
2022-07-15 16:19:50 +08:00
|
|
|
kfree(mem);
|
2022-07-09 00:15:44 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2021-06-19 11:40:41 +08:00
|
|
|
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
|
|
|
|
rmem->size >> PAGE_SHIFT);
|
2022-06-02 02:49:39 +08:00
|
|
|
swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, SWIOTLB_FORCE,
|
2022-07-09 00:15:44 +08:00
|
|
|
false, nareas);
|
2021-06-19 11:40:41 +08:00
|
|
|
mem->for_alloc = true;
|
|
|
|
|
|
|
|
rmem->priv = mem;
|
|
|
|
|
2022-01-25 00:40:17 +08:00
|
|
|
swiotlb_create_debugfs_files(mem, rmem->name);
|
2021-06-19 11:40:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
dev->dma_io_tlb_mem = mem;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
|
|
|
|
struct device *dev)
|
|
|
|
{
|
2021-07-20 21:38:24 +08:00
|
|
|
dev->dma_io_tlb_mem = &io_tlb_default_mem;
|
2021-06-19 11:40:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct reserved_mem_ops rmem_swiotlb_ops = {
|
|
|
|
.device_init = rmem_swiotlb_device_init,
|
|
|
|
.device_release = rmem_swiotlb_device_release,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
|
|
|
|
{
|
|
|
|
unsigned long node = rmem->fdt_node;
|
|
|
|
|
|
|
|
if (of_get_flat_dt_prop(node, "reusable", NULL) ||
|
|
|
|
of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
|
|
|
|
of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
|
|
|
|
of_get_flat_dt_prop(node, "no-map", NULL))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
rmem->ops = &rmem_swiotlb_ops;
|
|
|
|
pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
|
|
|
|
&rmem->base, (unsigned long)rmem->size / SZ_1M);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
|
2021-06-19 11:40:40 +08:00
|
|
|
#endif /* CONFIG_DMA_RESTRICTED_POOL */
|