UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'for-next/caches' into for-next/core 

Big cleanup of our cache maintenance routines, which were confusingly
named and inconsistent in their implementations.

* for-next/caches:
  arm64: Rename arm64-internal cache maintenance functions
  arm64: Fix cache maintenance function comments
  arm64: sync_icache_aliases to take end parameter instead of size
  arm64: __clean_dcache_area_pou to take end parameter instead of size
  arm64: __clean_dcache_area_pop to take end parameter instead of size
  arm64: __clean_dcache_area_poc to take end parameter instead of size
  arm64: __flush_dcache_area to take end parameter instead of size
  arm64: dcache_by_line_op to take end parameter instead of size
  arm64: __inval_dcache_area to take end parameter instead of size
  arm64: Fix comments to refer to correct function __flush_icache_range
  arm64: Move documentation of dcache_by_line_op
  arm64: assembler: remove user_alt
  arm64: Downgrade flush_icache_range to invalidate
  arm64: Do not enable uaccess for invalidate_icache_range
  arm64: Do not enable uaccess for flush_icache_range
  arm64: Apply errata to swsusp_arch_suspend_exit
  arm64: assembler: add conditional cache fixups
  arm64: assembler: replace `kaddr` with `addr`
This commit is contained in:
Will Deacon 2021-06-24 13:33:02 +01:00
parent 216fe62d6f fade9c2c6e
commit 25377204eb
27 changed files with 285 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
arch/arm64/include/asm/alternative-macros.h
View File

@ -197,11 +197,6 @@ alternative_endif
#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
.macro user_alt, label, oldinstr, newinstr, cond
9999: alternative_insn "\oldinstr", "\newinstr", \cond
_asm_extable 9999b, \label
.endm
#endif /* __ASSEMBLY__ */
/*

3
arch/arm64/include/asm/arch_gicv3.h
View File

@ -124,7 +124,8 @@ static inline u32 gic_read_rpr(void)
#define gic_read_lpir(c) readq_relaxed(c)
#define gic_write_lpir(v, c) writeq_relaxed(v, c)
#define gic_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
#define gic_flush_dcache_to_poc(a,l) \
dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l))
#define gits_read_baser(c) readq_relaxed(c)
#define gits_write_baser(v, c) writeq_relaxed(v, c)

82
arch/arm64/include/asm/assembler.h
View File

@ -130,15 +130,27 @@ alternative_endif
.endm
/*
* Emit an entry into the exception table
* Create an exception table entry for `insn`, which will branch to `fixup`
* when an unhandled fault is taken.
*/
.macro _asm_extable, from, to
.macro _asm_extable, insn, fixup
.pushsection __ex_table, "a"
.align 3
.long (\from - .), (\to - .)
.long (\insn - .), (\fixup - .)
.popsection
.endm
/*
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
* do nothing.
*/
.macro _cond_extable, insn, fixup
.ifnc \fixup,
_asm_extable \insn, \fixup
.endif
.endm
#define USER(l, x...) \
9999: x; \
_asm_extable 9999b, l
@ -383,51 +395,53 @@ alternative_cb_end
bfi \tcr, \tmp0, \pos, #3
.endm
/*
* Macro to perform a data cache maintenance for the interval
* [kaddr, kaddr + size)
*
* op: operation passed to dc instruction
* domain: domain used in dsb instruciton
* kaddr: starting virtual address of the region
* size: size of the region
* Corrupts: kaddr, size, tmp1, tmp2
*/
.macro __dcache_op_workaround_clean_cache, op, kaddr
.macro __dcache_op_workaround_clean_cache, op, addr
alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
dc \op, \kaddr
dc \op, \addr
alternative_else
dc civac, \kaddr
dc civac, \addr
alternative_endif
.endm
.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
/*
* Macro to perform a data cache maintenance for the interval
* [start, end)
*
* op: operation passed to dc instruction
* domain: domain used in dsb instruciton
* start: starting virtual address of the region
* end: end virtual address of the region
* fixup: optional label to branch to on user fault
* Corrupts: start, end, tmp1, tmp2
*/
.macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
dcache_line_size \tmp1, \tmp2
add \size, \kaddr, \size
sub \tmp2, \tmp1, #1
bic \kaddr, \kaddr, \tmp2
9998:
bic \start, \start, \tmp2
.Ldcache_op\@:
.ifc \op, cvau
__dcache_op_workaround_clean_cache \op, \kaddr
__dcache_op_workaround_clean_cache \op, \start
.else
.ifc \op, cvac
__dcache_op_workaround_clean_cache \op, \kaddr
__dcache_op_workaround_clean_cache \op, \start
.else
.ifc \op, cvap
sys 3, c7, c12, 1, \kaddr // dc cvap
sys 3, c7, c12, 1, \start // dc cvap
.else
.ifc \op, cvadp
sys 3, c7, c13, 1, \kaddr // dc cvadp
sys 3, c7, c13, 1, \start // dc cvadp
.else
dc \op, \kaddr
dc \op, \start
.endif
.endif
.endif
.endif
add \kaddr, \kaddr, \tmp1
cmp \kaddr, \size
b.lo 9998b
add \start, \start, \tmp1
cmp \start, \end
b.lo .Ldcache_op\@
dsb \domain
_cond_extable .Ldcache_op\@, \fixup
.endm
/*
@ -435,20 +449,22 @@ alternative_endif
* [start, end)
*
* start, end: virtual addresses describing the region
* label: A label to branch to on user fault.
* fixup: optional label to branch to on user fault
* Corrupts: tmp1, tmp2
*/
.macro invalidate_icache_by_line start, end, tmp1, tmp2, label
.macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup
icache_line_size \tmp1, \tmp2
sub \tmp2, \tmp1, #1
bic \tmp2, \start, \tmp2
9997:
USER(\label, ic ivau, \tmp2) // invalidate I line PoU
.Licache_op\@:
ic ivau, \tmp2 // invalidate I line PoU
add \tmp2, \tmp2, \tmp1
cmp \tmp2, \end
b.lo 9997b
b.lo .Licache_op\@
dsb ish
isb
_cond_extable .Licache_op\@, \fixup
.endm
/*

71
arch/arm64/include/asm/cacheflush.h
View File

@ -30,45 +30,58 @@
* the implementation assumes non-aliasing VIPT D-cache and (aliasing)
* VIPT I-cache.
*
* flush_icache_range(start, end)
* All functions below apply to the interval [start, end)
* - start - virtual start address (inclusive)
* - end - virtual end address (exclusive)
*
* Ensure coherency between the I-cache and the D-cache in the
* region described by start, end.
* - start - virtual start address
* - end - virtual end address
* caches_clean_inval_pou(start, end)
*
* invalidate_icache_range(start, end)
* Ensure coherency between the I-cache and the D-cache region to
* the Point of Unification.
*
* Invalidate the I-cache in the region described by start, end.
* - start - virtual start address
* - end - virtual end address
* caches_clean_inval_user_pou(start, end)
*
* __flush_cache_user_range(start, end)
* Ensure coherency between the I-cache and the D-cache region to
* the Point of Unification.
* Use only if the region might access user memory.
*
* Ensure coherency between the I-cache and the D-cache in the
* region described by start, end.
* - start - virtual start address
* - end - virtual end address
* icache_inval_pou(start, end)
*
* __flush_dcache_area(kaddr, size)
* Invalidate I-cache region to the Point of Unification.
*
* Ensure that the data held in page is written back.
* - kaddr - page address
* - size - region size
* dcache_clean_inval_poc(start, end)
*
* Clean and invalidate D-cache region to the Point of Coherency.
*
* dcache_inval_poc(start, end)
*
* Invalidate D-cache region to the Point of Coherency.
*
* dcache_clean_poc(start, end)
*
* Clean D-cache region to the Point of Coherency.
*
* dcache_clean_pop(start, end)
*
* Clean D-cache region to the Point of Persistence.
*
* dcache_clean_pou(start, end)
*
* Clean D-cache region to the Point of Unification.
*/
extern void __flush_icache_range(unsigned long start, unsigned long end);
extern int invalidate_icache_range(unsigned long start, unsigned long end);
extern void __flush_dcache_area(void *addr, size_t len);
extern void __inval_dcache_area(void *addr, size_t len);
extern void __clean_dcache_area_poc(void *addr, size_t len);
extern void __clean_dcache_area_pop(void *addr, size_t len);
extern void __clean_dcache_area_pou(void *addr, size_t len);
extern long __flush_cache_user_range(unsigned long start, unsigned long end);
extern void sync_icache_aliases(void *kaddr, unsigned long len);
extern void caches_clean_inval_pou(unsigned long start, unsigned long end);
extern void icache_inval_pou(unsigned long start, unsigned long end);
extern void dcache_clean_inval_poc(unsigned long start, unsigned long end);
extern void dcache_inval_poc(unsigned long start, unsigned long end);
extern void dcache_clean_poc(unsigned long start, unsigned long end);
extern void dcache_clean_pop(unsigned long start, unsigned long end);
extern void dcache_clean_pou(unsigned long start, unsigned long end);
extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end);
extern void sync_icache_aliases(unsigned long start, unsigned long end);
static inline void flush_icache_range(unsigned long start, unsigned long end)
{
__flush_icache_range(start, end);
caches_clean_inval_pou(start, end);
/*
* IPI all online CPUs so that they undergo a context synchronization
@ -122,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
extern void flush_dcache_page(struct page *);
static __always_inline void __flush_icache_all(void)
static __always_inline void icache_inval_all_pou(void)
{
if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
return;

2
arch/arm64/include/asm/efi.h
View File

@ -137,7 +137,7 @@ void efi_virtmap_unload(void);
static inline void efi_capsule_flush_cache_range(void *addr, int size)
{
__flush_dcache_area(addr, size);
dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size);
}
#endif /* _ASM_EFI_H */

7
arch/arm64/include/asm/kvm_mmu.h
View File

@ -180,7 +180,8 @@ static inline void *__kvm_vector_slot2addr(void *base,
struct kvm;
#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
#define kvm_flush_dcache_to_poc(a,l) \
dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l))
static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
{
@ -208,12 +209,12 @@ static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
{
if (icache_is_aliasing()) {
/* any kind of VIPT cache */
__flush_icache_all();
icache_inval_all_pou();
} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
void *va = page_address(pfn_to_page(pfn));
invalidate_icache_range((unsigned long)va,
icache_inval_pou((unsigned long)va,
(unsigned long)va + size);
}
}

2
arch/arm64/kernel/alternative.c
View File

@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu
*/
if (!is_module) {
dsb(ish);
__flush_icache_all();
icache_inval_all_pou();
isb();
/* Ignore ARM64_CB bit from feature mask */

9
arch/arm64/kernel/efi-entry.S
View File

@ -28,7 +28,8 @@ SYM_CODE_START(efi_enter_kernel)
* stale icache entries from before relocation.
*/
ldr w1, =kernel_size
bl __clean_dcache_area_poc
add x1, x0, x1
bl dcache_clean_poc
ic ialluis
/*
@ -36,8 +37,8 @@ SYM_CODE_START(efi_enter_kernel)
* so that we can safely disable the MMU and caches.
*/
adr x0, 0f
ldr w1, 3f
bl __clean_dcache_area_poc
adr x1, 3f
bl dcache_clean_poc
0:
/* Turn off Dcache and MMU */
mrs x0, CurrentEL
@ -64,5 +65,5 @@ SYM_CODE_START(efi_enter_kernel)
mov x2, xzr
mov x3, xzr
br x19
3:
SYM_CODE_END(efi_enter_kernel)
3: .long . - 0b

13
arch/arm64/kernel/head.S
View File

@ -118,8 +118,8 @@ SYM_CODE_START_LOCAL(preserve_boot_args)
dmb sy // needed before dc ivac with
// MMU off
mov x1, #0x20 // 4 x 8 bytes
b __inval_dcache_area // tail call
add x1, x0, #0x20 // 4 x 8 bytes
b dcache_inval_poc // tail call
SYM_CODE_END(preserve_boot_args)
/*
@ -269,8 +269,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
*/
adrp x0, init_pg_dir
adrp x1, init_pg_end
sub x1, x1, x0
bl __inval_dcache_area
bl dcache_inval_poc
/*
* Clear the init page tables.
@ -383,13 +382,11 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
adrp x0, idmap_pg_dir
adrp x1, idmap_pg_end
sub x1, x1, x0
bl __inval_dcache_area
bl dcache_inval_poc
adrp x0, init_pg_dir
adrp x1, init_pg_end
sub x1, x1, x0
bl __inval_dcache_area
bl dcache_inval_poc
ret x28
SYM_FUNC_END(__create_page_tables)

7
arch/arm64/kernel/hibernate-asm.S
View File

@ -45,7 +45,7 @@
* Because this code has to be copied to a 'safe' page, it can't call out to
* other functions by PC-relative address. Also remember that it may be
* mid-way through over-writing other functions. For this reason it contains
* code from flush_icache_range() and uses the copy_page() macro.
* code from caches_clean_inval_pou() and uses the copy_page() macro.
*
* This 'safe' page is mapped via ttbr0, and executed from there. This function
* switches to a copy of the linear map in ttbr1, performs the restore, then
@ -87,11 +87,12 @@ SYM_CODE_START(swsusp_arch_suspend_exit)
copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9
add x1, x10, #PAGE_SIZE
/* Clean the copied page to PoU - based on flush_icache_range() */
/* Clean the copied page to PoU - based on caches_clean_inval_pou() */
raw_dcache_line_size x2, x3
sub x3, x2, #1
bic x4, x10, x3
2: dc cvau, x4 /* clean D line / unified line */
2: /* clean D line / unified line */
alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE
add x4, x4, x2
cmp x4, x1
b.lo 2b

20
arch/arm64/kernel/hibernate.c
View File

@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length,
return -ENOMEM;
memcpy(page, src_start, length);
__flush_icache_range((unsigned long)page, (unsigned long)page + length);
caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length);
rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
if (rc)
return rc;
@ -240,8 +240,6 @@ static int create_safe_exec_page(void *src_start, size_t length,
return 0;
}
#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start))
#ifdef CONFIG_ARM64_MTE
static DEFINE_XARRAY(mte_pages);
@ -383,13 +381,18 @@ int swsusp_arch_suspend(void)
ret = swsusp_save();
} else {
/* Clean kernel core startup/idle code to PoC*/
dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end);
dcache_clean_range(__idmap_text_start, __idmap_text_end);
dcache_clean_inval_poc((unsigned long)__mmuoff_data_start,
(unsigned long)__mmuoff_data_end);
dcache_clean_inval_poc((unsigned long)__idmap_text_start,
(unsigned long)__idmap_text_end);
/* Clean kvm setup code to PoC? */
if (el2_reset_needed()) {
dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end);
dcache_clean_range(__hyp_text_start, __hyp_text_end);
dcache_clean_inval_poc(
(unsigned long)__hyp_idmap_text_start,
(unsigned long)__hyp_idmap_text_end);
dcache_clean_inval_poc((unsigned long)__hyp_text_start,
(unsigned long)__hyp_text_end);
}
swsusp_mte_restore_tags();
@ -474,7 +477,8 @@ int swsusp_arch_resume(void)
* The hibernate exit text contains a set of el2 vectors, that will
* be executed at el2 with the mmu off in order to reload hyp-stub.
*/
__flush_dcache_area(hibernate_exit, exit_size);
dcache_clean_inval_poc((unsigned long)hibernate_exit,
(unsigned long)hibernate_exit + exit_size);
/*
* KASLR will cause the el2 vectors to be in a different location in

3
arch/arm64/kernel/idreg-override.c
View File

@ -237,7 +237,8 @@ asmlinkage void __init init_feature_override(void)
for (i = 0; i < ARRAY_SIZE(regs); i++) {
if (regs[i]->override)
__flush_dcache_area(regs[i]->override,
dcache_clean_inval_poc((unsigned long)regs[i]->override,
(unsigned long)regs[i]->override +
sizeof(*regs[i]->override));
}
}

2
arch/arm64/kernel/image-vars.h
View File

@ -35,7 +35,7 @@ __efistub_strnlen = __pi_strnlen;
__efistub_strcmp = __pi_strcmp;
__efistub_strncmp = __pi_strncmp;
__efistub_strrchr = __pi_strrchr;
__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc;
__efistub_dcache_clean_poc = __pi_dcache_clean_poc;
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
__efistub___memcpy = __pi_memcpy;

2
arch/arm64/kernel/insn.c
View File

@ -198,7 +198,7 @@ int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
ret = aarch64_insn_write(tp, insn);
if (ret == 0)
__flush_icache_range((uintptr_t)tp,
caches_clean_inval_pou((uintptr_t)tp,
(uintptr_t)tp + AARCH64_INSN_SIZE);
return ret;

12
arch/arm64/kernel/kaslr.c
View File

@ -72,7 +72,9 @@ u64 __init kaslr_early_init(void)
* we end up running with module randomization disabled.
*/
module_alloc_base = (u64)_etext - MODULES_VSIZE;
__flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base));
dcache_clean_inval_poc((unsigned long)&module_alloc_base,
(unsigned long)&module_alloc_base +
sizeof(module_alloc_base));
/*
* Try to map the FDT early. If this fails, we simply bail,
@ -170,8 +172,12 @@ u64 __init kaslr_early_init(void)
module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21;
module_alloc_base &= PAGE_MASK;
__flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base));
__flush_dcache_area(&memstart_offset_seed, sizeof(memstart_offset_seed));
dcache_clean_inval_poc((unsigned long)&module_alloc_base,
(unsigned long)&module_alloc_base +
sizeof(module_alloc_base));
dcache_clean_inval_poc((unsigned long)&memstart_offset_seed,
(unsigned long)&memstart_offset_seed +
sizeof(memstart_offset_seed));
return offset;
}

30
arch/arm64/kernel/machine_kexec.c
View File

@ -68,10 +68,16 @@ int machine_kexec_post_load(struct kimage *kimage)
kimage->arch.kern_reloc = __pa(reloc_code);
kexec_image_info(kimage);
/* Flush the reloc_code in preparation for its execution. */
__flush_dcache_area(reloc_code, arm64_relocate_new_kernel_size);
flush_icache_range((uintptr_t)reloc_code, (uintptr_t)reloc_code +
arm64_relocate_new_kernel_size);
/*
* For execution with the MMU off, reloc_code needs to be cleaned to the
* PoC and invalidated from the I-cache.
*/
dcache_clean_inval_poc((unsigned long)reloc_code,
(unsigned long)reloc_code +
arm64_relocate_new_kernel_size);
icache_inval_pou((uintptr_t)reloc_code,
(uintptr_t)reloc_code +
arm64_relocate_new_kernel_size);
return 0;
}
@ -102,16 +108,18 @@ static void kexec_list_flush(struct kimage *kimage)
for (entry = &kimage->head; ; entry++) {
unsigned int flag;
void *addr;
unsigned long addr;
/* flush the list entries. */
__flush_dcache_area(entry, sizeof(kimage_entry_t));
dcache_clean_inval_poc((unsigned long)entry,
(unsigned long)entry +
sizeof(kimage_entry_t));
flag = *entry & IND_FLAGS;
if (flag == IND_DONE)
break;
addr = phys_to_virt(*entry & PAGE_MASK);
addr = (unsigned long)phys_to_virt(*entry & PAGE_MASK);
switch (flag) {
case IND_INDIRECTION:
@ -120,7 +128,7 @@ static void kexec_list_flush(struct kimage *kimage)
break;
case IND_SOURCE:
/* flush the source pages. */
__flush_dcache_area(addr, PAGE_SIZE);
dcache_clean_inval_poc(addr, addr + PAGE_SIZE);
break;
case IND_DESTINATION:
break;
@ -147,8 +155,10 @@ static void kexec_segment_flush(const struct kimage *kimage)
kimage->segment[i].memsz,
kimage->segment[i].memsz / PAGE_SIZE);
__flush_dcache_area(phys_to_virt(kimage->segment[i].mem),
kimage->segment[i].memsz);
dcache_clean_inval_poc(
(unsigned long)phys_to_virt(kimage->segment[i].mem),
(unsigned long)phys_to_virt(kimage->segment[i].mem) +
kimage->segment[i].memsz);
}
}

2
arch/arm64/kernel/probes/uprobes.c
View File

@ -21,7 +21,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
memcpy(dst, src, len);
/* flush caches (dcache/icache) */
sync_icache_aliases(dst, len);
sync_icache_aliases((unsigned long)dst, (unsigned long)dst + len);
kunmap_atomic(xol_page_kaddr);
}

7
arch/arm64/kernel/smp_spin_table.c
View File

@ -36,7 +36,7 @@ static void write_pen_release(u64 val)
unsigned long size = sizeof(secondary_holding_pen_release);
secondary_holding_pen_release = val;
__flush_dcache_area(start, size);
dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size);
}
@ -90,8 +90,9 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu)
* the boot protocol.
*/
writeq_relaxed(pa_holding_pen, release_addr);
__flush_dcache_area((__force void *)release_addr,
sizeof(*release_addr));
dcache_clean_inval_poc((__force unsigned long)release_addr,
(__force unsigned long)release_addr +
sizeof(*release_addr));
/*
* Send an event to wake up the secondary CPU.

2
arch/arm64/kernel/sys_compat.c
View File

@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)
dsb(ish);
}
ret = __flush_cache_user_range(start, start + chunk);
ret = caches_clean_inval_user_pou(start, start + chunk);
if (ret)
return ret;

2
arch/arm64/kvm/arm.c
View File

@ -1064,7 +1064,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
stage2_unmap_vm(vcpu->kvm);
else
__flush_icache_all();
icache_inval_all_pou();
}
vcpu_reset_hcr(vcpu);

4
arch/arm64/kvm/hyp/nvhe/cache.S
View File

@ -7,7 +7,7 @@
#include <asm/assembler.h>
#include <asm/alternative.h>
SYM_FUNC_START_PI(__flush_dcache_area)
SYM_FUNC_START_PI(dcache_clean_inval_poc)
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__flush_dcache_area)
SYM_FUNC_END_PI(dcache_clean_inval_poc)

3
arch/arm64/kvm/hyp/nvhe/setup.c
View File

@ -134,7 +134,8 @@ static void update_nvhe_init_params(void)
for (i = 0; i < hyp_nr_cpus; i++) {
params = per_cpu_ptr(&kvm_init_params, i);
params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
__flush_dcache_area(params, sizeof(*params));
dcache_clean_inval_poc((unsigned long)params,
(unsigned long)params + sizeof(*params));
}
}

2
arch/arm64/kvm/hyp/nvhe/tlb.c
View File

@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
* you should be running with VHE enabled.
*/
if (icache_is_vpipt())
__flush_icache_all();
icache_inval_all_pou();
__tlb_switch_to_host(&cxt);
}

13
arch/arm64/kvm/hyp/pgtable.c
View File

@ -839,8 +839,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
stage2_put_pte(ptep, mmu, addr, level, mm_ops);
if (need_flush) {
__flush_dcache_area(kvm_pte_follow(pte, mm_ops),
kvm_granule_size(level));
kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
dcache_clean_inval_poc((unsigned long)pte_follow,
(unsigned long)pte_follow +
kvm_granule_size(level));
}
if (childp)
@ -988,11 +991,15 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
struct kvm_pgtable *pgt = arg;
struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
kvm_pte_t pte = *ptep;
kvm_pte_t *pte_follow;
if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))
return 0;
__flush_dcache_area(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level));
pte_follow = kvm_pte_follow(pte, mm_ops);
dcache_clean_inval_poc((unsigned long)pte_follow,
(unsigned long)pte_follow +
kvm_granule_size(level));
return 0;
}

4
arch/arm64/lib/uaccess_flushcache.c
View File

@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt)
* barrier to order the cache maintenance against the memcpy.
*/
memcpy(dst, src, cnt);
__clean_dcache_area_pop(dst, cnt);
dcache_clean_pop((unsigned long)dst, (unsigned long)dst + cnt);
}
EXPORT_SYMBOL_GPL(memcpy_flushcache);
@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from,
rc = raw_copy_from_user(to, from, n);
/* See above */
__clean_dcache_area_pop(to, n - rc);
dcache_clean_pop((unsigned long)to, (unsigned long)to + n - rc);
return rc;
}

174
arch/arm64/mm/cache.S
View File

@ -15,7 +15,7 @@
#include <asm/asm-uaccess.h>
/*
* flush_icache_range(start,end)
* caches_clean_inval_pou_macro(start,end) [fixup]
*
* Ensure that the I and D caches are coherent within specified region.
* This is typically used when code has been written to a memory region,
@ -23,70 +23,54 @@
*
* - start - virtual start address of region
* - end - virtual end address of region
* - fixup - optional label to branch to on user fault
*/
SYM_FUNC_START(__flush_icache_range)
/* FALLTHROUGH */
/*
* __flush_cache_user_range(start,end)
*
* Ensure that the I and D caches are coherent within specified region.
* This is typically used when code has been written to a memory region,
* and will be executed.
*
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(__flush_cache_user_range)
uaccess_ttbr0_enable x2, x3, x4
.macro caches_clean_inval_pou_macro, fixup
alternative_if ARM64_HAS_CACHE_IDC
dsb ishst
b 7f
dsb ishst
b .Ldc_skip_\@
alternative_else_nop_endif
dcache_line_size x2, x3
sub x3, x2, #1
bic x4, x0, x3
1:
user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE
add x4, x4, x2
cmp x4, x1
b.lo 1b
dsb ish
7:
mov x2, x0
mov x3, x1
dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup
.Ldc_skip_\@:
alternative_if ARM64_HAS_CACHE_DIC
isb
b 8f
b .Lic_skip_\@
alternative_else_nop_endif
invalidate_icache_by_line x0, x1, x2, x3, 9f
8: mov x0, #0
1:
uaccess_ttbr0_disable x1, x2
ret
9:
mov x0, #-EFAULT
b 1b
SYM_FUNC_END(__flush_icache_range)
SYM_FUNC_END(__flush_cache_user_range)
invalidate_icache_by_line x0, x1, x2, x3, \fixup
.Lic_skip_\@:
.endm
/*
* invalidate_icache_range(start,end)
* caches_clean_inval_pou(start,end)
*
* Ensure that the I cache is invalid within specified region.
* Ensure that the I and D caches are coherent within specified region.
* This is typically used when code has been written to a memory region,
* and will be executed.
*
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(invalidate_icache_range)
alternative_if ARM64_HAS_CACHE_DIC
mov x0, xzr
isb
SYM_FUNC_START(caches_clean_inval_pou)
caches_clean_inval_pou_macro
ret
alternative_else_nop_endif
SYM_FUNC_END(caches_clean_inval_pou)
/*
* caches_clean_inval_user_pou(start,end)
*
* Ensure that the I and D caches are coherent within specified region.
* This is typically used when code has been written to a memory region,
* and will be executed.
*
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(caches_clean_inval_user_pou)
uaccess_ttbr0_enable x2, x3, x4
invalidate_icache_by_line x0, x1, x2, x3, 2f
caches_clean_inval_pou_macro 2f
mov x0, xzr
1:
uaccess_ttbr0_disable x1, x2
@ -94,60 +78,77 @@ alternative_else_nop_endif
2:
mov x0, #-EFAULT
b 1b
SYM_FUNC_END(invalidate_icache_range)
SYM_FUNC_END(caches_clean_inval_user_pou)
/*
* __flush_dcache_area(kaddr, size)
* icache_inval_pou(start,end)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* Ensure that the I cache is invalid within specified region.
*
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(icache_inval_pou)
alternative_if ARM64_HAS_CACHE_DIC
isb
ret
alternative_else_nop_endif
invalidate_icache_by_line x0, x1, x2, x3
ret
SYM_FUNC_END(icache_inval_pou)
/*
* dcache_clean_inval_poc(start, end)
*
* Ensure that any D-cache lines for the interval [start, end)
* are cleaned and invalidated to the PoC.
*
* - kaddr - kernel address
* - size - size in question
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START_PI(__flush_dcache_area)
SYM_FUNC_START_PI(dcache_clean_inval_poc)
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__flush_dcache_area)
SYM_FUNC_END_PI(dcache_clean_inval_poc)
/*
* __clean_dcache_area_pou(kaddr, size)
* dcache_clean_pou(start, end)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* Ensure that any D-cache lines for the interval [start, end)
* are cleaned to the PoU.
*
* - kaddr - kernel address
* - size - size in question
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(__clean_dcache_area_pou)
SYM_FUNC_START(dcache_clean_pou)
alternative_if ARM64_HAS_CACHE_IDC
dsb ishst
ret
alternative_else_nop_endif
dcache_by_line_op cvau, ish, x0, x1, x2, x3
ret
SYM_FUNC_END(__clean_dcache_area_pou)
SYM_FUNC_END(dcache_clean_pou)
/*
* __inval_dcache_area(kaddr, size)
* dcache_inval_poc(start, end)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* Ensure that any D-cache lines for the interval [start, end)
* are invalidated. Any partial lines at the ends of the interval are
* also cleaned to PoC to prevent data loss.
*
* - kaddr - kernel address
* - size - size in question
* - start - kernel start address of region
* - end - kernel end address of region
*/
SYM_FUNC_START_LOCAL(__dma_inv_area)
SYM_FUNC_START_PI(__inval_dcache_area)
SYM_FUNC_START_PI(dcache_inval_poc)
/* FALLTHROUGH */
/*
* __dma_inv_area(start, size)
* __dma_inv_area(start, end)
* - start - virtual start address of region
* - size - size in question
* - end - virtual end address of region
*/
add x1, x1, x0
dcache_line_size x2, x3
sub x3, x2, #1
tst x1, x3 // end cache line aligned?
@ -165,48 +166,48 @@ SYM_FUNC_START_PI(__inval_dcache_area)
b.lo 2b
dsb sy
ret
SYM_FUNC_END_PI(__inval_dcache_area)
SYM_FUNC_END_PI(dcache_inval_poc)
SYM_FUNC_END(__dma_inv_area)
/*
* __clean_dcache_area_poc(kaddr, size)
* dcache_clean_poc(start, end)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* Ensure that any D-cache lines for the interval [start, end)
* are cleaned to the PoC.
*
* - kaddr - kernel address
* - size - size in question
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START_LOCAL(__dma_clean_area)
SYM_FUNC_START_PI(__clean_dcache_area_poc)
SYM_FUNC_START_PI(dcache_clean_poc)
/* FALLTHROUGH */
/*
* __dma_clean_area(start, size)
* __dma_clean_area(start, end)
* - start - virtual start address of region
* - size - size in question
* - end - virtual end address of region
*/
dcache_by_line_op cvac, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__clean_dcache_area_poc)
SYM_FUNC_END_PI(dcache_clean_poc)
SYM_FUNC_END(__dma_clean_area)
/*
* __clean_dcache_area_pop(kaddr, size)
* dcache_clean_pop(start, end)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* Ensure that any D-cache lines for the interval [start, end)
* are cleaned to the PoP.
*
* - kaddr - kernel address
* - size - size in question
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START_PI(__clean_dcache_area_pop)
SYM_FUNC_START_PI(dcache_clean_pop)
alternative_if_not ARM64_HAS_DCPOP
b __clean_dcache_area_poc
b dcache_clean_poc
alternative_else_nop_endif
dcache_by_line_op cvap, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__clean_dcache_area_pop)
SYM_FUNC_END_PI(dcache_clean_pop)
/*
* __dma_flush_area(start, size)
@ -217,6 +218,7 @@ SYM_FUNC_END_PI(__clean_dcache_area_pop)
* - size - size in question
*/
SYM_FUNC_START_PI(__dma_flush_area)
add x1, x0, x1
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__dma_flush_area)
@ -228,6 +230,7 @@ SYM_FUNC_END_PI(__dma_flush_area)
* - dir - DMA direction
*/
SYM_FUNC_START_PI(__dma_map_area)
add x1, x0, x1
cmp w2, #DMA_FROM_DEVICE
b.eq __dma_inv_area
b __dma_clean_area
@ -240,6 +243,7 @@ SYM_FUNC_END_PI(__dma_map_area)
* - dir - DMA direction
*/
SYM_FUNC_START_PI(__dma_unmap_area)
add x1, x0, x1
cmp w2, #DMA_TO_DEVICE
b.ne __dma_inv_area
ret

29
arch/arm64/mm/flush.c
View File

@ -14,28 +14,25 @@
#include <asm/cache.h>
#include <asm/tlbflush.h>
void sync_icache_aliases(void *kaddr, unsigned long len)
void sync_icache_aliases(unsigned long start, unsigned long end)
{
unsigned long addr = (unsigned long)kaddr;
if (icache_is_aliasing()) {
__clean_dcache_area_pou(kaddr, len);
__flush_icache_all();
dcache_clean_pou(start, end);
icache_inval_all_pou();
} else {
/*
* Don't issue kick_all_cpus_sync() after I-cache invalidation
* for user mappings.
*/
__flush_icache_range(addr, addr + len);
caches_clean_inval_pou(start, end);
}
}
static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
unsigned long uaddr, void *kaddr,
unsigned long len)
static void flush_ptrace_access(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
if (vma->vm_flags & VM_EXEC)
sync_icache_aliases(kaddr, len);
sync_icache_aliases(start, end);
}
/*
@ -48,7 +45,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
unsigned long len)
{
memcpy(dst, src, len);
flush_ptrace_access(vma, page, uaddr, dst, len);
flush_ptrace_access(vma, (unsigned long)dst, (unsigned long)dst + len);
}
void __sync_icache_dcache(pte_t pte)
@ -56,7 +53,9 @@ void __sync_icache_dcache(pte_t pte)
struct page *page = pte_page(pte);
if (!test_bit(PG_dcache_clean, &page->flags)) {
sync_icache_aliases(page_address(page), page_size(page));
sync_icache_aliases((unsigned long)page_address(page),
(unsigned long)page_address(page) +
page_size(page));
set_bit(PG_dcache_clean, &page->flags);
}
}
@ -77,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page);
/*
* Additional functions defined in assembly.
*/
EXPORT_SYMBOL(__flush_icache_range);
EXPORT_SYMBOL(caches_clean_inval_pou);
#ifdef CONFIG_ARCH_HAS_PMEM_API
void arch_wb_cache_pmem(void *addr, size_t size)
{
/* Ensure order against any prior non-cacheable writes */
dmb(osh);
__clean_dcache_area_pop(addr, size);
dcache_clean_pop((unsigned long)addr, (unsigned long)addr + size);
}
EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
void arch_invalidate_pmem(void *addr, size_t size)
{
__inval_dcache_area(addr, size);
dcache_inval_poc((unsigned long)addr, (unsigned long)addr + size);
}
EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
#endif