Merge branch 'for-rmk/arm-mm-lpae' of git://git.kernel.org/pub/scm/linux/kernel/git/ssantosh/linux-keystone into devel-stable
This series extends the existing ARM v2p runtime patching for 64 bit. Needed for LPAE machines which have physical memory beyond 4GB.
This commit is contained in:
Russell King
commit
8754c4bf2a
|
|
@ -49,6 +49,7 @@ struct machine_desc {
|
|||
bool (*smp_init)(void);
|
||||
void (*fixup)(struct tag *, char **,
|
||||
struct meminfo *);
|
||||
void (*init_meminfo)(void);
|
||||
void (*reserve)(void);/* reserve mem blocks */
|
||||
void (*map_io)(void);/* IO mapping function */
|
||||
void (*init_early)(void);
|
||||
|
|
|
|||
|
|
@ -172,8 +172,14 @@
|
|||
* so that all we need to do is modify the 8-bit constant field.
|
||||
*/
|
||||
#define __PV_BITS_31_24 0x81000000
|
||||
#define __PV_BITS_7_0 0x81
|
||||
|
||||
extern phys_addr_t (*arch_virt_to_idmap) (unsigned long x);
|
||||
extern u64 __pv_phys_offset;
|
||||
extern u64 __pv_offset;
|
||||
extern void fixup_pv_table(const void *, unsigned long);
|
||||
extern const void *__pv_table_begin, *__pv_table_end;
|
||||
|
||||
extern unsigned long __pv_phys_offset;
|
||||
#define PHYS_OFFSET __pv_phys_offset
|
||||
|
||||
#define __pv_stub(from,to,instr,type) \
|
||||
|
|
@ -185,22 +191,58 @@ extern unsigned long __pv_phys_offset;
|
|||
: "=r" (to) \
|
||||
: "r" (from), "I" (type))
|
||||
|
||||
static inline unsigned long __virt_to_phys(unsigned long x)
|
||||
#define __pv_stub_mov_hi(t) \
|
||||
__asm__ volatile("@ __pv_stub_mov\n" \
|
||||
"1: mov %R0, %1\n" \
|
||||
" .pushsection .pv_table,\"a\"\n" \
|
||||
" .long 1b\n" \
|
||||
" .popsection\n" \
|
||||
: "=r" (t) \
|
||||
: "I" (__PV_BITS_7_0))
|
||||
|
||||
#define __pv_add_carry_stub(x, y) \
|
||||
__asm__ volatile("@ __pv_add_carry_stub\n" \
|
||||
"1: adds %Q0, %1, %2\n" \
|
||||
" adc %R0, %R0, #0\n" \
|
||||
" .pushsection .pv_table,\"a\"\n" \
|
||||
" .long 1b\n" \
|
||||
" .popsection\n" \
|
||||
: "+r" (y) \
|
||||
: "r" (x), "I" (__PV_BITS_31_24) \
|
||||
: "cc")
|
||||
|
||||
static inline phys_addr_t __virt_to_phys(unsigned long x)
|
||||
{
|
||||
unsigned long t;
|
||||
phys_addr_t t;
|
||||
|
||||
if (sizeof(phys_addr_t) == 4) {
|
||||
__pv_stub(x, t, "add", __PV_BITS_31_24);
|
||||
} else {
|
||||
__pv_stub_mov_hi(t);
|
||||
__pv_add_carry_stub(x, t);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
static inline unsigned long __phys_to_virt(unsigned long x)
|
||||
static inline unsigned long __phys_to_virt(phys_addr_t x)
|
||||
{
|
||||
unsigned long t;
|
||||
__pv_stub(x, t, "sub", __PV_BITS_31_24);
|
||||
return t;
|
||||
}
|
||||
|
||||
#else
|
||||
#define __virt_to_phys(x) ((x) - PAGE_OFFSET + PHYS_OFFSET)
|
||||
#define __phys_to_virt(x) ((x) - PHYS_OFFSET + PAGE_OFFSET)
|
||||
|
||||
static inline phys_addr_t __virt_to_phys(unsigned long x)
|
||||
{
|
||||
return (phys_addr_t)x - PAGE_OFFSET + PHYS_OFFSET;
|
||||
}
|
||||
|
||||
static inline unsigned long __phys_to_virt(phys_addr_t x)
|
||||
{
|
||||
return x - PHYS_OFFSET + PAGE_OFFSET;
|
||||
}
|
||||
|
||||
#endif
|
||||
#endif
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
|
@ -238,16 +280,31 @@ static inline phys_addr_t virt_to_phys(const volatile void *x)
|
|||
|
||||
static inline void *phys_to_virt(phys_addr_t x)
|
||||
{
|
||||
return (void *)(__phys_to_virt((unsigned long)(x)));
|
||||
return (void *)__phys_to_virt(x);
|
||||
}
|
||||
|
||||
/*
|
||||
* Drivers should NOT use these either.
|
||||
*/
|
||||
#define __pa(x) __virt_to_phys((unsigned long)(x))
|
||||
#define __va(x) ((void *)__phys_to_virt((unsigned long)(x)))
|
||||
#define __va(x) ((void *)__phys_to_virt((phys_addr_t)(x)))
|
||||
#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
|
||||
|
||||
/*
|
||||
* These are for systems that have a hardware interconnect supported alias of
|
||||
* physical memory for idmap purposes. Most cases should leave these
|
||||
* untouched.
|
||||
*/
|
||||
static inline phys_addr_t __virt_to_idmap(unsigned long x)
|
||||
{
|
||||
if (arch_virt_to_idmap)
|
||||
return arch_virt_to_idmap(x);
|
||||
else
|
||||
return __virt_to_phys(x);
|
||||
}
|
||||
|
||||
#define virt_to_idmap(x) __virt_to_idmap((unsigned long)(x))
|
||||
|
||||
/*
|
||||
* Virtual <-> DMA view memory address translations
|
||||
* Again, these are *only* valid on the kernel direct mapped RAM
|
||||
|
|
|
|||
|
|
@ -155,4 +155,5 @@ EXPORT_SYMBOL(__gnu_mcount_nc);
|
|||
|
||||
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
|
||||
EXPORT_SYMBOL(__pv_phys_offset);
|
||||
EXPORT_SYMBOL(__pv_offset);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -536,6 +536,14 @@ ENTRY(fixup_smp)
|
|||
ldmfd sp!, {r4 - r6, pc}
|
||||
ENDPROC(fixup_smp)
|
||||
|
||||
#ifdef __ARMEB_
|
||||
#define LOW_OFFSET 0x4
|
||||
#define HIGH_OFFSET 0x0
|
||||
#else
|
||||
#define LOW_OFFSET 0x0
|
||||
#define HIGH_OFFSET 0x4
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
|
||||
|
||||
/* __fixup_pv_table - patch the stub instructions with the delta between
|
||||
|
|
@ -546,17 +554,20 @@ ENDPROC(fixup_smp)
|
|||
__HEAD
|
||||
__fixup_pv_table:
|
||||
adr r0, 1f
|
||||
ldmia r0, {r3-r5, r7}
|
||||
sub r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET
|
||||
ldmia r0, {r3-r7}
|
||||
mvn ip, #0
|
||||
subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET
|
||||
add r4, r4, r3 @ adjust table start address
|
||||
add r5, r5, r3 @ adjust table end address
|
||||
add r7, r7, r3 @ adjust __pv_phys_offset address
|
||||
str r8, [r7] @ save computed PHYS_OFFSET to __pv_phys_offset
|
||||
add r6, r6, r3 @ adjust __pv_phys_offset address
|
||||
add r7, r7, r3 @ adjust __pv_offset address
|
||||
str r8, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_offset
|
||||
strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits
|
||||
mov r6, r3, lsr #24 @ constant for add/sub instructions
|
||||
teq r3, r6, lsl #24 @ must be 16MiB aligned
|
||||
THUMB( it ne @ cross section branch )
|
||||
bne __error
|
||||
str r6, [r7, #4] @ save to __pv_offset
|
||||
str r3, [r7, #LOW_OFFSET] @ save to __pv_offset low bits
|
||||
b __fixup_a_pv_table
|
||||
ENDPROC(__fixup_pv_table)
|
||||
|
||||
|
|
@ -565,10 +576,19 @@ ENDPROC(__fixup_pv_table)
|
|||
.long __pv_table_begin
|
||||
.long __pv_table_end
|
||||
2: .long __pv_phys_offset
|
||||
.long __pv_offset
|
||||
|
||||
.text
|
||||
__fixup_a_pv_table:
|
||||
adr r0, 3f
|
||||
ldr r6, [r0]
|
||||
add r6, r6, r3
|
||||
ldr r0, [r6, #HIGH_OFFSET] @ pv_offset high word
|
||||
ldr r6, [r6, #LOW_OFFSET] @ pv_offset low word
|
||||
mov r6, r6, lsr #24
|
||||
cmn r0, #1
|
||||
#ifdef CONFIG_THUMB2_KERNEL
|
||||
moveq r0, #0x200000 @ set bit 21, mov to mvn instruction
|
||||
lsls r6, #24
|
||||
beq 2f
|
||||
clz r7, r6
|
||||
|
|
@ -582,18 +602,28 @@ __fixup_a_pv_table:
|
|||
b 2f
|
||||
1: add r7, r3
|
||||
ldrh ip, [r7, #2]
|
||||
and ip, 0x8f00
|
||||
orr ip, r6 @ mask in offset bits 31-24
|
||||
tst ip, #0x4000
|
||||
and ip, #0x8f00
|
||||
orrne ip, r6 @ mask in offset bits 31-24
|
||||
orreq ip, r0 @ mask in offset bits 7-0
|
||||
strh ip, [r7, #2]
|
||||
ldrheq ip, [r7]
|
||||
biceq ip, #0x20
|
||||
orreq ip, ip, r0, lsr #16
|
||||
strheq ip, [r7]
|
||||
2: cmp r4, r5
|
||||
ldrcc r7, [r4], #4 @ use branch for delay slot
|
||||
bcc 1b
|
||||
bx lr
|
||||
#else
|
||||
moveq r0, #0x400000 @ set bit 22, mov to mvn instruction
|
||||
b 2f
|
||||
1: ldr ip, [r7, r3]
|
||||
bic ip, ip, #0x000000ff
|
||||
orr ip, ip, r6 @ mask in offset bits 31-24
|
||||
tst ip, #0xf00 @ check the rotation field
|
||||
orrne ip, ip, r6 @ mask in offset bits 31-24
|
||||
biceq ip, ip, #0x400000 @ clear bit 22
|
||||
orreq ip, ip, r0 @ mask in offset bits 7-0
|
||||
str ip, [r7, r3]
|
||||
2: cmp r4, r5
|
||||
ldrcc r7, [r4], #4 @ use branch for delay slot
|
||||
|
|
@ -602,28 +632,29 @@ __fixup_a_pv_table:
|
|||
#endif
|
||||
ENDPROC(__fixup_a_pv_table)
|
||||
|
||||
3: .long __pv_offset
|
||||
|
||||
ENTRY(fixup_pv_table)
|
||||
stmfd sp!, {r4 - r7, lr}
|
||||
ldr r2, 2f @ get address of __pv_phys_offset
|
||||
mov r3, #0 @ no offset
|
||||
mov r4, r0 @ r0 = table start
|
||||
add r5, r0, r1 @ r1 = table size
|
||||
ldr r6, [r2, #4] @ get __pv_offset
|
||||
bl __fixup_a_pv_table
|
||||
ldmfd sp!, {r4 - r7, pc}
|
||||
ENDPROC(fixup_pv_table)
|
||||
|
||||
.align
|
||||
2: .long __pv_phys_offset
|
||||
|
||||
.data
|
||||
.globl __pv_phys_offset
|
||||
.type __pv_phys_offset, %object
|
||||
__pv_phys_offset:
|
||||
.long 0
|
||||
.size __pv_phys_offset, . - __pv_phys_offset
|
||||
.quad 0
|
||||
.size __pv_phys_offset, . -__pv_phys_offset
|
||||
|
||||
.globl __pv_offset
|
||||
.type __pv_offset, %object
|
||||
__pv_offset:
|
||||
.long 0
|
||||
.quad 0
|
||||
.size __pv_offset, . -__pv_offset
|
||||
#endif
|
||||
|
||||
#include "head-common.S"
|
||||
|
|
|
|||
|
|
@ -73,6 +73,8 @@ __setup("fpe=", fpe_setup);
|
|||
#endif
|
||||
|
||||
extern void paging_init(const struct machine_desc *desc);
|
||||
extern void early_paging_init(const struct machine_desc *,
|
||||
struct proc_info_list *);
|
||||
extern void sanity_check_meminfo(void);
|
||||
extern enum reboot_mode reboot_mode;
|
||||
extern void setup_dma_zone(const struct machine_desc *desc);
|
||||
|
|
@ -878,6 +880,8 @@ void __init setup_arch(char **cmdline_p)
|
|||
parse_early_param();
|
||||
|
||||
sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL);
|
||||
|
||||
early_paging_init(mdesc, lookup_processor_type(read_cpuid_id()));
|
||||
sanity_check_meminfo();
|
||||
arm_memblock_init(&meminfo, mdesc);
|
||||
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ void __init smp_set_ops(struct smp_operations *ops)
|
|||
|
||||
static unsigned long get_arch_pgd(pgd_t *pgd)
|
||||
{
|
||||
phys_addr_t pgdir = virt_to_phys(pgd);
|
||||
phys_addr_t pgdir = virt_to_idmap(pgd);
|
||||
BUG_ON(pgdir & ARCH_PGD_MASK);
|
||||
return pgdir >> ARCH_PGD_SHIFT;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <asm/system_info.h>
|
||||
|
||||
pgd_t *idmap_pgd;
|
||||
phys_addr_t (*arch_virt_to_idmap) (unsigned long x);
|
||||
|
||||
#ifdef CONFIG_ARM_LPAE
|
||||
static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
|
||||
|
|
@ -67,8 +68,9 @@ static void identity_mapping_add(pgd_t *pgd, const char *text_start,
|
|||
unsigned long addr, end;
|
||||
unsigned long next;
|
||||
|
||||
addr = virt_to_phys(text_start);
|
||||
end = virt_to_phys(text_end);
|
||||
addr = virt_to_idmap(text_start);
|
||||
end = virt_to_idmap(text_end);
|
||||
pr_info("Setting up static identity map for 0x%lx - 0x%lx\n", addr, end);
|
||||
|
||||
prot |= PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
|
||||
|
||||
|
|
@ -90,8 +92,6 @@ static int __init init_static_idmap(void)
|
|||
if (!idmap_pgd)
|
||||
return -ENOMEM;
|
||||
|
||||
pr_info("Setting up static identity map for 0x%p - 0x%p\n",
|
||||
__idmap_text_start, __idmap_text_end);
|
||||
identity_mapping_add(idmap_pgd, __idmap_text_start,
|
||||
__idmap_text_end, 0);
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@
|
|||
#include <asm/highmem.h>
|
||||
#include <asm/system_info.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/procinfo.h>
|
||||
#include <asm/memory.h>
|
||||
|
||||
#include <asm/mach/arch.h>
|
||||
#include <asm/mach/map.h>
|
||||
|
|
@ -1315,6 +1317,86 @@ static void __init map_lowmem(void)
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM_LPAE
|
||||
/*
|
||||
* early_paging_init() recreates boot time page table setup, allowing machines
|
||||
* to switch over to a high (>4G) address space on LPAE systems
|
||||
*/
|
||||
void __init early_paging_init(const struct machine_desc *mdesc,
|
||||
struct proc_info_list *procinfo)
|
||||
{
|
||||
pmdval_t pmdprot = procinfo->__cpu_mm_mmu_flags;
|
||||
unsigned long map_start, map_end;
|
||||
pgd_t *pgd0, *pgdk;
|
||||
pud_t *pud0, *pudk, *pud_start;
|
||||
pmd_t *pmd0, *pmdk;
|
||||
phys_addr_t phys;
|
||||
int i;
|
||||
|
||||
if (!(mdesc->init_meminfo))
|
||||
return;
|
||||
|
||||
/* remap kernel code and data */
|
||||
map_start = init_mm.start_code;
|
||||
map_end = init_mm.brk;
|
||||
|
||||
/* get a handle on things... */
|
||||
pgd0 = pgd_offset_k(0);
|
||||
pud_start = pud0 = pud_offset(pgd0, 0);
|
||||
pmd0 = pmd_offset(pud0, 0);
|
||||
|
||||
pgdk = pgd_offset_k(map_start);
|
||||
pudk = pud_offset(pgdk, map_start);
|
||||
pmdk = pmd_offset(pudk, map_start);
|
||||
|
||||
mdesc->init_meminfo();
|
||||
|
||||
/* Run the patch stub to update the constants */
|
||||
fixup_pv_table(&__pv_table_begin,
|
||||
(&__pv_table_end - &__pv_table_begin) << 2);
|
||||
|
||||
/*
|
||||
* Cache cleaning operations for self-modifying code
|
||||
* We should clean the entries by MVA but running a
|
||||
* for loop over every pv_table entry pointer would
|
||||
* just complicate the code.
|
||||
*/
|
||||
flush_cache_louis();
|
||||
dsb();
|
||||
isb();
|
||||
|
||||
/* remap level 1 table */
|
||||
for (i = 0; i < PTRS_PER_PGD; pud0++, i++) {
|
||||
set_pud(pud0,
|
||||
__pud(__pa(pmd0) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
|
||||
pmd0 += PTRS_PER_PMD;
|
||||
}
|
||||
|
||||
/* remap pmds for kernel mapping */
|
||||
phys = __pa(map_start) & PMD_MASK;
|
||||
do {
|
||||
*pmdk++ = __pmd(phys | pmdprot);
|
||||
phys += PMD_SIZE;
|
||||
} while (phys < map_end);
|
||||
|
||||
flush_cache_all();
|
||||
cpu_switch_mm(pgd0, &init_mm);
|
||||
cpu_set_ttbr(1, __pa(pgd0) + TTBR1_OFFSET);
|
||||
local_flush_bp_all();
|
||||
local_flush_tlb_all();
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void __init early_paging_init(const struct machine_desc *mdesc,
|
||||
struct proc_info_list *procinfo)
|
||||
{
|
||||
if (mdesc->init_meminfo)
|
||||
mdesc->init_meminfo();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* paging_init() sets up the page tables, initialises the zone memory
|
||||
* maps, and sets up the zero page, bad page and bad page tables.
|
||||
|
|
|
|||
Loading…
Reference in New Issue