2019-06-04 16:11:33 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2006-03-27 21:58:25 +08:00
|
|
|
/*
|
|
|
|
|
* linux/arch/arm/kernel/head-common.S
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1994-2002 Russell King
|
|
|
|
|
* Copyright (c) 2003 ARM Limited
|
|
|
|
|
* All Rights Reserved
|
|
|
|
|
*/
|
2014-06-30 23:29:12 +08:00
|
|
|
#include <asm/assembler.h>
|
2006-03-27 21:58:25 +08:00
|
|
|
|
2008-03-05 13:50:07 +08:00
|
|
|
#define ATAG_CORE 0x54410001
|
|
|
|
|
#define ATAG_CORE_SIZE ((2*4 + 3*4) >> 2)
|
2009-10-02 00:43:29 +08:00
|
|
|
#define ATAG_CORE_SIZE_EMPTY ((2*4) >> 2)
|
2008-03-05 13:50:07 +08:00
|
|
|
|
2011-04-29 04:27:20 +08:00
|
|
|
#ifdef CONFIG_CPU_BIG_ENDIAN
|
|
|
|
|
#define OF_DT_MAGIC 0xd00dfeed
|
|
|
|
|
#else
|
|
|
|
|
#define OF_DT_MAGIC 0xedfe0dd0 /* 0xd00dfeed in big-endian */
|
|
|
|
|
#endif
|
|
|
|
|
|
2006-03-27 21:58:25 +08:00
|
|
|
/*
|
|
|
|
|
* Exception handling. Something went wrong and we can't proceed. We
|
|
|
|
|
* ought to tell the user, but since we don't have any guarantee that
|
|
|
|
|
* we're even running on the right architecture, we do virtually nothing.
|
|
|
|
|
*
|
|
|
|
|
* If CONFIG_DEBUG_LL is set we try to print out something about the error
|
|
|
|
|
* and hope for the best (useful if bootloader fails to pass a proper
|
|
|
|
|
* machine ID for example).
|
|
|
|
|
*/
|
2010-10-05 00:45:25 +08:00
|
|
|
__HEAD
|
2007-06-01 05:02:22 +08:00
|
|
|
|
|
|
|
|
/* Determine validity of the r2 atags pointer. The heuristic requires
|
|
|
|
|
* that the pointer be aligned, in the first 16k of physical RAM and
|
2011-04-29 04:27:20 +08:00
|
|
|
* that the ATAG_CORE marker is first and present. If CONFIG_OF_FLATTREE
|
|
|
|
|
* is selected, then it will also accept a dtb pointer. Future revisions
|
2007-06-01 05:02:22 +08:00
|
|
|
* of this function may be more lenient with the physical address and
|
|
|
|
|
* may also be able to move the ATAGS block if necessary.
|
|
|
|
|
*
|
|
|
|
|
* Returns:
|
2011-04-29 04:27:20 +08:00
|
|
|
* r2 either valid atags pointer, valid dtb pointer, or zero
|
2007-06-01 05:02:22 +08:00
|
|
|
* r5, r6 corrupted
|
|
|
|
|
*/
|
|
|
|
|
__vet_atags:
|
|
|
|
|
tst r2, #0x3 @ aligned?
|
|
|
|
|
bne 1f
|
|
|
|
|
|
2011-04-29 04:27:20 +08:00
|
|
|
ldr r5, [r2, #0]
|
|
|
|
|
#ifdef CONFIG_OF_FLATTREE
|
|
|
|
|
ldr r6, =OF_DT_MAGIC @ is it a DTB?
|
|
|
|
|
cmp r5, r6
|
|
|
|
|
beq 2f
|
|
|
|
|
#endif
|
|
|
|
|
cmp r5, #ATAG_CORE_SIZE @ is first tag ATAG_CORE?
|
2009-10-02 00:43:29 +08:00
|
|
|
cmpne r5, #ATAG_CORE_SIZE_EMPTY
|
2007-06-01 05:02:22 +08:00
|
|
|
bne 1f
|
|
|
|
|
ldr r5, [r2, #4]
|
|
|
|
|
ldr r6, =ATAG_CORE
|
|
|
|
|
cmp r5, r6
|
|
|
|
|
bne 1f
|
|
|
|
|
|
2014-06-30 23:29:12 +08:00
|
|
|
2: ret lr @ atag/dtb pointer is ok
|
2007-06-01 05:02:22 +08:00
|
|
|
|
|
|
|
|
1: mov r2, #0
|
2014-06-30 23:29:12 +08:00
|
|
|
ret lr
|
2008-08-28 18:22:32 +08:00
|
|
|
ENDPROC(__vet_atags)
|
2010-10-01 22:37:05 +08:00
|
|
|
|
2010-10-04 23:29:35 +08:00
|
|
|
/*
|
|
|
|
|
* The following fragment of code is executed with the MMU on in MMU mode,
|
|
|
|
|
* and uses absolute addresses; this is not position independent.
|
|
|
|
|
*
|
2019-10-10 17:12:20 +08:00
|
|
|
* r0 = cp#15 control register (exc_ret for M-class)
|
2010-10-04 23:29:35 +08:00
|
|
|
* r1 = machine ID
|
2011-04-29 04:27:20 +08:00
|
|
|
* r2 = atags/dtb pointer
|
2010-10-04 23:29:35 +08:00
|
|
|
* r9 = processor ID
|
|
|
|
|
*/
|
|
|
|
|
__INIT
|
|
|
|
|
__mmap_switched:
|
2017-08-25 03:54:47 +08:00
|
|
|
|
|
|
|
|
mov r7, r1
|
|
|
|
|
mov r8, r2
|
|
|
|
|
mov r10, r0
|
|
|
|
|
|
|
|
|
|
adr r4, __mmap_switched_data
|
|
|
|
|
mov fp, #0
|
|
|
|
|
|
2017-08-25 12:54:18 +08:00
|
|
|
#if defined(CONFIG_XIP_DEFLATED_DATA)
|
|
|
|
|
ARM( ldr sp, [r4], #4 )
|
|
|
|
|
THUMB( ldr sp, [r4] )
|
|
|
|
|
THUMB( add r4, #4 )
|
|
|
|
|
bl __inflate_kernel_data @ decompress .data to RAM
|
|
|
|
|
teq r0, #0
|
|
|
|
|
bne __error
|
|
|
|
|
#elif defined(CONFIG_XIP_KERNEL)
|
2017-08-25 03:54:47 +08:00
|
|
|
ARM( ldmia r4!, {r0, r1, r2, sp} )
|
|
|
|
|
THUMB( ldmia r4!, {r0, r1, r2, r3} )
|
|
|
|
|
THUMB( mov sp, r3 )
|
|
|
|
|
sub r2, r2, r1
|
2020-10-26 06:52:08 +08:00
|
|
|
bl __memcpy @ copy .data to RAM
|
2017-08-25 03:54:47 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
ARM( ldmia r4!, {r0, r1, sp} )
|
|
|
|
|
THUMB( ldmia r4!, {r0, r1, r3} )
|
|
|
|
|
THUMB( mov sp, r3 )
|
2018-01-20 01:17:46 +08:00
|
|
|
sub r2, r1, r0
|
|
|
|
|
mov r1, #0
|
2020-10-26 06:52:08 +08:00
|
|
|
bl __memset @ clear .bss
|
2017-08-25 03:54:47 +08:00
|
|
|
|
ARM: smp: Store current pointer in TPIDRURO register if available
Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.
Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.
<do_one_initcall>:
e92d 41f0 stmdb sp!, {r4, r5, r6, r7, r8, lr}
ee1d 4f70 mrc 15, 0, r4, cr13, cr0, {3}
4606 mov r6, r0
b094 sub sp, #80 ; 0x50
f8d4 34e8 ldr.w r3, [r4, #1256] ; 0x4e8 <- stack canary
9313 str r3, [sp, #76] ; 0x4c
f8d4 8004 ldr.w r8, [r4, #4] <- preempt count
Co-developed-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Tested-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
2021-09-18 16:44:37 +08:00
|
|
|
adr_l r0, init_task @ get swapper task_struct
|
ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems
On UP systems, only a single task can be 'current' at the same time,
which means we can use a global variable to track it. This means we can
also enable THREAD_INFO_IN_TASK for those systems, as in that case,
thread_info is accessed via current rather than the other way around,
removing the need to store thread_info at the base of the task stack.
This, in turn, permits us to enable IRQ stacks and vmap'ed stacks on UP
systems as well.
To partially mitigate the performance overhead of this arrangement, use
a ADD/ADD/LDR sequence with the appropriate PC-relative group
relocations to load the value of current when needed. This means that
accessing current will still only require a single load as before,
avoiding the need for a literal to carry the address of the global
variable in each function. However, accessing thread_info will now
require this load as well.
Acked-by: Linus Walleij <linus.walleij@linaro.org>
Acked-by: Nicolas Pitre <nico@fluxnic.net>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Marc Zyngier <maz@kernel.org>
Tested-by: Vladimir Murzin <vladimir.murzin@arm.com> # ARMv7M
2021-11-24 21:08:11 +08:00
|
|
|
set_current r0, r1
|
ARM: smp: Store current pointer in TPIDRURO register if available
Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.
Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.
<do_one_initcall>:
e92d 41f0 stmdb sp!, {r4, r5, r6, r7, r8, lr}
ee1d 4f70 mrc 15, 0, r4, cr13, cr0, {3}
4606 mov r6, r0
b094 sub sp, #80 ; 0x50
f8d4 34e8 ldr.w r3, [r4, #1256] ; 0x4e8 <- stack canary
9313 str r3, [sp, #76] ; 0x4c
f8d4 8004 ldr.w r8, [r4, #4] <- preempt count
Co-developed-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Tested-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
2021-09-18 16:44:37 +08:00
|
|
|
|
2017-08-25 03:54:47 +08:00
|
|
|
ldmia r4, {r0, r1, r2, r3}
|
|
|
|
|
str r9, [r0] @ Save processor ID
|
|
|
|
|
str r7, [r1] @ Save machine type
|
|
|
|
|
str r8, [r2] @ Save atags pointer
|
|
|
|
|
cmp r3, #0
|
|
|
|
|
strne r10, [r3] @ Save control register values
|
2020-10-26 06:55:16 +08:00
|
|
|
#ifdef CONFIG_KASAN
|
|
|
|
|
bl kasan_early_init
|
|
|
|
|
#endif
|
2017-10-04 02:14:38 +08:00
|
|
|
mov lr, #0
|
2010-10-04 23:29:35 +08:00
|
|
|
b start_kernel
|
|
|
|
|
ENDPROC(__mmap_switched)
|
|
|
|
|
|
|
|
|
|
.align 2
|
|
|
|
|
.type __mmap_switched_data, %object
|
|
|
|
|
__mmap_switched_data:
|
2017-08-25 03:54:47 +08:00
|
|
|
#ifdef CONFIG_XIP_KERNEL
|
2017-08-25 12:54:18 +08:00
|
|
|
#ifndef CONFIG_XIP_DEFLATED_DATA
|
2017-08-25 03:54:47 +08:00
|
|
|
.long _sdata @ r0
|
|
|
|
|
.long __data_loc @ r1
|
|
|
|
|
.long _edata_loc @ r2
|
2017-08-25 12:54:18 +08:00
|
|
|
#endif
|
2017-08-25 03:54:47 +08:00
|
|
|
.long __bss_stop @ sp (temporary stack in .bss)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
.long __bss_start @ r0
|
|
|
|
|
.long __bss_stop @ r1
|
|
|
|
|
.long init_thread_union + THREAD_START_SP @ sp
|
|
|
|
|
|
|
|
|
|
.long processor_id @ r0
|
|
|
|
|
.long __machine_arch_type @ r1
|
|
|
|
|
.long __atags_pointer @ r2
|
2012-01-16 17:34:31 +08:00
|
|
|
#ifdef CONFIG_CPU_CP15
|
2017-08-25 03:54:47 +08:00
|
|
|
.long cr_alignment @ r3
|
2012-01-16 17:34:31 +08:00
|
|
|
#else
|
2019-10-10 17:12:20 +08:00
|
|
|
M_CLASS(.long exc_ret) @ r3
|
|
|
|
|
AR_CLASS(.long 0) @ r3
|
2012-01-16 17:34:31 +08:00
|
|
|
#endif
|
2010-10-04 23:29:35 +08:00
|
|
|
.size __mmap_switched_data, . - __mmap_switched_data
|
|
|
|
|
|
2018-07-19 18:42:36 +08:00
|
|
|
__FINIT
|
|
|
|
|
.text
|
|
|
|
|
|
2010-10-04 23:29:35 +08:00
|
|
|
/*
|
|
|
|
|
* This provides a C-API version of __lookup_processor_type
|
|
|
|
|
*/
|
|
|
|
|
ENTRY(lookup_processor_type)
|
|
|
|
|
stmfd sp!, {r4 - r6, r9, lr}
|
|
|
|
|
mov r9, r0
|
|
|
|
|
bl __lookup_processor_type
|
|
|
|
|
mov r0, r5
|
|
|
|
|
ldmfd sp!, {r4 - r6, r9, pc}
|
|
|
|
|
ENDPROC(lookup_processor_type)
|
|
|
|
|
|
2010-10-01 22:37:05 +08:00
|
|
|
/*
|
|
|
|
|
* Read processor ID register (CP#15, CR0), and look up in the linker-built
|
|
|
|
|
* supported processor list. Note that we can't use the absolute addresses
|
|
|
|
|
* for the __proc_info lists since we aren't running with the MMU on
|
|
|
|
|
* (and therefore, we are not in the correct address space). We have to
|
|
|
|
|
* calculate the offset.
|
|
|
|
|
*
|
|
|
|
|
* r9 = cpuid
|
|
|
|
|
* Returns:
|
|
|
|
|
* r3, r4, r6 corrupted
|
|
|
|
|
* r5 = proc_info pointer in physical address space
|
|
|
|
|
* r9 = cpuid (preserved)
|
|
|
|
|
*/
|
|
|
|
|
__lookup_processor_type:
|
2020-09-14 16:25:06 +08:00
|
|
|
/*
|
|
|
|
|
* Look in <asm/procinfo.h> for information about the __proc_info
|
|
|
|
|
* structure.
|
|
|
|
|
*/
|
|
|
|
|
adr_l r5, __proc_info_begin
|
|
|
|
|
adr_l r6, __proc_info_end
|
2010-10-01 22:37:05 +08:00
|
|
|
1: ldmia r5, {r3, r4} @ value, mask
|
|
|
|
|
and r4, r4, r9 @ mask wanted bits
|
|
|
|
|
teq r3, r4
|
|
|
|
|
beq 2f
|
|
|
|
|
add r5, r5, #PROC_INFO_SZ @ sizeof(proc_info_list)
|
|
|
|
|
cmp r5, r6
|
|
|
|
|
blo 1b
|
|
|
|
|
mov r5, #0 @ unknown processor
|
2014-06-30 23:29:12 +08:00
|
|
|
2: ret lr
|
2010-10-01 22:37:05 +08:00
|
|
|
ENDPROC(__lookup_processor_type)
|
|
|
|
|
|
ARM: 7980/1: kernel: improve error message when LPAE config doesn't match CPU
Currently, when the kernel is configured with LPAE support, but the
CPU doesn't support it, the error message is fairly cryptic:
Error: unrecognized/unsupported processor variant (0x561f5811).
This messages is normally shown when there is an issue when comparing
the processor ID (CP15 0, c0, c0) with the values/masks described in
proc-v7.S. However, the same message is displayed when LPAE support is
enabled in the kernel configuration, but not available in the CPU,
after looking at ID_MMFR0 (CP15 0, c0, c1, 4). Having the same error
message is highly misleading.
This commit improves this by showing a different error message when
this situation occurs:
Error: Kernel with LPAE support, but CPU does not support LPAE.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2014-02-19 00:02:54 +08:00
|
|
|
__error_lpae:
|
|
|
|
|
#ifdef CONFIG_DEBUG_LL
|
|
|
|
|
adr r0, str_lpae
|
|
|
|
|
bl printascii
|
|
|
|
|
b __error
|
|
|
|
|
str_lpae: .asciz "\nError: Kernel with LPAE support, but CPU does not support LPAE.\n"
|
|
|
|
|
#else
|
|
|
|
|
b __error
|
|
|
|
|
#endif
|
|
|
|
|
.align
|
|
|
|
|
ENDPROC(__error_lpae)
|
|
|
|
|
|
2010-10-05 00:39:20 +08:00
|
|
|
__error_p:
|
|
|
|
|
#ifdef CONFIG_DEBUG_LL
|
|
|
|
|
adr r0, str_p1
|
|
|
|
|
bl printascii
|
|
|
|
|
mov r0, r9
|
|
|
|
|
bl printhex8
|
|
|
|
|
adr r0, str_p2
|
|
|
|
|
bl printascii
|
|
|
|
|
b __error
|
|
|
|
|
str_p1: .asciz "\nError: unrecognized/unsupported processor variant (0x"
|
|
|
|
|
str_p2: .asciz ").\n"
|
|
|
|
|
.align
|
|
|
|
|
#endif
|
|
|
|
|
ENDPROC(__error_p)
|
|
|
|
|
|
|
|
|
|
__error:
|
|
|
|
|
#ifdef CONFIG_ARCH_RPC
|
|
|
|
|
/*
|
|
|
|
|
* Turn the screen red on a error - RiscPC only.
|
|
|
|
|
*/
|
|
|
|
|
mov r0, #0x02000000
|
|
|
|
|
mov r3, #0x11
|
|
|
|
|
orr r3, r3, r3, lsl #8
|
|
|
|
|
orr r3, r3, r3, lsl #16
|
|
|
|
|
str r3, [r0], #4
|
|
|
|
|
str r3, [r0], #4
|
|
|
|
|
str r3, [r0], #4
|
|
|
|
|
str r3, [r0], #4
|
|
|
|
|
#endif
|
|
|
|
|
1: mov r0, r0
|
|
|
|
|
b 1b
|
|
|
|
|
ENDPROC(__error)
|