UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge patch series "riscv: Add GENERIC_ENTRY support" 

guoren@kernel.org <guoren@kernel.org> says:

From: Guo Ren <guoren@linux.alibaba.com>

The patches convert riscv to use the generic entry infrastructure from
kernel/entry/*. Some optimization for entry.S with new .macro and merge
ret_from_kernel_thread into ret_from_fork.

* b4-shazam-merge:
  riscv: entry: Consolidate general regs saving/restoring
  riscv: entry: Consolidate ret_from_kernel_thread into ret_from_fork
  riscv: entry: Remove extra level wrappers of trace_hardirqs_{on,off}
  riscv: entry: Convert to generic entry
  riscv: entry: Add noinstr to prevent instrumentation inserted
  riscv: ptrace: Remove duplicate operation

Link: https://lore.kernel.org/r/20230222033021.983168-1-guoren@kernel.org
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Palmer Dabbelt 2023-03-23 08:47:05 -07:00
parent 4b740779ac 45b32b946a
commit e45d6a52fe
No known key found for this signature in database
GPG Key ID: 2E1319F35FBB1889
20 changed files with 278 additions and 494 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/riscv/Kconfig
View File

@ -60,6 +60,7 @@ config RISCV
select GENERIC_ATOMIC64 if !64BIT
select GENERIC_CLOCKEVENTS_BROADCAST if SMP
select GENERIC_EARLY_IOREMAP
select GENERIC_ENTRY
select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IOREMAP if MMU

2
arch/riscv/include/asm/asm-prototypes.h
View File

@ -27,5 +27,7 @@ DECLARE_DO_ERROR_INFO(do_trap_break);
asmlinkage unsigned long get_overflow_stack(void);
asmlinkage void handle_bad_stack(struct pt_regs *regs);
asmlinkage void do_page_fault(struct pt_regs *regs);
asmlinkage void do_irq(struct pt_regs *regs);
#endif /* _ASM_RISCV_PROTOTYPES_H */

61
arch/riscv/include/asm/asm.h
View File

@ -69,6 +69,7 @@
#endif
#ifdef __ASSEMBLY__
#include <asm/asm-offsets.h>
/* Common assembly source macros */
@ -81,6 +82,66 @@
.endr
.endm
/* save all GPs except x1 ~ x5 */
.macro save_from_x6_to_x31
REG_S x6, PT_T1(sp)
REG_S x7, PT_T2(sp)
REG_S x8, PT_S0(sp)
REG_S x9, PT_S1(sp)
REG_S x10, PT_A0(sp)
REG_S x11, PT_A1(sp)
REG_S x12, PT_A2(sp)
REG_S x13, PT_A3(sp)
REG_S x14, PT_A4(sp)
REG_S x15, PT_A5(sp)
REG_S x16, PT_A6(sp)
REG_S x17, PT_A7(sp)
REG_S x18, PT_S2(sp)
REG_S x19, PT_S3(sp)
REG_S x20, PT_S4(sp)
REG_S x21, PT_S5(sp)
REG_S x22, PT_S6(sp)
REG_S x23, PT_S7(sp)
REG_S x24, PT_S8(sp)
REG_S x25, PT_S9(sp)
REG_S x26, PT_S10(sp)
REG_S x27, PT_S11(sp)
REG_S x28, PT_T3(sp)
REG_S x29, PT_T4(sp)
REG_S x30, PT_T5(sp)
REG_S x31, PT_T6(sp)
.endm
/* restore all GPs except x1 ~ x5 */
.macro restore_from_x6_to_x31
REG_L x6, PT_T1(sp)
REG_L x7, PT_T2(sp)
REG_L x8, PT_S0(sp)
REG_L x9, PT_S1(sp)
REG_L x10, PT_A0(sp)
REG_L x11, PT_A1(sp)
REG_L x12, PT_A2(sp)
REG_L x13, PT_A3(sp)
REG_L x14, PT_A4(sp)
REG_L x15, PT_A5(sp)
REG_L x16, PT_A6(sp)
REG_L x17, PT_A7(sp)
REG_L x18, PT_S2(sp)
REG_L x19, PT_S3(sp)
REG_L x20, PT_S4(sp)
REG_L x21, PT_S5(sp)
REG_L x22, PT_S6(sp)
REG_L x23, PT_S7(sp)
REG_L x24, PT_S8(sp)
REG_L x25, PT_S9(sp)
REG_L x26, PT_S10(sp)
REG_L x27, PT_S11(sp)
REG_L x28, PT_T3(sp)
REG_L x29, PT_T4(sp)
REG_L x30, PT_T5(sp)
REG_L x31, PT_T6(sp)
.endm
#endif /* __ASSEMBLY__ */
#endif /* _ASM_RISCV_ASM_H */

1
arch/riscv/include/asm/csr.h
View File

@ -40,7 +40,6 @@
#define SR_UXL _AC(0x300000000, UL) /* XLEN mask for U-mode */
#define SR_UXL_32 _AC(0x100000000, UL) /* XLEN = 32 for U-mode */
#define SR_UXL_64 _AC(0x200000000, UL) /* XLEN = 64 for U-mode */
#define SR_UXL_SHIFT 32
#endif
/* SATP flags */

11
arch/riscv/include/asm/entry-common.h Normal file
View File

@ -0,0 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_RISCV_ENTRY_COMMON_H
#define _ASM_RISCV_ENTRY_COMMON_H
#include <asm/stacktrace.h>
void handle_page_fault(struct pt_regs *regs);
void handle_break(struct pt_regs *regs);
#endif /* _ASM_RISCV_ENTRY_COMMON_H */

10
arch/riscv/include/asm/ptrace.h
View File

@ -53,6 +53,9 @@ struct pt_regs {
unsigned long orig_a0;
};
#define PTRACE_SYSEMU 0x1f
#define PTRACE_SYSEMU_SINGLESTEP 0x20
#ifdef CONFIG_64BIT
#define REG_FMT "%016lx"
#else
@ -121,8 +124,6 @@ extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs,
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
unsigned long frame_pointer);
int do_syscall_trace_enter(struct pt_regs *regs);
void do_syscall_trace_exit(struct pt_regs *regs);
/**
* regs_get_register() - get register value from its offset
@ -172,6 +173,11 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
return 0;
}
static inline int regs_irqs_disabled(struct pt_regs *regs)
{
return !(regs->status & SR_PIE);
}
#endif /* __ASSEMBLY__ */
#endif /* _ASM_RISCV_PTRACE_H */

5
arch/riscv/include/asm/stacktrace.h
View File

@ -16,4 +16,9 @@ extern void notrace walk_stackframe(struct task_struct *task, struct pt_regs *re
extern void dump_backtrace(struct pt_regs *regs, struct task_struct *task,
const char *loglvl);
static inline bool on_thread_stack(void)
{
return !(((unsigned long)(current->stack) ^ current_stack_pointer) & ~(THREAD_SIZE - 1));
}
#endif /* _ASM_RISCV_STACKTRACE_H */

21
arch/riscv/include/asm/syscall.h
View File

@ -74,5 +74,26 @@ static inline int syscall_get_arch(struct task_struct *task)
#endif
}
typedef long (*syscall_t)(ulong, ulong, ulong, ulong, ulong, ulong, ulong);
static inline void syscall_handler(struct pt_regs *regs, ulong syscall)
{
syscall_t fn;
#ifdef CONFIG_COMPAT
if ((regs->status & SR_UXL) == SR_UXL_32)
fn = compat_sys_call_table[syscall];
else
#endif
fn = sys_call_table[syscall];
regs->a0 = fn(regs->orig_a0, regs->a1, regs->a2,
regs->a3, regs->a4, regs->a5, regs->a6);
}
static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
{
return false;
}
asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t);
#endif /* _ASM_RISCV_SYSCALL_H */

13
arch/riscv/include/asm/thread_info.h
View File

@ -67,6 +67,7 @@ struct thread_info {
long kernel_sp; /* Kernel stack pointer */
long user_sp; /* User stack pointer */
int cpu;
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
};
/*
@ -89,26 +90,18 @@ struct thread_info {
* - pending work-to-be-done flags are in lowest half-word
* - other flags in upper half-word(s)
*/
#define TIF_SYSCALL_TRACE 0 /* syscall trace active */
#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
#define TIF_SIGPENDING 2 /* signal pending */
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */
#define TIF_MEMDIE 5 /* is terminating due to OOM killer */
#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
#define TIF_SYSCALL_AUDIT 7 /* syscall auditing */
#define TIF_SECCOMP 8 /* syscall secure computing */
#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */
#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */
#define TIF_32BIT 11 /* compat-mode 32bit process */
#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
#define _TIF_UPROBE (1 << TIF_UPROBE)
@ -116,8 +109,4 @@ struct thread_info {
(_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \
_TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
#define _TIF_SYSCALL_WORK \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \
_TIF_SECCOMP)
#endif /* _ASM_RISCV_THREAD_INFO_H */

2
arch/riscv/kernel/Makefile
View File

@ -68,8 +68,6 @@ obj-$(CONFIG_CPU_PM) += suspend_entry.o suspend.o
obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o
obj-$(CONFIG_TRACE_IRQFLAGS) += trace_irq.o
obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o
obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o
obj-$(CONFIG_RISCV_SBI) += sbi.o

321
arch/riscv/kernel/entry.S
View File

@ -14,11 +14,7 @@
#include <asm/asm-offsets.h>
#include <asm/errata_list.h>
#if !IS_ENABLED(CONFIG_PREEMPTION)
.set resume_kernel, restore_all
#endif
ENTRY(handle_exception)
SYM_CODE_START(handle_exception)
/*
* If coming from userspace, preserve the user thread pointer and load
* the kernel thread pointer. If we came from the kernel, the scratch
@ -46,32 +42,7 @@ _save_context:
REG_S x1, PT_RA(sp)
REG_S x3, PT_GP(sp)
REG_S x5, PT_T0(sp)
REG_S x6, PT_T1(sp)
REG_S x7, PT_T2(sp)
REG_S x8, PT_S0(sp)
REG_S x9, PT_S1(sp)
REG_S x10, PT_A0(sp)
REG_S x11, PT_A1(sp)
REG_S x12, PT_A2(sp)
REG_S x13, PT_A3(sp)
REG_S x14, PT_A4(sp)
REG_S x15, PT_A5(sp)
REG_S x16, PT_A6(sp)
REG_S x17, PT_A7(sp)
REG_S x18, PT_S2(sp)
REG_S x19, PT_S3(sp)
REG_S x20, PT_S4(sp)
REG_S x21, PT_S5(sp)
REG_S x22, PT_S6(sp)
REG_S x23, PT_S7(sp)
REG_S x24, PT_S8(sp)
REG_S x25, PT_S9(sp)
REG_S x26, PT_S10(sp)
REG_S x27, PT_S11(sp)
REG_S x28, PT_T3(sp)
REG_S x29, PT_T4(sp)
REG_S x30, PT_T5(sp)
REG_S x31, PT_T6(sp)
save_from_x6_to_x31
/*
* Disable user-mode memory access as it should only be set in the
@ -106,19 +77,8 @@ _save_context:
.option norelax
la gp, __global_pointer$
.option pop
#ifdef CONFIG_TRACE_IRQFLAGS
call __trace_hardirqs_off
#endif
#ifdef CONFIG_CONTEXT_TRACKING_USER
/* If previous state is in user mode, call user_exit_callable(). */
li a0, SR_PP
and a0, s1, a0
bnez a0, skip_context_tracking
call user_exit_callable
skip_context_tracking:
#endif
move a0, sp /* pt_regs */
la ra, ret_from_exception
/*
* MSB of cause differentiates between
@ -126,38 +86,13 @@ skip_context_tracking:
*/
bge s4, zero, 1f
la ra, ret_from_exception
/* Handle interrupts */
move a0, sp /* pt_regs */
la a1, generic_handle_arch_irq
jr a1
tail do_irq
1:
/*
* Exceptions run with interrupts enabled or disabled depending on the
* state of SR_PIE in m/sstatus.
*/
andi t0, s1, SR_PIE
beqz t0, 1f
/* kprobes, entered via ebreak, must have interrupts disabled. */
li t0, EXC_BREAKPOINT
beq s4, t0, 1f
#ifdef CONFIG_TRACE_IRQFLAGS
call __trace_hardirqs_on
#endif
csrs CSR_STATUS, SR_IE
1:
la ra, ret_from_exception
/* Handle syscalls */
li t0, EXC_SYSCALL
beq s4, t0, handle_syscall
/* Handle other exceptions */
slli t0, s4, RISCV_LGPTR
la t1, excp_vect_table
la t2, excp_vect_table_end
move a0, sp /* pt_regs */
add t0, t1, t0
/* Check if exception code lies within bounds */
bgeu t0, t2, 1f
@ -165,95 +100,16 @@ skip_context_tracking:
jr t0
1:
tail do_trap_unknown
SYM_CODE_END(handle_exception)
handle_syscall:
#ifdef CONFIG_RISCV_M_MODE
/*
* When running is M-Mode (no MMU config), MPIE does not get set.
* As a result, we need to force enable interrupts here because
* handle_exception did not do set SR_IE as it always sees SR_PIE
* being cleared.
*/
csrs CSR_STATUS, SR_IE
#endif
#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER)
/* Recover a0 - a7 for system calls */
REG_L a0, PT_A0(sp)
REG_L a1, PT_A1(sp)
REG_L a2, PT_A2(sp)
REG_L a3, PT_A3(sp)
REG_L a4, PT_A4(sp)
REG_L a5, PT_A5(sp)
REG_L a6, PT_A6(sp)
REG_L a7, PT_A7(sp)
#endif
/* save the initial A0 value (needed in signal handlers) */
REG_S a0, PT_ORIG_A0(sp)
/*
* Advance SEPC to avoid executing the original
* scall instruction on sret
*/
addi s2, s2, 0x4
REG_S s2, PT_EPC(sp)
/* Trace syscalls, but only if requested by the user. */
REG_L t0, TASK_TI_FLAGS(tp)
andi t0, t0, _TIF_SYSCALL_WORK
bnez t0, handle_syscall_trace_enter
check_syscall_nr:
/* Check to make sure we don't jump to a bogus syscall number. */
li t0, __NR_syscalls
la s0, sys_ni_syscall
/*
* Syscall number held in a7.
* If syscall number is above allowed value, redirect to ni_syscall.
*/
bgeu a7, t0, 3f
#ifdef CONFIG_COMPAT
REG_L s0, PT_STATUS(sp)
srli s0, s0, SR_UXL_SHIFT
andi s0, s0, (SR_UXL >> SR_UXL_SHIFT)
li t0, (SR_UXL_32 >> SR_UXL_SHIFT)
sub t0, s0, t0
bnez t0, 1f
/* Call compat_syscall */
la s0, compat_sys_call_table
j 2f
1:
#endif
/* Call syscall */
la s0, sys_call_table
2:
slli t0, a7, RISCV_LGPTR
add s0, s0, t0
REG_L s0, 0(s0)
3:
jalr s0
ret_from_syscall:
/* Set user a0 to kernel a0 */
REG_S a0, PT_A0(sp)
/*
* We didn't execute the actual syscall.
* Seccomp already set return value for the current task pt_regs.
* (If it was configured with SECCOMP_RET_ERRNO/TRACE)
*/
ret_from_syscall_rejected:
#ifdef CONFIG_DEBUG_RSEQ
move a0, sp
call rseq_syscall
#endif
/* Trace syscalls, but only if requested by the user. */
REG_L t0, TASK_TI_FLAGS(tp)
andi t0, t0, _TIF_SYSCALL_WORK
bnez t0, handle_syscall_trace_exit
/*
* The ret_from_exception must be called with interrupt disabled. Here is the
* caller list:
* - handle_exception
* - ret_from_fork
*/
SYM_CODE_START_NOALIGN(ret_from_exception)
REG_L s0, PT_STATUS(sp)
csrc CSR_STATUS, SR_IE
#ifdef CONFIG_TRACE_IRQFLAGS
call __trace_hardirqs_off
#endif
#ifdef CONFIG_RISCV_M_MODE
/* the MPP value is too large to be used as an immediate arg for addi */
li t0, SR_MPP
@ -261,17 +117,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception)
#else
andi s0, s0, SR_SPP
#endif
bnez s0, resume_kernel
SYM_CODE_END(ret_from_exception)
/* Interrupts must be disabled here so flags are checked atomically */
REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */
andi s1, s0, _TIF_WORK_MASK
bnez s1, resume_userspace_slow
resume_userspace:
#ifdef CONFIG_CONTEXT_TRACKING_USER
call user_enter_callable
#endif
bnez s0, 1f
/* Save unwound kernel stack pointer in thread_info */
addi s0, sp, PT_SIZE_ON_STACK
@ -282,18 +128,7 @@ resume_userspace:
* structures again.
*/
csrw CSR_SCRATCH, tp
restore_all:
#ifdef CONFIG_TRACE_IRQFLAGS
REG_L s1, PT_STATUS(sp)
andi t0, s1, SR_PIE
beqz t0, 1f
call __trace_hardirqs_on
j 2f
1:
call __trace_hardirqs_off
2:
#endif
REG_L a0, PT_STATUS(sp)
/*
* The current load reservation is effectively part of the processor's
@ -322,32 +157,7 @@ restore_all:
REG_L x3, PT_GP(sp)
REG_L x4, PT_TP(sp)
REG_L x5, PT_T0(sp)
REG_L x6, PT_T1(sp)
REG_L x7, PT_T2(sp)
REG_L x8, PT_S0(sp)
REG_L x9, PT_S1(sp)
REG_L x10, PT_A0(sp)
REG_L x11, PT_A1(sp)
REG_L x12, PT_A2(sp)
REG_L x13, PT_A3(sp)
REG_L x14, PT_A4(sp)
REG_L x15, PT_A5(sp)
REG_L x16, PT_A6(sp)
REG_L x17, PT_A7(sp)
REG_L x18, PT_S2(sp)
REG_L x19, PT_S3(sp)
REG_L x20, PT_S4(sp)
REG_L x21, PT_S5(sp)
REG_L x22, PT_S6(sp)
REG_L x23, PT_S7(sp)
REG_L x24, PT_S8(sp)
REG_L x25, PT_S9(sp)
REG_L x26, PT_S10(sp)
REG_L x27, PT_S11(sp)
REG_L x28, PT_T3(sp)
REG_L x29, PT_T4(sp)
REG_L x30, PT_T5(sp)
REG_L x31, PT_T6(sp)
restore_from_x6_to_x31
REG_L x2, PT_SP(sp)
@ -356,47 +166,10 @@ restore_all:
#else
sret
#endif
#if IS_ENABLED(CONFIG_PREEMPTION)
resume_kernel:
REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
bnez s0, restore_all
REG_L s0, TASK_TI_FLAGS(tp)
andi s0, s0, _TIF_NEED_RESCHED
beqz s0, restore_all
call preempt_schedule_irq
j restore_all
#endif
resume_userspace_slow:
/* Enter slow path for supplementary processing */
move a0, sp /* pt_regs */
move a1, s0 /* current_thread_info->flags */
call do_work_pending
j resume_userspace
/* Slow paths for ptrace. */
handle_syscall_trace_enter:
move a0, sp
call do_syscall_trace_enter
move t0, a0
REG_L a0, PT_A0(sp)
REG_L a1, PT_A1(sp)
REG_L a2, PT_A2(sp)
REG_L a3, PT_A3(sp)
REG_L a4, PT_A4(sp)
REG_L a5, PT_A5(sp)
REG_L a6, PT_A6(sp)
REG_L a7, PT_A7(sp)
bnez t0, ret_from_syscall_rejected
j check_syscall_nr
handle_syscall_trace_exit:
move a0, sp
call do_syscall_trace_exit
j ret_from_exception
SYM_CODE_END(ret_from_exception)
#ifdef CONFIG_VMAP_STACK
handle_kernel_stack_overflow:
SYM_CODE_START_LOCAL(handle_kernel_stack_overflow)
/*
* Takes the psuedo-spinlock for the shadow stack, in case multiple
* harts are concurrently overflowing their kernel stacks. We could
@ -464,32 +237,7 @@ restore_caller_reg:
REG_S x1, PT_RA(sp)
REG_S x3, PT_GP(sp)
REG_S x5, PT_T0(sp)
REG_S x6, PT_T1(sp)
REG_S x7, PT_T2(sp)
REG_S x8, PT_S0(sp)
REG_S x9, PT_S1(sp)
REG_S x10, PT_A0(sp)
REG_S x11, PT_A1(sp)
REG_S x12, PT_A2(sp)
REG_S x13, PT_A3(sp)
REG_S x14, PT_A4(sp)
REG_S x15, PT_A5(sp)
REG_S x16, PT_A6(sp)
REG_S x17, PT_A7(sp)
REG_S x18, PT_S2(sp)
REG_S x19, PT_S3(sp)
REG_S x20, PT_S4(sp)
REG_S x21, PT_S5(sp)
REG_S x22, PT_S6(sp)
REG_S x23, PT_S7(sp)
REG_S x24, PT_S8(sp)
REG_S x25, PT_S9(sp)
REG_S x26, PT_S10(sp)
REG_S x27, PT_S11(sp)
REG_S x28, PT_T3(sp)
REG_S x29, PT_T4(sp)
REG_S x30, PT_T5(sp)
REG_S x31, PT_T6(sp)
save_from_x6_to_x31
REG_L s0, TASK_TI_KERNEL_SP(tp)
csrr s1, CSR_STATUS
@ -505,23 +253,20 @@ restore_caller_reg:
REG_S s5, PT_TP(sp)
move a0, sp
tail handle_bad_stack
SYM_CODE_END(handle_kernel_stack_overflow)
#endif
END(handle_exception)
ENTRY(ret_from_fork)
la ra, ret_from_exception
tail schedule_tail
ENDPROC(ret_from_fork)
ENTRY(ret_from_kernel_thread)
SYM_CODE_START(ret_from_fork)
call schedule_tail
beqz s0, 1f /* not from kernel thread */
/* Call fn(arg) */
la ra, ret_from_exception
move a0, s1
jr s0
ENDPROC(ret_from_kernel_thread)
jalr s0
1:
move a0, sp /* pt_regs */
la ra, ret_from_exception
tail syscall_exit_to_user_mode
SYM_CODE_END(ret_from_fork)
/*
* Integer register context switch
@ -533,7 +278,7 @@ ENDPROC(ret_from_kernel_thread)
* The value of a0 and a1 must be preserved by this function, as that's how
* arguments are passed to schedule_tail.
*/
ENTRY(__switch_to)
SYM_FUNC_START(__switch_to)
/* Save context into prev->thread */
li a4, TASK_THREAD_RA
add a3, a0, a4
@ -570,7 +315,7 @@ ENTRY(__switch_to)
/* The offset of thread_info in task_struct is zero. */
move tp, a1
ret
ENDPROC(__switch_to)
SYM_FUNC_END(__switch_to)
#ifndef CONFIG_MMU
#define do_page_fault do_trap_unknown
@ -579,7 +324,7 @@ ENDPROC(__switch_to)
.section ".rodata"
.align LGREG
/* Exception vector table */
ENTRY(excp_vect_table)
SYM_CODE_START(excp_vect_table)
RISCV_PTR do_trap_insn_misaligned
ALT_INSN_FAULT(RISCV_PTR do_trap_insn_fault)
RISCV_PTR do_trap_insn_illegal
@ -588,7 +333,7 @@ ENTRY(excp_vect_table)
RISCV_PTR do_trap_load_fault
RISCV_PTR do_trap_store_misaligned
RISCV_PTR do_trap_store_fault
RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */
RISCV_PTR do_trap_ecall_u /* system call */
RISCV_PTR do_trap_ecall_s
RISCV_PTR do_trap_unknown
RISCV_PTR do_trap_ecall_m
@ -598,11 +343,11 @@ ENTRY(excp_vect_table)
RISCV_PTR do_trap_unknown
RISCV_PTR do_page_fault /* store page fault */
excp_vect_table_end:
END(excp_vect_table)
SYM_CODE_END(excp_vect_table)
#ifndef CONFIG_MMU
ENTRY(__user_rt_sigreturn)
SYM_CODE_START(__user_rt_sigreturn)
li a7, __NR_rt_sigreturn
scall
END(__user_rt_sigreturn)
SYM_CODE_END(__user_rt_sigreturn)
#endif

1
arch/riscv/kernel/head.h
View File

@ -10,7 +10,6 @@
extern atomic_t hart_lottery;
asmlinkage void do_page_fault(struct pt_regs *regs);
asmlinkage void __init setup_vm(uintptr_t dtb_pa);
#ifdef CONFIG_XIP_KERNEL
asmlinkage void __init __copy_data(void);

57
arch/riscv/kernel/mcount-dyn.S
View File

@ -66,66 +66,17 @@
REG_S x3, PT_GP(sp)
REG_S x4, PT_TP(sp)
REG_S x5, PT_T0(sp)
REG_S x6, PT_T1(sp)
REG_S x7, PT_T2(sp)
REG_S x8, PT_S0(sp)
REG_S x9, PT_S1(sp)
REG_S x10, PT_A0(sp)
REG_S x11, PT_A1(sp)
REG_S x12, PT_A2(sp)
REG_S x13, PT_A3(sp)
REG_S x14, PT_A4(sp)
REG_S x15, PT_A5(sp)
REG_S x16, PT_A6(sp)
REG_S x17, PT_A7(sp)
REG_S x18, PT_S2(sp)
REG_S x19, PT_S3(sp)
REG_S x20, PT_S4(sp)
REG_S x21, PT_S5(sp)
REG_S x22, PT_S6(sp)
REG_S x23, PT_S7(sp)
REG_S x24, PT_S8(sp)
REG_S x25, PT_S9(sp)
REG_S x26, PT_S10(sp)
REG_S x27, PT_S11(sp)
REG_S x28, PT_T3(sp)
REG_S x29, PT_T4(sp)
REG_S x30, PT_T5(sp)
REG_S x31, PT_T6(sp)
save_from_x6_to_x31
.endm
.macro RESTORE_ALL
REG_L t0, PT_EPC(sp)
REG_L x1, PT_RA(sp)
REG_L x2, PT_SP(sp)
REG_L x3, PT_GP(sp)
REG_L x4, PT_TP(sp)
REG_L x6, PT_T1(sp)
REG_L x7, PT_T2(sp)
REG_L x8, PT_S0(sp)
REG_L x9, PT_S1(sp)
REG_L x10, PT_A0(sp)
REG_L x11, PT_A1(sp)
REG_L x12, PT_A2(sp)
REG_L x13, PT_A3(sp)
REG_L x14, PT_A4(sp)
REG_L x15, PT_A5(sp)
REG_L x16, PT_A6(sp)
REG_L x17, PT_A7(sp)
REG_L x18, PT_S2(sp)
REG_L x19, PT_S3(sp)
REG_L x20, PT_S4(sp)
REG_L x21, PT_S5(sp)
REG_L x22, PT_S6(sp)
REG_L x23, PT_S7(sp)
REG_L x24, PT_S8(sp)
REG_L x25, PT_S9(sp)
REG_L x26, PT_S10(sp)
REG_L x27, PT_S11(sp)
REG_L x28, PT_T3(sp)
REG_L x29, PT_T4(sp)
REG_L x30, PT_T5(sp)
REG_L x31, PT_T6(sp)
/* Restore t0 with PT_EPC */
REG_L x5, PT_EPC(sp)
restore_from_x6_to_x31
addi sp, sp, PT_SIZE_ON_STACK
.endm

5
arch/riscv/kernel/process.c
View File

@ -34,7 +34,6 @@ EXPORT_SYMBOL(__stack_chk_guard);
#endif
extern asmlinkage void ret_from_fork(void);
extern asmlinkage void ret_from_kernel_thread(void);
void arch_cpu_idle(void)
{
@ -173,7 +172,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
/* Supervisor/Machine, irqs on: */
childregs->status = SR_PP | SR_PIE;
p->thread.ra = (unsigned long)ret_from_kernel_thread;
p->thread.s[0] = (unsigned long)args->fn;
p->thread.s[1] = (unsigned long)args->fn_arg;
} else {
@ -183,8 +181,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
if (clone_flags & CLONE_SETTLS)
childregs->tp = tls;
childregs->a0 = 0; /* Return value of fork() */
p->thread.ra = (unsigned long)ret_from_fork;
p->thread.s[0] = 0;
}
p->thread.ra = (unsigned long)ret_from_fork;
p->thread.sp = (unsigned long)childregs; /* kernel sp */
return 0;
}

44
arch/riscv/kernel/ptrace.c
View File

@ -19,9 +19,6 @@
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#define CREATE_TRACE_POINTS
#include <trace/events/syscalls.h>
enum riscv_regset {
REGSET_X,
#ifdef CONFIG_FPU
@ -212,7 +209,6 @@ unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
void ptrace_disable(struct task_struct *child)
{
clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE);
}
long arch_ptrace(struct task_struct *child, long request,
@ -229,46 +225,6 @@ long arch_ptrace(struct task_struct *child, long request,
return ret;
}
/*
* Allows PTRACE_SYSCALL to work. These are called from entry.S in
* {handle,ret_from}_syscall.
*/
__visible int do_syscall_trace_enter(struct pt_regs *regs)
{
if (test_thread_flag(TIF_SYSCALL_TRACE))
if (ptrace_report_syscall_entry(regs))
return -1;
/*
* Do the secure computing after ptrace; failures should be fast.
* If this fails we might have return value in a0 from seccomp
* (via SECCOMP_RET_ERRNO/TRACE).
*/
if (secure_computing() == -1)
return -1;
#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
trace_sys_enter(regs, syscall_get_nr(current, regs));
#endif
audit_syscall_entry(regs->a7, regs->a0, regs->a1, regs->a2, regs->a3);
return 0;
}
__visible void do_syscall_trace_exit(struct pt_regs *regs)
{
audit_syscall_exit(regs);
if (test_thread_flag(TIF_SYSCALL_TRACE))
ptrace_report_syscall_exit(regs, 0);
#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
trace_sys_exit(regs, regs_return_value(regs));
#endif
}
#ifdef CONFIG_COMPAT
static int compat_riscv_gpr_get(struct task_struct *target,
const struct user_regset *regset,

29
arch/riscv/kernel/signal.c
View File

@ -12,6 +12,7 @@
#include <linux/syscalls.h>
#include <linux/resume_user_mode.h>
#include <linux/linkage.h>
#include <linux/entry-common.h>
#include <asm/ucontext.h>
#include <asm/vdso.h>
@ -274,7 +275,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
signal_setup_done(ret, ksig, 0);
}
static void do_signal(struct pt_regs *regs)
void arch_do_signal_or_restart(struct pt_regs *regs)
{
struct ksignal ksig;
@ -311,29 +312,3 @@ static void do_signal(struct pt_regs *regs)
*/
restore_saved_sigmask();
}
/*
* Handle any pending work on the resume-to-userspace path, as indicated by
* _TIF_WORK_MASK. Entered from assembly with IRQs off.
*/
asmlinkage __visible void do_work_pending(struct pt_regs *regs,
unsigned long thread_info_flags)
{
do {
if (thread_info_flags & _TIF_NEED_RESCHED) {
schedule();
} else {
local_irq_enable();
if (thread_info_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);
/* Handle pending signal delivery */
if (thread_info_flags & (_TIF_SIGPENDING |
_TIF_NOTIFY_SIGNAL))
do_signal(regs);
if (thread_info_flags & _TIF_NOTIFY_RESUME)
resume_user_mode_work(regs);
}
local_irq_disable();
thread_info_flags = read_thread_flags();
} while (thread_info_flags & _TIF_WORK_MASK);
}

27
arch/riscv/kernel/trace_irq.c
View File

@ -1,27 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2022 Changbin Du <changbin.du@gmail.com>
*/
#include <linux/irqflags.h>
#include <linux/kprobes.h>
#include "trace_irq.h"
/*
* trace_hardirqs_on/off require the caller to setup frame pointer properly.
* Otherwise, CALLER_ADDR1 might trigger an pagging exception in kernel.
* Here we add one extra level so they can be safely called by low
* level entry code which $fp is used for other purpose.
*/
void __trace_hardirqs_on(void)
{
trace_hardirqs_on();
}
NOKPROBE_SYMBOL(__trace_hardirqs_on);
void __trace_hardirqs_off(void)
{
trace_hardirqs_off();
}
NOKPROBE_SYMBOL(__trace_hardirqs_off);

11
arch/riscv/kernel/trace_irq.h
View File

@ -1,11 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2022 Changbin Du <changbin.du@gmail.com>
*/
#ifndef __TRACE_IRQ_H
#define __TRACE_IRQ_H
void __trace_hardirqs_on(void);
void __trace_hardirqs_off(void);
#endif /* __TRACE_IRQ_H */

144
arch/riscv/kernel/traps.c
View File

@ -17,12 +17,14 @@
#include <linux/module.h>
#include <linux/irq.h>
#include <linux/kexec.h>
#include <linux/entry-common.h>
#include <asm/asm-prototypes.h>
#include <asm/bug.h>
#include <asm/csr.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
int show_unhandled_signals = 1;
@ -119,14 +121,22 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code,
}
#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
#define __trap_section __section(".xip.traps")
#define __trap_section __noinstr_section(".xip.traps")
#else
#define __trap_section
#define __trap_section noinstr
#endif
#define DO_ERROR_INFO(name, signo, code, str) \
asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
{ \
do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
#define DO_ERROR_INFO(name, signo, code, str) \
asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
{ \
if (user_mode(regs)) { \
irqentry_enter_from_user_mode(regs); \
do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
irqentry_exit_to_user_mode(regs); \
} else { \
irqentry_state_t state = irqentry_nmi_enter(regs); \
do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
irqentry_nmi_exit(regs, state); \
} \
}
DO_ERROR_INFO(do_trap_unknown,
@ -148,26 +158,50 @@ DO_ERROR_INFO(do_trap_store_misaligned,
int handle_misaligned_load(struct pt_regs *regs);
int handle_misaligned_store(struct pt_regs *regs);
asmlinkage void __trap_section do_trap_load_misaligned(struct pt_regs *regs)
asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
{
if (!handle_misaligned_load(regs))
return;
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - load address misaligned");
if (user_mode(regs)) {
irqentry_enter_from_user_mode(regs);
if (handle_misaligned_load(regs))
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - load address misaligned");
irqentry_exit_to_user_mode(regs);
} else {
irqentry_state_t state = irqentry_nmi_enter(regs);
if (handle_misaligned_load(regs))
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - load address misaligned");
irqentry_nmi_exit(regs, state);
}
}
asmlinkage void __trap_section do_trap_store_misaligned(struct pt_regs *regs)
asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
{
if (!handle_misaligned_store(regs))
return;
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - store (or AMO) address misaligned");
if (user_mode(regs)) {
irqentry_enter_from_user_mode(regs);
if (handle_misaligned_store(regs))
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - store (or AMO) address misaligned");
irqentry_exit_to_user_mode(regs);
} else {
irqentry_state_t state = irqentry_nmi_enter(regs);
if (handle_misaligned_store(regs))
do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
"Oops - store (or AMO) address misaligned");
irqentry_nmi_exit(regs, state);
}
}
#endif
DO_ERROR_INFO(do_trap_store_fault,
SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
DO_ERROR_INFO(do_trap_ecall_u,
SIGILL, ILL_ILLTRP, "environment call from U-mode");
DO_ERROR_INFO(do_trap_ecall_s,
SIGILL, ILL_ILLTRP, "environment call from S-mode");
DO_ERROR_INFO(do_trap_ecall_m,
@ -183,7 +217,7 @@ static inline unsigned long get_break_insn_length(unsigned long pc)
return GET_INSN_LENGTH(insn);
}
asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
void handle_break(struct pt_regs *regs)
{
#ifdef CONFIG_KPROBES
if (kprobe_single_step_handler(regs))
@ -213,7 +247,77 @@ asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
else
die(regs, "Kernel BUG");
}
NOKPROBE_SYMBOL(do_trap_break);
asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
{
if (user_mode(regs)) {
irqentry_enter_from_user_mode(regs);
handle_break(regs);
irqentry_exit_to_user_mode(regs);
} else {
irqentry_state_t state = irqentry_nmi_enter(regs);
handle_break(regs);
irqentry_nmi_exit(regs, state);
}
}
asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
{
if (user_mode(regs)) {
ulong syscall = regs->a7;
syscall = syscall_enter_from_user_mode(regs, syscall);
regs->epc += 4;
regs->orig_a0 = regs->a0;
if (syscall < NR_syscalls)
syscall_handler(regs, syscall);
else
regs->a0 = -ENOSYS;
syscall_exit_to_user_mode(regs);
} else {
irqentry_state_t state = irqentry_nmi_enter(regs);
do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
"Oops - environment call from U-mode");
irqentry_nmi_exit(regs, state);
}
}
#ifdef CONFIG_MMU
asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
{
irqentry_state_t state = irqentry_enter(regs);
handle_page_fault(regs);
local_irq_disable();
irqentry_exit(regs, state);
}
#endif
asmlinkage __visible noinstr void do_irq(struct pt_regs *regs)
{
struct pt_regs *old_regs;
irqentry_state_t state = irqentry_enter(regs);
irq_enter_rcu();
old_regs = set_irq_regs(regs);
handle_arch_irq(regs);
set_irq_regs(old_regs);
irq_exit_rcu();
irqentry_exit(regs, state);
}
#ifdef CONFIG_GENERIC_BUG
int is_valid_bugaddr(unsigned long pc)

6
arch/riscv/mm/fault.c
View File

@ -15,6 +15,7 @@
#include <linux/uaccess.h>
#include <linux/kprobes.h>
#include <linux/kfence.h>
#include <linux/entry-common.h>
#include <asm/ptrace.h>
#include <asm/tlbflush.h>
@ -204,7 +205,7 @@ static inline bool access_error(unsigned long cause, struct vm_area_struct *vma)
* This routine handles page faults. It determines the address and the
* problem, and then passes it off to one of the appropriate routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs)
void handle_page_fault(struct pt_regs *regs)
{
struct task_struct *tsk;
struct vm_area_struct *vma;
@ -251,7 +252,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
}
#endif
/* Enable interrupts if they were enabled in the parent context. */
if (likely(regs->status & SR_PIE))
if (!regs_irqs_disabled(regs))
local_irq_enable();
/*
@ -356,4 +357,3 @@ good_area:
}
return;
}
NOKPROBE_SYMBOL(do_page_fault);