From c5ae732a443e2600823b930457eaab6e25f69b32 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 4 Mar 2015 17:36:32 +0100 Subject: [PATCH 1/7] ppc: Remove unused cpp symbols in kvm headers These don't seem to be used anywhere. Acked-by: Rik van Riel Cc: Benjamin Herrenschmidt Cc: Alexander Graf Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- arch/powerpc/include/asm/kvm_book3s.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 942c7b1678e3..993090422690 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -106,10 +106,6 @@ struct kvmppc_vcpu_book3s { spinlock_t mmu_lock; }; -#define CONTEXT_HOST 0 -#define CONTEXT_GUEST 1 -#define CONTEXT_GUEST_END 2 - #define VSID_REAL 0x07ffffffffc00000ULL #define VSID_BAT 0x07ffffffffb00000ULL #define VSID_64K 0x0800000000000000ULL From c467ea763fd5d8795b7d1b5a78eb94b6ad8f66ad Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 4 Mar 2015 18:06:33 +0100 Subject: [PATCH 2/7] context_tracking: Rename context symbols to prepare for transition state Current context tracking symbols are designed to express living state. As such they are prefixed with "IN_": IN_USER, IN_KERNEL. Now we are going to use these symbols to also express state transitions such as context_tracking_enter(IN_USER) or context_tracking_exit(IN_USER). But while the "IN_" prefix works well to express entering a context, it's confusing to depict a context exit: context_tracking_exit(IN_USER) could mean two things: 1) We are exiting the current context to enter user context. 2) We are exiting the user context We want 2) but the reviewer may be confused and understand 1) So lets disambiguate these symbols and rename them to CONTEXT_USER and CONTEXT_KERNEL. Acked-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- arch/x86/kernel/traps.c | 2 +- include/linux/context_tracking.h | 2 +- include/linux/context_tracking_state.h | 6 +++--- kernel/context_tracking.c | 8 ++++---- kernel/sched/core.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 9d2073e2ecc9..756f74eed35d 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -123,7 +123,7 @@ enum ctx_state ist_enter(struct pt_regs *regs) * but we need to notify RCU. */ rcu_nmi_enter(); - prev_state = IN_KERNEL; /* the value is irrelevant. */ + prev_state = CONTEXT_KERNEL; /* the value is irrelevant. */ } /* diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 37b81bd51ec0..427b056dfd3d 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -43,7 +43,7 @@ static inline enum ctx_state exception_enter(void) static inline void exception_exit(enum ctx_state prev_ctx) { if (context_tracking_is_enabled()) { - if (prev_ctx == IN_USER) + if (prev_ctx == CONTEXT_USER) context_tracking_user_enter(); } } diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 97a81225d037..8076f875c324 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -13,8 +13,8 @@ struct context_tracking { */ bool active; enum ctx_state { - IN_KERNEL = 0, - IN_USER, + CONTEXT_KERNEL = 0, + CONTEXT_USER, } state; }; @@ -34,7 +34,7 @@ static inline bool context_tracking_cpu_is_enabled(void) static inline bool context_tracking_in_user(void) { - return __this_cpu_read(context_tracking.state) == IN_USER; + return __this_cpu_read(context_tracking.state) == CONTEXT_USER; } #else static inline bool context_tracking_in_user(void) { return false; } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 937ecdfdf258..8ad53c9d38b6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -75,7 +75,7 @@ void context_tracking_user_enter(void) WARN_ON_ONCE(!current->mm); local_irq_save(flags); - if ( __this_cpu_read(context_tracking.state) != IN_USER) { + if ( __this_cpu_read(context_tracking.state) != CONTEXT_USER) { if (__this_cpu_read(context_tracking.active)) { trace_user_enter(0); /* @@ -101,7 +101,7 @@ void context_tracking_user_enter(void) * OTOH we can spare the calls to vtime and RCU when context_tracking.active * is false because we know that CPU is not tickless. */ - __this_cpu_write(context_tracking.state, IN_USER); + __this_cpu_write(context_tracking.state, CONTEXT_USER); } local_irq_restore(flags); } @@ -129,7 +129,7 @@ void context_tracking_user_exit(void) return; local_irq_save(flags); - if (__this_cpu_read(context_tracking.state) == IN_USER) { + if (__this_cpu_read(context_tracking.state) == CONTEXT_USER) { if (__this_cpu_read(context_tracking.active)) { /* * We are going to run code that may use RCU. Inform @@ -139,7 +139,7 @@ void context_tracking_user_exit(void) vtime_user_exit(current); trace_user_exit(0); } - __this_cpu_write(context_tracking.state, IN_KERNEL); + __this_cpu_write(context_tracking.state, CONTEXT_KERNEL); } local_irq_restore(flags); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f0f831e8a345..06b9a00871e0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2818,7 +2818,7 @@ asmlinkage __visible void __sched schedule_user(void) * we find a better solution. * * NB: There are buggy callers of this function. Ideally we - * should warn if prev_state != IN_USER, but that will trigger + * should warn if prev_state != CONTEXT_USER, but that will trigger * too frequently to make sense yet. */ enum ctx_state prev_state = exception_enter(); From 3aab4f50bff89bdea5066a05d4f3c5fa25bc37c7 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 10 Feb 2015 15:27:50 -0500 Subject: [PATCH 3/7] context_tracking: Generalize context tracking APIs to support user and guest Generalize the context tracking APIs to support various nature of contexts. This is performed by splitting out the mechanism from context_tracking_user_enter and context_tracking_user_exit into context_tracking_enter and context_tracking_exit. The nature of the context we track is now detailed in a ctx_state parameter pushed to these APIs, allowing the same functions to not just track kernel <> user space switching, but also kernel <> guest transitions. But leave the old functions in order to avoid breaking ARM, which calls these functions from assembler code, and cannot easily use C enum parameters. Reviewed-by: Paul E. McKenney Signed-off-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- include/linux/context_tracking.h | 9 ++++--- kernel/context_tracking.c | 43 +++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 427b056dfd3d..7f1810a3b5a4 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -10,6 +10,8 @@ #ifdef CONFIG_CONTEXT_TRACKING extern void context_tracking_cpu_set(int cpu); +extern void context_tracking_enter(enum ctx_state state); +extern void context_tracking_exit(enum ctx_state state); extern void context_tracking_user_enter(void); extern void context_tracking_user_exit(void); extern void __context_tracking_task_switch(struct task_struct *prev, @@ -35,7 +37,8 @@ static inline enum ctx_state exception_enter(void) return 0; prev_ctx = this_cpu_read(context_tracking.state); - context_tracking_user_exit(); + if (prev_ctx != CONTEXT_KERNEL) + context_tracking_exit(prev_ctx); return prev_ctx; } @@ -43,8 +46,8 @@ static inline enum ctx_state exception_enter(void) static inline void exception_exit(enum ctx_state prev_ctx) { if (context_tracking_is_enabled()) { - if (prev_ctx == CONTEXT_USER) - context_tracking_user_enter(); + if (prev_ctx != CONTEXT_KERNEL) + context_tracking_enter(prev_ctx); } } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 8ad53c9d38b6..17715d811b71 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -39,15 +39,15 @@ void context_tracking_cpu_set(int cpu) } /** - * context_tracking_user_enter - Inform the context tracking that the CPU is going to - * enter userspace mode. + * context_tracking_enter - Inform the context tracking that the CPU is going + * enter user or guest space mode. * * This function must be called right before we switch from the kernel - * to userspace, when it's guaranteed the remaining kernel instructions - * to execute won't use any RCU read side critical section because this - * function sets RCU in extended quiescent state. + * to user or guest space, when it's guaranteed the remaining kernel + * instructions to execute won't use any RCU read side critical section + * because this function sets RCU in extended quiescent state. */ -void context_tracking_user_enter(void) +void context_tracking_enter(enum ctx_state state) { unsigned long flags; @@ -75,7 +75,7 @@ void context_tracking_user_enter(void) WARN_ON_ONCE(!current->mm); local_irq_save(flags); - if ( __this_cpu_read(context_tracking.state) != CONTEXT_USER) { + if ( __this_cpu_read(context_tracking.state) != state) { if (__this_cpu_read(context_tracking.active)) { trace_user_enter(0); /* @@ -101,24 +101,31 @@ void context_tracking_user_enter(void) * OTOH we can spare the calls to vtime and RCU when context_tracking.active * is false because we know that CPU is not tickless. */ - __this_cpu_write(context_tracking.state, CONTEXT_USER); + __this_cpu_write(context_tracking.state, state); } local_irq_restore(flags); } +NOKPROBE_SYMBOL(context_tracking_enter); + +void context_tracking_user_enter(void) +{ + context_tracking_enter(CONTEXT_USER); +} NOKPROBE_SYMBOL(context_tracking_user_enter); /** - * context_tracking_user_exit - Inform the context tracking that the CPU is - * exiting userspace mode and entering the kernel. + * context_tracking_exit - Inform the context tracking that the CPU is + * exiting user or guest mode and entering the kernel. * - * This function must be called after we entered the kernel from userspace - * before any use of RCU read side critical section. This potentially include - * any high level kernel code like syscalls, exceptions, signal handling, etc... + * This function must be called after we entered the kernel from user or + * guest space before any use of RCU read side critical section. This + * potentially include any high level kernel code like syscalls, exceptions, + * signal handling, etc... * * This call supports re-entrancy. This way it can be called from any exception * handler without needing to know if we came from userspace or not. */ -void context_tracking_user_exit(void) +void context_tracking_exit(enum ctx_state state) { unsigned long flags; @@ -129,7 +136,7 @@ void context_tracking_user_exit(void) return; local_irq_save(flags); - if (__this_cpu_read(context_tracking.state) == CONTEXT_USER) { + if (__this_cpu_read(context_tracking.state) == state) { if (__this_cpu_read(context_tracking.active)) { /* * We are going to run code that may use RCU. Inform @@ -143,6 +150,12 @@ void context_tracking_user_exit(void) } local_irq_restore(flags); } +NOKPROBE_SYMBOL(context_tracking_exit); + +void context_tracking_user_exit(void) +{ + context_tracking_exit(CONTEXT_USER); +} NOKPROBE_SYMBOL(context_tracking_user_exit); /** From a3a9c7dff25791f3a1041aa96e770af662d99a35 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 10 Feb 2015 15:27:51 -0500 Subject: [PATCH 4/7] context_tracking: Add stub context_tracking_is_enabled With code elsewhere doing something conditional on whether or not context tracking is enabled, we want a stub function that tells us context tracking is not enabled, when CONFIG_CONTEXT_TRACKING is not set. Reviewed-by: Paul E. McKenney Signed-off-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- include/linux/context_tracking_state.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 8076f875c324..ad4458588b47 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -39,6 +39,8 @@ static inline bool context_tracking_in_user(void) #else static inline bool context_tracking_in_user(void) { return false; } static inline bool context_tracking_active(void) { return false; } +static inline bool context_tracking_is_enabled(void) { return false; } +static inline bool context_tracking_cpu_is_enabled(void) { return false; } #endif /* CONFIG_CONTEXT_TRACKING */ #endif From 19fdd98b6253404c6bdd6927bde9f962729376f7 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 10 Feb 2015 15:27:52 -0500 Subject: [PATCH 5/7] context_tracking: Run vtime_user_enter/exit only when state == CONTEXT_USER Only run vtime_user_enter, vtime_user_exit, and the user enter & exit trace points when we are entering or exiting user state, respectively. The KVM code in guest_enter and guest_exit already take care of calling vtime_guest_enter and vtime_guest_exit, respectively. The RCU code only distinguishes between "idle" and "not idle or kernel". There should be no need to add an additional (unused) state there. Reviewed-by: Paul E. McKenney Signed-off-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- kernel/context_tracking.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 17715d811b71..a2c0866384e8 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -77,7 +77,6 @@ void context_tracking_enter(enum ctx_state state) local_irq_save(flags); if ( __this_cpu_read(context_tracking.state) != state) { if (__this_cpu_read(context_tracking.active)) { - trace_user_enter(0); /* * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be @@ -85,7 +84,10 @@ void context_tracking_enter(enum ctx_state state) * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency * on the tick. */ - vtime_user_enter(current); + if (state == CONTEXT_USER) { + trace_user_enter(0); + vtime_user_enter(current); + } rcu_user_enter(); } /* @@ -143,8 +145,10 @@ void context_tracking_exit(enum ctx_state state) * RCU core about that (ie: we may need the tick again). */ rcu_user_exit(); - vtime_user_exit(current); - trace_user_exit(0); + if (state == CONTEXT_USER) { + vtime_user_exit(current); + trace_user_exit(0); + } } __this_cpu_write(context_tracking.state, CONTEXT_KERNEL); } From efc1e2c9bcbab73797d7bc214014cb916d6a8eb5 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 10 Feb 2015 15:27:53 -0500 Subject: [PATCH 6/7] context_tracking: Export context_tracking_user_enter/exit Export context_tracking_user_enter/exit so it can be used by KVM. Reviewed-by: Paul E. McKenney Signed-off-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- kernel/context_tracking.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index a2c0866384e8..72d59a1a6eb6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -108,6 +108,7 @@ void context_tracking_enter(enum ctx_state state) local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_enter); +EXPORT_SYMBOL_GPL(context_tracking_enter); void context_tracking_user_enter(void) { @@ -155,6 +156,7 @@ void context_tracking_exit(enum ctx_state state) local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_exit); +EXPORT_SYMBOL_GPL(context_tracking_exit); void context_tracking_user_exit(void) { From 126a6a542446f1a49b9f3c69237c87df3eb4e6e1 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 10 Feb 2015 15:27:54 -0500 Subject: [PATCH 7/7] kvm,rcu,nohz: use RCU extended quiescent state when running KVM guest The host kernel is not doing anything while the CPU is executing a KVM guest VCPU, so it can be marked as being in an extended quiescent state, identical to that used when running user space code. The only exception to that rule is when the host handles an interrupt, which is already handled by the irq code, which calls rcu_irq_enter and rcu_irq_exit. The guest_enter and guest_exit functions already switch vtime accounting independent of context tracking. Leave those calls where they are, instead of moving them into the context tracking code. Reviewed-by: Paul E. McKenney Signed-off-by: Rik van Riel Cc: Paul E. McKenney Cc: Andy Lutomirski Cc: Will deacon Cc: Marcelo Tosatti Cc: Christian Borntraeger Cc: Luiz Capitulino Cc: Paolo Bonzini Signed-off-by: Frederic Weisbecker --- include/linux/context_tracking.h | 6 ++++++ include/linux/context_tracking_state.h | 1 + include/linux/kvm_host.h | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 7f1810a3b5a4..2821838256b4 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -81,10 +81,16 @@ static inline void guest_enter(void) vtime_guest_enter(current); else current->flags |= PF_VCPU; + + if (context_tracking_is_enabled()) + context_tracking_enter(CONTEXT_GUEST); } static inline void guest_exit(void) { + if (context_tracking_is_enabled()) + context_tracking_exit(CONTEXT_GUEST); + if (vtime_accounting_enabled()) vtime_guest_exit(current); else diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index ad4458588b47..6b7b96a32b75 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -15,6 +15,7 @@ struct context_tracking { enum ctx_state { CONTEXT_KERNEL = 0, CONTEXT_USER, + CONTEXT_GUEST, } state; }; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index d12b2104d19b..cc8c61c5459c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -766,7 +766,8 @@ static inline void kvm_guest_enter(void) * one time slice). Lets treat guest mode as quiescent state, just like * we do with user-mode execution. */ - rcu_virt_note_context_switch(smp_processor_id()); + if (!context_tracking_cpu_is_enabled()) + rcu_virt_note_context_switch(smp_processor_id()); } static inline void kvm_guest_exit(void)