tick/nohz: Evaluate the CPU expression after the static key

When tick_nohz_full_cpu() is called with smp_processor_id(), the latter
is unconditionally evaluated whether the static key is on or off. It is
not necessary in the off-case though, so make sure the cpu expression
is executed at the last moment.

Illustrate with the following test function:

	int tick_nohz_test(void)
	{
		return tick_nohz_full_cpu(smp_processor_id());
	}

The resulting code before was:

	mov    %gs:0x7eea92d1(%rip),%eax   # smp_processor_id() fetch
	nopl   0x0(%rax,%rax,1)
	xor    %eax,%eax
	retq
	cmpb   $0x0,0x29d393a(%rip)        # <tick_nohz_full_running>
	je     tick_nohz_test+0x29         # jump to below eax clear
	mov    %eax,%eax
	bt     %rax,0x29d3936(%rip)        # <tick_nohz_full_mask>
	setb   %al
	movzbl %al,%eax
	retq
	xor    %eax,%eax
	retq

Now it becomes:

	nopl   0x0(%rax,%rax,1)
	xor    %eax,%eax
	retq
	cmpb   $0x0,0x29d3871(%rip)        # <tick_nohz_full_running>
	je     tick_nohz_test+0x29         # jump to below eax clear
	mov    %gs:0x7eea91f0(%rip),%eax   # smp_processor_id() fetch, after static key
	mov    %eax,%eax
	bt     %rax,0x29d3866(%rip)        # <tick_nohz_full_mask>
	setb   %al
	movzbl %al,%eax
	retq
	xor    %eax,%eax
	retq

Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210512232924.150322-2-frederic@kernel.org
This commit is contained in:
Peter Zijlstra 2021-05-13 01:29:15 +02:00 committed by Ingo Molnar
parent 6efb943b86
commit f105dfec0a
1 changed files with 11 additions and 7 deletions

View File

@ -185,13 +185,17 @@ static inline bool tick_nohz_full_enabled(void)
return tick_nohz_full_running; return tick_nohz_full_running;
} }
static inline bool tick_nohz_full_cpu(int cpu) /*
{ * Check if a CPU is part of the nohz_full subset. Arrange for evaluating
if (!tick_nohz_full_enabled()) * the cpu expression (typically smp_processor_id()) _after_ the static
return false; * key.
*/
return cpumask_test_cpu(cpu, tick_nohz_full_mask); #define tick_nohz_full_cpu(_cpu) ({ \
} bool __ret = false; \
if (tick_nohz_full_enabled()) \
__ret = cpumask_test_cpu((_cpu), tick_nohz_full_mask); \
__ret; \
})
static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask)
{ {