UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

asm-generic/bitops: Always inline all bit manipulation helpers 

Make it consistent with the atomic/atomic-instrumented.h helpers.

And defconfig size is actually going down!

     text    data     bss     dec     hex filename
  22352096        8213152 1917164 32482412        1efa46c vmlinux.x86-64.defconfig.before
  22350551        8213184 1917164 32480899        1ef9e83 vmlinux.x86-64.defconfig.after

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Marco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20220113155357.4706-2-bp@alien8.de
This commit is contained in:
Borislav Petkov 2022-01-13 16:53:56 +01:00 committed by Peter Zijlstra
parent 61cc4534b6
commit acb13ea0ba
2 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/asm-generic/bitops/instrumented-atomic.h
View File

@ -23,7 +23,7 @@
* Note that @nr may be almost arbitrarily large; this function is not * Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity. * restricted to acting on a single-word quantity.
*/ */
static inline void set_bit(long nr, volatile unsigned long *addr) static __always_inline void set_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_set_bit(nr, addr); arch_set_bit(nr, addr);
@ -36,7 +36,7 @@ static inline void set_bit(long nr, volatile unsigned long *addr)
* *
* This is a relaxed atomic operation (no implied memory barriers). * This is a relaxed atomic operation (no implied memory barriers).
*/ */
static inline void clear_bit(long nr, volatile unsigned long *addr) static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_clear_bit(nr, addr); arch_clear_bit(nr, addr);
@ -52,7 +52,7 @@ static inline void clear_bit(long nr, volatile unsigned long *addr)
* Note that @nr may be almost arbitrarily large; this function is not * Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity. * restricted to acting on a single-word quantity.
*/ */
static inline void change_bit(long nr, volatile unsigned long *addr) static __always_inline void change_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_change_bit(nr, addr); arch_change_bit(nr, addr);
@ -65,7 +65,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@ -79,7 +79,7 @@ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@ -93,7 +93,7 @@ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));

16
include/asm-generic/bitops/instrumented-non-atomic.h
View File

@ -22,7 +22,7 @@
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __set_bit(long nr, volatile unsigned long *addr) static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___set_bit(nr, addr); arch___set_bit(nr, addr);
@ -37,7 +37,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr)
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __clear_bit(long nr, volatile unsigned long *addr) static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___clear_bit(nr, addr); arch___clear_bit(nr, addr);
@ -52,13 +52,13 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __change_bit(long nr, volatile unsigned long *addr) static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___change_bit(nr, addr); arch___change_bit(nr, addr);
} }
static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr) static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
{ {
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) { if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
/* /*
@ -90,7 +90,7 @@ static inline void __instrument_read_write_bitop(long nr, volatile unsigned long
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_set_bit(nr, addr); return arch___test_and_set_bit(nr, addr);
@ -104,7 +104,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_clear_bit(nr, addr); return arch___test_and_clear_bit(nr, addr);
@ -118,7 +118,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_change_bit(nr, addr); return arch___test_and_change_bit(nr, addr);
@ -129,7 +129,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
* @nr: bit number to test * @nr: bit number to test
* @addr: Address to start counting from * @addr: Address to start counting from
*/ */
static inline bool test_bit(long nr, const volatile unsigned long *addr) static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
{ {
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
return arch_test_bit(nr, addr); return arch_test_bit(nr, addr);