maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

tsan: avoid extra call indirection in unaligned access functions 

Currently unaligned access functions are defined in tsan_interface.cpp
and do a real call to MemoryAccess. This means we have a real call
and no read/write constant propagation.

Unaligned memory access can be quite hot for some programs
(observed on some compression algorithms with ~90% of unaligned accesses).

Move them to tsan_interface_inl.h to avoid the additional call
and enable constant propagation.
Also reorder the actual store and memory access handling for
__sanitizer_unaligned_store callbacks to enable tail calling
in MemoryAccess.

Depends on D107282.

Reviewed By: vitalybuka, melver

Differential Revision: https://reviews.llvm.org/D107283
This commit is contained in:
Dmitry Vyukov 2021-08-02 16:52:53 +02:00
parent 69396896fb
commit d77b476c19
2 changed files with 71 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
compiler-rt/lib/tsan/rtl/tsan_interface.cpp
View File

@ -59,77 +59,21 @@ void __tsan_write16_pc(void *addr, void *pc) {
// __tsan_unaligned_read/write calls are emitted by compiler.
void __tsan_unaligned_read2(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
}
void __tsan_unaligned_read4(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
}
void __tsan_unaligned_read8(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
}
void __tsan_unaligned_read16(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, kAccessRead);
}
void __tsan_unaligned_write2(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
}
void __tsan_unaligned_write4(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
}
void __tsan_unaligned_write8(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
uptr pc = CALLERPC;
ThreadState *thr = cur_thread();
UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
}
void __tsan_unaligned_write16(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, kAccessWrite);
uptr pc = CALLERPC;
ThreadState *thr = cur_thread();
UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
}
// __sanitizer_unaligned_load/store are for user instrumentation.
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
u16 __sanitizer_unaligned_load16(const uu16 *addr) {
__tsan_unaligned_read2(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
u32 __sanitizer_unaligned_load32(const uu32 *addr) {
__tsan_unaligned_read4(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
u64 __sanitizer_unaligned_load64(const uu64 *addr) {
__tsan_unaligned_read8(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
__tsan_unaligned_write2(addr);
*addr = v;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
__tsan_unaligned_write4(addr);
*addr = v;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
__tsan_unaligned_write8(addr);
*addr = v;
}
SANITIZER_INTERFACE_ATTRIBUTE
void *__tsan_get_current_fiber() {
return cur_thread();

63
compiler-rt/lib/tsan/rtl/tsan_interface_inl.h
View File

@ -82,6 +82,69 @@ void __tsan_write8_pc(void *addr, void *pc) {
MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite);
}
ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
}
ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
}
ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
}
ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
}
ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
}
ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) {
UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
}
extern "C" {
// __sanitizer_unaligned_load/store are for user instrumentation.
SANITIZER_INTERFACE_ATTRIBUTE
u16 __sanitizer_unaligned_load16(const uu16 *addr) {
__tsan_unaligned_read2(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
u32 __sanitizer_unaligned_load32(const uu32 *addr) {
__tsan_unaligned_read4(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
u64 __sanitizer_unaligned_load64(const uu64 *addr) {
__tsan_unaligned_read8(addr);
return *addr;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
*addr = v;
__tsan_unaligned_write2(addr);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
*addr = v;
__tsan_unaligned_write4(addr);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
*addr = v;
__tsan_unaligned_write8(addr);
}
}
void __tsan_vptr_update(void **vptr_p, void *new_val) {
if (*vptr_p == new_val)
return;