maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[InstrProfiling] Use weak alias for bias variable 

We need the compiler generated variable to override the weak symbol of
the same name inside the profile runtime, but using LinkOnceODRLinkage
results in weak symbol being emitted in which case the symbol selected
by the linker is going to depend on the order of inputs which can be
fragile.

This change replaces the use of weak definition inside the runtime with
a weak alias. We place the compiler generated symbol inside a COMDAT
group so dead definition can be garbage collected by the linker.

We also disable the use of runtime counter relocation on Darwin since
Mach-O doesn't support weak external references, but Darwin already uses
a different continous mode that relies on overmapping so runtime counter
relocation isn't needed there.

Differential Revision: https://reviews.llvm.org/D105176
This commit is contained in:
Petr Hosek 2021-07-08 13:44:05 -07:00
parent b988d69ea2
commit 54902e00d1
14 changed files with 166 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
compiler-rt/include/profile/InstrProfData.inc
View File

@ -664,6 +664,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
/* The variable that holds the name of the profile data
* specified via command line. */

1
compiler-rt/lib/profile/CMakeLists.txt
View File

@ -53,7 +53,6 @@ set(PROFILE_SOURCES
InstrProfiling.c
InstrProfilingInternal.c
InstrProfilingValue.c
InstrProfilingBiasVar.c
InstrProfilingBuffer.c
InstrProfilingFile.c
InstrProfilingMerge.c

7
compiler-rt/lib/profile/InstrProfiling.h
View File

@ -319,11 +319,4 @@ extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */
*/
extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */
/*!
* This variable is a weak symbol defined in InstrProfilingBiasVar.c. It
* allows compiler instrumentation to provide overriding definition with
* value from compiler command line. This variable has hidden visibility.
*/
COMPILER_RT_VISIBILITY extern intptr_t __llvm_profile_counter_bias;
#endif /* PROFILE_INSTRPROFILING_H_ */

15
compiler-rt/lib/profile/InstrProfilingBiasVar.c
View File

@ -1,15 +0,0 @@
/*===- InstrProfilingBiasVar.c - profile counter bias variable setup ------===*\
|*
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|* See https://llvm.org/LICENSE.txt for license information.
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|*
\*===----------------------------------------------------------------------===*/
#include "InstrProfiling.h"
/* The runtime should only provide its own definition of this symbol when the
* user has not specified one. Set this up by moving the runtime's copy of this
* symbol to an object file within the archive.
*/
COMPILER_RT_WEAK intptr_t __llvm_profile_counter_bias = -1;

11
compiler-rt/lib/profile/InstrProfilingBuffer.c
View File

@ -67,13 +67,20 @@ static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
return 0;
}
static int needsCounterPadding(void) {
#if defined(__APPLE__)
return __llvm_profile_is_continuous_mode_enabled();
#else
return 0;
#endif
}
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
uint64_t *PaddingBytesAfterNames) {
if (!__llvm_profile_is_continuous_mode_enabled() ||
lprofRuntimeCounterRelocation()) {
if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
*PaddingBytesAfterCounters = 0;
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);

237
compiler-rt/lib/profile/InstrProfilingFile.c
View File

@ -426,33 +426,6 @@ static void truncateCurrentFile(void) {
fclose(File);
}
#if !defined(__Fuchsia__) && !defined(_WIN32)
static void assertIsZero(int *i) {
if (*i)
PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
}
/* Write a partial profile to \p Filename, which is required to be backed by
* the open file object \p File. */
static int writeProfileWithFileObject(const char *Filename, FILE *File) {
setProfileFile(File);
int rc = writeFile(Filename);
if (rc)
PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
setProfileFile(NULL);
return rc;
}
/* Unlock the profile \p File and clear the unlock flag. */
static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
if (!*ProfileRequiresUnlock) {
PROF_WARN("%s", "Expected to require profile unlock\n");
}
lprofUnlockFileHandle(File);
*ProfileRequiresUnlock = 0;
}
#endif // !defined(__Fuchsia__) && !defined(_WIN32)
static int writeMMappedFile(FILE *OutputFile, char **Profile) {
if (!OutputFile)
return -1;
@ -481,83 +454,38 @@ static int writeMMappedFile(FILE *OutputFile, char **Profile) {
return 0;
}
static void relocateCounters(void) {
if (!__llvm_profile_is_continuous_mode_enabled() ||
!lprofRuntimeCounterRelocation())
return;
// TODO: Move these functions into InstrProfilingPlatform* files.
#if defined(__APPLE__)
static void assertIsZero(int *i) {
if (*i)
PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
}
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
const uint64_t *CountersBegin = __llvm_profile_begin_counters();
const uint64_t *CountersEnd = __llvm_profile_end_counters();
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
(DataSize * sizeof(__llvm_profile_data));
/* Write a partial profile to \p Filename, which is required to be backed by
* the open file object \p File. */
static int writeProfileWithFileObject(const char *Filename, FILE *File) {
setProfileFile(File);
int rc = writeFile(Filename);
if (rc)
PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
setProfileFile(NULL);
return rc;
}
int Length = getCurFilenameLength();
char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
const char *Filename = getCurFilename(FilenameBuf, 0);
if (!Filename)
return;
FILE *File = NULL;
char *Profile = NULL;
if (!doMerging()) {
File = fopen(Filename, "w+b");
if (!File)
return;
if (writeMMappedFile(File, &Profile) == -1) {
fclose(File);
return;
}
} else {
File = lprofOpenFileEx(Filename);
if (!File)
return;
uint64_t ProfileFileSize = 0;
if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
lprofUnlockFileHandle(File);
fclose(File);
return;
}
if (!ProfileFileSize) {
if (writeMMappedFile(File, &Profile) == -1) {
fclose(File);
return;
}
} else {
/* The merged profile has a non-zero length. Check that it is compatible
* with the data in this process. */
if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
fclose(File);
return;
}
}
lprofUnlockFileHandle(File);
/* Unlock the profile \p File and clear the unlock flag. */
static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
if (!*ProfileRequiresUnlock) {
PROF_WARN("%s", "Expected to require profile unlock\n");
}
/* Update the profile fields based on the current mapping. */
__llvm_profile_counter_bias =
(intptr_t)Profile - (uintptr_t)CountersBegin + CountersOffset;
/* Return the memory allocated for counters to OS. */
lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
lprofUnlockFileHandle(File);
*ProfileRequiresUnlock = 0;
}
static void initializeProfileForContinuousMode(void) {
if (!__llvm_profile_is_continuous_mode_enabled())
return;
#if defined(__Fuchsia__) || defined(_WIN32)
PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows.");
#else // defined(__Fuchsia__) || defined(_WIN32)
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
@ -683,8 +611,109 @@ static void initializeProfileForContinuousMode(void) {
if (ProfileRequiresUnlock)
unlockProfile(&ProfileRequiresUnlock, File);
#endif // defined(__Fuchsia__) || defined(_WIN32)
}
#elif defined(__ELF__) || defined(_WIN32)
#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
/* This variable is a weak external reference which could be used to detect
* whether or not the compiler defined this symbol. */
#if defined(_WIN32)
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
#pragma comment(linker, "/alternatename:" \
INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" \
INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
#else
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
__attribute__((weak, alias(INSTR_PROF_QUOTE(
INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
#endif
static void initializeProfileForContinuousMode(void) {
if (!__llvm_profile_is_continuous_mode_enabled())
return;
/* This symbol is defined by the compiler when runtime counter relocation is
* used and runtime provides a weak alias so we can check if it's defined. */
void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
if (BiasAddr == BiasDefaultAddr) {
PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined");
return;
}
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
const uint64_t *CountersBegin = __llvm_profile_begin_counters();
const uint64_t *CountersEnd = __llvm_profile_end_counters();
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
const uint64_t CountersOffset =
sizeof(__llvm_profile_header) + (DataSize * sizeof(__llvm_profile_data));
int Length = getCurFilenameLength();
char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
const char *Filename = getCurFilename(FilenameBuf, 0);
if (!Filename)
return;
FILE *File = NULL;
char *Profile = NULL;
if (!doMerging()) {
File = fopen(Filename, "w+b");
if (!File)
return;
if (writeMMappedFile(File, &Profile) == -1) {
fclose(File);
return;
}
} else {
File = lprofOpenFileEx(Filename);
if (!File)
return;
uint64_t ProfileFileSize = 0;
if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
lprofUnlockFileHandle(File);
fclose(File);
return;
}
if (!ProfileFileSize) {
if (writeMMappedFile(File, &Profile) == -1) {
fclose(File);
return;
}
} else {
/* The merged profile has a non-zero length. Check that it is compatible
* with the data in this process. */
if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
fclose(File);
return;
}
}
lprofUnlockFileHandle(File);
}
/* Update the profile fields based on the current mapping. */
INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
(intptr_t)Profile - (uintptr_t)CountersBegin +
CountersOffset;
/* Return the memory allocated for counters to OS. */
lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
}
#else
static void initializeProfileForContinuousMode(void) {
PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
}
#endif
static const char *DefaultProfileName = "default.profraw";
static void resetFilenameToDefault(void) {
@ -784,9 +813,14 @@ static int parseFilenamePattern(const char *FilenamePat,
FilenamePat);
return -1;
}
#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32)
__llvm_profile_set_page_size(getpagesize());
__llvm_profile_enable_continuous_mode();
#else
PROF_WARN("%s", "Continous mode is currently only supported for Mach-O,"
" ELF and COFF formats.");
return -1;
#endif
} else {
unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
if (!MergePoolSize)
@ -843,12 +877,8 @@ static void parseAndSetFilename(const char *FilenamePat,
}
truncateCurrentFile();
if (__llvm_profile_is_continuous_mode_enabled()) {
if (lprofRuntimeCounterRelocation())
relocateCounters();
else
initializeProfileForContinuousMode();
}
if (__llvm_profile_is_continuous_mode_enabled())
initializeProfileForContinuousMode();
}
/* Return buffer length that is required to store the current profile
@ -1004,9 +1034,6 @@ void __llvm_profile_initialize_file(void) {
ProfileNameSpecifier PNS = PNS_unknown;
int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
if (__llvm_profile_counter_bias != -1)
lprofSetRuntimeCounterRelocation(1);
EnvFilenamePat = getFilenamePatFromEnv();
if (EnvFilenamePat) {
/* Pass CopyFilenamePat = 1, to ensure that the filename would be valid

10
compiler-rt/lib/profile/InstrProfilingInternal.c
View File

@ -23,14 +23,4 @@ COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {
ProfileDumped = Value;
}
static unsigned RuntimeCounterRelocation = 0;
COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) {
return RuntimeCounterRelocation;
}
COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) {
RuntimeCounterRelocation = Value;
}
#endif

4
compiler-rt/lib/profile/InstrProfilingInternal.h
View File

@ -184,10 +184,6 @@ uint64_t lprofGetLoadModuleSignature();
unsigned lprofProfileDumped(void);
void lprofSetProfileDumped(unsigned);
/* Return non zero value if counters are being relocated at runtime. */
unsigned lprofRuntimeCounterRelocation(void);
void lprofSetRuntimeCounterRelocation(unsigned);
COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *);
COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer;
COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize;

18
compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c
View File

@ -34,16 +34,14 @@
#include "InstrProfilingInternal.h"
#include "InstrProfilingUtil.h"
/* This variable is an external reference to symbol defined by the compiler. */
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() {
return 1;
}
COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {}
COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) {
return 1;
}
COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) {}
static const char ProfileSinkName[] = "llvm-profile";
static inline void lprofWrite(const char *fmt, ...) {
@ -116,14 +114,6 @@ void __llvm_profile_initialize(void) {
return;
}
/* This symbol is defined as weak and initialized to -1 by the runtimer, but
* compiler will generate a strong definition initialized to 0 when runtime
* counter relocation is used. */
if (__llvm_profile_counter_bias == -1) {
lprofWrite("LLVM Profile: counter relocation at runtime is required\n");
return;
}
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
const uint64_t *CountersBegin = __llvm_profile_begin_counters();
@ -192,7 +182,7 @@ void __llvm_profile_initialize(void) {
lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName);
/* Update the profile fields based on the current mapping. */
__llvm_profile_counter_bias =
INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
(intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset;
/* Return the memory allocated for counters to OS. */

4
compiler-rt/lib/profile/InstrProfilingUtil.h
View File

@ -74,11 +74,11 @@ int lprofSuspendSigKill();
/* Restore previously suspended SIGKILL. */
void lprofRestoreSigKill();
inline size_t lprofRoundUpTo(size_t x, size_t boundary) {
static inline size_t lprofRoundUpTo(size_t x, size_t boundary) {
return (x + boundary - 1) & ~(boundary - 1);
}
inline size_t lprofRoundDownTo(size_t x, size_t boundary) {
static inline size_t lprofRoundDownTo(size_t x, size_t boundary) {
return x & ~(boundary - 1);
}

2
llvm/include/llvm/ProfileData/InstrProf.h
View File

@ -156,7 +156,7 @@ inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
}
inline StringRef getInstrProfCounterBiasVarName() {
return "__llvm_profile_counter_bias";
return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
}
/// Return the marker used to separate PGO names during serialization.

1
llvm/include/llvm/ProfileData/InstrProfData.inc
View File

@ -664,6 +664,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
/* The variable that holds the name of the profile data
* specified via command line. */

14
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
View File

@ -467,9 +467,14 @@ bool InstrProfiling::lowerIntrinsics(Function *F) {
}
bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
// Mach-O don't support weak external references.
if (TT.isOSBinFormatMachO())
return false;
if (RuntimeCounterRelocation.getNumOccurrences() > 0)
return RuntimeCounterRelocation;
// Fuchsia uses runtime counter relocation by default.
return TT.isOSFuchsia();
}
@ -690,10 +695,19 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
Type *Int64Ty = Type::getInt64Ty(M->getContext());
GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
if (!Bias) {
// Compiler must define this variable when runtime counter relocation
// is being used. Runtime has a weak external reference that is used
// to check whether that's the case or not.
Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
Constant::getNullValue(Int64Ty),
getInstrProfCounterBiasVarName());
Bias->setVisibility(GlobalVariable::HiddenVisibility);
// A definition that's weak (linkonce_odr) without being in a COMDAT
// section wouldn't lead to link errors, but it would lead to a dead
// data word from every TU but one. Putting it in COMDAT ensures there
// will be exactly one data slot in the link.
if (TT.supportsCOMDAT())
Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
}
LI = Builder.CreateLoad(Int64Ty, Bias);
}

3
llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
View File

@ -4,7 +4,8 @@
target triple = "x86_64-unknown-linux-gnu"
@__profn_foo = private constant [3 x i8] c"foo"
; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0
; RELOC: $__llvm_profile_counter_bias = comdat any
; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat
; CHECK-LABEL: define void @foo
; CHECK-NEXT: %pgocount = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i64 0, i64 0)