2014-03-20 06:10:27 +08:00
|
|
|
/*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\
|
|
|
|
|*
|
|
|
|
|* The LLVM Compiler Infrastructure
|
|
|
|
|*
|
|
|
|
|* This file is distributed under the University of Illinois Open Source
|
|
|
|
|* License. See LICENSE.TXT for details.
|
|
|
|
|*
|
|
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
|
|
|
|
#include "InstrProfiling.h"
|
2015-11-19 02:12:35 +08:00
|
|
|
#include <limits.h>
|
2015-12-04 09:02:24 +08:00
|
|
|
#include <stdio.h>
|
2015-11-19 02:12:35 +08:00
|
|
|
#include <stdlib.h>
|
2014-03-21 03:44:31 +08:00
|
|
|
#include <string.h>
|
2015-12-04 09:02:24 +08:00
|
|
|
#define INSTR_PROF_VALUE_PROF_DATA
|
|
|
|
#define INSTR_PROF_COMMON_API_IMPL
|
|
|
|
#include "InstrProfData.inc"
|
|
|
|
|
|
|
|
#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
|
|
|
|
#define PROF_OOM_RETURN(Msg) \
|
|
|
|
{ \
|
|
|
|
PROF_OOM(Msg) \
|
|
|
|
return 0; \
|
|
|
|
}
|
2014-03-20 06:10:27 +08:00
|
|
|
|
2015-12-04 12:22:59 +08:00
|
|
|
#ifdef _MIPS_ARCH
|
|
|
|
LLVM_LIBRARY_VISIBILITY
|
|
|
|
uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
|
|
|
|
void *R = *Ptr;
|
|
|
|
if (R == OldV) {
|
|
|
|
*Ptr = NewV;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#define BOOL_CMPXCHG(Ptr, OldV, NewV) BoolCmpXchg((void **)Ptr, OldV, NewV)
|
|
|
|
#else
|
|
|
|
#define BOOL_CMPXCHG(Ptr, OldV, NewV) \
|
|
|
|
__sync_bool_compare_and_swap(Ptr, OldV, NewV)
|
|
|
|
#endif
|
|
|
|
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
|
2015-11-24 01:26:53 +08:00
|
|
|
return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
|
|
|
|
: (INSTR_PROF_RAW_MAGIC_32);
|
2014-03-22 02:25:56 +08:00
|
|
|
}
|
2014-03-21 08:27:50 +08:00
|
|
|
|
2015-11-19 05:54:40 +08:00
|
|
|
/* Return the number of bytes needed to add to SizeInBytes to make it
|
2015-11-19 06:09:41 +08:00
|
|
|
* the result a multiple of 8.
|
|
|
|
*/
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY uint8_t
|
2015-11-19 05:54:40 +08:00
|
|
|
__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
|
2015-11-19 02:12:35 +08:00
|
|
|
return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
|
|
|
|
}
|
|
|
|
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_version(void) {
|
2015-11-24 01:26:53 +08:00
|
|
|
return INSTR_PROF_RAW_VERSION;
|
2014-03-20 06:10:27 +08:00
|
|
|
}
|
|
|
|
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY void __llvm_profile_reset_counters(void) {
|
2014-09-04 23:45:31 +08:00
|
|
|
uint64_t *I = __llvm_profile_begin_counters();
|
|
|
|
uint64_t *E = __llvm_profile_end_counters();
|
2014-03-21 03:44:31 +08:00
|
|
|
|
2015-11-19 05:54:40 +08:00
|
|
|
memset(I, 0, sizeof(uint64_t) * (E - I));
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
|
|
|
|
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
|
2015-11-19 04:20:15 +08:00
|
|
|
const __llvm_profile_data *DI;
|
|
|
|
for (DI = DataBegin; DI != DataEnd; ++DI) {
|
|
|
|
uint64_t CurrentVSiteCount = 0;
|
|
|
|
uint32_t VKI, i;
|
2015-11-21 03:41:02 +08:00
|
|
|
if (!DI->Values)
|
2015-11-19 02:12:35 +08:00
|
|
|
continue;
|
|
|
|
|
2015-11-21 03:41:02 +08:00
|
|
|
ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values;
|
|
|
|
|
|
|
|
for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
|
2015-11-19 02:12:35 +08:00
|
|
|
CurrentVSiteCount += DI->NumValueSites[VKI];
|
|
|
|
|
2015-11-19 04:20:15 +08:00
|
|
|
for (i = 0; i < CurrentVSiteCount; ++i) {
|
2015-11-21 03:41:02 +08:00
|
|
|
ValueProfNode *CurrentVNode = ValueCounters[i];
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
while (CurrentVNode) {
|
2015-11-24 01:09:40 +08:00
|
|
|
CurrentVNode->VData.Count = 0;
|
2015-11-19 02:12:35 +08:00
|
|
|
CurrentVNode = CurrentVNode->Next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
/* This method is only used in value profiler mock testing. */
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY void
|
2015-12-04 09:02:24 +08:00
|
|
|
__llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
|
|
|
|
uint32_t ValueKind, uint16_t NumValueSites) {
|
|
|
|
*((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
|
|
|
|
}
|
2015-11-21 02:31:45 +08:00
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
/* This method is only used in value profiler mock testing. */
|
|
|
|
LLVM_LIBRARY_VISIBILITY const __llvm_profile_data *
|
|
|
|
__llvm_profile_iterate_data(const __llvm_profile_data *Data) {
|
|
|
|
return Data + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This method is only used in value profiler mock testing. */
|
|
|
|
LLVM_LIBRARY_VISIBILITY void *
|
|
|
|
__llvm_get_function_addr(const __llvm_profile_data *Data) {
|
|
|
|
return Data->FunctionPointer;
|
|
|
|
}
|
2015-11-21 02:31:45 +08:00
|
|
|
|
2015-11-19 05:54:40 +08:00
|
|
|
/* Allocate an array that holds the pointers to the linked lists of
|
2015-11-19 06:09:41 +08:00
|
|
|
* value profile counter nodes. The number of element of the array
|
|
|
|
* is the total number of value profile sites instrumented. Returns
|
2015-12-04 09:02:24 +08:00
|
|
|
* 0 if allocation fails.
|
2015-11-19 06:09:41 +08:00
|
|
|
*/
|
2015-11-19 02:12:35 +08:00
|
|
|
|
2015-11-19 05:54:40 +08:00
|
|
|
static int allocateValueProfileCounters(__llvm_profile_data *Data) {
|
|
|
|
uint64_t NumVSites = 0;
|
|
|
|
uint32_t VKI;
|
2015-11-21 03:41:02 +08:00
|
|
|
for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
|
2015-11-19 05:54:40 +08:00
|
|
|
NumVSites += Data->NumValueSites[VKI];
|
|
|
|
|
2015-11-21 03:41:02 +08:00
|
|
|
ValueProfNode **Mem =
|
|
|
|
(ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
|
2015-11-19 05:54:40 +08:00
|
|
|
if (!Mem)
|
|
|
|
return 0;
|
2015-12-04 12:22:59 +08:00
|
|
|
if (!BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
|
2015-11-19 05:54:40 +08:00
|
|
|
free(Mem);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
|
|
|
|
uint64_t NumVSites = 0, I;
|
|
|
|
uint32_t VKI;
|
|
|
|
if (!Data->Values)
|
|
|
|
return;
|
|
|
|
for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
|
|
|
|
NumVSites += Data->NumValueSites[VKI];
|
|
|
|
for (I = 0; I < NumVSites; I++) {
|
|
|
|
ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
|
|
|
|
while (Node) {
|
|
|
|
ValueProfNode *Next = Node->Next;
|
|
|
|
free(Node);
|
|
|
|
Node = Next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
free(Data->Values);
|
|
|
|
}
|
|
|
|
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY void
|
2015-11-21 03:41:02 +08:00
|
|
|
__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
|
2015-11-19 05:54:40 +08:00
|
|
|
uint32_t CounterIndex) {
|
|
|
|
|
2015-11-24 01:09:40 +08:00
|
|
|
__llvm_profile_data *PData = (__llvm_profile_data *)Data;
|
|
|
|
if (!PData)
|
2015-11-19 02:12:35 +08:00
|
|
|
return;
|
|
|
|
|
2015-11-24 01:09:40 +08:00
|
|
|
if (!PData->Values) {
|
|
|
|
if (!allocateValueProfileCounters(PData))
|
2015-11-19 02:12:35 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-11-24 01:09:40 +08:00
|
|
|
ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
|
2015-11-21 03:41:02 +08:00
|
|
|
ValueProfNode *PrevVNode = NULL;
|
|
|
|
ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
uint8_t VDataCount = 0;
|
|
|
|
while (CurrentVNode) {
|
2015-11-24 01:09:40 +08:00
|
|
|
if (TargetValue == CurrentVNode->VData.Value) {
|
|
|
|
CurrentVNode->VData.Count++;
|
2015-11-19 02:12:35 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
PrevVNode = CurrentVNode;
|
|
|
|
CurrentVNode = CurrentVNode->Next;
|
|
|
|
++VDataCount;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (VDataCount >= UCHAR_MAX)
|
|
|
|
return;
|
|
|
|
|
2015-11-21 03:41:02 +08:00
|
|
|
CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
|
2015-11-19 02:12:35 +08:00
|
|
|
if (!CurrentVNode)
|
|
|
|
return;
|
|
|
|
|
2015-11-24 01:09:40 +08:00
|
|
|
CurrentVNode->VData.Value = TargetValue;
|
|
|
|
CurrentVNode->VData.Count++;
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
uint32_t Success = 0;
|
2015-11-21 03:41:02 +08:00
|
|
|
if (!ValueCounters[CounterIndex])
|
2015-12-04 12:22:59 +08:00
|
|
|
Success = BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
|
2015-11-19 02:12:35 +08:00
|
|
|
else if (PrevVNode && !PrevVNode->Next)
|
2015-12-04 12:22:59 +08:00
|
|
|
Success = BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
if (!Success) {
|
|
|
|
free(CurrentVNode);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2015-12-04 09:02:24 +08:00
|
|
|
|
|
|
|
/* For multi-threaded programs, while the profile is being dumped, other
|
|
|
|
threads may still be updating the value profile data and creating new
|
|
|
|
value entries. To accommadate this, we need to add extra bytes to the
|
|
|
|
data buffer. The size of the extra space is controlled by an environment
|
|
|
|
varaible. */
|
|
|
|
static unsigned getVprofExtraBytes() {
|
|
|
|
const char *ExtraStr = getenv("LLVM_VALUE_PROF_BUFFER_EXTRA");
|
|
|
|
if (!ExtraStr || !ExtraStr[0])
|
|
|
|
return 1024;
|
|
|
|
return (unsigned)atoi(ExtraStr);
|
|
|
|
}
|
2015-11-19 02:12:35 +08:00
|
|
|
|
2015-11-23 12:38:17 +08:00
|
|
|
LLVM_LIBRARY_VISIBILITY uint64_t
|
2015-11-19 05:54:40 +08:00
|
|
|
__llvm_profile_gather_value_data(uint8_t **VDataArray) {
|
2015-12-04 09:02:24 +08:00
|
|
|
size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
|
|
|
|
__llvm_profile_data *I;
|
|
|
|
if (!VDataArray)
|
|
|
|
PROF_OOM_RETURN("Failed to write value profile data ");
|
2015-11-19 02:12:35 +08:00
|
|
|
|
|
|
|
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
|
|
|
|
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
|
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
/*
|
|
|
|
* Compute the total Size of the buffer to hold ValueProfData
|
|
|
|
* structures for functions with value profile data.
|
|
|
|
*/
|
|
|
|
for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
|
|
|
|
ValueProfRuntimeRecord R;
|
|
|
|
/* Extract the value profile data info from the runtime. */
|
|
|
|
if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
|
|
|
|
PROF_OOM_RETURN("Failed to write value profile data ");
|
|
|
|
/* Compute the size of ValueProfData from this runtime record. */
|
|
|
|
if (getNumValueKindsRT(&R) != 0)
|
|
|
|
S += getValueProfDataSizeRT(&R);
|
|
|
|
finalizeValueProfRuntimeRecord(&R);
|
|
|
|
}
|
|
|
|
/* No value sites or no value profile data is collected. */
|
|
|
|
if (!S)
|
|
|
|
return 0;
|
2015-11-19 04:20:15 +08:00
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
Extra = getVprofExtraBytes();
|
|
|
|
BufferCapacity = S + Extra;
|
|
|
|
*VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
|
|
|
|
if (!*VDataArray)
|
|
|
|
PROF_OOM_RETURN("Failed to write value profile data ");
|
|
|
|
|
|
|
|
ValueProfData *VD = (ValueProfData *)(*VDataArray);
|
|
|
|
/*
|
|
|
|
* Extract value profile data and write into ValueProfData structure
|
|
|
|
* one by one. Note that new value profile data added to any value
|
|
|
|
* site (from another thread) after the ValueProfRuntimeRecord is
|
|
|
|
* initialized (when the profile data snapshot is taken) won't be
|
|
|
|
* collected. This is not a problem as those dropped value will have
|
|
|
|
* very low taken count.
|
|
|
|
*/
|
|
|
|
for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
|
|
|
|
ValueProfRuntimeRecord R;
|
|
|
|
if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
|
|
|
|
PROF_OOM_RETURN("Failed to write value profile data ");
|
|
|
|
if (getNumValueKindsRT(&R) == 0)
|
2015-11-19 02:12:35 +08:00
|
|
|
continue;
|
|
|
|
|
2015-12-04 09:02:24 +08:00
|
|
|
/* Record R has taken a snapshot of the VP data at this point. Newly
|
|
|
|
added VP data for this function will be dropped. */
|
|
|
|
/* Check if there is enough space. */
|
|
|
|
if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
|
|
|
|
PROF_ERR("Value profile data is dropped :%s \n",
|
|
|
|
"Out of buffer space. Use environment "
|
|
|
|
" LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
|
|
|
|
I->Values = 0;
|
2015-11-19 02:12:35 +08:00
|
|
|
}
|
2015-12-04 09:02:24 +08:00
|
|
|
|
|
|
|
serializeValueProfDataFromRT(&R, VD);
|
|
|
|
deallocateValueProfileCounters(I);
|
|
|
|
I->Values = VD;
|
|
|
|
finalizeValueProfRuntimeRecord(&R);
|
|
|
|
RealSize += VD->TotalSize;
|
|
|
|
VD = (ValueProfData *)((char *)VD + VD->TotalSize);
|
2015-11-19 02:12:35 +08:00
|
|
|
}
|
2015-12-04 09:02:24 +08:00
|
|
|
|
|
|
|
return RealSize;
|
2014-03-21 03:44:31 +08:00
|
|
|
}
|