diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index d94e41516c0b..cc2e25262a02 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -159,19 +159,28 @@ struct InstrProfStringTable { } }; +struct InstrProfValueData { + // Profiled value. + uint64_t Value; + // Number of times the value appears in the training run. + uint64_t Count; +}; + struct InstrProfValueSiteRecord { - /// Typedef for a single TargetValue-NumTaken pair. - typedef std::pair ValueDataPair; /// Value profiling data pairs at a given value site. - std::list ValueData; + std::list ValueData; InstrProfValueSiteRecord() { ValueData.clear(); } + template + InstrProfValueSiteRecord(InputIterator F, InputIterator L) + : ValueData(F, L) {} - /// Sort ValueData ascending by TargetValue + /// Sort ValueData ascending by Value void sortByTargetValues() { - ValueData.sort([](const ValueDataPair &left, const ValueDataPair &right) { - return left.first < right.first; - }); + ValueData.sort( + [](const InstrProfValueData &left, const InstrProfValueData &right) { + return left.Value < right.Value; + }); } /// Merge data from another InstrProfValueSiteRecord @@ -182,10 +191,9 @@ struct InstrProfValueSiteRecord { auto IE = ValueData.end(); for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE; ++J) { - while (I != IE && I->first < J->first) - ++I; - if (I != IE && I->first == J->first) { - I->second += J->second; + while (I != IE && I->Value < J->Value) ++I; + if (I != IE && I->Value == J->Value) { + I->Count += J->Count; ++I; continue; } @@ -202,15 +210,47 @@ struct InstrProfRecord { StringRef Name; uint64_t Hash; std::vector Counts; - std::vector IndirectCallSites; + typedef std::vector> ValueMapType; + + /// Return the number of value profile kinds with non-zero number + /// of profile sites. + inline uint32_t getNumValueKinds() const; + /// Return the number of instrumented sites for ValueKind. + inline uint32_t getNumValueSites(uint32_t ValueKind) const; + /// Return the total number of ValueData for ValueKind. + inline uint32_t getNumValueData(uint32_t ValueKind) const; + /// Return the number of value data collected for ValueKind at profiling + /// site: Site. + inline uint32_t getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const; + inline std::unique_ptr getValueForSite( + uint32_t ValueKind, uint32_t Site) const; + /// Reserve space for NumValueSites sites. + inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); + /// Add ValueData for ValueKind at value Site. + inline void addValueData(uint32_t ValueKind, uint32_t Site, + InstrProfValueData *VData, uint32_t N, + ValueMapType *HashKeys); + /// Merge Value Profile ddata from Src record to this record for ValueKind. + inline instrprof_error mergeValueProfData(uint32_t ValueKind, + InstrProfRecord &Src); + + /// Used by InstrProfWriter: update the value strings to commoned strings in + /// the writer instance. + inline void updateStrings(InstrProfStringTable *StrTab); + + private: + std::vector IndirectCallSites; const std::vector & getValueSitesForKind(uint32_t ValueKind) const { switch (ValueKind) { case IPVK_IndirectCallTarget: return IndirectCallSites; + default: + llvm_unreachable("Unknown value kind!"); } - llvm_unreachable("Unknown value kind!"); + return IndirectCallSites; } std::vector & @@ -219,8 +259,102 @@ struct InstrProfRecord { const_cast(this) ->getValueSitesForKind(ValueKind)); } + // Map indirect call target name hash to name string. + uint64_t remapValue(uint64_t Value, uint32_t ValueKind, + ValueMapType *HashKeys) { + if (!HashKeys) return Value; + switch (ValueKind) { + case IPVK_IndirectCallTarget: { + auto Result = + std::lower_bound(HashKeys->begin(), HashKeys->end(), Value, + [](const std::pair &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + assert(Result != HashKeys->end() && + "Hash does not match any known keys\n"); + Value = (uint64_t)Result->second; + break; + } + } + return Value; + } }; +uint32_t InstrProfRecord::getNumValueKinds() const { + uint32_t NumValueKinds = 0; + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) + NumValueKinds += !(getValueSitesForKind(Kind).empty()); + return NumValueKinds; +} + +uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { + return getValueSitesForKind(ValueKind).size(); +} + +uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const { + return getValueSitesForKind(ValueKind)[Site].ValueData.size(); +} + +std::unique_ptr InstrProfRecord::getValueForSite( + uint32_t ValueKind, uint32_t Site) const { + uint32_t N = getNumValueDataForSite(ValueKind, Site); + if (N == 0) return std::unique_ptr(nullptr); + + std::unique_ptr VD(new InstrProfValueData[N]); + uint32_t I = 0; + for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { + VD[I] = V; + I++; + } + assert(I == N); + + return std::move(VD); +} + +void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, + InstrProfValueData *VData, uint32_t N, + ValueMapType *HashKeys) { + for (uint32_t I = 0; I < N; I++) { + VData[I].Value = remapValue(VData[I].Value, ValueKind, HashKeys); + } + std::vector &ValueSites = + getValueSitesForKind(ValueKind); + if (N == 0) + ValueSites.push_back(InstrProfValueSiteRecord()); + else + ValueSites.emplace_back(VData, VData + N); +} + +void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { + std::vector &ValueSites = + getValueSitesForKind(ValueKind); + ValueSites.reserve(NumValueSites); +} + +instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind, + InstrProfRecord &Src) { + uint32_t ThisNumValueSites = getNumValueSites(ValueKind); + uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); + if (ThisNumValueSites != OtherNumValueSites) + return instrprof_error::value_site_count_mismatch; + std::vector &ThisSiteRecords = + getValueSitesForKind(ValueKind); + std::vector &OtherSiteRecords = + Src.getValueSitesForKind(ValueKind); + for (uint32_t I = 0; I < ThisNumValueSites; I++) + ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I]); + return instrprof_error::success; +} + +void InstrProfRecord::updateStrings(InstrProfStringTable *StrTab) { + if (!StrTab) return; + + Name = StrTab->insertString(Name); + for (auto &VSite : IndirectCallSites) + for (auto &VData : VSite.ValueData) + VData.Value = (uint64_t)StrTab->insertString((const char *)VData.Value); +} + namespace IndexedInstrProf { enum class HashT : uint32_t { MD5, diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 031b64b78728..69e899d806ea 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -278,6 +278,11 @@ private: /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; + /// Return the pointer to InstrProfRecord associated with FuncName + /// and FuncHash + ErrorOr getInstrProfRecord(StringRef FuncName, + uint64_t FuncHash); + /// Fill Counts with the profile data for the given function name. std::error_code getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index a80fc5619ce1..58dbeef65313 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -305,21 +305,23 @@ bool InstrProfLookupTrait::ReadValueProfilingData( return false; uint64_t ValueKindCount = endian::readNext(D); + InstrProfRecord &ProfRecord = DataBuffer.back(); for (uint32_t Kind = 0; Kind < ValueKindCount; ++Kind) { // Read value kind and number of value sites for kind. if (D + 2 * sizeof(uint64_t) > End) return false; + uint64_t ValueKind = endian::readNext(D); uint64_t ValueSiteCount = endian::readNext(D); - std::vector &ValueSites = - DataBuffer.back().getValueSitesForKind(ValueKind); - ValueSites.reserve(ValueSiteCount); + ProfRecord.reserveSites(ValueKind, ValueSiteCount); + for (uint64_t VSite = 0; VSite < ValueSiteCount; ++VSite) { // Read number of value data pairs at value site. if (D + sizeof(uint64_t) > End) return false; + uint64_t ValueDataCount = endian::readNext(D); @@ -327,25 +329,18 @@ bool InstrProfLookupTrait::ReadValueProfilingData( if (D + (ValueDataCount << 1) * sizeof(uint64_t) > End) return false; - InstrProfValueSiteRecord VSiteRecord; + std::unique_ptr VDataPtr( + ValueDataCount == 0 ? nullptr + : new InstrProfValueData[ValueDataCount]); + for (uint64_t VCount = 0; VCount < ValueDataCount; ++VCount) { - uint64_t Value = endian::readNext(D); - uint64_t NumTaken = endian::readNext(D); - switch (ValueKind) { - case IPVK_IndirectCallTarget: { - auto Result = - std::lower_bound(HashKeys.begin(), HashKeys.end(), Value, - [](const std::pair &LHS, - uint64_t RHS) { return LHS.first < RHS; }); - assert(Result != HashKeys.end() && - "Hash does not match any known keys\n"); - Value = (uint64_t)Result->second; - break; - } - } - VSiteRecord.ValueData.push_back(std::make_pair(Value, NumTaken)); + VDataPtr[VCount].Value = + endian::readNext(D); + VDataPtr[VCount].Count = + endian::readNext(D); } - ValueSites.push_back(std::move(VSiteRecord)); + ProfRecord.addValueData(ValueKind, VSite, VDataPtr.get(), ValueDataCount, + &HashKeys); } } return true; @@ -385,7 +380,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - DataBuffer.push_back(InstrProfRecord(K, Hash, std::move(CounterBuffer))); + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. if (FormatVersion > 2 && !ReadValueProfilingData(D, End)) { @@ -488,24 +483,30 @@ std::error_code IndexedInstrProfReader::readHeader() { return success(); } -std::error_code IndexedInstrProfReader::getFunctionCounts( - StringRef FuncName, uint64_t FuncHash, std::vector &Counts) { +ErrorOr IndexedInstrProfReader::getInstrProfRecord( + StringRef FuncName, uint64_t FuncHash) { ArrayRef Data; - std::error_code EC = Index.getRecords(FuncName, Data); if (EC != instrprof_error::success) return EC; - // Found it. Look for counters with the right hash. for (unsigned I = 0, E = Data.size(); I < E; ++I) { // Check for a match and fill the vector if there is one. if (Data[I].Hash == FuncHash) { - Counts = Data[I].Counts; - return success(); + return std::move(Data[I]); } } return error(instrprof_error::hash_mismatch); } +std::error_code IndexedInstrProfReader::getFunctionCounts( + StringRef FuncName, uint64_t FuncHash, std::vector &Counts) { + ErrorOr Record = getInstrProfRecord(FuncName, FuncHash); + if (std::error_code EC = Record.getError()) return EC; + + Counts = Record.get().Counts; + return success(); +} + std::error_code IndexedInstrProfReader::readNextRecord( InstrProfRecord &Record) { static unsigned RecordIndex = 0; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index e3018a92e5ef..333865395766 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -45,22 +45,23 @@ public: offset_type M = 0; for (const auto &ProfileData : *V) { + const InstrProfRecord &ProfRecord = ProfileData.second; M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector - M += ProfileData.second.Counts.size() * sizeof(uint64_t); + M += ProfRecord.Counts.size() * sizeof(uint64_t); // Value data M += sizeof(uint64_t); // Number of value kinds with value sites. for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { - const std::vector &ValueSites = - ProfileData.second.getValueSitesForKind(Kind); - if (ValueSites.empty()) - continue; + uint32_t NumValueSites = ProfRecord.getNumValueSites(Kind); + if (NumValueSites == 0) continue; M += sizeof(uint64_t); // Value kind M += sizeof(uint64_t); // The number of value sites for given value kind - for (InstrProfValueSiteRecord I : ValueSites) { + for (uint32_t I = 0; I < NumValueSites; I++) { M += sizeof(uint64_t); // Number of value data pairs at a value site - M += 2 * sizeof(uint64_t) * I.ValueData.size(); // Value data pairs + uint64_t NumValueDataForSite = + ProfRecord.getNumValueDataForSite(Kind, I); + M += 2 * sizeof(uint64_t) * NumValueDataForSite; // Value data pairs } } } @@ -78,36 +79,38 @@ public: using namespace llvm::support; endian::Writer LE(Out); for (const auto &ProfileData : *V) { + const InstrProfRecord &ProfRecord = ProfileData.second; + LE.write(ProfileData.first); // Function hash - LE.write(ProfileData.second.Counts.size()); - for (uint64_t I : ProfileData.second.Counts) - LE.write(I); + LE.write(ProfRecord.Counts.size()); + for (uint64_t I : ProfRecord.Counts) LE.write(I); // Compute the number of value kinds with value sites. - uint64_t NumValueKinds = 0; - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - NumValueKinds += - !(ProfileData.second.getValueSitesForKind(Kind).empty()); + uint64_t NumValueKinds = ProfRecord.getNumValueKinds(); LE.write(NumValueKinds); // Write value data for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { - const std::vector &ValueSites = - ProfileData.second.getValueSitesForKind(Kind); - if (ValueSites.empty()) - continue; + uint32_t NumValueSites = ProfRecord.getNumValueSites(Kind); + if (NumValueSites == 0) continue; LE.write(Kind); // Write value kind // Write number of value sites for current value kind - LE.write(ValueSites.size()); - for (InstrProfValueSiteRecord I : ValueSites) { + LE.write(NumValueSites); + + for (uint32_t I = 0; I < NumValueSites; I++) { // Write number of value data pairs at this value site - LE.write(I.ValueData.size()); - for (auto V : I.ValueData) { + uint64_t NumValueDataForSite = + ProfRecord.getNumValueDataForSite(Kind, I); + LE.write(NumValueDataForSite); + std::unique_ptr VD = + ProfRecord.getValueForSite(Kind, I); + + for (uint32_t V = 0; V < NumValueDataForSite; V++) { if (Kind == IPVK_IndirectCallTarget) - LE.write(ComputeHash((const char *)V.first)); + LE.write(ComputeHash((const char *)VD[V].Value)); else - LE.write(V.first); - LE.write(V.second); + LE.write(VD[V].Value); + LE.write(VD[V].Count); } } } @@ -131,24 +134,7 @@ static std::error_code combineInstrProfRecords(InstrProfRecord &Dest, } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { - - std::vector &SourceValueSites = - Source.getValueSitesForKind(Kind); - if (SourceValueSites.empty()) - continue; - - std::vector &DestValueSites = - Dest.getValueSitesForKind(Kind); - - if (DestValueSites.empty()) { - DestValueSites.swap(SourceValueSites); - continue; - } - - if (DestValueSites.size() != SourceValueSites.size()) - return instrprof_error::value_site_count_mismatch; - for (size_t I = 0, E = SourceValueSites.size(); I < E; ++I) - DestValueSites[I].mergeValueData(SourceValueSites[I]); + if (std::error_code EC = Dest.mergeValueProfData(Kind, Source)) return EC; } // We keep track of the max function count as we go for simplicity. @@ -159,11 +145,7 @@ static std::error_code combineInstrProfRecords(InstrProfRecord &Dest, } void InstrProfWriter::updateStringTableReferences(InstrProfRecord &I) { - I.Name = StringTable.insertString(I.Name); - for (auto &VSite : I.IndirectCallSites) - for (auto &VData : VSite.ValueData) - VData.first = - (uint64_t)StringTable.insertString((const char *)VData.first); + I.updateStrings(&StringTable); } std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I) { diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 382ac908da4d..48fc83c6971b 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -168,8 +168,8 @@ static int showInstrProfile(std::string Filename, bool ShowCounts, << " Counters: " << Func.Counts.size() << "\n" << " Function count: " << Func.Counts[0] << "\n"; if (ShowIndirectCallTargets) - OS << " Indirect Call Site Count: " << Func.IndirectCallSites.size() - << "\n"; + OS << " Indirect Call Site Count: " + << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; } if (Show && ShowCounts) @@ -184,11 +184,15 @@ static int showInstrProfile(std::string Filename, bool ShowCounts, OS << "]\n"; if (Show && ShowIndirectCallTargets) { + uint32_t NS = Func.getNumValueSites(IPVK_IndirectCallTarget); OS << " Indirect Target Results: \n"; - for (size_t I = 0, E = Func.IndirectCallSites.size(); I < E; ++I) { - for (auto V : Func.IndirectCallSites[I].ValueData) { + for (size_t I = 0; I < NS; ++I) { + uint32_t NV = Func.getNumValueDataForSite(IPVK_IndirectCallTarget, I); + std::unique_ptr VD = + Func.getValueForSite(IPVK_IndirectCallTarget, I); + for (uint32_t V = 0; V < NV; V++) { OS << "\t[ " << I << ", "; - OS << (const char *)V.first << ", " << V.second << " ]\n"; + OS << (const char *)VD[V].Value << ", " << VD[V].Count << " ]\n"; } } } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 0e0141b80ea2..9a58e072cfa7 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -67,6 +67,33 @@ TEST_F(InstrProfTest, write_and_read_one_function) { ASSERT_TRUE(++I == E); } +TEST_F(InstrProfTest, get_instr_prof_record) { + InstrProfRecord Record1("foo", 0x1234, {1, 2}); + InstrProfRecord Record2("foo", 0x1235, {3, 4}); + Writer.addRecord(std::move(Record1)); + Writer.addRecord(std::move(Record2)); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + ErrorOr R = Reader->getInstrProfRecord("foo", 0x1234); + ASSERT_TRUE(NoError(R.getError())); + ASSERT_EQ(2U, R.get().Counts.size()); + ASSERT_EQ(1U, R.get().Counts[0]); + ASSERT_EQ(2U, R.get().Counts[1]); + + R = Reader->getInstrProfRecord("foo", 0x1235); + ASSERT_TRUE(NoError(R.getError())); + ASSERT_EQ(2U, R.get().Counts.size()); + ASSERT_EQ(3U, R.get().Counts[0]); + ASSERT_EQ(4U, R.get().Counts[1]); + + R = Reader->getInstrProfRecord("foo", 0x5678); + ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.getError())); + + R = Reader->getInstrProfRecord("bar", 0x1234); + ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.getError())); +} + TEST_F(InstrProfTest, get_function_counts) { InstrProfRecord Record1("foo", 0x1234, {1, 2}); InstrProfRecord Record2("foo", 0x1235, {3, 4}); @@ -94,6 +121,178 @@ TEST_F(InstrProfTest, get_function_counts) { ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, EC)); } +TEST_F(InstrProfTest, get_icall_data_read_write) { + InstrProfRecord Record1("caller", 0x1234, {1, 2}); + InstrProfRecord Record2("callee1", 0x1235, {3, 4}); + InstrProfRecord Record3("callee2", 0x1235, {3, 4}); + InstrProfRecord Record4("callee3", 0x1235, {3, 4}); + + // 4 value sites. + Record1.reserveSites(IPVK_IndirectCallTarget, 4); + InstrProfValueData VD0[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}, + {(uint64_t) "callee3", 3}}; + Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, 0); + // No valeu profile data at the second site. + Record1.addValueData(IPVK_IndirectCallTarget, 1, 0, 0, 0); + InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}}; + Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, 0); + InstrProfValueData VD3[] = {{(uint64_t) "callee1", 1}}; + Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, 0); + + Writer.addRecord(std::move(Record1)); + Writer.addRecord(std::move(Record2)); + Writer.addRecord(std::move(Record3)); + Writer.addRecord(std::move(Record4)); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + ErrorOr R = Reader->getInstrProfRecord("caller", 0x1234); + ASSERT_TRUE(NoError(R.getError())); + ASSERT_EQ(4U, R.get().getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(3U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(0U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(2U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(1U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + + std::unique_ptr VD = + R.get().getValueForSite(IPVK_IndirectCallTarget, 0); + // Now sort the target acording to frequency. + std::sort(&VD[0], &VD[3], + [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { + return VD1.Count > VD2.Count; + }); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); +} + +TEST_F(InstrProfTest, get_icall_data_merge1) { + InstrProfRecord Record11("caller", 0x1234, {1, 2}); + InstrProfRecord Record12("caller", 0x1234, {1, 2}); + InstrProfRecord Record2("callee1", 0x1235, {3, 4}); + InstrProfRecord Record3("callee2", 0x1235, {3, 4}); + InstrProfRecord Record4("callee3", 0x1235, {3, 4}); + InstrProfRecord Record5("callee3", 0x1235, {3, 4}); + InstrProfRecord Record6("callee4", 0x1235, {3, 5}); + + // 5 value sites. + Record11.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD0[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}, + {(uint64_t) "callee3", 3}, + {(uint64_t) "callee4", 4}}; + Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, 0); + + // No valeu profile data at the second site. + Record11.addValueData(IPVK_IndirectCallTarget, 1, 0, 0, 0); + + InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}, + {(uint64_t) "callee3", 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, 0); + + InstrProfValueData VD3[] = {{(uint64_t) "callee1", 1}}; + Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, 0); + + InstrProfValueData VD4[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}, + {(uint64_t) "callee3", 3}}; + Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, 0); + + // A differnt record for the same caller. + Record12.reserveSites(IPVK_IndirectCallTarget, 5); + InstrProfValueData VD02[] = {{(uint64_t) "callee2", 5}, + {(uint64_t) "callee3", 3}}; + Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, 2, 0); + + // No valeu profile data at the second site. + Record12.addValueData(IPVK_IndirectCallTarget, 1, 0, 0, 0); + + InstrProfValueData VD22[] = {{(uint64_t) "callee2", 1}, + {(uint64_t) "callee3", 3}, + {(uint64_t) "callee4", 4}}; + Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, 3, 0); + + Record12.addValueData(IPVK_IndirectCallTarget, 3, 0, 0, 0); + + InstrProfValueData VD42[] = {{(uint64_t) "callee1", 1}, + {(uint64_t) "callee2", 2}, + {(uint64_t) "callee3", 3}}; + Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, 0); + + Writer.addRecord(std::move(Record11)); + // Merge profile data. + Writer.addRecord(std::move(Record12)); + + Writer.addRecord(std::move(Record2)); + Writer.addRecord(std::move(Record3)); + Writer.addRecord(std::move(Record4)); + Writer.addRecord(std::move(Record5)); + Writer.addRecord(std::move(Record6)); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + ErrorOr R = Reader->getInstrProfRecord("caller", 0x1234); + ASSERT_TRUE(NoError(R.getError())); + ASSERT_EQ(5U, R.get().getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(4U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(0U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(4U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(1U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(3U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + + std::unique_ptr VD = + R.get().getValueForSite(IPVK_IndirectCallTarget, 0); + // Now sort the target acording to frequency. + std::sort(&VD[0], &VD[4], + [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { + return VD1.Count > VD2.Count; + }); + ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(7U, VD[0].Count); + ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD[1].Count); + ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD[2].Count); + ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD[3].Count); + + std::unique_ptr VD_2( + R.get().getValueForSite(IPVK_IndirectCallTarget, 2)); + std::sort(&VD_2[0], &VD_2[4], + [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { + return VD1.Count > VD2.Count; + }); + ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_2[0].Count); + ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4")); + ASSERT_EQ(4U, VD_2[1].Count); + ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2")); + ASSERT_EQ(3U, VD_2[2].Count); + ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_2[3].Count); + + std::unique_ptr VD_3( + R.get().getValueForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1")); + ASSERT_EQ(1U, VD_3[0].Count); + + std::unique_ptr VD_4( + R.get().getValueForSite(IPVK_IndirectCallTarget, 4)); + std::sort(&VD_4[0], &VD_4[3], + [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { + return VD1.Count > VD2.Count; + }); + ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(6U, VD_4[0].Count); + ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(4U, VD_4[1].Count); + ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(2U, VD_4[2].Count); +} + TEST_F(InstrProfTest, get_max_function_count) { InstrProfRecord Record1("foo", 0x1234, {1ULL << 31, 2}); InstrProfRecord Record2("bar", 0, {1ULL << 63});