forked from OSchip/llvm-project
[libFuzzer] Remove mutation stats and weighted mutation selection.
Summary: This was an experimental feature. After evaluating it with: 1) https://github.com/google/fuzzer-test-suite/tree/master/engine-comparison 2) enabling on real world fuzz targets running at ClusterFuzz and OSS-Fuzz The following conclusions were made: 1) With fuzz targets that have reached a code coverage plateau, the feature does not improve libFuzzer's ability to discover new coverage and may actually negatively impact it. 2) With fuzz targets that have not yet reached a code coverage plateau, the feature might speed up new units discovery in some cases, but it is quite rare and hard to confirm with a high level on confidence. Revert of https://reviews.llvm.org/D48054 and https://reviews.llvm.org/D49621. Reviewers: metzman, morehouse Reviewed By: metzman, morehouse Subscribers: delcypher, #sanitizers, llvm-commits, kcc Differential Revision: https://reviews.llvm.org/D51455 llvm-svn: 340976
This commit is contained in:
Max Moroz
parent
594b5410a6
commit
8c95b48ba2
|
|
@ -615,8 +615,6 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
|
|||
Options.PrintNewCovPcs = Flags.print_pcs;
|
||||
Options.PrintNewCovFuncs = Flags.print_funcs;
|
||||
Options.PrintFinalStats = Flags.print_final_stats;
|
||||
Options.PrintMutationStats = Flags.print_mutation_stats;
|
||||
Options.UseWeightedMutations = Flags.use_weighted_mutations;
|
||||
Options.PrintCorpusStats = Flags.print_corpus_stats;
|
||||
Options.PrintCoverage = Flags.print_coverage;
|
||||
Options.PrintUnstableStats = Flags.print_unstable_stats;
|
||||
|
|
|
|||
|
|
@ -162,6 +162,3 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_server)
|
|||
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
|
||||
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
|
||||
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
|
||||
FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
|
||||
FUZZER_FLAG_INT(use_weighted_mutations, 0, "Experimental: If 1, fuzzing will "
|
||||
"favor mutations that perform better during runtime.")
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@
|
|||
|
||||
namespace fuzzer {
|
||||
static const size_t kMaxUnitSizeToPrint = 256;
|
||||
static const size_t kUpdateMutationWeightRuns = 10000;
|
||||
|
||||
thread_local bool Fuzzer::IsMyThread;
|
||||
|
||||
|
|
@ -361,7 +360,6 @@ void Fuzzer::PrintFinalStats() {
|
|||
TPC.DumpCoverage();
|
||||
if (Options.PrintCorpusStats)
|
||||
Corpus.PrintStats();
|
||||
if (Options.PrintMutationStats) MD.PrintMutationStats();
|
||||
if (!Options.PrintFinalStats)
|
||||
return;
|
||||
size_t ExecPerSec = execPerSec();
|
||||
|
|
@ -550,9 +548,6 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) {
|
|||
|
||||
void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
|
||||
TPC.RecordInitialStack();
|
||||
if (Options.UseWeightedMutations &&
|
||||
TotalNumberOfRuns % kUpdateMutationWeightRuns == 0)
|
||||
MD.UpdateDistribution();
|
||||
TotalNumberOfRuns++;
|
||||
assert(InFuzzingThread());
|
||||
if (SMR.IsClient())
|
||||
|
|
|
|||
|
|
@ -30,41 +30,34 @@ MutationDispatcher::MutationDispatcher(Random &Rand,
|
|||
DefaultMutators.insert(
|
||||
DefaultMutators.begin(),
|
||||
{
|
||||
// Initialize useful and total mutation counts as 1 in order to
|
||||
// have mutation stats (i.e. weights) with equal non-zero values.
|
||||
{&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 1, 1},
|
||||
{&MutationDispatcher::Mutate_InsertByte, "InsertByte", 1, 1},
|
||||
{&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
|
||||
{&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
|
||||
{&MutationDispatcher::Mutate_InsertRepeatedBytes,
|
||||
"InsertRepeatedBytes", 1, 1},
|
||||
{&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 1, 1},
|
||||
{&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 1, 1},
|
||||
{&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 1, 1},
|
||||
{&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 1,
|
||||
1},
|
||||
{&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 1,
|
||||
1},
|
||||
{&MutationDispatcher::Mutate_CopyPart, "CopyPart", 1, 1},
|
||||
{&MutationDispatcher::Mutate_CrossOver, "CrossOver", 1, 1},
|
||||
"InsertRepeatedBytes"},
|
||||
{&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
|
||||
{&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
|
||||
{&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
|
||||
{&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
|
||||
{&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
|
||||
{&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
|
||||
{&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
|
||||
{&MutationDispatcher::Mutate_AddWordFromManualDictionary,
|
||||
"ManualDict", 1, 1},
|
||||
"ManualDict"},
|
||||
{&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
|
||||
"PersAutoDict", 1, 1},
|
||||
"PersAutoDict"},
|
||||
});
|
||||
if(Options.UseCmp)
|
||||
DefaultMutators.push_back(
|
||||
{&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 1, 1});
|
||||
{&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
|
||||
|
||||
if (EF->LLVMFuzzerCustomMutator)
|
||||
Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 1, 1});
|
||||
Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
|
||||
else
|
||||
Mutators = DefaultMutators;
|
||||
|
||||
if (EF->LLVMFuzzerCustomCrossOver)
|
||||
Mutators.push_back(
|
||||
{&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 1, 1});
|
||||
|
||||
// For weighted mutation selection, init with uniform weights distribution.
|
||||
Stats.resize(Mutators.size());
|
||||
{&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
|
||||
}
|
||||
|
||||
static char RandCh(Random &Rand) {
|
||||
|
|
@ -471,7 +464,6 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() {
|
|||
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
|
||||
PersistentAutoDictionary.push_back({DE->GetW(), 1});
|
||||
}
|
||||
RecordUsefulMutations();
|
||||
}
|
||||
|
||||
void MutationDispatcher::PrintRecommendedDictionary() {
|
||||
|
|
@ -492,7 +484,8 @@ void MutationDispatcher::PrintRecommendedDictionary() {
|
|||
|
||||
void MutationDispatcher::PrintMutationSequence() {
|
||||
Printf("MS: %zd ", CurrentMutatorSequence.size());
|
||||
for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
|
||||
for (auto M : CurrentMutatorSequence)
|
||||
Printf("%s-", M.Name);
|
||||
if (!CurrentDictionaryEntrySequence.empty()) {
|
||||
Printf(" DE: ");
|
||||
for (auto DE : CurrentDictionaryEntrySequence) {
|
||||
|
|
@ -519,20 +512,13 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
|
|||
// Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
|
||||
// in which case they will return 0.
|
||||
// Try several times before returning un-mutated data.
|
||||
Mutator *M = nullptr;
|
||||
for (int Iter = 0; Iter < 100; Iter++) {
|
||||
// Even when using weighted mutations, fallback to the default selection in
|
||||
// 20% of cases.
|
||||
if (Options.UseWeightedMutations && Rand(5))
|
||||
M = &Mutators[WeightedIndex()];
|
||||
else
|
||||
M = &Mutators[Rand(Mutators.size())];
|
||||
size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
|
||||
auto M = Mutators[Rand(Mutators.size())];
|
||||
size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
|
||||
if (NewSize && NewSize <= MaxSize) {
|
||||
if (Options.OnlyASCII)
|
||||
ToASCII(Data, NewSize);
|
||||
CurrentMutatorSequence.push_back(M);
|
||||
M->TotalCount++;
|
||||
return NewSize;
|
||||
}
|
||||
}
|
||||
|
|
@ -573,34 +559,4 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) {
|
|||
{W, std::numeric_limits<size_t>::max()});
|
||||
}
|
||||
|
||||
void MutationDispatcher::RecordUsefulMutations() {
|
||||
for (auto M : CurrentMutatorSequence) M->UsefulCount++;
|
||||
}
|
||||
|
||||
void MutationDispatcher::PrintMutationStats() {
|
||||
Printf("\nstat::mutation_usefulness: ");
|
||||
UpdateMutationStats();
|
||||
for (size_t i = 0; i < Stats.size(); i++) {
|
||||
Printf("%.3f", 100 * Stats[i]);
|
||||
if (i < Stats.size() - 1)
|
||||
Printf(",");
|
||||
else
|
||||
Printf("\n");
|
||||
}
|
||||
}
|
||||
|
||||
void MutationDispatcher::UpdateMutationStats() {
|
||||
// Calculate usefulness statistic for each mutation
|
||||
for (size_t i = 0; i < Stats.size(); i++)
|
||||
Stats[i] =
|
||||
static_cast<double>(Mutators[i].UsefulCount) / Mutators[i].TotalCount;
|
||||
}
|
||||
|
||||
void MutationDispatcher::UpdateDistribution() {
|
||||
UpdateMutationStats();
|
||||
Distribution = std::discrete_distribution<size_t>(Stats.begin(), Stats.end());
|
||||
}
|
||||
|
||||
size_t MutationDispatcher::WeightedIndex() { return Distribution(GetRand()); }
|
||||
|
||||
} // namespace fuzzer
|
||||
|
|
|
|||
|
|
@ -93,29 +93,10 @@ public:
|
|||
|
||||
Random &GetRand() { return Rand; }
|
||||
|
||||
/// Records tally of mutations resulting in new coverage, for usefulness
|
||||
/// metric.
|
||||
void RecordUsefulMutations();
|
||||
|
||||
/// Outputs usefulness stats on command line if option is enabled.
|
||||
void PrintMutationStats();
|
||||
|
||||
/// Recalculates mutation stats based on latest run data.
|
||||
void UpdateMutationStats();
|
||||
|
||||
/// Sets weights based on mutation performance during fuzzer run.
|
||||
void UpdateDistribution();
|
||||
|
||||
/// Returns the index of a mutation based on how useful it has been.
|
||||
/// Favors mutations with higher usefulness ratios but can return any index.
|
||||
size_t WeightedIndex();
|
||||
|
||||
private:
|
||||
struct Mutator {
|
||||
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
|
||||
const char *Name;
|
||||
uint64_t UsefulCount;
|
||||
uint64_t TotalCount;
|
||||
};
|
||||
|
||||
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
|
||||
|
|
@ -154,7 +135,6 @@ public:
|
|||
Dictionary PersistentAutoDictionary;
|
||||
|
||||
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
|
||||
Vector<Mutator *> CurrentMutatorSequence;
|
||||
|
||||
static const size_t kCmpDictionaryEntriesDequeSize = 16;
|
||||
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
|
||||
|
|
@ -169,10 +149,7 @@ public:
|
|||
|
||||
Vector<Mutator> Mutators;
|
||||
Vector<Mutator> DefaultMutators;
|
||||
|
||||
// Used to weight mutations based on usefulness.
|
||||
Vector<double> Stats;
|
||||
std::discrete_distribution<size_t> Distribution;
|
||||
Vector<Mutator> CurrentMutatorSequence;
|
||||
};
|
||||
|
||||
} // namespace fuzzer
|
||||
|
|
|
|||
|
|
@ -52,8 +52,6 @@ struct FuzzingOptions {
|
|||
bool PrintNewCovPcs = false;
|
||||
int PrintNewCovFuncs = 0;
|
||||
bool PrintFinalStats = false;
|
||||
bool PrintMutationStats = false;
|
||||
bool UseWeightedMutations = false;
|
||||
bool PrintCorpusStats = false;
|
||||
bool PrintCoverage = false;
|
||||
bool PrintUnstableStats = false;
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest
|
||||
RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s --check-prefix=STAT
|
||||
|
||||
# Ensures there are some non-zero values in the usefulness percentages printed.
|
||||
STAT: stat::mutation_usefulness: {{[0-9]+\.[0-9]+}}
|
||||
|
||||
# Weighted mutations only trigger after first 10,000 runs, hence flag.
|
||||
RUN: not %run %t-MutationStatsTest -use_weighted_mutations=1 -seed=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=WEIGHTED
|
||||
|
||||
WEIGHTED: BINGO
|
||||
Loading…
Reference in New Issue