maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[libFuzzer] when shrinking the corpus, delete evicted files previously created by the current process 

llvm-svn: 283682
This commit is contained in:
Kostya Serebryany 2016-10-08 23:24:45 +00:00
parent 8ec7b4f588
commit c5325ed29d
8 changed files with 46 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
llvm/lib/Fuzzer/FuzzerCorpus.h
View File

@ -30,12 +30,13 @@ struct InputInfo {
// Stats.
size_t NumExecutedMutations = 0;
size_t NumSuccessfullMutations = 0;
bool MayDeleteFile = false;
};
class InputCorpus {
public:
static const size_t kFeatureSetSize = 1 << 16;
InputCorpus() {
InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) {
memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature));
memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature));
}
@ -58,7 +59,7 @@ class InputCorpus {
}
bool empty() const { return Inputs.empty(); }
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
void AddToCorpus(const Unit &U, size_t NumFeatures) {
void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile = false) {
assert(!U.empty());
uint8_t Hash[kSHA1NumBytes];
if (FeatureDebug)
@ -69,6 +70,7 @@ class InputCorpus {
InputInfo &II = *Inputs.back();
II.U = U;
II.NumFeatures = NumFeatures;
II.MayDeleteFile = MayDeleteFile;
memcpy(II.Sha1, Hash, kSHA1NumBytes);
UpdateCorpusDistribution();
ValidateFeatureSet();
@ -112,20 +114,27 @@ class InputCorpus {
Printf("\n");
}
void DeleteInput(size_t Idx) {
InputInfo &II = *Inputs[Idx];
if (!OutputCorpus.empty() && II.MayDeleteFile)
DeleteFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1)));
Unit().swap(II.U);
if (FeatureDebug)
Printf("EVICTED %zd\n", Idx);
}
bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) {
assert(NewSize);
Idx = Idx % kFeatureSetSize;
uint32_t OldSize = GetFeature(Idx);
if (OldSize == 0 || (Shrink && OldSize > NewSize)) {
if (OldSize > 0) {
InputInfo &II = *Inputs[SmallestElementPerFeature[Idx]];
size_t OldIdx = SmallestElementPerFeature[Idx];
InputInfo &II = *Inputs[OldIdx];
assert(II.NumFeatures > 0);
II.NumFeatures--;
if (II.NumFeatures == 0) {
Unit().swap(II.U);
if (FeatureDebug)
Printf("EVICTED %zd\n", SmallestElementPerFeature[Idx]);
}
if (II.NumFeatures == 0)
DeleteInput(OldIdx);
}
if (FeatureDebug)
Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize);
@ -191,6 +200,8 @@ private:
bool CountingFeatures = false;
uint32_t InputSizesPerFeature[kFeatureSetSize];
uint32_t SmallestElementPerFeature[kFeatureSetSize];
std::string OutputCorpus;
};
} // namespace fuzzer

6
llvm/lib/Fuzzer/FuzzerDefs.h
View File

@ -60,11 +60,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);
bool IsFile(const std::string &Path);
long GetEpoch(const std::string &Path);
std::string FileToString(const std::string &Path);
Unit FileToVector(const std::string &Path, size_t MaxSize = 0);
Unit FileToVector(const std::string &Path, size_t MaxSize = 0,
bool ExitOnError = true);
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
long *Epoch, size_t MaxSize);
long *Epoch, size_t MaxSize, bool ExitOnError);
void WriteToFile(const Unit &U, const std::string &Path);
void CopyFileToErr(const std::string &Path);
void DeleteFile(const std::string &Path);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
const std::string &FileName);

5
llvm/lib/Fuzzer/FuzzerDriver.cpp
View File

@ -443,7 +443,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
Random Rand(Seed);
MutationDispatcher MD(Rand, Options);
InputCorpus Corpus;
InputCorpus Corpus(Options.OutputCorpus);
Fuzzer F(Callback, Corpus, MD, Options);
for (auto &U: Dictionary)
@ -500,7 +500,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
UnitVector InitialCorpus;
for (auto &Inp : *Inputs) {
Printf("Loading corpus dir: %s\n", Inp.c_str());
ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, TemporaryMaxLen);
ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
TemporaryMaxLen, /*ExitOnError=*/false);
}
if (Options.MaxLen == 0) {

2
llvm/lib/Fuzzer/FuzzerFlags.def
View File

@ -56,7 +56,7 @@ FUZZER_FLAG_INT(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn"
FUZZER_FLAG_INT(workers, 0,
"Number of simultaneous worker processes to run the jobs."
" If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.")
FUZZER_FLAG_INT(reload, 10,
FUZZER_FLAG_INT(reload, 1,
"Reload the main corpus every <N> seconds to get new units"
" discovered by other processes. If 0, disabled")
FUZZER_FLAG_INT(report_slow_units, 10,

14
llvm/lib/Fuzzer/FuzzerIO.cpp
View File

@ -60,9 +60,9 @@ static void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
*Epoch = E;
}
Unit FileToVector(const std::string &Path, size_t MaxSize) {
Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) {
std::ifstream T(Path);
if (!T) {
if (ExitOnError && !T) {
Printf("No such directory: %s; exiting\n", Path.c_str());
exit(1);
}
@ -78,6 +78,10 @@ Unit FileToVector(const std::string &Path, size_t MaxSize) {
return Res;
}
void DeleteFile(const std::string &Path) {
unlink(Path.c_str());
}
std::string FileToString(const std::string &Path) {
std::ifstream T(Path);
return std::string((std::istreambuf_iterator<char>(T)),
@ -97,7 +101,7 @@ void WriteToFile(const Unit &U, const std::string &Path) {
}
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
long *Epoch, size_t MaxSize) {
long *Epoch, size_t MaxSize, bool ExitOnError) {
long E = Epoch ? *Epoch : 0;
std::vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
@ -108,7 +112,9 @@ void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
NumLoaded++;
if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024)
Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path);
V->push_back(FileToVector(X, MaxSize));
auto S = FileToVector(X, MaxSize, ExitOnError);
if (!S.empty())
V->push_back(S);
}
}

13
llvm/lib/Fuzzer/FuzzerLoop.cpp
View File

@ -386,7 +386,8 @@ void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus, MaxSize);
&EpochOfLastReadOfOutputCorpus, MaxSize,
/*ExitOnError*/ false);
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
bool Reloaded = false;
@ -605,9 +606,9 @@ void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
assert(MaxInputLen > 0);
UnitVector Initial, Extra;
ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen);
ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen, true);
for (auto &C : ExtraCorpora)
ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen);
ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen, true);
if (!Initial.empty()) {
Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
@ -685,8 +686,8 @@ void Fuzzer::MutateAndTestOne() {
StartTraceRecording();
II.NumExecutedMutations++;
if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) {
Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size},
NumFeatures);
Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures,
/*MayDeleteFile=*/true);
ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size});
CheckExitOnItem();
}
@ -711,7 +712,7 @@ void Fuzzer::Loop() {
if (duration_cast<seconds>(Now - LastCorpusReload).count() >=
Options.ReloadIntervalSec) {
RereadOutputCorpus(MaxInputLen);
LastCorpusReload = Now;
LastCorpusReload = system_clock::now();
}
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
break;

2
llvm/lib/Fuzzer/build.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
LIBFUZZER_SRC_DIR=$(dirname $0)
for f in $LIBFUZZER_SRC_DIR/*.cpp; do
clang -g -O2 -std=c++11 $f -c &
clang -g -O2 -fno-omit-frame-pointer -std=c++11 $f -c &
done
wait
rm -f libFuzzer.a

2
llvm/lib/Fuzzer/test/FuzzerUnittest.cpp
View File

@ -579,7 +579,7 @@ TEST(FuzzerUtil, Base64) {
TEST(Corpus, Distribution) {
Random Rand(0);
InputCorpus C;
InputCorpus C("");
size_t N = 10;
size_t TriesPerUnit = 1<<20;
for (size_t i = 0; i < N; i++)