maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[libFuzzer] more refactoring; change some of the exit codes (timeout, OOM, interrupt) so that the parent process can distinguish those 

llvm-svn: 353584
This commit is contained in:
Kostya Serebryany 2019-02-09 00:16:21 +00:00
parent 9197056419
commit 0fda9dcbe9
6 changed files with 57 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
compiler-rt/lib/fuzzer/FuzzerDriver.cpp
View File

@ -475,22 +475,27 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
void FuzzWithFork(const FuzzingOptions &Options,
const Vector<std::string> &Args,
const Vector<std::string> &Corpora) {
auto CFPath = TempPath(".fork");
Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
"be more resistant to crashes, timeouts, and OOMs\n");
Vector<SizedFile> Corpus;
for (auto &Dir : Corpora)
GetSizedFilesFromDir(Dir, &Corpus);
std::sort(Corpus.begin(), Corpus.end());
auto Files = CrashResistantMerge(Args, {}, Corpus, CFPath);
Printf("INFO: -fork=1: seed corpus analyzed, %zd seeds chosen, starting to "
"fuzz in separate processes\n", Files.size());
Vector<std::string> Files;
Set<uint32_t> Features;
if (!Corpus.empty()) {
auto CFPath = TempPath(".fork");
CrashResistantMerge(Args, {}, Corpus, &Files, {}, &Features, CFPath);
RemoveFile(CFPath);
}
Printf("INFO: -fork=1: %zd seeds, starting to fuzz\n", Files.size());
Command Cmd(Args);
Cmd.removeFlag("fork");
for (auto &C : Corpora) // Remove all corpora from the args.
Cmd.removeArgument(C);
if (Files.size() >= 2)
Cmd.addFlag("seed_inputs",
Files.back() + "," + Files[Files.size() - 2]);
@ -499,11 +504,13 @@ void FuzzWithFork(const FuzzingOptions &Options,
for (size_t i = 0; i < 1000; i++) {
Printf("RUN %s\n", Cmd.toString().c_str());
int ExitCode = ExecuteCommand(Cmd);
// TODO: sniff the crash, ignore OOMs and timeouts.
if (ExitCode == Options.InterruptExitCode)
exit(0);
if (ExitCode == Options.TimeoutExitCode || ExitCode == Options.OOMExitCode)
continue;
if (ExitCode != 0) break;
}
RemoveFile(CFPath);
exit(0);
}
@ -522,8 +529,11 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
std::sort(NewCorpus.begin(), NewCorpus.end());
std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt");
auto Files = CrashResistantMerge(Args, OldCorpus, NewCorpus, CFPath);
for (auto &Path : Files)
Vector<std::string> NewFiles;
Set<uint32_t> NewFeatures;
CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
CFPath);
for (auto &Path : NewFiles)
F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
// We are done, delete the control file if it was a temporary one.
if (!Flags.merge_control_file)

7
compiler-rt/lib/fuzzer/FuzzerLoop.cpp
View File

@ -131,7 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) {
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
_Exit(Options.ErrorExitCode); // Stop right now.
_Exit(Options.OOMExitCode); // Stop right now.
}
Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
@ -265,7 +265,8 @@ void Fuzzer::MaybeExitGracefully() {
void Fuzzer::InterruptCallback() {
Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
PrintFinalStats();
_Exit(0); // Stop right now, don't perform any at-exit actions.
// Stop right now, don't perform any at-exit actions.
_Exit(Options.InterruptExitCode);
}
NO_SANITIZE_MEMORY
@ -314,7 +315,7 @@ void Fuzzer::RssLimitCallback() {
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
_Exit(Options.ErrorExitCode); // Stop right now.
_Exit(Options.OOMExitCode); // Stop right now.
}
void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {

34
compiler-rt/lib/fuzzer/FuzzerMerge.cpp
View File

@ -122,25 +122,26 @@ size_t Merger::ApproximateMemoryConsumption() const {
// Decides which files need to be merged (add thost to NewFiles).
// Returns the number of new features added.
size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
Set<uint32_t> *AllFeatures,
Vector<std::string> *NewFiles) {
NewFiles->clear();
assert(NumFilesInFirstCorpus <= Files.size());
Set<uint32_t> AllFeatures(InitialFeatures);
*AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
auto &Cur = Files[i].Features;
AllFeatures.insert(Cur.begin(), Cur.end());
AllFeatures->insert(Cur.begin(), Cur.end());
}
size_t InitialNumFeatures = AllFeatures.size();
size_t InitialNumFeatures = AllFeatures->size();
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
Vector<uint32_t> Tmp;
std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
std::set_difference(Cur.begin(), Cur.end(), AllFeatures->begin(),
AllFeatures->end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
}
@ -160,12 +161,12 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
auto &Cur = Files[i].Features;
// Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(),
// Files[i].Size, Cur.size());
size_t OldSize = AllFeatures.size();
AllFeatures.insert(Cur.begin(), Cur.end());
if (AllFeatures.size() > OldSize)
size_t OldSize = AllFeatures->size();
AllFeatures->insert(Cur.begin(), Cur.end());
if (AllFeatures->size() > OldSize)
NewFiles->push_back(Files[i].Name);
}
return AllFeatures.size() - InitialNumFeatures;
return AllFeatures->size() - InitialNumFeatures;
}
Set<uint32_t> Merger::AllFeatures() const {
@ -248,10 +249,12 @@ static void WriteNewControlFile(const std::string &CFPath,
}
// Outer process. Does not call the target code and thus should not fail.
Vector<std::string>
CrashResistantMerge(const Vector<std::string> &Args,
void CrashResistantMerge(const Vector<std::string> &Args,
const Vector<SizedFile> &OldCorpus,
const Vector<SizedFile> &NewCorpus,
Vector<std::string> *NewFiles,
const Set<uint32_t> &InitialFeatures,
Set<uint32_t> *NewFeatures,
const std::string &CFPath) {
size_t NumAttempts = 0;
if (FileSize(CFPath)) {
@ -319,12 +322,9 @@ CrashResistantMerge(const Vector<std::string> &Args,
IF.close();
Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n",
M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
Set<uint32_t> InitialFeatures;
Vector<std::string> NewFiles;
size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles);
size_t NumNewFeatures = M.Merge(InitialFeatures, NewFeatures, NewFiles);
Printf("MERGE-OUTER: %zd new files with %zd new features added\n",
NewFiles.size(), NumNewFeatures);
return NewFiles;
NewFiles->size(), NumNewFeatures);
}
} // namespace fuzzer

13
compiler-rt/lib/fuzzer/FuzzerMerge.h
View File

@ -64,16 +64,19 @@ struct Merger {
bool Parse(const std::string &Str, bool ParseCoverage);
void ParseOrExit(std::istream &IS, bool ParseCoverage);
size_t Merge(const Set<uint32_t> &InitialFeatures,
Set<uint32_t> *NewFeatures,
Vector<std::string> *NewFiles);
size_t ApproximateMemoryConsumption() const;
Set<uint32_t> AllFeatures() const;
};
Vector<std::string>
CrashResistantMerge(const Vector<std::string> &Args,
const Vector<SizedFile> &OldCorpus,
const Vector<SizedFile> &NewCorpus,
const std::string &CFPath);
void CrashResistantMerge(const Vector<std::string> &Args,
const Vector<SizedFile> &OldCorpus,
const Vector<SizedFile> &NewCorpus,
Vector<std::string> *NewFiles,
const Set<uint32_t> &InitialFeatures,
Set<uint32_t> *NewFeatures,
const std::string &CFPath);
} // namespace fuzzer

4
compiler-rt/lib/fuzzer/FuzzerOptions.h
View File

@ -19,7 +19,9 @@ struct FuzzingOptions {
size_t MaxLen = 0;
size_t LenControl = 1000;
int UnitTimeoutSec = 300;
int TimeoutExitCode = 77;
int TimeoutExitCode = 70;
int OOMExitCode = 71;
int InterruptExitCode = 72;
int ErrorExitCode = 77;
int MaxTotalTimeSec = 0;
int RssLimitMb = 0;

10
compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp
View File

@ -644,8 +644,9 @@ static void Merge(const std::string &Input,
size_t NumNewFeatures) {
Merger M;
Vector<std::string> NewFiles;
Set<uint32_t> NewFeatures;
EXPECT_TRUE(M.Parse(Input, true));
EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFiles));
EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, Result);
}
@ -689,6 +690,7 @@ TEST(Merge, Good) {
Vector<std::string> NewFiles;
Set<uint32_t> NewFeatures;
EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n"
"STARTED 0 1000\nDONE 0 1 2 3\n"
@ -702,7 +704,7 @@ TEST(Merge, Good) {
EQ(M.Files[0].Features, {1, 2, 3});
EQ(M.Files[1].Features, {4, 5, 6});
EQ(M.Files[2].Features, {1, 3, 6});
EXPECT_EQ(0U, M.Merge({}, &NewFiles));
EXPECT_EQ(0U, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, {});
EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n"
@ -713,7 +715,7 @@ TEST(Merge, Good) {
EQ(M.Files[0].Features, {1, 2, 3});
EQ(M.Files[1].Features, {4, 5, 6});
EQ(M.Files[2].Features, {1, 3, 6});
EXPECT_EQ(3U, M.Merge({}, &NewFiles));
EXPECT_EQ(3U, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, {"B"});
// Same as the above, but with InitialFeatures.
@ -727,7 +729,7 @@ TEST(Merge, Good) {
InitialFeatures.insert(1);
InitialFeatures.insert(2);
InitialFeatures.insert(3);
EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFiles));
EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, &NewFiles));
EQ(NewFiles, {"B"});
}