Merge pull request #117 from rchildre3/update-libfuzzer-18

Update libfuzzer code to compiler-rt 18

CHANGELOG.md

@@ -9,7 +9,7 @@ Released YYYY-MM-DD.
 
 ### Changed
 
-* TODO (or remove section if none)
+* Updated to `libFuzzer` commit `3747cde5e84f` (`release/18.x`).
 
 ### Deprecated
 

libfuzzer/CMakeLists.txt

@@ -59,7 +59,7 @@ CHECK_CXX_SOURCE_COMPILES("
 
 set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND
+if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
   list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer)
@@ -135,7 +135,7 @@ add_compiler_rt_runtime(clang_rt.fuzzer_interceptors
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND
+if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
   macro(partially_link_libcxx name dir arch)

libfuzzer/FuzzerCommand.h

@@ -19,6 +19,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <thread>
 
 namespace fuzzer {
 
@@ -139,7 +140,7 @@ class Command final {
   // be the equivalent command line.
   std::string toString() const {
     std::stringstream SS;
-    for (auto arg : getArguments())
+    for (const auto &arg : getArguments())
       SS << arg << " ";
     if (hasOutputFile())
       SS << ">" << getOutputFile() << " ";

libfuzzer/FuzzerCorpus.h

@@ -18,6 +18,7 @@
 #include "FuzzerSHA1.h"
 #include "FuzzerTracePC.h"
 #include <algorithm>
+#include <bitset>
 #include <chrono>
 #include <numeric>
 #include <random>
@@ -77,7 +78,7 @@ struct InputInfo {
     SumIncidence = 0.0;
 
     // Apply add-one smoothing to locally discovered features.
-    for (auto F : FeatureFreqs) {
+    for (const auto &F : FeatureFreqs) {
       double LocalIncidence = F.second + 1;
       Energy -= LocalIncidence * log(LocalIncidence);
       SumIncidence += LocalIncidence;
@@ -382,6 +383,7 @@ class InputCorpus {
       }
 
       // Remove most abundant rare feature.
+      IsRareFeature[Delete] = false;
       RareFeatures[Delete] = RareFeatures.back();
       RareFeatures.pop_back();
 
@@ -397,6 +399,7 @@ class InputCorpus {
 
     // Add rare feature, handle collisions, and update energy.
     RareFeatures.push_back(Idx);
+    IsRareFeature[Idx] = true;
     GlobalFeatureFreqs[Idx] = 0;
     for (auto II : Inputs) {
       II->DeleteFeatureFreq(Idx);
@@ -450,9 +453,7 @@ class InputCorpus {
     uint16_t Freq = GlobalFeatureFreqs[Idx32]++;
 
     // Skip if abundant.
-    if (Freq > FreqOfMostAbundantRareFeature ||
-        std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) ==
-            RareFeatures.end())
+    if (Freq > FreqOfMostAbundantRareFeature || !IsRareFeature[Idx32])
       return;
 
     // Update global frequencies.
@@ -581,6 +582,7 @@ class InputCorpus {
   uint16_t FreqOfMostAbundantRareFeature = 0;
   uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
   std::vector<uint32_t> RareFeatures;
+  std::bitset<kFeatureSetSize> IsRareFeature;
 
   std::string OutputCorpus;
 };

libfuzzer/FuzzerDataFlowTrace.cpp

@@ -88,7 +88,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
 //   * a function with a less frequently executed code gets bigger weight.
 std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
   std::vector<double> Res(NumFunctions);
-  for (auto It : Functions) {
+  for (const auto &It : Functions) {
     auto FunctionID = It.first;
     auto Counters = It.second;
     assert(FunctionID < NumFunctions);

libfuzzer/FuzzerDriver.cpp

@@ -293,9 +293,12 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
   std::vector<std::thread> V;
   std::thread Pulse(PulseThread);
   Pulse.detach();
-  for (unsigned i = 0; i < NumWorkers; i++)
-    V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
-                            &HasErrors));
+  V.resize(NumWorkers);
+  for (unsigned i = 0; i < NumWorkers; i++) {
+    V[i] = std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
+                            &HasErrors);
+    SetThreadName(V[i], "FuzzerWorker");
+  }
   for (auto &T : V)
     T.join();
   return HasErrors ? 1 : 0;
@@ -463,7 +466,7 @@ int MinimizeCrashInput(const std::vector<std::string> &Args,
         CurrentFilePath = Flags.exact_artifact_path;
         WriteToFile(U, CurrentFilePath);
       }
-      Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n",
+      Printf("CRASH_MIN: failed to minimize beyond %s (%zu bytes), exiting\n",
              CurrentFilePath.c_str(), U.size());
       break;
     }
@@ -501,7 +504,6 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
   F->MinimizeCrashLoop(U);
   Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n");
   exit(0);
-  return 0;
 }
 
 void Merge(Fuzzer *F, FuzzingOptions &Options,
@@ -535,7 +537,7 @@ void Merge(Fuzzer *F, FuzzingOptions &Options,
 
 int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
                       UnitVector &Corpus) {
-  Printf("Started dictionary minimization (up to %d tests)\n",
+  Printf("Started dictionary minimization (up to %zu tests)\n",
          Dict.size() * Corpus.size() * 2);
 
   // Scores and usage count for each dictionary unit.
@@ -779,7 +781,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   if (!Options.FocusFunction.empty())
     Options.Entropic = false; // FocusFunction overrides entropic scheduling.
   if (Options.Entropic)
-    Printf("INFO: Running with entropic power schedule (0x%X, %d).\n",
+    Printf("INFO: Running with entropic power schedule (0x%zX, %zu).\n",
            Options.EntropicFeatureFrequencyThreshold,
            Options.EntropicNumberOfRarestFeatures);
   struct EntropicOptions Entropic;
@@ -797,7 +799,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   if (Flags.verbosity)
     Printf("INFO: Seed: %u\n", Seed);
 
-  if (Flags.collect_data_flow && !Flags.fork &&
+  if (Flags.collect_data_flow && Flags.data_flow_trace && !Flags.fork &&
       !(Flags.merge || Flags.set_cover_merge)) {
     if (RunIndividualFiles)
       return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
@@ -860,7 +862,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
         RunOneTest(F, Path.c_str(), Options.MaxLen);
       auto StopTime = system_clock::now();
       auto MS = duration_cast<milliseconds>(StopTime - StartTime).count();
-      Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS);
+      Printf("Executed %s in %ld ms\n", Path.c_str(), (long)MS);
     }
     Printf("***\n"
            "*** NOTE: fuzzing was not performed, you have only\n"

libfuzzer/FuzzerFlags.def

@@ -167,7 +167,7 @@ FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and "
     "purge_allocator_interval=-1 to disable this functionality.")
 FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. "
     "If >= 2 will also print stack traces.")
-FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon"
+FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon "
     "reaching this limit of RSS memory usage.")
 FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit "
     "if the target tries to allocate this number of Mb with one malloc call. "

libfuzzer/FuzzerFork.cpp

@@ -220,7 +220,7 @@ struct GlobalEnv {
       }
     }
     // if (!FilesToAdd.empty() || Job->ExitCode != 0)
-    Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
+    Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd "
            "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n",
            NumRuns, Cov.size(), Features.size(), Files.size(),
            Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes,

libfuzzer/FuzzerIO.cpp

@@ -65,7 +65,7 @@ std::string FileToString(const std::string &Path) {
 }
 
 void CopyFileToErr(const std::string &Path) {
-  Printf("%s", FileToString(Path).c_str());
+  Puts(FileToString(Path).c_str());
 }
 
 void WriteToFile(const Unit &U, const std::string &Path) {
@@ -151,6 +151,11 @@ void CloseStdout() {
   DiscardOutput(1);
 }
 
+void Puts(const char *Str) {
+  fputs(Str, OutputFile);
+  fflush(OutputFile);
+}
+
 void Printf(const char *Fmt, ...) {
   va_list ap;
   va_start(ap, Fmt);

libfuzzer/FuzzerIO.h

@@ -58,6 +58,7 @@ void CloseStdout();
 FILE *GetOutputFile();
 void SetOutputFile(FILE *NewOutputFile);
 
+void Puts(const char *Str);
 void Printf(const char *Fmt, ...);
 void VPrintf(bool Verbose, const char *Fmt, ...);
 

libfuzzer/FuzzerInternal.h

@@ -29,12 +29,11 @@ namespace fuzzer {
 
 using namespace std::chrono;
 
-class Fuzzer {
+class Fuzzer final {
 public:
-
   Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
-         FuzzingOptions Options);
-  ~Fuzzer();
+         const FuzzingOptions &Options);
+  ~Fuzzer() = delete;
   void Loop(std::vector<SizedFile> &CorporaFiles);
   void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles);
   void MinimizeCrashLoop(const Unit &U);
@@ -91,6 +90,7 @@ class Fuzzer {
 
   void HandleMalloc(size_t Size);
   static void MaybeExitGracefully();
+  static int InterruptExitCode();
   std::string WriteToOutputCorpus(const Unit &U);
 
 private:

libfuzzer/FuzzerLoop.cpp

@@ -136,7 +136,7 @@ void Fuzzer::HandleMalloc(size_t Size) {
 }
 
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
-               FuzzingOptions Options)
+               const FuzzingOptions &Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
   if (EF->__sanitizer_set_death_callback)
     EF->__sanitizer_set_death_callback(StaticDeathCallback);
@@ -160,8 +160,6 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
   memset(BaseSha1, 0, sizeof(BaseSha1));
 }
 
-Fuzzer::~Fuzzer() {}
-
 void Fuzzer::AllocateCurrentUnitData() {
   if (CurrentUnitData || MaxInputLen == 0)
     return;
@@ -262,6 +260,11 @@ void Fuzzer::MaybeExitGracefully() {
   _Exit(0);
 }
 
+int Fuzzer::InterruptExitCode() {
+  assert(F);
+  return F->Options.InterruptExitCode;
+}
+
 void Fuzzer::InterruptCallback() {
   Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
   PrintFinalStats();
@@ -296,7 +299,7 @@ void Fuzzer::AlarmCallback() {
     Printf("       and the timeout value is %d (use -timeout=N to change)\n",
            Options.UnitTimeoutSec);
     DumpCurrentUnit("timeout-");
-    Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
+    Printf("==%lu== ERROR: libFuzzer: timeout after %zu seconds\n", GetPid(),
            Seconds);
     PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
@@ -309,9 +312,8 @@ void Fuzzer::RssLimitCallback() {
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
-  Printf(
-      "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
-      GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
+  Printf("==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %dMb)\n",
+         GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
   PrintMemoryProfile();
   DumpCurrentUnit("oom-");
@@ -366,7 +368,7 @@ void Fuzzer::PrintFinalStats() {
   Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns);
   Printf("stat::average_exec_per_sec:     %zd\n", ExecPerSec);
   Printf("stat::new_units_added:          %zd\n", NumberOfNewUnitsAdded);
-  Printf("stat::slowest_unit_time_sec:    %zd\n", TimeOfLongestUnitInSeconds);
+  Printf("stat::slowest_unit_time_sec:    %ld\n", TimeOfLongestUnitInSeconds);
   Printf("stat::peak_rss_mb:              %zd\n", GetPeakRSSMb());
 }
 
@@ -450,7 +452,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
       static_cast<long>(static_cast<double>(TimeOfLongestUnitInSeconds) * 1.1);
   if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) {
     TimeOfLongestUnitInSeconds = TimeOfUnit;
-    Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
+    Printf("Slowest unit: %ld s:\n", TimeOfLongestUnitInSeconds);
     WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
   }
 }
@@ -797,7 +799,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) {
     TotalSize += File.Size;
   }
   if (Options.MaxLen == 0)
-    SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen));
+    SetMaxInputLen(std::clamp(MaxSize, kMinDefaultLen, kMaxSaneLen));
   assert(MaxInputLen > 0);
 
   // Test the callback with empty input and never try it again.

libfuzzer/FuzzerMerge.cpp

@@ -77,6 +77,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
   size_t ExpectedStartMarker = 0;
   const size_t kInvalidStartMarker = -1;
   size_t LastSeenStartMarker = kInvalidStartMarker;
+  bool HaveFtMarker = true;
   std::vector<uint32_t> TmpFeatures;
   std::set<uint32_t> PCs;
   while (std::getline(IS, Line, '\n')) {
@@ -93,12 +94,13 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
       LastSeenStartMarker = ExpectedStartMarker;
       assert(ExpectedStartMarker < Files.size());
       ExpectedStartMarker++;
+      HaveFtMarker = false;
     } else if (Marker == "FT") {
       // FT FILE_ID COV1 COV2 COV3 ...
       size_t CurrentFileIdx = N;
       if (CurrentFileIdx != LastSeenStartMarker)
         return false;
-      LastSeenStartMarker = kInvalidStartMarker;
+      HaveFtMarker = true;
       if (ParseCoverage) {
         TmpFeatures.clear();  // use a vector from outer scope to avoid resizes.
         while (ISS1 >> N)
@@ -108,6 +110,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
       }
     } else if (Marker == "COV") {
       size_t CurrentFileIdx = N;
+      if (CurrentFileIdx != LastSeenStartMarker)
+        return false;
       if (ParseCoverage)
         while (ISS1 >> N)
           if (PCs.insert(N).second)
@@ -116,7 +120,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
       return false;
     }
   }
-  if (LastSeenStartMarker != kInvalidStartMarker)
+  if (!HaveFtMarker && LastSeenStartMarker != kInvalidStartMarker)
     LastFailure = Files[LastSeenStartMarker].Name;
 
   FirstNotProcessedFile = ExpectedStartMarker;

libfuzzer/FuzzerMutate.cpp

@@ -521,7 +521,7 @@ void MutationDispatcher::PrintMutationSequence(bool Verbose) {
 
 std::string MutationDispatcher::MutationSequence() {
   std::string MS;
-  for (auto M : CurrentMutatorSequence) {
+  for (const auto &M : CurrentMutatorSequence) {
     MS += M.Name;
     MS += "-";
   }

libfuzzer/FuzzerTracePC.cpp

@@ -149,8 +149,8 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) {
 ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) {
 #if defined(__mips__)
   return PC + 8;
-#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \
-    defined(__aarch64__)
+#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) ||        \
+    defined(__aarch64__) || defined(__loongarch__)
   return PC + 4;
 #else
   return PC + 1;