Initial implementation of a record-level bitcode fuzzer.

Initial implementation of a record-level bitcod fuzzer for PNaCl
bitcode files. Uses "rand" for the random number generator, as well as
some simplistic rules for generating values for the bitcode records.

The simplistic rules use lists of weighted values, and values are
chosen from the list based on the corresponding weighted distribution.

Also fixes minor bug in the bitstream writer where no checks were
applied to see if an abbreviation definition was outside any scope.

BUG=https://code.google.com/p/nativeclient/issues/detail?id=4169
[email protected]

Review URL: https://codereview.chromium.org/1156103003

Author: KarlSchimpf

include/llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h

@@ -238,7 +238,9 @@ class NaClMungedBitcode {
 
   bool write(SmallVectorImpl<char> &Buffer, bool AddHeader,
              const WriteFlags &Flags) const {
-    return writeMaybeRepair(Buffer, AddHeader, Flags).NumErrors == 0;
+    WriteResults Results = writeMaybeRepair(Buffer, AddHeader, Flags);
+    return Results.NumErrors == 0
+        || (Flags.getTryToRecover() && Results.NumErrors == Results.NumRepairs);
   }
 
   bool write(SmallVectorImpl<char> &Buffer, bool AddHeader) const {

include/llvm/Bitcode/NaCl/NaClFuzz.h

@@ -0,0 +1,94 @@
+//===- NaClFuzz.h - Fuzz PNaCl bitcode records ------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a basic fuzzer for a list of PNaCl bitcode records.
+//
+// *** WARNING *** The implementation of the fuzzer uses a random
+// number generator.  As a result, this code is not thread safe.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_BITCODE_NACL_NACLFUZZ_H
+#define LLVM_BITCODE_NACL_NACLFUZZ_H
+
+#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
+#include "llvm/Bitcode/NaCl/NaClRandNumGen.h"
+
+namespace naclfuzz {
+
+using namespace llvm;
+
+/// \brief Fuzzes a list of editable bitcode records.
+class RecordFuzzer {
+  RecordFuzzer(const RecordFuzzer&) = delete;
+  void operator=(const RecordFuzzer&) = delete;
+public:
+  typedef NaClMungedBitcode::iterator iterator;
+
+  /// \brief The set of possible fuzzing actions.
+  enum EditAction {
+    /// \brief Inserts a new record into the list of bitcode records.
+    InsertRecord,
+    /// \brief Mutate contents of an existing bitcode record.
+    MutateRecord,
+    /// \brief Removes an existing record from the list of bitcode
+    /// records.
+    RemoveRecord,
+    /// \brief Replaces an existing record with a new bitcode record.
+    ReplaceRecord,
+    /// \brief Swaps two records in the bitcode record list.
+    SwapRecord
+  };
+
+  virtual ~RecordFuzzer();
+
+  /// \brief Generates a random mutation of the bitcode, using the
+  /// provided random number generator. Percentage (a value between 0
+  /// and 1 defined by Count/Base) is used to define the number of
+  /// fuzzing actions applied to the bitcode.  Returns true if fuzzing
+  /// succeeded.
+  ///
+  /// May be called an arbitrary number of times. Results are left in
+  /// the munged bitcode records passed into static method
+  /// createSimpleRecordFuzzer.
+  virtual bool fuzz(unsigned Count, unsigned Base=100) = 0;
+
+  /// \brief Shows how many times each record was edited in the
+  /// corresponding (input) bitcode, over all calls to fuzz.
+  virtual void showRecordDistribution(raw_ostream &Out) const = 0;
+
+  /// \brief Shows how many times each type of edit action was applied
+  /// to the corresponding bitcode, over all calls to fuzz.
+  virtual void showEditDistribution(raw_ostream &Out) const = 0;
+
+  // Creates an instance of a fuzzer for the given bitcode.
+  static RecordFuzzer
+  *createSimpleRecordFuzzer(NaClMungedBitcode &Bitcode,
+                            RandomNumberGenerator &RandGenerator);
+
+  /// Returns printable name for the edit action.
+  static const char *actionName(EditAction Action);
+
+protected:
+  RecordFuzzer(NaClMungedBitcode &Bitcode, RandomNumberGenerator &Generator);
+
+  // Holds the bitcode being munged.
+  NaClMungedBitcode &Bitcode;
+
+  // Hold the random number generator.
+  RandomNumberGenerator &Generator;
+
+  // Erases the last fuzzing result from the munged bitcode records
+  // in Bitcode.
+  virtual void clear();
+};
+
+} // end of namespace naclfuzz
+
+#endif // LLVM_BITCODE_NACL_NACLFUZZ_H

include/llvm/Bitcode/NaCl/NaClRandNumGen.h

@@ -0,0 +1,64 @@
+//===- NaClRandNumGen.h - random number generator ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a random number generator API for 64-bit unsigned
+// values, and a corresponding default implementation.
+//
+// *** WARNING *** One should assume that random number generators are not
+// thread safe.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_BITCODE_NACL_NACLRANDNUMGEN_H
+#define LLVM_BITCODE_NACL_NACLRANDNUMGEN_H
+
+#include "llvm/ADT/StringRef.h"
+#include <random>
+
+namespace naclfuzz {
+
+/// Defines API for a random number generator to use with fuzzing.
+class RandomNumberGenerator {
+  RandomNumberGenerator(const RandomNumberGenerator&) = delete;
+  void operator=(const RandomNumberGenerator&) = delete;
+public:
+  virtual ~RandomNumberGenerator();
+  /// Returns a random number.
+  virtual uint64_t operator()() = 0;
+  // Returns a random value in [0..Limit)
+  uint64_t chooseInRange(uint64_t Limit) {
+    return (*this)() % Limit;
+  }
+protected:
+  RandomNumberGenerator() {}
+};
+
+/// Defines a random number generator based on C++ generator std::mt19937_64.
+class DefaultRandomNumberGenerator : public RandomNumberGenerator {
+  DefaultRandomNumberGenerator(const DefaultRandomNumberGenerator&) = delete;
+  void operator=(const DefaultRandomNumberGenerator&) = delete;
+public:
+  DefaultRandomNumberGenerator(llvm::StringRef Seed);
+  uint64_t operator()() final;
+  ~DefaultRandomNumberGenerator() final {}
+  // Resets random number seed by salting the seed of constructor with Salt.
+  void saltSeed(uint64_t Salt);
+private:
+  // 64-bit Mersenne Twister by Matsumoto and Nishimura, 2000
+  // http://en.cppreference.com/w/cpp/numeric/random/mersenne_twister_engine
+  // This RNG is deterministically portable across C++11
+  // implementations.
+  std::mt19937_64 Generator;
+  // Seed for the random number generator.
+  std::string Seed;
+};
+
+} // end of namespace naclfuzz
+
+#endif // LLVM_BITCODE_NACL_NACLRANDNUMGEN_H

lib/Bitcode/NaCl/TestUtils/CMakeLists.txt

@@ -3,5 +3,8 @@ add_llvm_library(LLVMNaClBitTestUtils
   NaClBitcodeMungeReader.cpp
   NaClBitcodeMungeUtils.cpp
   NaClBitcodeMungeWriter.cpp
+  NaClFuzz.cpp
+  NaClRandNumGen.cpp
+  NaClSimpleRecordFuzzer.cpp
   )
 add_dependencies(LLVMNaClBitTestUtils intrinsics_gen)

lib/Bitcode/NaCl/TestUtils/NaClFuzz.cpp

@@ -0,0 +1,53 @@
+//===- NaClFuzz.cpp - Fuzz PNaCl bitcode records --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a basic fuzzer for a list of PNaCl bitcode records.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Bitcode/NaCl/NaClFuzz.h"
+
+using namespace llvm;
+
+namespace {
+
+// Names for edit actions.
+const char *ActionNameArray[] = {
+  "Insert",
+  "Mutate",
+  "Remove",
+  "Replace",
+  "Swap"
+};
+
+} // end of anonymous namespace
+
+namespace naclfuzz {
+
+const char *RecordFuzzer::actionName(EditAction Action) {
+  return Action < array_lengthof(ActionNameArray)
+                  ? ActionNameArray[Action] : "???";
+}
+
+RecordFuzzer::RecordFuzzer(NaClMungedBitcode &Bitcode,
+                           RandomNumberGenerator &Generator)
+    : Bitcode(Bitcode), Generator(Generator) {
+  if (Bitcode.getBaseRecords().empty())
+    report_fatal_error(
+        "Sorry, the fuzzer doesn't know how to fuzz an empty record list");
+}
+
+RecordFuzzer::~RecordFuzzer() {}
+
+void RecordFuzzer::clear() {
+  Bitcode.removeEdits();
+}
+
+} // end of namespace naclfuzz

lib/Bitcode/NaCl/TestUtils/NaClRandNumGen.cpp

@@ -0,0 +1,44 @@
+//===- NaClRandNumGen.cpp - Random number generator -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides a default implementation of a random number generator.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitcode/NaCl/NaClRandNumGen.h"
+
+#include <vector>
+#include <algorithm>
+
+namespace naclfuzz {
+
+// Put destructor in .cpp file guarantee vtable construction.
+RandomNumberGenerator::~RandomNumberGenerator() {}
+
+DefaultRandomNumberGenerator::DefaultRandomNumberGenerator(llvm::StringRef Seed)
+    : Seed(Seed) {
+  saltSeed(0);
+}
+
+uint64_t DefaultRandomNumberGenerator::operator()() {
+  return Generator();
+}
+
+void DefaultRandomNumberGenerator::saltSeed(uint64_t Salt) {
+  // Combine seed and salt and pass to generator.
+  std::vector<uint32_t> Data;
+  Data.reserve(Seed.size() + 2);
+  Data.push_back(static_cast<uint32_t>(Salt));
+  Data.push_back(static_cast<uint32_t>(Salt >> 32));
+  std::copy(Seed.begin(), Seed.end(), Data.end());
+  std::seed_seq SeedSeq(Data.begin(), Data.end());
+  Generator.seed(SeedSeq);
+}
+
+} // end of namespace naclfuzz