Run fuzzing as a travis cron job (#1662)

* Added fuzzing script to run all fuzzing targets on cron jobs.
* Added travis fuzzing script and `FUZZING_DURATION` to control fuzzing time.
* Given a total fuzzing duration, select 1 target to run for half of
  the duration, and divide the remaining duration over the rest of
  targets equally.

Author: Mina Farid

.travis.yml

@@ -178,6 +178,15 @@ jobs:
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
 
+    # Run fuzz tests only on cron jobs.
+    - stage: fuzz_test
+      env:
+        - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild
+      before_install:
+        - ./scripts/if_cron.sh ./scripts/install_prereqs.sh
+      script:
+        - ./scripts/if_cron.sh ./scripts/fuzzing_ci.sh
+
     # TODO(varconst): UBSan for CMake. UBSan failures are non-fatal by default,
     # need to make them fatal for the purposes of the test run.
 

Firestore/Example/Firestore.xcodeproj/xcshareddata/xcschemes/Firestore_FuzzTests_iOS.xcscheme

@@ -56,6 +56,11 @@
             value = "$(FUZZING_TARGET)"
             isEnabled = "YES">
          </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "FUZZING_DURATION"
+            value = "$(FUZZING_DURATION)"
+            isEnabled = "YES">
+         </EnvironmentVariable>
       </EnvironmentVariables>
       <AdditionalOptions>
       </AdditionalOptions>

Firestore/Example/FuzzTests/FSTFuzzTestsPrincipal.mm

@@ -14,24 +14,27 @@
  * limitations under the License.
  */
 
-#import <Foundation/NSObject.h>
-#include <string>
+#import <Foundation/Foundation.h>
 
-#include "LibFuzzer/FuzzerDefs.h"
+#include <cstdlib>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "Firestore/Example/FuzzTests/FuzzingTargets/FSTFuzzTestFieldPath.h"
 #include "Firestore/Example/FuzzTests/FuzzingTargets/FSTFuzzTestSerializer.h"
-
 #include "Firestore/core/src/firebase/firestore/util/log.h"
 #include "Firestore/core/src/firebase/firestore/util/string_apple.h"
+#include "LibFuzzer/FuzzerDefs.h"
+#include "absl/strings/str_join.h"
 
 namespace {
 
 using firebase::firestore::util::MakeString;
 namespace fuzzing = firebase::firestore::fuzzing;
 
-// A list of targets to fuzz test. Should be kept in sync with the method
-// GetFuzzingTarget().
+// A list of targets to fuzz test. Should be kept in sync with
+// GetFuzzingTarget() fuzzing_target_names map object.
 enum class FuzzingTarget { kNone, kSerializer, kFieldPath };
 
 // Directory to which crashing inputs are written. Must include the '/' at the
@@ -43,9 +46,14 @@
 // Default target is kNone if the environment variable is empty, not set, or
 // could not be interpreted. Should be kept in sync with FuzzingTarget.
 FuzzingTarget GetFuzzingTarget() {
-  char *fuzzing_target_env = std::getenv("FUZZING_TARGET");
+  std::unordered_map<std::string, FuzzingTarget> fuzzing_target_names;
+  fuzzing_target_names["NONE"] = FuzzingTarget::kNone;
+  fuzzing_target_names["SERIALIZER"] = FuzzingTarget::kSerializer;
+  fuzzing_target_names["FIELDPATH"] = FuzzingTarget::kFieldPath;
+
+  const char *fuzzing_target_env = std::getenv("FUZZING_TARGET");
 
-  if (fuzzing_target_env == nullptr) {
+  if (!fuzzing_target_env) {
     LOG_WARN("No value provided for FUZZING_TARGET environment variable.");
     return FuzzingTarget::kNone;
   }
@@ -56,19 +64,39 @@ FuzzingTarget GetFuzzingTarget() {
     LOG_WARN("No value provided for FUZZING_TARGET environment variable.");
     return FuzzingTarget::kNone;
   }
-  if (fuzzing_target == "NONE") {
-    return FuzzingTarget::kNone;
-  }
-  if (fuzzing_target == "SERIALIZER") {
-    return FuzzingTarget::kSerializer;
+
+  // Return the value of the fuzzing_target key if it exists in the
+  // fuzzing_target_names map.
+  if (fuzzing_target_names.find(fuzzing_target) != fuzzing_target_names.end()) {
+    return fuzzing_target_names[fuzzing_target];
   }
-  if (fuzzing_target == "FIELDPATH") {
-    return FuzzingTarget::kFieldPath;
+
+  // If the target is not found, print an error message with all available targets.
+  // The targets must be enclosed in curly brackets and separated by spaces. This format
+  // is needed by the script /firebase-ios-sdk/script/fuzzing_travis.sh, which parses
+  // this message to retrieve a list of the available targets.
+  std::vector<std::string> all_keys;
+  for (const auto &kv : fuzzing_target_names) {
+    all_keys.push_back(kv.first);
   }
-  LOG_WARN("Invalid fuzzing target: %s", fuzzing_target);
+  const std::string all_keys_str = absl::StrJoin(all_keys, " ");
+  LOG_WARN("Invalid fuzzing target: %s. Available targets: { %s }.", fuzzing_target, all_keys_str);
   return FuzzingTarget::kNone;
 }
 
+// Retrieves fuzzing duration from the FUZZING_DURATION environment variable.
+// Defaults to "0" if the environment variable is empty, which corresponds to
+// running indefinitely.
+std::string GetFuzzingDuration() {
+  const char *fuzzing_duration_env = std::getenv("FUZZING_DURATION");
+
+  if (!fuzzing_duration_env) {
+    return "0";
+  }
+
+  return std::string{fuzzing_duration_env};
+}
+
 // Simulates calling the main() function of libFuzzer (FuzzerMain.cpp).
 // Uses GetFuzzingTarget() to get the fuzzing target and sets libFuzzer's args
 // accordingly. It also calls an appropriate LLVMFuzzerTestOneInput-like method
@@ -121,13 +149,17 @@ int RunFuzzTestingMain() {
   // The directory in which libFuzzer writes crashing inputs.
   std::string prefix_arg = std::string("-artifact_prefix=") + MakeString(kCrashingInputsDirectory);
 
+  // Run fuzzing for the defined fuzzing duration.
+  std::string time_arg = "-max_total_time=" + GetFuzzingDuration();
+
   // Arguments to libFuzzer main() function should be added to this array,
   // e.g., dictionaries, corpus, number of runs, jobs, etc. The FuzzerDriver of
   // libFuzzer expects the non-const argument 'char ***argv' and it does not
   // modify it throughout the method.
   char *program_args[] = {
       const_cast<char *>("RunFuzzTestingMain"),  // First arg is program name.
       const_cast<char *>(prefix_arg.c_str()),    // Crashing inputs directory.
+      const_cast<char *>(time_arg.c_str()),      // Maximum total time.
       const_cast<char *>(dict_arg.c_str()),      // Dictionary arg.
       const_cast<char *>(corpus_arg.c_str())     // Corpus must be the last arg.
   };

scripts/fuzzing_ci.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+##
+# Copyright 2018 Google
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run fuzz testing using xcodebuild. Given a total fuzzing duration, select
+# a fuzzing target to run for half of the fuzzing duration. The remaining
+# duration is split equally over the rest of the fuzzing targets. Must be run
+# from the project root directory. The script is intended to execute on travis
+# cron jobs, but can run locally (on a macOS) from the project root.
+
+# Total time allowed for fuzzing in seconds (40 minutes for now).
+readonly ALLOWED_TIME=2400
+
+# An invalid target that is used to retrieve the list of available targets in
+# the returned error message.
+readonly INVALID_TARGET="INVALID_TARGET"
+# The NONE target that does not perform fuzzing. It is valid but useless.
+readonly NONE_TARGET="NONE"
+
+# Script exit codes.
+readonly EXIT_SUCCESS=0
+readonly EXIT_FAILURE=1
+
+# Get day of the year to use it when selecting a main target.
+readonly DOY="$(date +%j)"
+
+# Run fuzz testing only if executing on xcode >= 9 because fuzzing requires
+# Clang that supports -fsanitize-coverage=trace-pc-guard.
+xcode_version="$(xcodebuild -version | head -n 1)"
+xcode_version="${xcode_version/Xcode /}"
+xcode_major="${xcode_version/.*/}"
+
+if [[ "${xcode_major}" -lt 9 ]]; then
+  echo "Unsupported version of xcode."
+  exit ${EXIT_SUCCESS}
+fi
+
+# Helper to run fuzz tests using xcodebuild. Takes the fuzzing target as the
+# first argument and the fuzzing duration as the second argument.
+function run_xcode_fuzzing() {
+  xcodebuild \
+    -workspace 'Firestore/Example/Firestore.xcworkspace' \
+    -scheme 'Firestore_FuzzTests_iOS' \
+    -sdk 'iphonesimulator' \
+    -destination 'platform=iOS Simulator,name=iPhone 7' \
+    FUZZING_TARGET="$1" \
+    FUZZING_DURATION="$2" \
+    test
+}
+
+# Run fuzz testing with an INVALID_TARGET and grep the response message line
+# that contains the string 'Available targets: {'.
+fuzzing_targets_line="$(run_xcode_fuzzing ${INVALID_TARGET} "0" \
+    | grep "Available targets: {")"
+
+# The fuzzing_targets_line string contains { TARGET1 TARGET2 ... TARGETN }.
+# The following command extracts the targets and splits them into an array
+# in the variable all_fuzzing_targets.
+read -r -a all_fuzzing_targets <<< "$(echo ${fuzzing_targets_line} \
+    | cut -d "{" -f2 | cut -d "}" -f1)"
+
+# Remove the NONE target, if it exists.
+for i in "${!all_fuzzing_targets[@]}"; do
+  if [[ "${all_fuzzing_targets[${i}]}" == ${NONE_TARGET} ]]; then
+    unset all_fuzzing_targets[${i}]
+  fi
+done
+
+# Count all targets.
+readonly fuzzing_targets_count="${#all_fuzzing_targets[@]}"
+
+# Select a primary target to fuzz test for longer. Changes daily. The variable
+# todays_primary_target contains the index of the target selected as primary.
+todays_primary_target="$(( ${DOY} % ${fuzzing_targets_count} ))"
+
+# Spend half of allowed time on the selected primary target and half on the
+# remaining targets.
+primary_target_time="$(( ${ALLOWED_TIME} / 2 ))"
+other_targets_time="$(( ${ALLOWED_TIME} / 2 / $(( ${fuzzing_targets_count} - 1)) ))"
+
+script_return=${EXIT_SUCCESS}
+for i in "${!all_fuzzing_targets[@]}"; do
+  fuzzing_duration="${other_targets_time}"
+  if [[ "${i}" -eq "${todays_primary_target}" ]]; then
+    fuzzing_duration="${primary_target_time}"
+  fi
+  fuzzing_target="${all_fuzzing_targets[${i}]}"
+
+  echo "Running fuzzing target ${fuzzing_target} for ${fuzzing_duration} seconds..."
+  fuzzing_results="$(run_xcode_fuzzing "${fuzzing_target}" "${fuzzing_duration}")"
+  # Get the last line of the fuzzing results.
+  fuzzing_results_last_line="$(echo "${fuzzing_results}" | tail -n1)"
+  # If fuzzing succeeded, the last line will start with "Done"; otherwise,
+  # print all fuzzing results.
+  if [[ "${fuzzing_results_last_line}" == Done* ]]; then
+    echo "Fuzz testing for target ${fuzzing_target} succeeded. Ignore the ** TEST FAILED ** message."
+  else
+    echo "Error: Fuzz testing for target ${fuzzing_target} failed."
+    script_return=${EXIT_FAILURE}
+    echo "Fuzzing logs:"
+    echo "${fuzzing_results}"
+  fi
+done
+
+exit ${script_return}