fuzz: copy *.sh files from rust-bitcoin; tweak generate-files.sh

Author: apoelstra

fuzz/cycle.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Continuosly cycle over fuzz targets running each for 1 hour.
+# It uses chrt SCHED_IDLE so that other process takes priority.
+#
+# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
+
+set -e
+REPO_DIR=$(git rev-parse --show-toplevel)
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+while :
+do
+  for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    echo "Fuzzing target $targetName ($targetFile)"
+
+    # fuzz for one hour
+    HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
+    # minimize the corpus
+    HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
+  done
+done
+

fuzz/fuzz-util.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
+  find fuzz_targets/ -type f -name "*.rs"
+  popd > /dev/null || exit 1
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+targetFileToHFuzzInputArg() {
+  baseName=$(basename "$1")
+  dirName="${baseName%.*}"
+  if [ -d "hfuzz_input/$dirName" ]; then
+    echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
+  fi
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+# Utility function to avoid CI failures on Windows
+checkWindowsFiles() {
+  incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
+  if [ "$incorrectFilenames" -gt 0 ]; then
+    echo "Bailing early because there is a Windows-incompatible filename in the tree."
+    exit 2
+  fi
+}
+
+# Checks whether a fuzz case output some report, and dumps it in hex
+checkReport() {
+  reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
+  if [ -f "$reportFile" ]; then
+    cat "$reportFile"
+    for CASE in "hfuzz_workspace/$1/SIG"*; do
+      xxd -p -c10000 < "$CASE"
+    done
+    exit 1
+  fi
+}

fuzz/fuzz.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# Check that input files are correct Windows file names
+checkWindowsFiles
+
+if [ "$1" == "" ]; then
+  targetFiles="$(listTargetFiles)"
+else
+  targetFiles=fuzz_targets/"$1".rs
+fi
+
+cargo --version
+rustc --version
+
+# Testing
+cargo install --force honggfuzz --no-default-features
+for targetFile in $targetFiles; do
+  targetName=$(targetFileToName "$targetFile")
+  echo "Fuzzing target $targetName ($targetFile)"
+  if [ -d "hfuzz_input/$targetName" ]; then
+    HFUZZ_INPUT_ARGS="-f hfuzz_input/$targetName/input\""
+  else
+    HFUZZ_INPUT_ARGS=""
+  fi
+  HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run "$targetName"
+
+  checkReport "$targetName"
+done

fuzz/generate-files.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+set -e
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# 1. Generate fuzz/Cargo.toml
+cat > "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+[package]
+name = "descriptor-fuzz"
+edition = "2018"
+version = "0.0.1"
+authors = ["Generated by fuzz/generate-files.sh"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+honggfuzz = { version = "0.5.55", default-features = false }
+miniscript = { path = "..", features = [ "compiler" ] }
+
+regex = "1.4"
+EOF
+
+for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    cat >> "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+
+[[bin]]
+name = "$targetName"
+path = "$targetFile"
+EOF
+done
+
+# 2. Generate .github/workflows/fuzz.yml
+cat > "$REPO_DIR/.github/workflows/fuzz.yml" <<EOF
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+
+on:
+  push:
+    branches:
+      - master
+      - 'test-ci/**'
+  pull_request:
+
+jobs:
+  fuzz:
+    if: \${{ !github.event.act }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzz_target: [
+$(for name in $(listTargetNames); do echo "$name,"; done)
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-\${{ matrix.target }}-\${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58
+          override: true
+          profile: minimal
+      - name: fuzz
+        run: cd fuzz && ./fuzz.sh "\${{ matrix.fuzz_target }}"
+      - run: echo "\${{ matrix.fuzz_target }}" >executed_\${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: executed_\${{ matrix.fuzz_target }}
+          path: executed_\${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: \${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed
+EOF
+