More fuzztest cleanups

.github/workflows/fuzz.yml

@@ -0,0 +1,63 @@
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+
+on:
+  push:
+    branches:
+      - master
+      - 'test-ci/**'
+  pull_request:
+
+jobs:
+  fuzz:
+    if: ${{ !github.event.act }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzz_target: [
+roundtrip_miniscript_str,
+roundtrip_miniscript_script,
+parse_descriptor,
+roundtrip_semantic,
+parse_descriptor_secret,
+roundtrip_descriptor,
+roundtrip_concrete,
+compile_descriptor,
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58
+          override: true
+          profile: minimal
+      - name: fuzz
+        run: cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
+      - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: executed_${{ matrix.fuzz_target }}
+          path: executed_${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: ${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed

.github/workflows/rust.yml

@@ -3,25 +3,6 @@ on: [push, pull_request]
 name: Continuous integration
 
 jobs:
-  Fuzz:
-    name: Fuzz
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Crate
-        uses: actions/checkout@v2
-      - name: Install hongfuzz dependencies
-        run: sudo apt update && sudo apt install build-essential binutils-dev libunwind-dev libblocksruntime-dev liblzma-dev
-      - name: Checkout Toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.58.0
-          override: true
-      - name: Running fuzzer
-        env:
-          DO_FUZZ: true
-        run: ./contrib/test.sh
-
   Nightly:
     name: Nightly - Bench + Docs + Fmt
     runs-on: ubuntu-latest

contrib/test.sh

@@ -26,17 +26,6 @@ if cargo --version | grep "1\.47\.0"; then
     cargo update -p serde --precise 1.0.156
 fi
 
-# Fuzz if told to
-if [ "$DO_FUZZ" = true ]
-then
-    cd fuzz
-    cargo test --verbose
-    ./travis-fuzz.sh
-
-    # Exit out of the fuzzer, do not run other tests.
-    exit 0
-fi
-
 # Test bitcoind integration tests if told to (this only works with the stable toolchain)
 if [ "$DO_BITCOIND_TESTS" = true ]; then
     cd bitcoind-tests

fuzz/Cargo.toml

@@ -1,50 +1,47 @@
 [package]
 name = "descriptor-fuzz"
+edition = "2018"
 version = "0.0.1"
-authors = ["Automatically generated"]
+authors = ["Generated by fuzz/generate-files.sh"]
 publish = false
 
 [package.metadata]
 cargo-fuzz = true
 
-[features]
-afl_fuzz = ["afl"]
-honggfuzz_fuzz = ["honggfuzz"]
-
 [dependencies]
-honggfuzz = { version = "0.5.55", default-features = false, optional = true }
-afl = { version = "0.8", optional = true }
-regex = { version = "1.4"}
-miniscript = { path = "..", features = ["compiler"] }
+honggfuzz = { version = "0.5.55", default-features = false }
+miniscript = { path = "..", features = [ "compiler" ] }
+
+regex = "1.4"
 
 [[bin]]
-name = "roundtrip_descriptor"
-path = "fuzz_targets/roundtrip_descriptor.rs"
+name = "roundtrip_miniscript_str"
+path = "fuzz_targets/roundtrip_miniscript_str.rs"
 
 [[bin]]
 name = "roundtrip_miniscript_script"
 path = "fuzz_targets/roundtrip_miniscript_script.rs"
 
 [[bin]]
-name = "roundtrip_miniscript_str"
-path = "fuzz_targets/roundtrip_miniscript_str.rs"
-
-[[bin]]
-name = "roundtrip_concrete"
-path = "fuzz_targets/roundtrip_concrete.rs"
+name = "parse_descriptor"
+path = "fuzz_targets/parse_descriptor.rs"
 
 [[bin]]
 name = "roundtrip_semantic"
 path = "fuzz_targets/roundtrip_semantic.rs"
 
 [[bin]]
-name = "compile_descriptor"
-path = "fuzz_targets/compile_descriptor.rs"
+name = "parse_descriptor_secret"
+path = "fuzz_targets/parse_descriptor_secret.rs"
 
 [[bin]]
-name = "parse_descriptor"
-path = "fuzz_targets/parse_descriptor.rs"
+name = "roundtrip_descriptor"
+path = "fuzz_targets/roundtrip_descriptor.rs"
 
 [[bin]]
-name = "parse_descriptor_secret"
-path = "fuzz_targets/parse_descriptor_secret.rs"
+name = "roundtrip_concrete"
+path = "fuzz_targets/roundtrip_concrete.rs"
+
+[[bin]]
+name = "compile_descriptor"
+path = "fuzz_targets/compile_descriptor.rs"

fuzz/README.md

@@ -1,13 +1,119 @@
-# Fuzz Tests
+# Fuzzing
 
-Repository for fuzz testing Miniscript.
+`miniscript` has a fuzzing harness setup for use with honggfuzz.
 
-## How to reproduce crashes?
+To run the fuzz-tests as in CI -- briefly fuzzing every target -- simply
+run
 
-Travis should output a offending hex("048531e80700ae6400670000af5168" in the example)
-which you can use as shown. Copy and paste the following code lines into file reporting crashes and
-replace the hex with the offending hex.
-Refer to file [roundtrip_concrete.rs](./fuzz_targets/roundtrip_concrete.rs) for an example.
+    ./fuzz.sh
+
+in this directory.
+
+To build honggfuzz, you must have libunwind on your system, as well as
+libopcodes and libbfd from binutils **2.38** on your system. The most
+recently-released binutils 2.39 has changed their API in a breaking way.
+
+On Nix, you can obtain these libraries by running
+
+    nix-shell -p libopcodes_2_38 -p libunwind
+
+and then run fuzz.sh as above.
+
+# Fuzzing with weak cryptography
+
+You may wish to replace the hashing and signing code with broken crypto,
+which will be faster and enable the fuzzer to do otherwise impossible
+things such as forging signatures or finding preimages to hashes.
+
+Doing so may result in spurious bug reports since the broken crypto does
+not respect the encoding or algebraic invariants upheld by the real crypto. We
+would like to improve this but it's a nontrivial problem -- though not
+beyond the abilities of a motivated student with a few months of time.
+Please let us know if you are interested in taking this on!
+
+Meanwhile, to use the broken crypto, simply compile (and run the fuzzing
+scripts) with
+
+    RUSTFLAGS="--cfg=hashes_fuzz --cfg=secp256k1_fuzz"
+
+which will replace the hashing library with broken hashes, and the
+secp256k1 library with broken cryptography.
+
+Needless to say, NEVER COMPILE REAL CODE WITH THESE FLAGS because if a
+fuzzer can break your crypto, so can anybody.
+
+# Long-term fuzzing
+
+To see the full list of targets, the most straightforward way is to run
+
+    source ./fuzz-util.sh
+    listTargetNames
+
+To run each of them for an hour, run
+
+    ./cycle.sh
+
+To run a single fuzztest indefinitely, run
+
+    cargo hfuzz run <target>
+
+`cycle.sh` uses the `chrt` utility to try to reduce the priority of the
+jobs. If you would like to run for longer, the most straightforward way
+is to edit `cycle.sh` before starting. To run the fuzz-tests in parallel,
+you will need to implement a custom harness.
+
+# Adding fuzz tests
+
+All fuzz tests can be found in the `fuzz_target/` directory. Adding a new
+one is as simple as copying an existing one and editing the `do_test`
+function to do what you want.
+
+If your test clearly belongs to a specific crate, please put it in that
+crate's directory. Otherwise you can put it directly in `fuzz_target/`.
+
+If you need to add dependencies, edit the file `generate-files.sh` to add
+it to the generated `Cargo.toml`.
+
+Once you've added a fuzztest, regenerate the `Cargo.toml` and CI job by
+running
+
+    ./generate-files.sh
+
+Then to test your fuzztest, run
+
+    ./fuzz.sh <target>
+
+If it is working, you will see a rapid stream of data for many seconds
+(you can hit Ctrl+C to stop it early). If not, you should quickly see
+an error.
+
+# Reproducing Failures
+
+If a fuzztest fails, it will exit with a summary which looks something like
+
+```
+...
+ fuzzTarget      : hfuzz_target/x86_64-unknown-linux-gnu/release/hashes_sha256 
+CRASH:
+DESCRIPTION: 
+ORIG_FNAME: 00000000000000000000000000000000.00000000.honggfuzz.cov
+FUZZ_FNAME: hfuzz_workspace/hashes_sha256/SIGABRT.PC.7ffff7c8abc7.STACK.18826d9b64.CODE.-6.ADDR.0.INSTR.mov____%eax,%ebp.fuzz
+...
+=====================================================================
+fff400610004
+```
+
+The final line is a hex-encoded version of the input that caused the crash. You
+can test this directly by editing the `duplicate_crash` test to copy/paste the
+hex output into the call to `extend_vec_from_hex`. Then run the test with
+
+    cargo test
+
+Note that if you set your `RUSTFLAGS` while fuzzing (see above) you must make
+sure they are set the same way when running `cargo test`.
+
+If the `duplicate_crash` function is not present, please add it. A template is
+as follows:
 
 ```
 #[cfg(test)]

fuzz/cycle.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Continuosly cycle over fuzz targets running each for 1 hour.
+# It uses chrt SCHED_IDLE so that other process takes priority.
+#
+# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
+
+set -e
+REPO_DIR=$(git rev-parse --show-toplevel)
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+while :
+do
+  for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    echo "Fuzzing target $targetName ($targetFile)"
+
+    # fuzz for one hour
+    HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
+    # minimize the corpus
+    HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
+  done
+done
+

fuzz/fuzz-util.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
+  find fuzz_targets/ -type f -name "*.rs"
+  popd > /dev/null || exit 1
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+targetFileToHFuzzInputArg() {
+  baseName=$(basename "$1")
+  dirName="${baseName%.*}"
+  if [ -d "hfuzz_input/$dirName" ]; then
+    echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
+  fi
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+# Utility function to avoid CI failures on Windows
+checkWindowsFiles() {
+  incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
+  if [ "$incorrectFilenames" -gt 0 ]; then
+    echo "Bailing early because there is a Windows-incompatible filename in the tree."
+    exit 2
+  fi
+}
+
+# Checks whether a fuzz case output some report, and dumps it in hex
+checkReport() {
+  reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
+  if [ -f "$reportFile" ]; then
+    cat "$reportFile"
+    for CASE in "hfuzz_workspace/$1/SIG"*; do
+      xxd -p -c10000 < "$CASE"
+    done
+    exit 1
+  fi
+}