Add minreq usage

.github/workflows/fuzz.yml

@@ -0,0 +1,58 @@
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+
+on:
+  push:
+    branches:
+      - master
+      - 'test-ci/**'
+  pull_request:
+
+jobs:
+  fuzz:
+    if: ${{ !github.event.act }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzz_target: [
+simple_http,
+minreq_http,
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58
+          override: true
+          profile: minimal
+      - name: fuzz
+        run: if [[ "${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'; fi
+        run: cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
+      - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: executed_${{ matrix.fuzz_target }}
+          path: executed_${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: ${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed

Cargo.toml

@@ -19,6 +19,8 @@ rustdoc-args = ["--cfg", "docsrs"]
 default = [ "simple_http", "simple_tcp" ]
 # A bare-minimum HTTP transport.
 simple_http = [ "base64" ]
+# A transport that uses `minreq` as the HTTP client.
+minreq_http = [ "base64", "minreq" ]
 # Basic transport over a raw TcpListener
 simple_tcp = []
 # Basic transport over a raw UnixStream
@@ -32,7 +34,8 @@ serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = [ "raw_value" ] }
 
 base64 = { version = "0.13.0", optional = true }
+minreq = { version = "2.7.0", features = ["json-using-serde"], optional = true }
 socks = { version = "0.3.4", optional = true}
 
 [workspace]
-members = ["fuzz"]
+members = ["fuzz", "integration_test"]

fuzz/Cargo.toml

@@ -1,20 +1,21 @@
 [package]
 name = "jsonrpc-fuzz"
+edition = "2018"
 version = "0.0.1"
-authors = ["Automatically generated"]
+authors = ["Generated by fuzz/generate-files.sh"]
 publish = false
 
 [package.metadata]
 cargo-fuzz = true
 
-[features]
-honggfuzz_fuzz = ["honggfuzz"]
-
 [dependencies]
-honggfuzz = { version = "0.5", optional = true, default-features = false }
-jsonrpc = { path = ".." }
+honggfuzz = { version = "0.5.55", default-features = false }
+jsonrpc = { path = "..", features = ["minreq_http"] }
 
 [[bin]]
 name = "simple_http"
 path = "fuzz_targets/simple_http.rs"
 
+[[bin]]
+name = "minreq_http"
+path = "fuzz_targets/minreq_http.rs"

fuzz/cycle.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Continuosly cycle over fuzz targets running each for 1 hour.
+# It uses chrt SCHED_IDLE so that other process takes priority.
+#
+# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
+
+set -e
+REPO_DIR=$(git rev-parse --show-toplevel)
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+while :
+do
+  for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    echo "Fuzzing target $targetName ($targetFile)"
+
+    # fuzz for one hour
+    HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
+    # minimize the corpus
+    HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
+  done
+done

fuzz/fuzz-util.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
+  find fuzz_targets/ -type f -name "*.rs"
+  popd > /dev/null || exit 1
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+targetFileToHFuzzInputArg() {
+  baseName=$(basename "$1")
+  dirName="${baseName%.*}"
+  if [ -d "hfuzz_input/$dirName" ]; then
+    echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
+  fi
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+# Utility function to avoid CI failures on Windows
+checkWindowsFiles() {
+  incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
+  if [ "$incorrectFilenames" -gt 0 ]; then
+    echo "Bailing early because there is a Windows-incompatible filename in the tree."
+    exit 2
+  fi
+}
+
+# Checks whether a fuzz case output some report, and dumps it in hex
+checkReport() {
+  reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
+  if [ -f "$reportFile" ]; then
+    cat "$reportFile"
+    for CASE in "hfuzz_workspace/$1/SIG"*; do
+      xxd -p -c10000 < "$CASE"
+    done
+    exit 1
+  fi
+}

fuzz/fuzz.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# Check that input files are correct Windows file names
+checkWindowsFiles
+
+if [ "$1" == "" ]; then
+  targetFiles="$(listTargetFiles)"
+else
+  targetFiles=fuzz_targets/"$1".rs
+fi
+
+cargo --version
+rustc --version
+
+# Testing
+cargo install --force honggfuzz --no-default-features
+for targetFile in $targetFiles; do
+  targetName=$(targetFileToName "$targetFile")
+  echo "Fuzzing target $targetName ($targetFile)"
+  if [ -d "hfuzz_input/$targetName" ]; then
+    HFUZZ_INPUT_ARGS="-f hfuzz_input/$targetName/input\""
+  else
+    HFUZZ_INPUT_ARGS=""
+  fi
+  RUSTFLAGS="--cfg=jsonrpc_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run "$targetName"
+
+  checkReport "$targetName"
+done

fuzz/fuzz_targets/minreq_http.rs

@@ -0,0 +1,62 @@
+extern crate jsonrpc;
+
+// Note, tests are empty if "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`.
+
+#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set.
+fn do_test(data: &[u8]) {
+    #[cfg(jsonrpc_fuzz)]
+    {
+        use std::io;
+
+        use jsonrpc::minreq_http::MinreqHttpTransport;
+        use jsonrpc::minreq_http::FUZZ_TCP_SOCK;
+        use jsonrpc::Client;
+
+        *FUZZ_TCP_SOCK.lock().unwrap() = Some(io::Cursor::new(data.to_vec()));
+
+        let t = MinreqHttpTransport::builder()
+            .url("localhost:123")
+            .expect("parse url")
+            .basic_auth("".to_string(), None)
+            .build();
+
+        let client = Client::with_transport(t);
+        let request = client.build_request("uptime", &[]);
+        let _ = client.send_request(request);
+    }
+}
+
+fn main() {
+    loop {
+        honggfuzz::fuzz!(|data| {
+            do_test(data);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn extend_vec_from_hex(hex: &str) -> Vec<u8> {
+        let mut out = vec![];
+        let mut b = 0;
+        for (idx, c) in hex.as_bytes().iter().enumerate() {
+            b <<= 4;
+            match *c {
+                b'A'..=b'F' => b |= c - b'A' + 10,
+                b'a'..=b'f' => b |= c - b'a' + 10,
+                b'0'..=b'9' => b |= c - b'0',
+                _ => panic!("Bad hex"),
+            }
+            if (idx & 1) == 1 {
+                out.push(b);
+                b = 0;
+            }
+        }
+        out
+    }
+
+    #[test]
+    fn duplicate_crash() {
+        super::do_test(&extend_vec_from_hex("00"));
+    }
+}

fuzz/fuzz_targets/simple_http.rs

@@ -1,10 +1,10 @@
 extern crate jsonrpc;
 
-// Note, tests are empty "fuzzing" is not set but still show up in output of `cargo test --workspace`.
+// Note, tests are if empty "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`.
 
-#[allow(unused_variables)] // `data` is not used when "fuzzing" is not set.
+#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set.
 fn do_test(data: &[u8]) {
-    #[cfg(fuzzing)]
+    #[cfg(jsonrpc_fuzz)]
     {
         use std::io;
 
@@ -26,7 +26,6 @@ fn do_test(data: &[u8]) {
     }
 }
 
-#[cfg(feature = "honggfuzz")]
 fn main() {
     loop {
         honggfuzz::fuzz!(|data| {

fuzz/generate-files.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+set -e
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# 1. Generate fuzz/Cargo.toml
+cat > "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+[package]
+name = "jsonrpc-fuzz"
+edition = "2018"
+version = "0.0.1"
+authors = ["Generated by fuzz/generate-files.sh"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+honggfuzz = { version = "0.5.55", default-features = false }
+jsonrpc = { path = "..", features = ["minreq_http"] }
+EOF
+
+for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    cat >> "$REPO_DIR/fuzz/Cargo.toml" <<EOF
+
+[[bin]]
+name = "$targetName"
+path = "$targetFile"
+EOF
+done
+
+# 2. Generate .github/workflows/fuzz.yml
+cat > "$REPO_DIR/.github/workflows/fuzz.yml" <<EOF
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+
+on:
+  push:
+    branches:
+      - master
+      - 'test-ci/**'
+  pull_request:
+
+jobs:
+  fuzz:
+    if: \${{ !github.event.act }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzz_target: [
+$(for name in $(listTargetNames); do echo "$name,"; done)
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-\${{ matrix.target }}-\${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58
+          override: true
+          profile: minimal
+      - name: fuzz
+        run: if [[ "\${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'; fi
+        run: cd fuzz && ./fuzz.sh "\${{ matrix.fuzz_target }}"
+      - run: echo "\${{ matrix.fuzz_target }}" >executed_\${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: executed_\${{ matrix.fuzz_target }}
+          path: executed_\${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: \${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed
+EOF