Update github workflow to upload test coverage reports (#759)

* update github workflow to upload coverage reports * avoid using xtask * use relative path for codecov CI workflow * improve relative paths * update relative paths * update path * use coverage for fuzz * update lcov path * use release flag for fuzz tests and grcov * use grcov github action * use grcov config * use grcov github action test suggestion * revert to source based code coverage * create tmp directory * use release for fuzz grcov * update codecov lcov file path * use tmp runner folder for coverage data * correct runner path in gh actions * use github actions default env var RUNNER_TEMP * gh action tmp folder and fail_ci_if_error true * correct file path * add own job for test and coverage upload * optimise github actions * change github workflow * fix github actions workflow * provide flag for github action codecov * use debug profile for tests for better cov reports * add fuzz coverage * remove fuzz coverage * add RUST_BACKTRACE to cargo fuzz * update cargo test comment
zeitgeistpm · Jan 6, 2023 · 85cefa6 · 85cefa6
1 parent f476780
commit 85cefa6
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 19 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -27,18 +27,20 @@ jobs:
       - uses: actions-rs/[email protected]
         with:
           crate: taplo-cli
+
+      - name: Cache Dependencies
+        uses: Swatinem/rust-cache@v1
 
       - name: Format
         run: ./scripts/tests/format.sh --check
 
-  tests:
-    name: Tests
+  checks:
+    name: Checks
     runs-on: ubuntu-latest
     strategy:
       matrix:
         scripts: [
           "clippy",
-          "misc",
           "parachain",
           "standalone"
         ]
@@ -52,7 +54,7 @@ jobs:
       - name: Cache Dependencies
         uses: Swatinem/rust-cache@v1
 
-      - name: Tests
+      - name: Checks
         run: ./scripts/tests/${{ matrix.scripts }}.sh
   benchmark:
     name: Quick check benchmarks
@@ -63,8 +65,46 @@ jobs:
 
       - name: Install rust toolchain
         run: rustup show
-
+      
       - run: ./scripts/benchmarks/quick_check.sh
+
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          target: wasm32-unknown-unknown
+          default: true
+          override: true
+          profile: minimal
+          components: rustfmt, clippy, llvm-tools-preview
+
+      - uses: actions-rs/[email protected]
+        with:
+          crate: grcov
+          use-tool-cache: true
+
+      - name: Cache Dependencies
+        uses: Swatinem/rust-cache@v1
+
+      - name: Tests
+        run: ./scripts/tests/misc.sh
+
+      - name: Upload to codecov.io
+        uses: codecov/codecov-action@v3
+        with:
+          files: ${{ runner.temp }}/zeitgeist-test-coverage.lcov
+          fail_ci_if_error: true
+          flags: tests
+          verbose: true
+          name: unit-tests
+
   fuzz:
     name: Fuzz
     runs-on: ubuntu-latest
@@ -80,4 +120,7 @@ jobs:
           crate: cargo-fuzz
           use-tool-cache: true
 
+      - name: Cache Dependencies
+        uses: Swatinem/rust-cache@v1
+
       - run: ./scripts/tests/fuzz.sh
diff --git a/scripts/tests/aux-functions.sh b/scripts/tests/aux-functions.sh
@@ -15,5 +15,6 @@ test_package_with_feature() {
     local features=$2
 
     /bin/echo -e "\e[0;33m***** Testing '$package' with features '$features' *****\e[0m\n"
-    cargo test --features $features --manifest-path $package/Cargo.toml --no-default-features --release
+    # default rustc profile dev (debug) is used to stop for debug_assertions
+    CARGO_INCREMENTAL=0 RUSTFLAGS="-Cinstrument-coverage" LLVM_PROFILE_FILE="cargo-test-%p-%m.profraw" cargo test --features $features --manifest-path $package/Cargo.toml --no-default-features
 }
diff --git a/scripts/tests/fuzz.sh b/scripts/tests/fuzz.sh
@@ -33,24 +33,24 @@ RIKIDDO_WITH_CALCULATED_FEE_FACT=1750
 RIKIDDO_PALLET_FACT=1000
 
 # --- Prediction Market Pallet fuzz tests ---
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/prediction-markets/fuzz pm_full_workflow -- -runs=$RUNS
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/prediction-markets/fuzz pm_full_workflow -- -runs=$RUNS
 
 # --- Swaps Pallet fuzz tests ---
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz create_pool -- -runs=$(($(($RUNS * $CREATE_POOL_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_join -- -runs=$(($(($RUNS * $POOL_JOIN_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_join_with_exact_pool_amount -- -runs=$(($(($RUNS * $POOL_JOIN_WITH_EXACT_POOL_AMOUNT_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_join_with_exact_asset_amount -- -runs=$(($(($RUNS * $POOL_JOIN_WITH_EXACT_ASSET_AMOUNT_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz swap_exact_amount_in -- -runs=$(($(($RUNS * $SWAP_EXACT_AMOUNT_IN_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz swap_exact_amount_out -- -runs=$(($(($RUNS * $SWAP_EXACT_AMOUNT_OUT_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_exit_with_exact_asset_amount -- -runs=$(($(($RUNS * $POOL_EXIT_WITH_EXACT_ASSET_AMOUNT_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_exit_with_exact_pool_amount -- -runs=$(($(($RUNS * $POOL_EXIT_WITH_EXACT_POOL_AMOUNT_FACT)) / $BASE))
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/swaps/fuzz pool_exit -- -runs=$(($(($RUNS * $POOL_EXIT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz create_pool -- -runs=$(($(($RUNS * $CREATE_POOL_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_join -- -runs=$(($(($RUNS * $POOL_JOIN_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_join_with_exact_pool_amount -- -runs=$(($(($RUNS * $POOL_JOIN_WITH_EXACT_POOL_AMOUNT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_join_with_exact_asset_amount -- -runs=$(($(($RUNS * $POOL_JOIN_WITH_EXACT_ASSET_AMOUNT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz swap_exact_amount_in -- -runs=$(($(($RUNS * $SWAP_EXACT_AMOUNT_IN_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz swap_exact_amount_out -- -runs=$(($(($RUNS * $SWAP_EXACT_AMOUNT_OUT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_exit_with_exact_asset_amount -- -runs=$(($(($RUNS * $POOL_EXIT_WITH_EXACT_ASSET_AMOUNT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_exit_with_exact_pool_amount -- -runs=$(($(($RUNS * $POOL_EXIT_WITH_EXACT_POOL_AMOUNT_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/swaps/fuzz pool_exit -- -runs=$(($(($RUNS * $POOL_EXIT_FACT)) / $BASE))
 
 # --- Orderbook-v1 Pallet fuzz tests ---
-RUST_BACKTRACE=1 cargo fuzz run --fuzz-dir zrml/orderbook-v1/fuzz orderbook_v1_full_workflow -- -runs=$RUNS
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/orderbook-v1/fuzz orderbook_v1_full_workflow -- -runs=$RUNS
 
 # --- Rikiddo Pallet fuzz tests ---
-# Release is required here since it triggers debug assertions otherwise
+# Profile release is required here since it triggers debug assertions otherwise
 # Using the default RUNS multiplier, each fuzz test needs approx. 6-7 seconds
 RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz fee_sigmoid -- -runs=$(($(($RUNS * $FEE_SIGMOID_FACT)) / $BASE))
 RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz fixedi_to_fixedu_conversion -- -runs=$(($(($RUNS * $FIXEDI_TO_FIXEDU_FACT)) / $BASE))
@@ -64,4 +64,4 @@ RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz ema_marke
 RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz rikiddo_with_initial_fee -- -runs=$(($(($RUNS * $RIKIDDO_WITH_INITIAL_FEE_FACT)) / $BASE))
 RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz rikiddo_with_calculated_fee -- -runs=$(($(($RUNS * $RIKIDDO_WITH_CALCULATED_FEE_FACT)) / $BASE))
 # This actually needs approx. 107 seconds. Need to find a way to optimize fuzzing on-chain
-RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz rikiddo_pallet -- -runs=$(($(($RUNS * $RIKIDDO_PALLET_FACT)) / $BASE))
+RUST_BACKTRACE=1 cargo fuzz run --release --fuzz-dir zrml/rikiddo/fuzz rikiddo_pallet -- -runs=$(($(($RUNS * $RIKIDDO_PALLET_FACT)) / $BASE))
diff --git a/scripts/tests/misc.sh b/scripts/tests/misc.sh
@@ -26,3 +26,5 @@ do
     echo "TEST $package std,runtime-benchmarks"
   fi
 done
+
+grcov . --binary-path ./target/debug/deps/ -s . -t lcov --branch --ignore-not-existing --llvm --ignore '../*' --ignore "/*" -o $RUNNER_TEMP/zeitgeist-test-coverage.lcov
-Original file line number
+Diff line change
@@ Expand Up / @@ -26,3 +26,5 @@ do @@
         echo "TEST $package std,runtime-benchmarks"
       fi
     done
+    grcov . --binary-path ./target/debug/deps/ -s . -t lcov --branch --ignore-not-existing --llvm --ignore '../*' --ignore "/*" -o $RUNNER_TEMP/zeitgeist-test-coverage.lcov