Merge pull request #25 from itzmeanjan/update-bench-results

Update Benchmark Results

.github/workflows/test_ci.yml

@@ -1,5 +1,5 @@
 # Collects inspiration from https://github.com/itzmeanjan/dilithium/blob/15c2280f9448b3631a571ee2f33f8b0c695d4788/.github/workflows/test_ci.yml
-name: Test SHA3 Hash and Extendable Output Functions using CI
+name: Test SHA3 Hash and Extendable Output Functions
 
 on:
   push:
@@ -8,14 +8,14 @@ on:
     branches: [ "master" ]
 
 jobs:
-  build:
+  build-on-nix:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     # From https://github.com/marketplace/actions/actions-setup-cmake
     - name: Setup CMake
       uses: jwlawson/[email protected]

Makefile

@@ -1,13 +1,12 @@
-CXX = clang++
+CXX ?= clang++
 CXX_FLAGS = -std=c++20
 WARN_FLAGS = -Wall -Wextra -pedantic
 OPT_FLAGS = -O3 -march=native
 LINK_FLAGS = -flto
 I_FLAGS = -I ./include
 PERF_DEFS = -DCYCLES_PER_BYTE
 ASAN_FLAGS = -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=address # From https://clang.llvm.org/docs/AddressSanitizer.html
-UBSAN_FLAGS = -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=undefined -fsanitize=nullability # From https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
-                                                                                                                     # Note, *nullability* sanitization tests can't be run when compiled with GCC.
+UBSAN_FLAGS = -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=undefined # From https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
 
 SRC_DIR = include
 SHA3_SOURCES := $(wildcard $(SRC_DIR)/*.hpp)
@@ -99,7 +98,7 @@ $(PERF_BINARY): $(PERF_OBJECTS)
 
 perf: $(PERF_BINARY)
 	# Must build google-benchmark with libPFM, follow https://gist.github.com/itzmeanjan/05dc3e946f635d00c5e0b21aae6203a7
-	./$< --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=8 --benchmark_min_time=0.1s --benchmark_counters_tabular=true --benchmark_display_aggregates_only=true --benchmark_perf_counters=CYCLES
+	./$< --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=10 --benchmark_min_time=0.1s --benchmark_counters_tabular=true --benchmark_display_aggregates_only=true --benchmark_perf_counters=CYCLES
 
 .PHONY: format clean
 

benchmarks/bench_common.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <algorithm>
+#include <vector>
+
+const auto compute_min = [](const std::vector<double>& v) -> double {
+  return *std::min_element(v.begin(), v.end());
+};
+
+const auto compute_max = [](const std::vector<double>& v) -> double {
+  return *std::max_element(v.begin(), v.end());
+};

benchmarks/bench_hashing.cpp

@@ -1,9 +1,9 @@
+#include "bench_common.hpp"
 #include "sha3_224.hpp"
 #include "sha3_256.hpp"
 #include "sha3_384.hpp"
 #include "sha3_512.hpp"
 #include <benchmark/benchmark.h>
-#include <vector>
 
 // Benchmarks SHA3-224 hash function with variable length input message.
 void
@@ -136,16 +136,24 @@ bench_sha3_512(benchmark::State& state)
 BENCHMARK(bench_sha3_224)
   ->RangeMultiplier(4)
   ->Range(64, 16384)
-  ->Name("sha3_224");
+  ->Name("sha3_224")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);
 BENCHMARK(bench_sha3_256)
   ->RangeMultiplier(4)
   ->Range(64, 16384)
-  ->Name("sha3_256");
+  ->Name("sha3_256")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);
 BENCHMARK(bench_sha3_384)
   ->RangeMultiplier(4)
   ->Range(64, 16384)
-  ->Name("sha3_384");
+  ->Name("sha3_384")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);
 BENCHMARK(bench_sha3_512)
   ->RangeMultiplier(4)
   ->Range(64, 16384)
-  ->Name("sha3_512");
+  ->Name("sha3_512")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);

benchmarks/bench_keccak.cpp

@@ -1,3 +1,4 @@
+#include "bench_common.hpp"
 #include "keccak.hpp"
 #include "utils.hpp"
 #include <benchmark/benchmark.h>
@@ -24,4 +25,7 @@ bench_keccak_permutation(benchmark::State& state)
 #endif
 }
 
-BENCHMARK(bench_keccak_permutation)->Name("keccak-p[1600, 24]");
+BENCHMARK(bench_keccak_permutation)
+  ->Name("keccak-p[1600, 24]")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);

benchmarks/bench_xof.cpp

@@ -1,7 +1,7 @@
+#include "bench_common.hpp"
 #include "shake128.hpp"
 #include "shake256.hpp"
 #include <benchmark/benchmark.h>
-#include <vector>
 
 // Benchmarks SHAKE-128 extendable output function with variable length input
 // and squeezed output.
@@ -77,7 +77,11 @@ bench_shake256(benchmark::State& state)
 
 BENCHMARK(bench_shake128)
   ->ArgsProduct({ benchmark::CreateRange(64, 16384, 4), { 64 } })
-  ->Name("shake128");
+  ->Name("shake128")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);
 BENCHMARK(bench_shake256)
   ->ArgsProduct({ benchmark::CreateRange(64, 16384, 4), { 64 } })
-  ->Name("shake256");
+  ->Name("shake256")
+  ->ComputeStatistics("min", compute_min)
+  ->ComputeStatistics("max", compute_max);