Limit benchmark runtime to random sample of fuzzer cases (#510)

ron-rs · Sep 24, 2023 · 1ff0efa · 1ff0efa
1 parent 416b132
commit 1ff0efa
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 21 deletions.
diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml
@@ -21,8 +21,12 @@ jobs:
           mkdir -p fuzz/corpus/arbitrary
           unzip public.zip -d fuzz/corpus/arbitrary
           rm public.zip
+      - name: Generate benchmark seed
+        run: echo RON_FUZZ_BENCH_SEED=$(date +%s) >> $GITHUB_ENV
       - uses: juntyr/criterion-compare-action@check-cargo-install
         with:
           cwd: fuzz
           benchName: bench
           branchName: ${{ github.base_ref }}
+        env:
+          RON_FUZZ_BENCH_CASES: 100
diff --git a/fuzz/fuzz_targets/bench/main.rs b/fuzz/fuzz_targets/bench/main.rs
@@ -1,6 +1,7 @@
 use std::{
-    collections::{HashSet, VecDeque},
+    collections::{hash_map::DefaultHasher, HashMap, VecDeque},
     fs,
+    hash::{BuildHasher, Hasher},
     path::PathBuf,
 };
 
@@ -11,10 +12,27 @@ use ron::ser::PrettyConfig;
 #[path = "lib.rs"]
 mod typed_data;
 
+struct SeededHasher {
+    seed: u64,
+}
+
+impl BuildHasher for SeededHasher {
+    type Hasher = DefaultHasher;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        let mut hasher = DefaultHasher::new();
+        hasher.write_u64(self.seed);
+        hasher
+    }
+}
+
 fn main() -> anyhow::Result<()> {
     let mut criterion = Criterion::default().configure_from_args();
 
-    let mut cases = HashSet::new();
+    let seed = std::env::var("RON_FUZZ_BENCH_SEED")?.parse()?;
+    let max_cases = std::env::var("RON_FUZZ_BENCH_CASES")?.parse()?;
+
+    let mut cases = HashMap::with_capacity_and_hasher(0, SeededHasher { seed });
 
     let mut entries = VecDeque::new();
     entries.push_back(PathBuf::from("corpus/arbitrary"));
@@ -53,30 +71,34 @@ fn main() -> anyhow::Result<()> {
             .unwrap();
             let ron = ron::ser::to_string_pretty(&typed_data, typed_data.pretty_config()).unwrap();
 
-            if !cases.insert((ty.clone(), value.clone(), ron.clone())) {
+            if cases.insert((ty, value, ron), (entry, pretty)).is_some() {
                 continue;
             }
-
-            println!("{:=^80}", " benchmark case ");
-            println!("{:^80}", entry.to_string_lossy());
-            println!("{:=^80}", " type ");
-            println!("{ty}");
-            println!("{:=^80}", " value ");
-            println!("{value}");
-            println!("{:=^80}", " pretty config ");
-            println!("{pretty}");
-            println!("{:=^80}", " pretty ron ");
-            println!("{ron}");
-            println!("{:=^80}", "");
-
-            criterion.bench_function(&format!("{:?}", entry), |b| {
-                b.iter(|| {
-                    black_box(typed_data::roundtrip_arbitrary_typed_ron_or_panic(&data));
-                })
-            });
         }
     }
 
+    for ((ty, value, ron), (entry, pretty)) in cases.into_iter().take(max_cases) {
+        let data = fs::read(&entry).context("could not read corpus entry")?;
+
+        println!("{:=^80}", " benchmark case ");
+        println!("{:^80}", entry.to_string_lossy());
+        println!("{:=^80}", " type ");
+        println!("{ty}");
+        println!("{:=^80}", " value ");
+        println!("{value}");
+        println!("{:=^80}", " pretty config ");
+        println!("{pretty}");
+        println!("{:=^80}", " pretty ron ");
+        println!("{ron}");
+        println!("{:=^80}", "");
+
+        criterion.bench_function(&format!("{:?}", entry), |b| {
+            b.iter(|| {
+                black_box(typed_data::roundtrip_arbitrary_typed_ron_or_panic(&data));
+            })
+        });
+    }
+
     criterion.final_summary();
 
     Ok(())