add per file duration_ms to benchmarks for debug

benchmark/compare.py

@@ -19,11 +19,11 @@ def load_data(btype: str, dname: str, providers: list[str], results_dir: str) ->
 def process_selectors(dname: str, providers: list[str], results_dir: str):
     pdata, ptimes = load_data('selectors', dname, providers, results_dir)
     ret = []
-    for fname, gt in pdata[0].items():
+    for fname, (_meta, gt) in pdata[0].items():
         gt_set = set(gt)
         data = []
         for i in range(1, len(providers)): # skip ground_truth provider
-            d = set(pdata[i].get(fname, []))
+            d = set(pdata[i].get(fname, (0, []))[1])
             fp = list(d - gt_set)
             fn = list(gt_set - d)
             data.append([fp, fn])
@@ -192,7 +192,7 @@ def f(s):
 def process_functions(tname: str, dname: str, providers: list[str], results_dir: str, normalize_func):
     pdata, ptimes = load_data(tname, dname, providers, results_dir)
     ret = []
-    for fname, gt in pdata[0].items():
+    for fname, (_meta, gt) in pdata[0].items():
         func = []
         for sel, gt_val in gt.items():
             if gt_val == '' and tname == 'mutability':
@@ -201,7 +201,7 @@ def process_functions(tname: str, dname: str, providers: list[str], results_dir:
             data = []
             norm_gt_val = normalize_func(gt_val)
             for i in range(1, len(providers)): # skip ground_truth provider
-                val = pdata[i][fname][sel]
+                (_meta, val) = pdata[i][fname][sel]
                 norm_val = normalize_func(val)
                 if norm_val == norm_gt_val:
                     data.append([1])
@@ -272,11 +272,21 @@ def show_arguments_or_mutability(providers: list[str], all_results: list, show_e
             cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'evm-hound-rs', 'simple']
         elif cfg.mode == 'arguments':
             cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'simple']
-        else: # mutability
+        elif cfg.mode == 'mutability':
             cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'simple']
+        else:
+            cfg.providers = []
     print('Config:')
     print('\n'.join(f'  {field} = {getattr(cfg, field)}' for field in vars(cfg)), '\n')
 
+    if cfg.mode == 'selectors':
+        results = [process_selectors(d, cfg.providers, cfg.results_dir) for d in cfg.datasets]
+
+        if cfg.markdown:
+            markdown_selectors(cfg.providers, results)
+        else:
+            show_selectors(cfg.providers, results, cfg.show_errors)
+
     if cfg.mode == 'arguments':
         results = [process_arguments(d, cfg.providers, cfg.results_dir, cfg.normalize_args) for d in cfg.datasets]
         if cfg.markdown:
@@ -296,11 +306,3 @@ def show_arguments_or_mutability(providers: list[str], all_results: list, show_e
                 x['dataset'] += '/strict'
                 results.append(x)
             show_arguments_or_mutability(cfg.providers, results, cfg.show_errors)
-
-    else:
-        results = [process_selectors(d, cfg.providers, cfg.results_dir) for d in cfg.datasets]
-
-        if cfg.markdown:
-            markdown_selectors(cfg.providers, results)
-        else:
-            show_selectors(cfg.providers, results, cfg.show_errors)

benchmark/providers/etherscan/main.py

@@ -1,6 +1,8 @@
 import json
 import os
+import re
 import sys
+import time
 
 from Crypto.Hash import keccak
 
@@ -20,16 +22,25 @@ def join_inputs(inputs) -> str:
         n += ','
     return n[:-1]
 
-def process(abi, mode) -> dict[str,str]:
+def process(data, mode):
     ret = {}
-    for x in abi:
+    for x in data['abi']:
         if x['type'] != 'function':
             continue
         args = join_inputs(x['inputs'])
         n = f'{x["name"]}({args})'
         sg = sign(n.encode('ascii'))
-        ret[sg] = args if mode == 'arguments' else x.get('stateMutability', '')
-    return ret
+        if mode == 'arguments' or mode == 'selectors':
+            ret[sg] = args
+        elif mode == 'mutability':
+            ret[sg] = x.get('stateMutability', '')
+        else:
+            raise Exception(f'Unknown mode {mode}')
+
+    if mode == 'selectors':
+        return list(ret.keys())
+    else:
+        return ret
 
 if len(sys.argv) < 4:
     print('Usage: python3 main.py MODE INPUT_DIR OUTPUT_FILE')
@@ -44,8 +55,10 @@ def process(abi, mode) -> dict[str,str]:
 for fname in os.listdir(indir):
     with open(f'{indir}/{fname}', 'r') as fh:
         d = json.load(fh)
-        r = process(d['abi'], mode)
-        ret[fname] = list(r.keys()) if mode == 'selectors' else r
+        t0 = time.perf_counter()
+        r = process(d, mode)
+        duration_ms = int((time.perf_counter() - t0) * 1000)
+        ret[fname] = [duration_ms, r]
 
 with open(outfile, 'w') as fh:
     json.dump(ret, fh)

benchmark/providers/evm-hound-rs/src/main.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::io::{BufWriter, Write};
+use std::time::Instant;
 use std::{env, fs};
 
 #[derive(Debug, serde::Deserialize)]
@@ -22,7 +23,9 @@ fn main() -> std::io::Result<()> {
         std::process::exit(1);
     }
 
-    let mut ret: HashMap<String, Vec<String>> = HashMap::new();
+    type Meta = u64; // duration in ms
+    let mut ret: HashMap<String, (Meta, Vec<String>)> = HashMap::new();
+
     for entry in fs::read_dir(indir)? {
         let entry = entry?;
         let path = entry.path();
@@ -35,12 +38,12 @@ fn main() -> std::io::Result<()> {
             hex::decode(x).unwrap()
         };
 
-        let string_selectors: Vec<_> = evm_hound::selectors_from_bytecode(&code)
-            .into_iter()
-            .map(hex::encode)
-            .collect();
+        let now = Instant::now();
+        let r = evm_hound::selectors_from_bytecode(&code);
+        let duration_ms = now.elapsed().as_millis() as u64;
+        let string_selectors: Vec<_> = r.into_iter().map(hex::encode).collect();
 
-        ret.insert(fname, string_selectors);
+        ret.insert(fname, (duration_ms, string_selectors));
     }
 
     let file = fs::File::create(outfile)?;

benchmark/providers/evmole-js/main.mjs

@@ -1,48 +1,47 @@
-import {readdirSync, readFileSync, writeFileSync} from 'fs'
-import {parseArgs} from 'util'
+import { readdirSync, readFileSync, writeFileSync } from 'fs'
+import { hrtime } from 'process'
 
-import {functionArguments, functionSelectors, functionStateMutability} from 'evmole'
+import { functionArguments, functionSelectors, functionStateMutability } from 'evmole'
 
-const {
-  values: cfg,
-  positionals: cfg_positionals,
-} = parseArgs({
-  options: {
-    'filter-filename': {
-      type: 'string',
-    },
-    'filter-selector': {
-      type: 'string',
-    },
-  },
-  allowPositionals: true
-});
-
-if (cfg_positionals.length < 3) {
-  console.log('Usage: node main.js MODE INPUT_DIR OUTPUT_FILE [SELCTORS_FILE]')
+const argv = process.argv;
+if (argv.length < 5) {
+  console.log('Usage: node main.js MODE INPUT_DIR OUTPUT_FILE [SELECTORS_FILE]')
   process.exit(1)
 }
 
-const [mode, indir, outfile, ...cfg_rest] = cfg_positionals;
+const mode = argv[2];
+const indir = argv[3];
+const outfile = argv[4];
+
+const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {};
 
-const selectors = mode === 'selectors' ? {} : JSON.parse(readFileSync(cfg_rest[0]));
+function timeit(fn) {
+  const start_ts = hrtime.bigint();
+  const r = fn();
+  const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n);
+  return [duration_ms, r]
+}
+
+function extract(code, mode, fname) {
+  if (mode === 'selectors') {
+    let [duration_ms, r] = timeit(() => functionSelectors(code));
+    return [duration_ms, r];
+  } else if (mode === 'arguments') {
+    let [duration_ms, r] = timeit(() => selectors[fname][1].map((s) => [s, functionArguments(code, s)]));
+    return [duration_ms, Object.fromEntries(r)];
+  } else if (mode === 'mutability') {
+    let [duration_ms, r] = timeit(() => selectors[fname][1].map((s) => [s, functionStateMutability(code, s)]));
+  } else {
+    throw 'unsupported mode';
+  }
+}
 
 const res = Object.fromEntries(
-  readdirSync(indir)
-    .filter((file) => cfg['filter-filename'] === undefined || file.includes(cfg['filter-filename']))
-    .map((file) => {
-      const code = JSON.parse(readFileSync(`${indir}/${file}`))['code']
-      if (mode === 'selectors') {
-        return [file, functionSelectors(code)];
-      } else {
-        const fsel = cfg['filter-selector'] === undefined ? selectors[file] : [cfg['filter-selector']];
-        if (mode === 'arguments') {
-          return [file, Object.fromEntries(fsel.map((s) => [s, functionArguments(code, s)]))];
-        } else {
-          return [file, Object.fromEntries(fsel.map((s) => [s, functionStateMutability(code, s)]))];
-        }
-      }
-    }
+  readdirSync(indir).map(
+    file => [
+      file,
+      extract(JSON.parse(readFileSync(`${indir}/${file}`))['code'], mode, file)
+    ]
   )
 );
 writeFileSync(outfile, JSON.stringify(res), 'utf8');

benchmark/providers/evmole-py/main.py

@@ -1,17 +1,15 @@
 import argparse
 import json
 import os
+import time
 
-from evmole import function_selectors, function_arguments, function_state_mutability
-
+from evmole import function_arguments, function_selectors, function_state_mutability
 
 parser = argparse.ArgumentParser()
 parser.add_argument('mode', choices=['selectors', 'arguments', 'mutability'])
 parser.add_argument('input_dir')
 parser.add_argument('output_file')
 parser.add_argument('selectors_file', nargs='*')
-parser.add_argument('--filter-filename', required=False)
-parser.add_argument('--filter-selector', required=False)
 cfg = parser.parse_args()
 
 selectors = {}
@@ -21,21 +19,22 @@
 
 ret = {}
 for fname in os.listdir(cfg.input_dir):
-    if cfg.filter_filename is not None and cfg.filter_filename not in fname:
-        continue
-
     with open(f'{cfg.input_dir}/{fname}', 'r') as fh:
         d = json.load(fh)
         code = d['code']
+        t0 = time.perf_counter()
         if cfg.mode == 'selectors':
             r = function_selectors(code)
+        elif cfg.mode == 'arguments':
+            fsel = selectors[fname][1]
+            r = {s: function_arguments(code, s) for s in fsel}
+        elif cfg.mode == 'mutability':
+            fsel = selectors[fname][1]
+            r = {s: function_state_mutability(code, s) for s in fsel}
         else:
-            fsel = selectors[fname] if cfg.filter_selector is None else [cfg.filter_selector]
-            if cfg.mode == 'arguments':
-                r = {s: function_arguments(code, s) for s in fsel}
-            else:
-                r = {s: function_state_mutability(code, s) for s in fsel}
-        ret[fname] = r
+            raise Exception(f'Unknown mode {cfg.mode}')
+        duration_ms = int((time.perf_counter() - t0) * 1000)
+        ret[fname] = [duration_ms, r]
 
 with open(cfg.output_file, 'w') as fh:
     json.dump(ret, fh)