diff --git a/benchmark/compare.py b/benchmark/compare.py index b91ff71..97bf68b 100644 --- a/benchmark/compare.py +++ b/benchmark/compare.py @@ -19,11 +19,11 @@ def load_data(btype: str, dname: str, providers: list[str], results_dir: str) -> def process_selectors(dname: str, providers: list[str], results_dir: str): pdata, ptimes = load_data('selectors', dname, providers, results_dir) ret = [] - for fname, gt in pdata[0].items(): + for fname, (_meta, gt) in pdata[0].items(): gt_set = set(gt) data = [] for i in range(1, len(providers)): # skip ground_truth provider - d = set(pdata[i].get(fname, [])) + d = set(pdata[i].get(fname, (0, []))[1]) fp = list(d - gt_set) fn = list(gt_set - d) data.append([fp, fn]) @@ -192,7 +192,7 @@ def f(s): def process_functions(tname: str, dname: str, providers: list[str], results_dir: str, normalize_func): pdata, ptimes = load_data(tname, dname, providers, results_dir) ret = [] - for fname, gt in pdata[0].items(): + for fname, (_meta, gt) in pdata[0].items(): func = [] for sel, gt_val in gt.items(): if gt_val == '' and tname == 'mutability': @@ -201,7 +201,7 @@ def process_functions(tname: str, dname: str, providers: list[str], results_dir: data = [] norm_gt_val = normalize_func(gt_val) for i in range(1, len(providers)): # skip ground_truth provider - val = pdata[i][fname][sel] + (_meta, val) = pdata[i][fname][sel] norm_val = normalize_func(val) if norm_val == norm_gt_val: data.append([1]) @@ -272,11 +272,21 @@ def show_arguments_or_mutability(providers: list[str], all_results: list, show_e cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'evm-hound-rs', 'simple'] elif cfg.mode == 'arguments': cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'simple'] - else: # mutability + elif cfg.mode == 'mutability': cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'simple'] + else: + cfg.providers = [] print('Config:') print('\n'.join(f' {field} = {getattr(cfg, field)}' for field in vars(cfg)), '\n') + if cfg.mode == 'selectors': + results = [process_selectors(d, cfg.providers, cfg.results_dir) for d in cfg.datasets] + + if cfg.markdown: + markdown_selectors(cfg.providers, results) + else: + show_selectors(cfg.providers, results, cfg.show_errors) + if cfg.mode == 'arguments': results = [process_arguments(d, cfg.providers, cfg.results_dir, cfg.normalize_args) for d in cfg.datasets] if cfg.markdown: @@ -296,11 +306,3 @@ def show_arguments_or_mutability(providers: list[str], all_results: list, show_e x['dataset'] += '/strict' results.append(x) show_arguments_or_mutability(cfg.providers, results, cfg.show_errors) - - else: - results = [process_selectors(d, cfg.providers, cfg.results_dir) for d in cfg.datasets] - - if cfg.markdown: - markdown_selectors(cfg.providers, results) - else: - show_selectors(cfg.providers, results, cfg.show_errors) diff --git a/benchmark/providers/etherscan/main.py b/benchmark/providers/etherscan/main.py index 6bfe7b0..458edcc 100644 --- a/benchmark/providers/etherscan/main.py +++ b/benchmark/providers/etherscan/main.py @@ -1,6 +1,8 @@ import json import os +import re import sys +import time from Crypto.Hash import keccak @@ -20,16 +22,25 @@ def join_inputs(inputs) -> str: n += ',' return n[:-1] -def process(abi, mode) -> dict[str,str]: +def process(data, mode): ret = {} - for x in abi: + for x in data['abi']: if x['type'] != 'function': continue args = join_inputs(x['inputs']) n = f'{x["name"]}({args})' sg = sign(n.encode('ascii')) - ret[sg] = args if mode == 'arguments' else x.get('stateMutability', '') - return ret + if mode == 'arguments' or mode == 'selectors': + ret[sg] = args + elif mode == 'mutability': + ret[sg] = x.get('stateMutability', '') + else: + raise Exception(f'Unknown mode {mode}') + + if mode == 'selectors': + return list(ret.keys()) + else: + return ret if len(sys.argv) < 4: print('Usage: python3 main.py MODE INPUT_DIR OUTPUT_FILE') @@ -44,8 +55,10 @@ def process(abi, mode) -> dict[str,str]: for fname in os.listdir(indir): with open(f'{indir}/{fname}', 'r') as fh: d = json.load(fh) - r = process(d['abi'], mode) - ret[fname] = list(r.keys()) if mode == 'selectors' else r + t0 = time.perf_counter() + r = process(d, mode) + duration_ms = int((time.perf_counter() - t0) * 1000) + ret[fname] = [duration_ms, r] with open(outfile, 'w') as fh: json.dump(ret, fh) diff --git a/benchmark/providers/evm-hound-rs/src/main.rs b/benchmark/providers/evm-hound-rs/src/main.rs index a627b54..eb68285 100644 --- a/benchmark/providers/evm-hound-rs/src/main.rs +++ b/benchmark/providers/evm-hound-rs/src/main.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::io::{BufWriter, Write}; +use std::time::Instant; use std::{env, fs}; #[derive(Debug, serde::Deserialize)] @@ -22,7 +23,9 @@ fn main() -> std::io::Result<()> { std::process::exit(1); } - let mut ret: HashMap> = HashMap::new(); + type Meta = u64; // duration in ms + let mut ret: HashMap)> = HashMap::new(); + for entry in fs::read_dir(indir)? { let entry = entry?; let path = entry.path(); @@ -35,12 +38,12 @@ fn main() -> std::io::Result<()> { hex::decode(x).unwrap() }; - let string_selectors: Vec<_> = evm_hound::selectors_from_bytecode(&code) - .into_iter() - .map(hex::encode) - .collect(); + let now = Instant::now(); + let r = evm_hound::selectors_from_bytecode(&code); + let duration_ms = now.elapsed().as_millis() as u64; + let string_selectors: Vec<_> = r.into_iter().map(hex::encode).collect(); - ret.insert(fname, string_selectors); + ret.insert(fname, (duration_ms, string_selectors)); } let file = fs::File::create(outfile)?; diff --git a/benchmark/providers/evmole-js/main.mjs b/benchmark/providers/evmole-js/main.mjs index 835ffa6..be7276d 100644 --- a/benchmark/providers/evmole-js/main.mjs +++ b/benchmark/providers/evmole-js/main.mjs @@ -1,48 +1,47 @@ -import {readdirSync, readFileSync, writeFileSync} from 'fs' -import {parseArgs} from 'util' +import { readdirSync, readFileSync, writeFileSync } from 'fs' +import { hrtime } from 'process' -import {functionArguments, functionSelectors, functionStateMutability} from 'evmole' +import { functionArguments, functionSelectors, functionStateMutability } from 'evmole' -const { - values: cfg, - positionals: cfg_positionals, -} = parseArgs({ - options: { - 'filter-filename': { - type: 'string', - }, - 'filter-selector': { - type: 'string', - }, - }, - allowPositionals: true -}); - -if (cfg_positionals.length < 3) { - console.log('Usage: node main.js MODE INPUT_DIR OUTPUT_FILE [SELCTORS_FILE]') +const argv = process.argv; +if (argv.length < 5) { + console.log('Usage: node main.js MODE INPUT_DIR OUTPUT_FILE [SELECTORS_FILE]') process.exit(1) } -const [mode, indir, outfile, ...cfg_rest] = cfg_positionals; +const mode = argv[2]; +const indir = argv[3]; +const outfile = argv[4]; + +const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {}; -const selectors = mode === 'selectors' ? {} : JSON.parse(readFileSync(cfg_rest[0])); +function timeit(fn) { + const start_ts = hrtime.bigint(); + const r = fn(); + const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n); + return [duration_ms, r] +} + +function extract(code, mode, fname) { + if (mode === 'selectors') { + let [duration_ms, r] = timeit(() => functionSelectors(code)); + return [duration_ms, r]; + } else if (mode === 'arguments') { + let [duration_ms, r] = timeit(() => selectors[fname][1].map((s) => [s, functionArguments(code, s)])); + return [duration_ms, Object.fromEntries(r)]; + } else if (mode === 'mutability') { + let [duration_ms, r] = timeit(() => selectors[fname][1].map((s) => [s, functionStateMutability(code, s)])); + } else { + throw 'unsupported mode'; + } +} const res = Object.fromEntries( - readdirSync(indir) - .filter((file) => cfg['filter-filename'] === undefined || file.includes(cfg['filter-filename'])) - .map((file) => { - const code = JSON.parse(readFileSync(`${indir}/${file}`))['code'] - if (mode === 'selectors') { - return [file, functionSelectors(code)]; - } else { - const fsel = cfg['filter-selector'] === undefined ? selectors[file] : [cfg['filter-selector']]; - if (mode === 'arguments') { - return [file, Object.fromEntries(fsel.map((s) => [s, functionArguments(code, s)]))]; - } else { - return [file, Object.fromEntries(fsel.map((s) => [s, functionStateMutability(code, s)]))]; - } - } - } + readdirSync(indir).map( + file => [ + file, + extract(JSON.parse(readFileSync(`${indir}/${file}`))['code'], mode, file) + ] ) ); writeFileSync(outfile, JSON.stringify(res), 'utf8'); diff --git a/benchmark/providers/evmole-py/main.py b/benchmark/providers/evmole-py/main.py index c4e07bf..508c76d 100644 --- a/benchmark/providers/evmole-py/main.py +++ b/benchmark/providers/evmole-py/main.py @@ -1,17 +1,15 @@ import argparse import json import os +import time -from evmole import function_selectors, function_arguments, function_state_mutability - +from evmole import function_arguments, function_selectors, function_state_mutability parser = argparse.ArgumentParser() parser.add_argument('mode', choices=['selectors', 'arguments', 'mutability']) parser.add_argument('input_dir') parser.add_argument('output_file') parser.add_argument('selectors_file', nargs='*') -parser.add_argument('--filter-filename', required=False) -parser.add_argument('--filter-selector', required=False) cfg = parser.parse_args() selectors = {} @@ -21,21 +19,22 @@ ret = {} for fname in os.listdir(cfg.input_dir): - if cfg.filter_filename is not None and cfg.filter_filename not in fname: - continue - with open(f'{cfg.input_dir}/{fname}', 'r') as fh: d = json.load(fh) code = d['code'] + t0 = time.perf_counter() if cfg.mode == 'selectors': r = function_selectors(code) + elif cfg.mode == 'arguments': + fsel = selectors[fname][1] + r = {s: function_arguments(code, s) for s in fsel} + elif cfg.mode == 'mutability': + fsel = selectors[fname][1] + r = {s: function_state_mutability(code, s) for s in fsel} else: - fsel = selectors[fname] if cfg.filter_selector is None else [cfg.filter_selector] - if cfg.mode == 'arguments': - r = {s: function_arguments(code, s) for s in fsel} - else: - r = {s: function_state_mutability(code, s) for s in fsel} - ret[fname] = r + raise Exception(f'Unknown mode {cfg.mode}') + duration_ms = int((time.perf_counter() - t0) * 1000) + ret[fname] = [duration_ms, r] with open(cfg.output_file, 'w') as fh: json.dump(ret, fh) diff --git a/benchmark/providers/evmole-rs/src/main.rs b/benchmark/providers/evmole-rs/src/main.rs index ff51361..74f78d2 100644 --- a/benchmark/providers/evmole-rs/src/main.rs +++ b/benchmark/providers/evmole-rs/src/main.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::fs; use std::io::{BufWriter, Write}; +use std::time::Instant; use clap::{Parser, ValueEnum}; use hex::FromHex; @@ -37,9 +38,11 @@ struct Args { fn main() -> Result<(), Box> { let cfg = Args::parse(); - let selectors: HashMap> = match cfg.mode { + type Meta = u64; // duration in ms + + let selectors: HashMap)> = match cfg.mode { Mode::Selectors => HashMap::new(), - _ => { + Mode::Arguments | Mode::Mutability => { let file_content = fs::read_to_string(cfg.selectors_file.unwrap())?; serde_json::from_str(&file_content)? } @@ -51,8 +54,9 @@ fn main() -> Result<(), Box> { vec![] }; - let mut ret_selectors: HashMap> = HashMap::new(); - let mut ret_other: HashMap> = HashMap::new(); + + let mut ret_selectors: HashMap)> = HashMap::new(); + let mut ret_other: HashMap)> = HashMap::new(); for entry in fs::read_dir(cfg.input_dir)? { let entry = entry?; @@ -71,41 +75,78 @@ fn main() -> Result<(), Box> { hex::decode(v.code.strip_prefix("0x").expect("0x prefix expected"))? }; - // println!("processing {}", fname); + // eprintln!("processing {}", fname); match cfg.mode { Mode::Selectors => { + let now = Instant::now(); let r = evmole::function_selectors(&code, 0); - ret_selectors.insert(fname, r.iter().map(hex::encode).collect()); + ret_selectors.insert( + fname, + ( + now.elapsed().as_millis() as u64, + r.iter().map(hex::encode).collect(), + ), + ); } - Mode::Arguments | Mode::Mutability => { + Mode::Arguments => { let fsel = if !only_selector.is_empty() { &only_selector } else { - &selectors[&fname] + &selectors[&fname].1 }; + let now = Instant::now(); + let args = fsel + .iter() + .map(|s| { + let selector = <[u8; 4]>::from_hex(s).unwrap(); + ( + s.to_string(), + evmole::function_arguments(&code, &selector, 0), + ) + }) + .collect(); + + ret_other.insert(fname, (now.elapsed().as_millis() as u64, args)); + } + Mode::Mutability => { + let fsel = if !only_selector.is_empty() { + &only_selector + } else { + &selectors[&fname].1 + }; + + let now = Instant::now(); + let info = evmole::contract_info( + evmole::ContractInfoArgs::new(&code).with_state_mutability(), + ); + let dur = now.elapsed().as_millis() as u64; + let res: HashMap<_, _> = info + .functions + .into_iter() + .map(|f| (f.selector, f.state_mutability.unwrap())) + .collect(); ret_other.insert( fname, - fsel.iter() - .map(|s| { - let selector = <[u8; 4]>::from_hex(s).unwrap(); - ( - s.to_string(), - match cfg.mode { - Mode::Arguments => { - evmole::function_arguments(&code, &selector, 0) - } - Mode::Mutability => { - evmole::function_state_mutability(&code, &selector, 0) - .as_json_str() - .to_string() - } - _ => panic!("impossible mode"), - }, - ) - }) - .collect(), + ( + dur, + fsel.iter() + .map(|s| { + let selector = <[u8; 4]>::from_hex(s).unwrap(); + ( + s.to_string(), + if let Some(sm) = res.get(&selector) { + sm.as_json_str().to_string() + } else { + "".to_string() + }, // evmole::function_state_mutability(&code, &selector, 0) + // .as_json_str() + // .to_string() + ) + }) + .collect(), + ), ); } } diff --git a/benchmark/providers/heimdall-rs/src/main.rs b/benchmark/providers/heimdall-rs/src/main.rs index 808febf..e326c64 100644 --- a/benchmark/providers/heimdall-rs/src/main.rs +++ b/benchmark/providers/heimdall-rs/src/main.rs @@ -1,6 +1,9 @@ use std::collections::HashMap; +use std::fs; use std::io::{BufWriter, Write}; -use std::{env, fs}; +use std::time::Instant; + +use clap::{Parser, ValueEnum}; use heimdall_core::heimdall_decompiler::DecompilerArgsBuilder; @@ -9,28 +12,42 @@ struct Input { code: String, } +#[derive(ValueEnum, Clone, PartialEq)] +enum Mode { + Selectors, + Arguments, + Mutability, +} + +#[derive(Parser)] +struct Args { + mode: Mode, + + input_dir: String, + + output_file: String, + + selectors_file: Option, +} + #[tokio::main] async fn main() -> Result<(), Box> { - let args: Vec<_> = env::args().collect(); - if args.len() < 4 { - eprintln!("Usage: ./main MODE INPUT_DIR OUTPUT_FILE [SELECTORS_FILE]"); - std::process::exit(1); - } - let mode = &args[1]; - let indir = &args[2]; - let outfile = &args[3]; + let cfg = Args::parse(); - let selectors: HashMap> = if mode == "selectors" { - HashMap::new() - } else { - let file_content = fs::read_to_string(&args[4])?; - serde_json::from_str(&file_content)? + type Meta = u64; // duration in ms + + let selectors: HashMap)> = match cfg.mode { + Mode::Selectors => HashMap::new(), + Mode::Arguments | Mode::Mutability => { + let file_content = fs::read_to_string(cfg.selectors_file.unwrap())?; + serde_json::from_str(&file_content)? + } }; - let mut ret_selectors: HashMap> = HashMap::new(); - let mut ret_arguments: HashMap> = HashMap::new(); + let mut ret_selectors: HashMap)> = HashMap::new(); + let mut ret_other: HashMap)> = HashMap::new(); - for entry in fs::read_dir(indir)? { + for entry in fs::read_dir(cfg.input_dir)? { let entry = entry?; let path = entry.path(); let fname = entry.file_name().to_str().unwrap().to_string(); @@ -39,75 +56,133 @@ async fn main() -> Result<(), Box> { let v: Input = serde_json::from_str(&file_content)?; v.code }; + match cfg.mode { + Mode::Selectors => { + let dargs = DecompilerArgsBuilder::new() + .target(hex_code) + .skip_resolving(true) + .build()?; + let now = Instant::now(); + let result = heimdall_core::heimdall_decompiler::decompile(dargs).await; + let duration_ms = now.elapsed().as_millis() as u64; + let r = match result { + Err(e) => { + println!("got error for {}: {}", fname, e); + vec![] + } + Ok(v) => v + .abi + .functions + .keys() + .map(|x| x.strip_prefix("Unresolved_").unwrap().to_string()) + .collect(), + }; + ret_selectors.insert(fname, (duration_ms, r)); + } + Mode::Arguments => { + let dargs = DecompilerArgsBuilder::new() + .target(hex_code) + .skip_resolving(true) + .build()?; + let now = Instant::now(); + let result = heimdall_core::heimdall_decompiler::decompile(dargs).await; + let duration_ms = now.elapsed().as_millis() as u64; + let r = match result { + Err(e) => { + println!("got error for {}: {}", fname, e); + selectors[&fname] + .1 + .iter() + .map(|s| (s.to_string(), "not_found".to_string())) + .collect() + } + Ok(v) => { + let args: HashMap = v + .abi + .functions + .iter() + .map(|(name, v)| { + let selector = + name.strip_prefix("Unresolved_").unwrap().to_string(); + let arguments = + v[0].inputs.iter().map(|v| v.ty.to_string()).collect(); + (selector, arguments) + }) + .collect(); - let dargs = DecompilerArgsBuilder::new() - .target(hex_code) - .skip_resolving(true) - .build()?; - - // println!("processing {}", fname); - - //if fname == "0x940259178FbF021e625510919BC2FF0B944E5613.json" { - // if mode == "arguments" { - // let r: HashMap = selectors[&fname].iter().map(|s| (s.to_string(), "not_found".to_string())).collect(); - // ret_arguments.insert(fname, r); - // } else { - // ret_selectors.insert(fname, Vec::new()); - // } - // continue - //} - - let result = heimdall_core::heimdall_decompiler::decompile(dargs).await; - if let Err(e) = result { - println!("got error for {}: {}", fname, e); - if mode == "selectors" { - ret_selectors.insert(fname, Vec::new()); - } else { - let r: HashMap = selectors[&fname].iter().map(|s| (s.to_string(), "not_found".to_string())).collect(); - ret_arguments.insert(fname, r); + selectors[&fname] + .1 + .iter() + .map(|s| { + ( + s.to_string(), + match args.get(s) { + Some(v) => v.to_string(), + None => "not_found".to_string(), + }, + ) + }) + .collect() + } + }; + ret_other.insert(fname, (duration_ms, r)); + } + Mode::Mutability => { + let dargs = DecompilerArgsBuilder::new() + .target(hex_code) + .skip_resolving(true) + .build()?; + let now = Instant::now(); + let result = heimdall_core::heimdall_decompiler::decompile(dargs).await; + let duration_ms = now.elapsed().as_millis() as u64; + let r = match result { + Err(e) => { + println!("got error for {}: {}", fname, e); + selectors[&fname] + .1 + .iter() + .map(|s| (s.to_string(), "not_found".to_string())) + .collect() + } + Ok(v) => { + let args: HashMap = v + .abi + .functions + .iter() + .map(|(name, v)| { + let selector = + name.strip_prefix("Unresolved_").unwrap().to_string(); + let mutability = v[0].state_mutability.as_json_str().to_string(); + (selector, mutability) + }) + .collect(); + + selectors[&fname] + .1 + .iter() + .map(|s| { + ( + s.to_string(), + match args.get(s) { + Some(v) => v.to_string(), + None => "not_found".to_string(), + }, + ) + }) + .collect() + } + }; + ret_other.insert(fname, (duration_ms, r)); } - continue - } - let abi = result?.abi.functions; - - if mode == "selectors" { - let r: Vec = abi - .keys() - .map(|x| x.strip_prefix("Unresolved_").unwrap().to_string()) - .collect(); - ret_selectors.insert(fname, r); - } else { - let args: HashMap = abi - .iter() - .map(|(name, v)| { - let selector = name.strip_prefix("Unresolved_").unwrap().to_string(); - let info = if mode == "arguments" { - let a: Vec<_> = v[0].inputs.iter().map(|v| v.ty.to_string()).collect(); - a.join(",") - } else { - v[0].state_mutability.as_json_str().to_string() - }; - (selector, info) - }) - .collect(); - - let r: HashMap = selectors[&fname] - .iter() - .map(|s| match args.get(s) { - Some(v) => (s.to_string(), v.to_string()), - None => (s.to_string(), "not_found".to_string()), - }) - .collect(); - ret_arguments.insert(fname, r); } } - let file = fs::File::create(outfile)?; + let file = fs::File::create(cfg.output_file)?; let mut bw = BufWriter::new(file); - if mode == "selectors" { + if cfg.mode == Mode::Selectors { let _ = serde_json::to_writer(&mut bw, &ret_selectors); } else { - let _ = serde_json::to_writer(&mut bw, &ret_arguments); + let _ = serde_json::to_writer(&mut bw, &ret_other); } bw.flush()?; diff --git a/benchmark/providers/sevm/main.mjs b/benchmark/providers/sevm/main.mjs index 0bade80..4143871 100644 --- a/benchmark/providers/sevm/main.mjs +++ b/benchmark/providers/sevm/main.mjs @@ -1,4 +1,5 @@ -import {readdirSync, readFileSync, writeFileSync} from 'fs' +import { readdirSync, readFileSync, writeFileSync } from 'fs' +import { hrtime } from 'process' import { Contract } from 'sevm'; @@ -9,34 +10,39 @@ if (argv.length < 5) { } const mode = argv[2]; -if (mode != 'selectors' && mode != 'mutability') { - console.log('Only "selectors" and "mutability" modes are supported, got ', mode) - process.exit(1) -} const indir = argv[3]; const outfile = argv[4]; -const selectors = mode === 'selectors' ? {} : JSON.parse(readFileSync(argv[5])); +const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {}; -function extract(code, mode, fname) { - let funcs; - try { - funcs = new Contract(code).functions; - } catch(e) { - funcs = {}; - } +function timeit(fn) { + const start_ts = hrtime.bigint(); + const r = fn(); + const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n); + return [duration_ms, r] +} - if (mode == 'selectors') { - return Object.keys(funcs) - } else { - return Object.fromEntries(selectors[fname].map((s) => { - const fn = funcs[s]; +function extract(code, mode, fname) { + const [duration_ms, contract] = timeit(() => { + try { + return new Contract(code); + } catch (e) { + // console.log(e); + } + }); + if (mode === 'selectors') { + return [duration_ms, Object.keys(contract ? contract.functions : {})] + } else if (mode === 'mutability') { + return [duration_ms, Object.fromEntries(selectors[fname][1].map((s) => { + const fn = contract ? contract.functions[s] : undefined; if (fn === undefined) { return [s, 'selnotfound']; } else { return [s, fn.constant ? 'view' : (fn.payable ? 'payable' : 'nonpayable')]; } - })); + }))]; + } else { + throw 'unsupported mode'; } } diff --git a/benchmark/providers/simple/main.py b/benchmark/providers/simple/main.py index 06d7a2d..05388e4 100644 --- a/benchmark/providers/simple/main.py +++ b/benchmark/providers/simple/main.py @@ -1,6 +1,7 @@ import json import os import sys +import time def extract_selectors(code: bytes) -> list[str]: @@ -35,13 +36,17 @@ def extract_selectors(code: bytes) -> list[str]: with open(f'{indir}/{fname}', 'r') as fh: d = json.load(fh) code = bytes.fromhex(d['code'][2:]) + t0 = time.perf_counter() if mode == 'arguments': r = {s: '' for s in selectors[fname]} elif mode == 'mutability': r = {s: 'nonpayable' for s in selectors[fname]} - else: + elif mode == 'selectors': r = extract_selectors(code) - ret[fname] = r + else: + raise Exception(f'Unknown mode {mode}') + duration_ms = int((time.perf_counter() - t0) * 1000) + ret[fname] = [duration_ms, r] with open(outfile, 'w') as fh: json.dump(ret, fh) diff --git a/benchmark/providers/whatsabi/main.mjs b/benchmark/providers/whatsabi/main.mjs index 38585b2..a151c51 100644 --- a/benchmark/providers/whatsabi/main.mjs +++ b/benchmark/providers/whatsabi/main.mjs @@ -1,4 +1,5 @@ -import {readdirSync, readFileSync, writeFileSync} from 'fs' +import { readdirSync, readFileSync, writeFileSync } from 'fs' +import { hrtime } from 'process' import { whatsabi } from "@shazow/whatsabi"; @@ -9,24 +10,28 @@ if (argv.length < 5) { } const mode = argv[2]; -if (mode != 'selectors' && mode != 'mutability') { - console.log('Only "selectors" and "mutability" modes are supported, got ', mode) - process.exit(1) -} const indir = argv[3]; const outfile = argv[4]; -const selectors = mode === 'selectors' ? {} : JSON.parse(readFileSync(argv[5])); +const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {}; + +function timeit(fn) { + const start_ts = hrtime.bigint(); + const r = fn(); + const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n); + return [duration_ms, r] +} function extract(code, mode, fname) { - if (mode == 'selectors') { - return whatsabi.selectorsFromBytecode(code).map(x => x.slice(2)); // remove '0x' prefix - } else { // mutability - const abi = whatsabi.abiFromBytecode(code); + if (mode === 'selectors') { + const [duration_ms, r] = timeit(() => whatsabi.selectorsFromBytecode(code)) + return [duration_ms, r.map(x => x.slice(2))]; // remove '0x' prefix + } else if (mode === 'mutability') { + const [duration_ms, abi] = timeit(() => whatsabi.abiFromBytecode(code)); const smut = Object.fromEntries(abi.filter((v) => v.type == 'function').map((v) => [v.selector, v.stateMutability])); - return Object.fromEntries(selectors[fname].map((s) => { - return [s, smut[`0x${s}`] || 'selnotfound']; - })); + return [duration_ms, Object.fromEntries(selectors[fname][1].map((s) => [s, smut[`0x${s}`] || 'selnotfound']))]; + } else { + throw 'unsupported mode'; } }