From 0caaad9ce6cee6d166d3f5dbb5fb551c2447abd6 Mon Sep 17 00:00:00 2001 From: Maxim Andreev Date: Thu, 5 Dec 2024 05:49:48 +0000 Subject: [PATCH] update benchmark results --- README.md | 127 ++++++++++-------------- benchmark/compare.py | 6 +- benchmark/providers/whatsabi/Dockerfile | 2 +- 3 files changed, 56 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index ad0e7ea..5372479 100644 --- a/README.md +++ b/README.md @@ -37,15 +37,7 @@ console.log( contractInfo(code, {selectors:true, arguments:true, stateMutability // arguments: 'uint32,address,uint224', // stateMutability: 'pure' // }, -// { -// selector: 'b69ef8a8', -// bytecodeOffset: 68, -// arguments: '', -// stateMutability: 'pure' -// } -// ], -// storage: undefined -// } +// ... ``` ### Rust @@ -68,15 +60,7 @@ println!("{:?}", evmole::contract_info( // arguments: Some([Uint(32), Address, Uint(224)]), // state_mutability: Some(Pure) // }, -// Function { -// selector: [182, 158, 248, 168], -// bytecode_offset: 68, -// arguments: Some([]), -// state_mutability: Some(Pure) -// } -// ]), -// storage: None -// } +// ... ``` ### Python @@ -97,14 +81,7 @@ print( contract_info(code, selectors=True, arguments=True, state_mutability=True # bytecode_offset=52, # arguments=uint32,address,uint224, # state_mutability=pure), -# Function( -# selector=b69ef8a8, -# bytecode_offset=68, -# arguments=, -# state_mutability=pure) -# ], -# storage=None -# ) +# ... ``` ### Foundry @@ -181,12 +158,12 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Time - 0.5s · 0.7s · 0.6s - 3.2s - 47s(*) - 0.5s - 341s(*) - 1.8s + 0.4s · 0.6s · 0.4s + 2.8s + 35s(*) + 0.4s + 343s(*) + 1.6s @@ -228,12 +205,12 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Time - 4.5s · 8.5s · 7.8s - 54s - 2472s(*) - 6.1s - 8576s(*) - 46s + 4.3s · 9.0s · 7.2s + 53s + 1124s(*) + 5.8s + 8572s(*) + 44s @@ -275,12 +252,12 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Time - 0.4s · 0.6s · 0.4s - 2.4s - 57s(*) - 0.4s - 27s(*) - 1.1s + 0.3s · 0.6s · 0.4s + 2.0s + 46s(*) + 0.3s + 28s(*) + 1.0s @@ -304,37 +281,37 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Time - 1.1s · 4.8s · 1.4s - 342s(*) - 0.7s + 1.0s · 2.1s · 1.1s + 344s(*) + 0.5s random50k
1171102
functions Errors - 4.5% 🥇
52662 + 4.5% 🥇
52670 19.4%
227612 54.9%
643213 Time - 25s · 127 · 32 - 8544s(*) - 9.5s + 20s · 49s · 24s + 8517s(*) + 8.1s vyper
21244
functions Errors - 48.5% 🥇
10300 + 48.5% 🥇
10299 100.0%
21244 56.8%
12077 Time - 0.8s · 3.0s · 1.0s + 0.7s · 1.4s · 0.8s 28s(*) - 0.6s + 0.5s @@ -365,45 +342,45 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Errors strict - 18.7% 🥇
4558 - 79.3%
19370 + 18.6% 🥇
4555 + 79.4%
19393 59.0%
14417 54.9%
13403 60.9%
14864 Time - 9.5s · 19s · 9.4s + 8.9s · 16s · 8.9s 3.8s - 46s(*) + 33s(*) 339s(*) - 0.7s + 0.5s random50k
1160861
functions Errors - 0.0% 🥇
35 + 0.0% 🥇
44 30.2%
351060 0.3%
3370 - 11.6%
134195 + 11.6%
134196 2.2%
24961 Errors strict - 6.7% 🥇
77945 - 58.1%
674922 + 6.8% 🥇
78923 + 58.2%
675111 55.7%
646831 - 27.7%
321494 + 27.7%
321495 57.7%
670318 Time - 214s · 471s · 240s - 85s - 2335s(*) - 8151s(*) - 9.4s + 169s · 334s · 172s + 88s + 1131s(*) + 8267s(*) + 8.2s @@ -417,7 +394,7 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Errors strict - 4.2% 🥇
888 + 4.0% 🥇
854 100.0%
21166 90.2%
19092 100.0%
21166 @@ -425,17 +402,17 @@ $ cast selectors --resolve $(cast code 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc Time - 10s · 15s · 10s - 2.4s - 52s(*) + 9.5s · 13s · 9.7s + 2.1s + 38s(*) 28s(*) - 0.6s + 0.5s See [benchmark/README.md](./benchmark/) for the methodology and commands to reproduce these results -versions: evmole v0.5.1; whatsabi v0.14.1; sevm v0.7.3; evm-hound-rs v0.1.4; heimdall-rs v0.8.4 +versions: evmole v0.6.0; whatsabi v0.17.0; sevm v0.7.3; evm-hound-rs v0.1.4; heimdall-rs v0.8.4 (*): sevm and heimdall-rs are full decompilers, not limited to extracting function selectors diff --git a/benchmark/compare.py b/benchmark/compare.py index e5864de..271d358 100644 --- a/benchmark/compare.py +++ b/benchmark/compare.py @@ -269,11 +269,11 @@ def show_arguments_or_mutability(providers: list[str], all_results: list, show_e cfg = parser.parse_args() if cfg.providers is None: if cfg.mode == 'selectors': - cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'evm-hound-rs', 'simple'] + cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'evm-hound-rs', 'heimdall-rs', 'simple'] elif cfg.mode == 'arguments': - cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'simple'] + cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'heimdall-rs', 'simple'] elif cfg.mode == 'mutability': - cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'simple'] + cfg.providers = ['etherscan', 'evmole-rs', 'evmole-js', 'evmole-py', 'whatsabi', 'sevm', 'heimdall-rs', 'simple'] else: cfg.providers = [] print('Config:') diff --git a/benchmark/providers/whatsabi/Dockerfile b/benchmark/providers/whatsabi/Dockerfile index 7d8b161..ef262e3 100644 --- a/benchmark/providers/whatsabi/Dockerfile +++ b/benchmark/providers/whatsabi/Dockerfile @@ -2,6 +2,6 @@ FROM node:22 WORKDIR /app -RUN npm install @shazow/whatsabi@0.14.1 +RUN npm install @shazow/whatsabi@0.17.0 COPY main.mjs /app ENTRYPOINT ["node", "/app/main.mjs"]