Use native Python and update results (#19)

ziyadedher · Oct 14, 2022 · ea26bc0 · ea26bc0
1 parent da8eb4f
commit ea26bc0
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -6,16 +6,16 @@
 
 evm-bench makes it easy to compare EVM performance in a scalable, standardized, and portable way.
 
-|                         | py-evm.cpython | py-evm.pypy | pyrevm  | revm   |
-| ----------------------- | -------------- | ----------- | ------- | ------ |
-| erc20.approval-transfer | 1.9362s        | 424.2ms     | 16.8ms  | 9.8ms  |
-| erc20.mint              | 1.8968s        | 374ms       | 15ms    | 5.6ms  |
-| erc20.transfer          | 2.7296s        | 482.6ms     | 23.4ms  | 11.4ms |
-| snailtracer             | 37.861s        | 7.409s      | 131.7ms | 60.7ms |
-| ten-thousand-hashes     | 4.1496s        | 632ms       | 17ms    | 6.8ms  |
-|                         |                |             |         |        |
-| **sum**                 | 48.6s          | 9.32s       | 203ms   | 94.3ms |
-| **relative**            | 515x           | 98.8x       | 2.15x   | 1.00x  |
+|                         | py-evm.cpython | py-evm.pypy | pyrevm | revm   |
+| ----------------------- | -------------- | ----------- | ------ | ------ |
+| erc20.approval-transfer | 1.4258s        | 377.2ms     | 16.8ms | 10ms   |
+| erc20.mint              | 1.4092s        | 347.8ms     | 14.4ms | 5.6ms  |
+| erc20.transfer          | 1.9988s        | 420.8ms     | 22.6ms | 11.4ms |
+| snailtracer             | 17.857s        | 5.17s       | 138ms  | 58ms   |
+| ten-thousand-hashes     | 3.4344s        | 665.6ms     | 18.2ms | 7ms    |
+|                         |                |             |        |        |
+| **sum**                 | 26.1252s       | 6.9814s     | 210ms  | 92ms   |
+| **relative**            | 284x           | 75.9x       | 2.28x  | 1.00x  |
 
 To reproduce these results, check out [usage with the evm-bench suite below](#with-the-evm-bench-suite).
 

diff --git a/benchmarks/snailtracer/benchmark.evm-bench.json b/benchmarks/snailtracer/benchmark.evm-bench.json
@@ -1,7 +1,7 @@
 {
   "$schema": "../schema.json",
   "name": "snailtracer",
-  "num-runs": 3,
+  "num-runs": 1,
   "solc-version": "0.4.26",
   "contract": "SnailTracer.sol",
   "calldata": "30627b7c"

diff --git a/runners/py-evm/cpython/entry.sh b/runners/py-evm/cpython/entry.sh
@@ -4,7 +4,7 @@ set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 cd $SCRIPT_DIR
-poetry env use python3.9 >&2
+poetry env use python3 >&2
 poetry install >&2
 poetry update >&2
 poetry run python ../runner.py $@
diff --git a/runners/py-evm/poetry.lock b/runners/py-evm/poetry.lock
diff --git a/runners/py-evm/pyproject.toml b/runners/py-evm/pyproject.toml
@@ -5,7 +5,7 @@ description = ""
 authors = ["Ziyad Edher <[email protected]>"]
 
 [tool.poetry.dependencies]
-python = "^3.9,<3.10"
+python = "^3.9"
 py-evm = "^0.6.0-alpha.1"
 
 [tool.poetry.dev-dependencies]

diff --git a/runners/py-evm/runner.py b/runners/py-evm/runner.py
@@ -1,6 +1,6 @@
-import argparse
-from typing import Final, cast
+from typing import cast, Final
 
+import argparse
 import pathlib
 import time
 
@@ -44,7 +44,7 @@ def _construct_chain() -> eth.chains.base.MiningChain:
     chain = chain_class.from_genesis(
         eth.db.atomic.AtomicDB(),
         genesis_params={
-            "difficulty": 100,
+            "difficulty": 1,
             "gas_limit": 2 * GAS_LIMIT,
         },
     )

diff --git a/src/results.rs b/src/results.rs
@@ -101,31 +101,52 @@ pub fn print_results(results_file_path: &Path) -> Result<(), Box<dyn error::Erro
     let mut runs = results.runs.into_iter().collect::<Vec<_>>();
     runs.sort_by_key(|(b, _)| b.clone());
 
+    let mut runner_times = HashMap::<String, Vec<Duration>>::new();
+
     let mut builder = Builder::default();
     for (benchmark_name, benchmark_runs) in runs.iter() {
         let vals = runner_names.iter().map(|runner_name| {
             let run = benchmark_runs.get(runner_name)?;
-            Some(
-                run.run_times
-                    .iter()
-                    .fold(Duration::ZERO, |a, v| a + v.clone())
-                    .div_f64(run.run_times.len() as f64),
-            )
+            let avg_run_time = run
+                .run_times
+                .iter()
+                .fold(Duration::ZERO, |a, v| a + v.clone())
+                .div_f64(run.run_times.len() as f64);
+            runner_times
+                .entry(runner_name.clone())
+                .or_default()
+                .push(avg_run_time);
+            Some(avg_run_time)
         });
+
         let mut record = vec![benchmark_name.clone()];
         record.extend(
             vals.map(|val| Some(format!("{:?}", val?)))
-                .map(|s| s.unwrap_or("".into())),
+                .map(|s| s.unwrap_or_default()),
         );
         builder.add_record(record);
     }
 
+    let average_runner_times = runner_times
+        .into_iter()
+        .map(|(name, times)| (name, times.iter().sum::<Duration>()))
+        .collect::<HashMap<String, Duration>>();
+    let mut record = vec!["sum".to_string()];
+    record.extend(
+        runner_names
+            .iter()
+            .map(|runner_name| average_runner_times.get(runner_name))
+            .map(|val| Some(format!("{:?}", val?)))
+            .map(|s| s.unwrap_or_default()),
+    );
+    builder.add_record(record);
+
     let mut columns = vec!["".to_owned()];
     columns.extend(runner_names);
     builder.set_columns(columns);
 
     let mut table = builder.build();
-    table.with(Style::rounded());
+    table.with(Style::markdown());
     println!("{}", table);
 
     Ok(())