From fae1d3c40879309ee17e784efd00daab53fa3f55 Mon Sep 17 00:00:00 2001 From: acheronfail Date: Thu, 11 Jan 2024 21:12:02 +1030 Subject: [PATCH] also include bytecode sizes --- .gitignore | 4 +- justfile | 93 ++++++++++++++++++++++++++++++---------------- scripts/summary.js | 58 ++++++++++++++++++++--------- 3 files changed, 105 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index 7f8a616..37b425c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ results/*.json *.out *.dump *.beam +*.tasty *.class *.json *.o @@ -15,4 +16,5 @@ scala/ scripts/summary.md CMD VERSION -STATS \ No newline at end of file +STATS +SIZE diff --git a/justfile b/justfile index 6b73ca2..7a844d5 100644 --- a/justfile +++ b/justfile @@ -32,20 +32,25 @@ docker-measure what: docker-measure-all: docker run --rm -ti --platform 'linux/amd64' -v "$PWD:{{mount}}" {{tag}} just measure-all -measure-all: +_all: + just -l | grep -v 'build-all' | grep 'build-' | cut -d'-' -f2- | xargs + +build what: + rm -f CMD VERSION STATS SIZE + just build-{{what}} + +build-all: #!/usr/bin/env bash - set -exuo pipefail + set -euxo pipefail - for lang in $(just -l | grep 'build-' | cut -d'-' -f2- | xargs); do - just test "$lang"; - just measure "$lang"; + failed=() + for lang in $(just _all); do + if ! just build "$lang"; then + failed+=("$lang") + fi done - cd scripts && npm start - -build what: - rm -f count CMD VERSION - just build-{{what}} + echo "${failed[@]}" run what: just build {{what}} @@ -82,12 +87,23 @@ measure what: hyperfine $args --shell=none --export-json "$out" "$(cat CMD)" jq '.results[0] | del(.exit_codes)' "$out" | sponge "$out" jq '. += {"name":"{{what}}","version":"'"$(cat VERSION)"'"}' "$out" | sponge "$out" - if [ -x count ]; then - jq '. += {"size":'"$(stat -c '%s' count)"'}' "$out" | sponge "$out" + if [[ -f SIZE ]]; then + jq '. += {"size":"'"$(cat SIZE)"'"}' "$out" | sponge "$out" fi timers $(cat CMD) >/dev/null 2> STATS jq '. += {"max_rss":'$(rg -oP '(?:max_rss:\s*)(\d+)' -r '$1' ./STATS)'}' "$out" | sponge "$out" +measure-all: + #!/usr/bin/env bash + set -exuo pipefail + + for lang in $(just _all); do + just test "$lang"; + just measure "$lang"; + done + + cd scripts && npm start + summary: cd scripts && npm start -- --results ../results cat scripts/summary.md @@ -126,51 +142,66 @@ test what: fi done +test-all: + #!/usr/bin/env bash + set -euxo pipefail + + for lang in $(just _all); do + just test "$lang"; + done + +# total byte size of all passed files +_size +files: (_check "paste" "bc" "stat") + stat -c '%s' {{files}} | paste -sd+ | bc > SIZE +# define size type (used in summary) +_sizet type: + echo -n {{type}} >> SIZE + # languages -build-c-gcc: (_check "gcc") +build-c-gcc: (_check "gcc") && (_size "./count") gcc --version | head -1 > VERSION gcc -O3 -o count ./count.c echo './count {{i}}' > CMD -build-c-clang: (_check "clang") +build-c-clang: (_check "clang") && (_size "./count") clang --version | head -1 > VERSION clang -O3 -o count ./count.c echo './count {{i}}' > CMD -build-cpp-gcc: (_check "g++") +build-cpp-gcc: (_check "g++") && (_size "./count") g++ --version | head -1 > VERSION g++ -O3 -o count ./count.cpp echo './count {{i}}' > CMD -build-cpp-clang: (_check "clang++") +build-cpp-clang: (_check "clang++") && (_size "./count") clang++ --version | head -1 > VERSION clang++ -O3 -o count ./count.cpp echo './count {{i}}' > CMD -build-rust: (_check "rustc") +build-rust: (_check "rustc") && (_size "./count") rustc --version > VERSION rustc -C opt-level=3 ./count.rs echo './count {{i}}' > CMD -build-fortran: (_check "gfortran") +build-fortran: (_check "gfortran") && (_size "./count") gfortran --version | head -1 > VERSION gfortran -O3 -o count ./count.f90 echo './count {{i}}' > CMD -build-java: (_check "javac java") +build-java: (_check "javac java") && (_size "./count.java") (_sizet "bytecode") javac --version > VERSION java --version | head -1 >> VERSION javac count.java echo 'java count {{i}}' > CMD -build-scala: (_check "scalac scala") +build-scala: (_check "scalac scala") && (_size "count.class" "count$.class" "count.tasty") (_sizet "bytecode") scalac -version > VERSION 2>&1 scala -version >> VERSION 2>&1 scalac count.scala echo 'scala count {{i}}' > CMD -build-kotlin: (_check "kotlinc java") +build-kotlin: (_check "kotlinc java") && (_size "count.jar") (_sizet "bytecode") kotlinc -version > VERSION 2>&1 java --version | head -1 >> VERSION kotlinc count.kt -include-runtime -d count.jar @@ -196,7 +227,7 @@ build-bun: (_check "bun") bun --version > VERSION echo 'bun run count.js {{i}}' > CMD -build-zig: (_check "zig") +build-zig: (_check "zig") && (_size "count") zig version > VERSION zig build-exe -O ReleaseFast ./count.zig echo './count {{i}}' > CMD @@ -205,12 +236,12 @@ build-perl: (_check "perl") perl --version | grep version > VERSION echo 'perl ./count.pl {{i}}' > CMD -build-haskell: (_check "ghc") +build-haskell: (_check "ghc") && (_size "count") ghc --version > VERSION ghc count.hs echo './count {{i}}' > CMD -build-go: (_check "go") +build-go: (_check "go") && (_size "count") go version > VERSION go build -o count count.go echo './count {{i}}' > CMD @@ -219,7 +250,7 @@ build-php: (_check "php") php --version | head -1 > VERSION echo 'php ./count.php {{i}}' > CMD -build-erlang: (_check "erlc erl") +build-erlang: (_check "erlc erl") && (_size "count.beam") (_sizet "bytecode") erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell > VERSION erlc count.erl echo 'erl -noshell -s count start {{i}}' > CMD @@ -228,13 +259,13 @@ build-crystal: (_check "crystal") crystal version | xargs > VERSION echo 'crystal run ./count.cr -- {{i}}' > CMD -build-assembly: (_check "nasm") +build-assembly: (_check "nasm") && (_size "count") nasm --version > VERSION nasm -f bin -o count ./count.asm chmod +x ./count echo './count {{i}}' > CMD -build-cobol: (_check "cobc") +build-cobol: (_check "cobc") && (_size "count") cobc --version | head -1 > VERSION cobc -O3 -free -x -o count count.cbl echo './count {{i}}' > CMD @@ -247,12 +278,12 @@ build-coffeescript: (_check "coffee") coffee --version > VERSION echo 'coffee ./count.coffee {{i}}' > CMD -build-nim: (_check "nim") +build-nim: (_check "nim") && (_size "count") nim --version | head -1 > VERSION nim compile --opt:speed ./count.nim echo './count {{i}}' > CMD -build-prolog: (_check "swipl") +build-prolog: (_check "swipl") && (_size "count") swipl --version > VERSION swipl -s count.pro -g "main" -t halt -- 1 echo './count {{i}}' > CMD @@ -265,7 +296,7 @@ build-tcl: (_check "tclsh") echo 'puts $tcl_version;exit 0' | tclsh > VERSION echo 'tclsh ./count.tcl {{i}}' > CMD -build-pascal: (_check "fpc") +build-pascal: (_check "fpc") && (_size "count") fpc -iW > VERSION fpc -O3 ./count.pas echo './count {{i}}' > CMD @@ -278,7 +309,7 @@ build-forth: (_check "gforth") gforth --version > VERSION 2>&1 echo 'gforth ./count.fth {{i}}' > CMD -build-csharp: (_check "mcs mono") +build-csharp: (_check "mcs mono") && (_size "count.exe") (_sizet "bytecode") mcs --version > VERSION mono --version | head -1 >> VERSION mcs -o+ ./count.cs diff --git a/scripts/summary.js b/scripts/summary.js index ed873cf..3602291 100644 --- a/scripts/summary.js +++ b/scripts/summary.js @@ -10,12 +10,23 @@ const args = minimist(process.argv.slice(2)); const resultsDir = args.results; if (!resultsDir) throw new Error('Please pass --results'); +const SIZE_T_BINARY = 'binary'; +const sizeTypes = new Set([SIZE_T_BINARY]); const results = await Promise.all( readdirSync(resultsDir) .filter((name) => name.endsWith('.json')) .map(async (name) => { const text = await readFile(join(resultsDir, name), 'utf-8'); - return JSON.parse(text); + const json = JSON.parse(text); + + if ('size' in json) { + const [bytes, type] = json.size.split('\n'); + json.size = parseInt(bytes); + json.sizeType = type ?? SIZE_T_BINARY; + sizeTypes.add(json.sizeType); + } + + return json; }) ); @@ -27,8 +38,8 @@ await writeFile( - - + +
Execution timeBinary sizeMax Memory Usage*Binary size1Max Memory Usage2
@@ -49,19 +60,30 @@ ${markdownTable( -${markdownTable( - [ - ['#', 'name', 'size'], - ...results - .slice() - .sort((a, b) => a.name.localeCompare(b.name)) - .sort((a, b) => (a.size ?? Infinity) - (b.size ?? Infinity)) - .map(({ name, size }, i) => [i + 1, wrap(name), size ? formatSize(size, { minimumFractionDigits: 7 }) : '-']), - ], - { - align: ['l', 'l', 'r'], - } -)} +${[...sizeTypes.values()] + .map( + (sizeType) => + `**${sizeType}**:\n` + + markdownTable( + [ + ['#', 'name', 'size'], + ...results + .slice() + .filter((x) => x.sizeType === sizeType) + .sort((a, b) => a.name.localeCompare(b.name)) + .sort((a, b) => (a.size ?? Infinity) - (b.size ?? Infinity)) + .map(({ name, size }, i) => [ + i + 1, + wrap(name), + size ? formatSize(size, { minimumFractionDigits: 7 }) : '-', + ]), + ], + { + align: ['l', 'l', 'r'], + } + ) + ) + .join('\n\n')} @@ -83,8 +105,8 @@ ${markdownTable(
-> \`*\`: Getting the \`max_rss\` isn't 100% reliable for very small binary sizes -> this appears to be [a limitation of the linux kernel](https://github.com/acheronfail/timeRS/blob/master/LIMITATIONS.md). +> - 1: only includes compiled files (i.e., does not include runtimes or libraries required for execution) +> - 2: Getting the \`max_rss\` isn't 100% reliable for very small binary sizes. This appears to be [a limitation of the linux kernel](https://github.com/acheronfail/timeRS/blob/master/LIMITATIONS.md). ${markdownTable( [