Skip to content

Commit

Permalink
also include bytecode sizes
Browse files Browse the repository at this point in the history
  • Loading branch information
@acheronfail
acheronfail committed Jan 11, 2024
1 parent fae13eb commit fae1d3c
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 50 deletions.
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ results/*.json
*.out
*.dump
*.beam
*.tasty
*.class
*.json
*.o
Expand All @@ -15,4 +16,5 @@ scala/
scripts/summary.md
CMD
VERSION
STATS
STATS
SIZE
93 changes: 62 additions & 31 deletions justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,20 +32,25 @@ docker-measure what:
docker-measure-all:
docker run --rm -ti --platform 'linux/amd64' -v "$PWD:{{mount}}" {{tag}} just measure-all

measure-all:
_all:
just -l | grep -v 'build-all' | grep 'build-' | cut -d'-' -f2- | xargs

build what:
rm -f CMD VERSION STATS SIZE
just build-{{what}}

build-all:
#!/usr/bin/env bash
set -exuo pipefail
set -euxo pipefail
for lang in $(just -l | grep 'build-' | cut -d'-' -f2- | xargs); do
just test "$lang";
just measure "$lang";
failed=()
for lang in $(just _all); do
if ! just build "$lang"; then
failed+=("$lang")
fi
done

cd scripts && npm start

build what:
rm -f count CMD VERSION
just build-{{what}}
echo "${failed[@]}"

run what:
just build {{what}}
Expand Down Expand Up @@ -82,12 +87,23 @@ measure what:
hyperfine $args --shell=none --export-json "$out" "$(cat CMD)"
jq '.results[0] | del(.exit_codes)' "$out" | sponge "$out"
jq '. += {"name":"{{what}}","version":"'"$(cat VERSION)"'"}' "$out" | sponge "$out"
if [ -x count ]; then
jq '. += {"size":'"$(stat -c '%s' count)"'}' "$out" | sponge "$out"
if [[ -f SIZE ]]; then
jq '. += {"size":"'"$(cat SIZE)"'"}' "$out" | sponge "$out"
fi
timers $(cat CMD) >/dev/null 2> STATS
jq '. += {"max_rss":'$(rg -oP '(?:max_rss:\s*)(\d+)' -r '$1' ./STATS)'}' "$out" | sponge "$out"

measure-all:
#!/usr/bin/env bash
set -exuo pipefail
for lang in $(just _all); do
just test "$lang";
just measure "$lang";
done

cd scripts && npm start

summary:
cd scripts && npm start -- --results ../results
cat scripts/summary.md
Expand Down Expand Up @@ -126,51 +142,66 @@ test what:
fi
done

test-all:
#!/usr/bin/env bash
set -euxo pipefail
for lang in $(just _all); do
just test "$lang";
done

# total byte size of all passed files
_size +files: (_check "paste" "bc" "stat")
stat -c '%s' {{files}} | paste -sd+ | bc > SIZE
# define size type (used in summary)
_sizet type:
echo -n {{type}} >> SIZE

# languages

build-c-gcc: (_check "gcc")
build-c-gcc: (_check "gcc") && (_size "./count")
gcc --version | head -1 > VERSION
gcc -O3 -o count ./count.c
echo './count {{i}}' > CMD

build-c-clang: (_check "clang")
build-c-clang: (_check "clang") && (_size "./count")
clang --version | head -1 > VERSION
clang -O3 -o count ./count.c
echo './count {{i}}' > CMD

build-cpp-gcc: (_check "g++")
build-cpp-gcc: (_check "g++") && (_size "./count")
g++ --version | head -1 > VERSION
g++ -O3 -o count ./count.cpp
echo './count {{i}}' > CMD

build-cpp-clang: (_check "clang++")
build-cpp-clang: (_check "clang++") && (_size "./count")
clang++ --version | head -1 > VERSION
clang++ -O3 -o count ./count.cpp
echo './count {{i}}' > CMD

build-rust: (_check "rustc")
build-rust: (_check "rustc") && (_size "./count")
rustc --version > VERSION
rustc -C opt-level=3 ./count.rs
echo './count {{i}}' > CMD

build-fortran: (_check "gfortran")
build-fortran: (_check "gfortran") && (_size "./count")
gfortran --version | head -1 > VERSION
gfortran -O3 -o count ./count.f90
echo './count {{i}}' > CMD

build-java: (_check "javac java")
build-java: (_check "javac java") && (_size "./count.java") (_sizet "bytecode")
javac --version > VERSION
java --version | head -1 >> VERSION
javac count.java
echo 'java count {{i}}' > CMD

build-scala: (_check "scalac scala")
build-scala: (_check "scalac scala") && (_size "count.class" "count$.class" "count.tasty") (_sizet "bytecode")
scalac -version > VERSION 2>&1
scala -version >> VERSION 2>&1
scalac count.scala
echo 'scala count {{i}}' > CMD

build-kotlin: (_check "kotlinc java")
build-kotlin: (_check "kotlinc java") && (_size "count.jar") (_sizet "bytecode")
kotlinc -version > VERSION 2>&1
java --version | head -1 >> VERSION
kotlinc count.kt -include-runtime -d count.jar
Expand All @@ -196,7 +227,7 @@ build-bun: (_check "bun")
bun --version > VERSION
echo 'bun run count.js {{i}}' > CMD

build-zig: (_check "zig")
build-zig: (_check "zig") && (_size "count")
zig version > VERSION
zig build-exe -O ReleaseFast ./count.zig
echo './count {{i}}' > CMD
Expand All @@ -205,12 +236,12 @@ build-perl: (_check "perl")
perl --version | grep version > VERSION
echo 'perl ./count.pl {{i}}' > CMD

build-haskell: (_check "ghc")
build-haskell: (_check "ghc") && (_size "count")
ghc --version > VERSION
ghc count.hs
echo './count {{i}}' > CMD

build-go: (_check "go")
build-go: (_check "go") && (_size "count")
go version > VERSION
go build -o count count.go
echo './count {{i}}' > CMD
Expand All @@ -219,7 +250,7 @@ build-php: (_check "php")
php --version | head -1 > VERSION
echo 'php ./count.php {{i}}' > CMD

build-erlang: (_check "erlc erl")
build-erlang: (_check "erlc erl") && (_size "count.beam") (_sizet "bytecode")
erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell > VERSION
erlc count.erl
echo 'erl -noshell -s count start {{i}}' > CMD
Expand All @@ -228,13 +259,13 @@ build-crystal: (_check "crystal")
crystal version | xargs > VERSION
echo 'crystal run ./count.cr -- {{i}}' > CMD

build-assembly: (_check "nasm")
build-assembly: (_check "nasm") && (_size "count")
nasm --version > VERSION
nasm -f bin -o count ./count.asm
chmod +x ./count
echo './count {{i}}' > CMD

build-cobol: (_check "cobc")
build-cobol: (_check "cobc") && (_size "count")
cobc --version | head -1 > VERSION
cobc -O3 -free -x -o count count.cbl
echo './count {{i}}' > CMD
Expand All @@ -247,12 +278,12 @@ build-coffeescript: (_check "coffee")
coffee --version > VERSION
echo 'coffee ./count.coffee {{i}}' > CMD

build-nim: (_check "nim")
build-nim: (_check "nim") && (_size "count")
nim --version | head -1 > VERSION
nim compile --opt:speed ./count.nim
echo './count {{i}}' > CMD

build-prolog: (_check "swipl")
build-prolog: (_check "swipl") && (_size "count")
swipl --version > VERSION
swipl -s count.pro -g "main" -t halt -- 1
echo './count {{i}}' > CMD
Expand All @@ -265,7 +296,7 @@ build-tcl: (_check "tclsh")
echo 'puts $tcl_version;exit 0' | tclsh > VERSION
echo 'tclsh ./count.tcl {{i}}' > CMD

build-pascal: (_check "fpc")
build-pascal: (_check "fpc") && (_size "count")
fpc -iW > VERSION
fpc -O3 ./count.pas
echo './count {{i}}' > CMD
Expand All @@ -278,7 +309,7 @@ build-forth: (_check "gforth")
gforth --version > VERSION 2>&1
echo 'gforth ./count.fth {{i}}' > CMD

build-csharp: (_check "mcs mono")
build-csharp: (_check "mcs mono") && (_size "count.exe") (_sizet "bytecode")
mcs --version > VERSION
mono --version | head -1 >> VERSION
mcs -o+ ./count.cs
Expand Down
58 changes: 40 additions & 18 deletions scripts/summary.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,23 @@ const args = minimist(process.argv.slice(2));
const resultsDir = args.results;
if (!resultsDir) throw new Error('Please pass --results');

const SIZE_T_BINARY = 'binary';
const sizeTypes = new Set([SIZE_T_BINARY]);
const results = await Promise.all(
readdirSync(resultsDir)
.filter((name) => name.endsWith('.json'))
.map(async (name) => {
const text = await readFile(join(resultsDir, name), 'utf-8');
return JSON.parse(text);
const json = JSON.parse(text);

if ('size' in json) {
const [bytes, type] = json.size.split('\n');
json.size = parseInt(bytes);
json.sizeType = type ?? SIZE_T_BINARY;
sizeTypes.add(json.sizeType);
}

return json;
})
);

Expand All @@ -27,8 +38,8 @@ await writeFile(
<table>
<tr>
<th>Execution time</th>
<th>Binary size</th>
<th>Max Memory Usage*</th>
<th>Binary size<sup>1</sup></th>
<th>Max Memory Usage<sup>2</sup></th>
</tr>
<tr>
<td>
Expand All @@ -49,19 +60,30 @@ ${markdownTable(
</td>
<td>
${markdownTable(
[
['#', 'name', 'size'],
...results
.slice()
.sort((a, b) => a.name.localeCompare(b.name))
.sort((a, b) => (a.size ?? Infinity) - (b.size ?? Infinity))
.map(({ name, size }, i) => [i + 1, wrap(name), size ? formatSize(size, { minimumFractionDigits: 7 }) : '-']),
],
{
align: ['l', 'l', 'r'],
}
)}
${[...sizeTypes.values()]
.map(
(sizeType) =>
`**${sizeType}**:\n` +
markdownTable(
[
['#', 'name', 'size'],
...results
.slice()
.filter((x) => x.sizeType === sizeType)
.sort((a, b) => a.name.localeCompare(b.name))
.sort((a, b) => (a.size ?? Infinity) - (b.size ?? Infinity))
.map(({ name, size }, i) => [
i + 1,
wrap(name),
size ? formatSize(size, { minimumFractionDigits: 7 }) : '-',
]),
],
{
align: ['l', 'l', 'r'],
}
)
)
.join('\n\n')}
</td>
<td>
Expand All @@ -83,8 +105,8 @@ ${markdownTable(
</tr>
</table>
> \`*\`: Getting the \`max_rss\` isn't 100% reliable for very small binary sizes
> this appears to be [a limitation of the linux kernel](https://github.com/acheronfail/timeRS/blob/master/LIMITATIONS.md).
> - <sup>1</sup>: only includes compiled files (i.e., does not include runtimes or libraries required for execution)
> - <sup>2</sup>: Getting the \`max_rss\` isn't 100% reliable for very small binary sizes. This appears to be [a limitation of the linux kernel](https://github.com/acheronfail/timeRS/blob/master/LIMITATIONS.md).
${markdownTable(
[
Expand Down

0 comments on commit fae1d3c

Please sign in to comment.