[DMS-72] Implement benchmarks for PersistentOrderedMap

collections/Makefile

@@ -9,7 +9,7 @@ motoko:
 rust:
 	$(call build,rust)
 
-build: motoko rust
+build: motoko # rust
 
 perf:
 	$(call perf,collections,perf.sh)

collections/motoko/dfx.json

@@ -1,40 +1,16 @@
 {
   "canisters": {
-    "hashmap": {
-      "type": "motoko",
-      "main": "src/hashmap.mo"
-    },
-    "triemap": {
-      "type": "motoko",
-      "main": "src/triemap.mo"
-    },
     "rbtree": {
       "type": "motoko",
       "main": "src/rbtree.mo"
     },
-    "splay": {
-      "type": "motoko",
-      "main": "src/splay.mo"
-    },
-    "heap": {
-      "type": "motoko",
-      "main": "src/heap.mo"
-    },
-    "btreemap": {
-      "type": "motoko",
-      "main": "src/btreemap.mo"
-    },
-    "zhenya_hashmap": {
-      "type": "motoko",
-      "main": "src/ZhenyaHashmap.mo"
-    },
-    "buffer": {
+    "persistentmap": {
       "type": "motoko",
-      "main": "src/buffer.mo"
+      "main": "src/persistentmap.mo"
     },
-    "vector": {
+    "persistentmap_baseline": {
       "type": "motoko",
-      "main": "src/vector.mo"
+      "main": "src/persistentmap_baseline.mo"
     }
   },
   "defaults": {

collections/motoko/mops.template.toml

@@ -1,5 +1,6 @@
 [dependencies]
-base = "https://github.com/dfinity/motoko-base#moc-$MOC_VERSION"
+base = "https://github.com/serokell/motoko-base#$TEST_BRANCH"
+base-unopt = "https://github.com/serokell/motoko-base#$BASELINE_BRANCH"
 stableheapbtreemap = "1.3.0"
 map = "9.0.0"
 splay = "0.1.0"

collections/motoko/src/persistentmap.mo

@@ -0,0 +1,58 @@
+import Map "mo:base/PersistentOrderedMap";
+import Nat64 "mo:base/Nat64";
+import Nat32 "mo:base/Nat32";
+import Iter "mo:base/Iter";
+import Option "mo:base/Option";
+import Random "random";
+import Profiling "../../../utils/motoko/Profiling";
+
+actor {
+    stable let profiling = Profiling.init();
+
+    let mapOps = Map.MapOps<Nat64>(Nat64.compare);
+    stable var rbMap = mapOps.empty<Nat64>();
+    let rand = Random.new(null, 42);
+
+    public func generate(size: Nat32) : async () {
+        let rand = Random.new(?size, 1);
+        let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
+        rbMap := mapOps.fromIter(iter);
+    };
+    public query func get_mem() : async (Nat,Nat,Nat) {
+        Random.get_memory()
+    };
+    public func batch_get(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func batch_put(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            let k = Option.get<Nat64>(rand.next(), 0);
+            rbMap := mapOps.put(rbMap, k, k);
+        }
+    };
+    public func batch_remove(n : Nat) : async () {
+        let rand = Random.new(null, 1);
+        for (_ in Iter.range(1, n)) {
+            rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func foldLeft() : async () {
+        ignore mapOps.foldLeft<Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func foldRight() : async () {
+        ignore mapOps.foldRight<Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func mapfilter() : async () {
+        ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
+          switch (key % 2) {
+            case 1 { null };
+            case _ { ?value }
+          };
+        })
+    };
+    public func map() : async () {
+      ignore mapOps.map<Nat64, Nat64>(rbMap, func (key, value) {key + value})
+    };
+}

collections/motoko/src/persistentmap_baseline.mo

@@ -0,0 +1,58 @@
+import Map "mo:base-unopt/PersistentOrderedMap";
+import Nat64 "mo:base/Nat64";
+import Nat32 "mo:base/Nat32";
+import Iter "mo:base/Iter";
+import Option "mo:base/Option";
+import Random "random";
+import Profiling "../../../utils/motoko/Profiling";
+
+actor {
+    stable let profiling = Profiling.init();
+
+    let mapOps = Map.MapOps<Nat64>(Nat64.compare);
+    stable var rbMap = mapOps.empty<Nat64>();
+    let rand = Random.new(null, 42);
+
+    public func generate(size: Nat32) : async () {
+        let rand = Random.new(?size, 1);
+        let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
+        rbMap := mapOps.fromIter(iter);
+    };
+    public query func get_mem() : async (Nat,Nat,Nat) {
+        Random.get_memory()
+    };
+    public func batch_get(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func batch_put(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            let k = Option.get<Nat64>(rand.next(), 0);
+            rbMap := mapOps.put(rbMap, k, k);
+        }
+    };
+    public func batch_remove(n : Nat) : async () {
+        let rand = Random.new(null, 1);
+        for (_ in Iter.range(1, n)) {
+            rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func foldLeft() : async () {
+        ignore mapOps.foldLeft<Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func foldRight() : async () {
+        ignore mapOps.foldRight<Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func mapfilter() : async () {
+        ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
+          switch (key % 2) {
+            case 1 { null };
+            case _ { ?value }
+          };
+        })
+    };
+    public func map() : async () {
+      ignore mapOps.map<Nat64, Nat64>(rbMap, func (key, value) {key + value})
+    };
+}

collections/motoko/tests/perf.sh

@@ -5,11 +5,13 @@ let hashmap = install(wasm_profiling("../.dfx/local/canisters/hashmap/hashmap.wa
 let triemap = install(wasm_profiling("../.dfx/local/canisters/triemap/triemap.wasm"), null, null);
 let rbtree = install(wasm_profiling("../.dfx/local/canisters/rbtree/rbtree.wasm"), null, null);
 let splay = install(wasm_profiling("../.dfx/local/canisters/splay/splay.wasm"), null, null);
+let persistentmap = install(wasm_profiling("../.dfx/local/canisters/persistentmap/persistentmap.wasm"), null, null);
 
 call hashmap.__toggle_tracing();
 call triemap.__toggle_tracing();
 call rbtree.__toggle_tracing();
 call splay.__toggle_tracing();
+call persistentmap.__toggle_tracing();
 call hashmap.generate(50000);
 let _ = get_memory(hashmap);
 call triemap.generate(50000);
@@ -18,15 +20,19 @@ call rbtree.generate(50000);
 let _ = get_memory(rbtree);
 call splay.generate(50000);
 let _ = get_memory(splay);
+call persistentmap.generate(50000);
+let _ = get_memory(persistentmap);
 
 call hashmap.__toggle_tracing();
 call triemap.__toggle_tracing();
 call rbtree.__toggle_tracing();
 call splay.__toggle_tracing();
+call persistentmap.__toggle_tracing();
 call hashmap.batch_get(50);
 call triemap.batch_get(50);
 call rbtree.batch_get(50);
 call splay.batch_get(50);
+call persistentmap.batch_get(50);
 
 call hashmap.batch_put(50);
 let _ = get_memory(hashmap);
@@ -36,9 +42,11 @@ call rbtree.batch_put(50);
 let _ = get_memory(rbtree);
 call splay.batch_put(50);
 let _ = get_memory(splay);
+call persistentmap.batch_put(50);
+let _ = get_memory(persistentmap);
 
 call hashmap.batch_remove(50);
 call triemap.batch_remove(50);
 call rbtree.batch_remove(50);
 call splay.batch_remove(50);
-
+call persistentmap.batch_remove(50);

collections/perf.sh

@@ -3,25 +3,27 @@ load "../prelude.sh";
 
 // use smaller page_limit to speed things up, since the whole trace is too large even with 256M.
 let mo_config = record { start_page = 16; page_limit = 128 };
-let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
-let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
+// let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
+// let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
 let rbtree = wasm_profiling("motoko/.dfx/local/canisters/rbtree/rbtree.wasm", mo_config);
-let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
-let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
-let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
-let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
-let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
-let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);
-
-let rs_config = record { start_page = 1; page_limit = 128 };
-let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
-let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
-let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
-let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
-let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
-let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
-let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
-let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);
+let persistentmap = wasm_profiling("motoko/.dfx/local/canisters/persistentmap/persistentmap.wasm", mo_config);
+let persistentmap_baseline = wasm_profiling("motoko/.dfx/local/canisters/persistentmap_baseline/persistentmap_baseline.wasm", mo_config);
+// let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
+// let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
+// let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
+// let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
+// let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
+// let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);
+
+// let rs_config = record { start_page = 1; page_limit = 128 };
+// let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
+// let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
+// let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
+// let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
+// let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
+// let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
+// let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
+// let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);
 
 //let movm_rs = wasm_profiling("rust/.dfx/local/canisters/movm/movm.wasm");
 //let movm_dynamic_rs = wasm_profiling("rust/.dfx/local/canisters/movm_dynamic/movm_dynamic.wasm");
@@ -63,12 +65,63 @@ function perf(wasm, title, init, batch) {
   uninstall(cid);
 };
 
-let init_size = 1_000_000;
+
 let batch_size = 50;
-output(file, "\n## Map\n\n| |binary_size|generate 1m|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
+let sizes = vec {100; 1000; 10_000; 100_000; 1_000_000};
+
+output(file, stringify("\n## Map comparison\n\n| |binary_size|generate|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n"));
+
+function compare_rb_maps(init_size){
+  perf(persistentmap_baseline, stringify("persistentmap_baseline_", init_size), init_size, batch_size);
+  perf(persistentmap, stringify("persistentmap_", init_size), init_size, batch_size);
+  perf(rbtree, stringify("rbtree_", init_size), init_size, batch_size);
+};
+
+sizes.map(compare_rb_maps);
+
+function perf_persistent_map(wasm, title, init) {
+  let cid = install(wasm, encode (), null);
+
+  output(file, stringify("|", title, "|", init, "|"));
+  call cid.__toggle_tracing();
+  call cid.generate(init);
+
+  call cid.__toggle_tracing();
+  call cid.foldLeft();
+  let svg = stringify(title, "_foldLeft_", init, ".svg");
+  output(file, stringify("[", __cost__, "](", svg, ")|"));
+  flamegraph(cid, "persistentmap.foldLeft", svg);
+
+  call cid.foldRight();
+  let svg = stringify(title, "_foldRight_", init, ".svg");
+  output(file, stringify("[", __cost__, "](", svg, ")|"));
+  flamegraph(cid, "persistentmap.foldRight", svg);
+
+  call cid.mapfilter();
+  let svg = stringify(title, "_mapfilter_", init, ".svg");
+  output(file, stringify("[", __cost__, "](", svg, ")|"));
+  flamegraph(cid, "persistentmap.mapfilter", svg);
+
+  call cid.map();
+  let svg = stringify(title, "_map_", init, ".svg");
+  output(file, stringify("[", __cost__, "](", svg, ")|\n"));
+  flamegraph(cid, "persistentmap.map", svg);
+
+  uninstall(cid);
+};
+
+function compare_persistent_maps(init){
+  perf_persistent_map(persistentmap, "persistentmap", init);
+  perf_persistent_map(persistentmap_baseline, "persistentmap_baseline", init);
+};
+
+output(file, stringify("\n## Persistent map API\n\n| |size|foldLeft|foldRight|mapfilter|map|\n|--:|--:|--:|--:|--:|--:|\n"));
+
+sizes.map(compare_persistent_maps);
+
+/*
 perf(hashmap, "hashmap", init_size, batch_size);
 perf(triemap, "triemap", init_size, batch_size);
-perf(rbtree, "rbtree", init_size, batch_size);
 perf(splay, "splay", init_size, batch_size);
 perf(btree, "btree", init_size, batch_size);
 perf(zhenya, "zhenya_hashmap", init_size, batch_size);
@@ -97,7 +150,6 @@ perf(heap_stable_rs, "heap_stable_rs", init_size, batch_size);
 perf(vector_rs, "vec_rs", init_size, batch_size);
 perf(vector_stable_rs, "vec_stable_rs", init_size, batch_size);
 
-/*
 let movm_size = 10000;
 output(file, "\n## MoVM\n\n| |binary_size|generate 10k|max mem|batch_get 50|batch_put 50|batch_remove 50|\n|--:|--:|--:|--:|--:|--:|--:|\n");
 perf(hashmap, "hashmap", movm_size);