Merge pull request #19 from MatrixAI/feature-control

Introduce Snapshot Isolation OCC to DBTransaction
MatrixAI · Jun 30, 2022 · 8fd0c9a · 8fd0c9a
2 parents a279855 + 879c5e2
commit 8fd0c9a
Show file tree

Hide file tree

Showing 112 changed files with 13,206 additions and 2,431 deletions.
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,2 @@
+BasedOnStyle: Google
+SortIncludes: false
diff --git a/.eslintrc b/.eslintrc
@@ -50,6 +50,11 @@
         "ignoreConsecutiveComments": true
       }
     ],
+    "curly": [
+      "error",
+      "multi-line",
+      "consistent"
+    ],
     "import/order": [
       "error",
       {
@@ -137,6 +142,10 @@
         "format": ["PascalCase"],
         "trailingUnderscore": "allowSingleOrDouble"
       },
+      {
+        "selector": "enumMember",
+        "format": ["PascalCase", "UPPER_CASE"]
+      },
       {
         "selector": "objectLiteralProperty",
         "format": null

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -65,13 +65,17 @@ check:test:
     - >
         nix-shell --run '
         npm run build --verbose;
-        npm test -- --ci;
+        npm test -- --ci --coverage;
         '
   artifacts:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -114,17 +118,21 @@ build:linux:
     - >
         nix-shell --run '
         npm run build --verbose;
-        npm test -- --ci;
+        npm test -- --ci --coverage;
         '
   artifacts:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
     paths:
       - ./prebuilds/
       # Only the build:linux preserves the dist
       - ./dist
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -150,7 +158,7 @@ build:windows:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
     paths:
       - ./prebuilds/
   rules:
@@ -169,6 +177,7 @@ build:macos:
     HOMEBREW_NO_INSTALL_UPGRADE: "true"
     HOMEBREW_NO_INSTALL_CLEANUP: "true"
   before_script:
+    - eval "$(brew shellenv)"
     - brew install node@16
     - brew link --overwrite node@16
     - brew install [email protected]
@@ -184,7 +193,7 @@ build:macos:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
     paths:
       - ./prebuilds/
   rules:
@@ -209,6 +218,26 @@ build:prerelease:
         nix-shell --run '
         npm publish --tag prerelease --access public;
         '
+    - >
+      for d in prebuilds/*; do
+        tar \
+          --create \
+          --verbose \
+          --file="prebuilds/$(basename $d).tar" \
+          --directory=prebuilds \
+          "$(basename $d)";
+      done
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        gh release \
+          create "$CI_COMMIT_TAG" \
+          prebuilds/*.tar \
+          --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+          --notes "" \
+          --prerelease \
+          --target staging \
+          --repo "$GH_PROJECT_PATH";
+        '
   after_script:
     - rm -f ./.npmrc
   rules:
@@ -272,6 +301,25 @@ release:distribution:
         nix-shell --run '
         npm publish --access public;
         '
+    - >
+      for d in prebuilds/*; do
+        tar \
+          --create \
+          --verbose \
+          --file="prebuilds/$(basename $d).tar" \
+          --directory=prebuilds \
+          "$(basename $d)";
+      done
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        gh release \
+          create "$CI_COMMIT_TAG" \
+          prebuilds/*.tar \
+          --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+          --notes "" \
+          --target master \
+          --repo "$GH_PROJECT_PATH";
+        '
   after_script:
     - rm -f ./.npmrc
   rules:

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "deps/snappy/snappy"]
+	path = deps/snappy/snappy
+	url = https://github.com/google/snappy.git
+[submodule "deps/rocksdb/rocksdb"]
+	path = deps/rocksdb/rocksdb
+	url = https://github.com/facebook/rocksdb.git
diff --git a/README.md b/README.md
@@ -5,12 +5,76 @@ master: [![pipeline status](https://gitlab.com/MatrixAI/open-source/js-db/badges
 
 DB is library managing key value state for MatrixAI's JavaScript/TypeScript applications.
 
+This forks classic-level's C++ binding code around LevelDB 1.20. Differences from classic-level:
+
+* Uses TypeScript from ground-up
+* Supports Snapshot-Isolation based transactions via `DBTransaction`
+* API supports "key paths" which can be used to manipulate "levels" of nested keys
+* Value encryption (key-encryption is not supported yet) - requires additional work with block-encryption
+
 ## Installation
 
 ```sh
 npm install --save @matrixai/db
 ```
 
+## Usage
+
+
+```ts
+import { DB } from '@matrixai/db';
+
+async function main () {
+
+  const key = Buffer.from([
+    0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
+    0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
+  ]);
+
+  const encrypt = async (
+    key: ArrayBuffer,
+    plainText: ArrayBuffer
+  ): Promise<ArrayBuffer> {
+    return plainText;
+  };
+
+  const decrypt = async (
+    key: ArrayBuffer,
+    cipherText: ArrayBuffer
+  ): Promise<ArrayBuffer | undefined> {
+    return cipherText;
+  }
+
+  const db = await DB.createDB({
+    dbPath: './tmp/db',
+    crypto: {
+      key,
+      ops: { encrypt, decrypt },
+    },
+    fresh: true,
+  });
+
+  await db.put(['level', Buffer.from([0x30, 0x30]), 'a'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x31]), 'b'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x32]), 'c'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x33]), 'c'], 'value');
+
+  console.log(await db.get(['level', Buffer.from([0x30, 0x32]), 'c']));
+
+  await db.del(['level', Buffer.from([0x30, 0x32]), 'c']);
+
+  for await (const [kP, v] of db.iterator({
+    lt: [Buffer.from([0x30, 0x32]), ''],
+  }, ['level'])) {
+    console.log(kP, v);
+  }
+
+  await db.stop();
+}
+
+main();
+```
+
 ## Development
 
 Run `nix-shell`, and once you're inside, you can use:

diff --git a/benches/DB1KiB.ts b/benches/DB1KiB.ts
@@ -58,9 +58,7 @@ async function main() {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;
diff --git a/benches/DB1MiB.ts b/benches/DB1MiB.ts
@@ -20,13 +20,13 @@ async function main() {
   const summary = await b.suite(
     'DB1MiB',
     b.add('get 1 MiB of data', async () => {
-      await db.put('1kib', data1MiB, true);
+      await db.put('1mib', data1MiB, true);
       return async () => {
-        await db.get('1kib', true);
+        await db.get('1mib', true);
       };
     }),
     b.add('put 1 MiB of data', async () => {
-      await db.put('1kib', data1MiB, true);
+      await db.put('1mib', data1MiB, true);
     }),
     b.add('put zero data', async () => {
       await db.put('0', data0, true);
@@ -58,9 +58,7 @@ async function main() {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;
diff --git a/benches/index.ts b/benches/index.ts
@@ -20,9 +20,7 @@ async function main(): Promise<void> {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;
diff --git a/benches/results/DB1KiB.chart.html b/benches/results/DB1KiB.chart.html
@@ -16,7 +16,7 @@
   </head>
   <body>
     <div style="max-width: 800px">
-      <canvas id="chart1654000546839" width="16" height="9"></canvas>
+      <canvas id="chart1656585967881" width="16" height="9"></canvas>
     </div>
     <script>
       const format = (num) => {
@@ -34,10 +34,10 @@
 
         return chunked.map((chunk) => chunk.join("")).join(" ");
       };
-      const ctx1654000546839 = document
-        .getElementById("chart1654000546839")
+      const ctx1656585967881 = document
+        .getElementById("chart1656585967881")
         .getContext("2d");
-      const chart1654000546839 = new Chart(ctx1654000546839, {
+      const chart1656585967881 = new Chart(ctx1656585967881, {
         type: "bar",
         data: {
           labels: [
@@ -48,7 +48,7 @@
           ],
           datasets: [
             {
-              data: [55227, 28769, 33029, 16935],
+              data: [43375, 24504, 27503, 13785],
               backgroundColor: [
                 "rgba(63, 142, 252, 0.8)",
                 "rgba(116, 165, 127, 0.8)",