increased memory in buildSnarks.ts

clrfund · Sep 4, 2020 · 61b65ab · 61b65ab
1 parent 1e6bff1
commit 61b65ab
Show file tree

Hide file tree

Showing 18 changed files with 290 additions and 18,305 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,35 +1,32 @@
 version: 2
 jobs:
   build:
-    docker:
-      - image: circleci/node:11.14.0
+    machine: true
 
     working_directory: ~/maci/
 
     steps:
-      - checkout:
-          path: ~/maci/
-
-      - restore_cache:
-          keys:
-            - v1.2-dependencies-{{ checksum "circuits/scripts/installZkutil.sh" }}
-
       - run:
-          name: Install zkutil
-          command: ./circuits/scripts/installZkutil.sh
+          name: Install Node 11.14.0
+          command: |
+            export NVM_DIR="/opt/circleci/.nvm"
+            [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+            nvm install 11.14.0
+            nvm alias default v11
+            npm i -g npm
+            touch $BASH_ENV
+            echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV
+            echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV
+            echo 'nvm install v11.14.0' >> $BASH_ENV
+            echo 'nvm alias default v11.14.0' >> $BASH_ENV
 
-      - save_cache:
-          paths:
-            - ~/.cargo/bin/zkutil
-          key: v1.2-dependencies-{{ checksum "circuits/scripts/installZkutil.sh" }}
 
-      - run:
-          name: Install solc
-          command: wget https://github.com/ethereum/solidity/releases/download/v0.5.16/solc-static-linux && chmod a+x solc-static-linux && sudo mv solc-static-linux /usr/bin/solc
+      - checkout:
+          path: ~/maci/
 
       - restore_cache:
           keys:
-            - v1.2-dependencies-{{ checksum "package-lock.json" }}
+            - v1.3-dependencies-{{ checksum "package-lock.json" }}
 
       - run: 
           name: Install NPM dependencies
@@ -38,41 +35,51 @@ jobs:
       - save_cache:
           paths:
             - node_modules
-          key: v1.2-dependencies-{{ checksum "package-lock.json" }}
+          key: v1.3-dependencies-{{ checksum "package-lock.json" }}
 
       - restore_cache:
           keys:
-            - v1.2-dependencies-{{ checksum "contracts/package-lock.json" }}-{{ checksum "circuits/package-lock.json" }}-{{ checksum "config/package-lock.json" }}-{{ checksum "domainobjs/package-lock.json" }}-{{ checksum "cli/package-lock.json" }}-{{ checksum "core/package-lock.json" }}-{{ checksum "integrationTests/package-lock.json" }}
+            - v1.3-dependencies-{{ checksum "contracts/package-lock.json" }}-{{ checksum "circuits/package-lock.json" }}-{{ checksum "config/package-lock.json" }}-{{ checksum "domainobjs/package-lock.json" }}-{{ checksum "cli/package-lock.json" }}-{{ checksum "core/package-lock.json" }}-{{ checksum "integrationTests/package-lock.json" }}--{{ checksum "crypto/package-lock.json" }}
 
       - run: npm run bootstrap && npm run build
 
       - save_cache:
           paths:
-            - contracts/node_modules
+            - circuits/node_modules
+            - cli/node_modules
             - config/node_modules
+            - contracts/node_modules
+            - crypto/node_modules
             - core/node_modules
-            - circuits/node_modules
             - domainobjs/node_modules
-          key: v1.2-dependencies-{{ checksum "contracts/package-lock.json" }}-{{ checksum "circuits/package-lock.json" }}-{{ checksum "config/package-lock.json" }}-{{ checksum "domainobjs/package-lock.json" }}-{{ checksum "cli/package-lock.json" }}-{{ checksum "core/package-lock.json" }}-{{ checksum "integrationTests/package-lock.json" }}
-
-      # checksum the circuit code
-      - run:
-          name: Checksum snark files
-          command: cd circuits/ && ./scripts/checksumSnarks.sh
+            - integrationTests/node_modules
+          key: v1.3-dependencies-{{ checksum "contracts/package-lock.json" }}-{{ checksum "circuits/package-lock.json" }}-{{ checksum "config/package-lock.json" }}-{{ checksum "domainobjs/package-lock.json" }}-{{ checksum "cli/package-lock.json" }}-{{ checksum "core/package-lock.json" }}-{{ checksum "integrationTests/package-lock.json" }}--{{ checksum "crypto/package-lock.json" }}
 
       - restore_cache:
-          name: restore-snark-cache
           keys:
-            - v1.2.3-dependencies-{{ checksum "circuits/build/.snark_checksum" }}
+            - v1.2-dependencies-{{ checksum "circuits/scripts/installZkutil.sh" }}
+
+      - run:
+          name: Install zkutil
+          command: ./circuits/scripts/installZkutil.sh
+
+      - save_cache:
+          paths:
+            - ~/.cargo/bin/zkutil
+          key: v1.2-dependencies-{{ checksum "circuits/scripts/installZkutil.sh" }}
+
+      - run:
+          name: Install solc
+          command: wget https://github.com/ethereum/solidity/releases/download/v0.5.16/solc-static-linux && chmod a+x solc-static-linux && sudo mv solc-static-linux /usr/bin/solc
 
       # run trusted setups
       - run:
           name: Build the batch update state tree snark
-          command: cd circuits && npm run buildBatchUpdateStateTreeSnark
+          command: cd circuits && NODE_OPTIONS=--max-old-space-size=8192 node build/buildSnarks.js -i circom/prod/batchUpdateStateTreeVerifier.circom -j build/batchUstCircuit.json -p build/batchUstPk.json -v build/batchUstVk.json -s build/BatchUpdateStateTreeVerifier.sol -vs BatchUpdateStateTreeVerifier -m build/batchUst.params && ./scripts/buildBatchUpdateStateTreeSnark.sh
 
       - run:
           name: Build the quad vote tally snark
-          command: cd circuits && npm run buildQuadVoteTallySnark
+          command: cd circuits && NODE_OPTIONS=--max-old-space-size=8192 node build/buildSnarks.js -i circom/prod/quadVoteTally.circom -j build/qvtCircuit.json -p build/qvtPk.bin -v build/qvtVk.json -s build/QuadVoteTallyVerifier.sol -vs QuadVoteTallyVerifier -m build/qvt.params && ./scripts/buildQuadVoteTallySnark.sh
 
       # cache generated snark circuit and keys
       - save_cache:

diff --git a/.travis.yml b/.travis.yml
@@ -7,6 +7,14 @@ install:
   - npm i
   - npm run bootstrap
   - npm run build
+
+script:
+  - cd crypto && npm run test
   - ./circuits/scripts/installZkutil.sh
-  - wget https://github.com/ethereum/solidity/releases/download/v0.5.17/solc-static-linux && chmod a+x solc-static-linux && sudo mv solc-static-linux /usr/bin/solc
-  - cd circuits && npm run buildBatchUpdateStateTreeSnark
+  - cd circuits && npm run buildBatchUpdateStateTreeSnark && npm run buildQuadVoteTallySnark
+  - cd circuits && ./scripts/runTestsInCircleCi.sh
+  - cd contracts && ./scripts/compileSol.sh
+  - cd domainobjs && npm run test
+  - cd core && npm run test
+  - cd contracts && ./scripts/runTestsInCircleCi.sh
+  - cd integrationTests && ./scripts/runTestsInCircleCi.sh
diff --git a/circuits/jest.config.js b/circuits/jest.config.js
@@ -4,7 +4,7 @@ module.exports = {
         "^.+\\.tsx?$": 'ts-jest'
     },
     testPathIgnorePatterns: [
-        "/build/",
+        "<rootDir>/build/",
         "/node_modules/",
     ],
     testRegex: '/__tests__/.*\\.test\\.ts$',

diff --git a/circuits/scripts/buildBatchUpdateStateTreeSnark.sh b/circuits/scripts/buildBatchUpdateStateTreeSnark.sh
@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 cd ..
 mkdir -p build
 
-NODE_OPTIONS=--max-old-space-size=4096 node build/buildSnarks.js -i circom/prod/batchUpdateStateTreeVerifier.circom -j build/batchUstCircuit.json -p build/batchUstPk.json -v build/batchUstVk.json -s build/BatchUpdateStateTreeVerifier.sol -vs BatchUpdateStateTreeVerifier -m build/batchUst.params
+NODE_OPTIONS=--max-old-space-size=8192 node build/buildSnarks.js -i circom/prod/batchUpdateStateTreeVerifier.circom -j build/batchUstCircuit.json -p build/batchUstPk.json -v build/batchUstVk.json -s build/BatchUpdateStateTreeVerifier.sol -vs BatchUpdateStateTreeVerifier -m build/batchUst.params
 
 echo 'Copying BatchUpdateStateTreeVerifier.sol to contracts/sol.'
 cp ./build/BatchUpdateStateTreeVerifier.sol ../contracts/sol/
diff --git a/circuits/scripts/buildQuadVoteTallySnark.sh b/circuits/scripts/buildQuadVoteTallySnark.sh
@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 cd ..
 mkdir -p build
 
-NODE_OPTIONS=--max-old-space-size=4096 node build/buildSnarks.js -i circom/prod/quadVoteTally.circom -j build/qvtCircuit.json -p build/qvtPk.bin -v build/qvtVk.json -s build/QuadVoteTallyVerifier.sol -vs QuadVoteTallyVerifier -m build/qvt.params
+NODE_OPTIONS=--max-old-space-size=8192 node build/buildSnarks.js -i circom/prod/quadVoteTally.circom -j build/qvtCircuit.json -p build/qvtPk.bin -v build/qvtVk.json -s build/QuadVoteTallyVerifier.sol -vs QuadVoteTallyVerifier -m build/qvt.params
 
 echo 'Copying QuadVoteTallyVerifier.sol to contracts/sol.'
 cp ./build/QuadVoteTallyVerifier.sol ../contracts/sol/
diff --git a/circuits/ts/__tests__/VerifySignature.test.ts b/circuits/ts/__tests__/VerifySignature.test.ts
@@ -5,7 +5,6 @@ const compiler = require('circom')
 import {
     stringifyBigInts,
     verifySignature,
-    sign,
     bigInt,
     hash11,
 } from 'maci-crypto'

diff --git a/circuits/ts/buildSnarks.ts b/circuits/ts/buildSnarks.ts
@@ -122,7 +122,7 @@ const main = () => {
     }
 
     // Set memory options for node
-    shell.env['NODE_OPTIONS'] = '--max-old-space-size=4096'
+    shell.env['NODE_OPTIONS'] = '--max-old-space-size=8192'
 
     // Check if the circuitJsonOut file exists and if we should not override files
     const circuitJsonOutFileExists = fileExists(circuitJsonOut)
@@ -134,7 +134,7 @@ const main = () => {
     } else {
         console.log(`Compiling ${inputFile}...`)
         // Compile the .circom file
-        shell.exec(`node ${circomPath} ${inputFile} -o ${circuitJsonOut}`)
+        shell.exec(`node --max-old-space-size=8192 ${circomPath} ${inputFile} -o ${circuitJsonOut}`)
         console.log('Generated', circuitJsonOut)
     }
 

diff --git a/cli/jest.config.js b/cli/jest.config.js
@@ -1,29 +1,29 @@
 module.exports = {
-    verbose: true,
-    transform: {
-        "^.+\\.tsx?$": 'ts-jest'
-    },
-    testPathIgnorePatterns: [
-        "/build/",
-        "/node_modules/",
-    ],
-    testRegex: '/__tests__/.*\\.test\\.ts$',
-    moduleFileExtensions: [
-        'ts',
-        'tsx',
-        'js',
-        'jsx',
-        'json',
-        'node'
-    ],
-    globals: {
-        'ts-jest': {
-            diagnostics: {
-                // Do not fail on TS compilation errors
-                // https://kulshekhar.github.io/ts-jest/user/config/diagnostics#do-not-fail-on-first-error
-                warnOnly: true
-            }
-        }
-    },
-    testEnvironment: 'node'
+  verbose: true,
+  transform: {
+    "^.+\\.tsx?$": 'ts-jest'
+  },
+  testPathIgnorePatterns: [
+    "<rootDir>/build/",
+    "/node_modules/",
+  ],
+  testRegex: '/__tests__/.*\\.test\\.ts$',
+  moduleFileExtensions: [
+    'ts',
+    'tsx',
+    'js',
+    'jsx',
+    'json',
+    'node'
+  ],
+  globals: {
+    'ts-jest': {
+      diagnostics: {
+        // Do not fail on TS compilation errors
+        // https://kulshekhar.github.io/ts-jest/user/config/diagnostics#do-not-fail-on-first-error
+        warnOnly: true
+      }
+    }
+  },
+  testEnvironment: 'node'
 }