Colossus 3.6.0

storage-node/CHANGELOG.md

@@ -1,3 +1,11 @@
+### 3.6.0
+
+- Collosus can now store multiple keys in it's keyring.
+- The `--accountUri` and `--password` args can be used multiple times to add multiple keys. This adds support for worker to use different transactor accounts for each bucket.
+- Added `--keyStore` argument for all commands to configure a directory containing multiple key files to the keyring.
+- Server can run and to serve specific buckets, by passing a comma separated list of bucket ids with the `--buckets` argument.
+- Renamed `--operatorId` argument to `--workerId` in operator commands for consistency.
+
 ### 3.5.1
 
 - **FIX** `sendExtrinsic`: The send extrinsic function (which is a wrapper around PolkadotJS `tx.signAndSend` function) has been fixed to handle the case when tx has been finalized before the callback registered in `tx.signAndSend` would run.

storage-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "3.5.1",
+  "version": "3.6.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"

storage-node/src/command-base/ApiCommandBase.ts

@@ -1,20 +1,23 @@
 import { Command, flags } from '@oclif/command'
 import { createApi } from '../services/runtime/api'
-import { getAccountFromJsonFile, getAlicePair, getAccountFromUri } from '../services/runtime/accounts'
-import { parseBagId } from '../services/helpers/bagTypes'
+import { addAccountFromJsonFile, addAlicePair, addAccountFromUri } from '../services/runtime/accounts'
 import { KeyringPair } from '@polkadot/keyring/types'
-import { ApiPromise } from '@polkadot/api'
+import { ApiPromise, Keyring } from '@polkadot/api'
+import { cryptoWaitReady } from '@polkadot/util-crypto'
 import logger from '../services/logger'
 import ExitCodes from './ExitCodes'
 import { CLIError } from '@oclif/errors'
 import { Input } from '@oclif/parser'
-import _ from 'lodash'
+import path from 'path'
+import fs from 'fs'
+import { JOYSTREAM_ADDRESS_PREFIX } from '@joystream/types'
 
 /**
  * Parent class for all runtime-based commands. Defines common functions.
  */
 export default abstract class ApiCommandBase extends Command {
   private api: ApiPromise | null = null
+  private keyring: Keyring | null = null
 
   static flags = {
     help: flags.help({ char: 'h' }),
@@ -26,35 +29,27 @@ export default abstract class ApiCommandBase extends Command {
     }),
     keyFile: flags.string({
       char: 'k',
-      description: 'Key file for the account. Mandatory in non-dev environment.',
+      description: 'Path to key file to add to the keyring.',
     }),
     password: flags.string({
       char: 'p',
-      description: 'Key file password (optional). Could be overriden by ACCOUNT_PWD environment variable.',
+      description:
+        'Password to unlock keyfiles. Multiple passwords can be passed, to try against all files. If not specified a single password can be set in ACCOUNT_PWD environment variable.',
+      multiple: true,
+      // only fits one password and flag will be a string, otherwise the flag will be an array of strings
+      env: 'ACCOUNT_PWD',
     }),
     accountUri: flags.string({
       char: 'y',
       description:
-        'Account URI (optional). Has a priority over the keyFile and password flags. Could be overriden by ACCOUNT_URI environment variable.',
+        'Account URI (optional). If not specified a single key can be set in ACCOUNT_URI environment variable.',
+      // only fits one key and flag will be a string, otherwise the flag will be an array of strings
+      env: 'ACCOUNT_URI',
+      multiple: true,
     }),
-  }
-
-  static extraFlags = {
-    bagId: flags.build({
-      parse: (value: string) => {
-        return parseBagId(value)
-      },
-      description: `Bag ID. Format: {bag_type}:{sub_type}:{id}.
-    - Bag types: 'static', 'dynamic'
-    - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
-    - Id:
-      - absent for 'static:council'
-      - working group name for 'static:wg'
-      - integer for 'dynamic:member' and 'dynamic:channel'
-    Examples:
-    - static:council
-    - static:wg:storage
-    - dynamic:member:4`,
+    // Path to a single keyfile or a folder
+    keyStore: flags.string({
+      description: 'Path to a folder with multiple key files to load into keystore.',
     }),
   }
 
@@ -88,14 +83,20 @@ export default abstract class ApiCommandBase extends Command {
    * default value (ws://localhost:9944)
    */
   async init(): Promise<void> {
+    await cryptoWaitReady()
+    this.keyring = new Keyring({ type: 'sr25519', ss58Format: JOYSTREAM_ADDRESS_PREFIX })
+
     // Oclif hack: https://github.com/oclif/oclif/issues/225#issuecomment-490555119
     /* eslint-disable @typescript-eslint/no-explicit-any */
     const { flags } = this.parse(<Input<any>>this.constructor)
 
+    // Add all keys to the keystore
+    await this.loadKeys(flags)
+
     // Some dev commands doesn't contain flags variables.
     const apiUrl = flags.apiUrl ?? 'ws://localhost:9944'
 
-    logger.info(`Initialized runtime connection: ${apiUrl}`)
+    logger.info(`Initializing runtime connection to: ${apiUrl}`)
     try {
       this.api = await createApi(apiUrl)
     } catch (err) {
@@ -123,61 +124,132 @@ export default abstract class ApiCommandBase extends Command {
     logger.info('Development mode is ON.')
   }
 
+  tryAddKeyFile(file: string, passwords: string[]): void {
+    if (path.parse(file).ext.toLowerCase() !== '.json') return
+    logger.info(`Adding key from ${file}`)
+    const keyring = this.getKeyring()
+    const pair = addAccountFromJsonFile(file, keyring)
+    if (pair.isLocked) {
+      // Try passwords until one of them works
+      passwords.forEach((passw) => {
+        if (!pair.isLocked) return
+        try {
+          pair.unlock(passw)
+        } catch {
+          //
+        }
+      })
+    }
+
+    // If pair is still locked, then none of the passwords worked.
+    if (pair.isLocked) {
+      this.warn(`Could not unlock keyfile ${file}`)
+    }
+  }
+
   /**
-   * Returns the intialized account KeyringPair instance. Loads the account
-   * JSON-file or loads 'Alice' Keypair when in the development mode.
-   *
-   * @param dev - indicates the development mode (optional).
-   * @param keyFile - key file path (optional).
-   * @param password - password for the key file (optional).
-   * @param accountURI - accountURI (optional). Overrides keyFile and password flags.
-   * @returns KeyringPair instance.
+   * Loads all supplied keys into the keyring.
+   * Accounts passed as SURI, and any JSON-files passed with keyFile and
+   * files found in the keyStore directory.
+   * Since there could be multiple files using the same password, and some may not be locked, its is hard
+   * to distringuish which password argument corresponds to which keyfile. So we try all provided passwords
+   * to unlock keyfiles.
    */
-  getAccount(flags: { dev: boolean; keyFile?: string; password?: string; accountUri?: string }): KeyringPair {
-    // Select account URI variable from flags key and environment variable.
-    let accountUri = flags.accountUri ?? ''
-    if (!_.isEmpty(process.env.ACCOUNT_URI)) {
-      if (!_.isEmpty(flags.accountUri)) {
-        logger.warn(
-          `Both enviroment variable and command line argument were provided for the account URI. Environment variable has a priority.`
-        )
-      }
-      accountUri = process.env.ACCOUNT_URI ?? ''
+  async loadKeys(flags: {
+    dev: boolean
+    keyFile?: string
+    password?: string | string[]
+    accountUri?: string | string[]
+    keyStore?: string
+  }): Promise<void> {
+    const keyring = this.getKeyring()
+    const { dev, password, keyFile, accountUri, keyStore } = flags
+    // Create the Alice account for development mode.
+
+    // in dev mode do not add anything other than the dev account
+    if (dev) {
+      addAlicePair(keyring)
+      return
     }
 
-    // Select password variable from flags key and environment variable.
-    let password = flags.password
-    if (!_.isEmpty(process.env.ACCOUNT_PWD)) {
-      if (!_.isEmpty(flags.password)) {
-        logger.warn(
-          `Both enviroment variable and command line argument were provided for the password. Environment variable has a priority.`
-        )
-      }
-      password = process.env.ACCOUNT_PWD ?? ''
+    // Multiple passwords, or single password passed as env variable
+    let passwords: string[] = []
+    if (Array.isArray(password)) {
+      passwords = passwords.concat(password)
+    } else if (password) {
+      passwords.push(password)
     }
 
-    const keyFile = flags.keyFile ?? ''
-    // Create the Alice account for development mode.
-    if (flags.dev) {
-      return getAlicePair()
+    // Single keyfile
+    if (keyFile) {
+      this.tryAddKeyFile(keyFile, passwords)
     }
-    // Create an account using account URI
-    else if (!_.isEmpty(accountUri)) {
-      return getAccountFromUri(accountUri)
+
+    // Multiple Account SURIs, or single SURI passed as env variable
+    let accountSuris: string[] = []
+    if (Array.isArray(accountUri)) {
+      accountSuris = accountSuris.concat(accountUri)
+    } else if (accountUri) {
+      accountSuris.push(accountUri)
     }
-    // Create an account using the keyFile and password.
-    else if (!_.isEmpty(keyFile)) {
-      const account = getAccountFromJsonFile(keyFile)
-      account.unlock(password)
+    accountSuris.forEach((suri) => addAccountFromUri(suri, keyring))
 
-      return account
+    if (keyStore) {
+      const stat = await fs.promises.stat(keyStore)
+      if (!stat.isDirectory) {
+        return this.error(`keyStore path is not a directory: ${keyStore}`)
+      }
+      const files = await fs.promises.readdir(keyStore)
+      files.forEach((file) => this.tryAddKeyFile(path.join(keyStore, file), passwords))
     }
-    // Cannot create any account for these parameters.
-    else {
-      this.error('Keyfile or account URI must be set.')
+  }
+
+  private getKeyring(): Keyring {
+    if (!this.keyring) {
+      throw new CLIError('Keyring is not ready!', {
+        exit: ExitCodes.KeyringNotReady,
+      })
+    }
+    return this.keyring
+  }
+
+  /**
+   * Returns the intialized account KeyringPair instance by the address.
+   *
+   * @param address - address to fetch keypair for from the keyring.
+   * @returns KeyringPair instance.
+   */
+  getKeyringPair(address: string): KeyringPair {
+    const keyring = this.getKeyring()
+    return keyring.getPair(address)
+  }
+
+  /**
+   * Returns true if keypair contains corresponding address and is unlocked.
+   *
+   * @param address - address to fetch keypair for from the keyring.
+   * @returns boolean
+   */
+  hasKeyringPair(address: string): boolean {
+    const keyring = this.getKeyring()
+    try {
+      const pair = keyring.getPair(address)
+      return !pair.isLocked
+    } catch (err) {
+      logger.warn(err)
+      return false
     }
   }
 
+  /**
+   * Returns addresses of all unlocked KeyPairs stored in the keyring.
+   * @returns string[]
+   */
+  getUnlockedAccounts(): string[] {
+    const keyring = this.getKeyring()
+    return keyring.pairs.filter((pair) => !pair.isLocked).map((pair) => pair.address)
+  }
+
   /**
    * Helper-function for exit after the CLI command. It changes the exit code
    * depending on the previous extrinsic call success.

storage-node/src/command-base/CustomFlags.ts

@@ -0,0 +1,38 @@
+import { flags } from '@oclif/command'
+import ExitCodes from './ExitCodes'
+import { CLIError } from '@oclif/errors'
+import { parseBagId } from '../services/helpers/bagTypes'
+
+export const customFlags = {
+  // 'integer array' oclif flag.
+  integerArr: flags.build({
+    parse: (value: string) => {
+      const arr: number[] = value.split(',').map((v) => {
+        if (!/^-?\d+$/.test(v)) {
+          throw new CLIError(`Expected comma-separated integers, but received: ${value}`, {
+            exit: ExitCodes.InvalidIntegerArray,
+          })
+        }
+        return parseInt(v)
+      })
+      return arr
+    },
+  }),
+  // BagId
+  bagId: flags.build({
+    parse: (value: string) => {
+      return parseBagId(value)
+    },
+    description: `Bag ID. Format: {bag_type}:{sub_type}:{id}.
+  - Bag types: 'static', 'dynamic'
+  - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
+  - Id:
+    - absent for 'static:council'
+    - working group name for 'static:wg'
+    - integer for 'dynamic:member' and 'dynamic:channel'
+  Examples:
+  - static:council
+  - static:wg:storage
+  - dynamic:member:4`,
+  }),
+}

storage-node/src/command-base/ExitCodes.ts

@@ -13,6 +13,7 @@ enum ExitCodes {
   ServerError,
   ApiError = 200,
   UnsuccessfulRuntimeCall,
+  KeyringNotReady,
 
   // NOTE: never exceed exit code 255 or it will be modulated by `256` and create problems
 }

storage-node/src/command-base/LeaderCommandBase.ts

@@ -0,0 +1,48 @@
+import { KeyringPair } from '@polkadot/keyring/types'
+import ExitCodes from './ExitCodes'
+import { CLIError } from '@oclif/errors'
+import { getLeadRoleAccount } from '../services/runtime/queries'
+
+import ApiCommandBase from './ApiCommandBase'
+
+/**
+ * Parent class for all leader commands. Ensure lead role key is in the keystore.
+ */
+export default abstract class LeaderCommandBase extends ApiCommandBase {
+  private roleAccount: string | undefined
+
+  /**
+   * Initilizes the runtime API using the URL from the command line or the
+   * default value (ws://localhost:9944)
+   */
+  async init(): Promise<void> {
+    await super.init()
+
+    const api = await this.getApi()
+    const leadRoleAccount = await getLeadRoleAccount(api)
+    if (leadRoleAccount) {
+      this.roleAccount = leadRoleAccount
+    } else {
+      this.error('Lead is not set')
+    }
+
+    if (!this.hasKeyringPair(leadRoleAccount)) {
+      this.error(`Keyring does not contain leader role key ${leadRoleAccount}`)
+    }
+  }
+
+  /**
+   * Returns the intialized account KeyringPair instance of the lead's role key.
+   *
+   * @returns KeyringPair instance.
+   */
+  getAccount(): KeyringPair {
+    // should not be called if roleAccount was not initialized
+    if (!this.roleAccount) {
+      throw new CLIError('getAccount called before command init', {
+        exit: ExitCodes.KeyringNotReady,
+      })
+    }
+    return this.getKeyringPair(this.roleAccount)
+  }
+}