Merge pull request #71 from worldcoin/POP-1707/load-default-configs-f…

…rom-env-vars-allow-overrides-from-cli

[POP-1707] Load default configs from env vars and allow overrides from CLI

.env.mpc1.dist

@@ -0,0 +1,32 @@
+# Credentials, not read in by the app, will get replaced with an IAM profile for EKS node
+AWS_ACCESS_KEY_ID=x
+AWS_SECRET_ACCESS_KEY=x
+
+# General Environment Configuration
+SMPC__ENVIRONMENT=development
+
+# Service Configuration
+SMPC__SERVICE__SERVICE_NAME=smpcv2-server
+
+# Database Configuration
+SMPC__DATABASE__URL=postgres://user:password@localhost/dbname
+SMPC__DATABASE__MIGRATE=true
+SMPC__DATABASE__CREATE=false
+
+# AWS Configuration
+SMPC__AWS__REGION=eu-north-1
+
+# Servers Configuration
+SMPC__SERVERS__CODES_ENGINE_PORT=4000
+SMPC__SERVERS__MASKS_ENGINE_PORT=4001
+SMPC__SERVERS__BATCH_CODES_ENGINE_PORT=4002
+SMPC__SERVERS__BATCH_MASKS_ENGINE_PORT=4003
+SMPC__SERVERS__PHASE_2_BATCH_PORT=4004
+SMPC__SERVERS__PHASE_2_PORT=4005
+
+# App configuration
+SMPC__PARTY_ID=0
+SMPC__REQUESTS_QUEUE_URL=https://sqs.eu-north-1.amazonaws.com/654654380399/mpc1.fifo
+SMPC__RESULTS_TOPIC_ARN=arn:aws:sns:eu-north-1:654654380399:mpc-results-topic
+SMPC__PATH=/opt/dlami/nvme/
+SMPC__KMS_KEY_IDS='["077788e2-9eeb-4044-859b-34496cfd500b", "896353dc-5ea5-42d4-9e4e-f65dd8169dee", "42bb01f5-8380-48b4-b1f1-929463a587fb"]'

.env.mpc2.dist

@@ -0,0 +1,33 @@
+# Credentials, not read in by the app, will get replaced with an IAM profile for EKS node
+AWS_ACCESS_KEY_ID=x
+AWS_SECRET_ACCESS_KEY=x
+
+# General Environment Configuration
+SMPC__ENVIRONMENT=development
+
+# Service Configuration
+SMPC__SERVICE__SERVICE_NAME=smpcv2-server
+
+# Database Configuration
+SMPC__DATABASE__URL=postgres://user:password@localhost/dbname
+SMPC__DATABASE__MIGRATE=true
+SMPC__DATABASE__CREATE=false
+
+# AWS Configuration
+SMPC__AWS__REGION=eu-north-1
+
+# Servers Configuration
+SMPC__SERVERS__CODES_ENGINE_PORT=4000
+SMPC__SERVERS__MASKS_ENGINE_PORT=4001
+SMPC__SERVERS__BATCH_CODES_ENGINE_PORT=4002
+SMPC__SERVERS__BATCH_MASKS_ENGINE_PORT=4003
+SMPC__SERVERS__PHASE_2_BATCH_PORT=4004
+SMPC__SERVERS__PHASE_2_PORT=4005
+
+# App configuration
+SMPC__PARTY_ID=1
+SMPC__REQUESTS_QUEUE_URL=https://sqs.eu-north-1.amazonaws.com/654654380399/mpc2.fifo
+SMPC__RESULTS_TOPIC_ARN=arn:aws:sns:eu-north-1:654654380399:mpc-results-topic
+SMPC__PATH=/opt/dlami/nvme/
+SMPC__BOOTSTRAP_URL=10.15.32.27
+SMPC__KMS_KEY_IDS='["077788e2-9eeb-4044-859b-34496cfd500b", "896353dc-5ea5-42d4-9e4e-f65dd8169dee", "42bb01f5-8380-48b4-b1f1-929463a587fb"]'

.env.mpc3.dist

@@ -0,0 +1,33 @@
+# Credentials, not read in by the app, will get replaced with an IAM profile for EKS node
+AWS_ACCESS_KEY_ID=x
+AWS_SECRET_ACCESS_KEY=x
+
+# General Environment Configuration
+SMPC__ENVIRONMENT=development
+
+# Service Configuration
+SMPC__SERVICE__SERVICE_NAME=smpcv2-server
+
+# Database Configuration
+SMPC__DATABASE__URL=postgres://user:password@localhost/dbname
+SMPC__DATABASE__MIGRATE=true
+SMPC__DATABASE__CREATE=false
+
+# AWS Configuration
+SMPC__AWS__REGION=eu-north-1
+
+# Servers Configuration
+SMPC__SERVERS__CODES_ENGINE_PORT=4000
+SMPC__SERVERS__MASKS_ENGINE_PORT=4001
+SMPC__SERVERS__BATCH_CODES_ENGINE_PORT=4002
+SMPC__SERVERS__BATCH_MASKS_ENGINE_PORT=4003
+SMPC__SERVERS__PHASE_2_BATCH_PORT=4004
+SMPC__SERVERS__PHASE_2_PORT=4005
+
+# App configuration
+SMPC__PARTY_ID=2
+SMPC__REQUESTS_QUEUE_URL=https://sqs.eu-north-1.amazonaws.com/654654380399/mpc3.fifo
+SMPC__RESULTS_TOPIC_ARN=arn:aws:sns:eu-north-1:654654380399:mpc-results-topic
+SMPC__PATH=/opt/dlami/nvme/
+SMPC__BOOTSTRAP_URL=10.15.32.27
+SMPC__KMS_KEY_IDS='["077788e2-9eeb-4044-859b-34496cfd500b", "896353dc-5ea5-42d4-9e4e-f65dd8169dee", "42bb01f5-8380-48b4-b1f1-929463a587fb"]'

.gitignore

@@ -2,3 +2,4 @@
 *.db
 .vscode/
 .idea/
+.env

README.md

@@ -4,14 +4,58 @@
 - Node PoC implementation in `src/bin/server.rs`
 - Example client in `src/bin/client.rs`
 
+
+#### Running the server without config override
+```
+AWS_REGION=eu-north-1 AWS_ACCESS_KEY_ID=xxx AWS_SECRET_ACCESS_KEY=xxx cargo run --release --bin server
+```
+
+#### Running the server with override
 ```
 AWS_REGION=eu-north-1 AWS_ACCESS_KEY_ID=xxx AWS_SECRET_ACCESS_KEY=xxx cargo run --release --bin server -- --party-id {0,1,2} --queue https://sqs.eu-north-1.amazonaws.com/xxx/mpc1.fifo
 ```
 
+#### Running the client
 ```
 AWS_ACCESS_KEY_ID=xxx AWS_SECRET_ACCESS_KEY=xxx cargo run --release --bin client -- --topic-arn arn:aws:sns:eu-north-1:xxx:mpc.fifo --db-index 2 --n-repeat 32
 ```
 
+### Server configuration
+
+Please note that this mechanism does not apply to the client part.
+
+
+#### Environment variables
+Application can be completely configured via environment variables. To see the list of variables please check out contents of the `.env*` files.
+
+**Important!**
+Please note that there is a dist file per an instance of MPC node. Before running, please make sure to rename the correct dist file to `.env`.
+
+For now the environment variables are read in via a `dotenvy` crate for the ease of development.
+
+#### CLI arguments
+Not to force developers into a change of approach to work that has been established, overloading the environment variables with CLI arguments is also possible. 
+List of possible overrides is represented by the following `struct`:
+```rust
+pub struct Opt {
+    #[structopt(long)]
+    requests_queue_url: Option<String>,
+
+    #[structopt(long)]
+    results_topic_arn: Option<String>,
+
+    #[structopt(long)]
+    party_id: Option<usize>,
+
+    #[structopt(long)]
+    bootstrap_url: Option<String>,
+
+    #[structopt(long)]
+    path: Option<String>,
+}
+```
+Please note that due to ambiguity, all the arguments need to be provided using their full names.
+
 ### Dependencies
 
 Requires a NVIDIA graphics card with recent drivers and CUDA libraries.
@@ -42,4 +86,3 @@ If you are using `cargo test` with non-standard library paths, you might need [a
 ## Streams and Synchronization in V2 (`src/bin/server.rs`)
 > TODO: dedup between query and previous is not yet implemented
 ![streams](mpc-iris-streams-v2.png)
-

scripts/sync_to_hosts.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Check if username is provided as a parameter
+if [ -z "$1" ]; then
+    echo "No username provided."
+    echo "Usage: $0 <username>"
+    exit 1
+fi
+
+# Assign the provided username to a variable
+USER_NAME=$1
+
+# Define the target directory
+TARGET_DIR="/home/ubuntu/${USER_NAME}"
+
+# Define the hosts, order is important, indices are used to identify .env files
+TARGET_HOSTS=(
+"[email protected]"
+"[email protected]"
+"[email protected]"
+)
+
+# Define the directories to exclude
+EXCLUDE_DIRS=(".idea" ".git" "target")
+
+# Create the exclude options for rsync
+EXCLUDE_OPTIONS=()
+for EXCLUDE_DIR in "${EXCLUDE_DIRS[@]}"; do
+    EXCLUDE_OPTIONS+=("--exclude=$EXCLUDE_DIR")
+done
+
+# Get the name of the current working directory
+CURRENT_DIR_NAME=$(basename "$PWD")
+
+## Loop through each host using an index, starting with 1
+for i in "${!TARGET_HOSTS[@]}"; do
+   INDEX=$((i + 1))
+   HOST=${TARGET_HOSTS[$i]}
+   echo "Syncing $CURRENT_DIR_NAME to $HOST:$TARGET_DIR (Target Host $INDEX)"
+
+   ## Make sure that the correct environment file is copied to the correct host
+   cp .env.mpc$INDEX.dist .env
+
+   ## Sync the current directory to the target host
+   rsync -avz --delete "${EXCLUDE_OPTIONS[@]}" "$PWD" "$HOST:$TARGET_DIR"
+   if [ $? -eq 0 ]; then
+       echo "Sync to $HOST completed successfully."
+   else
+       echo "Sync to $HOST failed."
+   fi
+done