final test adjustments

johannha · Sep 6, 2024 · cab5f21 · cab5f21
1 parent 6078c88
commit cab5f21
Show file tree

Hide file tree

Showing 29 changed files with 1,023 additions and 316 deletions.
diff --git a/cloud_setup/ansible/hosts.ini b/cloud_setup/ansible/hosts.ini
@@ -1,8 +1,8 @@
 [cloud_vms]
-vm1 ansible_host=34.107.69.117 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
-vm2 ansible_host=35.242.219.9 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
+vm1 ansible_host=35.246.167.161 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
+#vm2 ansible_host=35.242.219.9 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
 #vm3 ansible_host=4.185.20.177 ansible_user=azureadmin ansible_ssh_private_key_file=/Users/johann/.ssh/azure
 
 [zkp_vms]
-vm1 ansible_host=34.107.2.249 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
+#vm1 ansible_host=34.107.2.249 ansible_user=johann ansible_ssh_private_key_file=/Users/johann/.ssh/gcloud
 
diff --git a/cloud_setup/ansible/prepare.yml b/cloud_setup/ansible/prepare.yml
@@ -75,35 +75,41 @@
 
     - name: Copy node server
       ansible.builtin.synchronize:
-        src: "../../node_server/"
-        dest: "/home/johann/node_server"
+        src: "../../node_code/"
+        dest: "/home/johann/node_code"
         recursive: yes
 
-    - name: Copy local txt file to .env file on VM
-      ansible.builtin.copy:
-        src: "{{ lookup('vars', 'env_files')[inventory_hostname] }}"
-        dest: "/home/johann/node_server/.env"
-
-    - name: Copy training data to VM
+    - name: Copy neural network code
       ansible.builtin.synchronize:
-        src: "{{ lookup('vars', 'train_files')[inventory_hostname] }}"
-        dest: "/home/johann/node_server/data/train-images.idx3-ubyte"
+        src: "../../neural_network/"
+        dest: "/home/johann/neural_network"
         recursive: yes
 
-    - name: Copy label data to VM
-      ansible.builtin.synchronize:
-        src: "{{ lookup('vars', 'label_files')[inventory_hostname] }}"
-        dest: "/home/johann/node_server/data/train-labels.idx1-ubyte"
-        recursive: yes
+    # - name: Copy local txt file to .env file on VM
+    #   ansible.builtin.copy:
+    #     src: "{{ lookup('vars', 'env_files')[inventory_hostname] }}"
+    #     dest: "/home/johann/node_server/.env"
 
-    - name: Copy test images to VM
-      ansible.builtin.synchronize:
-        src: "./config/test_data/t10k-images.idx3-ubyte"
-        dest: "/home/johann/node_server/data/t10k-images.idx3-ubyte"
-        recursive: yes
+    # - name: Copy training data to VM
+    #   ansible.builtin.synchronize:
+    #     src: "{{ lookup('vars', 'train_files')[inventory_hostname] }}"
+    #     dest: "/home/johann/node_server/data/train-images.idx3-ubyte"
+    #     recursive: yes
 
-    - name: Copy test labels to VM
-      ansible.builtin.synchronize:
-        src: "./config/test_data/t10k-labels.idx1-ubyte"
-        dest: "/home/johann/node_server/data/t10k-labels.idx1-ubyte"
-        recursive: yes
+    # - name: Copy label data to VM
+    #   ansible.builtin.synchronize:
+    #     src: "{{ lookup('vars', 'label_files')[inventory_hostname] }}"
+    #     dest: "/home/johann/node_server/data/train-labels.idx1-ubyte"
+    #     recursive: yes
+
+    # - name: Copy test images to VM
+    #   ansible.builtin.synchronize:
+    #     src: "./config/test_data/t10k-images.idx3-ubyte"
+    #     dest: "/home/johann/node_server/data/t10k-images.idx3-ubyte"
+    #     recursive: yes
+
+    # - name: Copy test labels to VM
+    #   ansible.builtin.synchronize:
+    #     src: "./config/test_data/t10k-labels.idx1-ubyte"
+    #     dest: "/home/johann/node_server/data/t10k-labels.idx1-ubyte"
+    #     recursive: yes
diff --git a/cloud_setup/terraform/terraform.tfstate b/cloud_setup/terraform/terraform.tfstate
@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.6.2",
-  "serial": 804,
+  "serial": 860,
   "lineage": "e8aeae9e-78e8-c855-b237-26011b8c43c1",
   "outputs": {},
   "resources": [],

diff --git a/cloud_setup/terraform/terraform.tfstate.backup b/cloud_setup/terraform/terraform.tfstate.backup
@@ -1,19 +1,17 @@
 {
   "version": 4,
   "terraform_version": "1.6.2",
-  "serial": 795,
+  "serial": 853,
   "lineage": "e8aeae9e-78e8-c855-b237-26011b8c43c1",
   "outputs": {
     "public_ip_addresses": {
       "value": {
-        "aggregator": "34.107.69.117",
-        "client1": "35.242.219.9"
+        "aggregator": "35.246.132.173"
       },
       "type": [
         "object",
         {
-          "aggregator": "string",
-          "client1": "string"
+          "aggregator": "string"
         }
       ]
     }
@@ -29,9 +27,9 @@
           "index_key": 0,
           "schema_version": 0,
           "attributes": {
-            "address": "34.141.28.93",
+            "address": "34.159.51.229",
             "address_type": "EXTERNAL",
-            "creation_timestamp": "2024-07-24T03:23:34.112-07:00",
+            "creation_timestamp": "2024-08-06T03:35:44.991-07:00",
             "description": "",
             "effective_labels": {},
             "id": "projects/master-420616/regions/europe-west3/addresses/aggregator",
@@ -54,36 +52,6 @@
           },
           "sensitive_attributes": [],
           "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
-        },
-        {
-          "index_key": 1,
-          "schema_version": 0,
-          "attributes": {
-            "address": "34.89.163.7",
-            "address_type": "EXTERNAL",
-            "creation_timestamp": "2024-07-24T03:23:34.046-07:00",
-            "description": "",
-            "effective_labels": {},
-            "id": "projects/master-420616/regions/europe-west3/addresses/client1",
-            "ip_version": "",
-            "ipv6_endpoint_type": "",
-            "label_fingerprint": "42WmSpB8rSM=",
-            "labels": null,
-            "name": "client1",
-            "network": "",
-            "network_tier": "PREMIUM",
-            "prefix_length": 0,
-            "project": "master-420616",
-            "purpose": "",
-            "region": "europe-west3",
-            "self_link": "https://www.googleapis.com/compute/v1/projects/master-420616/regions/europe-west3/addresses/client1",
-            "subnetwork": "",
-            "terraform_labels": {},
-            "timeouts": null,
-            "users": []
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
         }
       ]
     },
@@ -106,7 +74,7 @@
                 "protocol": "tcp"
               }
             ],
-            "creation_timestamp": "2024-07-24T03:23:45.623-07:00",
+            "creation_timestamp": "2024-08-06T03:36:07.574-07:00",
             "deny": [],
             "description": "",
             "destination_ranges": [],
@@ -190,7 +158,7 @@
             "guest_accelerator": [],
             "hostname": "",
             "id": "projects/master-420616/zones/europe-west3-c/instances/aggregator",
-            "instance_id": "2590552563242592441",
+            "instance_id": "1658486458909737538",
             "label_fingerprint": "42WmSpB8rSM=",
             "labels": null,
             "machine_type": "n2d-standard-2",
@@ -206,135 +174,7 @@
               {
                 "access_config": [
                   {
-                    "nat_ip": "34.107.69.117",
-                    "network_tier": "PREMIUM",
-                    "public_ptr_domain_name": "",
-                    "security_policy": ""
-                  }
-                ],
-                "alias_ip_range": [],
-                "internal_ipv6_prefix_length": 0,
-                "ipv6_access_config": [],
-                "ipv6_access_type": "",
-                "ipv6_address": "",
-                "name": "nic0",
-                "network": "https://www.googleapis.com/compute/v1/projects/master-420616/global/networks/fl-network",
-                "network_attachment": "",
-                "network_ip": "10.0.1.3",
-                "nic_type": "",
-                "queue_count": 0,
-                "security_policy": "",
-                "stack_type": "IPV4_ONLY",
-                "subnetwork": "https://www.googleapis.com/compute/v1/projects/master-420616/regions/europe-west3/subnetworks/my-custom-subnet",
-                "subnetwork_project": "master-420616"
-              }
-            ],
-            "network_performance_config": [],
-            "params": [],
-            "project": "master-420616",
-            "reservation_affinity": [],
-            "resource_policies": null,
-            "scheduling": [
-              {
-                "automatic_restart": false,
-                "instance_termination_action": "",
-                "local_ssd_recovery_timeout": [],
-                "maintenance_interval": "",
-                "max_run_duration": [],
-                "min_node_cpus": 0,
-                "node_affinities": [],
-                "on_host_maintenance": "TERMINATE",
-                "preemptible": true,
-                "provisioning_model": ""
-              }
-            ],
-            "scratch_disk": [],
-            "self_link": "https://www.googleapis.com/compute/v1/projects/master-420616/zones/europe-west3-c/instances/aggregator",
-            "service_account": [],
-            "shielded_instance_config": [
-              {
-                "enable_integrity_monitoring": true,
-                "enable_secure_boot": false,
-                "enable_vtpm": true
-              }
-            ],
-            "tags": null,
-            "tags_fingerprint": "42WmSpB8rSM=",
-            "terraform_labels": {},
-            "timeouts": null,
-            "zone": "europe-west3-c"
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiNiJ9",
-          "dependencies": [
-            "google_compute_network.vpc_network",
-            "google_compute_subnetwork.default"
-          ]
-        },
-        {
-          "index_key": 1,
-          "schema_version": 6,
-          "attributes": {
-            "advanced_machine_features": [],
-            "allow_stopping_for_update": null,
-            "attached_disk": [],
-            "boot_disk": [
-              {
-                "auto_delete": true,
-                "device_name": "persistent-disk-0",
-                "disk_encryption_key_raw": "",
-                "disk_encryption_key_sha256": "",
-                "initialize_params": [
-                  {
-                    "enable_confidential_compute": false,
-                    "image": "https://www.googleapis.com/compute/beta/projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20240426",
-                    "labels": {},
-                    "provisioned_iops": 0,
-                    "provisioned_throughput": 0,
-                    "resource_manager_tags": null,
-                    "size": 10,
-                    "type": "pd-standard"
-                  }
-                ],
-                "kms_key_self_link": "",
-                "mode": "READ_WRITE",
-                "source": "https://www.googleapis.com/compute/v1/projects/master-420616/zones/europe-west3-c/disks/client1"
-              }
-            ],
-            "can_ip_forward": false,
-            "confidential_instance_config": [
-              {
-                "confidential_instance_type": "SEV_SNP",
-                "enable_confidential_compute": true
-              }
-            ],
-            "cpu_platform": "AMD Milan",
-            "current_status": "RUNNING",
-            "deletion_protection": false,
-            "description": "",
-            "desired_status": null,
-            "effective_labels": {},
-            "enable_display": false,
-            "guest_accelerator": [],
-            "hostname": "",
-            "id": "projects/master-420616/zones/europe-west3-c/instances/client1",
-            "instance_id": "8394365552777062585",
-            "label_fingerprint": "42WmSpB8rSM=",
-            "labels": null,
-            "machine_type": "n2d-standard-2",
-            "metadata": {
-              "enable-oslogin": "FALSE",
-              "ssh-keys": "johann:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5fQr0OrO2BsvDk64VjgpfheGeD4KmwA0nBMlIn+3JrG1jEmJ9xaT4EPZOMWBULxI0t4MxXwMamroAjSjQNKK/p7WzdtPUzqpiSkCpHYDfZr3BVtYDEsEwBNDiX+QS/hycdJpfwjVrN3Vlrb7OAxcW2Hy1TIoedan4SutK66rM9Yd1wiQNPcUYlu0IXnCmCG2BwgIsNJURkuEJ8gbd3Cp4+5o542sUTUkwv95nNmNrkR0XnwHaMf+hqNpPpn5GSSlaVTINQgit8b9tSE1gHPA3LH5wPYtoGIgG/JKb37+nBJvK2bVJDyd3yADsQIzh6TWPen/sb/JrPQRxi46Jhoex johann"
-            },
-            "metadata_fingerprint": "fBfWloZDuDw=",
-            "metadata_startup_script": null,
-            "min_cpu_platform": "AMD Milan",
-            "name": "client1",
-            "network_interface": [
-              {
-                "access_config": [
-                  {
-                    "nat_ip": "35.242.219.9",
+                    "nat_ip": "35.246.132.173",
                     "network_tier": "PREMIUM",
                     "public_ptr_domain_name": "",
                     "security_policy": ""
@@ -377,7 +217,7 @@
               }
             ],
             "scratch_disk": [],
-            "self_link": "https://www.googleapis.com/compute/v1/projects/master-420616/zones/europe-west3-c/instances/client1",
+            "self_link": "https://www.googleapis.com/compute/v1/projects/master-420616/zones/europe-west3-c/instances/aggregator",
             "service_account": [],
             "shielded_instance_config": [
               {
@@ -420,7 +260,7 @@
             "mtu": 1460,
             "name": "fl-network",
             "network_firewall_policy_enforcement_order": "AFTER_CLASSIC_FIREWALL",
-            "numeric_id": "4745288985076394186",
+            "numeric_id": "8102035454248632943",
             "project": "master-420616",
             "routing_mode": "REGIONAL",
             "self_link": "https://www.googleapis.com/compute/v1/projects/master-420616/global/networks/fl-network",
@@ -440,7 +280,7 @@
         {
           "schema_version": 0,
           "attributes": {
-            "creation_timestamp": "2024-07-24T03:23:46.143-07:00",
+            "creation_timestamp": "2024-08-06T03:36:08.315-07:00",
             "description": "",
             "external_ipv6_prefix": "",
             "fingerprint": null,
@@ -481,7 +321,7 @@
         {
           "schema_version": 0,
           "attributes": {
-            "id": "rg-divine-hare",
+            "id": "rg-tidy-drake",
             "keepers": null,
             "length": 2,
             "prefix": "rg",

diff --git a/cloud_setup/terraform/variables.tf b/cloud_setup/terraform/variables.tf
@@ -43,6 +43,6 @@ variable "username" {
 variable "vm_count" {
   type       = number
   description = "The number of virtual machines including the aggregator."
-  default     = 2
+  default     = 1
   nullable = false
 }
diff --git a/linux_out/start_vm.exe b/linux_out/start_vm.exe
diff --git a/neural_network/data/lm.bin b/neural_network/data/lm.bin
diff --git a/neural_network/data/results_iid/aggregated.bin b/neural_network/data/results_iid/aggregated.bin
diff --git a/neural_network/src/double hidden layer/train-network2.cpp b/neural_network/src/double hidden layer/train-network2.cpp
@@ -2,7 +2,7 @@
 #include <iostream>
 #include <chrono>
 #include <Eigen/Dense>
-
+#include <signal.h>
 #include "../functions.h"
 #include "train-network2.h"
 #include "network2.h"

diff --git a/neural_network/start.exe b/neural_network/start.exe
diff --git a/neural_network/test/aggr_worker.py b/neural_network/test/aggr_worker.py
@@ -7,8 +7,8 @@
     executable_path = "./start.exe"
     with open("./local_worker.txt", "w") as f:
         f.write("amount,execution_time\n")
-        for i in range(10):
-            print(f"Running aggregate {i + 1}")
+        for i in range(1, 21):
+            print(f"Running aggregate {i}")
             start_time = time.time()
             process = subprocess.Popen(
                 [executable_path, "aggregate", str(i)], cwd="../"