Merge branch 'master' into sendToGitHub/remove-errors-logs

armadaproject · Feb 6, 2025 · 636ca51 · 636ca51
2 parents 55f3061 + 6e52825
commit 636ca51
Show file tree

Hide file tree

Showing 221 changed files with 4,943 additions and 1,170 deletions.
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -197,6 +197,7 @@ dockers:
       - scheduler
       - scheduleringester
     extra_files:
+      - config/logging.yaml
       - config/server/config.yaml
       - config/executor/config.yaml
       - config/binoculars/config.yaml
@@ -217,6 +218,7 @@ dockers:
       - lookoutv2
       - lookoutingesterv2
     extra_files:
+      - config/logging.yaml
       - config/lookoutv2/config.yaml
       - config/lookoutingesterv2/config.yaml
       - internal/lookout/ui
@@ -245,6 +247,7 @@ dockers:
       - lookoutingesterv2
       - fakeexecutor
     extra_files:
+      - config/logging.yaml
       - config/server/config.yaml
       - config/executor/config.yaml
       - config/binoculars/config.yaml
@@ -271,6 +274,7 @@ dockers:
       - server
     extra_files:
       - config/server/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/server/Dockerfile
 
   - id: executor
@@ -285,6 +289,7 @@ dockers:
       - executor
     extra_files:
       - config/executor/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/executor/Dockerfile
 
   - id: fakeexecutor
@@ -299,6 +304,7 @@ dockers:
       - fakeexecutor
     extra_files:
       - config/executor/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/fakeexecutor/Dockerfile
 
   - id: armadaloadtester
@@ -337,6 +343,7 @@ dockers:
       - lookoutingesterv2
     extra_files:
       - config/lookoutingesterv2/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/lookoutingesterv2/Dockerfile
 
   - id: lookoutv2
@@ -357,6 +364,7 @@ dockers:
       - pkg/api/schedulerobjects/api.swagger.json
       - config/lookoutv2/config.yaml
       - config/lookoutingesterv2/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/lookoutv2/Dockerfile
 
   - id: eventingester
@@ -371,6 +379,7 @@ dockers:
       - eventingester
     extra_files:
       - config/eventingester/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/eventingester/Dockerfile
 
   - id: scheduler
@@ -385,6 +394,7 @@ dockers:
       - scheduler
     extra_files:
       - config/scheduler/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/scheduler/Dockerfile
 
   - id: scheduleringester
@@ -399,6 +409,7 @@ dockers:
       - scheduleringester
     extra_files:
       - config/scheduleringester/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/scheduleringester/Dockerfile
 
   - id: binoculars
@@ -413,6 +424,7 @@ dockers:
       - binoculars
     extra_files:
       - config/binoculars/config.yaml
+      - config/logging.yaml
     dockerfile: ./build/binoculars/Dockerfile
 
   - id: armadactl

diff --git a/.mergify.yml b/.mergify.yml
@@ -9,6 +9,6 @@ pull_request_rules:
           - "#approved-reviews-by>=2"
           - and: 
             - "#approved-reviews-by>=1"
-            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)"
+            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)"
         title:
           Two are checks required.
diff --git a/.run/Event Ingester.run.xml b/.run/Event Ingester.run.xml
@@ -3,6 +3,7 @@
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
       <env name="ARMADA_REDIS_ADDRS" value="localhost:6379" />
       <env name="ARMADA_METRICSPORT" value="9011" />
@@ -13,4 +14,4 @@
     <filePath value="$PROJECT_DIR$/cmd/eventingester/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Executor.run.xml b/.run/Executor.run.xml
@@ -6,6 +6,7 @@
       <env name="ARMADA_APICONNECTION_FORCENOTLS" value="true" />
       <env name="ARMADA_EXECUTORAPICONNECTION_ARMADAURL" value="localhost:50052" />
       <env name="ARMADA_EXECUTORAPICONNECTION_FORCENOTLS" value="true" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_HTTPPORT" value="8085" />
       <env name="HOME" value="$USER_HOME$/" />
       <env name="KUBECONFIG" value="$PROJECT_DIR$/.kube/external/config" />
@@ -17,4 +18,4 @@
     <filePath value="$PROJECT_DIR$/cmd/executor/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Lookout Ingester V2.run.xml b/.run/Lookout Ingester V2.run.xml
@@ -3,6 +3,7 @@
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_METRICS_PORT" value="9006" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />

diff --git a/.run/LookoutV2.run.xml b/.run/LookoutV2.run.xml
@@ -4,6 +4,7 @@
     <working_directory value="$PROJECT_DIR$" />
     <envs>
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
       <env name="ARMADA_UICONFIG_ARMADAAPIBASEURL" value="http://localhost:8080" />

diff --git a/.run/Scheduler Ingester.run.xml b/.run/Scheduler Ingester.run.xml
@@ -3,6 +3,7 @@
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
     </envs>
@@ -12,4 +13,4 @@
     <filePath value="$PROJECT_DIR$/cmd/scheduleringester/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Scheduler.run.xml b/.run/Scheduler.run.xml
@@ -6,6 +6,7 @@
     <envs>
       <env name="ARMADA_ARMADAAPI_ARMADAURL" value="localhost:50051" />
       <env name="ARMADA_HTTP_PORT" value="8084" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_METRICS_PORT" value="9004" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />

diff --git a/.run/Server.run.xml b/.run/Server.run.xml
@@ -7,6 +7,7 @@
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
       <env name="ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_EVENTSAPIREDIS_ADDRS" value="localhost:6379" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_METRICSPORT" value="9005" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />

diff --git a/.run/lookoutv2PostgresMigration.run.xml b/.run/lookoutv2PostgresMigration.run.xml
@@ -5,6 +5,7 @@
     <parameters value="--migrateDatabase" />
     <envs>
       <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
     </envs>
@@ -14,4 +15,4 @@
     <filePath value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/schedulerPostgresMigration.run.xml b/.run/schedulerPostgresMigration.run.xml
@@ -9,6 +9,7 @@
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
       <env name="ARMADA_REDIS_ADDRS" value="localhost:6379" />
+      <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
     </envs>
     <kind value="FILE" />
     <package value="github.com/armadaproject/armada" />

diff --git a/build/binoculars/Dockerfile b/build/binoculars/Dockerfile
@@ -9,6 +9,7 @@ USER armada
 
 COPY binoculars /app/
 COPY config/binoculars/config.yaml /app/config/binoculars/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 
 WORKDIR /app
 

diff --git a/build/bundles/armada/Dockerfile b/build/bundles/armada/Dockerfile
@@ -7,6 +7,9 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 
+# Logging (share between all components)
+COPY config/logging.yaml /app/config/logging.yaml
+
 # Server
 COPY config/server/config.yaml /app/config/server/config.yaml
 

diff --git a/build/bundles/lookout/Dockerfile b/build/bundles/lookout/Dockerfile
@@ -8,6 +8,9 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada-loo
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 
+# Logging (shared between both components)
+COPY config/logging.yaml /app/config/logging.yaml
+
 COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.yaml
 
 COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml

diff --git a/build/eventingester/Dockerfile b/build/eventingester/Dockerfile
@@ -9,5 +9,7 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 COPY eventingester /app/
 COPY config/eventingester/config.yaml /app/config/eventingester/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
+
 WORKDIR /app
 ENTRYPOINT ["./eventingester"]
diff --git a/build/executor/Dockerfile b/build/executor/Dockerfile
@@ -10,6 +10,7 @@ USER armada
 
 COPY executor /app/
 COPY config/executor/config.yaml /app/config/executor/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 
 WORKDIR /app
 

diff --git a/build/fakeexecutor/Dockerfile b/build/fakeexecutor/Dockerfile
@@ -9,6 +9,7 @@ USER armada
 
 COPY fakeexecutor /app/
 COPY config/executor/config.yaml /app/config/executor/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 
 WORKDIR /app
 

diff --git a/build/lookoutingesterv2/Dockerfile b/build/lookoutingesterv2/Dockerfile
@@ -10,6 +10,7 @@ USER armada
 
 COPY lookoutingesterv2 /app/
 COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 
 WORKDIR /app
 

diff --git a/build/lookoutv2/Dockerfile b/build/lookoutv2/Dockerfile
@@ -27,5 +27,6 @@ COPY lookoutv2 /app/
 COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml
 COPY lookoutingesterv2 /app/
 COPY config/lookoutingesterv2/ /app/config/lookoutingesterv2
+COPY config/logging.yaml /app/config/logging.yaml
 WORKDIR /app
 ENTRYPOINT ["./lookoutv2"]
diff --git a/build/scheduler/Dockerfile b/build/scheduler/Dockerfile
@@ -9,5 +9,6 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 COPY scheduler /app/
 COPY config/scheduler/config.yaml /app/config/scheduler/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 WORKDIR /app
 ENTRYPOINT ["./scheduler"]
diff --git a/build/scheduleringester/Dockerfile b/build/scheduleringester/Dockerfile
@@ -9,5 +9,6 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 COPY scheduleringester /app/
 COPY config/scheduleringester/config.yaml /app/config/scheduleringester/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 WORKDIR /app
 ENTRYPOINT ["./scheduleringester"]
diff --git a/build/server/Dockerfile b/build/server/Dockerfile
@@ -10,6 +10,7 @@ USER armada
 
 COPY server /app/
 COPY config/server/config.yaml /app/config/server/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
 
 WORKDIR /app
 

diff --git a/client/python/armada_client/asyncio_client.py b/client/python/armada_client/asyncio_client.py
@@ -105,6 +105,7 @@ def __init__(
         event_timeout: timedelta = timedelta(minutes=15),
     ) -> None:
         self.submit_stub = submit_pb2_grpc.SubmitStub(channel)
+        self.queue_stub = submit_pb2_grpc.QueueServiceStub(channel)
         self.event_stub = event_pb2_grpc.EventStub(channel)
         self.job_stub = job_pb2_grpc.JobsStub(channel)
         self.event_timeout = event_timeout
@@ -390,7 +391,7 @@ async def create_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty:
         :param queue: A queue to create.
         """
 
-        response = await self.submit_stub.CreateQueue(queue)
+        response = await self.queue_stub.CreateQueue(queue)
         return response
 
     async def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty:
@@ -400,7 +401,7 @@ async def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty:
         :param queue: A queue to update.
         """
 
-        response = await self.submit_stub.UpdateQueue(queue)
+        response = await self.queue_stub.UpdateQueue(queue)
         return response
 
     async def create_queues(
@@ -413,7 +414,7 @@ async def create_queues(
         """
 
         queue_list = submit_pb2.QueueList(queues=queues)
-        response = await self.submit_stub.CreateQueues(queue_list)
+        response = await self.queue_stub.CreateQueues(queue_list)
         return response
 
     async def update_queues(
@@ -426,7 +427,7 @@ async def update_queues(
         """
 
         queue_list = submit_pb2.QueueList(queues=queues)
-        response = await self.submit_stub.UpdateQueues(queue_list)
+        response = await self.queue_stub.UpdateQueues(queue_list)
         return response
 
     async def delete_queue(self, name: str) -> None:
@@ -438,7 +439,7 @@ async def delete_queue(self, name: str) -> None:
         :return: None
         """
         request = submit_pb2.QueueDeleteRequest(name=name)
-        await self.submit_stub.DeleteQueue(request)
+        await self.queue_stub.DeleteQueue(request)
 
     async def get_queue(self, name: str) -> submit_pb2.Queue:
         """Get the queue by name.
@@ -449,9 +450,23 @@ async def get_queue(self, name: str) -> submit_pb2.Queue:
         :return: A queue object. See the api definition.
         """
         request = submit_pb2.QueueGetRequest(name=name)
-        response = await self.submit_stub.GetQueue(request)
+        response = await self.queue_stub.GetQueue(request)
         return response
 
+    async def get_queues(self) -> list:
+        """Retrieves all queues
+        :return: List containing all queues.
+        """
+        queues = []
+        request = submit_pb2.StreamingQueueGetRequest()
+        async for message in self.queue_stub.GetQueues(request):
+            event_type = message.WhichOneof("event")
+            if event_type == "queue":
+                queues.append(message.queue)
+            elif event_type == "end":
+                break
+        return queues
+
     @staticmethod
     def unwatch_events(event_stream) -> None:
         """Closes gRPC event streams