updated user-mode data collection and support for flux resource manager

.github/workflows/test-user.yml → .github/workflows/test-user-pull.yml

@@ -1,12 +1,12 @@
-name: User mode
+name: User mode - Pull
 on:
   push:
     branches:  [ main, dev ]
   pull_request:
     branches: [ main, dev ]
 jobs:
   test:
-    name: User-level Omnistat
+    name: User-level Omnistat (pull/prometheus)
     runs-on: ubuntu-22.04
     strategy:
       matrix:
@@ -39,4 +39,4 @@ jobs:
         run: pip3 install -r test/requirements.txt
       - name: Run tests
         working-directory: ./test
-        run: pytest -v test_job_user.py
+        run: pytest -v test_job_user_pull.py

.github/workflows/test-user-push.yml

@@ -0,0 +1,42 @@
+name: User mode - Push
+on:
+  push:
+    branches:  [ main, dev ]
+  pull_request:
+    branches: [ main, dev ]
+jobs:
+  test:
+    name: User-level Omnistat (push/victoria)
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        execution: [ source, package ]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Comment out GPU devices (not available in GitHub)
+        run: sed -i "/devices:/,+2 s/^/#/" test/docker/slurm/compose.yaml
+      - name: Disable SMI collector (won't work in GitHub)
+        run: >
+          sed -i "s/enable_rocm_smi = True/enable_rocm_smi = False/" \
+            test/docker/slurm/omnistat-user.config
+      - name: Start containerized environment
+        env:
+          TEST_OMNISTAT_EXECUTION: ${{ matrix.execution }}
+        run: docker compose -f test/docker/slurm/compose.yaml -f test/docker/slurm/compose-user.yaml up -d
+      - name: Wait for user-level Omnistat
+        run: >
+          timeout 5m bash -c \
+            'for i in controller node1 node2; do \
+               until [[ $(docker logs -n 1 slurm-$i) == READY ]]; do \
+                 echo "Waiting for $i..."; \
+                 sleep 5; \
+               done \
+             done'
+      - name: Display node logs
+        run: for i in node1 node2; do docker logs slurm-$i; done
+      - name: Install test dependencies
+        run: pip3 install -r test/requirements.txt
+      - name: Run tests
+        working-directory: ./test
+        run: pytest -s -v test_job_user_push.py

docker/compose.prometheus.yaml

@@ -0,0 +1,30 @@
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.retention.time=1y'
+      - '--query.max-samples=500000000'
+    ports:
+      - 9090:9090
+    restart: unless-stopped
+    volumes:
+      - ./prometheus:/etc/prometheus
+      - ./prometheus-data:/prometheus
+    user: "${PROMETHEUS_USER}"
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+      - GF_USERS_DEFAULT_THEME=light
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Editor
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/json-models/index.json
+    volumes:
+      - ./grafana:/etc/grafana/provisioning

docker/compose.yaml

@@ -1,18 +1,14 @@
 services:
   prometheus:
-    image: prom/prometheus
+    image: victoriametrics/victoria-metrics
     container_name: prometheus
     command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.retention.time=1y'
-      - '--query.max-samples=500000000'
+      - '-httpListenAddr=:9090'
     ports:
       - 9090:9090
     restart: unless-stopped
     volumes:
-      - ./prometheus:/etc/prometheus
-      - ./prometheus-data:/prometheus
-    user: "${PROMETHEUS_USER}"
+      - ./prometheus-data/vicdata/:/victoria-metrics-data
   grafana:
     image: grafana/grafana
     container_name: grafana