Enable CI integration tests for discovery node (#1047)

.github/workflows/integration.yaml

@@ -4,6 +4,8 @@ on:
   push:
     tags:
       - '**'
+    branches:
+      - '**'
   workflow_dispatch:
 
 jobs:
@@ -141,20 +143,60 @@ jobs:
         run: docker logs sequencer
 
   discovery:
-    name: Simple discovery test
+    name: Simple discovery sequencer test
     runs-on: ubuntu-latest
     timeout-minutes: 5
     needs: images
     env:
       IMAGE_NAME: ${{ github.repository }}
       REF_NAME: ${{ github.ref_name }}
+      DEVICE_ID: ${{'AHU-1'}}
     steps:
-      - name: Execute discovery node
+      - name: Setup Environment
+        run: |
+          sudo apt-get install moreutils
+          git clone https://github.com/faucetsdn/udmi_site_model.git
+          ln -s udmi_site_model/ site_model
+          (cd site_model; git log -n 1)
+          jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json
+          jq . site_model/cloud_iot_config.json
+          docker network create udminet --subnet 192.168.99.0/24
+      - name: Start UDMIS container
+        run: |
+          export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME
+          docker run $IMAGE_TAG cat bin/actualize > /tmp/actualize.sh
+          cat /tmp/actualize.sh && bash /tmp/actualize.sh
+      - name: Generate keys
+        run: |
+          docker run --net udminet --name keygen -v $(realpath site_model):/root/site_model \
+            ghcr.io/$IMAGE_NAME:validator-$REF_NAME bin/keygen CERT site_model/devices/$DEVICE_ID
+      - name: Registrar run
+        run: |
+          docker run --net udminet --name registrar -v $(realpath site_model):/root/site_model \
+            ghcr.io/$IMAGE_NAME:validator-$REF_NAME bin/registrar site_model/cloud_iot_config.json
+      - name: Start discoverynode
         run: |
           export IMAGE_TAG=ghcr.io/$IMAGE_NAME:misc-$REF_NAME
           # This currently fails (no config), but just check that it actually runs.
-          docker run $IMAGE_TAG discoverynode/bin/run | fgrep "Loading config"
-
+            docker run -d \
+              --net udminet \
+              -v $(realpath site_model):/var/site_model \
+              --name discoverynode \
+              $IMAGE_TAG \
+              bin/run /var/site_model //mqtt/udmis AHU-1 vendor.range=0x65,28179023,20231,,,,,,,,,,,,,,,,
+      - name: Sequencer run
+        run: |
+          docker run --net udminet --name sequencer -v $(realpath site_model):/root/site_model \
+            ghcr.io/$IMAGE_NAME:validator-$REF_NAME bin/sequencer site_model/cloud_iot_config.json \
+            scan_single_future
+      - name: Sequencer results
+        run: |
+          cat site_model/out/devices/$DEVICE_ID/results.md
+          [[ $(cat site_model/out/devices/$DEVICE_ID/results.md | grep -Ec ".*discovery.* pass") == 1 ]]
+      - name: Discoverynode logs
+        if: ${{ !cancelled() }}
+        run: docker logs discoverynode
+
   udmif:
     name: UDMIF unit tests
     runs-on: ubuntu-latest

.github/workflows/testing.yml

@@ -352,46 +352,3 @@ jobs:
       - name: Itemized validation
         if: ${{ !cancelled() }}
         run: bin/test_itemcheck
-
-  discovery:
-    name: Discovery Tests
-    runs-on: ubuntu-24.04
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          fetch-tags: true
-      - uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-      - name: Setup prerequisites
-        run: |
-          bin/setup_base
-          bin/clone_model
-          ln -s sites/udmi_site_model/ site_model
-          (cd site_model; git log -n 1)
-          docker network create udminet --subnet 192.168.99.0/24
-      - name: Build UDMIS
-        run: |
-          udmis/bin/build check
-          bin/container udmis build --no-check latest
-          echo Built local UDMIS
-      - name: Start UDMIS container
-        run: |
-          docker run -d --net udminet --name udmis -p 8883:8883 \
-            -v $PWD/site_model:/root/site_model \
-            -v $PWD/var/tmp:/tmp \
-            -v $PWD/var/etcd:/root/udmi/var/etcd \
-            -v $PWD/var/mosquitto:/etc/mosquitto \
-            udmis udmi/bin/start_local block site_model/cloud_iot_config.json
-          for count in `seq 0 30`; do
-            echo Waiting for UDMIS startup $((30 - count))
-            [[ ! -f var/tmp/pod_ready.txt ]] || break
-            (docker ps | fgrep -q udmis) || break
-            sleep 1
-          done
-          ls -l var/tmp/pod_ready.txt 2>&1
-      - name: Run Tests
-        run: misc/discoverynode/testing/e2e/test_local site_model || true

misc/Dockerfile.misc

@@ -1,12 +1,11 @@
 FROM debian:12-slim
 
-WORKDIR /root
+COPY discoverynode/ /root/discoverynode/
+WORKDIR /root/discoverynode
 
 RUN apt-get update && \
-    apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev sudo && \
+    apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev sudo jq moreutils && \
     python3 -m venv venv && \
     venv/bin/pip install --upgrade pip setuptools wheel
 
-ADD discoverynode/src/requirements.txt /tmp/
-RUN venv/bin/pip install --disable-pip-version-check -r /tmp/requirements.txt
-ADD discoverynode/ discoverynode/
+RUN venv/bin/pip install --disable-pip-version-check -r src/requirements.txt

misc/discoverynode/README.md

@@ -168,4 +168,6 @@ TODO
 - Unit tests - `~/venv/bin/python3 -m pytest tests/`
 - Integration tests - TODO
 
-##
+## Building binary
+
+Use `bin/build_binary`

misc/discoverynode/bin/build_binary

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+set -x
+ROOT_DIR=$(realpath $(dirname $0)/..)
+
+TMP_DIR=$(mktemp -d)
+OUT_DIR=$ROOT_DIR/dist
+OUT_FILE=$OUT_DIR/discoverynode
+
+echo Building binary to 
+cat >$TMP_DIR/build.sh <<-EOF
+            set -x
+            mkdir /build
+            cp -r /src /build
+            cd /build/src
+            pip3 install -r requirements.txt
+            pyinstaller --onefile --hidden-import udmi main.py
+            mv dist/main /tmp/main
+EOF
+
+docker pull ghcr.io/noursaidi/discoverybuilder:latest 
+docker run --rm --volume $ROOT_DIR/src:/src --volume $TMP_DIR:/tmp ghcr.io/noursaidi/discoverybuilder:latest /bin/bash /tmp/build.sh
+mkdir -p $OUT_DIR
+mv $TMP_DIR/main $OUT_FILE
+chmod 7755 $OUT_FILE

misc/discoverynode/bin/run

@@ -11,8 +11,8 @@ ROOT_DIR=$(realpath $(dirname $0)/..)
 BASE_CONFIG=$(realpath $ROOT_DIR/etc/base_config.json)
 TMP_CONFIG="/tmp/discoverynode_config.json"
 
-if [[ $# -ne 3 ]]; then
-  error Usage: $0 SITE_MODEL TARGET DEVICE_ID
+if (( $# -le 4 )); then
+  error Usage: $0 SITE_MODEL TARGET DEVICE_ID [OPTIONS]
 fi
 
 site_model=$1
@@ -65,12 +65,8 @@ EOF
   )
 
 elif [[ $provider == mqtt ]]; then
-  if [[ $project != localhost ]]; then
-    error only localhost supported
-  fi
-
   substitutions=$(cat <<EOF
-    .mqtt.host|="localhost" |
+    .mqtt.host|="$project" |
     .mqtt.port|=8883 |
     .mqtt.region|="$region" |
     .mqtt.project_id|="$project" |
@@ -88,6 +84,12 @@ else
   exit
 fi
 
+shift 3
 
 cat $TMP_CONFIG | jq -r  "$substitutions" | sponge $TMP_CONFIG
+
+for option in "$@"; do
+  cat $TMP_CONFIG | jq -r ".${option/=/|=\"}\"" | sponge $TMP_CONFIG
+done
+
 $ROOT_DIR/venv/bin/python3 $ROOT_DIR/src/main.py --config_file=$TMP_CONFIG

misc/discoverynode/discoverybuilder.Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+
+WORKDIR /tmp
+
+RUN apt update && apt -y install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget 
+RUN wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz 
+RUN wget https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz
+
+RUN apt -y remove openssl
+RUN tar -zxf openssl-1.1.1g.tar.gz
+RUN tar -xzvf Python-3.12.8.tgz
+
+WORKDIR /tmp/openssl-1.1.1g
+RUN ./config
+RUN make
+RUN make install
+RUN cp -r libssl.so.1.1 /usr/lib && cp -r libcrypto.so.1.1 /usr/lib
+
+WORKDIR /tmp/Python-3.12.8
+RUN ./configure --enable-optimizations --with-openssl=/usr/local --with-ensurepip=install CFLAGS="-I/usr/include" LDFLAGS="-Wl,-rpath /usr/local/lib" --enable-shared --prefix=/usr/local
+RUN make
+RUN make install
+
+RUN python3 -m pip install pyinstaller
+
+WORKDIR /root

misc/discoverynode/src/main.py

@@ -36,7 +36,8 @@ def get_arguments():
   parser.add_argument(
       "--config_file",
       type=str,
-      help="path to config file"
+      help="path to config file",
+      required=True
   )
   return parser.parse_args()
 

misc/discoverynode/src/udmi/core.py

@@ -144,7 +144,7 @@ def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True):
 
     if vendor:
       number_discovery = udmi.discovery.numbers.NumberDiscovery(
-          self.state, self.publish_discovery
+          self.state, self.publish_discovery, range=self.config.get("vendor", {}).get("range"),
       )
 
       self.add_config_route(

misc/discoverynode/src/udmi/discovery/numbers.py

@@ -11,9 +11,10 @@ class NumberDiscovery(discovery.DiscoveryController):
 
   scan_family = "vendor"
 
-  def __init__(self, state, publisher):
+  def __init__(self, state, publisher, *, range):
     self.cancelled = None
     self.task_thread = None
+    self.range = range
     super().__init__(state, publisher)
 
   def start_discovery(self):
@@ -26,13 +27,19 @@ def start_discovery(self):
   @discovery.catch_exceptions_to_state
   @discovery.main_task
   def discoverer(self):
-    for i in itertools.count(1):
+    if self.range:
+      iterator = self.range.split(",")
+    else:
+      iterator = itertools.count(1)
+
+    for i in iterator:
       if self.cancelled:
         return
-      result = DiscoveryEvent(
-          generation=self.generation, scan_family=self.scan_family, scan_addr=str(i)
-      )
-      self.publish(result)
+      if i:
+        result = DiscoveryEvent(
+            generation=self.generation, scan_family=self.scan_family, scan_addr=str(i)
+        )
+        self.publish(result)
       time.sleep(1)
 
   def stop_discovery(self):

misc/discoverynode/src/udmi/discovery/passive.py

@@ -182,15 +182,14 @@ def queue_worker(self):
                 and ip is not None
                 and ip not in self.addresses_seen
             ):
-              if ipaddress.ip_address(ip).is_private:
-                self.addresses_seen.add(ip)
-                self.device_records.add(
-                    PassiveScanRecord(
-                        addr=ip,
-                        hostname=self.get_host(ip),
-                        mac=getattr(item[scapy.layers.inet.Ether], x),
-                    )
-                )
+              self.addresses_seen.add(ip)
+              self.device_records.add(
+                  PassiveScanRecord(
+                      addr=ip,
+                      hostname=self.get_host(ip),
+                      mac=getattr(item[scapy.layers.inet.Ether], x),
+                  )
+              )
       except queue.Empty:
         if self.cancel_threads.is_set():
           return

misc/discoverynode/src/udmi/publishers/mqtt.py

@@ -236,14 +236,17 @@ def make_client(self):
         )
 
         if self.autentication_mechanism == "jwt_gcp":
-            client.tls_set(tls_version=ssl.PROTOCOL_TLSv1_2)
+            client.tls_set(self.ca_file, tls_version=ssl.PROTOCOL_TLSv1_2)
         elif self.autentication_mechanism == "udmi_local":
             client.tls_set(
                 self.ca_file,
                 tls_version=ssl.PROTOCOL_TLSv1_2,
                 keyfile=self.private_key_file,
                 certfile=self.cert_file,
             )
+            # I don't know why this doesn't like the UDMIS certs
+            # ssl.SSLError: [SSL] PEM lib (_ssl.c:3874)
+            # TODO: Investigate SSL errors with local UDMIS
             client.tls_insecure_set(True)
         else:
             raise RuntimeError("unknown authentication mechanism")