Merge branch 'master' into lighting

grafnu · Jan 24, 2025 · 6b2fec3 · 6b2fec3
2 parents 46ae3fd + 0c8962b
commit 6b2fec3
Show file tree

Hide file tree

Showing 58 changed files with 2,749 additions and 2,394 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -22,7 +22,6 @@ jobs:
         container: [ "udmis", "validator", "pubber", "misc" ]
     env:
       PUSH_REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
       CONTAINER: ${{ matrix.container }}
       REF_NAME: ${{ github.ref_name }}
     steps:
@@ -40,11 +39,13 @@ jobs:
           registry: ${{ env.PUSH_REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Convert repository name to lowercase
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
       - name: Container build and push
         run: |
           revhash=$(git log -n 1 --pretty=format:"%h")
           IMAGE_HASH=g${revhash:0:8}
-          PUSH_REPO=$PUSH_REGISTRY/${IMAGE_NAME,,}
+          PUSH_REPO=$PUSH_REGISTRY/$IMAGE_NAME
 
           TAG_BASE=$PUSH_REPO:$CONTAINER
           PUSH_TAG=${TAG_BASE}-$IMAGE_HASH
@@ -66,6 +67,28 @@ jobs:
           echo
           docker history $PUSH_TAG
 
+  maven:
+    name: Publish maven package (pubber)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    timeout-minutes: 5
+    env:
+      GITHUB_ACTOR: ${{ github.actor }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: Publish maven package
+        run:  pubber/bin/publish
+
   simple:
     name: Simple sequence test
     runs-on: ubuntu-latest
@@ -76,7 +99,6 @@ jobs:
       matrix:
         device_id: [ "AHU-1", "AHU-22", "GAT-123" ]
     env:
-      IMAGE_NAME: ${{ github.repository }}
       REF_NAME: ${{ github.ref_name }}
       DEVICE_ID: ${{ matrix.device_id }}
     steps:
@@ -89,6 +111,8 @@ jobs:
           jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json
           jq . site_model/cloud_iot_config.json
           docker network create udminet --subnet 192.168.99.0/24
+      - name: Convert repository name to lowercase
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
       - name: Start UDMIS container
         run: |
           export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME
@@ -148,7 +172,6 @@ jobs:
     timeout-minutes: 5
     needs: images
     env:
-      IMAGE_NAME: ${{ github.repository }}
       REF_NAME: ${{ github.ref_name }}
       DEVICE_ID: ${{'AHU-1'}}
     steps:
@@ -161,6 +184,8 @@ jobs:
           jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json
           jq . site_model/cloud_iot_config.json
           docker network create udminet --subnet 192.168.99.0/24
+      - name: Convert repository name to lowercase
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
       - name: Start UDMIS container
         run: |
           export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME

diff --git a/common/src/main/java/com/google/udmi/util/CertManager.java b/common/src/main/java/com/google/udmi/util/CertManager.java
@@ -5,9 +5,13 @@
 import static java.lang.String.format;
 
 import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.security.KeyStore;
 import java.security.PrivateKey;
 import java.security.Security;
@@ -49,6 +53,9 @@ public class CertManager {
   private final File crtFile;
   private final char[] password;
   private final Transport transport;
+  private final String caCertificate;
+  private final String clientCertificate;
+  private final String clientPrivateKey;
 
   {
     Security.addProvider(new BouncyCastleProvider());
@@ -61,6 +68,9 @@ public CertManager(File caCrtFile, File clientDir, Transport transport,
       String passString, Consumer<String> logging) {
     this.caCrtFile = caCrtFile;
     this.transport = transport;
+    this.caCertificate = null;
+    this.clientCertificate = null;
+    this.clientPrivateKey = null;
 
     if (Transport.SSL.equals(transport)) {
       String prefix = keyPrefix(clientDir);
@@ -78,6 +88,18 @@ public CertManager(File caCrtFile, File clientDir, Transport transport,
     }
   }
 
+  public CertManager(String caCertificate, String clientCertificate, String clientPrivateKey,
+      Transport transport, String passString) {
+    caCrtFile = null;
+    crtFile = null;
+    keyFile = null;
+    this.transport = transport;
+    this.password = passString.toCharArray();
+    this.caCertificate = caCertificate;
+    this.clientCertificate = clientCertificate;
+    this.clientPrivateKey = clientPrivateKey;
+  }
+
   private String keyPrefix(File clientDir) {
     File rsaCrtFile = new File(clientDir, "rsa_private.crt");
     File ecCrtFile = new File(clientDir, "ec_private.crt");
@@ -92,18 +114,27 @@ private String keyPrefix(File clientDir) {
   public SSLSocketFactory getCertSocketFactory() throws Exception {
     CertificateFactory certFactory = CertificateFactory.getInstance(X509_FACTORY);
 
+    InputStream caCertStream = caCrtFile != null
+        ? new FileInputStream(caCrtFile)
+        : new ByteArrayInputStream(caCertificate.getBytes());
     final X509Certificate caCert;
-    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(caCrtFile))) {
+    try (BufferedInputStream bis = new BufferedInputStream(caCertStream)) {
       caCert = (X509Certificate) certFactory.generateCertificate(bis);
     }
 
+    InputStream clientCertStream = crtFile != null
+        ? new FileInputStream(crtFile)
+        : new ByteArrayInputStream(clientCertificate.getBytes());
     final X509Certificate clientCert;
-    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(crtFile))) {
+    try (BufferedInputStream bis = new BufferedInputStream(clientCertStream)) {
       clientCert = (X509Certificate) certFactory.generateCertificate(bis);
     }
 
+    Reader keyReader = keyFile != null
+        ? new FileReader(keyFile)
+        : new StringReader(clientPrivateKey);
     final PrivateKey privateKey;
-    try (PEMParser pemParser = new PEMParser(new FileReader(keyFile))) {
+    try (PEMParser pemParser = new PEMParser(keyReader)) {
       Object pemObject = pemParser.readObject();
       if (pemObject instanceof PEMEncryptedKeyPair keyPair) {
         PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password);
@@ -114,7 +145,8 @@ public SSLSocketFactory getCertSocketFactory() throws Exception {
         privateKey = converter.getPrivateKey(keyPair);
       } else {
         throw new RuntimeException(format("Unknown pem file type %s from %s",
-            pemObject.getClass().getSimpleName(), keyFile.getAbsolutePath()));
+            pemObject.getClass().getSimpleName(),
+            keyFile == null ? "" : keyFile.getAbsolutePath()));
       }
     }
 

diff --git a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java
@@ -28,6 +28,7 @@ public class CommandLineProcessor {
   private static final int LINUX_ERROR_CODE = -1;
   private final Object target;
   private final Method showHelpMethod;
+  private List<String> usageForms;
 
   Map<CommandLineOption, Method> optionMap = new TreeMap<>(
       (a, b) -> CASE_INSENSITIVE_ORDER.compare(getSortArg(a), getSortArg(b)));
@@ -69,6 +70,13 @@ public CommandLineProcessor(Object target) {
     checkState(duplicateLong.isEmpty(), "duplicate short form command line option");
   }
 
+  public CommandLineProcessor(Object target, List<String> usageForms) {
+    this(target);
+    if (usageForms != null && !usageForms.isEmpty()) {
+      this.usageForms = usageForms;
+    }
+  }
+
   private Method getShowHelpMethod() {
     try {
       return CommandLineProcessor.class.getDeclaredMethod("showUsage");
@@ -91,6 +99,10 @@ private void showUsage() {
    */
   public void showUsage(String message) {
     ifNotNullThen(message, m -> System.err.println(m));
+    ifNotNullThen(usageForms, () -> {
+      System.err.println("Usage forms:");
+      usageForms.forEach(form -> System.err.printf("    %s%n", form));
+    });
     System.err.println("Options supported:");
     optionMap.forEach((option, method) -> System.err.printf("  %s %12s  %s%n",
         option.short_form(), option.arg_name(), option.description()));

diff --git a/docs/tools/registrar.md b/docs/tools/registrar.md
@@ -39,22 +39,33 @@ be used to specific specific device(s) to register (rather than all).
 ```
 Usage:
 
-bin/registrar site_path [project_id] [options] [devices...]
+bin/registrar site_model project_spec [options] [devices...]
 
-bin/registrar config_file
+bin/registrar site_spec [options] [devices...]
 ```
 
-* `config_file`: Path to a configuration file which contains configuration options;
-* `site_path`: The _directory_ containing the site model, or a model-with-project _file_ directly.
-* `project_id`: The project ID that contains the target registry. The project ID can be prepended with iot_provider:
+* `site_model`: The path to the _directory_ containing the site model, or a model-with-project _file_ directly.
+* `project_spec`: The project ID that contains the target registry. The project ID can be prepended with iot_provider:
   * `//clearblade/PROJECT_ID` for a public ClearBlade project.
   * `//gbos/PROJECT_ID` for a Google operated ClearBlade project.
+* `site_spec`: Path to a configuration file which contains configuration options;
 * `options`: Various options to impact behavior:
-  * `-u` Update.
-  * `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry)
+  * `-a` Set alternate registry 
   * `-b` Block unknown devices.
-  * `-x` Delete unknown devices from the registry.
+  * `-c` Count of registries to be created
+  * `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry)
+  * `-e` Set registry suffix
+  * `-f` Set PubSub feed topic
+  * `-h` Show help and exit
+  * `-l` Set idle limit
+  * `-m` Initial metadata model out
   * `-n` Number of thread counts.
+  * `-p` Set Project ID
+  * `-q` Query only, registry to not be updated
+  * `-r` Set tool root path
+  * `-s` Set site path
+  * `-t` Do not validate metadata
+  * `-x` Delete unknown devices from the registry.
 * `devices`: Multiple device entries for limited registration. Can be just the device name
   (`AHU-12`), or path to device (`site/devices/AHU-12`) for use with file-name glob.
 

diff --git a/etc/validator.out b/etc/validator.out
@@ -339,15 +339,15 @@ sites/udmi_site_model/out/devices/AHU-22/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "1 schema violations found",
-    "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "1 schema violations found",
-    "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -382,15 +382,15 @@ sites/udmi_site_model/out/devices/AHU-22/state_system.out
   "sub_folder" : "system",
   "sub_type" : "state",
   "status" : {
-    "message" : "1 schema violations found",
-    "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "1 schema violations found",
-    "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -795,14 +795,14 @@ sites/udmi_site_model/out/devices/SNS-4/state.out
   "sub_type" : "state",
   "status" : {
     "message" : "Multiple validation errors",
-    "detail" : "1 schema violations found; Device has missing points: split_threshold, triangulating_axis",
+    "detail" : "2 schema violations found; Device has missing points: split_threshold, triangulating_axis",
     "category" : "validation.device.multiple",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "1 schema violations found",
-    "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -850,15 +850,15 @@ sites/udmi_site_model/out/devices/SNS-4/state_system.out
   "sub_folder" : "system",
   "sub_type" : "state",
   "status" : {
-    "message" : "1 schema violations found",
-    "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "1 schema violations found",
-    "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
+    "message" : "2 schema violations found",
+    "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500

diff --git a/pubber/.idea/runConfigurations/Swarm.xml b/pubber/.idea/runConfigurations/Swarm.xml
diff --git a/pubber/bin/publish b/pubber/bin/publish
@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+UDMI_ROOT=$(realpath $(dirname $0)/../..)
+cd $UDMI_ROOT
+
+DEFAULT_TASK="publish"
+task=${1:-$DEFAULT_TASK}
+
+# Validate the task
+VALID_TASKS=("publish" "publishToMavenLocal")
+if [[ ! " ${VALID_TASKS[@]} " =~ " $task " ]]; then
+    echo "Usage: $0 [publishTask]"
+    echo "  publishTask: Task to execute. Default is '$DEFAULT_TASK'."
+    echo "  Valid tasks are: ${VALID_TASKS[*]}"
+    echo
+    echo "Error: Invalid task '$task'"
+    exit 1
+fi
+
+source $UDMI_ROOT/etc/shell_common.sh
+
+echo pubber/gradlew -p pubber $task
+echo $UDMI_VERSION
+$UDMI_ROOT/pubber/gradlew -p pubber $task