Merge branch 'master' into model

.gencode_hash.txt

@@ -41,7 +41,7 @@ fa94295bc0c97518634c7232e7e55e1c923d7eb42bfa3004651ed183e1ccbb40  gencode/java/u
 2e7c0912c6f43b8dae9e9149b7361d3640bde4f01de6e13d8e487fe01be2c2c0  gencode/java/udmi/schema/Bucket.java
 79b678b524b3edce833fe56691dcd8c47b53a12f7491e761121408e6e7ceb5e3  gencode/java/udmi/schema/BuildingTranslation.java
 27ca4f61bd43beca0871b465f3906785a1030a4526605f1fa9b10e3adda8df05  gencode/java/udmi/schema/CapabilityValidationState.java
-ec164962f2f00924ecb41ae07c2e01d6bf027951ccc605e0210988370b987973  gencode/java/udmi/schema/Category.java
+224c6be2cb6d90a47d181fa30ba1418cf6cc5385ec1c010fe112728382a70216  gencode/java/udmi/schema/Category.java
 65a892c39e687a00e66e02f7ace2e17f07d3633a1b20b3dfd1e4ac3ec72a8788  gencode/java/udmi/schema/CloudConfigModel.java
 5fcf687fa0167889a63a1fbaafc8c3f2d9ac16609c8e804fdce8171903c37a1f  gencode/java/udmi/schema/CloudModel.java
 051b4a7c85a2f1a310787f88f336be2fc9b87c99a94ac2f970955d2fe1daa931  gencode/java/udmi/schema/CloudQuery.java

bin/pull_mqtt

@@ -55,12 +55,13 @@ sudo mosquitto_sub $SERVER_OPTS -R -F "%j" -t $topic_filter |
         out_path=$OUT_BASE/$registryId/devices/$deviceId/${timepath}/${usetime}_${subFolder}_${subType}
         out_dir=$(dirname $out_path)
         mkdir -p $out_dir
-        echo $(realpath --relative-to $OUT_BASE $out_path).json $topic
+        echo $(realpath --relative-to $UDMI_ROOT $out_path).json
         echo $json > $out_path.json
         echo {} | jq ".deviceRegistryId=\"$registryId\" | \
              .subFolder=\"$subFolder\" |
              .subType=\"$subType\" |
              .deviceId=\"$deviceId\"" > $out_path.attr
+        echo $topic > $out_path.mqtt
     done
 
 echo Finished mqtt message capture at $(date -u -Is)

bin/test_regclean

@@ -34,21 +34,23 @@ pubber_config=/tmp/pubber_config.json
 pubber_log=out/pubber.log
 serial_no=clean-$RANDOM
 device_id=AHU-1
+gateway_id=GAT-123
 fake_id=ACME-2301
 fake_device=$site_path/devices/$fake_id
 fake_metadata=$fake_device/metadata.json
 device_dir=$site_path/devices/$device_id
 
 function check_csv_file {
-    expected=$1
+    the_device=$1
+    expected=$2
     more $csv_file | cat
 
-    fake_state=`fgrep $fake_id $csv_file | cut -d, -f3`
-    [[ ${fake_state# } == $expected ]] || fail Device $fake_id not $expected
-    [[ `fgrep BLOCK $csv_file | wc -l` -le 1 ]] || fail More than one blocked device
+    state=`fgrep $the_device $csv_file | cut -d, -f3`
+    [[ ${state# } == $expected ]] || fail Device $the_device not $expected
+    [[ `fgrep BLOCK $csv_file | wc -l` -le 1 ]] || fail More than one matching blocked device
 
     device_num_id=`fgrep AHU-22 $csv_file | cut -d, -f2`
-    [[ $(echo $device_num_id) == $expected_num_id ]] || fail Device num_id $device_num_id does not match expected $expected_num_id
+    [[ $(echo $device_num_id) == $expected_num_id ]] || fail Device AHU-22 num_id $device_num_id does not match expected $expected_num_id
 
     sns_line=`fgrep -n SNS-4 $csv_file | cut -d: -f1`
     [[ $sns_line == 6 ]] || fail Device SNS-4 not found on expected output csv line, was $sns_line expected 6
@@ -168,45 +170,55 @@ jq '.iot_provider = "dynamic"' $iot_config | sponge $iot_config
 jq '.project_id = "this-is-not-right"' $iot_config | sponge $iot_config
 jq '.udmi_namespace = "bunny"' $iot_config | sponge $iot_config
 
-echo Create one device to test that it works...
-echo bin/registrar $site_args $device_id
-bin/registrar $site_args $device_id
-
-echo Check $device_id to make sure that it works...
-echo bin/pubber $pubber_config
-result=0
-timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$?
-[[ $result == 0 ]] || fail Improper pubber failure code $result
-
 echo Clean out the registry to make sure devices get removed...
 echo bin/registrar $site_args -d -x
 bin/registrar $site_args -d -x
 
 echo Checking reported cloud version info
 jq .cloud_version.udmi_ref $output_file
 
-echo Pubber config from $pubber_config:
-cat $pubber_config
+echo Cleaning message capture output out/registries/
+rm -rf out/registries/
 
 echo Check for failure in running pubber...
 echo bin/pubber $pubber_config
 result=0
 timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$?
 [[ $result == 255 ]] || fail Improper pubber failure code $result
 
-echo Now recreate the registry from scratch!
+echo Pubber config from $pubber_config:
+cat $pubber_config
+
+echo Create one device to test that it works...
+echo bin/registrar $site_args $device_id
+bin/registrar $site_args $device_id
+
+echo Check $device_id to make sure that it works...
+echo bin/pubber $pubber_config
+result=0
+timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$?
+[[ $result == 0 ]] || fail Improper pubber failure code $result
+
+echo Now recreate the complete registry from scratch!
 echo bin/registrar $site_args
 bin/registrar $site_args
 
-check_csv_file ERROR
+check_csv_file $fake_id ERROR
 
 echo Remove fake device from site model, so it will be blocked
 rm -rf $fake_device
 echo $UDMI_ROOT/bin/registrar $full_args -b
 # Test from other-directory for proper path handling
 (cd /tmp; $UDMI_ROOT/bin/registrar $full_args -b)
 
-check_csv_file BLOCK
+check_csv_file $fake_id BLOCK
+check_csv_file $gateway_id READ
+
+echo Delete the gateway to make sure stuff happens...
+echo bin/registrar $site_args -d $gateway_id
+bin/registrar $site_args -d $gateway_id
+
+check_csv_file $gateway_id CLEAN
 
 echo Check $device_id again, but this time with success...
 echo bin/pubber $pubber_config
@@ -216,6 +228,10 @@ timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$?
 
 echo Successful pubber check passed on clean registry.
 
+echo Registering everything to end up with everything in place
+echo bin/registrar $site_args
+bin/registrar $site_args
+
 if [[ -n $solo_arg ]]; then
     echo Deleting singular entry $solo_arg...
     echo bin/registrar $site_args -d $solo_arg

bin/test_registrar

@@ -1,26 +1,27 @@
 #!/bin/bash -e
 
-ROOT_DIR=$(dirname $0)/..
-cd ${ROOT_DIR}
+UDMI_ROOT=$(dirname $0)/..
+cd ${UDMI_ROOT}
+source etc/shell_common.sh
 
 bin/clone_model
 
 TEST_SITE=sites/udmi_site_model
 find $TEST_SITE -name out | xargs rm -rf
 (cd $TEST_SITE; git checkout .)
 
-exit_status=clean
+exit_status=
 echo '###################################' starting test section processing
 
 bin/registrar ${TEST_SITE}
 
 output_file=${TEST_SITE}/out/registration_summary.json
 
-clean_devices=$(jq -r .Clean[] $output_file | wc -l)
+clean_devices=$(jq -r .status $output_file | fgrep CLEAN | wc -l)
 EXPECTED_CLEAN=3  # AHU-1 has a bad site location, so does not match as clean.
 
 echo Found ${clean_devices} clean devices, expecting $EXPECTED_CLEAN.
-[ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status=expected_clean
+[ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status+=" expected_clean"
 echo '###################################' end test section expected_clean: $exit_status
 
 device_files=$(ls -d ${TEST_SITE}/devices/* | fgrep -v XXX)
@@ -33,7 +34,7 @@ generated_config=sites/udmi_site_model/devices/AHU-1/out/generated_config.json
 [[ "Degrees-Celsius" == $(jq -r .pointset.points.filter_differential_pressure_sensor.units $generated_config) ]] || (echo Units mismatch; touch $error_file)
 if [[ -f $error_file ]]; then
     echo Errors found testing generated config.
-    exit_status=generated_config
+    exit_status+=" generated_config"
 fi
 echo '###################################' end test section generated_config: $exit_status
 
@@ -62,7 +63,7 @@ function check_norm {
 sm_devices=$(check_norm)
 echo Devices with correct site_defaults: ${sm_devices}
 
-[ "${sm_devices}" == "${device_files}" ] || exit_status=site_defaults
+[ "${sm_devices}" == "${device_files}" ] || exit_status+=" site_defaults"
 echo '###################################' end test section site_defaults: $exit_status
 
 mv ${TEST_SITE}/site_defaults.json ${TEST_SITE}/_site_defaults.json
@@ -73,19 +74,19 @@ base_devices=sites/udmi_site_model/devices/AHU-1
 sm_devices=$(check_norm)
 echo Devices with correct no-defaults site_defaults: ${sm_devices}
 echo Expecting: $base_devices
-[ "${sm_devices}" == "${base_devices}" ] || exit_status=no_defaults
+[ "${sm_devices}" == "${base_devices}" ] || exit_status+=" no_defaults"
 echo '###################################' end test section no_defaults: $exit_status
 
 
 config_points=`jq '.pointset.points | length' $TEST_SITE/devices/AHU-22/out/generated_config.json`
-[[ $config_points == 3 ]] || exit_status=missing_points
+[[ $config_points == 3 ]] || exit_status+=" missing_points"
 echo '###################################' end test section missing_points: $exit_status
 
 config_points=`jq .pointset.points $TEST_SITE/devices/SNS-4/out/generated_config.json`
-[[ $config_points == null ]] || exit_status=configured_points
+[[ $config_points == null ]] || exit_status+=" configured_points"
 echo '###################################' end test section configured_points: $exit_status
 
 mv ${TEST_SITE}/_site_defaults.json ${TEST_SITE}/site_defaults.json
 
-echo Done with registrar tests, results are \'$exit_status\' \(expected to be \'clean\'\)
-[[ $exit_status == clean ]]
+[[ -z $exit_status ]] || fail Failed registrar tests: $exit_status
+echo Registrar standalone tests passed.

bin/test_runlocal

@@ -12,13 +12,16 @@ cat out/message_capture.log
 
 echo Captured $(wc -l out/message_capture.log) messages.
 
-systems=$(find out/registries/ -name \*update_model.json | xargs jq .system | fgrep -v null | wc -l)
-[[ $systems == 196 ]] || fail Expected 196 system operations, found $systems
+systems=$(find out/registries/ -name \*update_model.json | xargs jq .system | fgrep -v null | wc -l) || true
+[[ $systems == 269 ]] || fail Expected 269 system operations, found $systems
 
-deletes=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep DELETE | wc -l)
+deletes=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep DELETE | wc -l) || true
 [[ $deletes == 1 ]] || fail Expected 1 delete operations, found $deletes
 
-binds=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BIND | wc -l)
-[[ $binds == 2 ]] || fail Expected 2 bind operations, found $binds
+binds=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BIND | wc -l) || true
+[[ $binds == 3 ]] || fail Expected 3 bind operations, found $binds
+
+blocks=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BLOCK | wc -l) || true
+[[ $blocks == 1 ]] || fail Expected 1 block operations, found $blocks
 
 echo Done with successful runlocal validation.

common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java

@@ -0,0 +1,41 @@
+package com.google.daq.mqtt.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple class for a keyed map of exceptions.
+ */
+public class ErrorMap extends HashMap<String, Exception> {
+
+  private final String description;
+
+  public ErrorMap(String description) {
+    super();
+    this.description = description;
+  }
+
+  /**
+   * Throw an exception if there's something in the map.
+   */
+  public void throwIfNotEmpty() {
+    if (!isEmpty()) {
+      throw asException();
+    }
+  }
+
+  public ErrorMapException asException() {
+    return new ErrorMapException();
+  }
+
+  public class ErrorMapException extends RuntimeException {
+
+    ErrorMapException() {
+      super(description);
+    }
+
+    public Map<String, Exception> getMap() {
+      return ErrorMap.this;
+    }
+  }
+}

common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java

@@ -2,6 +2,7 @@
 
 import static com.google.udmi.util.GeneralUtils.multiTrim;
 
+import com.google.daq.mqtt.util.ErrorMap.ErrorMapException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -25,12 +26,18 @@ public class ExceptionMap extends RuntimeException {
   private static final byte[] SEPARATOR_BYTES = ": ".getBytes();
   private static final String ERROR_FORMAT_INDENT = "  ";
 
-  final Map<String, Exception> exceptions = new TreeMap<>();
+  final Map<ExceptionCategory, Exception> exceptions = new TreeMap<>();
 
   public ExceptionMap(String description) {
     super(description);
   }
 
+  public enum ExceptionCategory {
+    missing, extra, out, validation, loading, writing, site_metadata, initializing, sample,
+    registering, envelope, credentials, samples, files, binding, creating, updating, schema,
+    configuring, status
+  }
+
   /**
    * Format the given exception with indicated level.
    *
@@ -46,12 +53,15 @@ private static ErrorTree format(Throwable e, final String prefix, final String i
     errorTree.prefix = prefix;
     errorTree.message = multiTrim(e.getMessage());
     final String newPrefix = prefix + indent;
-    if (e instanceof ExceptionMap) {
+    if (e instanceof ExceptionMap exceptionMap) {
       if (e.getCause() != null) {
         errorTree.child = format(e.getCause(), newPrefix, indent);
       }
-      ((ExceptionMap) e)
-          .forEach((key, sub) -> errorTree.children.put(key, format(sub, newPrefix, indent)));
+      exceptionMap.forEach((key, sub) ->
+          errorTree.children.put(key.toString(), format(sub, newPrefix, indent)));
+    } else if (e instanceof ErrorMapException errorMap) {
+      errorMap.getMap().forEach((key, sub) ->
+          errorTree.children.put(key, format(sub, newPrefix, indent)));
     } else if (e instanceof ValidationException) {
       ((ValidationException) e)
           .getCausingExceptions()
@@ -69,7 +79,7 @@ private static ErrorTree format(Throwable e, final String prefix, final String i
     return errorTree;
   }
 
-  private void forEach(BiConsumer<String, Exception> consumer) {
+  private void forEach(BiConsumer<ExceptionCategory, Exception> consumer) {
     exceptions.forEach(consumer);
   }
 
@@ -96,13 +106,13 @@ public void throwIfNotEmpty() {
    * @param key       entry key
    * @param exception exception to add
    */
-  public void put(String key, Exception exception) {
+  public void put(ExceptionCategory key, Exception exception) {
     if (exceptions.put(key, exception) != null) {
       throw new IllegalArgumentException("Exception key already defined: " + key);
     }
   }
 
-  public Stream<Map.Entry<String, Exception>> stream() {
+  public Stream<Map.Entry<ExceptionCategory, Exception>> stream() {
     return exceptions.entrySet().stream();
   }
 
@@ -118,7 +128,7 @@ public int size() {
   /**
    * Execute the action and capture into the map if it throws an exception.
    */
-  public void capture(String category, Runnable action) {
+  public void capture(ExceptionCategory category, Runnable action) {
     try {
       action.run();
     } catch (Exception e) {