From 8792d148681bb9b2bbe7a8da8d5112c6bbcbda8a Mon Sep 17 00:00:00 2001 From: Trevor Date: Mon, 27 Jan 2025 21:27:34 -0800 Subject: [PATCH] Updates to registrar tool (#1075) --- .gencode_hash.txt | 2 +- bin/pull_mqtt | 3 +- bin/test_regclean | 56 ++++++---- bin/test_registrar | 25 ++--- bin/test_runlocal | 13 ++- .../com/google/daq/mqtt/util/ErrorMap.java | 41 +++++++ .../google/daq/mqtt/util/ExceptionMap.java | 26 +++-- .../udmi/util/CommandLineProcessor.java | 20 ++-- .../java/com/google/udmi/util/Common.java | 4 +- .../com/google/udmi/util/GeneralUtils.java | 6 +- .../java/com/google/udmi/util/SiteModel.java | 15 ++- etc/Category.java | 2 +- gencode/java/udmi/schema/Category.java | 2 +- .../basic/devices/AHU-1/expected/errors.map | 4 +- .../basic/devices/GAT-123/expected/errors.map | 2 +- .../basic/devices/GAT-72/expected/errors.map | 2 +- .../configs/devices/AHU-1/expected/errors.map | 6 +- .../devices/GAT-123/expected/errors.map | 4 +- .../devices/GAT-456/expected/errors.map | 4 +- .../devices/GAT-123/expected/errors.map | 2 +- .../devices/CGW-1/expected/errors.map | 2 +- .../devices/CGW-12/expected/errors.map | 2 +- .../devices/DDC-54/expected/errors.map | 2 +- .../devices/DWN-1/expected/errors.map | 2 +- .../devices/DWN-3/expected/errors.map | 2 +- .../devices/GAT-123/expected/errors.map | 2 +- .../missing/devices/AHU-1/expected/errors.map | 4 +- .../devices/AHU-22/expected/errors.map | 19 ++-- .../devices/GAT-123/expected/errors.map | 2 +- .../access/ClearBladeIotAccessProvider.java | 2 +- .../udmi/service/core/ReflectProcessor.java | 7 +- validator/.idea/compiler.xml | 3 + ...wngrade_site.xml => Registrar_missing.xml} | 4 +- validator/build.gradle | 4 + .../iot/core/proxy/IotReflectorClient.java | 3 +- .../daq/mqtt/registrar/LocalDevice.java | 57 ++++++---- .../google/daq/mqtt/registrar/Registrar.java | 101 +++++++++++------- .../google/daq/mqtt/util/CloudIotManager.java | 9 +- .../google/daq/mqtt/util/PubSubClient.java | 2 +- .../mqtt/validator/MessageReadingClient.java | 12 ++- .../google/daq/mqtt/validator/Validator.java | 11 +- .../daq/mqtt/registrar/RegistrarTest.java | 10 +- .../google/daq/mqtt/validator/TestBase.java | 18 ++-- 43 files changed, 336 insertions(+), 183 deletions(-) create mode 100644 common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java rename validator/.idea/runConfigurations/{Registrar_downgrade_site.xml => Registrar_missing.xml} (72%) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index a3c5916954..7421e14711 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -41,7 +41,7 @@ fa94295bc0c97518634c7232e7e55e1c923d7eb42bfa3004651ed183e1ccbb40 gencode/java/u 2e7c0912c6f43b8dae9e9149b7361d3640bde4f01de6e13d8e487fe01be2c2c0 gencode/java/udmi/schema/Bucket.java 79b678b524b3edce833fe56691dcd8c47b53a12f7491e761121408e6e7ceb5e3 gencode/java/udmi/schema/BuildingTranslation.java 27ca4f61bd43beca0871b465f3906785a1030a4526605f1fa9b10e3adda8df05 gencode/java/udmi/schema/CapabilityValidationState.java -ec164962f2f00924ecb41ae07c2e01d6bf027951ccc605e0210988370b987973 gencode/java/udmi/schema/Category.java +224c6be2cb6d90a47d181fa30ba1418cf6cc5385ec1c010fe112728382a70216 gencode/java/udmi/schema/Category.java 65a892c39e687a00e66e02f7ace2e17f07d3633a1b20b3dfd1e4ac3ec72a8788 gencode/java/udmi/schema/CloudConfigModel.java 5fcf687fa0167889a63a1fbaafc8c3f2d9ac16609c8e804fdce8171903c37a1f gencode/java/udmi/schema/CloudModel.java 051b4a7c85a2f1a310787f88f336be2fc9b87c99a94ac2f970955d2fe1daa931 gencode/java/udmi/schema/CloudQuery.java diff --git a/bin/pull_mqtt b/bin/pull_mqtt index 400b5bda9e..bfc91704be 100755 --- a/bin/pull_mqtt +++ b/bin/pull_mqtt @@ -55,12 +55,13 @@ sudo mosquitto_sub $SERVER_OPTS -R -F "%j" -t $topic_filter | out_path=$OUT_BASE/$registryId/devices/$deviceId/${timepath}/${usetime}_${subFolder}_${subType} out_dir=$(dirname $out_path) mkdir -p $out_dir - echo $(realpath --relative-to $OUT_BASE $out_path).json $topic + echo $(realpath --relative-to $UDMI_ROOT $out_path).json echo $json > $out_path.json echo {} | jq ".deviceRegistryId=\"$registryId\" | \ .subFolder=\"$subFolder\" | .subType=\"$subType\" | .deviceId=\"$deviceId\"" > $out_path.attr + echo $topic > $out_path.mqtt done echo Finished mqtt message capture at $(date -u -Is) diff --git a/bin/test_regclean b/bin/test_regclean index 14e3c69932..8dacafcc71 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -34,21 +34,23 @@ pubber_config=/tmp/pubber_config.json pubber_log=out/pubber.log serial_no=clean-$RANDOM device_id=AHU-1 +gateway_id=GAT-123 fake_id=ACME-2301 fake_device=$site_path/devices/$fake_id fake_metadata=$fake_device/metadata.json device_dir=$site_path/devices/$device_id function check_csv_file { - expected=$1 + the_device=$1 + expected=$2 more $csv_file | cat - fake_state=`fgrep $fake_id $csv_file | cut -d, -f3` - [[ ${fake_state# } == $expected ]] || fail Device $fake_id not $expected - [[ `fgrep BLOCK $csv_file | wc -l` -le 1 ]] || fail More than one blocked device + state=`fgrep $the_device $csv_file | cut -d, -f3` + [[ ${state# } == $expected ]] || fail Device $the_device not $expected + [[ `fgrep BLOCK $csv_file | wc -l` -le 1 ]] || fail More than one matching blocked device device_num_id=`fgrep AHU-22 $csv_file | cut -d, -f2` - [[ $(echo $device_num_id) == $expected_num_id ]] || fail Device num_id $device_num_id does not match expected $expected_num_id + [[ $(echo $device_num_id) == $expected_num_id ]] || fail Device AHU-22 num_id $device_num_id does not match expected $expected_num_id sns_line=`fgrep -n SNS-4 $csv_file | cut -d: -f1` [[ $sns_line == 6 ]] || fail Device SNS-4 not found on expected output csv line, was $sns_line expected 6 @@ -168,16 +170,6 @@ jq '.iot_provider = "dynamic"' $iot_config | sponge $iot_config jq '.project_id = "this-is-not-right"' $iot_config | sponge $iot_config jq '.udmi_namespace = "bunny"' $iot_config | sponge $iot_config -echo Create one device to test that it works... -echo bin/registrar $site_args $device_id -bin/registrar $site_args $device_id - -echo Check $device_id to make sure that it works... -echo bin/pubber $pubber_config -result=0 -timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$? -[[ $result == 0 ]] || fail Improper pubber failure code $result - echo Clean out the registry to make sure devices get removed... echo bin/registrar $site_args -d -x bin/registrar $site_args -d -x @@ -185,8 +177,8 @@ bin/registrar $site_args -d -x echo Checking reported cloud version info jq .cloud_version.udmi_ref $output_file -echo Pubber config from $pubber_config: -cat $pubber_config +echo Cleaning message capture output out/registries/ +rm -rf out/registries/ echo Check for failure in running pubber... echo bin/pubber $pubber_config @@ -194,11 +186,24 @@ result=0 timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$? [[ $result == 255 ]] || fail Improper pubber failure code $result -echo Now recreate the registry from scratch! +echo Pubber config from $pubber_config: +cat $pubber_config + +echo Create one device to test that it works... +echo bin/registrar $site_args $device_id +bin/registrar $site_args $device_id + +echo Check $device_id to make sure that it works... +echo bin/pubber $pubber_config +result=0 +timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$? +[[ $result == 0 ]] || fail Improper pubber failure code $result + +echo Now recreate the complete registry from scratch! echo bin/registrar $site_args bin/registrar $site_args -check_csv_file ERROR +check_csv_file $fake_id ERROR echo Remove fake device from site model, so it will be blocked rm -rf $fake_device @@ -206,7 +211,14 @@ echo $UDMI_ROOT/bin/registrar $full_args -b # Test from other-directory for proper path handling (cd /tmp; $UDMI_ROOT/bin/registrar $full_args -b) -check_csv_file BLOCK +check_csv_file $fake_id BLOCK +check_csv_file $gateway_id READ + +echo Delete the gateway to make sure stuff happens... +echo bin/registrar $site_args -d $gateway_id +bin/registrar $site_args -d $gateway_id + +check_csv_file $gateway_id CLEAN echo Check $device_id again, but this time with success... echo bin/pubber $pubber_config @@ -216,6 +228,10 @@ timeout 30s bin/pubber $pubber_config 2> $pubber_log || result=$? echo Successful pubber check passed on clean registry. +echo Registering everything to end up with everything in place +echo bin/registrar $site_args +bin/registrar $site_args + if [[ -n $solo_arg ]]; then echo Deleting singular entry $solo_arg... echo bin/registrar $site_args -d $solo_arg diff --git a/bin/test_registrar b/bin/test_registrar index d5bce4e1ad..f30efc492b 100755 --- a/bin/test_registrar +++ b/bin/test_registrar @@ -1,7 +1,8 @@ #!/bin/bash -e -ROOT_DIR=$(dirname $0)/.. -cd ${ROOT_DIR} +UDMI_ROOT=$(dirname $0)/.. +cd ${UDMI_ROOT} +source etc/shell_common.sh bin/clone_model @@ -9,18 +10,18 @@ TEST_SITE=sites/udmi_site_model find $TEST_SITE -name out | xargs rm -rf (cd $TEST_SITE; git checkout .) -exit_status=clean +exit_status= echo '###################################' starting test section processing bin/registrar ${TEST_SITE} output_file=${TEST_SITE}/out/registration_summary.json -clean_devices=$(jq -r .Clean[] $output_file | wc -l) +clean_devices=$(jq -r .status $output_file | fgrep CLEAN | wc -l) EXPECTED_CLEAN=3 # AHU-1 has a bad site location, so does not match as clean. echo Found ${clean_devices} clean devices, expecting $EXPECTED_CLEAN. -[ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status=expected_clean +[ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status+=" expected_clean" echo '###################################' end test section expected_clean: $exit_status device_files=$(ls -d ${TEST_SITE}/devices/* | fgrep -v XXX) @@ -33,7 +34,7 @@ generated_config=sites/udmi_site_model/devices/AHU-1/out/generated_config.json [[ "Degrees-Celsius" == $(jq -r .pointset.points.filter_differential_pressure_sensor.units $generated_config) ]] || (echo Units mismatch; touch $error_file) if [[ -f $error_file ]]; then echo Errors found testing generated config. - exit_status=generated_config + exit_status+=" generated_config" fi echo '###################################' end test section generated_config: $exit_status @@ -62,7 +63,7 @@ function check_norm { sm_devices=$(check_norm) echo Devices with correct site_defaults: ${sm_devices} -[ "${sm_devices}" == "${device_files}" ] || exit_status=site_defaults +[ "${sm_devices}" == "${device_files}" ] || exit_status+=" site_defaults" echo '###################################' end test section site_defaults: $exit_status mv ${TEST_SITE}/site_defaults.json ${TEST_SITE}/_site_defaults.json @@ -73,19 +74,19 @@ base_devices=sites/udmi_site_model/devices/AHU-1 sm_devices=$(check_norm) echo Devices with correct no-defaults site_defaults: ${sm_devices} echo Expecting: $base_devices -[ "${sm_devices}" == "${base_devices}" ] || exit_status=no_defaults +[ "${sm_devices}" == "${base_devices}" ] || exit_status+=" no_defaults" echo '###################################' end test section no_defaults: $exit_status config_points=`jq '.pointset.points | length' $TEST_SITE/devices/AHU-22/out/generated_config.json` -[[ $config_points == 3 ]] || exit_status=missing_points +[[ $config_points == 3 ]] || exit_status+=" missing_points" echo '###################################' end test section missing_points: $exit_status config_points=`jq .pointset.points $TEST_SITE/devices/SNS-4/out/generated_config.json` -[[ $config_points == null ]] || exit_status=configured_points +[[ $config_points == null ]] || exit_status+=" configured_points" echo '###################################' end test section configured_points: $exit_status mv ${TEST_SITE}/_site_defaults.json ${TEST_SITE}/site_defaults.json -echo Done with registrar tests, results are \'$exit_status\' \(expected to be \'clean\'\) -[[ $exit_status == clean ]] +[[ -z $exit_status ]] || fail Failed registrar tests: $exit_status +echo Registrar standalone tests passed. diff --git a/bin/test_runlocal b/bin/test_runlocal index f4d5afbc33..a72c5ac8db 100755 --- a/bin/test_runlocal +++ b/bin/test_runlocal @@ -12,13 +12,16 @@ cat out/message_capture.log echo Captured $(wc -l out/message_capture.log) messages. -systems=$(find out/registries/ -name \*update_model.json | xargs jq .system | fgrep -v null | wc -l) -[[ $systems == 196 ]] || fail Expected 196 system operations, found $systems +systems=$(find out/registries/ -name \*update_model.json | xargs jq .system | fgrep -v null | wc -l) || true +[[ $systems == 269 ]] || fail Expected 269 system operations, found $systems -deletes=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep DELETE | wc -l) +deletes=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep DELETE | wc -l) || true [[ $deletes == 1 ]] || fail Expected 1 delete operations, found $deletes -binds=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BIND | wc -l) -[[ $binds == 2 ]] || fail Expected 2 bind operations, found $binds +binds=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BIND | wc -l) || true +[[ $binds == 3 ]] || fail Expected 3 bind operations, found $binds + +blocks=$(find out/registries/ -name \*update_model.json | xargs jq .cloud.operation | fgrep BLOCK | wc -l) || true +[[ $blocks == 1 ]] || fail Expected 1 block operations, found $blocks echo Done with successful runlocal validation. diff --git a/common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java b/common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java new file mode 100644 index 0000000000..cee835895a --- /dev/null +++ b/common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java @@ -0,0 +1,41 @@ +package com.google.daq.mqtt.util; + +import java.util.HashMap; +import java.util.Map; + +/** + * Simple class for a keyed map of exceptions. + */ +public class ErrorMap extends HashMap { + + private final String description; + + public ErrorMap(String description) { + super(); + this.description = description; + } + + /** + * Throw an exception if there's something in the map. + */ + public void throwIfNotEmpty() { + if (!isEmpty()) { + throw asException(); + } + } + + public ErrorMapException asException() { + return new ErrorMapException(); + } + + public class ErrorMapException extends RuntimeException { + + ErrorMapException() { + super(description); + } + + public Map getMap() { + return ErrorMap.this; + } + } +} \ No newline at end of file diff --git a/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java b/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java index 730a4a4586..c1e90fd062 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java +++ b/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java @@ -2,6 +2,7 @@ import static com.google.udmi.util.GeneralUtils.multiTrim; +import com.google.daq.mqtt.util.ErrorMap.ErrorMapException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -25,12 +26,18 @@ public class ExceptionMap extends RuntimeException { private static final byte[] SEPARATOR_BYTES = ": ".getBytes(); private static final String ERROR_FORMAT_INDENT = " "; - final Map exceptions = new TreeMap<>(); + final Map exceptions = new TreeMap<>(); public ExceptionMap(String description) { super(description); } + public enum ExceptionCategory { + missing, extra, out, validation, loading, writing, site_metadata, initializing, sample, + registering, envelope, credentials, samples, files, binding, creating, updating, schema, + configuring, status + } + /** * Format the given exception with indicated level. * @@ -46,12 +53,15 @@ private static ErrorTree format(Throwable e, final String prefix, final String i errorTree.prefix = prefix; errorTree.message = multiTrim(e.getMessage()); final String newPrefix = prefix + indent; - if (e instanceof ExceptionMap) { + if (e instanceof ExceptionMap exceptionMap) { if (e.getCause() != null) { errorTree.child = format(e.getCause(), newPrefix, indent); } - ((ExceptionMap) e) - .forEach((key, sub) -> errorTree.children.put(key, format(sub, newPrefix, indent))); + exceptionMap.forEach((key, sub) -> + errorTree.children.put(key.toString(), format(sub, newPrefix, indent))); + } else if (e instanceof ErrorMapException errorMap) { + errorMap.getMap().forEach((key, sub) -> + errorTree.children.put(key, format(sub, newPrefix, indent))); } else if (e instanceof ValidationException) { ((ValidationException) e) .getCausingExceptions() @@ -69,7 +79,7 @@ private static ErrorTree format(Throwable e, final String prefix, final String i return errorTree; } - private void forEach(BiConsumer consumer) { + private void forEach(BiConsumer consumer) { exceptions.forEach(consumer); } @@ -96,13 +106,13 @@ public void throwIfNotEmpty() { * @param key entry key * @param exception exception to add */ - public void put(String key, Exception exception) { + public void put(ExceptionCategory key, Exception exception) { if (exceptions.put(key, exception) != null) { throw new IllegalArgumentException("Exception key already defined: " + key); } } - public Stream> stream() { + public Stream> stream() { return exceptions.entrySet().stream(); } @@ -118,7 +128,7 @@ public int size() { /** * Execute the action and capture into the map if it throws an exception. */ - public void capture(String category, Runnable action) { + public void capture(ExceptionCategory category, Runnable action) { try { action.run(); } catch (Exception e) { diff --git a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java index a5d36e4982..714cb33294 100644 --- a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java +++ b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java @@ -3,9 +3,9 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static java.lang.String.CASE_INSENSITIVE_ORDER; import static java.util.Objects.nonNull; +import com.google.common.base.Strings; import java.lang.reflect.Method; import java.lang.reflect.Type; import java.util.ArrayList; @@ -31,11 +31,11 @@ public class CommandLineProcessor { private List usageForms; Map optionMap = new TreeMap<>( - (a, b) -> CASE_INSENSITIVE_ORDER.compare(getSortArg(a), getSortArg(b))); + (a, b) -> getSortArg(a).compareTo(getSortArg(b))); private static String getSortArg(CommandLineOption k) { String shortForm = k.short_form(); - return shortForm.isBlank() ? k.long_form().substring(1, 2) : shortForm.substring(1, 2); + return shortForm.isBlank() ? k.long_form() : shortForm; } /** @@ -54,20 +54,21 @@ public CommandLineProcessor(Object target) { Set options = optionMap.keySet(); List>> duplicateShort = options.stream() - .filter(x -> nonNull(x.short_form())) + .filter(x -> !Strings.isNullOrEmpty(x.short_form())) .collect(Collectors.groupingBy(CommandLineOption::short_form)) .entrySet().stream() .filter(e -> e.getValue().size() > 1) .toList(); - checkState(duplicateShort.isEmpty(), "duplicate short form command line option"); + checkState(duplicateShort.isEmpty(), + "duplicate short form command line option: " + duplicateShort); List>> duplicateLong = options.stream() - .filter(x -> nonNull(x.short_form())) - .collect(Collectors.groupingBy(CommandLineOption::short_form)) + .filter(x -> !Strings.isNullOrEmpty(x.long_form())) + .collect(Collectors.groupingBy(CommandLineOption::long_form)) .entrySet().stream() .filter(e -> e.getValue().size() > 1) .toList(); - checkState(duplicateLong.isEmpty(), "duplicate short form command line option"); + checkState(duplicateLong.isEmpty(), "duplicate long form command line option " + duplicateLong); } public CommandLineProcessor(Object target, List usageForms) { @@ -156,8 +157,9 @@ private boolean processArgEntry(String arg, CommandLineOption option, Method met } else if (option.short_form().isBlank() && option.long_form().isBlank()) { throw new IllegalArgumentException( "Neither long nor short form not defined for " + method.getName()); - } else + } else { return !arg.startsWith("-"); + } } catch (Exception e) { throw new IllegalArgumentException("Processing command line method " + method.getName(), e); } diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java index 8263702ceb..421192122d 100644 --- a/common/src/main/java/com/google/udmi/util/Common.java +++ b/common/src/main/java/com/google/udmi/util/Common.java @@ -8,6 +8,7 @@ import com.google.common.base.Strings; import com.google.common.reflect.ClassPath; import com.google.common.reflect.ClassPath.ClassInfo; +import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; import java.util.ArrayList; import java.util.List; import java.util.MissingFormatArgumentException; @@ -36,7 +37,7 @@ public abstract class Common { public static final String UPGRADED_FROM = "upgraded_from"; public static final String DOWNGRADED_FROM = "downgraded_from"; public static final String CLOUD_VERSION_KEY = "cloud_version"; - public static final String SITE_METADATA_KEY = "site_metadata"; + public static final String SITE_METADATA_KEY = ExceptionCategory.site_metadata.toString(); public static final String UDMI_VERSION_KEY = "udmi_version"; public static final String UDMI_VERSION_ENV = "UDMI_VERSION"; public static final String UDMI_COMMIT_ENV = "UDMI_COMMIT"; @@ -49,6 +50,7 @@ public abstract class Common { public static final String REGISTRY_ID_PROPERTY_KEY = "deviceRegistryId"; public static final String DEFAULT_REGION = "us-central1"; public static final String DEVICE_ID_KEY = "deviceId"; + public static final String DEVICE_NUM_KEY = "deviceNumId"; public static final String GATEWAY_ID_KEY = "gatewayId"; public static final String SOURCE_KEY = "source"; public static final String NO_SITE = "--"; diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 5c860645c3..8ead465d55 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -298,18 +298,18 @@ private static JsonGenerator getPrettyPrinterGenerator(OutputStream outputStream } } + @SuppressWarnings("unchecked") public static Map getSubMap(Map input, String field) { - //noinspection unchecked return (Map) input.get(field); } + @SuppressWarnings("unchecked") public static Map getSubMapNull(Map input, String field) { - //noinspection unchecked return ifNotNullGet(input, map -> (Map) map.get(field)); } + @SuppressWarnings("unchecked") public static Map getSubMapDefault(Map input, String field) { - //noinspection unchecked return ifNotNullGet(input, map -> (Map) map.computeIfAbsent(field, k -> new HashMap<>())); } diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 155a3ffb83..aab7994c7e 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -4,7 +4,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.udmi.util.Common.DEFAULT_REGION; import static com.google.udmi.util.Common.NO_SITE; -import static com.google.udmi.util.Common.SITE_METADATA_KEY; import static com.google.udmi.util.Common.UDMI_COMMIT_ENV; import static com.google.udmi.util.Common.UDMI_REF_ENV; import static com.google.udmi.util.Common.UDMI_TIMEVER_ENV; @@ -36,6 +35,7 @@ import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.google.common.collect.ImmutableList; import com.google.daq.mqtt.util.ExceptionMap; +import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; import java.io.File; import java.util.Arrays; import java.util.Collection; @@ -338,12 +338,12 @@ public Set getDeviceIds() { public SiteMetadata loadSiteMetadata() { ObjectNode siteMetadataObject = null; File siteMetadataFile = new File(new File(sitePath), SITE_METADATA_FILE); - siteMetadataExceptionMap = new ExceptionMap(SITE_METADATA_KEY); + siteMetadataExceptionMap = new ExceptionMap("Site metadata validation"); try { siteMetadataObject = loadFileRequired(ObjectNode.class, siteMetadataFile); return convertToStrict(SiteMetadata.class, siteMetadataObject); } catch (Exception e) { - siteMetadataExceptionMap.put(SITE_METADATA_KEY, e); + siteMetadataExceptionMap.put(ExceptionCategory.site_metadata, e); return convertTo(SiteMetadata.class, siteMetadataObject); } } @@ -605,6 +605,15 @@ public String getSiteName() { return exeConfig.site_name; } + public void resetRegistryId(String altRegistry) { + String previousId = getRegistryId(); + getExecutionConfiguration().registry_id = altRegistry; + getExecutionConfiguration().alt_registry = null; + String newId = getRegistryId(); + checkState(!newId.equals(previousId), "resetting to unchanged registry id"); + System.err.printf("Switched target registry from %s to %s%n", previousId, newId); + } + public static class MetadataException extends Metadata { public final File file; diff --git a/etc/Category.java b/etc/Category.java index 1f2e4ef28e..356215a512 100644 --- a/etc/Category.java +++ b/etc/Category.java @@ -12,7 +12,7 @@ // This class is manually curated, auto-generated, and then copied into the gencode directory. // Look for the proper source and don't be fooled! Ultimately sourced from docs/specs/categories.md public class Category { - public static final Map LEVEL = new HashMap(); + public static final Map LEVEL = new HashMap<>(); @@ gencode stuff goes here } diff --git a/gencode/java/udmi/schema/Category.java b/gencode/java/udmi/schema/Category.java index ee349f6394..dd0c5c81ba 100644 --- a/gencode/java/udmi/schema/Category.java +++ b/gencode/java/udmi/schema/Category.java @@ -12,7 +12,7 @@ // This class is manually curated, auto-generated, and then copied into the gencode directory. // Look for the proper source and don't be fooled! Ultimately sourced from docs/specs/categories.md public class Category { - public static final Map LEVEL = new HashMap(); + public static final Map LEVEL = new HashMap<>(); // System is in the process of (re)starting and essentially offline diff --git a/tests/sites/basic/devices/AHU-1/expected/errors.map b/tests/sites/basic/devices/AHU-1/expected/errors.map index 4d9ffdddb2..01b0becfe0 100644 --- a/tests/sites/basic/devices/AHU-1/expected/errors.map +++ b/tests/sites/basic/devices/AHU-1/expected/errors.map @@ -1,7 +1,7 @@ Exceptions for AHU-1 While loading credentials for local device AHU-1 Found 0 credentials + expected files mismatch + Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] 1 schema violations found /system/location/site: ECMA 262 regex "^[A-Z]{2}-[A-Z]{3,4}-[A-Z0-9]{2,9}$" does not match input string "ZZ-TRI-FECTA-AHU" - expected files - Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/basic/devices/GAT-123/expected/errors.map b/tests/sites/basic/devices/GAT-123/expected/errors.map index 6a82432a88..557025243d 100644 --- a/tests/sites/basic/devices/GAT-123/expected/errors.map +++ b/tests/sites/basic/devices/GAT-123/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for GAT-123 While loading credentials for local device GAT-123 Found 0 credentials - expected files + expected files mismatch Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] diff --git a/tests/sites/basic/devices/GAT-72/expected/errors.map b/tests/sites/basic/devices/GAT-72/expected/errors.map index 385057b082..961828110b 100644 --- a/tests/sites/basic/devices/GAT-72/expected/errors.map +++ b/tests/sites/basic/devices/GAT-72/expected/errors.map @@ -1,7 +1,7 @@ Exceptions for GAT-72 While loading credentials for local device GAT-72 Found 0 credentials - expected files + expected files mismatch Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] While converting device config generation specified without scan_interval_sec parameter diff --git a/tests/sites/configs/devices/AHU-1/expected/errors.map b/tests/sites/configs/devices/AHU-1/expected/errors.map index 53a887d7e8..7d40b38476 100644 --- a/tests/sites/configs/devices/AHU-1/expected/errors.map +++ b/tests/sites/configs/devices/AHU-1/expected/errors.map @@ -1,11 +1,11 @@ Exceptions for AHU-1 While loading credentials for local device AHU-1 Found 0 credentials - 1 schema violations found - /system/location/site: ECMA 262 regex "^[A-Z]{2}-[A-Z]{3,4}-[A-Z0-9]{2,9}$" does not match input string "ZZ-TRI-FECTA-AHU" - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] While converting device config While reading config file: non-existant.file While lREDACTED/sites/configs/devices/AHU-1/config/non-existant.file tests/sites/configs/devices/AHU-1/config/non-existant.file + 1 schema violations found + /system/location/site: ECMA 262 regex "^[A-Z]{2}-[A-Z]{3,4}-[A-Z0-9]{2,9}$" does not match input string "ZZ-TRI-FECTA-AHU" diff --git a/tests/sites/configs/devices/GAT-123/expected/errors.map b/tests/sites/configs/devices/GAT-123/expected/errors.map index 7f8c393404..27930826ad 100644 --- a/tests/sites/configs/devices/GAT-123/expected/errors.map +++ b/tests/sites/configs/devices/GAT-123/expected/errors.map @@ -1,7 +1,7 @@ Exceptions for GAT-123 While loading credentials for local device GAT-123 Found 0 credentials + expected files mismatch + Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] 1 schema violations found /system/location: object has missing required properties (["site"]) - expected files - Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] diff --git a/tests/sites/configs/devices/GAT-456/expected/errors.map b/tests/sites/configs/devices/GAT-456/expected/errors.map index 5ac0845354..0745665a00 100644 --- a/tests/sites/configs/devices/GAT-456/expected/errors.map +++ b/tests/sites/configs/devices/GAT-456/expected/errors.map @@ -1,7 +1,7 @@ Exceptions for GAT-456 While loading credentials for local device GAT-456 Found 0 credentials + expected files mismatch + Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] 1 schema violations found /system/location: object has missing required properties (["site"]) - expected files - Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] diff --git a/tests/sites/discovery/devices/GAT-123/expected/errors.map b/tests/sites/discovery/devices/GAT-123/expected/errors.map index 6a82432a88..557025243d 100644 --- a/tests/sites/discovery/devices/GAT-123/expected/errors.map +++ b/tests/sites/discovery/devices/GAT-123/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for GAT-123 While loading credentials for local device GAT-123 Found 0 credentials - expected files + expected files mismatch Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] diff --git a/tests/sites/downgrade/devices/CGW-1/expected/errors.map b/tests/sites/downgrade/devices/CGW-1/expected/errors.map index 6e8c83561c..9b0f5c8d66 100644 --- a/tests/sites/downgrade/devices/CGW-1/expected/errors.map +++ b/tests/sites/downgrade/devices/CGW-1/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for CGW-1 While loading credentials for local device CGW-1 Found 0 credentials - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/downgrade/devices/CGW-12/expected/errors.map b/tests/sites/downgrade/devices/CGW-12/expected/errors.map index d9c2a1ae55..dd2f123278 100644 --- a/tests/sites/downgrade/devices/CGW-12/expected/errors.map +++ b/tests/sites/downgrade/devices/CGW-12/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for CGW-12 While loading credentials for local device CGW-12 Found 0 credentials - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/downgrade/devices/DDC-54/expected/errors.map b/tests/sites/downgrade/devices/DDC-54/expected/errors.map index 2aa6c9872a..196a2115f4 100644 --- a/tests/sites/downgrade/devices/DDC-54/expected/errors.map +++ b/tests/sites/downgrade/devices/DDC-54/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for DDC-54 While loading credentials for local device DDC-54 Found 0 credentials - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/downgrade/devices/DWN-1/expected/errors.map b/tests/sites/downgrade/devices/DWN-1/expected/errors.map index 2152d3968c..4fcf0e333f 100644 --- a/tests/sites/downgrade/devices/DWN-1/expected/errors.map +++ b/tests/sites/downgrade/devices/DWN-1/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for DWN-1 While loading credentials for local device DWN-1 Found 0 credentials - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/downgrade/devices/DWN-3/expected/errors.map b/tests/sites/downgrade/devices/DWN-3/expected/errors.map index 7b284b3c3c..2ea553526f 100644 --- a/tests/sites/downgrade/devices/DWN-3/expected/errors.map +++ b/tests/sites/downgrade/devices/DWN-3/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for DWN-3 While loading credentials for local device DWN-3 Found 0 credentials - expected files + expected files mismatch Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/mapping/devices/GAT-123/expected/errors.map b/tests/sites/mapping/devices/GAT-123/expected/errors.map index 6cfd072b3b..d1a594b8ef 100644 --- a/tests/sites/mapping/devices/GAT-123/expected/errors.map +++ b/tests/sites/mapping/devices/GAT-123/expected/errors.map @@ -1,6 +1,6 @@ Exceptions for GAT-123 While loading credentials for local device GAT-123 Credential cloud.auth_type definition missing + Invalid auth type null 1 schema violations found object has missing required properties (["system","version"]) - Invalid auth type null diff --git a/tests/sites/missing/devices/AHU-1/expected/errors.map b/tests/sites/missing/devices/AHU-1/expected/errors.map index 4d9ffdddb2..01b0becfe0 100644 --- a/tests/sites/missing/devices/AHU-1/expected/errors.map +++ b/tests/sites/missing/devices/AHU-1/expected/errors.map @@ -1,7 +1,7 @@ Exceptions for AHU-1 While loading credentials for local device AHU-1 Found 0 credentials + expected files mismatch + Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] 1 schema violations found /system/location/site: ECMA 262 regex "^[A-Z]{2}-[A-Z]{3,4}-[A-Z0-9]{2,9}$" does not match input string "ZZ-TRI-FECTA-AHU" - expected files - Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/missing/devices/AHU-22/expected/errors.map b/tests/sites/missing/devices/AHU-22/expected/errors.map index 6c9ae65ca3..00f0761819 100644 --- a/tests/sites/missing/devices/AHU-22/expected/errors.map +++ b/tests/sites/missing/devices/AHU-22/expected/errors.map @@ -1,10 +1,11 @@ Exceptions for AHU-22 - While converting device config - While converting point filter_differential_pressure_sensor - protocol ref FAXLE12.present_value does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? - While converting device config - While converting point filter_alarm_pressure_status - protocol ref MCN8.present_value does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? - While converting device config - While converting point filter_differential_pressure - protocol ref AI2.differential does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? + schema validation errors + While converting device config + While converting point filter_differential_pressure_sensor + protocol ref FAXLE12.present_value does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? + While converting device config + While converting point filter_alarm_pressure_status + protocol ref MCN8.present_value does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? + While converting device config + While converting point filter_differential_pressure + protocol ref AI2.differential does not match expression bacnet://(0|[1-9][0-9]*)/([A-Z]{2,4}):(0|[1-9][0-9]*)(#[_a-z]+)? diff --git a/tests/sites/missing/devices/GAT-123/expected/errors.map b/tests/sites/missing/devices/GAT-123/expected/errors.map index 6a82432a88..557025243d 100644 --- a/tests/sites/missing/devices/GAT-123/expected/errors.map +++ b/tests/sites/missing/devices/GAT-123/expected/errors.map @@ -1,5 +1,5 @@ Exceptions for GAT-123 While loading credentials for local device GAT-123 Found 0 credentials - expected files + expected files mismatch Missing files: [ec_private.pem, ec_private.pkcs8, ec_public.pem] diff --git a/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java b/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java index e6538ee2aa..5ecbac323e 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java @@ -252,7 +252,7 @@ private CloudModel bindDevicesToGateway(String registryId, String gatewayId, boolean toBind = cloudModel.operation == BIND; bindDevicesGateways(registryId, ImmutableSet.of(gatewayId), deviceIds, toBind, progress); CloudModel reply = new CloudModel(); - reply.num_id = deviceIds.size() > 0 ? EMPTY_RETURN_RECEIPT : null; + reply.num_id = EMPTY_RETURN_RECEIPT; reply.operation = cloudModel.operation; return reply; } diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java index e9a720a262..23e3a3b643 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java @@ -284,8 +284,13 @@ private CloudModel queryDeviceState(Envelope attributes) { } private CloudModel reflectModel(Envelope attributes, CloudModel request) { - ifNotNullThen(extractModel(request), model -> publish(makeTargetEnvelope(attributes), model)); + CloudModel modelResult = updateModel(attributes, request); + ifNotNullThen(extractModel(request), + model -> publish(makeTargetEnvelope(attributes), model)); + return modelResult; + } + private CloudModel updateModel(Envelope attributes, CloudModel request) { if (request.operation == PREVIEW) { // Nothing more to do in this case since this is just preview, meaning no impact. return previewReflectResponse(attributes, request); diff --git a/validator/.idea/compiler.xml b/validator/.idea/compiler.xml index b589d56e9f..c72a0b5020 100644 --- a/validator/.idea/compiler.xml +++ b/validator/.idea/compiler.xml @@ -3,4 +3,7 @@ + + \ No newline at end of file diff --git a/validator/.idea/runConfigurations/Registrar_downgrade_site.xml b/validator/.idea/runConfigurations/Registrar_missing.xml similarity index 72% rename from validator/.idea/runConfigurations/Registrar_downgrade_site.xml rename to validator/.idea/runConfigurations/Registrar_missing.xml index 1feb983ea0..ca5079b3be 100644 --- a/validator/.idea/runConfigurations/Registrar_downgrade_site.xml +++ b/validator/.idea/runConfigurations/Registrar_missing.xml @@ -1,8 +1,8 @@ - +