From 3c73593893018bd4eabccfc2c1a00d99ada2a4f5 Mon Sep 17 00:00:00 2001 From: Khyati Mahendru Date: Thu, 23 Jan 2025 05:39:53 +0000 Subject: [PATCH 1/2] Update registrar usage forms in its help function & update docs (#1059) --- .../udmi/util/CommandLineProcessor.java | 12 +++++++++ docs/tools/registrar.md | 27 +++++++++++++------ .../google/daq/mqtt/registrar/Registrar.java | 7 ++++- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java index 621f0a8980..a5d36e4982 100644 --- a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java +++ b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java @@ -28,6 +28,7 @@ public class CommandLineProcessor { private static final int LINUX_ERROR_CODE = -1; private final Object target; private final Method showHelpMethod; + private List usageForms; Map optionMap = new TreeMap<>( (a, b) -> CASE_INSENSITIVE_ORDER.compare(getSortArg(a), getSortArg(b))); @@ -69,6 +70,13 @@ public CommandLineProcessor(Object target) { checkState(duplicateLong.isEmpty(), "duplicate short form command line option"); } + public CommandLineProcessor(Object target, List usageForms) { + this(target); + if (usageForms != null && !usageForms.isEmpty()) { + this.usageForms = usageForms; + } + } + private Method getShowHelpMethod() { try { return CommandLineProcessor.class.getDeclaredMethod("showUsage"); @@ -91,6 +99,10 @@ private void showUsage() { */ public void showUsage(String message) { ifNotNullThen(message, m -> System.err.println(m)); + ifNotNullThen(usageForms, () -> { + System.err.println("Usage forms:"); + usageForms.forEach(form -> System.err.printf(" %s%n", form)); + }); System.err.println("Options supported:"); optionMap.forEach((option, method) -> System.err.printf(" %s %12s %s%n", option.short_form(), option.arg_name(), option.description())); diff --git a/docs/tools/registrar.md b/docs/tools/registrar.md index 51e412e0c4..1da2cc44c9 100644 --- a/docs/tools/registrar.md +++ b/docs/tools/registrar.md @@ -39,22 +39,33 @@ be used to specific specific device(s) to register (rather than all). ``` Usage: -bin/registrar site_path [project_id] [options] [devices...] +bin/registrar site_model project_spec [options] [devices...] -bin/registrar config_file +bin/registrar site_spec [options] [devices...] ``` -* `config_file`: Path to a configuration file which contains configuration options; -* `site_path`: The _directory_ containing the site model, or a model-with-project _file_ directly. -* `project_id`: The project ID that contains the target registry. The project ID can be prepended with iot_provider: +* `site_model`: The path to the _directory_ containing the site model, or a model-with-project _file_ directly. +* `project_spec`: The project ID that contains the target registry. The project ID can be prepended with iot_provider: * `//clearblade/PROJECT_ID` for a public ClearBlade project. * `//gbos/PROJECT_ID` for a Google operated ClearBlade project. +* `site_spec`: Path to a configuration file which contains configuration options; * `options`: Various options to impact behavior: - * `-u` Update. - * `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry) + * `-a` Set alternate registry * `-b` Block unknown devices. - * `-x` Delete unknown devices from the registry. + * `-c` Count of registries to be created + * `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry) + * `-e` Set registry suffix + * `-f` Set PubSub feed topic + * `-h` Show help and exit + * `-l` Set idle limit + * `-m` Initial metadata model out * `-n` Number of thread counts. + * `-p` Set Project ID + * `-q` Query only, registry to not be updated + * `-r` Set tool root path + * `-s` Set site path + * `-t` Do not validate metadata + * `-x` Delete unknown devices from the registry. * `devices`: Multiple device entries for limited registration. Can be just the device name (`AHU-12`), or path to device (`site/devices/AHU-12`) for use with file-name glob. diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index aec186b366..0c32cd73f6 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -45,6 +45,7 @@ import com.github.fge.jsonschema.main.JsonSchema; import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets.SetView; @@ -131,7 +132,11 @@ public class Registrar { private final Map schemas = new HashMap<>(); private final String generation = JsonUtil.isoConvert(); private final Set summarizers = new HashSet<>(); - private final CommandLineProcessor commandLineProcessor = new CommandLineProcessor(this); + private final List usageForms = ImmutableList.of( + "bin/registrar site_model project_spec [options] [devices...]", + "bin/registrar site_spec [options] [devices...]"); + private final CommandLineProcessor commandLineProcessor = new CommandLineProcessor(this, + usageForms); private CloudIotManager cloudIotManager; private File schemaBase; private PubSubPusher updatePusher; From 0c8962b97f5804ef1187ddd72ad2e974c75c8d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Cing=C3=B6z?= Date: Fri, 24 Jan 2025 11:51:41 +0100 Subject: [PATCH 2/2] Cleanup Client Implementation and Java Library (#1074) --- .github/workflows/integration.yaml | 33 +- .../com/google/udmi/util/CertManager.java | 40 +- etc/validator.out | 30 +- pubber/.idea/runConfigurations/Swarm.xml | 17 - pubber/bin/publish | 24 + pubber/build.gradle | 22 + pubber/src/main/java/daq/pubber/Pubber.java | 701 +----------------- .../daq/pubber/PubberDiscoveryManager.java | 172 ----- .../java/daq/pubber/PubberGatewayManager.java | 148 ---- .../main/java/daq/pubber/PubberManager.java | 31 - .../daq/pubber/PubberPointsetManager.java | 214 ------ .../java/daq/pubber/PubberProxyDevice.java | 90 --- .../java/daq/pubber/PubberPubSubClient.java | 92 --- .../java/daq/pubber/PubberSystemManager.java | 238 ------ .../daq/pubber/{ => impl}/PubberFeatures.java | 8 +- .../java/daq/pubber/impl/PubberManager.java | 223 ++++++ .../daq/pubber/impl/host/PubberProxyHost.java | 61 ++ .../pubber/impl/host/PubberPublisherHost.java | 484 ++++++++++++ .../manager}/PubberDeviceManager.java | 58 +- .../impl/manager/PubberDiscoveryManager.java | 82 ++ .../impl/manager/PubberGatewayManager.java | 99 +++ .../manager}/PubberLocalnetManager.java | 72 +- .../impl/manager/PubberPointsetManager.java | 130 ++++ .../impl/manager/PubberSystemManager.java | 102 +++ .../{ => impl/point}/PubberRandomBoolean.java | 2 +- .../{ => impl/point}/PubberRandomPoint.java | 6 +- .../provider}/PubberBacnetProvider.java | 4 +- .../provider}/PubberFamilyProvider.java | 2 +- .../{ => impl/provider}/PubberIpProvider.java | 33 +- .../provider}/PubberVendorProvider.java | 31 +- .../main/java/udmi/lib/base/BasicPoint.java | 8 +- .../java/udmi/lib/base/ListPublisher.java | 9 +- .../main/java/udmi/lib/base/ManagerBase.java | 24 +- .../main/java/udmi/lib/base/MqttDevice.java | 6 +- .../java/udmi/lib/base/MqttPublisher.java | 125 ++-- .../java/udmi/lib/client/GatewayManager.java | 134 ---- .../ProxyHost.java} | 52 +- .../lib/client/host/PublisherHost.java} | 623 +++++++++++----- .../client/{ => manager}/DeviceManager.java | 41 +- .../{ => manager}/DiscoveryManager.java | 124 +++- .../lib/client/manager/GatewayManager.java | 198 +++++ .../client/{ => manager}/LocalnetManager.java | 9 +- .../client/{ => manager}/PointsetManager.java | 150 +++- .../client/{ => manager}/SystemManager.java | 250 +++++-- .../main/java/udmi/lib/intf/ManagerHost.java | 7 +- .../lib/{client => intf}/SubBlockManager.java | 5 +- .../java/udmi/lib/intf/UdmiPublisher.java | 18 - .../CatchingScheduledThreadPoolExecutor.java | 12 +- .../test/java/daq/pubber/IpManagerTest.java | 1 + .../src/test/java/daq/pubber/PubberTest.java | 35 +- .../daq/pubber/SupportedFeaturesTest.java | 1 + .../lib}/ListPublisherTest.java | 4 +- .../pubber => udmi/lib}/MqttDeviceTest.java | 4 +- .../lib/{base => }/MqttPublisherTest.java | 6 +- .../{daq/pubber => udmi/lib}/TestBase.java | 2 +- 55 files changed, 2712 insertions(+), 2385 deletions(-) delete mode 100644 pubber/.idea/runConfigurations/Swarm.xml create mode 100755 pubber/bin/publish delete mode 100644 pubber/src/main/java/daq/pubber/PubberDiscoveryManager.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberGatewayManager.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberManager.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberPointsetManager.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberProxyDevice.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberPubSubClient.java delete mode 100644 pubber/src/main/java/daq/pubber/PubberSystemManager.java rename pubber/src/main/java/daq/pubber/{ => impl}/PubberFeatures.java (89%) create mode 100644 pubber/src/main/java/daq/pubber/impl/PubberManager.java create mode 100644 pubber/src/main/java/daq/pubber/impl/host/PubberProxyHost.java create mode 100644 pubber/src/main/java/daq/pubber/impl/host/PubberPublisherHost.java rename pubber/src/main/java/daq/pubber/{ => impl/manager}/PubberDeviceManager.java (71%) create mode 100644 pubber/src/main/java/daq/pubber/impl/manager/PubberDiscoveryManager.java create mode 100644 pubber/src/main/java/daq/pubber/impl/manager/PubberGatewayManager.java rename pubber/src/main/java/daq/pubber/{ => impl/manager}/PubberLocalnetManager.java (79%) create mode 100644 pubber/src/main/java/daq/pubber/impl/manager/PubberPointsetManager.java create mode 100644 pubber/src/main/java/daq/pubber/impl/manager/PubberSystemManager.java rename pubber/src/main/java/daq/pubber/{ => impl/point}/PubberRandomBoolean.java (96%) rename pubber/src/main/java/daq/pubber/{ => impl/point}/PubberRandomPoint.java (92%) rename pubber/src/main/java/daq/pubber/{ => impl/provider}/PubberBacnetProvider.java (98%) rename pubber/src/main/java/daq/pubber/{ => impl/provider}/PubberFamilyProvider.java (88%) rename pubber/src/main/java/daq/pubber/{ => impl/provider}/PubberIpProvider.java (83%) rename pubber/src/main/java/daq/pubber/{ => impl/provider}/PubberVendorProvider.java (74%) delete mode 100644 pubber/src/main/java/udmi/lib/client/GatewayManager.java rename pubber/src/main/java/udmi/lib/client/{ProxyDeviceHost.java => host/ProxyHost.java} (67%) rename pubber/src/main/java/{daq/pubber/PubberUdmiPublisher.java => udmi/lib/client/host/PublisherHost.java} (69%) rename pubber/src/main/java/udmi/lib/client/{ => manager}/DeviceManager.java (83%) rename pubber/src/main/java/udmi/lib/client/{ => manager}/DiscoveryManager.java (60%) create mode 100644 pubber/src/main/java/udmi/lib/client/manager/GatewayManager.java rename pubber/src/main/java/udmi/lib/client/{ => manager}/LocalnetManager.java (91%) rename pubber/src/main/java/udmi/lib/client/{ => manager}/PointsetManager.java (50%) rename pubber/src/main/java/udmi/lib/client/{ => manager}/SystemManager.java (50%) rename pubber/src/main/java/udmi/lib/{client => intf}/SubBlockManager.java (84%) delete mode 100644 pubber/src/main/java/udmi/lib/intf/UdmiPublisher.java rename pubber/src/test/java/{daq/pubber => udmi/lib}/ListPublisherTest.java (97%) rename pubber/src/test/java/{daq/pubber => udmi/lib}/MqttDeviceTest.java (98%) rename pubber/src/test/java/udmi/lib/{base => }/MqttPublisherTest.java (97%) rename pubber/src/test/java/{daq/pubber => udmi/lib}/TestBase.java (98%) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3a822da574..a290797039 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -22,7 +22,6 @@ jobs: container: [ "udmis", "validator", "pubber", "misc" ] env: PUSH_REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} CONTAINER: ${{ matrix.container }} REF_NAME: ${{ github.ref_name }} steps: @@ -40,11 +39,13 @@ jobs: registry: ${{ env.PUSH_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Convert repository name to lowercase + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Container build and push run: | revhash=$(git log -n 1 --pretty=format:"%h") IMAGE_HASH=g${revhash:0:8} - PUSH_REPO=$PUSH_REGISTRY/${IMAGE_NAME,,} + PUSH_REPO=$PUSH_REGISTRY/$IMAGE_NAME TAG_BASE=$PUSH_REPO:$CONTAINER PUSH_TAG=${TAG_BASE}-$IMAGE_HASH @@ -66,6 +67,28 @@ jobs: echo docker history $PUSH_TAG + maven: + name: Publish maven package (pubber) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + timeout-minutes: 5 + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + - name: Publish maven package + run: pubber/bin/publish + simple: name: Simple sequence test runs-on: ubuntu-latest @@ -76,7 +99,6 @@ jobs: matrix: device_id: [ "AHU-1", "AHU-22", "GAT-123" ] env: - IMAGE_NAME: ${{ github.repository }} REF_NAME: ${{ github.ref_name }} DEVICE_ID: ${{ matrix.device_id }} steps: @@ -89,6 +111,8 @@ jobs: jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json jq . site_model/cloud_iot_config.json docker network create udminet --subnet 192.168.99.0/24 + - name: Convert repository name to lowercase + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Start UDMIS container run: | export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME @@ -148,7 +172,6 @@ jobs: timeout-minutes: 5 needs: images env: - IMAGE_NAME: ${{ github.repository }} REF_NAME: ${{ github.ref_name }} DEVICE_ID: ${{'AHU-1'}} steps: @@ -161,6 +184,8 @@ jobs: jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json jq . site_model/cloud_iot_config.json docker network create udminet --subnet 192.168.99.0/24 + - name: Convert repository name to lowercase + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Start UDMIS container run: | export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME diff --git a/common/src/main/java/com/google/udmi/util/CertManager.java b/common/src/main/java/com/google/udmi/util/CertManager.java index 7f10d440f5..afb34a93e9 100644 --- a/common/src/main/java/com/google/udmi/util/CertManager.java +++ b/common/src/main/java/com/google/udmi/util/CertManager.java @@ -5,9 +5,13 @@ import static java.lang.String.format; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; import java.security.KeyStore; import java.security.PrivateKey; import java.security.Security; @@ -49,6 +53,9 @@ public class CertManager { private final File crtFile; private final char[] password; private final Transport transport; + private final String caCertificate; + private final String clientCertificate; + private final String clientPrivateKey; { Security.addProvider(new BouncyCastleProvider()); @@ -61,6 +68,9 @@ public CertManager(File caCrtFile, File clientDir, Transport transport, String passString, Consumer logging) { this.caCrtFile = caCrtFile; this.transport = transport; + this.caCertificate = null; + this.clientCertificate = null; + this.clientPrivateKey = null; if (Transport.SSL.equals(transport)) { String prefix = keyPrefix(clientDir); @@ -78,6 +88,18 @@ public CertManager(File caCrtFile, File clientDir, Transport transport, } } + public CertManager(String caCertificate, String clientCertificate, String clientPrivateKey, + Transport transport, String passString) { + caCrtFile = null; + crtFile = null; + keyFile = null; + this.transport = transport; + this.password = passString.toCharArray(); + this.caCertificate = caCertificate; + this.clientCertificate = clientCertificate; + this.clientPrivateKey = clientPrivateKey; + } + private String keyPrefix(File clientDir) { File rsaCrtFile = new File(clientDir, "rsa_private.crt"); File ecCrtFile = new File(clientDir, "ec_private.crt"); @@ -92,18 +114,27 @@ private String keyPrefix(File clientDir) { public SSLSocketFactory getCertSocketFactory() throws Exception { CertificateFactory certFactory = CertificateFactory.getInstance(X509_FACTORY); + InputStream caCertStream = caCrtFile != null + ? new FileInputStream(caCrtFile) + : new ByteArrayInputStream(caCertificate.getBytes()); final X509Certificate caCert; - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(caCrtFile))) { + try (BufferedInputStream bis = new BufferedInputStream(caCertStream)) { caCert = (X509Certificate) certFactory.generateCertificate(bis); } + InputStream clientCertStream = crtFile != null + ? new FileInputStream(crtFile) + : new ByteArrayInputStream(clientCertificate.getBytes()); final X509Certificate clientCert; - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(crtFile))) { + try (BufferedInputStream bis = new BufferedInputStream(clientCertStream)) { clientCert = (X509Certificate) certFactory.generateCertificate(bis); } + Reader keyReader = keyFile != null + ? new FileReader(keyFile) + : new StringReader(clientPrivateKey); final PrivateKey privateKey; - try (PEMParser pemParser = new PEMParser(new FileReader(keyFile))) { + try (PEMParser pemParser = new PEMParser(keyReader)) { Object pemObject = pemParser.readObject(); if (pemObject instanceof PEMEncryptedKeyPair keyPair) { PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password); @@ -114,7 +145,8 @@ public SSLSocketFactory getCertSocketFactory() throws Exception { privateKey = converter.getPrivateKey(keyPair); } else { throw new RuntimeException(format("Unknown pem file type %s from %s", - pemObject.getClass().getSimpleName(), keyFile.getAbsolutePath())); + pemObject.getClass().getSimpleName(), + keyFile == null ? "" : keyFile.getAbsolutePath())); } } diff --git a/etc/validator.out b/etc/validator.out index da6b3a0c04..e07acdec44 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -339,15 +339,15 @@ sites/udmi_site_model/out/devices/AHU-22/state.out "sub_folder" : "update", "sub_type" : "state", "status" : { - "message" : "1 schema violations found", - "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "1 schema violations found", - "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -382,15 +382,15 @@ sites/udmi_site_model/out/devices/AHU-22/state_system.out "sub_folder" : "system", "sub_type" : "state", "status" : { - "message" : "1 schema violations found", - "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "1 schema violations found", - "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -795,14 +795,14 @@ sites/udmi_site_model/out/devices/SNS-4/state.out "sub_type" : "state", "status" : { "message" : "Multiple validation errors", - "detail" : "1 schema violations found; Device has missing points: split_threshold, triangulating_axis", + "detail" : "2 schema violations found; Device has missing points: split_threshold, triangulating_axis", "category" : "validation.device.multiple", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "1 schema violations found", - "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -850,15 +850,15 @@ sites/udmi_site_model/out/devices/SNS-4/state_system.out "sub_folder" : "system", "sub_type" : "state", "status" : { - "message" : "1 schema violations found", - "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "1 schema violations found", - "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "2 schema violations found", + "detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 diff --git a/pubber/.idea/runConfigurations/Swarm.xml b/pubber/.idea/runConfigurations/Swarm.xml deleted file mode 100644 index 5cf22ca9f7..0000000000 --- a/pubber/.idea/runConfigurations/Swarm.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - \ No newline at end of file diff --git a/pubber/bin/publish b/pubber/bin/publish new file mode 100755 index 0000000000..d81cf81a0d --- /dev/null +++ b/pubber/bin/publish @@ -0,0 +1,24 @@ +#!/bin/bash -e + +UDMI_ROOT=$(realpath $(dirname $0)/../..) +cd $UDMI_ROOT + +DEFAULT_TASK="publish" +task=${1:-$DEFAULT_TASK} + +# Validate the task +VALID_TASKS=("publish" "publishToMavenLocal") +if [[ ! " ${VALID_TASKS[@]} " =~ " $task " ]]; then + echo "Usage: $0 [publishTask]" + echo " publishTask: Task to execute. Default is '$DEFAULT_TASK'." + echo " Valid tasks are: ${VALID_TASKS[*]}" + echo + echo "Error: Invalid task '$task'" + exit 1 +fi + +source $UDMI_ROOT/etc/shell_common.sh + +echo pubber/gradlew -p pubber $task +echo $UDMI_VERSION +$UDMI_ROOT/pubber/gradlew -p pubber $task diff --git a/pubber/build.gradle b/pubber/build.gradle index 4a6b88dfbf..ed91fd68c9 100644 --- a/pubber/build.gradle +++ b/pubber/build.gradle @@ -16,6 +16,7 @@ plugins { id 'java' id 'jacoco' id 'checkstyle' + id 'maven-publish' } group 'daq-pubber' @@ -76,6 +77,27 @@ repositories { maven { url 'https://jitpack.io' } } +publishing { + repositories { + maven { + url = "https://maven.pkg.github.com/${System.getenv("GITHUB_REPOSITORY")}" + name = "GitHubPackages" + credentials { + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") + } + } + } + publications { + mavenJava(MavenPublication) { + groupId = 'com.google.udmi' + artifactId = 'lib' + version = System.getenv("UDMI_VERSION") + from components.java + } + } +} + dependencies { implementation 'io.jsonwebtoken:jjwt:0.9.1' implementation 'javax.xml.bind:jaxb-api:2.3.1' diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 6b06dee3fc..11d743243a 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -1,702 +1,43 @@ package daq.pubber; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; -import static com.google.udmi.util.GeneralUtils.catchToFalse; -import static com.google.udmi.util.GeneralUtils.catchToNull; -import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; -import static com.google.udmi.util.GeneralUtils.fromJsonFile; -import static com.google.udmi.util.GeneralUtils.fromJsonFileStrict; -import static com.google.udmi.util.GeneralUtils.fromJsonString; -import static com.google.udmi.util.GeneralUtils.getFileBytes; -import static com.google.udmi.util.GeneralUtils.getTimestamp; -import static com.google.udmi.util.GeneralUtils.ifNotNullGet; -import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.GeneralUtils.ifTrueThen; -import static com.google.udmi.util.GeneralUtils.isTrue; -import static com.google.udmi.util.GeneralUtils.optionsString; -import static com.google.udmi.util.GeneralUtils.setClockSkew; -import static com.google.udmi.util.GeneralUtils.stackTraceString; -import static com.google.udmi.util.GeneralUtils.toJsonFile; -import static com.google.udmi.util.JsonUtil.safeSleep; -import static com.google.udmi.util.JsonUtil.stringify; -import static java.lang.String.format; -import static java.util.Objects.requireNonNullElse; -import static java.util.Optional.ofNullable; -import static udmi.schema.EndpointConfiguration.Protocol.MQTT; - -import com.google.udmi.util.CertManager; -import com.google.udmi.util.SiteModel; -import com.google.udmi.util.SiteModel.MetadataException; -import daq.pubber.PubberPubSubClient.Bundle; -import java.io.File; -import java.io.PrintStream; -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Function; +import daq.pubber.impl.host.PubberPublisherHost; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import udmi.lib.base.MqttDevice; -import udmi.lib.client.DeviceManager; -import udmi.lib.client.SystemManager; -import udmi.lib.intf.FamilyProvider; -import udmi.schema.Config; -import udmi.schema.DevicePersistent; -import udmi.schema.EndpointConfiguration; -import udmi.schema.EndpointConfiguration.Protocol; -import udmi.schema.Envelope; -import udmi.schema.Envelope.SubFolder; -import udmi.schema.Level; -import udmi.schema.Metadata; -import udmi.schema.Operation.SystemMode; -import udmi.schema.PubberConfiguration; -import udmi.schema.PubberOptions; -import udmi.util.SchemaVersion; /** - * IoT Core UDMI Device Emulator. + * Main Class for running UDMI publisher. */ -public class Pubber extends PubberManager implements PubberUdmiPublisher { - - public static final String PUBBER_OUT = "pubber/out"; - public static final String PERSISTENT_STORE_FILE = "persistent_data.json"; - public static final String PERSISTENT_TMP_FORMAT = "/tmp/pubber_%s_" + PERSISTENT_STORE_FILE; - public static final String CA_CRT = "ca.crt"; - - public static final Logger LOG = LoggerFactory.getLogger(Pubber.class); - private static final String HOSTNAME = System.getenv("HOSTNAME"); - - private static final String PUBSUB_SITE = "PubSub"; +public class Pubber { - private static final Map MESSAGE_COUNTS = new ConcurrentHashMap<>(); - private static final int CONNECT_RETRIES = 10; - private static final AtomicInteger retriesRemaining = new AtomicInteger(CONNECT_RETRIES); - private static final long RESTART_DELAY_MS = 1000; - - private static final Duration CLOCK_SKEW = Duration.ofMinutes(30); - private static final int STATE_SPAM_SEC = 5; // Expected config-state response time. - - private final File outDir; - private final ReentrantLock stateLock = new ReentrantLock(); - public PrintStream logPrintWriter; - protected DevicePersistent persistentData; - private CountDownLatch configLatch; - private MqttDevice deviceTarget; - private long lastStateTimeMs; - private PubberPubSubClient pubSubClient; - private Function connectionDone; - private String workingEndpoint; - private String attemptedEndpoint; - private EndpointConfiguration extractedEndpoint; - private SiteModel siteModel; - private SchemaVersion targetSchema; - private int deviceUpdateCount = -1; - private PubberDeviceManager deviceManager; - private boolean isGatewayDevice; + static final Logger LOG = LoggerFactory.getLogger(Pubber.class); + static final String USAGE = "Usage: config_file or { project_id site_path/ device_id serial_no }"; /** - * Start an instance from a configuration file. - * - * @param configPath Path to configuration file. - */ - public Pubber(String configPath) { - super(null, loadConfiguration(configPath)); - setClockSkew(isTrue(options.skewClock) ? CLOCK_SKEW : Duration.ZERO); - Protocol protocol = requireNonNullElse( - ifNotNullGet(config.endpoint, endpoint -> endpoint.protocol), MQTT); - checkArgument(MQTT.equals(protocol), "protocol mismatch"); - outDir = new File(PUBBER_OUT); - ifTrueThen(options.spamState, () -> schedulePeriodic(STATE_SPAM_SEC, this::markStateDirty)); - } - - /** - * Start an instance from explicit args. - * - * @param iotProject GCP project - * @param sitePath Path to site_model - * @param deviceId Device ID to emulate - * @param serialNo Serial number of the device - */ - public Pubber(String iotProject, String sitePath, String deviceId, String serialNo) { - super(null, makeExplicitConfiguration(iotProject, sitePath, deviceId, serialNo)); - outDir = new File(PUBBER_OUT + "/" + serialNo); - if (!outDir.exists()) { - checkState(outDir.mkdirs(), "could not make out dir " + outDir.getAbsolutePath()); - } - if (PUBSUB_SITE.equals(sitePath)) { - pubSubClient = new PubberPubSubClient(iotProject, deviceId); - } - } - - private static PubberConfiguration loadConfiguration(String configPath) { - File configFile = new File(configPath); - try { - return sanitizeConfiguration(fromJsonFileStrict(configFile, PubberConfiguration.class)); - } catch (Exception e) { - throw new RuntimeException("While configuring from " + configFile.getAbsolutePath(), e); - } - } - - private static PubberConfiguration makeExplicitConfiguration(String iotProject, String sitePath, - String deviceId, String serialNo) { - PubberConfiguration configuration = new PubberConfiguration(); - configuration.iotProject = iotProject; - configuration.sitePath = sitePath; - configuration.deviceId = deviceId; - configuration.serialNo = serialNo; - configuration.options = new PubberOptions(); - return configuration; - } - - /** - * Start a pubber instance with command line args. - * + * Start UDMI publisher with command line args. */ public static void main(String[] args) { try { - boolean swarm = args.length > 1 && PUBSUB_SITE.equals(args[1]); - if (swarm) { - swarmPubber(args); - } else { - singularPubber(args); - } - LOG.info("Done with main"); - } catch (Exception e) { - LOG.error("Exception starting pubber: " + friendlyStackTrace(e)); - e.printStackTrace(); - System.exit(-1); - } - } - - static Pubber singularPubber(String[] args) { - Pubber pubber = null; - try { - if (args.length == 1) { - pubber = new Pubber(args[0]); - } else if (args.length == 4) { - pubber = new Pubber(args[0], args[1], args[2], args[3]); - } else { - throw new IllegalArgumentException( - "Usage: config_file or { project_id site_path/ device_id serial_no }"); - } - pubber.initialize(); - pubber.startConnection(deviceId -> { - LOG.info(format("Connection closed/finished for %s", deviceId)); - return true; - }); - } catch (Exception e) { - if (pubber != null) { - pubber.shutdown(); - } - throw new RuntimeException("While starting singular pubber", e); - } - return pubber; - } - - private static void swarmPubber(String[] args) throws InterruptedException { - if (args.length != 4) { - throw new IllegalArgumentException( - "Usage: { project_id PubSub pubsub_subscription instance_count }"); - } - String projectId = args[0]; - String siteName = args[1]; - String feedName = args[2]; - int instances = Integer.parseInt(args[3]); - LOG.info(format("Starting %d pubber instances", instances)); - for (int instance = 0; instance < instances; instance++) { - String serialNo = format("%s-%d", HOSTNAME, (instance + 1)); - startFeedListener(projectId, siteName, feedName, serialNo); - } - LOG.info(format("Started all %d pubber instances", instances)); - } - - private static void startFeedListener(String projectId, String siteName, String feedName, - String serialNo) { - Pubber pubber = new Pubber(projectId, siteName, feedName, serialNo); - try { - LOG.info("Starting feed listener " + serialNo); - pubber.initialize(); - pubber.startConnection(deviceId -> { - LOG.error("Connection terminated, restarting listener"); - startFeedListener(projectId, siteName, feedName, serialNo); - return false; - }); - pubber.shutdown(); - } catch (Exception e) { - LOG.error("Exception starting instance " + serialNo, e); - pubber.shutdown(); - startFeedListener(projectId, siteName, feedName, serialNo); - } - } - - private static PubberConfiguration sanitizeConfiguration(PubberConfiguration configuration) { - if (configuration.options == null) { - configuration.options = new PubberOptions(); - } - return configuration; - } - - @Override - public FamilyProvider getLocalnetProvider(String family) { - return deviceManager.getLocalnetProvider(family); - } - - @Override - public void initializeDevice() { - deviceManager = new PubberDeviceManager(this, config); - - if (config.sitePath != null) { - PubberFeatures.writeFeatureFile(config.sitePath, deviceManager); - siteModel = new SiteModel(config.sitePath); - siteModel.initialize(); - if (config.endpoint == null) { - config.endpoint = siteModel.makeEndpointConfig(config.iotProject, deviceId); - } - if (!siteModel.allDeviceIds().contains(config.deviceId)) { - throw new IllegalArgumentException( - "Device ID " + config.deviceId + " not found in site model"); - } - Metadata metadata = siteModel.getMetadata(config.deviceId); - processDeviceMetadata(metadata); - deviceManager.setSiteModel(siteModel); - } else if (pubSubClient != null) { - pullDeviceMessage(); - } - - PubberFeatures.setFeatureSwap(config.options.featureEnableSwap); - initializePersistentStore(); - - info(format("Starting pubber %s, serial %s, mac %s, gateway %s, options %s", - config.deviceId, config.serialNo, config.macAddr, - config.gatewayId, optionsString(config.options))); - - markStateDirty(); - } - - @Override - public void initializePersistentStore() { - checkState(persistentData == null, "persistent data already loaded"); - File persistentStore = getPersistentStore(); - - if (isTrue(config.options.noPersist)) { - info("Resetting persistent store " + persistentStore.getAbsolutePath()); - persistentData = newDevicePersistent(); - } else { - info("Initializing from persistent store " + persistentStore.getAbsolutePath()); - persistentData = - persistentStore.exists() ? fromJsonFile(persistentStore, DevicePersistent.class) - : newDevicePersistent(); - } - - persistentData.restart_count = requireNonNullElse(persistentData.restart_count, 0) + 1; - - // If the persistentData contains endpoint configuration, prioritize using that. - // Otherwise, use the endpoint configuration that came from the Pubber config file on start. - if (persistentData.endpoint != null) { - info("Loading endpoint from persistent data"); - config.endpoint = persistentData.endpoint; - } else if (config.endpoint != null) { - info("Loading endpoint into persistent data from configuration"); - persistentData.endpoint = config.endpoint; - } else { - error( - "Neither configuration nor persistent data supplies endpoint configuration"); - } - - writePersistentStore(); - } - - @Override - public void writePersistentStore() { - checkState(persistentData != null, "persistent data not defined"); - toJsonFile(getPersistentStore(), persistentData); - warn("Updating persistent store:\n" + stringify(persistentData)); - deviceManager.setPersistentData(persistentData); - } - - private File getPersistentStore() { - return siteModel == null ? new File(format(PERSISTENT_TMP_FORMAT, deviceId)) : - new File(siteModel.getDeviceWorkingDir(deviceId), PERSISTENT_STORE_FILE); - } - - @Override - public void markStateDirty(long delayMs) { - stateDirty.set(true); - if (delayMs >= 0) { - try { - executor.schedule(this::flushDirtyState, delayMs, TimeUnit.MILLISECONDS); - } catch (Exception e) { - System.err.println("Rejecting state publish after " + delayMs + " " + e); - } - } - } - - private void pullDeviceMessage() { - while (true) { + PubberPublisherHost publisher = null; try { - info("Waiting for swarm configuration"); - Envelope attributes = new Envelope(); - Bundle pull = pubSubClient.pull(); - attributes.subFolder = SubFolder.valueOf(pull.attributes.get("subFolder")); - if (!SubFolder.SWARM.equals(attributes.subFolder)) { - error("Ignoring message with subFolder " + attributes.subFolder); - continue; + if (args.length == 1) { + publisher = new PubberPublisherHost(args[0]); + } else if (args.length == 4) { + publisher = new PubberPublisherHost(args[0], args[1], args[2], args[3]); + } else { + throw new IllegalArgumentException(USAGE); } - attributes.deviceId = pull.attributes.get("deviceId"); - attributes.deviceRegistryId = pull.attributes.get("deviceRegistryId"); - attributes.deviceRegistryLocation = pull.attributes.get("deviceRegistryLocation"); - - return; + publisher.initialize(); + publisher.startConnection(); } catch (Exception e) { - error("Error pulling swarm message", e); - safeSleep(INITIAL_THRESHOLD_SEC); - } - } - } - - private void processDeviceMetadata(Metadata metadata) { - if (metadata instanceof MetadataException metadataException) { - throw new RuntimeException("While processing metadata file " + metadataException.file, - metadataException.exception); - } - targetSchema = ifNotNullGet(metadata.system.device_version, SchemaVersion::fromKey); - ifNotNullThen(targetSchema, version -> warn("Emulating UDMI version " + version.key())); - - if (config.serialNo == null) { - config.serialNo = catchToNull(() -> metadata.system.serial_no); - } - if (config.gatewayId == null) { - config.gatewayId = catchToNull(() -> metadata.gateway.gateway_id); - } - - config.algorithm = config.gatewayId == null - ? catchToNull(() -> metadata.cloud.auth_type.value()) - : catchToNull(() -> siteModel.getAuthType(config.gatewayId).value()); - - info("Configured with auth_type " + config.algorithm); - - isGatewayDevice = catchToFalse(() -> metadata.gateway.proxy_ids != null); - - deviceManager.setMetadata(metadata); - } - - @Override - public synchronized void periodicUpdate() { - try { - deviceUpdateCount++; - checkSmokyFailure(); - deferredConfigActions(); - sendEmptyMissingBadEvents(); - maybeTweakState(); - flushDirtyState(); - } catch (Exception e) { - error("Fatal error during execution", e); - resetConnection(getWorkingEndpoint()); - } - } - - @Override - public void startConnection(Function connectionDone) { - String nonce = String.valueOf(System.currentTimeMillis()); - warn(format("Starting connection %s with %d", nonce, retriesRemaining.get())); - try { - this.connectionDone = connectionDone; - while (retriesRemaining.getAndDecrement() > 0) { - if (attemptConnection()) { - return; + if (publisher != null) { + publisher.shutdown(); } + throw new RuntimeException("While starting main", e); } - throw new RuntimeException("Failed connection attempt after retries"); - } catch (Exception e) { - throw new RuntimeException("While attempting to start connection", e); - } finally { - warn(format("Ending connection %s with %d", nonce, retriesRemaining.get())); - } - } - - private boolean attemptConnection() { - try { - deviceManager.stop(); - super.stop(); - disconnectMqtt(); - initializeMqtt(); - registerMessageHandlers(); - connect(); - configLatchWait(); - deviceManager.activate(); - return true; - } catch (Exception e) { - error("While waiting for connection start", e); - } - error("Attempt failed, retries remaining: " + retriesRemaining.get()); - safeSleep(RESTART_DELAY_MS); - return false; - } - - @Override - public void shutdown() { - warn("Initiating device shutdown"); - - if (deviceState.system != null && deviceState.system.operation != null) { - deviceState.system.operation.mode = SystemMode.SHUTDOWN; - } - - if (isConnected()) { - captureExceptions("Publishing shutdown state", this::publishSynchronousState); - } - ifNotNullThen(deviceManager, dm -> captureExceptions("Device manager shutdown", dm::shutdown)); - captureExceptions("Pubber sender shutdown", super::shutdown); - captureExceptions("Disconnecting mqtt", this::disconnectMqtt); - } - - @Override - public PubberConfiguration getConfig() { - return config; - } - - @Override - public void initializeMqtt() { - checkNotNull(config.deviceId, "configuration deviceId not defined"); - if (siteModel != null && config.keyFile != null) { - config.keyFile = siteModel.getDeviceKeyFile(config.deviceId); - } - ensureKeyBytes(); - checkState(deviceTarget == null, "mqttPublisher already defined"); - EndpointConfiguration endpoint = config.endpoint; - endpoint.gatewayId = ofNullable(config.gatewayId).orElse(config.deviceId); - endpoint.deviceId = config.deviceId; - endpoint.noConfigAck = options.noConfigAck; - endpoint.keyBytes = config.keyBytes; - endpoint.algorithm = config.algorithm; - augmentEndpoint(endpoint); - String keyPassword = siteModel.getDevicePassword(config.deviceId); - debug("Extracted device password from " + siteModel.getDeviceKeyFile(config.deviceId)); - String targetDeviceId = getTargetDeviceId(siteModel, config.deviceId); - CertManager certManager = new CertManager(new File(siteModel.getReflectorDir(), CA_CRT), - siteModel.getDeviceDir(targetDeviceId), endpoint.transport, keyPassword, - this::info); - deviceTarget = new MqttDevice(endpoint, this::publisherException, certManager); - publishDirtyState(); - } - - protected void augmentEndpoint(EndpointConfiguration endpoint) { - } - - private String getTargetDeviceId(SiteModel siteModel, String deviceId) { - Metadata metadata = siteModel.getMetadata(deviceId); - return ofNullable(catchToNull(() -> metadata.gateway.gateway_id)).orElse(deviceId); - } - - @Override - public byte[] ensureKeyBytes() { - if (config.keyBytes == null) { - checkNotNull(config.keyFile, "configuration keyFile not defined"); - info("Loading device key bytes from " + config.keyFile); - config.keyBytes = getFileBytes(config.keyFile); - config.keyFile = null; - } - return (byte[]) config.keyBytes; - } - - @Override - public synchronized void reconnect() { - while (retriesRemaining.getAndDecrement() > 0) { - if (attemptConnection()) { - return; - } - } - error("Connection retry failed, giving up."); - deviceManager.systemLifecycle(SystemMode.TERMINATE); - } - - @Override - public void persistEndpoint(EndpointConfiguration endpoint) { - notice("Persisting connection endpoint"); - persistentData.endpoint = endpoint; - writePersistentStore(); - } - - @Override - public synchronized void resetConnection(String targetEndpoint) { - try { - config.endpoint = fromJsonString(targetEndpoint, - EndpointConfiguration.class); - retriesRemaining.set(CONNECT_RETRIES); - startConnection(connectionDone); + LOG.info("Done with main"); } catch (Exception e) { - throw new RuntimeException("While resetting connection", e); - } - } - - @Override - public String traceTimestamp(String messageBase) { - int serial = MESSAGE_COUNTS.computeIfAbsent(messageBase, key -> new AtomicInteger()) - .incrementAndGet(); - String timestamp = getTimestamp().replace("Z", format(".%03dZ", serial)); - return messageBase + (isTrue(config.options.messageTrace) ? ("_" + timestamp) : ""); - } - - private void trace(String message) { - cloudLog(message, Level.TRACE); - } - - @Override - public void debug(String message) { - cloudLog(message, Level.DEBUG); - } - - @Override - public void info(String message) { - cloudLog(message, Level.INFO); - } - - @Override - public void warn(String message) { - cloudLog(message, Level.WARNING); - } - - @Override - public void error(String message) { - cloudLog(message, Level.ERROR); - } - - @Override - public void error(String message, Throwable e) { - if (e == null) { - error(message); - return; + LOG.error("Exception starting main", e); + System.exit(-1); } - String longMessage = message + ": " + e.getMessage(); - cloudLog(longMessage, Level.ERROR); - deviceManager.localLog(message, Level.TRACE, getTimestamp(), stackTraceString(e)); - } - - - @Override - public AtomicBoolean getStateDirty() { - return stateDirty; - } - - @Override - public SchemaVersion getTargetSchema() { - return targetSchema; - } - - @Override - public void setLastStateTimeMs(long lastStateTimeMs) { - this.lastStateTimeMs = lastStateTimeMs; - } - - @Override - public long getLastStateTimeMs() { - return this.lastStateTimeMs; - } - - @Override - public CountDownLatch getConfigLatch() { - return this.configLatch; - } - - @Override - public File getOutDir() { - return this.outDir; - } - - @Override - public PubberOptions getOptions() { - return options; - } - - @Override - public Lock getStateLock() { - return stateLock; - } - - @Override - public EndpointConfiguration getExtractedEndpoint() { - return this.extractedEndpoint; - } - - @Override - public void setExtractedEndpoint(EndpointConfiguration endpointConfiguration) { - this.extractedEndpoint = endpointConfiguration; - } - - @Override - public String getWorkingEndpoint() { - return workingEndpoint; - } - - @Override - public void setAttemptedEndpoint(String attemptedEndpoint) { - this.attemptedEndpoint = attemptedEndpoint; - } - - @Override - public String getAttemptedEndpoint() { - return this.attemptedEndpoint; - } - - @Override - public Config getDeviceConfig() { - return deviceConfig; - } - - @Override - public DeviceManager getDeviceManager() { - return deviceManager; - } - - @Override - public MqttDevice getDeviceTarget() { - return deviceTarget; - } - - @Override - public void setDeviceTarget(MqttDevice deviceTarget) { - this.deviceTarget = deviceTarget; - } - - @Override - public boolean isGatewayDevice() { - return isGatewayDevice; - } - - @Override - public void setWorkingEndpoint(String jsonString) { - this.workingEndpoint = jsonString; - } - - @Override - public void setConfigLatch(CountDownLatch countDownLatch) { - this.configLatch = countDownLatch; - } - - @Override - public boolean isConnected() { - return deviceTarget != null && deviceTarget.isActive(); - } - - @Override - public int getDeviceUpdateCount() { - return deviceUpdateCount; - } - - @Override - public Map> getLogMap() { - return SystemManager.getLogMap().apply(LOG); - } - - public SiteModel getSiteModel() { - return siteModel; } } diff --git a/pubber/src/main/java/daq/pubber/PubberDiscoveryManager.java b/pubber/src/main/java/daq/pubber/PubberDiscoveryManager.java deleted file mode 100644 index 523c3c0db5..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberDiscoveryManager.java +++ /dev/null @@ -1,172 +0,0 @@ -package daq.pubber; - -import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.GeneralUtils.ifTrueThen; -import static com.google.udmi.util.JsonUtil.getNowInstant; -import static com.google.udmi.util.JsonUtil.isoConvert; -import static daq.pubber.PubberUdmiPublisher.DEVICE_START_TIME; -import static java.lang.String.format; -import static udmi.schema.FamilyDiscoveryState.Phase.ACTIVE; -import static udmi.schema.FamilyDiscoveryState.Phase.STOPPED; - -import com.google.udmi.util.SiteModel; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicInteger; -import udmi.lib.ProtocolFamily; -import udmi.lib.client.DiscoveryManager; -import udmi.lib.intf.FamilyProvider; -import udmi.lib.intf.ManagerHost; -import udmi.schema.DiscoveryConfig; -import udmi.schema.DiscoveryEvents; -import udmi.schema.DiscoveryState; -import udmi.schema.Enumerations; -import udmi.schema.FamilyDiscoveryState; -import udmi.schema.PointPointsetModel; -import udmi.schema.PubberConfiguration; -import udmi.schema.RefDiscovery; -import udmi.schema.SystemDiscoveryData; - -/** - * Manager wrapper for discovery functionality in pubber. - */ -public class PubberDiscoveryManager extends PubberManager implements DiscoveryManager { - - public static final int SCAN_DURATION_SEC = 10; - - private final PubberDeviceManager deviceManager; - private DiscoveryState discoveryState; - private DiscoveryConfig discoveryConfig; - private SiteModel siteModel; - - public PubberDiscoveryManager(ManagerHost host, PubberConfiguration configuration, - PubberDeviceManager deviceManager) { - super(host, configuration); - this.deviceManager = deviceManager; - } - - /** - * Get a ref value that describes a point for self enumeration. - */ - public static RefDiscovery getModelPointRef(Entry entry, - boolean swapPointRef) { - RefDiscovery refDiscovery = new RefDiscovery(); - PointPointsetModel model = entry.getValue(); - refDiscovery.writable = model.writable; - refDiscovery.units = model.units; - refDiscovery.point = swapPointRef ? model.ref : entry.getKey(); - return refDiscovery; - } - - /** - * Updates discovery enumeration. - * - * @param config Discovery Configuration. - */ - public void updateDiscoveryEnumeration(DiscoveryConfig config) { - Date enumerationGeneration = config.generation; - if (enumerationGeneration == null) { - discoveryState.generation = null; - return; - } - if (discoveryState.generation != null - && !enumerationGeneration.after(discoveryState.generation)) { - return; - } - discoveryState.generation = enumerationGeneration; - info("Discovery enumeration at " + isoConvert(enumerationGeneration)); - DiscoveryEvents discoveryEvent = new DiscoveryEvents(); - discoveryEvent.scan_family = ProtocolFamily.IOT; - discoveryEvent.generation = enumerationGeneration; - Enumerations depths = config.enumerations; - discoveryEvent.points = maybeEnumerate(depths.points, () -> enumeratePoints(deviceId)); - discoveryEvent.features = maybeEnumerate(depths.features, PubberFeatures::getFeatures); - discoveryEvent.families = maybeEnumerate(depths.families, deviceManager::enumerateFamilies); - host.publish(discoveryEvent); - } - - /** - * Starts a discovery scan. - * - * @param family Discovery scan family. - * @param scanGeneration Scan generation. - */ - public void startDiscoveryScan(String family, Date scanGeneration) { - info("Discovery scan starting " + family + " generation " + isoConvert(scanGeneration)); - Date stopTime = Date.from(getNowInstant().plusSeconds(SCAN_DURATION_SEC)); - final FamilyDiscoveryState familyDiscoveryState = ensureFamilyDiscoveryState(family); - scheduleFuture(stopTime, () -> discoveryScanComplete(family, scanGeneration)); - familyDiscoveryState.generation = scanGeneration; - familyDiscoveryState.phase = ACTIVE; - AtomicInteger sendCount = new AtomicInteger(); - familyDiscoveryState.active_count = sendCount.get(); - updateState(); - discoveryProvider(family).startScan(shouldEnumerate(family), - (deviceId, discoveryEvent) -> ifNotNullThen(discoveryEvent.scan_addr, addr -> { - info(format("Discovered %s device %s for gen %s", family, addr, - isoConvert(scanGeneration))); - discoveryEvent.scan_family = family; - discoveryEvent.generation = scanGeneration; - discoveryEvent.system = new SystemDiscoveryData(); - discoveryEvent.system.ancillary = new HashMap<>(); - discoveryEvent.system.ancillary.put("device-name", deviceId); - familyDiscoveryState.active_count = sendCount.incrementAndGet(); - updateState(); - host.publish(discoveryEvent); - })); - } - - private FamilyProvider discoveryProvider(String family) { - return host.getLocalnetProvider(family); - } - - private void discoveryScanComplete(String family, Date scanGeneration) { - try { - FamilyDiscoveryState familyDiscoveryState = ensureFamilyDiscoveryState(family); - ifTrueThen(scanGeneration.equals(familyDiscoveryState.generation), - () -> { - discoveryProvider(family).stopScan(); - familyDiscoveryState.phase = STOPPED; - updateState(); - scheduleDiscoveryScan(family); - }); - } catch (Exception e) { - throw new RuntimeException("While completing discovery scan " + family, e); - } - } - - @Override - public Date getDeviceStartTime() { - return DEVICE_START_TIME; - } - - private Map enumeratePoints(String deviceId) { - return siteModel.getMetadata(deviceId).pointset.points; - } - - public void setSiteModel(SiteModel siteModel) { - this.siteModel = siteModel; - } - - @Override - public DiscoveryState getDiscoveryState() { - return discoveryState; - } - - @Override - public void setDiscoveryState(DiscoveryState discoveryState) { - this.discoveryState = discoveryState; - } - - @Override - public DiscoveryConfig getDiscoveryConfig() { - return discoveryConfig; - } - - @Override - public void setDiscoveryConfig(DiscoveryConfig discoveryConfig) { - this.discoveryConfig = discoveryConfig; - } -} diff --git a/pubber/src/main/java/daq/pubber/PubberGatewayManager.java b/pubber/src/main/java/daq/pubber/PubberGatewayManager.java deleted file mode 100644 index fe9a14df06..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberGatewayManager.java +++ /dev/null @@ -1,148 +0,0 @@ -package daq.pubber; - -import static com.google.udmi.util.GeneralUtils.catchToNull; -import static com.google.udmi.util.GeneralUtils.ifNotNullGet; -import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.GeneralUtils.ifNullThen; -import static com.google.udmi.util.GeneralUtils.ifTrueGet; -import static com.google.udmi.util.GeneralUtils.ifTrueThen; -import static com.google.udmi.util.GeneralUtils.isTrue; -import static java.lang.String.format; -import static java.util.Optional.ofNullable; -import static java.util.function.Predicate.not; -import static udmi.schema.Category.GATEWAY_PROXY_TARGET; - -import com.google.udmi.util.SiteModel; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import udmi.lib.ProtocolFamily; -import udmi.lib.client.GatewayManager; -import udmi.lib.client.ProxyDeviceHost; -import udmi.lib.intf.ManagerHost; -import udmi.schema.GatewayConfig; -import udmi.schema.GatewayState; -import udmi.schema.Level; -import udmi.schema.Metadata; -import udmi.schema.PubberConfiguration; - -/** - * Manager for UDMI gateway functionality. - */ -public class PubberGatewayManager extends PubberManager implements GatewayManager { - - private Map proxyDevices; - private SiteModel siteModel; - private Metadata metadata; - private GatewayState gatewayState; - - public PubberGatewayManager(ManagerHost host, PubberConfiguration configuration) { - super(host, configuration); - } - - public void setMetadata(Metadata metadata) { - this.metadata = metadata; - proxyDevices = ifNotNullGet(metadata.gateway, g -> createProxyDevices(g.proxy_ids)); - } - - @Override - public void activate() { - ifNotNullThen(proxyDevices, p -> CompletableFuture.runAsync(() -> p.values() - .parallelStream() - .forEach(ProxyDeviceHost::activate))); - } - - @Override - public ProxyDeviceHost makeExtraDevice() { - return new PubberProxyDevice(getHost(), EXTRA_PROXY_DEVICE, config); - } - - /** - * Update gateway operation based off of a gateway configuration block. This happens in two - * slightly different forms, one for the gateway proper (primarily indicating what devices - * should be proxy targets), and the other for the proxy devices themselves. - */ - @Override - public void updateConfig(GatewayConfig gateway) { - if (gateway == null) { - gatewayState = null; - updateState(); - return; - } - ifNullThen(gatewayState, () -> gatewayState = new GatewayState()); - - ifNotNullThen(proxyDevices, - p -> ifTrueThen(p.containsKey(EXTRA_PROXY_DEVICE), this::configExtraDevice)); - - if (gateway.proxy_ids == null || gateway.target != null) { - try { - String addr = catchToNull(() -> gateway.target.addr); - String family = ofNullable(catchToNull(() -> gateway.target.family)) - .orElse(ProtocolFamily.VENDOR); - validateGatewayFamily(family, addr); - setGatewayStatus(GATEWAY_PROXY_TARGET, Level.DEBUG, "gateway target family " + family); - } catch (Exception e) { - setGatewayStatus(GATEWAY_PROXY_TARGET, Level.ERROR, e.getMessage()); - } - } - updateState(); - } - - @Override - public void shutdown() { - super.shutdown(); - ifNotNullThen(proxyDevices, p -> p.values().forEach(ProxyDeviceHost::shutdown)); - } - - @Override - public void stop() { - super.stop(); - ifNotNullThen(proxyDevices, p -> p.values().forEach(ProxyDeviceHost::stop)); - } - - public void setSiteModel(SiteModel siteModel) { - this.siteModel = siteModel; - processMetadata(); - } - - void processMetadata() { - ifNotNullThen(proxyDevices, p -> p.values().forEach(proxy -> { - Metadata localMetadata = ifNotNullGet(siteModel, s -> s.getMetadata(proxy.getDeviceId())); - localMetadata = ofNullable(localMetadata).orElse(new Metadata()); - proxy.setMetadata(localMetadata); - })); - } - - @Override - public Metadata getMetadata() { - return metadata; - } - - @Override - public GatewayState getGatewayState() { - return gatewayState; - } - - @Override - public Map getProxyDevices() { - return proxyDevices; - } - - @Override - public Map createProxyDevices(List proxyIds) { - List deviceIds = ofNullable(proxyIds).orElseGet(ArrayList::new); - String firstId = deviceIds.stream().sorted().findFirst().orElse(null); - String noProxyId = ifTrueGet(isTrue(options.noProxy), () -> firstId); - ifNotNullThen(noProxyId, id -> warn(format("Not proxying device %s", noProxyId))); - List filteredList = deviceIds.stream().filter(not(id -> id.equals(noProxyId))).toList(); - Map devices = GatewayManager.super.createProxyDevices(filteredList); - ifTrueThen(options.extraDevice, () -> devices.put(EXTRA_PROXY_DEVICE, makeExtraDevice())); - return devices; - } - - @Override - public ProxyDeviceHost createProxyDevice(ManagerHost host, String id) { - return new PubberProxyDevice(host, id, config); - } -} diff --git a/pubber/src/main/java/daq/pubber/PubberManager.java b/pubber/src/main/java/daq/pubber/PubberManager.java deleted file mode 100644 index 9ea103a893..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberManager.java +++ /dev/null @@ -1,31 +0,0 @@ -package daq.pubber; - -import static java.util.Optional.ofNullable; - -import udmi.lib.base.ManagerBase; -import udmi.lib.intf.ManagerHost; -import udmi.schema.PubberConfiguration; -import udmi.schema.PubberOptions; - -/** - * Common manager class for all Pubber managers. Mainly just holds on to configuration and options. - */ -public class PubberManager extends ManagerBase { - - protected final PubberConfiguration config; - protected final PubberOptions options; - - /** - * New instance aye. - */ - public PubberManager(ManagerHost host, PubberConfiguration configuration) { - super(host, configuration.deviceId); - config = configuration; - options = configuration.options; - } - - @Override - protected int getIntervalSec(Integer sampleRateSec) { - return ofNullable(options.fixedSampleRate).orElse(super.getIntervalSec(sampleRateSec)); - } -} diff --git a/pubber/src/main/java/daq/pubber/PubberPointsetManager.java b/pubber/src/main/java/daq/pubber/PubberPointsetManager.java deleted file mode 100644 index 5384e549b4..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberPointsetManager.java +++ /dev/null @@ -1,214 +0,0 @@ -package daq.pubber; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.udmi.util.GeneralUtils.ifNotNullGet; -import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.GeneralUtils.ifNotTrueThen; -import static com.google.udmi.util.GeneralUtils.ifNullThen; -import static com.google.udmi.util.GeneralUtils.ifTrueGet; -import static com.google.udmi.util.GeneralUtils.ifTrueThen; -import static java.util.Objects.requireNonNull; -import static java.util.Objects.requireNonNullElseGet; -import static java.util.Optional.ofNullable; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import udmi.lib.client.PointsetManager; -import udmi.lib.intf.AbstractPoint; -import udmi.lib.intf.ManagerHost; -import udmi.schema.PointPointsetConfig; -import udmi.schema.PointPointsetModel; -import udmi.schema.PointPointsetState; -import udmi.schema.PointPointsetState.Value_state; -import udmi.schema.PointsetConfig; -import udmi.schema.PointsetModel; -import udmi.schema.PointsetState; -import udmi.schema.PubberConfiguration; - -/** - * Helper class to manage the operation of a pointset block. - */ -public class PubberPointsetManager extends PubberManager implements PointsetManager { - - private static final Set BOOLEAN_UNITS = ImmutableSet.of("No-units"); - - private static final Map DEFAULT_POINTS = ImmutableMap.of( - "recalcitrant_angle", makePointPointsetModel(true, 50, 50, "Celsius"), - "faulty_finding", makePointPointsetModel(true, 40, 0, "deg"), - "superimposition_reading", new PointPointsetModel() - ); - private final ExtraPointsetEvent pointsetEvent = new ExtraPointsetEvent(); - private final Map managedPoints = new HashMap<>(); - private int pointsetUpdateCount = -1; - private PointsetState pointsetState; - - /** - * Create a new instance attached to the given host. - */ - public PubberPointsetManager(ManagerHost host, PubberConfiguration configuration) { - super(host, configuration); - setExtraField(options.extraField); - updateState(); - } - - private static PointPointsetModel makePointPointsetModel(boolean writable, int value, - double tolerance, String units) { - PointPointsetModel pointMetadata = new PointPointsetModel(); - pointMetadata.writable = writable; - pointMetadata.baseline_value = value; - pointMetadata.baseline_tolerance = tolerance; - pointMetadata.units = units; - return pointMetadata; - } - - private AbstractPoint makePoint(String name, PointPointsetModel point) { - if (BOOLEAN_UNITS.contains(point.units)) { - return new PubberRandomBoolean(name, point); - } else { - return new PubberRandomPoint(name, point); - } - } - - @Override - public PointPointsetState getPointState(AbstractPoint point) { - PointPointsetState pointState = PointsetManager.super.getPointState(point); - // Tweak for testing: erroneously apply an applied state here. - ifTrueThen(point.getName().equals(options.extraPoint), - () -> pointState.value_state = ofNullable(pointState.value_state).orElse( - Value_state.APPLIED)); - return ifTrueGet(options.noPointState, PointPointsetState::new, pointState); - } - - /** - * Set the underlying static model for this pointset. This is information that would NOT be - * normally available for a device, but would, e.g. be programmed directly into a device. It's - * only available here since this is a reference pseudo-device for testing. - * - * @param model pointset model - */ - @Override - public void setPointsetModel(PointsetModel model) { - Map points = - ifNotNullGet(model, m -> requireNonNullElseGet(model.points, HashMap::new), DEFAULT_POINTS); - - ifNotNullThen(options.missingPoint, - x -> requireNonNull(points.remove(x), "missing point not in pointset metadata")); - - points.forEach((name, point) -> addPoint(makePoint(name, point))); - } - - /** - * updates the data of all points in the managedPoints map. - */ - @Override - public void updatePoints() { - managedPoints.values().forEach(point -> { - try { - point.updateData(); - } catch (Exception ex) { - error("Unable to update point data", ex); - } - updateState(point); - }); - } - - private void updatePointConfig(AbstractPoint point, PointPointsetConfig pointConfig) { - ifNotTrueThen(options.noWriteback, () -> { - point.setConfig(pointConfig); - updateState(point); - }); - } - - /** - * Updates the configuration of pointset points. - */ - @Override - public void updatePointsetPointsConfig(PointsetConfig config) { - // If there is no pointset config, then ensure that there's no pointset state. - if (config == null) { - pointsetState = null; - updateState(); - return; - } - - // Known that pointset config exists, so ensure that a pointset state also exists. - ifNullThen(pointsetState, () -> { - pointsetState = new PointsetState(); - pointsetState.points = new HashMap<>(); - pointsetEvent.points = new HashMap<>(); - }); - - // Update each internally managed point with its specific pointset config (if any). - Map points = ofNullable(config.points).orElseGet(HashMap::new); - ifNotNullThen(options.missingPoint, points::remove); - managedPoints.forEach((name, point) -> updatePointConfig(point, points.get(name))); - pointsetState.state_etag = config.state_etag; - - // Calculate the differences between the config points and state points. - Set configuredPoints = points.keySet(); - Set statePoints = pointsetState.points.keySet(); - Set missingPoints = Sets.difference(configuredPoints, statePoints).immutableCopy(); - final Set extraPoints = Sets.difference(statePoints, configuredPoints).immutableCopy(); - - // Restore points in config but not state, implicitly creating pointset.points.X as needed. - missingPoints.forEach(name -> { - debug("Restoring unknown point " + name); - restorePoint(name); - }); - - // Suspend points in state but not config, implicitly removing pointset.points.X as needed. - extraPoints.forEach(key -> { - debug("Clearing extraneous point " + key); - suspendPoint(key); - }); - - // Ensure that the logic was correct, and that state points match config points. - checkState(pointsetState.points.keySet().equals(points.keySet()), - "state/config pointset mismatch"); - - // Special testing provisions for forcing an extra point (designed to cause a violation). - ifNotNullThen(options.extraPoint, extraPoint -> pointsetEvent.points.put(extraPoint, - PointsetManager.extraPointsetEvent())); - - // Mark device state as dirty, so the system will send a consolidated state update. - updateState(); - } - - @Override - public void periodicUpdate() { - try { - if (pointsetState != null) { - pointsetUpdateCount++; - updatePoints(); - sendDevicePoints(); - } - } catch (Exception e) { - error("Fatal error during execution", e); - } - } - - @Override - public int getPointsetUpdateCount() { - return pointsetUpdateCount; - } - - @Override - public udmi.lib.client.PointsetManager.ExtraPointsetEvent getPointsetEvent() { - return pointsetEvent; - } - - @Override - public Map getManagedPoints() { - return managedPoints; - } - - @Override - public PointsetState getPointsetState() { - return pointsetState; - } - -} diff --git a/pubber/src/main/java/daq/pubber/PubberProxyDevice.java b/pubber/src/main/java/daq/pubber/PubberProxyDevice.java deleted file mode 100644 index fd15c39ede..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberProxyDevice.java +++ /dev/null @@ -1,90 +0,0 @@ -package daq.pubber; - -import static com.google.udmi.util.GeneralUtils.catchToNull; -import static com.google.udmi.util.GeneralUtils.deepCopy; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import udmi.lib.client.DeviceManager; -import udmi.lib.client.ProxyDeviceHost; -import udmi.lib.intf.ManagerHost; -import udmi.schema.Metadata; -import udmi.schema.PubberConfiguration; - -/** - * Wrapper for a complete device construct. - */ -public class PubberProxyDevice extends PubberManager implements ProxyDeviceHost { - - private static final long STATE_INTERVAL_MS = 1000; - final PubberDeviceManager deviceManager; - final Pubber pubberHost; - private final AtomicBoolean active = new AtomicBoolean(); - - /** - * New instance. - */ - public PubberProxyDevice(ManagerHost host, String id, PubberConfiguration pubberConfig) { - super(host, makeProxyConfiguration(host, id, pubberConfig)); - // Simple shortcut to get access to some foundational mechanisms inside of Pubber. - pubberHost = (Pubber) host; - deviceManager = new PubberDeviceManager(this, makeProxyConfiguration(host, id, - pubberConfig)); - deviceManager.setSiteModel(pubberHost.getSiteModel()); - executor.scheduleAtFixedRate(this::publishDirtyState, STATE_INTERVAL_MS, STATE_INTERVAL_MS, - TimeUnit.MILLISECONDS); - } - - private static PubberConfiguration makeProxyConfiguration(ManagerHost host, String id, - PubberConfiguration config) { - PubberConfiguration proxyConfiguration = deepCopy(config); - proxyConfiguration.deviceId = id; - Metadata metadata = ((Pubber) host).getSiteModel().getMetadata(id); - proxyConfiguration.serialNo = catchToNull(() -> metadata.system.serial_no); - return proxyConfiguration; - } - - @Override - public void shutdown() { - deviceManager.shutdown(); - isActive().set(false); - } - - @Override - public void stop() { - deviceManager.stop(); - isActive().set(false); - } - - private void publishDirtyState() { - if (stateDirty.getAndSet(false)) { - publish(deviceId, deviceState); - } - } - - @Override - public void publish(String targetId, Object message) { - publish(message); - } - - @Override - public DeviceManager getDeviceManager() { - return deviceManager; - } - - @Override - public PubberUdmiPublisher getUdmiPublisher() { - return pubberHost; - } - - @Override - public ManagerHost getManagerHost() { - return host; - } - - @Override - public AtomicBoolean isActive() { - return active; - } - -} diff --git a/pubber/src/main/java/daq/pubber/PubberPubSubClient.java b/pubber/src/main/java/daq/pubber/PubberPubSubClient.java deleted file mode 100644 index 6817c365ea..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberPubSubClient.java +++ /dev/null @@ -1,92 +0,0 @@ -package daq.pubber; - -import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import java.util.List; -import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.threeten.bp.Duration; - -/** - * Wrapper class for a PubSub client. - */ -public class PubberPubSubClient { - - private static final Logger LOG = LoggerFactory.getLogger(PubberPubSubClient.class); - - private final String subscriptionName; - private final GrpcSubscriberStub subscriber; - - /** - * Subscribe to the given subscription. - * - * @param projectId GCP project id - * @param subscriptionId PubSub subscription - */ - public PubberPubSubClient(String projectId, String subscriptionId) { - subscriptionName = ProjectSubscriptionName.format(projectId, subscriptionId); - LOG.info("Using PubSub subscription " + subscriptionName); - try { - SubscriberStubSettings.Builder subSettingsBuilder = - SubscriberStubSettings.newBuilder(); - subSettingsBuilder - .pullSettings() - .setSimpleTimeoutNoRetries(Duration.ofDays(1)) - .build(); - SubscriberStubSettings build = subSettingsBuilder.build(); - subscriber = GrpcSubscriberStub.create(build); - - } catch (Exception e) { - throw new RuntimeException("While connecting to subscription " + subscriptionName, e); - } - } - - /** - * Pull a message from the subscription. - * - * @return Bundle of pulled message. - */ - public Bundle pull() { - PullRequest pullRequest = - PullRequest.newBuilder() - .setMaxMessages(1) - .setSubscription(subscriptionName) - .build(); - - List messages; - do { - PullResponse pullResponse = subscriber.pullCallable().call(pullRequest); - messages = pullResponse.getReceivedMessagesList(); - } while (messages.size() == 0); - - if (messages.size() != 1) { - throw new RuntimeException("Did not receive singular message"); - } - ReceivedMessage message = messages.get(0); - - AcknowledgeRequest acknowledgeRequest = - AcknowledgeRequest.newBuilder() - .setSubscription(subscriptionName) - .addAckIds(message.getAckId()) - .build(); - - subscriber.acknowledgeCallable().call(acknowledgeRequest); - - Bundle bundle = new Bundle(); - bundle.body = message.getMessage().getData().toStringUtf8(); - bundle.attributes = message.getMessage().getAttributesMap(); - return bundle; - } - - static class Bundle { - - String body; - Map attributes; - } -} diff --git a/pubber/src/main/java/daq/pubber/PubberSystemManager.java b/pubber/src/main/java/daq/pubber/PubberSystemManager.java deleted file mode 100644 index 460b12688d..0000000000 --- a/pubber/src/main/java/daq/pubber/PubberSystemManager.java +++ /dev/null @@ -1,238 +0,0 @@ -package daq.pubber; - -import static com.google.udmi.util.GeneralUtils.catchOrElse; -import static com.google.udmi.util.GeneralUtils.catchToNull; -import static com.google.udmi.util.GeneralUtils.getTimestamp; -import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.GeneralUtils.ifNotTrueGet; -import static com.google.udmi.util.GeneralUtils.ifTrueThen; -import static com.google.udmi.util.GeneralUtils.isTrue; -import static com.google.udmi.util.JsonUtil.isoConvert; -import static java.lang.String.format; -import static java.util.Optional.ofNullable; - -import com.google.common.collect.ImmutableList; -import com.google.udmi.util.CleanDateFormat; -import java.io.File; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import udmi.lib.client.SystemManager; -import udmi.lib.intf.ManagerHost; -import udmi.schema.Entry; -import udmi.schema.Level; -import udmi.schema.Metadata; -import udmi.schema.Operation; -import udmi.schema.Operation.SystemMode; -import udmi.schema.PubberConfiguration; -import udmi.schema.StateSystemHardware; -import udmi.schema.StateSystemOperation; -import udmi.schema.SystemConfig; - -/** - * Support manager for system stuff. - */ -public class PubberSystemManager extends PubberManager implements SystemManager { - - private static final String PUBBER_LOG = "device.log"; - private static final String DEFAULT_MAKE = "bos"; - private static final String DEFAULT_MODEL = "pubber"; - private static final String DEFAULT_SOFTWARE_KEY = "firmware"; - private static final String DEFAULT_SOFTWARE_VALUE = "v1"; - private static final Date DEVICE_START_TIME = PubberUdmiPublisher.DEVICE_START_TIME; - - private final List logentries = new ArrayList<>(); - private final ExtraSystemState systemState; - private final ManagerHost host; - private SystemConfig systemConfig; - private boolean publishingLog; - - /** - * New instance. - */ - public PubberSystemManager(ManagerHost host, PubberConfiguration configuration) { - super(host, configuration); - this.host = host; - - if (host instanceof Pubber pubberHost) { - initializeLogger(pubberHost); - info("Device start time is " + isoConvert(DEVICE_START_TIME)); - } - - systemState = new ExtraSystemState(); - systemState.operation = new StateSystemOperation(); - - if (!isTrue(options.noLastStart)) { - systemState.operation.last_start = DEVICE_START_TIME; - } - - systemState.hardware = isTrue(options.noHardware) ? null : new StateSystemHardware(); - - systemState.operation.operational = true; - systemState.operation.mode = SystemMode.INITIAL; - if (host instanceof Pubber) { - systemState.serial_no = configuration.serialNo; - } - systemState.last_config = new Date(0); - - ifNotNullThen(options.extraField, value -> systemState.extraField = value); - - updateState(); - } - - private void initializeLogger(Pubber host) { - File outDir = new File(Pubber.PUBBER_OUT); - try { - outDir.mkdirs(); - host.logPrintWriter = new PrintStream(new File(outDir, PUBBER_LOG)); - host.logPrintWriter.println("Pubber log started at " + getTimestamp()); - } catch (Exception e) { - throw new RuntimeException("While initializing out dir " + outDir.getAbsolutePath(), e); - } - } - - @Override - public void shutdown() { - super.shutdown(); - if (host instanceof Pubber pubberHost && pubberHost.logPrintWriter != null) { - pubberHost.logPrintWriter.close(); - } - } - - @Override - public ExtraSystemState getSystemState() { - return this.systemState; - } - - @Override - public Date getDeviceStartTime() { - return DEVICE_START_TIME; - } - - @Override - public void periodicUpdate() { - sendSystemEvent(); - } - - @Override - public void updateConfig(SystemConfig system, Date timestamp) { - SystemManager.super.updateConfig(system, ifNotTrueGet(options.noLastConfig, () -> timestamp)); - } - - @Override - public void systemLifecycle(SystemMode mode) { - systemState.operation.mode = mode; - try { - host.update(null); - } catch (Exception e) { - error("Squashing error publishing state while shutting down", e); - } - int exitCode = EXIT_CODE_MAP.getOrDefault(mode, UNKNOWN_MODE_EXIT_CODE); - error("Stopping system with extreme prejudice, restart " + mode + " with code " + exitCode); - System.exit(exitCode); - } - - @Override - public void localLog(String message, Level level, String timestamp, String detail) { - String detailPostfix = detail == null ? "" : ":\n" + detail; - String logMessage = format("%s %s%s", timestamp, message, detailPostfix); - SystemManager.getLogMap().apply(Pubber.LOG).get(level).accept(logMessage); - try { - PrintStream stream; - if (host instanceof Pubber pubberHost) { - stream = pubberHost.logPrintWriter; - } else if (host instanceof PubberProxyDevice proxyHost) { - stream = proxyHost.pubberHost.logPrintWriter; - } else { - throw new RuntimeException("While writing log output file: Unknown host"); - } - stream.println(logMessage); - stream.flush(); - } catch (Exception e) { - throw new RuntimeException("While writing log output file", e); - } - } - - @Override - public void setHardwareSoftware(Metadata metadata) { - ExtraSystemState state = getSystemState(); - state.hardware.make = catchOrElse(() -> metadata.system.hardware.make, () -> DEFAULT_MAKE); - - state.hardware.model = catchOrElse(() -> metadata.system.hardware.model, () -> DEFAULT_MODEL); - - state.software = new HashMap<>(); - Map metadataSoftware = catchToNull(() -> metadata.system.software); - if (metadataSoftware == null) { - state.software.put(DEFAULT_SOFTWARE_KEY, DEFAULT_SOFTWARE_VALUE); - } else { - state.software = metadataSoftware; - } - - if (options.softwareFirmwareValue != null) { - state.software.put("firmware", options.softwareFirmwareValue); - } - } - - @Override - public void maybeRestartSystem() { - SystemManager.super.maybeRestartSystem(); - SystemConfig system = ofNullable(getSystemConfig()).orElseGet(SystemConfig::new); - Operation operation = ofNullable(system.operation).orElseGet(Operation::new); - Date configLastStart = operation.last_start; - if (configLastStart != null && isTrue(options.smokeCheck) - && CleanDateFormat.dateEquals(getDeviceStartTime(), configLastStart)) { - error(format("Device start time %s matches, smoke check indicating success!", - isoConvert(configLastStart))); - systemLifecycle(SystemMode.SHUTDOWN); - } - } - - @Override - public boolean shouldLogLevel(int level) { - if (options.fixedLogLevel != null) { - return level >= options.fixedLogLevel; - } - return SystemManager.super.shouldLogLevel(level); - } - - @Override - public synchronized void publishLogMessage(Entry report) { - if (shouldLogLevel(report.level)) { - ifTrueThen(options.badLevel, () -> report.level = 0); - logentries.add(report); - } - } - - @Override - public synchronized List getLogentries() { - if (isTrue(options.noLog)) { - return null; - } - List entries = ImmutableList.copyOf(logentries); - logentries.clear(); - return entries; - } - - @Override - public boolean getPublishingLog() { - return publishingLog; - } - - @Override - public SystemConfig getSystemConfig() { - return systemConfig; - } - - @Override - public void setSystemConfig(SystemConfig systemConfig) { - this.systemConfig = systemConfig; - } - - @Override - public void setPublishingLog(boolean publishingLog) { - this.publishingLog = publishingLog; - } -} diff --git a/pubber/src/main/java/daq/pubber/PubberFeatures.java b/pubber/src/main/java/daq/pubber/impl/PubberFeatures.java similarity index 89% rename from pubber/src/main/java/daq/pubber/PubberFeatures.java rename to pubber/src/main/java/daq/pubber/impl/PubberFeatures.java index f10cfcc3f9..bcb41bf27c 100644 --- a/pubber/src/main/java/daq/pubber/PubberFeatures.java +++ b/pubber/src/main/java/daq/pubber/impl/PubberFeatures.java @@ -1,8 +1,9 @@ -package daq.pubber; +package daq.pubber.impl; import static com.google.udmi.util.GeneralUtils.getTimestamp; import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.JsonUtil.writeFile; +import static java.lang.String.format; import static udmi.schema.Bucket.DISCOVERY_SCAN; import static udmi.schema.Bucket.ENDPOINT_CONFIG; import static udmi.schema.Bucket.ENUMERATION; @@ -16,6 +17,7 @@ import static udmi.schema.FeatureDiscovery.FeatureStage.PREVIEW; import static udmi.schema.FeatureDiscovery.FeatureStage.STABLE; +import daq.pubber.impl.manager.PubberDeviceManager; import java.io.File; import java.util.HashMap; import java.util.Map; @@ -56,12 +58,12 @@ private static void add(Bucket featureBucket, FeatureStage stage) { public static void writeFeatureFile(String sitePath, PubberDeviceManager deviceManager) { File path = new File(sitePath, PUBBER_FEATURES_JSON); try { - String message = "Writing pubber feature file to " + path.getAbsolutePath(); + String message = format("Writing pubber feature file to %s", path.getAbsolutePath()); deviceManager.localLog(message, Level.NOTICE, getTimestamp(), null); path.getParentFile().mkdirs(); writeFile(FEATURES_MAP, path); } catch (Exception e) { - throw new RuntimeException("While making dir for " + path.getAbsolutePath()); + throw new RuntimeException(format("While making dir for %s", path.getAbsolutePath())); } } diff --git a/pubber/src/main/java/daq/pubber/impl/PubberManager.java b/pubber/src/main/java/daq/pubber/impl/PubberManager.java new file mode 100644 index 0000000000..4e76c6e826 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/PubberManager.java @@ -0,0 +1,223 @@ +package daq.pubber.impl; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.GeneralUtils.catchToNull; +import static com.google.udmi.util.GeneralUtils.deepCopy; +import static com.google.udmi.util.GeneralUtils.fromJsonFileStrict; +import static com.google.udmi.util.GeneralUtils.ifNullThen; +import static com.google.udmi.util.GeneralUtils.ifTrueThen; +import static com.google.udmi.util.GeneralUtils.isTrue; +import static java.lang.String.format; +import static java.util.Optional.ofNullable; + +import com.google.udmi.util.SiteModel; +import daq.pubber.impl.host.PubberPublisherHost; +import java.io.File; +import udmi.lib.base.ManagerBase; +import udmi.lib.intf.ManagerHost; +import udmi.schema.Metadata; +import udmi.schema.PubberConfiguration; +import udmi.schema.PubberOptions; + +/** + * Common manager class for all Pubber managers. Mainly just holds on to configuration and options. + */ +public class PubberManager extends ManagerBase { + + protected static final String LOG_PATH = "pubber/out"; + protected static final String PERSISTENT_STORE_FILE = "persistent_data.json"; + protected static final String PERSISTENT_TMP_FORMAT = "/tmp/pubber_%s_" + PERSISTENT_STORE_FILE; + + protected final PubberConfiguration config; + protected final PubberOptions options; + protected File outDir; + + /** + * New instance. + */ + public PubberManager(ManagerHost host, PubberConfiguration configuration) { + super(host, configuration.deviceId); + config = configuration; + options = configuration.options; + } + + @Override + protected int getIntervalSec(Integer sampleRateSec) { + return ofNullable(options.fixedSampleRate).orElse(super.getIntervalSec(sampleRateSec)); + } + + public PubberOptions getOptions() { + return options; + } + + public PubberConfiguration getConfig() { + return config; + } + + protected static PubberConfiguration loadConfiguration(String configPath) { + File configFile = new File(configPath); + try { + PubberConfiguration fromFile = fromJsonFileStrict(configFile, PubberConfiguration.class); + ifNullThen(fromFile.options, () -> fromFile.options = new PubberOptions()); + return fromFile; + } catch (Exception e) { + throw new RuntimeException( + format("While configuring from %s", configFile.getAbsolutePath()), e); + } + } + + protected static PubberConfiguration makeExplicitConfiguration( + String iotProject, String sitePath, String deviceId, String serialNo) { + PubberConfiguration configuration = new PubberConfiguration(); + configuration.iotProject = iotProject; + configuration.sitePath = sitePath; + configuration.deviceId = deviceId; + configuration.serialNo = serialNo; + configuration.options = new PubberOptions(); + return configuration; + } + + protected static PubberConfiguration makeProxyConfiguration( + ManagerHost host, String id, PubberConfiguration config) { + PubberConfiguration proxyConfiguration = deepCopy(config); + proxyConfiguration.deviceId = id; + Metadata metadata = ((PubberPublisherHost) host).getSiteModel().getMetadata(id); + proxyConfiguration.serialNo = catchToNull(() -> metadata.system.serial_no); + return proxyConfiguration; + } + + protected static File createOutDir(String serialNo) { + File dir = new File(serialNo == null ? LOG_PATH : format("%s/%s", LOG_PATH, serialNo)); + ifTrueThen(!dir.exists(), () -> + checkState(dir.mkdirs(), format("Could not make out dir %s", dir.getAbsolutePath()))); + return dir; + } + + protected static File getPersistentStore(SiteModel siteModel, String deviceId) { + return siteModel == null + ? new File(format(PERSISTENT_TMP_FORMAT, deviceId)) + : new File(siteModel.getDeviceWorkingDir(deviceId), PERSISTENT_STORE_FILE); + } + + // + public String getRedirectRegistry() { + return options.redirectRegistry; + } + + public boolean isBadVersion() { + return isTrue(options.badVersion); + } + + public boolean isNoFolder() { + return isTrue(options.noFolder); + } + + public boolean isNoState() { + return isTrue(options.noState); + } + + public boolean isBadState() { + return isTrue(options.badState); + } + + public boolean isTweakState() { + return isTrue(options.tweakState); + } + + public boolean isEmptyMissing() { + return isTrue(options.emptyMissing); + } + + public boolean isBarfConfig() { + return isTrue(options.barfConfig); + } + + public boolean isSmokeCheck() { + return isTrue(options.smokeCheck); + } + + public boolean isConfigStateDelay() { + return isTrue(options.configStateDelay); + } + + public boolean isBadCategory() { + return isTrue(options.badCategory); + } + + public boolean isNoStatus() { + return isTrue(options.noStatus); + } + + public boolean isDupeState() { + return isTrue(options.dupeState); + } + + public boolean isMessageTrace() { + return isTrue(options.messageTrace); + } + + public boolean isSkewClock() { + return isTrue(options.skewClock); + } + + public boolean isSpamState() { + return isTrue(options.spamState); + } + + public boolean isNoLog() { + return isTrue(options.noLog); + } + + public boolean isBadLevel() { + return isTrue(options.badLevel); + } + + public boolean isNoLastConfig() { + return isTrue(options.noLastConfig); + } + + public boolean isNoLastStart() { + return isTrue(options.noLastStart); + } + + public boolean isNoHardware() { + return isTrue(options.noHardware); + } + + public String getExtraField() { + return options.extraField; + } + + public Integer getFixedLogLevel() { + return options.fixedLogLevel; + } + + public String getSoftwareFirmwareValue() { + return options.softwareFirmwareValue; + } + + public boolean isNoWriteback() { + return isTrue(options.noWriteback); + } + + public boolean isNoPointState() { + return isTrue(options.noPointState); + } + + public String getExtraPoint() { + return options.extraPoint; + } + + public String getMissingPoint() { + return options.missingPoint; + } + + public boolean isNoProxy() { + return isTrue(options.noProxy); + } + + public boolean isExtraDevice() { + return isTrue(options.extraDevice); + } + // +} diff --git a/pubber/src/main/java/daq/pubber/impl/host/PubberProxyHost.java b/pubber/src/main/java/daq/pubber/impl/host/PubberProxyHost.java new file mode 100644 index 0000000000..7fe8a6bd95 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/host/PubberProxyHost.java @@ -0,0 +1,61 @@ +package daq.pubber.impl.host; + +import daq.pubber.impl.PubberManager; +import daq.pubber.impl.manager.PubberDeviceManager; +import java.util.concurrent.atomic.AtomicBoolean; +import udmi.lib.client.host.ProxyHost; +import udmi.lib.client.host.PublisherHost; +import udmi.lib.client.manager.DeviceManager; +import udmi.lib.intf.ManagerHost; +import udmi.schema.PubberConfiguration; + +/** + * Wrapper for a complete device construct. + */ +public class PubberProxyHost extends PubberManager implements ProxyHost { + + private final AtomicBoolean active = new AtomicBoolean(); + private final PubberDeviceManager deviceManager; + private final PubberPublisherHost publisherHost; + + /** + * New instance. + */ + public PubberProxyHost(ManagerHost host, String id, PubberConfiguration pubberConfig) { + super(host, makeProxyConfiguration(host, id, pubberConfig)); + publisherHost = (PubberPublisherHost) host; + deviceManager = new PubberDeviceManager(this, makeProxyConfiguration(host, id, pubberConfig)); + deviceManager.setSiteModel(publisherHost.getSiteModel()); + schedulePeriodic(STATE_INTERVAL_SEC, this::publishDirtyState); + } + + @Override + public DeviceManager getDeviceManager() { + return deviceManager; + } + + @Override + public PublisherHost getPublisherHost() { + return publisherHost; + } + + @Override + public ManagerHost getManagerHost() { + return host; + } + + @Override + public AtomicBoolean isActive() { + return active; + } + + @Override + public void shutdown() { + ProxyHost.super.shutdown(); + } + + @Override + public void stop() { + ProxyHost.super.stop(); + } +} diff --git a/pubber/src/main/java/daq/pubber/impl/host/PubberPublisherHost.java b/pubber/src/main/java/daq/pubber/impl/host/PubberPublisherHost.java new file mode 100644 index 0000000000..a1c41f8334 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/host/PubberPublisherHost.java @@ -0,0 +1,484 @@ +package daq.pubber.impl.host; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.GeneralUtils.catchToFalse; +import static com.google.udmi.util.GeneralUtils.catchToNull; +import static com.google.udmi.util.GeneralUtils.fromJsonFile; +import static com.google.udmi.util.GeneralUtils.getFileBytes; +import static com.google.udmi.util.GeneralUtils.ifNotNullGet; +import static com.google.udmi.util.GeneralUtils.ifNotNullThen; +import static com.google.udmi.util.GeneralUtils.ifNullThen; +import static com.google.udmi.util.GeneralUtils.isTrue; +import static com.google.udmi.util.GeneralUtils.optionsString; +import static com.google.udmi.util.GeneralUtils.toJsonFile; +import static com.google.udmi.util.JsonUtil.safeSleep; +import static com.google.udmi.util.JsonUtil.stringify; +import static java.lang.String.format; +import static java.util.Objects.requireNonNullElse; +import static java.util.Optional.ofNullable; +import static udmi.schema.EndpointConfiguration.Protocol.MQTT; + +import com.google.udmi.util.CertManager; +import com.google.udmi.util.SiteModel; +import daq.pubber.impl.PubberFeatures; +import daq.pubber.impl.PubberManager; +import daq.pubber.impl.manager.PubberDeviceManager; +import java.io.File; +import java.io.PrintStream; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import udmi.lib.base.MqttDevice; +import udmi.lib.client.host.PublisherHost; +import udmi.lib.client.manager.DeviceManager; +import udmi.schema.DevicePersistent; +import udmi.schema.EndpointConfiguration; +import udmi.schema.Metadata; +import udmi.schema.Operation; +import udmi.util.SchemaVersion; + +/** + * IoT Core UDMI Device Emulator. + */ +public class PubberPublisherHost extends PubberManager implements PublisherHost { + + private static final int CONNECT_RETRIES = 10; + + private PubberDeviceManager deviceManager; + private SiteModel siteModel; + + /** + * Start an instance from a configuration file. + * + * @param configPath Path to configuration file. + */ + public PubberPublisherHost(String configPath) { + super(null, loadConfiguration(configPath)); + outDir = createOutDir(null); + } + + /** + * Start an instance from explicit args. + * + * @param iotProject GCP project + * @param sitePath Path to site_model + * @param deviceId Device ID to emulate + * @param serialNo Serial number of the device + */ + public PubberPublisherHost(String iotProject, String sitePath, String deviceId, String serialNo) { + super(null, makeExplicitConfiguration(iotProject, sitePath, deviceId, serialNo)); + outDir = createOutDir(serialNo); + } + + @Override + public void initialize() { + EndpointConfiguration.Protocol protocol = requireNonNullElse( + ifNotNullGet(config.endpoint, endpoint -> endpoint.protocol), MQTT); + checkArgument(MQTT.equals(protocol), "Protocol mismatch"); + PublisherHost.super.initialize(); + } + + @Override + public void initializeDevice() { + deviceManager = new PubberDeviceManager(this, config); + + if (config.sitePath != null) { + PubberFeatures.writeFeatureFile(config.sitePath, deviceManager); + siteModel = new SiteModel(config.sitePath); + siteModel.initialize(); + if (config.endpoint == null) { + config.endpoint = siteModel.makeEndpointConfig(config.iotProject, deviceId); + } + if (!siteModel.allDeviceIds().contains(config.deviceId)) { + throw new IllegalArgumentException( + format("Device ID %s not found in site model", config.deviceId)); + } + Metadata metadata = siteModel.getMetadata(config.deviceId); + processDeviceMetadata(metadata); + deviceManager.setSiteModel(siteModel); + } + + PubberFeatures.setFeatureSwap(config.options.featureEnableSwap); + initializePersistentStore(); + + info(format("Starting pubber %s, serial %s, mac %s, gateway %s, options %s", + config.deviceId, config.serialNo, config.macAddr, + config.gatewayId, optionsString(config.options))); + + markStateDirty(); + } + + @Override + public void initializePersistentStore() { + checkState(persistentData == null, "Persistent data already loaded"); + File persistentStore = getPersistentStore(siteModel, deviceId); + + if (isTrue(config.options.noPersist)) { + info(format("Resetting persistent store %s", persistentStore.getAbsolutePath())); + persistentData = newDevicePersistent(); + } else { + info(format("Initializing from persistent store %s", persistentStore.getAbsolutePath())); + persistentData = persistentStore.exists() + ? fromJsonFile(persistentStore, DevicePersistent.class) + : newDevicePersistent(); + } + + persistentData.restart_count = requireNonNullElse(persistentData.restart_count, 0) + 1; + + // If the persistentData contains endpoint configuration, prioritize using that. + // Otherwise, use the endpoint configuration that came from the Pubber config file on start. + if (persistentData.endpoint != null) { + info("Loading endpoint from persistent data"); + config.endpoint = persistentData.endpoint; + } else if (config.endpoint != null) { + info("Loading endpoint into persistent data from configuration"); + persistentData.endpoint = config.endpoint; + } else { + error("Neither configuration nor persistent data supplies endpoint configuration"); + } + + writePersistentStore(); + } + + @Override + public void writePersistentStore() { + checkState(persistentData != null, "Persistent data not defined"); + toJsonFile(getPersistentStore(siteModel, deviceId), persistentData); + warn(format("Updating persistent store:\n%s", stringify(persistentData))); + deviceManager.setPersistentData(persistentData); + } + + private void processDeviceMetadata(Metadata metadata) { + if (metadata instanceof SiteModel.MetadataException metadataException) { + throw new RuntimeException( + format("While processing metadata file %s", metadataException.file), + metadataException.exception); + } + targetSchema = ifNotNullGet(metadata.system.device_version, SchemaVersion::fromKey); + ifNotNullThen(targetSchema, + version -> warn(format("Emulating UDMI version %s", version.key()))); + + ifNullThen(config.serialNo, + () -> config.serialNo = catchToNull(() -> metadata.system.serial_no)); + ifNullThen(config.gatewayId, + () -> config.gatewayId = catchToNull(() -> metadata.gateway.gateway_id)); + + config.algorithm = config.gatewayId == null + ? catchToNull(() -> metadata.cloud.auth_type.value()) + : catchToNull(() -> siteModel.getAuthType(config.gatewayId).value()); + + info(format("Configured with auth_type %s", config.algorithm)); + isGatewayDevice = catchToFalse(() -> metadata.gateway.proxy_ids != null); + deviceManager.setMetadata(metadata); + } + + @Override + public void startConnection() { + String nonce = String.valueOf(System.currentTimeMillis()); + warn(format("Starting connection %s with %d", nonce, retriesCount.get())); + try { + while (retriesCount.getAndIncrement() < CONNECT_RETRIES) { + if (attemptConnection()) { + return; + } + } + throw new RuntimeException("Failed connection attempt after retries"); + } catch (Exception e) { + throw new RuntimeException("While attempting to start connection", e); + } finally { + warn(format("Ending connection %s with %d", nonce, retriesCount.get())); + } + } + + private boolean attemptConnection() { + try { + deviceManager.stop(); + super.stop(); + disconnectMqtt(); + initializeMqtt(); + registerMessageHandlers(); + connect(); + configLatchWait(); + deviceManager.activate(); + return true; + } catch (Exception e) { + error("While waiting for connection start", e); + } + error(format("Attempt #%s failed", retriesCount.get())); + safeSleep(RESTART_DELAY_MS); + return false; + } + + @Override + public void initializeMqtt() { + checkNotNull(config.deviceId, "Configuration deviceId not defined"); + if (siteModel != null && config.keyFile != null) { + config.keyFile = siteModel.getDeviceKeyFile(config.deviceId); + } + ensureKeyBytes(); + checkState(deviceTarget == null, "MQTT target already defined"); + EndpointConfiguration endpoint = config.endpoint; + endpoint.gatewayId = ofNullable(config.gatewayId).orElse(config.deviceId); + endpoint.deviceId = config.deviceId; + endpoint.noConfigAck = options.noConfigAck; + endpoint.keyBytes = config.keyBytes; + endpoint.algorithm = config.algorithm; + augmentEndpoint(endpoint); + String keyPassword = siteModel.getDevicePassword(config.deviceId); + debug(format("Extracted device password from %s", siteModel.getDeviceKeyFile(config.deviceId))); + String targetDeviceId = getTargetDeviceId(siteModel, config.deviceId); + CertManager certManager = new CertManager(new File(siteModel.getReflectorDir(), "ca.crt"), + siteModel.getDeviceDir(targetDeviceId), endpoint.transport, keyPassword, this::info); + deviceTarget = new MqttDevice(endpoint, this::publisherException, certManager); + publishDirtyState(); + } + + protected void augmentEndpoint(EndpointConfiguration endpoint) { + } + + private String getTargetDeviceId(SiteModel siteModel, String deviceId) { + Metadata metadata = siteModel.getMetadata(deviceId); + return ofNullable(catchToNull(() -> metadata.gateway.gateway_id)).orElse(deviceId); + } + + @Override + public void ensureKeyBytes() { + if (config.keyBytes == null) { + checkNotNull(config.keyFile, "Configuration keyFile not defined"); + info(format("Loading device key bytes from %s", config.keyFile)); + config.keyBytes = getFileBytes(config.keyFile); + config.keyFile = null; + } + } + + @Override + public synchronized void reconnect() { + while (retriesCount.getAndIncrement() < CONNECT_RETRIES) { + if (attemptConnection()) { + return; + } + } + error("Connection retry failed, giving up."); + deviceManager.systemLifecycle(Operation.SystemMode.TERMINATE); + } + + @Override + public String getLogPath() { + return LOG_PATH; + } + + public SiteModel getSiteModel() { + return siteModel; + } + + // + private final Map messageCounts = new ConcurrentHashMap<>(); + private final AtomicInteger retriesCount = new AtomicInteger(0); + private final ReentrantLock stateLock = new ReentrantLock(); + + private CountDownLatch configLatch; + private MqttDevice deviceTarget; + private long lastStateTimeMs; + private String workingEndpoint; + private String attemptedEndpoint; + private EndpointConfiguration extractedEndpoint; + private SchemaVersion targetSchema; + private int deviceUpdateCount = -1; + private boolean isGatewayDevice; + private PrintStream logPrintWriter; + + public DevicePersistent persistentData; + + @Override + public void periodicSchedule(int sec, Runnable runnable) { + schedulePeriodic(sec, runnable); + } + + @Override + public void schedule(long ms, Runnable runnable) { + executor.schedule(runnable, ms, TimeUnit.MILLISECONDS); + } + + @Override + public void periodicUpdate() { + PublisherHost.super.periodicUpdate(); + } + + @Override + public void shutdown() { + PublisherHost.super.shutdown(super::shutdown); + } + + @Override + public void debug(String message) { + PublisherHost.super.debug(message); + } + + @Override + public void info(String message) { + PublisherHost.super.info(message); + } + + @Override + public void warn(String message) { + PublisherHost.super.warn(message); + } + + @Override + public void error(String message) { + PublisherHost.super.error(message); + } + + @Override + public void error(String message, Throwable e) { + PublisherHost.super.error(message, e); + } + + @Override + public Map getMessageCounts() { + return messageCounts; + } + + @Override + public AtomicBoolean getStateDirty() { + return stateDirty; + } + + @Override + public SchemaVersion getTargetSchema() { + return targetSchema; + } + + @Override + public void setLastStateTimeMs(long lastStateTimeMs) { + this.lastStateTimeMs = lastStateTimeMs; + } + + @Override + public long getLastStateTimeMs() { + return lastStateTimeMs; + } + + @Override + public CountDownLatch getConfigLatch() { + return configLatch; + } + + @Override + public File getOutDir() { + return outDir; + } + + @Override + public Lock getStateLock() { + return stateLock; + } + + @Override + public EndpointConfiguration getExtractedEndpoint() { + return extractedEndpoint; + } + + @Override + public void setExtractedEndpoint(EndpointConfiguration endpointConfiguration) { + this.extractedEndpoint = endpointConfiguration; + } + + @Override + public String getWorkingEndpoint() { + return workingEndpoint; + } + + @Override + public void setAttemptedEndpoint(String attemptedEndpoint) { + this.attemptedEndpoint = attemptedEndpoint; + } + + @Override + public String getAttemptedEndpoint() { + return attemptedEndpoint; + } + + @Override + public DeviceManager getDeviceManager() { + return deviceManager; + } + + @Override + public MqttDevice getDeviceTarget() { + return deviceTarget; + } + + @Override + public void setDeviceTarget(MqttDevice targetDevice) { + this.deviceTarget = targetDevice; + } + + @Override + public boolean isGatewayDevice() { + return isGatewayDevice; + } + + @Override + public void setWorkingEndpoint(String workingEndpoint) { + this.workingEndpoint = workingEndpoint; + } + + @Override + public void setConfigLatch(CountDownLatch configLatch) { + this.configLatch = configLatch; + } + + @Override + public int getDeviceUpdateCount() { + return deviceUpdateCount; + } + + @Override + public void incrementDeviceUpdateCount() { + deviceUpdateCount++; + } + + @Override + public PrintStream getLogPrintWriter() { + return logPrintWriter; + } + + @Override + public void setLogPrintWriter(PrintStream logPrintWriter) { + this.logPrintWriter = logPrintWriter; + } + + @Override + public String getIotProject() { + return config.iotProject; + } + + @Override + public EndpointConfiguration getEndpoint() { + return config.endpoint; + } + + @Override + public void setEndpoint(EndpointConfiguration endpoint) { + config.endpoint = endpoint; + } + + @Override + public AtomicInteger getRetriesCount() { + return retriesCount; + } + + @Override + public DevicePersistent getPersistentData() { + return persistentData; + } + // +} diff --git a/pubber/src/main/java/daq/pubber/PubberDeviceManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberDeviceManager.java similarity index 71% rename from pubber/src/main/java/daq/pubber/PubberDeviceManager.java rename to pubber/src/main/java/daq/pubber/impl/manager/PubberDeviceManager.java index b34845ece4..a28ef3165a 100644 --- a/pubber/src/main/java/daq/pubber/PubberDeviceManager.java +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberDeviceManager.java @@ -1,17 +1,17 @@ -package daq.pubber; +package daq.pubber.impl.manager; import com.google.udmi.util.SiteModel; +import daq.pubber.impl.PubberManager; import java.util.ArrayList; import java.util.List; -import udmi.lib.client.DeviceManager; -import udmi.lib.client.DiscoveryManager; -import udmi.lib.client.GatewayManager; -import udmi.lib.client.LocalnetManager; -import udmi.lib.client.PointsetManager; -import udmi.lib.client.SubBlockManager; -import udmi.lib.client.SystemManager; +import udmi.lib.client.manager.DeviceManager; +import udmi.lib.client.manager.DiscoveryManager; +import udmi.lib.client.manager.GatewayManager; +import udmi.lib.client.manager.LocalnetManager; +import udmi.lib.client.manager.PointsetManager; +import udmi.lib.client.manager.SystemManager; import udmi.lib.intf.ManagerHost; -import udmi.schema.Config; +import udmi.lib.intf.SubBlockManager; import udmi.schema.PubberConfiguration; /** @@ -19,11 +19,11 @@ */ public class PubberDeviceManager extends PubberManager implements DeviceManager { - private final PubberPointsetManager pointsetManager; private final PubberSystemManager systemManager; private final PubberLocalnetManager localnetManager; - private final PubberGatewayManager gatewayManager; + private final PubberPointsetManager pointsetManager; private final PubberDiscoveryManager discoveryManager; + private final PubberGatewayManager gatewayManager; private final List subManagers = new ArrayList<>(); /** @@ -42,15 +42,14 @@ public PubberDeviceManager(ManagerHost host, PubberConfiguration configuration) gatewayManager = addManager(new PubberGatewayManager(host, configuration)); } - private T addManager(T manager) { - // Keep the resulting list in reverse order for proper shutdown semantics. - subManagers.add(0, manager); - return manager; + @Override + public SystemManager getSystemManager() { + return systemManager; } @Override - public void updateConfig(Config config) { - DeviceManager.super.updateConfig(config); + public LocalnetManager getLocalnetManager() { + return localnetManager; } @Override @@ -59,13 +58,8 @@ public PointsetManager getPointsetManager() { } @Override - public SystemManager getSystemManager() { - return systemManager; - } - - @Override - public LocalnetManager getLocalnetManager() { - return localnetManager; + public DiscoveryManager getDiscoveryManager() { + return discoveryManager; } @Override @@ -74,30 +68,24 @@ public GatewayManager getGatewayManager() { } @Override - public DiscoveryManager getDiscoveryManager() { - return discoveryManager; + public List getSubmanagers() { + return subManagers; } - /** - * Shutdown everything, including sub-managers. - */ @Override public void shutdown() { - subManagers.forEach(SubBlockManager::shutdown); + DeviceManager.super.shutdown(); } - /** - * Stop periodic senders. - */ @Override public void stop() { - subManagers.forEach(SubBlockManager::stop); + DeviceManager.super.stop(); } /** * Set the site model. */ - protected void setSiteModel(SiteModel siteModel) { + public void setSiteModel(SiteModel siteModel) { discoveryManager.setSiteModel(siteModel); gatewayManager.setSiteModel(siteModel); localnetManager.setSiteModel(siteModel); diff --git a/pubber/src/main/java/daq/pubber/impl/manager/PubberDiscoveryManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberDiscoveryManager.java new file mode 100644 index 0000000000..29f55edb23 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberDiscoveryManager.java @@ -0,0 +1,82 @@ +package daq.pubber.impl.manager; + + +import com.google.udmi.util.SiteModel; +import daq.pubber.impl.PubberFeatures; +import daq.pubber.impl.PubberManager; +import java.util.HashMap; +import java.util.Map; +import udmi.lib.client.manager.DeviceManager; +import udmi.lib.client.manager.DiscoveryManager; +import udmi.lib.intf.ManagerHost; +import udmi.schema.DiscoveryConfig; +import udmi.schema.DiscoveryEvents; +import udmi.schema.DiscoveryState; +import udmi.schema.FeatureDiscovery; +import udmi.schema.PointPointsetModel; +import udmi.schema.PubberConfiguration; +import udmi.schema.SystemDiscoveryData; + +/** + * Manager wrapper for discovery functionality in pubber. + */ +public class PubberDiscoveryManager extends PubberManager implements DiscoveryManager { + + private final PubberDeviceManager deviceManager; + private DiscoveryState discoveryState; + private DiscoveryConfig discoveryConfig; + + private SiteModel siteModel; + + public PubberDiscoveryManager(ManagerHost host, PubberConfiguration configuration, + PubberDeviceManager deviceManager) { + super(host, configuration); + this.deviceManager = deviceManager; + } + + @Override + public DeviceManager getDeviceManager() { + return deviceManager; + } + + @Override + public DiscoveryState getDiscoveryState() { + return discoveryState; + } + + @Override + public void setDiscoveryState(DiscoveryState discoveryState) { + this.discoveryState = discoveryState; + } + + @Override + public DiscoveryConfig getDiscoveryConfig() { + return discoveryConfig; + } + + @Override + public void setDiscoveryConfig(DiscoveryConfig discoveryConfig) { + this.discoveryConfig = discoveryConfig; + } + + @Override + public Map getFeatures() { + return PubberFeatures.getFeatures(); + } + + public void setSiteModel(SiteModel siteModel) { + this.siteModel = siteModel; + } + + @Override + public Map enumeratePoints(String deviceId) { + return siteModel.getMetadata(deviceId).pointset.points; + } + + @Override + public void postDiscoveryProcess(String deviceId, DiscoveryEvents discoveryEvent) { + discoveryEvent.system = new SystemDiscoveryData(); + discoveryEvent.system.ancillary = new HashMap<>(); + discoveryEvent.system.ancillary.put("device-name", deviceId); + } +} diff --git a/pubber/src/main/java/daq/pubber/impl/manager/PubberGatewayManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberGatewayManager.java new file mode 100644 index 0000000000..969f3c7e94 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberGatewayManager.java @@ -0,0 +1,99 @@ +package daq.pubber.impl.manager; + +import static com.google.udmi.util.GeneralUtils.ifNotNullGet; +import static com.google.udmi.util.GeneralUtils.ifNotNullThen; +import static java.util.Optional.ofNullable; + +import com.google.udmi.util.SiteModel; +import daq.pubber.impl.PubberManager; +import daq.pubber.impl.host.PubberProxyHost; +import java.util.Map; +import udmi.lib.client.host.ProxyHost; +import udmi.lib.client.manager.GatewayManager; +import udmi.lib.intf.ManagerHost; +import udmi.schema.GatewayConfig; +import udmi.schema.GatewayState; +import udmi.schema.Metadata; +import udmi.schema.PubberConfiguration; + +/** + * Manager for UDMI gateway functionality. + */ +public class PubberGatewayManager extends PubberManager implements GatewayManager { + + private Map proxyDevices; + private GatewayState gatewayState; + private Metadata metadata; + + public PubberGatewayManager(ManagerHost host, PubberConfiguration configuration) { + super(host, configuration); + } + + @Override + public Metadata getMetadata() { + return metadata; + } + + @Override + public void setMetadata(Metadata metadata) { + this.metadata = metadata; + GatewayManager.super.setMetadata(metadata); + } + + @Override + public GatewayState getGatewayState() { + return gatewayState; + } + + @Override + public void setGatewayState(GatewayState gatewayState) { + this.gatewayState = gatewayState; + } + + @Override + public Map getProxyDevices() { + return proxyDevices; + } + + @Override + public void setProxyDevices(Map proxyDevices) { + this.proxyDevices = proxyDevices; + } + + @Override + public void shutdown() { + super.shutdown(); + GatewayManager.super.shutdown(); + } + + @Override + public void stop() { + super.stop(); + GatewayManager.super.stop(); + } + + @Override + public ProxyHost createProxyDevice(ManagerHost host, String id) { + return new PubberProxyHost(host, id, config); + } + + @Override + public ProxyHost makeExtraDevice() { + return new PubberProxyHost(getHost(), EXTRA_PROXY_DEVICE, config); + } + + @Override + public void syncDevices(GatewayConfig gatewayConfig) { + } + + /** + * Set site model. + */ + public void setSiteModel(SiteModel siteModel) { + ifNotNullThen(proxyDevices, p -> p.values().forEach(proxy -> { + Metadata localMetadata = ifNotNullGet(siteModel, s -> s.getMetadata(proxy.getDeviceId())); + localMetadata = ofNullable(localMetadata).orElse(new Metadata()); + proxy.setMetadata(localMetadata); + })); + } +} diff --git a/pubber/src/main/java/daq/pubber/PubberLocalnetManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberLocalnetManager.java similarity index 79% rename from pubber/src/main/java/daq/pubber/PubberLocalnetManager.java rename to pubber/src/main/java/daq/pubber/impl/manager/PubberLocalnetManager.java index 3ee53cc88b..c7f5c7dcef 100644 --- a/pubber/src/main/java/daq/pubber/PubberLocalnetManager.java +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberLocalnetManager.java @@ -1,14 +1,21 @@ -package daq.pubber; +package daq.pubber.impl.manager; import static com.google.udmi.util.GeneralUtils.catchToNull; +import static java.lang.String.format; import com.google.udmi.util.SiteModel; +import daq.pubber.impl.PubberManager; +import daq.pubber.impl.provider.PubberBacnetProvider; +import daq.pubber.impl.provider.PubberFamilyProvider; +import daq.pubber.impl.provider.PubberIpProvider; +import daq.pubber.impl.provider.PubberVendorProvider; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; import udmi.lib.ProtocolFamily; -import udmi.lib.client.LocalnetManager; +import udmi.lib.client.host.PublisherHost; +import udmi.lib.client.manager.LocalnetManager; import udmi.lib.intf.FamilyProvider; import udmi.lib.intf.ManagerHost; import udmi.schema.LocalnetConfig; @@ -21,8 +28,8 @@ */ public class PubberLocalnetManager extends PubberManager implements LocalnetManager { - private final LocalnetState localnetState; private final Map localnetProviders; + private final LocalnetState localnetState; private LocalnetConfig localnetConfig; static Map> LOCALNET_PROVIDERS = Map.of( @@ -42,16 +49,39 @@ public PubberLocalnetManager(ManagerHost host, PubberConfiguration configuration localnetProviders = new HashMap<>(); } + @Override + public LocalnetConfig getLocalnetConfig() { + return localnetConfig; + } + + @Override + public void setLocalnetConfig(LocalnetConfig localnetConfig) { + this.localnetConfig = localnetConfig; + } + + @Override + public LocalnetState getLocalnetState() { + return this.localnetState; + } + + @Override + public Map getLocalnetProviders() { + // Silly type downgrade from PubberFamilyProvider to FamilyProvider. + return localnetProviders.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + } + /** * Instantiate a family provider. */ PubberFamilyProvider instantiateProvider(String family) { try { - return LOCALNET_PROVIDERS.get(family).getDeclaredConstructor( - ManagerHost.class, String.class, String.class) + return LOCALNET_PROVIDERS.get(family) + .getDeclaredConstructor(ManagerHost.class, String.class, String.class) .newInstance(this, family, config.deviceId); } catch (Exception e) { - throw new RuntimeException("While creating instance of " + LOCALNET_PROVIDERS.get(family), e); + throw new RuntimeException(format("While creating instance of %s", + LOCALNET_PROVIDERS.get(family)), e); } } @@ -64,11 +94,11 @@ public void setSiteModel(SiteModel siteModel) { localnetProviders.put(family, instantiateProvider(family)); return; } - if (providerClass == PubberBacnetProvider.class && host instanceof Pubber) { + if (providerClass == PubberBacnetProvider.class && host instanceof PublisherHost) { localnetProviders.put(family, instantiateProvider(family)); return; } - if (providerClass == PubberIpProvider.class && host instanceof Pubber pubberHost) { + if (providerClass == PubberIpProvider.class && host instanceof PublisherHost) { Metadata metadata = siteModel.getMetadata(getDeviceId()); String addr = catchToNull(() -> metadata.localnet.families.get(family).addr); if (addr != null) { @@ -76,30 +106,6 @@ public void setSiteModel(SiteModel siteModel) { } } }); - localnetProviders.forEach((key, value) -> value.setSiteModel(siteModel)); + localnetProviders.values().forEach(value -> value.setSiteModel(siteModel)); } - - @Override - public LocalnetState getLocalnetState() { - return this.localnetState; - } - - - @Override - public LocalnetConfig getLocalnetConfig() { - return localnetConfig; - } - - @Override - public void setLocalnetConfig(LocalnetConfig localnetConfig) { - this.localnetConfig = localnetConfig; - } - - @Override - public Map getLocalnetProviders() { - // Silly type downgrade from PubberFamilyProvider to FamilyProvider. - return localnetProviders.entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); - } - } diff --git a/pubber/src/main/java/daq/pubber/impl/manager/PubberPointsetManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberPointsetManager.java new file mode 100644 index 0000000000..840b67bded --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberPointsetManager.java @@ -0,0 +1,130 @@ +package daq.pubber.impl.manager; + +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import daq.pubber.impl.PubberManager; +import daq.pubber.impl.point.PubberRandomBoolean; +import daq.pubber.impl.point.PubberRandomPoint; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import udmi.lib.client.manager.PointsetManager; +import udmi.lib.intf.AbstractPoint; +import udmi.lib.intf.ManagerHost; +import udmi.schema.PointPointsetConfig; +import udmi.schema.PointPointsetModel; +import udmi.schema.PointsetState; +import udmi.schema.PubberConfiguration; + +/** + * Helper class to manage the operation of a pointset block. + */ +public class PubberPointsetManager extends PubberManager implements PointsetManager { + + private final ExtraPointsetEvent pointsetEvent = new ExtraPointsetEvent(); + private final Map managedPoints = new HashMap<>(); + + private int pointsetUpdateCount = -1; + private PointsetState pointsetState; + + /** + * Create a new instance attached to the given host. + */ + public PubberPointsetManager(ManagerHost host, PubberConfiguration configuration) { + super(host, configuration); + setExtraField(options.extraField); + updateState(); + } + + @Override + public int getPointsetUpdateCount() { + return pointsetUpdateCount; + } + + @Override + public void incrementUpdateCount() { + pointsetUpdateCount++; + } + + @Override + public ExtraPointsetEvent getPointsetEvent() { + return pointsetEvent; + } + + @Override + public Map getManagedPoints() { + return managedPoints; + } + + @Override + public PointsetState getPointsetState() { + return pointsetState; + } + + @Override + public void setPointsetState(PointsetState pointsetState) { + this.pointsetState = pointsetState; + } + + @Override + public void periodicUpdate() { + PointsetManager.super.periodicUpdate(); + } + + @Override + public AbstractPoint makePoint(String name, PointPointsetModel point) { + if (point.units.equals("No-units")) { + return new PubberRandomBoolean(name, point); + } else { + return new PubberRandomPoint(name, point); + } + } + + @Override + public boolean syncPoints(Map points) { + // Calculate the differences between the config points and state points. + Set configuredPoints = points.keySet(); + Set statePoints = pointsetState.points.keySet(); + Set missingPoints = Sets.difference(configuredPoints, statePoints).immutableCopy(); + final Set extraPoints = Sets.difference(statePoints, configuredPoints).immutableCopy(); + + // Restore points in config but not state, implicitly creating pointset.points.X as needed. + missingPoints.forEach(name -> { + debug(format("Restoring unknown point %s", name)); + restorePoint(name); + }); + + // Suspend points in state but not config, implicitly removing pointset.points.X as needed. + extraPoints.forEach(key -> { + debug(format("Clearing extraneous point %s", key)); + suspendPoint(key); + }); + + // Ensure that the logic was correct, and that state points match config points. + checkState(pointsetState.points.keySet().equals(points.keySet()), + "state/config pointset mismatch"); + + return !missingPoints.isEmpty() || !extraPoints.isEmpty(); + } + + @Override + public Map getTestPoints() { + return ImmutableMap.of( + "recalcitrant_angle", makePointPointsetModel(50, 50, "Celsius"), + "faulty_finding", makePointPointsetModel(40, 0, "deg"), + "superimposition_reading", new PointPointsetModel()); + } + + private static PointPointsetModel makePointPointsetModel(int value, double tolerance, + String units) { + PointPointsetModel pointMetadata = new PointPointsetModel(); + pointMetadata.writable = true; + pointMetadata.baseline_value = value; + pointMetadata.baseline_tolerance = tolerance; + pointMetadata.units = units; + return pointMetadata; + } +} diff --git a/pubber/src/main/java/daq/pubber/impl/manager/PubberSystemManager.java b/pubber/src/main/java/daq/pubber/impl/manager/PubberSystemManager.java new file mode 100644 index 0000000000..df1de1cc68 --- /dev/null +++ b/pubber/src/main/java/daq/pubber/impl/manager/PubberSystemManager.java @@ -0,0 +1,102 @@ +package daq.pubber.impl.manager; + +import static com.google.udmi.util.GeneralUtils.ifNullThen; +import static java.lang.String.format; + +import daq.pubber.impl.PubberManager; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import udmi.lib.client.host.PublisherHost; +import udmi.lib.client.manager.SystemManager; +import udmi.lib.intf.ManagerHost; +import udmi.schema.Entry; +import udmi.schema.Metadata; +import udmi.schema.Operation.SystemMode; +import udmi.schema.PubberConfiguration; +import udmi.schema.SystemConfig; + +/** + * Support manager for system stuff. + */ +public class PubberSystemManager extends PubberManager implements SystemManager { + + private final List logs = new ArrayList<>(); + private final AtomicBoolean publishLock = new AtomicBoolean(false); + private final ExtraSystemState systemState = new ExtraSystemState(); + + private SystemConfig systemConfig; + + /** + * New instance. + */ + public PubberSystemManager(ManagerHost host, PubberConfiguration configuration) { + super(host, configuration); + initialize(host, configuration.serialNo); + } + + @Override + public ExtraSystemState getSystemState() { + return systemState; + } + + @Override + public SystemConfig getSystemConfig() { + return systemConfig; + } + + @Override + public void setSystemConfig(SystemConfig systemConfig) { + this.systemConfig = systemConfig; + } + + @Override + public List getLogs() { + return logs; + } + + @Override + public AtomicBoolean getPublishLock() { + return publishLock; + } + + @Override + public void shutdown() { + super.shutdown(); + SystemManager.super.shutdown(); + } + + @Override + public void periodicUpdate() { + SystemManager.super.periodicUpdate(); + } + + @Override + public void systemLifecycle(SystemMode mode) { + systemState.operation.mode = mode; + try { + host.update(null); + } catch (Exception e) { + error("Squashing error publishing state while shutting down", e); + } + int exitCode = EXIT_CODE_MAP.getOrDefault(mode, UNKNOWN_MODE_EXIT_CODE); + error( + format("Stopping system with extreme prejudice, restart %s with code %s", mode, exitCode)); + System.exit(exitCode); + } + + @Override + public void setHardwareSoftware(Metadata metadata) { + SystemManager.super.setHardwareSoftware(metadata); + if (getHost() instanceof PublisherHost) { + ExtraSystemState state = getSystemState(); + ifNullThen(state.hardware.make, () -> state.hardware.make = "bos"); + ifNullThen(state.hardware.model, () -> state.hardware.model = "pubber"); + ifNullThen(state.software, () -> { + state.software = new HashMap<>(); + state.software.put(DEFAULT_SOFTWARE_KEY, "v1"); + }); + } + } +} diff --git a/pubber/src/main/java/daq/pubber/PubberRandomBoolean.java b/pubber/src/main/java/daq/pubber/impl/point/PubberRandomBoolean.java similarity index 96% rename from pubber/src/main/java/daq/pubber/PubberRandomBoolean.java rename to pubber/src/main/java/daq/pubber/impl/point/PubberRandomBoolean.java index e042091121..5b088164e2 100644 --- a/pubber/src/main/java/daq/pubber/PubberRandomBoolean.java +++ b/pubber/src/main/java/daq/pubber/impl/point/PubberRandomBoolean.java @@ -1,4 +1,4 @@ -package daq.pubber; +package daq.pubber.impl.point; import udmi.lib.base.BasicPoint; import udmi.lib.intf.AbstractPoint; diff --git a/pubber/src/main/java/daq/pubber/PubberRandomPoint.java b/pubber/src/main/java/daq/pubber/impl/point/PubberRandomPoint.java similarity index 92% rename from pubber/src/main/java/daq/pubber/PubberRandomPoint.java rename to pubber/src/main/java/daq/pubber/impl/point/PubberRandomPoint.java index 930cb31d94..09176ca412 100644 --- a/pubber/src/main/java/daq/pubber/PubberRandomPoint.java +++ b/pubber/src/main/java/daq/pubber/impl/point/PubberRandomPoint.java @@ -1,4 +1,6 @@ -package daq.pubber; +package daq.pubber.impl.point; + +import static java.lang.String.format; import udmi.lib.base.BasicPoint; import udmi.lib.intf.AbstractPoint; @@ -37,7 +39,7 @@ private double convertValue(Object baselineValue, double defaultBaselineValue) { if (baselineValue instanceof Integer) { return (double) (int) baselineValue; } - throw new RuntimeException("Unknown value type " + baselineValue.getClass()); + throw new RuntimeException(format("Unknown value type %s", baselineValue.getClass())); } @Override diff --git a/pubber/src/main/java/daq/pubber/PubberBacnetProvider.java b/pubber/src/main/java/daq/pubber/impl/provider/PubberBacnetProvider.java similarity index 98% rename from pubber/src/main/java/daq/pubber/PubberBacnetProvider.java rename to pubber/src/main/java/daq/pubber/impl/provider/PubberBacnetProvider.java index 716e93abd7..4b16398734 100644 --- a/pubber/src/main/java/daq/pubber/PubberBacnetProvider.java +++ b/pubber/src/main/java/daq/pubber/impl/provider/PubberBacnetProvider.java @@ -1,4 +1,4 @@ -package daq.pubber; +package daq.pubber.impl.provider; import static com.google.common.base.Preconditions.checkState; import static com.google.udmi.util.GeneralUtils.catchToElse; @@ -20,7 +20,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import udmi.lib.base.ManagerBase; -import udmi.lib.client.LocalnetManager; +import udmi.lib.client.manager.LocalnetManager; import udmi.lib.intf.ManagerHost; import udmi.schema.DiscoveryEvents; import udmi.schema.FamilyLocalnetState; diff --git a/pubber/src/main/java/daq/pubber/PubberFamilyProvider.java b/pubber/src/main/java/daq/pubber/impl/provider/PubberFamilyProvider.java similarity index 88% rename from pubber/src/main/java/daq/pubber/PubberFamilyProvider.java rename to pubber/src/main/java/daq/pubber/impl/provider/PubberFamilyProvider.java index 269d6e29f5..c938862033 100644 --- a/pubber/src/main/java/daq/pubber/PubberFamilyProvider.java +++ b/pubber/src/main/java/daq/pubber/impl/provider/PubberFamilyProvider.java @@ -1,4 +1,4 @@ -package daq.pubber; +package daq.pubber.impl.provider; import com.google.udmi.util.SiteModel; import udmi.lib.intf.FamilyProvider; diff --git a/pubber/src/main/java/daq/pubber/PubberIpProvider.java b/pubber/src/main/java/daq/pubber/impl/provider/PubberIpProvider.java similarity index 83% rename from pubber/src/main/java/daq/pubber/PubberIpProvider.java rename to pubber/src/main/java/daq/pubber/impl/provider/PubberIpProvider.java index c8b1864a77..c9e44a97f3 100644 --- a/pubber/src/main/java/daq/pubber/PubberIpProvider.java +++ b/pubber/src/main/java/daq/pubber/impl/provider/PubberIpProvider.java @@ -1,7 +1,8 @@ -package daq.pubber; +package daq.pubber.impl.provider; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.runtimeExec; +import static java.lang.String.format; import static java.util.Optional.ofNullable; import com.google.common.annotations.VisibleForTesting; @@ -18,7 +19,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import udmi.lib.base.ManagerBase; -import udmi.lib.client.LocalnetManager; +import udmi.lib.client.manager.LocalnetManager; import udmi.lib.intf.ManagerHost; import udmi.schema.FamilyLocalnetState; @@ -52,8 +53,11 @@ public PubberIpProvider(ManagerHost host, String family, String deviceId) { populateInterfaceAddresses(); } + /** + * Get default interface. + */ @VisibleForTesting - static String getDefaultInterfaceStatic(List routeLines) { + public static String getDefaultInterfaceStatic(List routeLines) { AtomicReference currentInterface = new AtomicReference<>(); AtomicInteger currentMaxMetric = new AtomicInteger(Integer.MAX_VALUE); routeLines.forEach(line -> { @@ -69,7 +73,7 @@ static String getDefaultInterfaceStatic(List routeLines) { } } } catch (Exception e) { - throw new RuntimeException("While processing ip route line: " + line, e); + throw new RuntimeException(format("While processing ip route line: %s", line), e); } }); return currentInterface.get(); @@ -97,7 +101,7 @@ static String getDefaultInterfaceStatic(List routeLines) { * */ @VisibleForTesting - static Map getInterfaceAddressesStatic(List strings) { + public static Map getInterfaceAddressesStatic(List strings) { Map interfaceMap = new HashMap<>(); strings.forEach(line -> { for (Pattern pattern : familyPatterns) { @@ -116,13 +120,13 @@ private String getDefaultInterface() { try { routeLines = runtimeExec("ip", "route"); } catch (Exception e) { - error("Could not execute ip route command: " + friendlyStackTrace(e)); + error(format("Could not execute ip route command: %s", friendlyStackTrace(e))); return null; } try { return getDefaultInterfaceStatic(routeLines); } catch (Exception e) { - error("Could not infer default interface: " + friendlyStackTrace(e)); + error(format("Could not infer default interface: %s", friendlyStackTrace(e))); return null; } } @@ -130,7 +134,7 @@ private String getDefaultInterface() { /** * Parse the output of ip route/addr and turn it into a family addr map. * - *

Start with default route with lowest metric, and then parse the interface addresses. + *

Start with default route with the lowest metric, and then parse the interface addresses. * * 

    * peringknife@peringknife-glaptop4:~/udmi$ ip route
@@ -151,9 +155,10 @@ private String getDefaultInterface() {
    */
   private void populateInterfaceAddresses() {
     String defaultInterface = getDefaultInterface();
-    info("Using addresses from default interface: " + defaultInterface + " for family: " + family);
-    Map interfaceAddresses = ofNullable(
-        getInterfaceAddresses(defaultInterface)).orElse(ImmutableMap.of());
+    info(format("Using addresses from default interface: %s for family: %s",
+        defaultInterface, family));
+    Map interfaceAddresses = ofNullable(getInterfaceAddresses(defaultInterface))
+        .orElse(ImmutableMap.of());
     interfaceAddresses.entrySet().forEach(this::addStateMapEntry);
   }
 
@@ -164,7 +169,7 @@ private void addStateMapEntry(Entry entry) {
     }
     FamilyLocalnetState stateEntry = new FamilyLocalnetState();
     stateEntry.addr = entry.getValue();
-    info("Family " + family + " address is " + stateEntry.addr);
+    info(format("Family %s address is %s", family, stateEntry.addr));
     localnetHost.update(family, stateEntry);
   }
 
@@ -176,14 +181,14 @@ private Map getInterfaceAddresses(String defaultInterface) {
     try {
       strings = runtimeExec("ip", "addr", "show", "dev", defaultInterface);
     } catch (Exception e) {
-      error("Could not execute ip addr command: " + friendlyStackTrace(e));
+      error(format("Could not execute ip addr command: %s", friendlyStackTrace(e)));
       return null;
     }
 
     try {
       return getInterfaceAddressesStatic(strings);
     } catch (Exception e) {
-      error("Could not infer interface addresses: " + friendlyStackTrace(e));
+      error(format("Could not infer interface addresses: %s", friendlyStackTrace(e)));
       return null;
     }
   }
diff --git a/pubber/src/main/java/daq/pubber/PubberVendorProvider.java b/pubber/src/main/java/daq/pubber/impl/provider/PubberVendorProvider.java
similarity index 74%
rename from pubber/src/main/java/daq/pubber/PubberVendorProvider.java
rename to pubber/src/main/java/daq/pubber/impl/provider/PubberVendorProvider.java
index a86ed2d32d..22370aa308 100644
--- a/pubber/src/main/java/daq/pubber/PubberVendorProvider.java
+++ b/pubber/src/main/java/daq/pubber/impl/provider/PubberVendorProvider.java
@@ -1,8 +1,9 @@
-package daq.pubber;
+package daq.pubber.impl.provider;
 
 import static com.google.udmi.util.GeneralUtils.catchToNull;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
 import static com.google.udmi.util.GeneralUtils.ifTrueGet;
+import static java.lang.String.format;
 import static java.util.Objects.nonNull;
 import static java.util.Objects.requireNonNull;
 import static java.util.Optional.ofNullable;
@@ -16,7 +17,7 @@
 import java.util.Map.Entry;
 import java.util.function.BiConsumer;
 import udmi.lib.base.ManagerBase;
-import udmi.lib.client.LocalnetManager;
+import udmi.lib.client.manager.LocalnetManager;
 import udmi.lib.intf.ManagerHost;
 import udmi.schema.DiscoveryEvents;
 import udmi.schema.FamilyLocalnetState;
@@ -54,17 +55,28 @@ private Map getDiscoveredRefs(Metadata entry) {
         .collect(toMap(Entry::getKey, Entry::getValue));
   }
 
-  private Entry pointsetToRef(Entry entry) {
-    return new SimpleEntry<>(entry.getValue().ref,
-        PubberDiscoveryManager.getModelPointRef(entry, false));
+  private Entry pointsetToRef(Entry e) {
+    return new SimpleEntry<>(e.getValue().ref, getModelPointRef(e));
+  }
+
+  /**
+   * Get a ref value that describes a point for self enumeration.
+   */
+  private static RefDiscovery getModelPointRef(Entry entry) {
+    RefDiscovery refDiscovery = new RefDiscovery();
+    PointPointsetModel model = entry.getValue();
+    refDiscovery.writable = model.writable;
+    refDiscovery.units = model.units;
+    refDiscovery.point = entry.getKey();
+    return refDiscovery;
   }
 
   private void updateStateAddress() {
     selfAddr = catchToNull(
         () -> siteModel.getMetadata(deviceId).localnet.families.get(VENDOR).addr);
-    ifNotNullThen(selfAddr, x -> {
+    ifNotNullThen(selfAddr, addr -> {
       FamilyLocalnetState stateEntry = new FamilyLocalnetState();
-      stateEntry.addr = selfAddr;
+      stateEntry.addr = addr;
       localnetHost.update(VENDOR, stateEntry);
     });
   }
@@ -77,9 +89,8 @@ public void setSiteModel(SiteModel siteModel) {
 
   @Override
   public void startScan(boolean enumerate, BiConsumer publisher) {
-    requireNonNull(selfAddr, "no local address defined for family " + VENDOR);
-    siteModel.forEachMetadata(
-        entry -> publisher.accept(entry.getKey(), augmentSend(entry, enumerate)));
+    requireNonNull(selfAddr, format("No local address defined for family %s", VENDOR));
+    siteModel.forEachMetadata(e -> publisher.accept(e.getKey(), augmentSend(e, enumerate)));
   }
 
   @Override
diff --git a/pubber/src/main/java/udmi/lib/base/BasicPoint.java b/pubber/src/main/java/udmi/lib/base/BasicPoint.java
index d347ad074b..0ec7583bdc 100644
--- a/pubber/src/main/java/udmi/lib/base/BasicPoint.java
+++ b/pubber/src/main/java/udmi/lib/base/BasicPoint.java
@@ -3,6 +3,7 @@
 import static com.google.udmi.util.GeneralUtils.deepCopy;
 import static com.google.udmi.util.GeneralUtils.getNow;
 import static com.google.udmi.util.GeneralUtils.isTrue;
+import static java.lang.String.format;
 
 import java.util.Objects;
 import udmi.lib.intf.AbstractPoint;
@@ -20,11 +21,12 @@
  */
 public abstract class BasicPoint implements AbstractPoint {
 
-  protected final String name;
   protected final PointPointsetEvents data = new PointPointsetEvents();
   private final PointPointsetState state = new PointPointsetState();
+  protected final String name;
   private final boolean writable;
   private final String pointRef;
+
   protected boolean written;
   private boolean dirty;
 
@@ -133,7 +135,7 @@ public void updateStateConfig(PointPointsetConfig config) {
   @Override
   public RefDiscovery enumerate() {
     RefDiscovery point = new RefDiscovery();
-    point.description = getClass().getSimpleName() + " " + getName();
+    point.description = format("%s %s", getClass().getSimpleName(), getName());
     point.writable = writable ? true : null;
     populateEnumeration(point);
     return point;
@@ -141,7 +143,7 @@ public RefDiscovery enumerate() {
 
   private Entry createEntryFrom(String category, String message) {
     Entry entry = new Entry();
-    entry.detail = String.format("Point %s (writable %s)", name, writable);
+    entry.detail = format("Point %s (writable %s)", name, writable);
     entry.timestamp = getNow();
     entry.message = message;
     entry.category = category;
diff --git a/pubber/src/main/java/udmi/lib/base/ListPublisher.java b/pubber/src/main/java/udmi/lib/base/ListPublisher.java
index f904e87ad7..44e5b61ad4 100644
--- a/pubber/src/main/java/udmi/lib/base/ListPublisher.java
+++ b/pubber/src/main/java/udmi/lib/base/ListPublisher.java
@@ -2,6 +2,7 @@
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+import static java.lang.String.format;
 
 import com.google.udmi.util.JsonUtil;
 import java.util.AbstractMap.SimpleEntry;
@@ -22,15 +23,16 @@
 public class ListPublisher implements Publisher {
 
   private final ExecutorService publisherExecutor = Executors.newFixedThreadPool(1);
+  private final Map, Class>> handlers = new HashMap<>();
+
   private List messages = new ArrayList<>();
   private String usePrefix;
-  private final Map, Class>> handlers = new HashMap<>();
 
   public ListPublisher(Consumer onError) {
   }
 
   public static String getMessageString(String deviceId, String topic, Object message) {
-    return String.format("%s/%s/%s", deviceId, topic, JsonUtil.stringify(message));
+    return format("%s/%s/%s", deviceId, topic, JsonUtil.stringify(message));
   }
 
   /**
@@ -70,7 +72,7 @@ public void connect(String deviceId, boolean clean) {
 
   @Override
   public void publish(String deviceId, String topicSuffix, Object message, Runnable callback) {
-    String useTopic = usePrefix + "/" + topicSuffix;
+    String useTopic = format("%s/%s", usePrefix, topicSuffix);
     messages.add(getMessageString(deviceId, useTopic, message));
     ifNotNullThen(callback, () -> publisherExecutor.submit(callback));
   }
@@ -82,7 +84,6 @@ public boolean isActive() {
 
   @Override
   public void close() {
-
   }
 
   @Override
diff --git a/pubber/src/main/java/udmi/lib/base/ManagerBase.java b/pubber/src/main/java/udmi/lib/base/ManagerBase.java
index 6a4e46a048..6af407e9ba 100644
--- a/pubber/src/main/java/udmi/lib/base/ManagerBase.java
+++ b/pubber/src/main/java/udmi/lib/base/ManagerBase.java
@@ -13,8 +13,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import udmi.lib.client.SubBlockManager;
 import udmi.lib.intf.ManagerHost;
+import udmi.lib.intf.SubBlockManager;
 import udmi.schema.Config;
 import udmi.schema.DiscoveryState;
 import udmi.schema.GatewayState;
@@ -30,20 +30,21 @@
  */
 public abstract class ManagerBase implements SubBlockManager {
 
-  public static final int DISABLED_INTERVAL = 0;
   protected static final int DEFAULT_REPORT_SEC = 10;
+  public static final int DISABLED_INTERVAL = 0;
   public static final int INITIAL_THRESHOLD_SEC = 5;
+
   protected final AtomicInteger sendRateSec = new AtomicInteger(DEFAULT_REPORT_SEC);
-  protected final ManagerHost host;
   protected final Config deviceConfig = new Config();
   protected final State deviceState = new State();
   protected final ScheduledExecutorService executor = new CatchingScheduledThreadPoolExecutor(1);
   protected final AtomicBoolean stateDirty = new AtomicBoolean();
+  private final AtomicInteger eventCount = new AtomicInteger();
+  protected final ManagerHost host;
   protected final String deviceId;
+
   protected ScheduledFuture periodicSender;
   protected ScheduledFuture initialUpdate;
-  private final AtomicInteger eventCount = new AtomicInteger();
-
 
   /**
    * New instance.
@@ -66,7 +67,8 @@ public static void updateStateHolder(State state, Object update) {
     try {
       checkTarget = markerClass ? ((Class) update).getConstructor().newInstance() : update;
     } catch (Exception e) {
-      throw new RuntimeException("Could not create marker instance of class " + update.getClass());
+      throw new RuntimeException(
+        format("Could not create marker instance of class %s", update.getClass()));
     }
     if (checkTarget instanceof SystemState) {
       state.system = (SystemState) checkValue;
@@ -80,7 +82,7 @@ public static void updateStateHolder(State state, Object update) {
       state.discovery = (DiscoveryState) checkValue;
     } else {
       throw new RuntimeException(
-          "Unrecognized update type " + checkTarget.getClass().getSimpleName());
+        format("Unrecognized update type %s", checkTarget.getClass().getSimpleName()));
     }
   }
 
@@ -93,8 +95,8 @@ public void updateState(Object state) {
    * Schedule a future for the futureTask parameter.
    */
   public ScheduledFuture scheduleFuture(Date futureTime, Runnable futureTask) {
-    if (executor.isShutdown() || executor.isTerminated()) {
-      throw new RuntimeException("Executor shutdown/terminated, not scheduling");
+    if (executor.isShutdown()) {
+      throw new RuntimeException("Executor shutdown, not scheduling");
     }
     long delay = Math.max(futureTime.getTime() - getNow().getTime(), 0);
     debug(format("Scheduling future in %dms", delay));
@@ -237,6 +239,10 @@ public State getDeviceState() {
     return deviceState;
   }
 
+  public Config getDeviceConfig() {
+    return deviceConfig;
+  }
+
   public AtomicBoolean getStateDirty() {
     return stateDirty;
   }
diff --git a/pubber/src/main/java/udmi/lib/base/MqttDevice.java b/pubber/src/main/java/udmi/lib/base/MqttDevice.java
index 4d88472b57..2ceaa70a3d 100644
--- a/pubber/src/main/java/udmi/lib/base/MqttDevice.java
+++ b/pubber/src/main/java/udmi/lib/base/MqttDevice.java
@@ -33,8 +33,7 @@ public MqttDevice(EndpointConfiguration configuration, Consumer onErr
     this.certManager = certManager;
     deviceId = requireNonNull(configuration.deviceId, "deviceId not specified");
     publisher = getPublisher(configuration, onError);
-    ifNotNullThen(configuration.topic_prefix,
-        prefix -> publisher.setDeviceTopicPrefix(deviceId, prefix));
+    ifNotNullThen(configuration.topic_prefix, p -> publisher.setDeviceTopicPrefix(deviceId, p));
   }
 
   /**
@@ -51,8 +50,7 @@ public MqttDevice(String deviceId, MqttDevice target) {
     certManager = null;
   }
 
-  Publisher getPublisher(EndpointConfiguration configuration,
-      Consumer onError) {
+  Publisher getPublisher(EndpointConfiguration configuration, Consumer onError) {
     return TEST_PREFIX.equals(configuration.topic_prefix)
         ? new ListPublisher(onError)
         : new MqttPublisher(configuration, onError, certManager);
diff --git a/pubber/src/main/java/udmi/lib/base/MqttPublisher.java b/pubber/src/main/java/udmi/lib/base/MqttPublisher.java
index 5d808fa677..d88a05ff7c 100644
--- a/pubber/src/main/java/udmi/lib/base/MqttPublisher.java
+++ b/pubber/src/main/java/udmi/lib/base/MqttPublisher.java
@@ -98,26 +98,24 @@ public class MqttPublisher implements Publisher {
   private static final String LOCAL_MQTT_PREFIX = "/r/";
 
   private final Semaphore connectionLock = new Semaphore(1);
-
   private final Map mqttClients = new ConcurrentHashMap<>();
-  private final Map reauthTimes = new ConcurrentHashMap<>();
-  ReentrantLock reconnectLock = new ReentrantLock();
-
+  private final Map reAuthTimes = new ConcurrentHashMap<>();
+  private final ReentrantLock reconnectLock = new ReentrantLock();
   private final ExecutorService publisherExecutor =
       Executors.newFixedThreadPool(PUBLISH_THREAD_COUNT);
-
-  private final String registryId;
-  private final String projectId;
-  private final String cloudRegion;
-
   private final AtomicInteger publishCounter = new AtomicInteger(0);
   private final AtomicInteger errorCounter = new AtomicInteger(0);
   private final Map> handlers = new ConcurrentHashMap<>();
   private final Map> handlersType = new ConcurrentHashMap<>();
-  private final Consumer onError;
+
+  private final String registryId;
+  private final String projectId;
+  private final String cloudRegion;
   private final String deviceId;
   private final CertManager certManager;
   private final EndpointConfiguration configuration;
+  private final Consumer onError;
+
   private CountDownLatch connectionLatch;
   private String topicPrefixPrefix;
 
@@ -145,8 +143,7 @@ public MqttPublisher(EndpointConfiguration configuration, Consumer on
   }
 
   private boolean isGcpIotCore(EndpointConfiguration configuration) {
-    return configuration.client_id != null && configuration.client_id.startsWith(
-        GCP_CLIENT_PREFIX);
+    return configuration.client_id != null && configuration.client_id.startsWith(GCP_CLIENT_PREFIX);
   }
 
   private String getClientId(String targetId) {
@@ -170,13 +167,14 @@ public boolean isActive() {
   @Override
   public void publish(String deviceId, String topicSuffix, Object data, Runnable callback) {
     Preconditions.checkNotNull(deviceId, "publish deviceId");
-    debug("Publishing in background " + topicSuffix);
-    Object marked =
-        topicSuffix.startsWith(EVENT_MARK_PREFIX) ? decorateMessage(topicSuffix, data) : data;
+    debug(format("Publishing in background %s", topicSuffix));
+    Object marked = topicSuffix.startsWith(EVENT_MARK_PREFIX)
+        ? decorateMessage(topicSuffix, data)
+        : data;
     try {
       publisherExecutor.submit(() -> publishCore(deviceId, topicSuffix, marked, callback));
     } catch (Exception e) {
-      throw new RuntimeException("While publishing to topic suffix " + topicSuffix, e);
+      throw new RuntimeException(format("While publishing to topic suffix %s", topicSuffix), e);
     }
   }
 
@@ -204,9 +202,9 @@ private void publishCore(String deviceId, String topicSuffix, Object data, Runna
     try {
       String payload = getMessagePayload(data);
       String sendTopic = getSendTopic(deviceId, getMessageTopic(topicSuffix, data));
-      debug("Sending message to " + sendTopic);
+      debug(format("Sending message to %s", sendTopic));
       if (!sendMessage(deviceId, sendTopic, payload.getBytes())) {
-        debug("Queue message for retry");
+        debug(format("Queue message for retry %s %s", topicSuffix, deviceId));
         safeSleep(ATTACH_DELAY_MS);
         if (isActive()) {
           publisherExecutor.submit(() -> publishCore(deviceId, topicSuffix, data, callback));
@@ -217,19 +215,12 @@ private void publishCore(String deviceId, String topicSuffix, Object data, Runna
         callback.run();
       }
     } catch (Exception e) {
-      if (!isActive()) {
-        return;
-      }
-      errorCounter.incrementAndGet();
-      warn(format("Publish %s failed for %s: %s", topicSuffix, deviceId, e));
-      if (getGatewayId() == null) {
-        closeMqttClient(deviceId);
-        if (mqttClients.isEmpty()) {
-          warn("Last client closed, shutting down connection.");
+      if (isActive()) {
+        errorCounter.incrementAndGet();
+        warn(format("Publish %s failed for %s: %s", topicSuffix, deviceId, e));
+        if (!isProxyDevice(deviceId)) {
           reconnect();
         }
-      } else if (getGatewayId().equals(deviceId)) {
-        reconnect();
       }
     }
   }
@@ -248,7 +239,8 @@ private synchronized void reconnect() {
   }
 
   private String getMessageTopic(String deviceId, String topic) {
-    return ofNullable(topicPrefixPrefix).orElse(DEFAULT_TOPIC_PREFIX) + deviceId + "/" + topic;
+    return format("%s%s/%s",
+        ofNullable(topicPrefixPrefix).orElse(DEFAULT_TOPIC_PREFIX), deviceId, topic);
   }
 
   @SuppressWarnings("unchecked")
@@ -284,7 +276,7 @@ private void closeMqttClient(String deviceId) {
           }
           removed.close();
         } catch (Exception e) {
-          error("Error closing MQTT client: " + e, deviceId, null, "stop", e);
+          error(format("Error closing MQTT client: %s", e), deviceId, null, "stop", e);
         }
       }
     }
@@ -333,14 +325,14 @@ private MqttClient newProxyClient(String deviceId) {
     try {
       startupLatchWait(connectionLatch, "gateway startup exchange");
       String topic = getMessageTopic(deviceId, MqttDevice.ATTACH_TOPIC);
-      info("Publishing attach message " + topic);
+      info(format("Publishing attach message %s", topic));
       byte[] mqttMessage = EMPTY_STRING.getBytes(StandardCharsets.UTF_8);
       mqttClient.setTimeToWait(ATTACH_DELAY_MS);
       mqttClientPublish(mqttClient, topic, mqttMessage);
       subscribeToUpdates(mqttClient, deviceId);
       return mqttClient;
     } catch (Exception e) {
-      throw new RuntimeException("While binding client " + deviceId, e);
+      throw new RuntimeException(format("While binding client %s", deviceId), e);
     } finally {
       mqttClient.setTimeToWait(timeToWait);
     }
@@ -353,14 +345,13 @@ private void mqttClientPublish(MqttClient mqttClient, String topic, byte[] mqttM
 
   private void startupLatchWait(CountDownLatch gatewayLatch, String designator) {
     try {
-      int waitTimeSec = ofNullable(configuration.config_sync_sec)
-          .orElse(DEFAULT_CONFIG_WAIT_SEC);
+      int waitTimeSec = ofNullable(configuration.config_sync_sec).orElse(DEFAULT_CONFIG_WAIT_SEC);
       int useWaitTime = waitTimeSec == 0 ? DEFAULT_CONFIG_WAIT_SEC : waitTimeSec;
       if (useWaitTime > 0 && !gatewayLatch.await(useWaitTime, TimeUnit.SECONDS)) {
-        throw new RuntimeException("Latch timeout " + designator);
+        throw new RuntimeException(format("Latch timeout %s", designator));
       }
     } catch (Exception e) {
-      throw new RuntimeException("While waiting for " + designator, e);
+      throw new RuntimeException(format("While waiting for %s", designator), e);
     }
   }
 
@@ -370,11 +361,11 @@ private MqttClient newMqttClient(String deviceId) {
       String clientId = getClientId(deviceId);
       String brokerUrl = getBrokerUrl();
       MqttClient mqttClient = getMqttClient(clientId, brokerUrl);
-      info("Creating new client to " + brokerUrl + " as " + clientId);
+      info(format("Creating new client to %s as %s", brokerUrl, clientId));
       return mqttClient;
     } catch (Exception e) {
       errorCounter.incrementAndGet();
-      throw new RuntimeException("Creating new MQTT client " + deviceId, e);
+      throw new RuntimeException(format("Creating new MQTT client %s", deviceId), e);
     }
   }
 
@@ -398,15 +389,15 @@ private MqttClient newDirectClient(String deviceId) {
       options.setConnectionTimeout(INITIALIZE_TIME_MS);
 
       configureAuth(options);
-      reauthTimes.put(deviceId, Instant.now().plusSeconds(TOKEN_EXPIRY_MINUTES * 60 / 2));
+      reAuthTimes.put(deviceId, Instant.now().plusSeconds(TOKEN_EXPIRY_MINUTES * 60 / 2));
       connectionLatch = new CountDownLatch(1);
 
-      info("Attempting connection to " + getClientId(deviceId));
+      info(format("Attempting connection to %s", getClientId(deviceId)));
       mqttClient.connect(options);
       subscribeToUpdates(mqttClient, deviceId);
       return mqttClient;
     } catch (Exception e) {
-      throw new RuntimeException("While connecting mqtt client " + deviceId, e);
+      throw new RuntimeException(format("While connecting mqtt client %s", deviceId), e);
     } finally {
       connectionLock.release();
     }
@@ -417,7 +408,7 @@ private SocketFactory getSocketFactory() {
         .orElse(SSLSocketFactory.getDefault());
   }
 
-  private void configureAuth(MqttConnectOptions options) throws Exception {
+  private void configureAuth(MqttConnectOptions options) {
     options.setSocketFactory(getSocketFactory());
     if (configuration.auth_provider == null) {
       info("No endpoint auth_provider found, using gcp defaults");
@@ -431,17 +422,17 @@ private void configureAuth(MqttConnectOptions options) throws Exception {
     }
   }
 
-  private void configureAuth(MqttConnectOptions options, Jwt jwt) throws Exception {
+  private void configureAuth(MqttConnectOptions options, Jwt jwt) {
     String audience = jwt == null ? projectId : jwt.audience;
-    info("Auth using audience " + audience);
+    info(format("Auth using audience %s", audience));
     options.setUserName(UNUSED_ACCOUNT_NAME);
-    info("Key hash " + Hashing.sha256().hashBytes((byte[]) configuration.keyBytes));
+    info(format("Key hash %s", Hashing.sha256().hashBytes((byte[]) configuration.keyBytes)));
     options.setPassword(createJwt(audience, (byte[]) configuration.keyBytes,
         configuration.algorithm).toCharArray());
   }
 
   private void configureAuth(MqttConnectOptions options, Basic basic) {
-    info("Auth using username " + basic.username);
+    info(format("Auth using username %s", basic.username));
     options.setUserName(basic.username);
     options.setPassword(basic.password.toCharArray());
   }
@@ -449,17 +440,15 @@ private void configureAuth(MqttConnectOptions options, Basic basic) {
   /**
    * Create a Cloud IoT JWT for the given project id, signed with the given private key.
    */
-  private String createJwt(String audience, byte[] privateKeyBytes, String algorithm)
-      throws Exception {
+  private String createJwt(String audience, byte[] privateKeyBytes, String algorithm) {
     DateTime now = new DateTime();
     // Create a JWT to authenticate this device. The device will be disconnected after the token
     // expires, and will have to reconnect with a new token. The audience field should always be set
     // to the GCP project id.
-    JwtBuilder jwtBuilder =
-        Jwts.builder()
-            .setIssuedAt(now.toDate())
-            .setExpiration(now.plusMinutes(TOKEN_EXPIRY_MINUTES).toDate())
-            .setAudience(audience);
+    JwtBuilder jwtBuilder = Jwts.builder()
+        .setIssuedAt(now.toDate())
+        .setExpiration(now.plusMinutes(TOKEN_EXPIRY_MINUTES).toDate())
+        .setAudience(audience);
 
     if (algorithm.equals("RS256") || algorithm.equals("RS256_X509")) {
       PrivateKey privateKey = loadKeyBytes(privateKeyBytes, "RSA");
@@ -469,7 +458,7 @@ private String createJwt(String audience, byte[] privateKeyBytes, String algorit
       return jwtBuilder.signWith(SignatureAlgorithm.ES256, privateKey).compact();
     } else {
       throw new IllegalArgumentException(
-          "Invalid algorithm " + algorithm + ". Should be one of 'RS256' or 'ES256'.");
+          format("Invalid algorithm %s should be one of 'RS256' or 'ES256'.", algorithm));
     }
   }
 
@@ -482,8 +471,7 @@ private String getBrokerUrl() {
   }
 
   private void subscribeToUpdates(MqttClient client, String deviceId) {
-    int configQos =
-        isTrue(configuration.noConfigAck) ? QOS_AT_MOST_ONCE : QOS_AT_LEAST_ONCE;
+    int configQos = isTrue(configuration.noConfigAck) ? QOS_AT_MOST_ONCE : QOS_AT_LEAST_ONCE;
     if (configuration.recv_id == null) {
       subscribeTopic(client, getMessageTopic(deviceId, MqttDevice.CONFIG_TOPIC), configQos);
       subscribeTopic(client, getMessageTopic(deviceId, MqttDevice.ERRORS_TOPIC), QOS_AT_MOST_ONCE);
@@ -497,7 +485,7 @@ private void subscribeTopic(MqttClient client, String updateTopic, int mqttQos)
       client.subscribe(updateTopic, mqttQos);
       info(format("Subscribed to mqtt topic %s (qos %d)", updateTopic, mqttQos));
     } catch (MqttException e) {
-      throw new RuntimeException("While subscribing to MQTT topic " + updateTopic, e);
+      throw new RuntimeException(format("While subscribing to MQTT topic %s", updateTopic), e);
     }
   }
 
@@ -570,8 +558,8 @@ private void debug(String message) {
     LOG.debug(message);
   }
 
-  private boolean sendMessage(String deviceId, String mqttTopic,
-      byte[] mqttMessage) throws Exception {
+  private boolean sendMessage(String deviceId, String mqttTopic, byte[] mqttMessage)
+      throws Exception {
     MqttClient connectedClient = getActiveClient(deviceId);
     if (connectedClient == null) {
       return false;
@@ -602,12 +590,12 @@ private void safeSleep(long timeoutMs) {
 
   private boolean checkAuthentication(String targetId) {
     String authId = ofNullable(getGatewayId()).orElse(targetId);
-    Instant reAuthTime = reauthTimes.get(authId);
+    Instant reAuthTime = reAuthTimes.get(authId);
     if (reAuthTime == null || Instant.now().isBefore(reAuthTime)) {
       return true;
     }
-    warn("Authentication retry time reached for " + authId);
-    reauthTimes.remove(authId);
+    warn(format("Authentication retry time reached for %s", authId));
+    reAuthTimes.remove(authId);
     reconnect();
     return false;
   }
@@ -621,7 +609,7 @@ private MqttClient getConnectedClient(String deviceId) {
         return mqttClients.computeIfAbsent(deviceId, this::newDirectClient);
       }
     } catch (Exception e) {
-      throw new RuntimeException("While getting mqtt client " + deviceId + ": " + e, e);
+      throw new RuntimeException(format("While getting mqtt client %s : %s", deviceId, e), e);
     }
   }
 
@@ -637,7 +625,7 @@ private String getGatewayId() {
   /**
    * Load a PKCS8 encoded keyfile from the given path.
    */
-  private PrivateKey loadKeyBytes(byte[] keyBytes, String algorithm) throws Exception {
+  private PrivateKey loadKeyBytes(byte[] keyBytes, String algorithm) {
     try {
       PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
       KeyFactory kf = KeyFactory.getInstance(algorithm);
@@ -714,7 +702,6 @@ public static class FakeTopic {
    * Injected state.
    */
   public static class InjectedState extends InjectedMessage {
-
   }
 
   private class MqttCallbackHandler implements MqttCallback {
@@ -755,8 +742,8 @@ private void messageArrivedCore(String topic, MqttMessage message) {
       Consumer handler = handlers.get(handlerKey);
       Class type = handlersType.get(handlerKey);
       if (handler == null) {
-        error("Missing handler " + handlerKey, deviceId, messageType, "receive",
-            new RuntimeException("No registered handler for topic " + topic));
+        error(format("Missing handler %s", handlerKey), deviceId, messageType, "receive",
+            new RuntimeException(format("No registered handler for topic %s", topic)));
         handlersType.put(handlerKey, Object.class);
         handlers.put(handlerKey, this::ignoringHandler);
         return;
@@ -765,7 +752,7 @@ private void messageArrivedCore(String topic, MqttMessage message) {
 
       final Object payload;
       try {
-        if (message.toString().length() == 0) {
+        if (message.toString().isEmpty()) {
           payload = null;
         } else {
           payload = OBJECT_MAPPER.readValue(message.toString(), type);
diff --git a/pubber/src/main/java/udmi/lib/client/GatewayManager.java b/pubber/src/main/java/udmi/lib/client/GatewayManager.java
deleted file mode 100644
index 0534a23dee..0000000000
--- a/pubber/src/main/java/udmi/lib/client/GatewayManager.java
+++ /dev/null
@@ -1,134 +0,0 @@
-package udmi.lib.client;
-
-import static com.google.udmi.util.GeneralUtils.catchToNull;
-import static com.google.udmi.util.GeneralUtils.getNow;
-import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
-import static java.lang.String.format;
-import static java.util.Optional.ofNullable;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.concurrent.ConcurrentHashMap;
-import udmi.lib.ProtocolFamily;
-import udmi.lib.intf.ManagerHost;
-import udmi.schema.Config;
-import udmi.schema.Entry;
-import udmi.schema.GatewayConfig;
-import udmi.schema.GatewayState;
-import udmi.schema.Level;
-import udmi.schema.Metadata;
-import udmi.schema.PointPointsetConfig;
-import udmi.schema.PointsetConfig;
-
-/**
- * Gateway client.
- */
-public interface GatewayManager extends SubBlockManager {
-
-  String EXTRA_PROXY_DEVICE = "XXX-1";
-  String EXTRA_PROXY_POINT = "xxx_conflagration";
-  Metadata getMetadata();
-
-  void setMetadata(Metadata metadata);
-
-  GatewayState getGatewayState();
-
-  Map getProxyDevices();
-
-  /**
-   * Creates a map of proxy devices.
-   *
-   * @param proxyIds A list of device IDs to create proxies for.
-   * @return A map where each key-value pair represents a device ID and its corresponding proxy
-   * @throws NoSuchElementException if no first element exists in the stream
-   */
-  default Map createProxyDevices(List proxyIds) {
-    if (proxyIds == null) {
-      return Map.of();
-    }
-
-    Map devices = new ConcurrentHashMap<>();
-    proxyIds.forEach(id -> devices.put(id, createProxyDevice(getHost(), id)));
-
-    return devices;
-  }
-
-  ProxyDeviceHost createProxyDevice(ManagerHost host, String id);
-
-  ProxyDeviceHost makeExtraDevice();
-
-  default void activate() {
-    throw new UnsupportedOperationException("Not supported yet.");
-  }
-
-  /**
-   * Publish log message for target device.
-   */
-  default void publishLogMessage(Entry logEntry, String targetId) {
-    ifNotNullThen(getProxyDevices(), p ->
-          ifNotNullThen(p.getOrDefault(targetId, null), pd ->
-                pd.getDeviceManager().publishLogMessage(logEntry, targetId)));
-  }
-
-  /**
-   * Set device status for target device.
-   */
-  default void setStatus(Entry report, String targetId) {
-    ifNotNullThen(getProxyDevices(), p ->
-          ifNotNullThen(p.getOrDefault(targetId, null), pd ->
-                pd.getDeviceManager().setStatus(report, targetId)));
-  }
-
-  /**
-   * Sets gateway status.
-   *
-   */
-  default void setGatewayStatus(String category, Level level, String message) {
-    // TODO: Implement a map or tree or something to properly handle different error sources.
-    getGatewayState().status = new Entry();
-    getGatewayState().status.category = category;
-    getGatewayState().status.level = level.value();
-    getGatewayState().status.message = message;
-    getGatewayState().status.timestamp = getNow();
-  }
-
-  /**
-   * Updates the state of the gateway.
-   */
-  default void updateState() {
-    updateState(ofNullable((Object) getGatewayState()).orElse(GatewayState.class));
-  }
-
-  /**
-   * Validates the given gateway family.
-   */
-  default void validateGatewayFamily(String family, String addr) {
-    if (!ProtocolFamily.FAMILIES.contains(family)) {
-      throw new IllegalArgumentException("Unrecognized address family " + family);
-    }
-
-    String expectedAddr = catchToNull(() -> getMetadata().localnet.families.get(family).addr);
-
-    if (expectedAddr != null && !expectedAddr.equals(addr)) {
-      throw new IllegalStateException(
-          format("Family address was %s, expected %s", addr, expectedAddr));
-    }
-  }
-
-  /**
-   * Configures the extra device with default settings.
-   *
-   */
-  default void configExtraDevice() {
-    Config config = new Config();
-    config.pointset = new PointsetConfig();
-    config.pointset.points = new HashMap<>();
-    PointPointsetConfig pointPointsetConfig = new PointPointsetConfig();
-    config.pointset.points.put(EXTRA_PROXY_POINT, pointPointsetConfig);
-    getProxyDevices().get(EXTRA_PROXY_DEVICE).configHandler(config);
-  }
-
-  void updateConfig(GatewayConfig gateway);
-}
diff --git a/pubber/src/main/java/udmi/lib/client/ProxyDeviceHost.java b/pubber/src/main/java/udmi/lib/client/host/ProxyHost.java
similarity index 67%
rename from pubber/src/main/java/udmi/lib/client/ProxyDeviceHost.java
rename to pubber/src/main/java/udmi/lib/client/host/ProxyHost.java
index 6acb68e7b6..d387f658ca 100644
--- a/pubber/src/main/java/udmi/lib/client/ProxyDeviceHost.java
+++ b/pubber/src/main/java/udmi/lib/client/host/ProxyHost.java
@@ -1,4 +1,4 @@
-package udmi.lib.client;
+package udmi.lib.client.host;
 
 import static com.google.udmi.util.GeneralUtils.friendlyStackTrace;
 import static com.google.udmi.util.JsonUtil.safeSleep;
@@ -6,12 +6,13 @@
 import static udmi.lib.base.ManagerBase.updateStateHolder;
 import static udmi.lib.base.MqttPublisher.DEFAULT_CONFIG_WAIT_SEC;
 
+import java.util.Date;
 import java.util.concurrent.atomic.AtomicBoolean;
 import udmi.lib.base.MqttDevice;
+import udmi.lib.client.manager.DeviceManager;
 import udmi.lib.intf.FamilyProvider;
 import udmi.lib.intf.ManagerHost;
 import udmi.lib.intf.ManagerLog;
-import udmi.lib.intf.UdmiPublisher;
 import udmi.schema.Config;
 import udmi.schema.Metadata;
 import udmi.schema.State;
@@ -19,11 +20,13 @@
 /**
  * Proxy Device host provider.
  */
-public interface ProxyDeviceHost extends ManagerHost, ManagerLog {
+public interface ProxyHost extends ManagerHost, ManagerLog {
+
+  int STATE_INTERVAL_SEC = 1;
 
   DeviceManager getDeviceManager();
 
-  UdmiPublisher getUdmiPublisher();
+  PublisherHost getPublisherHost();
 
   ManagerHost getManagerHost();
 
@@ -38,11 +41,11 @@ public interface ProxyDeviceHost extends ManagerHost, ManagerLog {
    * Logs information or errors based on the success or failure of these operations.
    */
   default void activate() {
-    while (getUdmiPublisher().isConnected() && !isActive().get()) {
+    while (getPublisherHost().isConnected() && !isActive().get()) {
       try {
         isActive().set(false);
-        info("Activating proxy device " + getDeviceId());
-        MqttDevice mqttDevice = getUdmiPublisher().getMqttDevice(getDeviceId());
+        info(format("Activating proxy device %s", getDeviceId()));
+        MqttDevice mqttDevice = getPublisherHost().getMqttDevice(getDeviceId());
         if (mqttDevice == null) {
           throw new RuntimeException("Publisher is not connected");
         }
@@ -64,19 +67,39 @@ default void activate() {
    * @param config The configuration to be applied.
    */
   default void configHandler(Config config) {
-    getUdmiPublisher().configPreprocess(getDeviceId(), config);
+    getPublisherHost().configPreprocess(getDeviceId(), config);
     getDeviceManager().updateConfig(config);
-    getUdmiPublisher().publisherConfigLog("apply", null, getDeviceId());
+    getPublisherHost().publisherConfigLog("apply", null, getDeviceId());
+  }
+
+  default void shutdown() {
+    getDeviceManager().shutdown();
+    isActive().set(false);
   }
 
-  void shutdown();
+  default void stop() {
+    getDeviceManager().stop();
+    isActive().set(false);
+  }
+
+  /**
+   * Publish dirty state.
+   */
+  default void publishDirtyState() {
+    if (getStateDirty().getAndSet(false)) {
+      publish(getDeviceId(), getDeviceState());
+    }
+  }
 
-  void stop();
+  @Override
+  default void publish(String targetId, Object message) {
+    publish(message);
+  }
 
   @Override
   default void publish(Object message) {
     if (isActive().get()) {
-      getUdmiPublisher().publish(getDeviceId(), message);
+      getPublisherHost().publish(getDeviceId(), message);
     }
   }
 
@@ -99,4 +122,9 @@ default void setMetadata(Metadata metadata) {
   AtomicBoolean getStateDirty();
 
   State getDeviceState();
+
+  @Override
+  default Date getStartTime() {
+    return getDeviceManager().getSystemManager().getStartTime();
+  }
 }
diff --git a/pubber/src/main/java/daq/pubber/PubberUdmiPublisher.java b/pubber/src/main/java/udmi/lib/client/host/PublisherHost.java
similarity index 69%
rename from pubber/src/main/java/daq/pubber/PubberUdmiPublisher.java
rename to pubber/src/main/java/udmi/lib/client/host/PublisherHost.java
index ca42f9a0d6..a11a0d9433 100644
--- a/pubber/src/main/java/daq/pubber/PubberUdmiPublisher.java
+++ b/pubber/src/main/java/udmi/lib/client/host/PublisherHost.java
@@ -1,4 +1,4 @@
-package daq.pubber;
+package udmi.lib.client.host;
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.udmi.util.GeneralUtils.catchToNull;
@@ -9,9 +9,7 @@
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
 import static com.google.udmi.util.GeneralUtils.ifTrueThen;
-import static com.google.udmi.util.GeneralUtils.isGetTrue;
-import static com.google.udmi.util.GeneralUtils.isNotTrue;
-import static com.google.udmi.util.GeneralUtils.isTrue;
+import static com.google.udmi.util.GeneralUtils.setClockSkew;
 import static com.google.udmi.util.GeneralUtils.stackTraceString;
 import static com.google.udmi.util.GeneralUtils.toJsonFile;
 import static com.google.udmi.util.GeneralUtils.toJsonString;
@@ -27,15 +25,17 @@
 import static udmi.lib.base.MqttDevice.ERRORS_TOPIC;
 import static udmi.lib.base.MqttDevice.STATE_TOPIC;
 import static udmi.lib.base.MqttPublisher.DEFAULT_CONFIG_WAIT_SEC;
-import static udmi.lib.client.SystemManager.UDMI_PUBLISHER_LOG_CATEGORY;
+import static udmi.lib.client.manager.SystemManager.UDMI_PUBLISHER_LOG_CATEGORY;
+import static udmi.schema.BlobBlobsetConfig.BlobPhase.FINAL;
 import static udmi.schema.BlobsetConfig.SystemBlobsets.IOT_ENDPOINT_CONFIG;
+import static udmi.schema.Envelope.SubType.STATE;
 
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMap.Builder;
 import com.google.udmi.util.GeneralUtils;
 import com.google.udmi.util.MessageDowngrader;
 import com.google.udmi.util.SiteModel;
 import java.io.File;
+import java.io.PrintStream;
 import java.lang.reflect.Field;
 import java.time.Duration;
 import java.time.Instant;
@@ -49,25 +49,22 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
-import java.util.function.Consumer;
-import java.util.function.Function;
 import org.apache.http.ConnectionClosedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import udmi.lib.base.GatewayError;
 import udmi.lib.base.MqttDevice;
-import udmi.lib.base.MqttPublisher.FakeTopic;
-import udmi.lib.base.MqttPublisher.InjectedMessage;
-import udmi.lib.base.MqttPublisher.InjectedState;
-import udmi.lib.base.MqttPublisher.PublisherException;
-import udmi.lib.client.DeviceManager;
-import udmi.lib.client.PointsetManager;
-import udmi.lib.client.PointsetManager.ExtraPointsetEvent;
-import udmi.lib.client.SystemManager.ExtraSystemState;
-import udmi.lib.intf.UdmiPublisher;
+import udmi.lib.base.MqttPublisher;
+import udmi.lib.client.manager.DeviceManager;
+import udmi.lib.client.manager.PointsetManager;
+import udmi.lib.client.manager.SystemManager;
+import udmi.lib.intf.FamilyProvider;
+import udmi.lib.intf.ManagerHost;
 import udmi.schema.BlobBlobsetConfig;
-import udmi.schema.BlobBlobsetConfig.BlobPhase;
 import udmi.schema.BlobBlobsetState;
-import udmi.schema.BlobsetConfig.SystemBlobsets;
+import udmi.schema.BlobsetConfig;
 import udmi.schema.BlobsetState;
 import udmi.schema.Category;
 import udmi.schema.Config;
@@ -75,88 +72,61 @@
 import udmi.schema.DiscoveryEvents;
 import udmi.schema.EndpointConfiguration;
 import udmi.schema.Entry;
-import udmi.schema.Envelope.SubType;
 import udmi.schema.Level;
-import udmi.schema.Operation.SystemMode;
+import udmi.schema.Operation;
 import udmi.schema.PointsetEvents;
-import udmi.schema.PubberConfiguration;
-import udmi.schema.PubberOptions;
 import udmi.schema.State;
 import udmi.schema.SystemEvents;
 import udmi.schema.SystemState;
 import udmi.util.SchemaVersion;
 
 /**
- * UDMI publisher client.
+ * Abstract interface for some kind of publishing stuff.
  */
-public interface PubberUdmiPublisher extends UdmiPublisher {
-
-  String DATA_URL_JSON_BASE64 = "data:application/json;base64,";
+public interface PublisherHost extends ManagerHost {
 
+  String LOG_FILE = "device.log";
+  Logger LOG = LoggerFactory.getLogger(PublisherHost.class);
   String UDMI_VERSION = SchemaVersion.CURRENT.key();
-  Date DEVICE_START_TIME = getRoundedStartTime();
   String BROKEN_VERSION = "1.4.";
+  String DATA_URL_JSON_BASE64 = "data:application/json;base64,";
   int STATE_THROTTLE_MS = 2000;
   int DEFAULT_REPORT_SEC = 10;
+  int FORCED_STATE_TIME_MS = 10000;
+  long INJECT_MESSAGE_DELAY_MS = 1000; // Delay to make sure testing is stable.
+  long RESTART_DELAY_MS = 1000;
+  Duration SMOKE_CHECK_TIME = Duration.ofMinutes(5);
+  int STATE_SPAM_SEC = 5; // Expected config-state response time.
+  Duration CLOCK_SKEW = Duration.ofMinutes(30);
   String SYSTEM_CATEGORY_FORMAT = "system.%s.%s";
+  String RAW_EVENT_TOPIC = "events";
+  String SYSTEM_EVENT_TOPIC = "events/system";
   ImmutableMap, String> MESSAGE_TOPIC_SUFFIX_MAP =
-      new Builder, String>()
+      new ImmutableMap.Builder, String>()
           .put(State.class, STATE_TOPIC)
-          .put(ExtraSystemState.class, STATE_TOPIC) // Used for badState option
+          .put(SystemManager.ExtraSystemState.class, STATE_TOPIC) // Used for badState option
           .put(SystemEvents.class, getEventsSuffix("system"))
           .put(PointsetEvents.class, getEventsSuffix("pointset"))
-          .put(ExtraPointsetEvent.class, getEventsSuffix("pointset"))
-          .put(InjectedMessage.class, getEventsSuffix("system"))
-          .put(FakeTopic.class, getEventsSuffix("racoon"))
-          .put(InjectedState.class, STATE_TOPIC)
+          .put(PointsetManager.ExtraPointsetEvent.class, getEventsSuffix("pointset"))
+          .put(MqttPublisher.InjectedMessage.class, getEventsSuffix("system"))
+          .put(MqttPublisher.FakeTopic.class, getEventsSuffix("racoon"))
+          .put(MqttPublisher.InjectedState.class, STATE_TOPIC)
           .put(DiscoveryEvents.class, getEventsSuffix("discovery"))
           .build();
   Map INVALID_REPLACEMENTS = ImmutableMap.of(
       "events/blobset", "\"\"",
       "events/discovery", "{}",
       "events/gateway", "{ \"testing\": \"This is prematurely terminated",
-      "events/mapping", "{ NOT VALID JSON!"
-  );
+      "events/mapping", "{ NOT VALID JSON!");
   List INVALID_KEYS = new ArrayList<>(INVALID_REPLACEMENTS.keySet());
   String CORRUPT_STATE_MESSAGE = "!&*@(!*&@!";
-  long INJECT_MESSAGE_DELAY_MS = 1000; // Delay to make sure testing is stable.
-  int FORCED_STATE_TIME_MS = 10000;
-  Duration SMOKE_CHECK_TIME = Duration.ofMinutes(5);
-  String RAW_EVENT_TOPIC = "events";
-  String SYSTEM_EVENT_TOPIC = "events/system";
-
-  State getDeviceState();
-
-  Config getDeviceConfig();
-
-  DeviceManager getDeviceManager();
-
-  MqttDevice getDeviceTarget();
-
-  void setDeviceTarget(MqttDevice deviceTarget);
-
-  boolean isGatewayDevice();
-
-  static String getEventsSuffix(String suffixSuffix) {
-    return MqttDevice.EVENTS_TOPIC + "/" + suffixSuffix;
-  }
-
-  /**
-   * Retrieves the start time of the current second, with milliseconds removed for precise
-   * comparison.
-   */
-  static Date getRoundedStartTime() {
-    long timestamp = getNow().getTime();
-    // Remove ms so that rounded conversions preserve equality.
-    return new Date(timestamp - (timestamp % 1000));
-  }
 
   /**
    * Acquires and validates blob data from a given URL encoded in Base64 format.
    */
   static String acquireBlobData(String url, String sha256) {
     if (!url.startsWith(DATA_URL_JSON_BASE64)) {
-      throw new RuntimeException("URL encoding not supported: " + url);
+      throw new RuntimeException(format("URL encoding not supported: %s", url));
     }
     byte[] dataBytes = Base64.getDecoder().decode(url.substring(DATA_URL_JSON_BASE64.length()));
     String dataSha256 = GeneralUtils.sha256(dataBytes);
@@ -166,6 +136,54 @@ static String acquireBlobData(String url, String sha256) {
     return new String(dataBytes);
   }
 
+  Config getDeviceConfig();
+
+  /**
+   * Extracts the configuration blob with the specified name, if it exists and is in the final
+   * phase.
+   */
+  default String extractConfigBlob(String blobName) {
+    // TODO: Refactor to get any blob meta parameters.
+    try {
+      HashMap blobs = catchToNull(() -> getDeviceConfig().blobset.blobs);
+      if (blobs == null) {
+        return null;
+      }
+      BlobBlobsetConfig blobBlobsetConfig = blobs.get(blobName);
+      if (blobBlobsetConfig != null && FINAL.equals(blobBlobsetConfig.phase)) {
+        return acquireBlobData(blobBlobsetConfig.url, blobBlobsetConfig.sha256);
+      }
+      return null;
+    } catch (Exception e) {
+      EndpointConfiguration endpointConfiguration = new EndpointConfiguration();
+      endpointConfiguration.error = e.toString();
+      return stringify(endpointConfiguration);
+    }
+  }
+
+  default boolean isConnected() {
+    return getDeviceTarget() != null && getDeviceTarget().isActive();
+  }
+
+  DeviceManager getDeviceManager();
+
+  @Override
+  default Date getStartTime() {
+    return getDeviceManager().getSystemManager().getStartTime();
+  }
+
+  State getDeviceState();
+
+  MqttDevice getDeviceTarget();
+
+  void setDeviceTarget(MqttDevice deviceTarget);
+
+  boolean isGatewayDevice();
+
+  static String getEventsSuffix(String suffixSuffix) {
+    return format("%s/%s", MqttDevice.EVENTS_TOPIC, suffixSuffix);
+  }
+
   /**
    * Augments a given {@code message} object with the current timestamp and version information.
    */
@@ -188,7 +206,38 @@ default void markStateDirty() {
     markStateDirty(0);
   }
 
-  void markStateDirty(long delayMs);
+  /**
+   * Mark state dirty.
+   */
+  default void markStateDirty(long delayMs) {
+    getStateDirty().set(true);
+    if (delayMs >= 0) {
+      try {
+        schedule(delayMs, this::flushDirtyState);
+      } catch (Exception e) {
+        LOG.error("Rejecting state publish after {}", delayMs, e);
+      }
+    }
+  }
+
+  /**
+   * Periodic update.
+   */
+  default void periodicUpdate() {
+    synchronized (this) {
+      try {
+        incrementDeviceUpdateCount();
+        checkSmokyFailure();
+        deferredConfigActions();
+        sendEmptyMissingBadEvents();
+        maybeTweakState();
+        flushDirtyState();
+      } catch (Exception e) {
+        error("Fatal error during execution", e);
+        resetConnection(getWorkingEndpoint());
+      }
+    }
+  }
 
   /**
    * Publishes a dirty state by resetting the internal state flag to clean.
@@ -209,7 +258,7 @@ default void update(Object update) {
     updateStateHolder(getDeviceState(), update);
     markStateDirty();
     if (update instanceof SystemState) {
-      ifTrueThen(getOptions().dupeState, this::sendPartialState);
+      ifTrueThen(isDupeState(), this::sendPartialState);
     }
   }
 
@@ -245,24 +294,19 @@ default void captureExceptions(String action, Runnable runnable) {
   default void disconnectMqtt() {
     if (getDeviceTarget() != null) {
       captureExceptions("Shutting down MQTT publisher executor",
-          () -> getDeviceTarget().shutdown());
+              () -> getDeviceTarget().shutdown());
       captureExceptions("Closing MQTT publisher", () -> getDeviceTarget().close());
       setDeviceTarget(null);
     }
   }
 
-  static String getGatewayId(String targetId, PubberConfiguration configuration) {
-    return ofNullable(configuration.gatewayId).orElse(
-        targetId.equals(configuration.deviceId) ? null : configuration.deviceId);
-  }
-
   /**
    * Registers the necessary message handlers for device configuration and error handling based on
    * whether the device is a gateway or a proxy device.
    */
   default void registerMessageHandlers() {
     getDeviceTarget().registerHandler(CONFIG_TOPIC, this::configHandler, Config.class);
-    String gatewayId = getGatewayId(getDeviceId(), getConfig());
+    String gatewayId = getGatewayId(getDeviceId());
     if (isGatewayDevice()) {
       // In this case, this is the gateway so register the appropriate error handler directly.
       getDeviceTarget().registerHandler(ERRORS_TOPIC, this::errorHandler, GatewayError.class);
@@ -292,11 +336,11 @@ default MqttDevice getMqttDevice(String proxyId) {
    */
   default void connect() {
     try {
-      warn("Creating new config latch for " + getDeviceId());
+      warn(format("Creating new config latch for %s", getDeviceId()));
       setConfigLatch(new CountDownLatch(1));
       getDeviceTarget().connect();
       info("Connection complete.");
-      setWorkingEndpoint(toJsonString(getConfig().endpoint));
+      setWorkingEndpoint(toJsonString(getEndpoint()));
     } catch (Exception e) {
       throw new RuntimeException("Connection error", e);
     }
@@ -321,31 +365,31 @@ default void publisherConfigLog(String phase, Exception e, String deviceId) {
    */
   default void publisherHandler(String type, String phase, Throwable cause, String targetId) {
     if (cause != null) {
-      error("Error receiving message " + type, cause);
-      if (isTrue(getConfig().options.barfConfig)) {
+      error(format("Error receiving message %s", type), cause);
+      if (isBarfConfig()) {
         error("Restarting system because of restart-on-error configuration setting");
-        getDeviceManager().systemLifecycle(SystemMode.RESTART);
+        getDeviceManager().systemLifecycle(Operation.SystemMode.RESTART);
         return;
       }
     }
-    String usePhase = isTrue(getOptions().badCategory) ? "apply" : phase;
-    String category = type == null ? UDMI_PUBLISHER_LOG_CATEGORY :
-        format(SYSTEM_CATEGORY_FORMAT, type, usePhase);
+    String usePhase = isBadCategory() ? "apply" : phase;
+    String category = type == null
+        ? UDMI_PUBLISHER_LOG_CATEGORY
+        : format(SYSTEM_CATEGORY_FORMAT, type, usePhase);
     Entry report = entryFromException(category, cause);
     getDeviceManager().localLog(report);
     publishLogMessage(report, targetId);
     registerSystemStatus(report, targetId);
   }
 
-  void error(String s);
-
   /**
    * Register a system status entry.
    */
   default void registerSystemStatus(Entry report, String targetId) {
-    if (isNotTrue(getOptions().noStatus)) {
-      getDeviceManager().setStatus(report, targetId);
+    if (isNoStatus()) {
+      return;
     }
+    getDeviceManager().setStatus(report, targetId);
   }
 
   /**
@@ -354,7 +398,7 @@ default void registerSystemStatus(Entry report, String targetId) {
    * this case appropriately.
    */
   default void publishConfigStateUpdate() {
-    if (isTrue(getConfig().options.configStateDelay)) {
+    if (isConfigStateDelay()) {
       delayNextStateUpdate();
     }
     publishAsynchronousState();
@@ -388,7 +432,7 @@ default Entry entryFromException(String category, Throwable e) {
     entry.category = category;
     entry.timestamp = getNow();
     entry.message = success ? "success"
-        : e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
+            : e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
     entry.detail = success ? null : exceptionDetail(e);
     Level successLevel = Category.LEVEL.computeIfAbsent(category, key -> Level.INFO);
     entry.level = (success ? successLevel : Level.ERROR).value();
@@ -413,9 +457,9 @@ default void configHandler(Config config) {
     try {
       configPreprocess(getDeviceId(), config);
       debug(format("Config update %s%s", getDeviceId(), getDeviceManager().getTestingTag()),
-          toJsonString(config));
+              toJsonString(config));
       if (getConfigLatch().getCount() > 0) {
-        warn("Received config for config latch " + getDeviceId());
+        warn(format("Received config for config latch %s", getDeviceId()));
         getConfigLatch().countDown();
       }
       processConfigUpdate(config);
@@ -429,7 +473,7 @@ default void configHandler(Config config) {
   CountDownLatch getConfigLatch();
 
   private void gatewayHandler(Config gatewayConfig) {
-    warn("Ignoring configuration for gateway " + getGatewayId(getDeviceId(), getConfig()));
+    warn(format("Ignoring configuration for gateway %s", getGatewayId(getDeviceId())));
   }
 
   private void errorHandler(GatewayError error) {
@@ -443,11 +487,12 @@ private void errorHandler(GatewayError error) {
    * @param configMsg The configuration message to be processed.
    */
   default void configPreprocess(String targetId, Config configMsg) {
-    String gatewayId = getGatewayId(targetId, getConfig());
-    String suffix = ifNotNullGet(gatewayId, x -> "_" + targetId, "");
+    String gatewayId = getGatewayId(targetId);
+    String suffix = ifNotNullGet(gatewayId, x -> format("_%s", targetId), "");
     String deviceType = ifNotNullGet(gatewayId, x -> "Proxy", "Device");
     info(format("%s %s config handler", deviceType, targetId));
-    File configOut = new File(getOutDir(), format("%s.json", traceTimestamp("config" + suffix)));
+    File configOut = new File(getOutDir(),
+        format("%s.json", traceTimestamp(format("config%s", suffix))));
     toJsonFile(configOut, configMsg);
   }
 
@@ -463,9 +508,9 @@ private void processConfigUpdate(Config configMsg) {
     try {
       updateInterval(DEFAULT_REPORT_SEC);
       if (configMsg != null) {
-        if (configMsg.system == null && isTrue(getConfig().options.barfConfig)) {
+        if (configMsg.system == null && isBarfConfig()) {
           error("Empty config system block and configured to restart on bad config!");
-          getDeviceManager().systemLifecycle(SystemMode.RESTART);
+          getDeviceManager().systemLifecycle(Operation.SystemMode.RESTART);
           return;
         }
         GeneralUtils.copyFields(configMsg, getDeviceConfig(), true);
@@ -473,7 +518,7 @@ private void processConfigUpdate(Config configMsg) {
         getDeviceManager().updateConfig(configMsg);
         extractEndpointBlobConfig();
       } else {
-        info(getTimestamp() + " defaulting empty config");
+        info(format("%s defaulting empty config", getTimestamp()));
       }
     } finally {
       getStateLock().unlock();
@@ -486,11 +531,12 @@ private void processConfigUpdate(Config configMsg) {
    * Check smoky failure.
    */
   default void checkSmokyFailure() {
-    if (isTrue(getConfig().options.smokeCheck)
-        && Instant.now().minus(SMOKE_CHECK_TIME).isAfter(DEVICE_START_TIME.toInstant())) {
+    Date startTime = getDeviceManager().getSystemManager().getStartTime();
+    if (isSmokeCheck() && startTime != null
+        && Instant.now().minus(SMOKE_CHECK_TIME).isAfter(startTime.toInstant())) {
       error(format("Smoke check failed after %sm, terminating run.",
-          SMOKE_CHECK_TIME.getSeconds() / 60));
-      getDeviceManager().systemLifecycle(SystemMode.TERMINATE);
+              SMOKE_CHECK_TIME.getSeconds() / 60));
+      getDeviceManager().systemLifecycle(Operation.SystemMode.TERMINATE);
     }
   }
 
@@ -504,8 +550,8 @@ default void deferredConfigActions() {
     DeviceManager deviceManager = getDeviceManager();
     deviceManager.maybeRestartSystem();
     SystemState systemState = deviceManager.getSystemManager().getSystemState();
-    SystemMode mode = catchToNull(() -> systemState.operation.mode);
-    if (SystemMode.INITIAL.equals(mode) || SystemMode.ACTIVE.equals(mode)) {
+    Operation.SystemMode mode = catchToNull(() -> systemState.operation.mode);
+    if (Operation.SystemMode.INITIAL.equals(mode) || Operation.SystemMode.ACTIVE.equals(mode)) {
       // Do redirect after restart system check, since this might take a long time.
       maybeRedirectEndpoint(getExtractedEndpoint());
     }
@@ -518,15 +564,14 @@ default void deferredConfigActions() {
    * each message sent to stabilize the output order for testing purposes.
    */
   default void sendEmptyMissingBadEvents() {
-    if (!isTrue(getConfig().options.emptyMissing)) {
+    if (!isEmptyMissing()) {
       return;
     }
 
     final int explicitPhases = 3;
 
-    checkState(
-        PointsetManager.MESSAGE_REPORT_INTERVAL > explicitPhases + INVALID_REPLACEMENTS.size() + 1,
-        "not enough space for hacky messages");
+    checkState(PointsetManager.MESSAGE_REPORT_INTERVAL > explicitPhases
+        + INVALID_REPLACEMENTS.size() + 1, "not enough space for hacky messages");
     int phase = (getDeviceUpdateCount() + PointsetManager.MESSAGE_REPORT_INTERVAL / 2)
         % PointsetManager.MESSAGE_REPORT_INTERVAL;
 
@@ -534,25 +579,25 @@ default void sendEmptyMissingBadEvents() {
 
     if (phase == 0) {
       flushDirtyState();
-      InjectedState invalidState = new InjectedState();
+      MqttPublisher.InjectedState invalidState = new MqttPublisher.InjectedState();
       invalidState.REPLACE_MESSAGE_WITH = CORRUPT_STATE_MESSAGE;
       warn("Sending badly formatted state as per configuration");
       publishStateMessage(invalidState);
     } else if (phase == 1) {
-      InjectedMessage invalidEvent = new InjectedMessage();
+      MqttPublisher.InjectedMessage invalidEvent = new MqttPublisher.InjectedMessage();
       invalidEvent.field = "bunny";
       warn("Sending badly formatted message with extra field");
       publishDeviceMessage(invalidEvent);
     } else if (phase == 2) {
-      FakeTopic invalidTopic = new FakeTopic();
+      MqttPublisher.FakeTopic invalidTopic = new MqttPublisher.FakeTopic();
       warn("Sending badly formatted message with fake topic");
       publishDeviceMessage(invalidTopic);
     } else if (phase < INVALID_REPLACEMENTS.size() + explicitPhases) {
       String key = INVALID_KEYS.get(phase - explicitPhases);
-      InjectedMessage replacedEvent = new InjectedMessage();
+      MqttPublisher.InjectedMessage replacedEvent = new MqttPublisher.InjectedMessage();
       replacedEvent.REPLACE_TOPIC_WITH = key;
       replacedEvent.REPLACE_MESSAGE_WITH = INVALID_REPLACEMENTS.get(key);
-      warn("Sending badly formatted message of type " + key);
+      warn(format("Sending badly formatted message of type %s", key));
       publishDeviceMessage(replacedEvent);
     }
     safeSleep(INJECT_MESSAGE_DELAY_MS);
@@ -562,7 +607,7 @@ default void sendEmptyMissingBadEvents() {
    * Maybe tweak state.
    */
   default void maybeTweakState() {
-    if (!isTrue(getOptions().tweakState)) {
+    if (!isTweakState()) {
       return;
     }
     int phase = getDeviceUpdateCount() % 2;
@@ -574,8 +619,6 @@ default void maybeTweakState() {
     }
   }
 
-  PubberOptions getOptions();
-
   Lock getStateLock();
 
   // TODO: Consider refactoring this to either return or change an instance variable, not both.
@@ -596,7 +639,7 @@ default EndpointConfiguration extractEndpointBlobConfig() {
       if (getExtractedEndpoint() != null) {
         if (getDeviceConfig().blobset.blobs.containsKey(IOT_ENDPOINT_CONFIG.value())) {
           BlobBlobsetConfig config = getDeviceConfig()
-              .blobset.blobs.get(IOT_ENDPOINT_CONFIG.value());
+                  .blobset.blobs.get(IOT_ENDPOINT_CONFIG.value());
           getExtractedEndpoint().generation = config.generation;
         }
       }
@@ -610,7 +653,7 @@ default EndpointConfiguration extractEndpointBlobConfig() {
 
   void setExtractedEndpoint(EndpointConfiguration endpointConfiguration);
 
-  private void removeBlobsetBlobState(SystemBlobsets blobId) {
+  private void removeBlobsetBlobState(BlobsetConfig.SystemBlobsets blobId) {
     if (getDeviceState().blobset == null) {
       return;
     }
@@ -631,11 +674,11 @@ private void removeBlobsetBlobState(SystemBlobsets blobId) {
    * logic.
    */
   default void maybeRedirectEndpoint(EndpointConfiguration extractedEndpoint) {
-    String redirectRegistry = getConfig().options.redirectRegistry;
-    String currentSignature = toJsonString(getConfig().endpoint);
-    String extractedSignature =
-        redirectRegistry == null ? toJsonString(extractedEndpoint)
-            : redirectedEndpoint(redirectRegistry);
+    String redirectRegistry = getRedirectRegistry();
+    String currentSignature = toJsonString(getEndpoint());
+    String extractedSignature = redirectRegistry == null
+        ? toJsonString(extractedEndpoint)
+        : redirectedEndpoint(redirectRegistry);
 
     if (extractedSignature == null) {
       setAttemptedEndpoint(null);
@@ -646,7 +689,7 @@ default void maybeRedirectEndpoint(EndpointConfiguration extractedEndpoint) {
     BlobBlobsetState endpointState = ensureBlobsetState(IOT_ENDPOINT_CONFIG);
 
     if (extractedSignature.equals(currentSignature)
-        || extractedSignature.equals(getAttemptedEndpoint())) {
+            || extractedSignature.equals(getAttemptedEndpoint())) {
       return; // No need to redirect anything!
     }
 
@@ -660,7 +703,7 @@ default void maybeRedirectEndpoint(EndpointConfiguration extractedEndpoint) {
 
       if (extractedEndpoint.error != null) {
         setAttemptedEndpoint(extractedSignature);
-        endpointState.phase = BlobPhase.FINAL;
+        endpointState.phase = BlobBlobsetConfig.BlobPhase.FINAL;
         Exception applyError = new RuntimeException(extractedEndpoint.error);
         endpointState.status = exceptionStatus(applyError, Category.BLOBSET_BLOB_APPLY);
         publishSynchronousState();
@@ -668,20 +711,21 @@ default void maybeRedirectEndpoint(EndpointConfiguration extractedEndpoint) {
       }
     }
 
-    info("New config blob endpoint detected:\n" + stringify(parseJson(extractedSignature)));
+    info(format("New config blob endpoint detected:\n%s",
+        stringify(parseJson(extractedSignature))));
 
     try {
       setAttemptedEndpoint(extractedSignature);
-      endpointState.phase = BlobPhase.APPLY;
+      endpointState.phase = BlobBlobsetConfig.BlobPhase.APPLY;
       publishSynchronousState();
       resetConnection(extractedSignature);
       persistEndpoint(extractedEndpoint);
-      endpointState.phase = BlobPhase.FINAL;
+      endpointState.phase = BlobBlobsetConfig.BlobPhase.FINAL;
       markStateDirty();
     } catch (Exception e) {
       try {
         error("Reconfigure failed, attempting connection to last working endpoint", e);
-        endpointState.phase = BlobPhase.FINAL;
+        endpointState.phase = BlobBlobsetConfig.BlobPhase.FINAL;
         endpointState.status = exceptionStatus(e, Category.BLOBSET_BLOB_APPLY);
         resetConnection(getWorkingEndpoint());
         publishAsynchronousState();
@@ -699,13 +743,9 @@ default void maybeRedirectEndpoint(EndpointConfiguration extractedEndpoint) {
 
   String getAttemptedEndpoint();
 
-  default void notice(String message) {
-    cloudLog(message, Level.NOTICE);
-  }
-
   private String redirectedEndpoint(String redirectRegistry) {
     try {
-      EndpointConfiguration endpoint = deepCopy(getConfig().endpoint);
+      EndpointConfiguration endpoint = deepCopy(getEndpoint());
       endpoint.client_id = getClientId(redirectRegistry);
       return toJsonString(endpoint);
     } catch (Exception e) {
@@ -729,40 +769,17 @@ default Entry exceptionStatus(Exception e, String category) {
   /**
    * Ensures the {@code blobset} and its {@code blobs} map are initialized in the device state.
    */
-  default BlobBlobsetState ensureBlobsetState(SystemBlobsets iotEndpointConfig) {
+  default BlobBlobsetState ensureBlobsetState(BlobsetConfig.SystemBlobsets iotEndpointConfig) {
     getDeviceState().blobset = ofNullable(getDeviceState().blobset).orElseGet(BlobsetState::new);
     getDeviceState().blobset.blobs = ofNullable(getDeviceState().blobset.blobs)
-        .orElseGet(HashMap::new);
+            .orElseGet(HashMap::new);
     return getDeviceState().blobset.blobs.computeIfAbsent(iotEndpointConfig.value(),
-        key -> new BlobBlobsetState());
-  }
-
-  default String getClientId(String forRegistry) {
-    String cloudRegion = SiteModel.parseClientId(getConfig().endpoint.client_id).cloudRegion;
-    return SiteModel.getClientId(getConfig().iotProject, cloudRegion, forRegistry, getDeviceId());
+            key -> new BlobBlobsetState());
   }
 
-  /**
-   * Extracts the configuration blob with the specified name, if it exists and is in the final
-   * phase.
-   */
-  default String extractConfigBlob(String blobName) {
-    // TODO: Refactor to get any blob meta parameters.
-    try {
-      if (getDeviceConfig() == null || getDeviceConfig().blobset == null
-          || getDeviceConfig().blobset.blobs == null) {
-        return null;
-      }
-      BlobBlobsetConfig blobBlobsetConfig = getDeviceConfig().blobset.blobs.get(blobName);
-      if (blobBlobsetConfig != null && BlobPhase.FINAL.equals(blobBlobsetConfig.phase)) {
-        return acquireBlobData(blobBlobsetConfig.url, blobBlobsetConfig.sha256);
-      }
-      return null;
-    } catch (Exception e) {
-      EndpointConfiguration endpointConfiguration = new EndpointConfiguration();
-      endpointConfiguration.error = e.toString();
-      return stringify(endpointConfiguration);
-    }
+  private String getClientId(String forRegistry) {
+    String cloudRegion = SiteModel.parseClientId(getEndpoint().client_id).cloudRegion;
+    return SiteModel.getClientId(getIotProject(), cloudRegion, forRegistry, getDeviceId());
   }
 
   default void publishLogMessage(Entry logEntry, String targetId) {
@@ -819,8 +836,8 @@ default void publishStateMessage() {
     getStateDirty().set(false);
     getDeviceState().timestamp = getNow();
     info(format("Update state %s last_config %s", isoConvert(getDeviceState().timestamp),
-        isoConvert(getDeviceState().system.last_config)));
-    publishStateMessage(isTrue(getOptions().badState) ? getDeviceState().system : getDeviceState());
+            isoConvert(getDeviceState().system.last_config)));
+    publishStateMessage(isBadState() ? getDeviceState().system : getDeviceState());
   }
 
   /**
@@ -861,7 +878,7 @@ default void publishStateMessageRaw(Object stateToSend) {
 
     try {
       debug(format("State update %s%s", getDeviceId(), getDeviceManager().getTestingTag()),
-          toJsonString(stateToSend));
+              toJsonString(stateToSend));
     } catch (Exception e) {
       throw new RuntimeException("While converting new device state", e);
     }
@@ -871,7 +888,7 @@ default void publishStateMessageRaw(Object stateToSend) {
       latch.countDown();
     });
     try {
-      if (shouldSendState() && !latch.await(INITIAL_THRESHOLD_SEC, TimeUnit.SECONDS)) {
+      if (!isNoState() && !latch.await(INITIAL_THRESHOLD_SEC, TimeUnit.SECONDS)) {
         throw new RuntimeException("Timeout waiting for state send");
       }
     } catch (Exception e) {
@@ -879,10 +896,6 @@ default void publishStateMessageRaw(Object stateToSend) {
     }
   }
 
-  default boolean shouldSendState() {
-    return !isGetTrue(() -> getConfig().options.noState);
-  }
-
   default void publishDeviceMessage(Object message) {
     publishDeviceMessage(null, message);
   }
@@ -902,38 +915,38 @@ default void publishDeviceMessage(String targetId, Object message, Runnable call
     }
     String topicSuffix = MESSAGE_TOPIC_SUFFIX_MAP.get(message.getClass());
     if (topicSuffix == null) {
-      error("Unknown message class " + message.getClass());
+      error(format("Unknown message class %s", message.getClass()));
       return;
     }
 
-    if (!shouldSendState() && topicSuffix.equals(STATE_TOPIC)) {
+    if (isNoState() && topicSuffix.equals(STATE_TOPIC)) {
       warn("Squelching state update as per configuration");
       return;
     }
 
-    if (isTrue(getOptions().noFolder) && topicSuffix.equals(SYSTEM_EVENT_TOPIC)) {
+    if (isNoFolder() && topicSuffix.equals(SYSTEM_EVENT_TOPIC)) {
       topicSuffix = RAW_EVENT_TOPIC;
     }
 
     String useId = ofNullable(targetId).orElseGet(this::getDeviceId);
-    augmentDeviceMessage(message, getNow(), isTrue(getOptions().badVersion));
+    augmentDeviceMessage(message, getNow(), isBadVersion());
     Object downgraded = downgradeMessage(message);
     getDeviceTarget().publish(useId, topicSuffix, downgraded, callback);
     String messageBase = topicSuffix.replace("/", "_");
-    String gatewayId = getGatewayId(useId, getConfig());
-    String suffix = ifNotNullGet(gatewayId, x -> "_" + useId, "");
+    String gatewayId = getGatewayId(useId);
+    String suffix = ifNotNullGet(gatewayId, x -> format("_%s", useId), "");
     File messageOut = new File(getOutDir(), format("%s.json",
-        traceTimestamp(messageBase + suffix)));
+            traceTimestamp(messageBase + suffix)));
 
     try {
       toJsonFile(messageOut, downgraded);
     } catch (Exception e) {
-      throw new RuntimeException("While writing " + messageOut.getAbsolutePath(), e);
+      throw new RuntimeException(format("While writing %s", messageOut.getAbsolutePath()), e);
     }
   }
 
   private Object downgradeMessage(Object message) {
-    MessageDowngrader messageDowngrader = new MessageDowngrader(SubType.STATE.value(), message);
+    MessageDowngrader messageDowngrader = new MessageDowngrader(STATE.value(), message);
     return ifNotNullGet(getTargetSchema(), messageDowngrader::downgrade, message);
   }
 
@@ -950,14 +963,12 @@ default void cloudLog(String message, Level level, String detail) {
     if (getDeviceManager() != null) {
       getDeviceManager().cloudLog(message, level, detail);
     } else {
-      String detailPostfix = detail == null ? "" : ":\n" + detail;
+      String detailPostfix = detail == null ? "" : format(":\n%s", detail);
       String logMessage = format("%s%s", message, detailPostfix);
-      getLogMap().get(level).accept(logMessage);
+      SystemManager.getLogMap().apply(LOG).get(level).accept(logMessage);
     }
   }
 
-  Map> getLogMap();
-
   /**
    * Initializes the system by calling {@link #initializeDevice()} and {@link #initializeMqtt()}.
    *
@@ -965,17 +976,18 @@ default void cloudLog(String message, Level level, String detail) {
    */
   default void initialize() {
     try {
+      setClockSkew(isSkewClock() ? CLOCK_SKEW : Duration.ZERO);
+      ifTrueThen(isSpamState(), () -> periodicSchedule(STATE_SPAM_SEC, this::markStateDirty));
       initializeDevice();
-      initializeMqtt();
     } catch (Exception e) {
-      shutdown();
+      shutdown(null);
       throw new RuntimeException("While initializing main UDMI publisher class", e);
     }
   }
 
-  default void debug(String message, String detail) {
-    cloudLog(message, Level.DEBUG, detail);
-  }
+  void periodicSchedule(int sec, Runnable update);
+
+  void schedule(long ms, Runnable update);
 
   void initializeDevice();
 
@@ -985,11 +997,24 @@ default void debug(String message, String detail) {
 
   void writePersistentStore();
 
-  void startConnection(Function connectionDone);
+  void startConnection();
 
   void reconnect();
 
-  void resetConnection(String targetEndpoint);
+  /**
+   * Reset connection.
+   */
+  default void resetConnection(String targetEndpoint) {
+    synchronized (this) {
+      try {
+        setEndpoint(fromJsonString(targetEndpoint, EndpointConfiguration.class));
+        getRetriesCount().set(0);
+        startConnection();
+      } catch (Exception e) {
+        throw new RuntimeException("While resetting connection", e);
+      }
+    }
+  }
 
   /**
    * Flushes the dirty state by publishing an asynchronous state change.
@@ -1000,7 +1025,7 @@ default void flushDirtyState() {
     }
   }
 
-  byte[] ensureKeyBytes();
+  void ensureKeyBytes();
 
   /**
    * Handles exceptions related to the publisher and
@@ -1009,19 +1034,24 @@ default void flushDirtyState() {
    * @param toReport the exception to be handled;
    */
   default void publisherException(Exception toReport) {
-    if (toReport instanceof PublisherException r) {
+    if (toReport instanceof MqttPublisher.PublisherException r) {
       publisherHandler(r.getType(), r.getPhase(), r.getCause(), r.getDeviceId());
     } else if (toReport instanceof ConnectionClosedException) {
       warn("Connection closed, attempting reconnect...");
       reconnect();
     } else {
-      error("Unknown exception type " + toReport.getClass(), toReport);
+      error(format("Unknown exception type %s", toReport.getClass()), toReport);
     }
   }
 
-  void persistEndpoint(EndpointConfiguration endpoint);
-
-  String traceTimestamp(String messageBase);
+  /**
+   * Persist endpoint.
+   */
+  default void persistEndpoint(EndpointConfiguration endpoint) {
+    notice("Persisting connection endpoint");
+    getPersistentData().endpoint = endpoint;
+    writePersistentStore();
+  }
 
   /**
    * Configures a wait time for the configuration latch and waits until it is acquired.
@@ -1031,7 +1061,7 @@ default void publisherException(Exception toReport) {
    */
   default void configLatchWait() {
     try {
-      int waitTimeSec = ofNullable(getConfig().endpoint.config_sync_sec)
+      int waitTimeSec = ofNullable(getEndpoint().config_sync_sec)
           .orElse(DEFAULT_CONFIG_WAIT_SEC);
       int useWaitTime = waitTimeSec == 0 ? DEFAULT_CONFIG_WAIT_SEC : waitTimeSec;
       warn(format("Start waiting %ds for config latch for %s", useWaitTime, getDeviceId()));
@@ -1043,13 +1073,198 @@ default void configLatchWait() {
     }
   }
 
-  PubberConfiguration getConfig();
+  /**
+   * Initialize logger.
+   */
+  default void initializeLogger() {
+    File outDir = new File(getLogPath());
+    try {
+      PrintStream printStream = new PrintStream(new File(outDir, LOG_FILE));
+      setLogPrintWriter(printStream);
+      printStream.printf("UDMI log started at %s%n", getTimestamp());
+    } catch (Exception e) {
+      String outPath = outDir.getAbsolutePath();
+      throw new RuntimeException(format("While initializing out dir %s", outPath), e);
+    }
+  }
+
+  String getLogPath();
 
-  void shutdown();
+  /**
+   * Shutdown logPrintWriter.
+   */
+  default void shutdownLogger() {
+    PrintStream logPrintWriter = getLogPrintWriter();
+    if (logPrintWriter != null) {
+      logPrintWriter.close();
+    }
+  }
 
-  String getDeviceId();
+  PrintStream getLogPrintWriter();
 
-  boolean isConnected();
+  void setLogPrintWriter(PrintStream logPrintWriter);
+
+  /**
+   * Shutdown device.
+   */
+  default void shutdown(Runnable shutdownTask) {
+    warn("Initiating device shutdown");
+
+    State deviceState = getDeviceState();
+    if (deviceState.system != null && deviceState.system.operation != null) {
+      deviceState.system.operation.mode = Operation.SystemMode.SHUTDOWN;
+    }
+
+    if (isConnected()) {
+      captureExceptions("Publishing shutdown state", this::publishSynchronousState);
+    }
+    ifNotNullThen(getDeviceManager(),
+        dm -> captureExceptions("Device manager shutdown", dm::shutdown));
+    ifNotNullThen(shutdownTask, t -> captureExceptions("PublisherHost sender shutdown", t));
+    captureExceptions("Disconnecting mqtt", this::disconnectMqtt);
+  }
+
+  String getDeviceId();
 
   int getDeviceUpdateCount();
+
+  void incrementDeviceUpdateCount();
+
+  default FamilyProvider getLocalnetProvider(String family) {
+    return getDeviceManager().getLocalnetProvider(family);
+  }
+
+  /**
+   * Get gateway ID.
+   */
+  default String getGatewayId(String targetId) {
+    if (isGatewayDevice() && !targetId.equals(getDeviceId())) {
+      return getDeviceId();
+    }
+    return null;
+  }
+
+  /**
+   * Trace timestamp.
+   */
+  default String traceTimestamp(String messageBase) {
+    int serial = getMessageCounts().computeIfAbsent(messageBase, key -> new AtomicInteger())
+            .incrementAndGet();
+    String timestamp = getTimestamp().replace("Z", format(".%03dZ", serial));
+    return messageBase + (isMessageTrace() ? format("_%s", timestamp) : "");
+  }
+
+  default void notice(String message) {
+    cloudLog(message, Level.NOTICE);
+  }
+
+  default void debug(String message, String detail) {
+    cloudLog(message, Level.DEBUG, detail);
+  }
+
+  @Override
+  default void debug(String message) {
+    cloudLog(message, Level.DEBUG);
+  }
+
+  @Override
+  default void info(String message) {
+    cloudLog(message, Level.INFO);
+  }
+
+  @Override
+  default void warn(String message) {
+    cloudLog(message, Level.WARNING);
+  }
+
+  default void error(String message) {
+    cloudLog(message, Level.ERROR);
+  }
+
+  @Override
+  default void error(String message, Throwable e) {
+    if (e == null) {
+      error(message);
+      return;
+    }
+    String longMessage = format("%s: %s", message, e.getMessage());
+    cloudLog(longMessage, Level.ERROR);
+    getDeviceManager().localLog(message, Level.TRACE, getTimestamp(), stackTraceString(e));
+  }
+
+  String getIotProject();
+
+  EndpointConfiguration getEndpoint();
+
+  void setEndpoint(EndpointConfiguration endpointConfiguration);
+
+  AtomicInteger getRetriesCount();
+
+  DevicePersistent getPersistentData();
+
+  Map getMessageCounts();
+
+  default String getRedirectRegistry() {
+    return null;
+  }
+
+  default boolean isBadVersion() {
+    return false;
+  }
+
+  default boolean isNoFolder() {
+    return false;
+  }
+
+  default boolean isNoState() {
+    return false;
+  }
+
+  default boolean isBadState() {
+    return false;
+  }
+
+  default boolean isTweakState() {
+    return false;
+  }
+
+  default boolean isEmptyMissing() {
+    return false;
+  }
+
+  default boolean isBarfConfig() {
+    return false;
+  }
+
+  default boolean isSmokeCheck() {
+    return false;
+  }
+
+  default boolean isConfigStateDelay() {
+    return false;
+  }
+
+  default boolean isBadCategory() {
+    return false;
+  }
+
+  default boolean isNoStatus() {
+    return false;
+  }
+
+  default boolean isDupeState() {
+    return false;
+  }
+
+  default boolean isMessageTrace() {
+    return false;
+  }
+
+  default boolean isSkewClock() {
+    return false;
+  }
+
+  default boolean isSpamState() {
+    return false;
+  }
 }
diff --git a/pubber/src/main/java/udmi/lib/client/DeviceManager.java b/pubber/src/main/java/udmi/lib/client/manager/DeviceManager.java
similarity index 83%
rename from pubber/src/main/java/udmi/lib/client/DeviceManager.java
rename to pubber/src/main/java/udmi/lib/client/manager/DeviceManager.java
index c605556d7c..b0661e293f 100644
--- a/pubber/src/main/java/udmi/lib/client/DeviceManager.java
+++ b/pubber/src/main/java/udmi/lib/client/manager/DeviceManager.java
@@ -1,7 +1,9 @@
-package udmi.lib.client;
+package udmi.lib.client.manager;
 
+import java.util.List;
 import java.util.Map;
 import udmi.lib.intf.FamilyProvider;
+import udmi.lib.intf.SubBlockManager;
 import udmi.schema.Config;
 import udmi.schema.DevicePersistent;
 import udmi.schema.Entry;
@@ -14,17 +16,27 @@
  * Device client.
  */
 public interface DeviceManager extends SubBlockManager {
-  
-  PointsetManager getPointsetManager();
+
+  List getSubmanagers();
 
   SystemManager getSystemManager();
 
   LocalnetManager getLocalnetManager();
 
-  GatewayManager getGatewayManager();
+  PointsetManager getPointsetManager();
 
   DiscoveryManager getDiscoveryManager();
 
+  GatewayManager getGatewayManager();
+
+  /**
+   * Keep the resulting list in reverse order for proper shutdown semantics.
+   */
+  default  T addManager(T manager) {
+    getSubmanagers().add(0, manager);
+    return manager;
+  }
+
   default Map enumerateFamilies() {
     return getLocalnetManager().enumerateFamilies();
   }
@@ -42,10 +54,6 @@ default void setMetadata(Metadata metadata) {
     getGatewayManager().setMetadata(metadata);
   }
 
-  default void activate() {
-    getGatewayManager().activate();
-  }
-
   default void systemLifecycle(SystemMode mode) {
     getSystemManager().systemLifecycle(mode);
   }
@@ -109,4 +117,21 @@ default FamilyProvider getLocalnetProvider(String family) {
     return getLocalnetManager().getLocalnetProvider(family);
   }
 
+  default void activate() {
+    getGatewayManager().activate();
+  }
+
+  /**
+   * Stop periodic senders.
+   */
+  default void stop() {
+    getSubmanagers().forEach(SubBlockManager::stop);
+  }
+
+  /**
+   * Shutdown everything, including sub-managers.
+   */
+  default void shutdown() {
+    getSubmanagers().forEach(SubBlockManager::shutdown);
+  }
 }
diff --git a/pubber/src/main/java/udmi/lib/client/DiscoveryManager.java b/pubber/src/main/java/udmi/lib/client/manager/DiscoveryManager.java
similarity index 60%
rename from pubber/src/main/java/udmi/lib/client/DiscoveryManager.java
rename to pubber/src/main/java/udmi/lib/client/manager/DiscoveryManager.java
index 36b955c2cf..6d6c30e263 100644
--- a/pubber/src/main/java/udmi/lib/client/DiscoveryManager.java
+++ b/pubber/src/main/java/udmi/lib/client/manager/DiscoveryManager.java
@@ -1,4 +1,4 @@
-package udmi.lib.client;
+package udmi.lib.client.manager;
 
 import static com.google.udmi.util.GeneralUtils.catchToNull;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
@@ -6,10 +6,12 @@
 import static com.google.udmi.util.GeneralUtils.ifNullThen;
 import static com.google.udmi.util.GeneralUtils.ifTrueGet;
 import static com.google.udmi.util.GeneralUtils.ifTrueThen;
+import static com.google.udmi.util.JsonUtil.getNowInstant;
 import static com.google.udmi.util.JsonUtil.isoConvert;
 import static java.lang.Math.floorMod;
 import static java.lang.String.format;
 import static java.util.Optional.ofNullable;
+import static udmi.schema.FamilyDiscoveryState.Phase.ACTIVE;
 import static udmi.schema.FamilyDiscoveryState.Phase.PENDING;
 import static udmi.schema.FamilyDiscoveryState.Phase.STOPPED;
 
@@ -20,17 +22,28 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
+import udmi.lib.ProtocolFamily;
+import udmi.lib.intf.FamilyProvider;
+import udmi.lib.intf.SubBlockManager;
 import udmi.schema.DiscoveryConfig;
+import udmi.schema.DiscoveryEvents;
 import udmi.schema.DiscoveryState;
+import udmi.schema.Enumerations;
 import udmi.schema.Enumerations.Depth;
 import udmi.schema.FamilyDiscoveryConfig;
 import udmi.schema.FamilyDiscoveryState;
+import udmi.schema.FeatureDiscovery;
+import udmi.schema.PointPointsetModel;
 
 /**
  * Discovery client.
  */
 public interface DiscoveryManager extends SubBlockManager {
+
+  int SCAN_DURATION_SEC = 10;
+
   /**
    * Determines whether enumeration to a specific depth level is required.
    *
@@ -44,6 +57,40 @@ private static boolean shouldEnumerateTo(Depth depth) {
     });
   }
 
+  /**
+   * Updates discovery enumeration.
+   *
+   * @param config Discovery Configuration.
+   */
+  default void updateDiscoveryEnumeration(DiscoveryConfig config) {
+    Date enumerationGeneration = config.generation;
+    if (enumerationGeneration == null) {
+      getDiscoveryState().generation = null;
+      return;
+    }
+    if (getDiscoveryState().generation != null
+        && !enumerationGeneration.after(getDiscoveryState().generation)) {
+      return;
+    }
+    getDiscoveryState().generation = enumerationGeneration;
+    info(format("Discovery enumeration at %s", isoConvert(enumerationGeneration)));
+    DiscoveryEvents discoveryEvent = new DiscoveryEvents();
+    discoveryEvent.scan_family = ProtocolFamily.IOT;
+    discoveryEvent.generation = enumerationGeneration;
+    Enumerations depths = config.enumerations;
+    discoveryEvent.points = maybeEnumerate(depths.points, () -> enumeratePoints(getDeviceId()));
+    discoveryEvent.features = maybeEnumerate(depths.features, this::getFeatures);
+    discoveryEvent.families = maybeEnumerate(depths.families,
+        () -> getDeviceManager().enumerateFamilies());
+    getHost().publish(discoveryEvent);
+  }
+
+  Map getFeatures();
+
+  Map enumeratePoints(String deviceId);
+
+  DeviceManager getDeviceManager();
+
   default  Map maybeEnumerate(Depth depth, Supplier> supplier) {
     return ifTrueGet(shouldEnumerateTo(depth), supplier);
   }
@@ -82,9 +129,10 @@ default void scheduleDiscoveryScan(String family) {
       return;
     }
 
-    Date configGeneration = ofNullable(rawGeneration).orElse(getDeviceStartTime());
+    Date startTime = getHost().getStartTime();
+    Date configGeneration = ofNullable(rawGeneration).orElse(startTime);
     FamilyDiscoveryState familyDiscoveryState = ensureFamilyDiscoveryState(family);
-    Date baseGeneration = ofNullable(familyDiscoveryState.generation).orElse(getDeviceStartTime());
+    Date baseGeneration = ofNullable(familyDiscoveryState.generation).orElse(startTime);
 
     final Date startGeneration;
     if (interval > 0) {
@@ -102,7 +150,7 @@ default void scheduleDiscoveryScan(String family) {
       return;
     }
 
-    info("Discovery scan generation " + family + " pending at " + isoConvert(startGeneration));
+    info(format("Discovery scan generation %s pending at %s", family, isoConvert(startGeneration)));
     familyDiscoveryState.generation = startGeneration;
     familyDiscoveryState.phase = PENDING;
     updateState();
@@ -151,8 +199,7 @@ default FamilyDiscoveryState ensureFamilyDiscoveryState(String family) {
       // If there is no need for family state, then return a floating bucket for results.
       return new FamilyDiscoveryState();
     }
-    return getDiscoveryState().families.computeIfAbsent(
-        family, key -> new FamilyDiscoveryState());
+    return getDiscoveryState().families.computeIfAbsent(family, key -> new FamilyDiscoveryState());
   }
 
   /**
@@ -200,13 +247,10 @@ default int getScanInterval(String family) {
         catchToNull(() -> getFamilyDiscoveryConfig(family).scan_interval_sec)).orElse(0);
   }
 
-
   default boolean shouldEnumerate(String family) {
     return shouldEnumerateTo(getFamilyDiscoveryConfig(family).depth);
   }
 
-  Date getDeviceStartTime();
-
   DiscoveryState getDiscoveryState();
 
   void setDiscoveryState(DiscoveryState discoveryState);
@@ -217,7 +261,65 @@ default boolean shouldEnumerate(String family) {
 
   ScheduledFuture scheduleFuture(Date startGeneration, Runnable runnable);
 
-  void startDiscoveryScan(String family, Date scanGeneration);
+  /**
+   * Starts a discovery scan.
+   *
+   * @param family Discovery scan family.
+   * @param scanGeneration Scan generation.
+   */
+  default void startDiscoveryScan(String family, Date scanGeneration) {
+    info(format("Discovery scan starting %s generation %s", family, isoConvert(scanGeneration)));
+    Date stopTime = Date.from(getNowInstant().plusSeconds(SCAN_DURATION_SEC));
+    final FamilyDiscoveryState familyDiscoveryState = ensureFamilyDiscoveryState(family);
+    scheduleFuture(stopTime, () -> discoveryScanComplete(family, scanGeneration));
+    familyDiscoveryState.generation = scanGeneration;
+    familyDiscoveryState.phase = ACTIVE;
+    AtomicInteger sendCount = new AtomicInteger();
+    familyDiscoveryState.active_count = sendCount.get();
+    updateState();
+    startDiscoveryForFamily(family, scanGeneration, familyDiscoveryState, sendCount);
+  }
+
+  /**
+   * Starts a discovery scan for given family.
+   */
+  default void startDiscoveryForFamily(String family, Date scanGeneration,
+      FamilyDiscoveryState familyDiscoveryState, AtomicInteger sendCount) {
+    discoveryProvider(family).startScan(shouldEnumerate(family), (deviceId, discoveryEvent) -> {
+      ifNotNullThen(discoveryEvent.scan_addr, addr -> {
+        info(
+            format("Discovered %s device %s for gen %s", family, addr, isoConvert(scanGeneration)));
+        discoveryEvent.scan_family = family;
+        discoveryEvent.generation = scanGeneration;
+        postDiscoveryProcess(deviceId, discoveryEvent);
+
+        familyDiscoveryState.active_count = sendCount.incrementAndGet();
+        updateState();
+        getHost().publish(discoveryEvent);
+      });
+    });
+  }
+
+  /**
+   * Discovery scan completed.
+   */
+  default void discoveryScanComplete(String family, Date scanGeneration) {
+    try {
+      FamilyDiscoveryState familyDiscoveryState = ensureFamilyDiscoveryState(family);
+      ifTrueThen(scanGeneration.equals(familyDiscoveryState.generation), () -> {
+        discoveryProvider(family).stopScan();
+        familyDiscoveryState.phase = STOPPED;
+        updateState();
+        scheduleDiscoveryScan(family);
+      });
+    } catch (Exception e) {
+      throw new RuntimeException(format("While completing discovery scan %s", family), e);
+    }
+  }
+
+  default FamilyProvider discoveryProvider(String family) {
+    return getHost().getLocalnetProvider(family);
+  }
 
-  void updateDiscoveryEnumeration(DiscoveryConfig config);
+  void postDiscoveryProcess(String deviceId, DiscoveryEvents discoveryEvents);
 }
diff --git a/pubber/src/main/java/udmi/lib/client/manager/GatewayManager.java b/pubber/src/main/java/udmi/lib/client/manager/GatewayManager.java
new file mode 100644
index 0000000000..e587455b8b
--- /dev/null
+++ b/pubber/src/main/java/udmi/lib/client/manager/GatewayManager.java
@@ -0,0 +1,198 @@
+package udmi.lib.client.manager;
+
+import static com.google.udmi.util.GeneralUtils.catchToNull;
+import static com.google.udmi.util.GeneralUtils.getNow;
+import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
+import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+import static com.google.udmi.util.GeneralUtils.ifNullThen;
+import static com.google.udmi.util.GeneralUtils.ifTrueGet;
+import static com.google.udmi.util.GeneralUtils.ifTrueThen;
+import static java.lang.String.format;
+import static java.util.Optional.ofNullable;
+import static java.util.function.Predicate.not;
+import static udmi.schema.Category.GATEWAY_PROXY_TARGET;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import udmi.lib.ProtocolFamily;
+import udmi.lib.client.host.ProxyHost;
+import udmi.lib.intf.ManagerHost;
+import udmi.lib.intf.SubBlockManager;
+import udmi.schema.Config;
+import udmi.schema.Entry;
+import udmi.schema.GatewayConfig;
+import udmi.schema.GatewayState;
+import udmi.schema.Level;
+import udmi.schema.Metadata;
+import udmi.schema.PointPointsetConfig;
+import udmi.schema.PointsetConfig;
+
+/**
+ * Gateway client.
+ */
+public interface GatewayManager extends SubBlockManager {
+
+  String EXTRA_PROXY_DEVICE = "XXX-1";
+  String EXTRA_PROXY_POINT = "xxx_conflagration";
+
+  Metadata getMetadata();
+
+  default void setMetadata(Metadata metadata) {
+    setProxyDevices(ifNotNullGet(metadata.gateway, g -> createProxyDevices(g.proxy_ids)));
+  }
+
+  GatewayState getGatewayState();
+
+  Map getProxyDevices();
+
+  void setProxyDevices(Map proxyDevices);
+
+  /**
+   * Creates a map of proxy devices.
+   *
+   * @param proxyIds A list of device IDs to create proxies for.
+   * @return A map where each key-value pair represents a device ID and its corresponding proxy
+   * @throws NoSuchElementException if no first element exists in the stream
+   */
+  default Map createProxyDevices(List proxyIds) {
+    List deviceIds = ofNullable(proxyIds).orElseGet(ArrayList::new);
+    String firstId = deviceIds.stream().sorted().findFirst().orElse(null);
+    String noProxyId = ifTrueGet(isNoProxy(), () -> firstId);
+    ifNotNullThen(noProxyId, id -> warn(format("Not proxying device %s", id)));
+    List filteredList = deviceIds.stream().filter(not(id -> id.equals(noProxyId))).toList();
+    Map devices = new ConcurrentHashMap<>();
+    filteredList.forEach(id -> devices.put(id, createProxyDevice(getHost(), id)));
+    ifTrueThen(isExtraDevice(), () -> devices.put(EXTRA_PROXY_DEVICE, makeExtraDevice()));
+    return devices;
+  }
+
+  ProxyHost createProxyDevice(ManagerHost host, String id);
+
+  ProxyHost makeExtraDevice();
+
+  default void activate() {
+    ifNotNullThen(getProxyDevices(), p -> CompletableFuture.runAsync(() -> p.values()
+        .parallelStream().forEach(ProxyHost::activate)));
+  }
+
+  /**
+   * Publish log message for target device.
+   */
+  default void publishLogMessage(Entry logEntry, String targetId) {
+    ifNotNullThen(getProxyDevices(), p ->
+          ifNotNullThen(p.getOrDefault(targetId, null), pd ->
+                pd.getDeviceManager().publishLogMessage(logEntry, targetId)));
+  }
+
+  /**
+   * Set device status for target device.
+   */
+  default void setStatus(Entry report, String targetId) {
+    ifNotNullThen(getProxyDevices(), p ->
+          ifNotNullThen(p.getOrDefault(targetId, null), pd ->
+                pd.getDeviceManager().setStatus(report, targetId)));
+  }
+
+  /**
+   * Sets gateway status.
+   *
+   */
+  default void setGatewayStatus(String category, Level level, String message) {
+    // TODO: Implement a map or tree or something to properly handle different error sources.
+    getGatewayState().status = new Entry();
+    getGatewayState().status.category = category;
+    getGatewayState().status.level = level.value();
+    getGatewayState().status.message = message;
+    getGatewayState().status.timestamp = getNow();
+  }
+
+  /**
+   * Updates the state of the gateway.
+   */
+  default void updateState() {
+    updateState(ofNullable((Object) getGatewayState()).orElse(GatewayState.class));
+  }
+
+  /**
+   * Validates the given gateway family.
+   */
+  default void validateGatewayFamily(String family, String addr) {
+    if (!ProtocolFamily.FAMILIES.contains(family)) {
+      throw new IllegalArgumentException(format("Unrecognized address family %s", family));
+    }
+    String expectedAddr = catchToNull(() -> getMetadata().localnet.families.get(family).addr);
+    if (expectedAddr != null && !expectedAddr.equals(addr)) {
+      throw new IllegalStateException(
+          format("Family address was %s, expected %s", addr, expectedAddr));
+    }
+  }
+
+  /**
+   * Configures the extra device with default settings.
+   */
+  default void configExtraDevice() {
+    Config config = new Config();
+    config.pointset = new PointsetConfig();
+    config.pointset.points = new HashMap<>();
+    PointPointsetConfig pointPointsetConfig = new PointPointsetConfig();
+    config.pointset.points.put(EXTRA_PROXY_POINT, pointPointsetConfig);
+    getProxyDevices().get(EXTRA_PROXY_DEVICE).configHandler(config);
+  }
+
+  /**
+   * Update gateway operation based off of a gateway configuration block. This happens in two
+   * slightly different forms, one for the gateway proper (primarily indicating what devices
+   * should be proxy targets), and the other for the proxy devices themselves.
+   */
+  default void updateConfig(GatewayConfig gatewayConfig) {
+    if (gatewayConfig == null) {
+      setGatewayState(null);
+      updateState();
+      return;
+    }
+    ifNullThen(getGatewayState(), () -> setGatewayState(new GatewayState()));
+
+    ifNotNullThen(getProxyDevices(),
+        p -> ifTrueThen(p.containsKey(EXTRA_PROXY_DEVICE), this::configExtraDevice));
+
+    if (gatewayConfig.proxy_ids == null || gatewayConfig.target != null) {
+      try {
+        String addr = catchToNull(() -> gatewayConfig.target.addr);
+        String family = ofNullable(catchToNull(() -> gatewayConfig.target.family))
+                .orElse(ProtocolFamily.VENDOR);
+        validateGatewayFamily(family, addr);
+        setGatewayStatus(GATEWAY_PROXY_TARGET, Level.DEBUG,
+                format("gateway target family %s", family));
+      } catch (Exception e) {
+        setGatewayStatus(GATEWAY_PROXY_TARGET, Level.ERROR, e.getMessage());
+      }
+    }
+    syncDevices(gatewayConfig);
+    updateState();
+  }
+
+  void syncDevices(GatewayConfig gatewayConfig);
+
+  void setGatewayState(GatewayState gatewayState);
+
+  default void shutdown() {
+    ifNotNullThen(getProxyDevices(), p -> p.values().forEach(ProxyHost::shutdown));
+  }
+
+  default void stop() {
+    ifNotNullThen(getProxyDevices(), p -> p.values().forEach(ProxyHost::stop));
+  }
+
+  default boolean isNoProxy() {
+    return false;
+  }
+
+  default boolean isExtraDevice() {
+    return false;
+  }
+}
diff --git a/pubber/src/main/java/udmi/lib/client/LocalnetManager.java b/pubber/src/main/java/udmi/lib/client/manager/LocalnetManager.java
similarity index 91%
rename from pubber/src/main/java/udmi/lib/client/LocalnetManager.java
rename to pubber/src/main/java/udmi/lib/client/manager/LocalnetManager.java
index 04c00acb81..86281801d4 100644
--- a/pubber/src/main/java/udmi/lib/client/LocalnetManager.java
+++ b/pubber/src/main/java/udmi/lib/client/manager/LocalnetManager.java
@@ -1,11 +1,13 @@
-package udmi.lib.client;
+package udmi.lib.client.manager;
 
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
 import static java.util.stream.Collectors.toMap;
 
+import java.util.Date;
 import java.util.Map;
 import udmi.lib.intf.FamilyProvider;
 import udmi.lib.intf.ManagerHost;
+import udmi.lib.intf.SubBlockManager;
 import udmi.schema.FamilyDiscovery;
 import udmi.schema.FamilyLocalnetState;
 import udmi.schema.LocalnetConfig;
@@ -16,7 +18,6 @@
  */
 public interface LocalnetManager extends ManagerHost, SubBlockManager {
 
-
   LocalnetConfig getLocalnetConfig();
 
   void setLocalnetConfig(LocalnetConfig localnetConfig);
@@ -79,4 +80,8 @@ default void publish(String targetId, Object message) {
     getHost().publish(targetId, message);
   }
 
+  @Override
+  default Date getStartTime() {
+    throw new RuntimeException("Not yet implemented");
+  }
 }
diff --git a/pubber/src/main/java/udmi/lib/client/PointsetManager.java b/pubber/src/main/java/udmi/lib/client/manager/PointsetManager.java
similarity index 50%
rename from pubber/src/main/java/udmi/lib/client/PointsetManager.java
rename to pubber/src/main/java/udmi/lib/client/manager/PointsetManager.java
index 097108fbf5..9952d62d74 100644
--- a/pubber/src/main/java/udmi/lib/client/PointsetManager.java
+++ b/pubber/src/main/java/udmi/lib/client/manager/PointsetManager.java
@@ -1,13 +1,20 @@
-package udmi.lib.client;
+package udmi.lib.client.manager;
 
 import static com.google.udmi.util.GeneralUtils.getNow;
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+import static com.google.udmi.util.GeneralUtils.ifNotTrueThen;
+import static com.google.udmi.util.GeneralUtils.ifNullThen;
+import static com.google.udmi.util.GeneralUtils.ifTrueGet;
+import static com.google.udmi.util.GeneralUtils.ifTrueThen;
 import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+import static java.util.Objects.requireNonNullElseGet;
 import static java.util.Optional.ofNullable;
 import static udmi.schema.Category.POINTSET_POINT_INVALID;
 import static udmi.schema.Category.POINTSET_POINT_INVALID_VALUE;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Stream;
@@ -15,7 +22,9 @@
 import udmi.lib.intf.ManagerHost;
 import udmi.lib.intf.ManagerLog;
 import udmi.schema.Entry;
+import udmi.schema.PointPointsetConfig;
 import udmi.schema.PointPointsetEvents;
+import udmi.schema.PointPointsetModel;
 import udmi.schema.PointPointsetState;
 import udmi.schema.PointsetConfig;
 import udmi.schema.PointsetEvents;
@@ -34,14 +43,12 @@ public interface PointsetManager extends ManagerLog {
    *
    * @return A new {@code PointPointsetEvents} instance with the present value initialized to 100.
    */
-
   static PointPointsetEvents extraPointsetEvent() {
     PointPointsetEvents pointPointsetEvent = new PointPointsetEvents();
     pointPointsetEvent.present_value = 100;
     return pointPointsetEvent;
   }
 
-
   ExtraPointsetEvent getPointsetEvent();
 
   Map getManagedPoints();
@@ -50,15 +57,12 @@ static PointPointsetEvents extraPointsetEvent() {
 
   PointsetState getPointsetState();
 
-  default void setPointsetModel(PointsetModel pointset) {
-    throw new UnsupportedOperationException("Not supported yet.");
-  }
+  void setPointsetState(PointsetState ppointsetState);
 
   default void setExtraField(String extraField) {
     ifNotNullThen(extraField, field -> getPointsetEvent().extraField = field);
   }
 
-
   default void addPoint(AbstractPoint point) {
     getManagedPoints().put(point.getName(), point);
   }
@@ -86,7 +90,11 @@ default void restorePoint(String pointName) {
    * @return tweaked point state.
    */
   default PointPointsetState getPointState(AbstractPoint point) {
-    return point.getState();
+    PointPointsetState pointState = point.getState();
+    // Tweak for testing: erroneously apply an applied state here.
+    ifTrueThen(point.getName().equals(getExtraPoint()), () -> pointState.value_state =
+        ofNullable(pointState.value_state).orElse(PointPointsetState.Value_state.APPLIED));
+    return ifTrueGet(isNoPointState(), PointPointsetState::new, pointState);
   }
 
   default void suspendPoint(String pointName) {
@@ -96,7 +104,6 @@ default void suspendPoint(String pointName) {
 
   /**
    *  Updates the state of a PointsetState object.
-   *
    */
   default void updateState() {
     updateState(ofNullable((Object) getPointsetState()).orElse(PointsetState.class));
@@ -104,11 +111,48 @@ default void updateState() {
 
   void updateState(Object state);
 
+  void incrementUpdateCount();
+
+  /**
+   * Periodic update.
+   */
+  default void periodicUpdate() {
+    try {
+      if (getPointsetState() != null) {
+        incrementUpdateCount();
+        updatePoints();
+        sendDevicePoints();
+      }
+    } catch (Exception e) {
+      error("Fatal error during execution", e);
+    }
+  }
+
+  AbstractPoint makePoint(String name, PointPointsetModel point);
+
+  default Map getTestPoints() {
+    return new HashMap<>();
+  }
+
+  /**
+   * Set the underlying static model for this pointset. This is information that would NOT be
+   * normally available for a device, but would, e.g. be programmed directly into a device. It's
+   * only available here since this is a reference pseudo-device for testing.
+   *
+   * @param model pointset model
+   */
+  default void setPointsetModel(PointsetModel model) {
+    Map points = ifNotNullGet(model,
+        m -> requireNonNullElseGet(m.points, HashMap::new), getTestPoints());
+    ifNotNullThen(getMissingPoint(),
+        x -> requireNonNull(points.remove(x), "Missing point not in pointset metadata"));
+    points.forEach((name, point) -> addPoint(makePoint(name, point)));
+  }
 
   /**
    * Updates the state of a specific point.
    */
-  default void updateState(AbstractPoint point) {
+  default void updatePoint(AbstractPoint point) {
     String pointName = point.getName();
 
     if (!getPointsetState().points.containsKey(pointName)) {
@@ -133,7 +177,7 @@ default PointPointsetState invalidPoint(String pointName) {
     pointPointsetState.status = new Entry();
     pointPointsetState.status.category = POINTSET_POINT_INVALID;
     pointPointsetState.status.level = POINTSET_POINT_INVALID_VALUE;
-    pointPointsetState.status.message = "Unknown configured point " + pointName;
+    pointPointsetState.status.message = format("Unknown configured point %s", pointName);
     pointPointsetState.status.timestamp = getNow();
     return pointPointsetState;
   }
@@ -157,7 +201,7 @@ default void updateConfig(PointsetConfig config) {
   default void sendDevicePoints() {
     if (getPointsetUpdateCount() % MESSAGE_REPORT_INTERVAL == 0) {
       info(format("sending %s message #%d with %d points", getDeviceId(), getPointsetUpdateCount(),
-              getPointsetEvent().points.size()));
+          getPointsetEvent().points.size()));
     }
     getHost().publish(getPointsetEvent());
   }
@@ -175,11 +219,87 @@ class ExtraPointsetEvent extends PointsetEvents {
     public Object extraField;
   }
 
-  void updatePoints();
-
-  void updatePointsetPointsConfig(PointsetConfig config);
 
   String getDeviceId();
 
   ManagerHost getHost();
+
+  /**
+   * Update points.
+   */
+  default void updatePoints() {
+    getManagedPoints().values().forEach(point -> {
+      try {
+        point.updateData();
+      } catch (Exception ex) {
+        error("Unable to update point data", ex);
+      }
+      updatePoint(point);
+    });
+  }
+
+  boolean syncPoints(Map points);
+
+  /**
+   * Updates the configuration of pointset points.
+   */
+  default void updatePointsetPointsConfig(PointsetConfig config) {
+    // If there is no pointset config, then ensure that there's no pointset state.
+    if (config == null) {
+      setPointsetState(null);
+      updateState();
+      return;
+    }
+
+    // Known that pointset config exists, so ensure that a pointset state also exists.
+    ifNullThen(getPointsetState(), () -> {
+      setPointsetState(new PointsetState());
+      getPointsetState().points = new HashMap<>();
+      getPointsetEvent().points = new HashMap<>();
+    });
+
+    // Update each internally managed point with its specific pointset config (if any).
+    Map points = ofNullable(config.points).orElseGet(HashMap::new);
+    ifNotNullThen(getMissingPoint(), points::remove);
+    syncPoints(points);
+    getManagedPoints().forEach((name, point) -> updatePointConfig(point, points.get(name)));
+    getPointsetState().state_etag = config.state_etag;
+
+    // Special testing provisions for forcing an extra point (designed to cause a violation).
+    ifNotNullThen(getExtraPoint(), extraPoint ->
+        getPointsetEvent().points.put(extraPoint, PointsetManager.extraPointsetEvent()));
+
+    // Mark device state as dirty, so the system will send a consolidated state update.
+    updateState();
+  }
+
+  /**
+   * Update point config.
+   */
+  default void updatePointConfig(AbstractPoint point, PointPointsetConfig pointConfig) {
+    ifNotTrueThen(isNoWriteback(), () -> {
+      try {
+        point.setConfig(pointConfig);
+      } catch (Exception ex) {
+        error("Unable to set point config", ex);
+      }
+      updatePoint(point);
+    });
+  }
+
+  default boolean isNoWriteback() {
+    return false;
+  }
+
+  default boolean isNoPointState() {
+    return false;
+  }
+
+  default String getExtraPoint() {
+    return null;
+  }
+
+  default String getMissingPoint() {
+    return null;
+  }
 }
diff --git a/pubber/src/main/java/udmi/lib/client/SystemManager.java b/pubber/src/main/java/udmi/lib/client/manager/SystemManager.java
similarity index 50%
rename from pubber/src/main/java/udmi/lib/client/SystemManager.java
rename to pubber/src/main/java/udmi/lib/client/manager/SystemManager.java
index 8ecf985e27..8b718e8e18 100644
--- a/pubber/src/main/java/udmi/lib/client/SystemManager.java
+++ b/pubber/src/main/java/udmi/lib/client/manager/SystemManager.java
@@ -1,22 +1,36 @@
-package udmi.lib.client;
+package udmi.lib.client.manager;
 
+import static com.google.udmi.util.CleanDateFormat.dateEquals;
 import static com.google.udmi.util.GeneralUtils.catchToNull;
+import static com.google.udmi.util.GeneralUtils.getNow;
 import static com.google.udmi.util.GeneralUtils.getTimestamp;
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
+import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+import static com.google.udmi.util.GeneralUtils.ifNotTrueGet;
+import static com.google.udmi.util.GeneralUtils.ifNullThen;
+import static com.google.udmi.util.GeneralUtils.ifTrueThen;
 import static com.google.udmi.util.JsonUtil.isoConvert;
 import static com.google.udmi.util.JsonUtil.stringify;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNullElse;
 import static java.util.Optional.ofNullable;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import java.io.PrintStream;
 import java.time.Instant;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import org.slf4j.Logger;
+import udmi.lib.client.host.ProxyHost;
+import udmi.lib.client.host.PublisherHost;
+import udmi.lib.intf.ManagerHost;
+import udmi.lib.intf.SubBlockManager;
 import udmi.schema.DevicePersistent;
 import udmi.schema.Entry;
 import udmi.schema.Level;
@@ -24,6 +38,8 @@
 import udmi.schema.Metrics;
 import udmi.schema.Operation;
 import udmi.schema.Operation.SystemMode;
+import udmi.schema.StateSystemHardware;
+import udmi.schema.StateSystemOperation;
 import udmi.schema.SystemConfig;
 import udmi.schema.SystemEvents;
 import udmi.schema.SystemState;
@@ -33,15 +49,38 @@
  */
 public interface SystemManager extends SubBlockManager {
 
+  String DEFAULT_SOFTWARE_KEY = "firmware";
   String UDMI_PUBLISHER_LOG_CATEGORY = "device.log";
-
   long BYTES_PER_MEGABYTE = 1024 * 1024;
-
+  Integer UNKNOWN_MODE_EXIT_CODE = -1;
   Map EXIT_CODE_MAP = ImmutableMap.of(
       SystemMode.SHUTDOWN, 0, // Indicates expected clean shutdown (success).
       SystemMode.RESTART, 192, // Indicate process to be explicitly restarted.
       SystemMode.TERMINATE, 193); // Indicates expected shutdown (failure code).
-  Integer UNKNOWN_MODE_EXIT_CODE = -1;
+
+  /**
+   * Initialize system state.
+   */
+  default void initialize(ManagerHost host, String serialNo) {
+    ExtraSystemState state = getSystemState();
+    state.operation = new StateSystemOperation();
+    state.operation.operational = true;
+    state.operation.mode = SystemMode.INITIAL;
+    state.operation.last_start = getRoundedStartTime();
+    state.hardware = new StateSystemHardware();
+    state.last_config = new Date(0);
+
+    if (host instanceof PublisherHost publisher) {
+      publisher.initializeLogger();
+      info(format("Device start time is %s", isoConvert(getStartTime())));
+    }
+
+    ifTrueThen(host instanceof PublisherHost, () -> state.serial_no = serialNo);
+    ifTrueThen(isNoLastStart(), () -> state.operation.last_start = null);
+    ifTrueThen(isNoHardware(), () -> state.hardware = null);
+    ifNotNullThen(getExtraField(), value -> state.extraField = value);
+    updateState();
+  }
 
   /**
    * Builds a map of logger consumers.
@@ -57,15 +96,64 @@ static Function>> getLogMap() {
         .build();
   }
 
-  List getLogentries();
+  List getLogs();
+
+  /**
+   * Get log entries.
+   */
+  default List getLogEntries() {
+    List logs = getLogs();
+    synchronized (logs) {
+      if (isNoLog()) {
+        return null;
+      }
+      List entries = ImmutableList.copyOf(logs);
+      logs.clear();
+      return entries;
+    }
+  }
+
+  /**
+   * Publish log message.
+   */
+  default void publishLogMessage(Entry report) {
+    List logs = getLogs();
+    synchronized (logs) {
+      if (shouldLogLevel(report.level)) {
+        ifTrueThen(isBadLevel(), () -> report.level = 0);
+        logs.add(report);
+      }
+    }
+  }
 
-  boolean getPublishingLog();
+  AtomicBoolean getPublishLock();
 
   SystemConfig getSystemConfig();
 
   void systemLifecycle(SystemMode mode);
 
-  void localLog(String message, Level trace, String timestamp, String detail);
+  /**
+   * Device local log.
+   */
+  default void localLog(String message, Level level, String timestamp, String detail) {
+    String detailPostfix = detail == null ? "" : ":\n" + detail;
+    String logMessage = format("%s (%s) -> %s%s", timestamp, getDeviceId(), message, detailPostfix);
+    SystemManager.getLogMap().apply(PublisherHost.LOG).get(level).accept(logMessage);
+    try {
+      PrintStream stream;
+      if (getHost() instanceof PublisherHost publisher) {
+        stream = publisher.getLogPrintWriter();
+      } else if (getHost() instanceof ProxyHost proxyHost) {
+        stream = proxyHost.getPublisherHost().getLogPrintWriter();
+      } else {
+        throw new RuntimeException("While writing log output file: Unknown host");
+      }
+      stream.println(logMessage);
+      stream.flush();
+    } catch (Exception e) {
+      throw new RuntimeException("While writing log output file", e);
+    }
+  }
 
   /**
    * Local log.
@@ -81,7 +169,16 @@ default void localLog(Entry entry) {
    * Retrieves the hardware and software from metadata.
    *
    */
-  void setHardwareSoftware(Metadata metadata);
+  default void setHardwareSoftware(Metadata metadata) {
+    ExtraSystemState state = getSystemState();
+    state.hardware.make = catchToNull(() -> metadata.system.hardware.make);
+    state.hardware.model = catchToNull(() -> metadata.system.hardware.model);
+    state.software = catchToNull(() -> metadata.system.software);
+    ifNotNullThen(getSoftwareFirmwareValue(), value ->  {
+      ifNullThen(state.software, () -> state.software = new HashMap<>());
+      state.software.put(DEFAULT_SOFTWARE_KEY, value);
+    });
+  }
 
   ExtraSystemState getSystemState();
 
@@ -106,8 +203,7 @@ default void maybeRestartSystem() {
     SystemMode configMode = operation.mode;
     SystemMode stateMode = getSystemState().operation.mode;
 
-    if (SystemMode.ACTIVE.equals(stateMode)
-        && SystemMode.RESTART.equals(configMode)) {
+    if (SystemMode.ACTIVE.equals(stateMode) && SystemMode.RESTART.equals(configMode)) {
       error("System mode requesting device restart");
       systemLifecycle(SystemMode.RESTART);
       return;
@@ -119,17 +215,23 @@ default void maybeRestartSystem() {
     }
 
     Date configLastStart = operation.last_start;
-    if (configLastStart != null) {
-      if (getDeviceStartTime().before(configLastStart)) {
-        error(format("Device start time %s before last config start %s, terminating.",
-            isoConvert(getDeviceStartTime()), isoConvert(configLastStart)));
-        systemLifecycle(SystemMode.TERMINATE);
-      }
+    Date startTime = getStartTime();
+    if (configLastStart != null && startTime != null && startTime.before(configLastStart)) {
+      error(format("Device start time %s before last config start %s, terminating.",
+          isoConvert(getStartTime()), isoConvert(configLastStart)));
+      systemLifecycle(SystemMode.TERMINATE);
     }
-  }
 
+    if (configLastStart != null && isSmokeCheck() && dateEquals(getStartTime(), configLastStart)) {
+      error(format("Device start time %s matches, smoke check indicating success!",
+          isoConvert(configLastStart)));
+      systemLifecycle(SystemMode.SHUTDOWN);
+    }
+  }
 
-  Date getDeviceStartTime();
+  default Date getStartTime() {
+    return catchToNull(() -> getSystemState().operation.last_start);
+  }
 
   default void updateState() {
     getHost().update(getSystemState());
@@ -142,14 +244,14 @@ default void updateState() {
   default void sendSystemEvent() {
     SystemEvents systemEvent = getSystemEvent();
     systemEvent.metrics = new Metrics();
-    if (!(getHost() instanceof ProxyDeviceHost)) {
+    if (!(getHost() instanceof ProxyHost)) {
       Runtime runtime = Runtime.getRuntime();
       systemEvent.metrics.mem_free_mb = (double) runtime.freeMemory() / BYTES_PER_MEGABYTE;
       systemEvent.metrics.mem_total_mb = (double) runtime.totalMemory() / BYTES_PER_MEGABYTE;
       systemEvent.metrics.store_total_mb = Double.NaN;
     }
     systemEvent.event_no = incrementEventCount();
-    systemEvent.logentries = getLogentries();
+    systemEvent.logentries = getLogEntries();
     getHost().publish(systemEvent);
   }
 
@@ -170,76 +272,67 @@ default void updateConfig(SystemConfig system, Date timestamp) {
     Integer newBase = catchToNull(() -> system.testing.config_base);
     if (oldBase != null && oldBase.equals(newBase)
         && !stringify(getSystemConfig()).equals(stringify(system))) {
-      error("Panic! Duplicate config_base detected: " + oldBase);
+      error(format("Panic! Duplicate config_base detected: %s", oldBase));
       System.exit(-22);
     }
     setSystemConfig(system);
     updateInterval(ifNotNullGet(system, config -> config.metrics_rate_sec));
-    getSystemState().last_config = timestamp;
+    getSystemState().last_config = ifNotTrueGet(isNoLastConfig(), () -> timestamp);
     updateState();
   }
 
   void setSystemConfig(SystemConfig system);
 
-  /**
-   * Publish log message.
-   *
-   */
-  void publishLogMessage(Entry report);
-
   /**
    * Check if we should log at the level provided.
    */
   default boolean shouldLogLevel(int level) {
-    Integer minLoglevel = ifNotNullGet(getSystemConfig(), config -> getSystemConfig().min_loglevel);
+    Integer fixedLogLevel = getFixedLogLevel();
+    if (fixedLogLevel != null) {
+      return level >= fixedLogLevel;
+    }
+    Integer minLoglevel = ifNotNullGet(getSystemConfig(), config -> config.min_loglevel);
     return level >= requireNonNullElse(minLoglevel, Level.INFO.value());
   }
 
   /**
    * Logs a message with specified level and detail. If publishing is enabled,
    * it will publish the log message using the `udmiPublisherLogMessage` method.
-   *
    */
   default void cloudLog(String message, Level level, String detail) {
     String timestamp = getTimestamp();
     localLog(message, level, timestamp, detail);
-
-    if (getPublishingLog()) {
-      return;
-    }
-
-    try {
-      setPublishingLog(true);
-      udmiPublisherLogMessage(message, level, timestamp, detail);
-    } catch (Exception e) {
-      localLog("Error publishing log message: " + e, Level.ERROR, timestamp, null);
-    } finally {
-      setPublishingLog(false);
+    AtomicBoolean publishLock = getPublishLock();
+    if (publishLock.compareAndSet(false, true)) {
+      try {
+        publishLog(message, level, timestamp, detail);
+      } catch (Exception e) {
+        localLog(format("Error publishing log message: %s", e), Level.ERROR, timestamp, null);
+      } finally {
+        publishLock.set(false);
+      }
     }
   }
 
-  void setPublishingLog(boolean b);
-
   /**
    * Get a testing tag.
    */
   default String getTestingTag() {
     SystemConfig config = getSystemConfig();
-    return config == null || config.testing == null
-        || config.testing.sequence_name == null ? ""
+    return config == null || config.testing == null || config.testing.sequence_name == null
+        ? ""
         : format(" (%s)", config.testing.sequence_name);
   }
 
   /**
    * Log a message.
    */
-  default void udmiPublisherLogMessage(String logMessage, Level level, String timestamp,
-      String detail) {
+  default void publishLog(String message, Level level, String timestamp, String detail) {
     Entry logEntry = new Entry();
     logEntry.category = UDMI_PUBLISHER_LOG_CATEGORY;
     logEntry.level = level.value();
     logEntry.timestamp = Date.from(Instant.parse(timestamp));
-    logEntry.message = logMessage;
+    logEntry.message = message;
     logEntry.detail = detail;
     publishLogMessage(logEntry);
   }
@@ -249,6 +342,16 @@ default void setStatus(Entry report) {
     updateState();
   }
 
+  /**
+   * Retrieves the start time of the current second,
+   * with milliseconds removed for precise comparison.
+   */
+  default Date getRoundedStartTime() {
+    long timestamp = getNow().getTime();
+    // Remove ms so that rounded conversions preserve equality.
+    return new Date(timestamp - (timestamp % 1000));
+  }
+
   /**
    * Extra system state with extra field.
    */
@@ -259,7 +362,54 @@ class ExtraSystemState extends SystemState {
 
   void stop();
 
-  void shutdown();
+  /**
+   * Shutdown.
+   */
+  default void shutdown() {
+    if (getHost() instanceof PublisherHost publisher) {
+      publisher.shutdownLogger();
+    }
+  }
+
+  default void periodicUpdate() {
+    sendSystemEvent();
+  }
 
   void error(String message);
+
+  default boolean isSmokeCheck() {
+    return false;
+  }
+
+  default boolean isNoLog() {
+    return false;
+  }
+
+  default boolean isBadLevel() {
+    return false;
+  }
+
+  default boolean isNoLastConfig() {
+    return false;
+  }
+
+  default boolean isNoLastStart() {
+    return false;
+  }
+
+  default boolean isNoHardware() {
+    return false;
+  }
+
+  default String getExtraField() {
+    return null;
+  }
+
+  default Integer getFixedLogLevel() {
+    return null;
+  }
+
+  default String getSoftwareFirmwareValue() {
+    return null;
+  }
 }
diff --git a/pubber/src/main/java/udmi/lib/intf/ManagerHost.java b/pubber/src/main/java/udmi/lib/intf/ManagerHost.java
index 428e72b77a..ebdfdc9b46 100644
--- a/pubber/src/main/java/udmi/lib/intf/ManagerHost.java
+++ b/pubber/src/main/java/udmi/lib/intf/ManagerHost.java
@@ -1,9 +1,12 @@
 package udmi.lib.intf;
 
+import java.util.Date;
+
 /**
- * Collection of methods for how a manager can/should interface with it's host class.
+ * Collection of methods for how a manager can/should interface with its host class.
  */
 public interface ManagerHost extends ManagerLog {
+
   void update(Object update);
 
   default void publish(Object message) {
@@ -13,4 +16,6 @@ default void publish(Object message) {
   void publish(String targetId, Object message);
 
   FamilyProvider getLocalnetProvider(String family);
+
+  Date getStartTime();
 }
diff --git a/pubber/src/main/java/udmi/lib/client/SubBlockManager.java b/pubber/src/main/java/udmi/lib/intf/SubBlockManager.java
similarity index 84%
rename from pubber/src/main/java/udmi/lib/client/SubBlockManager.java
rename to pubber/src/main/java/udmi/lib/intf/SubBlockManager.java
index 52d26e4cd2..10d8aad6cd 100644
--- a/pubber/src/main/java/udmi/lib/client/SubBlockManager.java
+++ b/pubber/src/main/java/udmi/lib/intf/SubBlockManager.java
@@ -1,13 +1,10 @@
-package udmi.lib.client;
+package udmi.lib.intf;
 
 import java.util.Date;
 import java.util.concurrent.ScheduledFuture;
-import udmi.lib.intf.ManagerHost;
-import udmi.lib.intf.ManagerLog;
 
 /**
  * Interface for providing main manager functionalities.
- *
  */
 public interface SubBlockManager extends ManagerLog {
 
diff --git a/pubber/src/main/java/udmi/lib/intf/UdmiPublisher.java b/pubber/src/main/java/udmi/lib/intf/UdmiPublisher.java
deleted file mode 100644
index 5b138d1d9a..0000000000
--- a/pubber/src/main/java/udmi/lib/intf/UdmiPublisher.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package udmi.lib.intf;
-
-import udmi.lib.base.MqttDevice;
-import udmi.schema.Config;
-
-/**
- * Abstract interface for some kind of publishing stuff.
- */
-public interface UdmiPublisher extends ManagerHost {
-
-  boolean isConnected();
-
-  MqttDevice getMqttDevice(String deviceId);
-
-  void configPreprocess(String deviceId, Config config);
-
-  void publisherConfigLog(String phase, Exception e, String deviceId);
-}
diff --git a/pubber/src/main/java/udmi/util/CatchingScheduledThreadPoolExecutor.java b/pubber/src/main/java/udmi/util/CatchingScheduledThreadPoolExecutor.java
index 581657c1b7..462da50bf8 100644
--- a/pubber/src/main/java/udmi/util/CatchingScheduledThreadPoolExecutor.java
+++ b/pubber/src/main/java/udmi/util/CatchingScheduledThreadPoolExecutor.java
@@ -1,12 +1,19 @@
 package udmi.util;
 
+import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Thread executor wrapper that does a better job of exposing exceptions during execution.
  */
 public class CatchingScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor {
 
+  public static final Logger LOG =
+      LoggerFactory.getLogger(CatchingScheduledThreadPoolExecutor.class);
+
   /**
    * Create a new executor.
    *
@@ -17,10 +24,7 @@ public CatchingScheduledThreadPoolExecutor(int corePoolSize) {
   }
 
   protected void afterExecute(Runnable r, Throwable t) {
-    if (t != null) {
-      System.err.println("Exception during scheduled execution:");
-      t.printStackTrace();
-    }
+    ifNotNullThen(t, () -> LOG.error("Exception during scheduled execution", t));
     super.afterExecute(r, null);
   }
 }
diff --git a/pubber/src/test/java/daq/pubber/IpManagerTest.java b/pubber/src/test/java/daq/pubber/IpManagerTest.java
index 13332acffa..299001a24e 100644
--- a/pubber/src/test/java/daq/pubber/IpManagerTest.java
+++ b/pubber/src/test/java/daq/pubber/IpManagerTest.java
@@ -4,6 +4,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import daq.pubber.impl.provider.PubberIpProvider;
 import java.util.List;
 import java.util.Map;
 import org.junit.Test;
diff --git a/pubber/src/test/java/daq/pubber/PubberTest.java b/pubber/src/test/java/daq/pubber/PubberTest.java
index f9881fdd48..c4efbd4acd 100644
--- a/pubber/src/test/java/daq/pubber/PubberTest.java
+++ b/pubber/src/test/java/daq/pubber/PubberTest.java
@@ -11,11 +11,14 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.udmi.util.JsonUtil;
+import daq.pubber.impl.host.PubberPublisherHost;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import org.junit.After;
 import org.junit.Test;
+import udmi.lib.TestBase;
+import udmi.lib.client.host.PublisherHost;
 import udmi.schema.BlobBlobsetConfig;
 import udmi.schema.BlobBlobsetConfig.BlobPhase;
 import udmi.schema.BlobsetConfig;
@@ -38,7 +41,7 @@ public class PubberTest extends TestBase {
   private static final String TEST_REDIRECT_HOSTNAME = "mqtt-redirect.google.com";
   private static final String DATA_URL_PREFIX = "data:application/json;base64,";
   private static final EndpointConfiguration TEST_ENDPOINT = getEndpointConfiguration(null);
-  private static final EndpointConfiguration TEST_REDIRECT_ENDPOINT = 
+  private static final EndpointConfiguration TEST_REDIRECT_ENDPOINT =
       getEndpointConfiguration(TEST_REDIRECT_HOSTNAME);
   private static final String ENDPOINT_BLOB = JsonUtil.stringify(TEST_ENDPOINT);
   private static final String ENDPOINT_REDIRECT_BLOB = JsonUtil.stringify(TEST_REDIRECT_ENDPOINT);
@@ -49,7 +52,7 @@ private enum PubberUnderTestFeatures {
     OptionsNoPersist
   }
 
-  private class PubberUnderTest extends Pubber {
+  private class PubberUnderTest extends PubberPublisherHost {
 
     private HashMap testFeatures = new HashMap<>();
 
@@ -97,8 +100,7 @@ protected void augmentEndpoint(EndpointConfiguration endpoint) {
   private PubberUnderTest singularPubber(String[] args) {
     PubberUnderTest pubber = new PubberUnderTest(args[0], args[1], args[2], args[3]);
     pubber.initialize();
-    pubber.startConnection(deviceId -> {
-      return true; });
+    pubber.startConnection();
     return pubber;
   }
 
@@ -178,14 +180,14 @@ public void missingDevice() {
   @Test
   public void parseDataUrl() {
     String testBlobDataUrl = DATA_URL_PREFIX + encodeBase64(TEST_BLOB_DATA);
-    String blobData = PubberUdmiPublisher.acquireBlobData(testBlobDataUrl, sha256(TEST_BLOB_DATA));
+    String blobData = PublisherHost.acquireBlobData(testBlobDataUrl, sha256(TEST_BLOB_DATA));
     assertEquals("extracted blob data", blobData, TEST_BLOB_DATA);
   }
 
   @Test(expected = RuntimeException.class)
   public void badDataUrl() {
     String testBlobDataUrl = DATA_URL_PREFIX + encodeBase64(TEST_BLOB_DATA + "XXXX");
-    PubberUdmiPublisher.acquireBlobData(testBlobDataUrl, sha256(TEST_BLOB_DATA));
+    PublisherHost.acquireBlobData(testBlobDataUrl, sha256(TEST_BLOB_DATA));
   }
 
   @Test
@@ -221,13 +223,13 @@ public void augmentDeviceMessageTest() {
     State testMessage = new State();
 
     assertNull(testMessage.timestamp);
-    PubberUdmiPublisher.augmentDeviceMessage(testMessage, new Date(), false);
-    assertEquals(testMessage.version, Pubber.UDMI_VERSION);
+    PublisherHost.augmentDeviceMessage(testMessage, new Date(), false);
+    assertEquals(testMessage.version, PublisherHost.UDMI_VERSION);
     assertNotEquals(testMessage.timestamp, null);
 
     testMessage.timestamp = new Date(1241);
-    PubberUdmiPublisher.augmentDeviceMessage(testMessage, new Date(), false);
-    assertEquals(testMessage.version, Pubber.UDMI_VERSION);
+    PublisherHost.augmentDeviceMessage(testMessage, new Date(), false);
+    assertEquals(testMessage.version, PublisherHost.UDMI_VERSION);
     assertNotEquals(testMessage.timestamp, new Date(1241));
   }
 
@@ -239,9 +241,7 @@ public void initializePersistentStoreNullTest() {
     testFeatures.put(PubberUnderTestFeatures.noInitializePersistentStore, true);
     pubber = new PubberUnderTest(TEST_PROJECT, TEST_SITE, TEST_DEVICE, SERIAL_NO, testFeatures);
     pubber.initialize();
-    pubber.startConnection(deviceId -> {
-      return true;
-    });
+    pubber.startConnection();
 
     // Prepare test.
     testPersistentData.endpoint = null;
@@ -259,9 +259,7 @@ public void initializePersistentStoreFromConfigTest() {
     testFeatures.put(PubberUnderTestFeatures.noInitializePersistentStore, true);
     pubber = new PubberUnderTest(TEST_PROJECT, TEST_SITE, TEST_DEVICE, SERIAL_NO, testFeatures);
     pubber.initialize();
-    pubber.startConnection(deviceId -> {
-      return true;
-    });
+    pubber.startConnection();
 
     // Prepare test.
     testPersistentData.endpoint = null;
@@ -276,12 +274,11 @@ public void initializePersistentStoreFromConfigTest() {
   @Test
   public void initializePersistentStoreFromPersistentDataTest() {
     // Initialize the test Pubber.
-    HashMap testFeatures = new HashMap<
-        PubberUnderTestFeatures, Boolean>();
+    HashMap testFeatures = new HashMap<>();
     testFeatures.put(PubberUnderTestFeatures.noInitializePersistentStore, true);
     pubber = new PubberUnderTest(TEST_PROJECT, TEST_SITE, TEST_DEVICE, SERIAL_NO, testFeatures);
     pubber.initialize();
-    pubber.startConnection(deviceId -> true);
+    pubber.startConnection();
 
     // Prepare test.
     testPersistentData.endpoint = getEndpointConfiguration("persistent");
diff --git a/pubber/src/test/java/daq/pubber/SupportedFeaturesTest.java b/pubber/src/test/java/daq/pubber/SupportedFeaturesTest.java
index fc9e8e2529..9c1a241bae 100644
--- a/pubber/src/test/java/daq/pubber/SupportedFeaturesTest.java
+++ b/pubber/src/test/java/daq/pubber/SupportedFeaturesTest.java
@@ -5,6 +5,7 @@
 import static org.junit.Assert.assertTrue;
 import static udmi.schema.Bucket.ENUMERATION_FEATURES;
 
+import daq.pubber.impl.PubberFeatures;
 import java.util.Map;
 import org.junit.Test;
 import udmi.schema.Bucket;
diff --git a/pubber/src/test/java/daq/pubber/ListPublisherTest.java b/pubber/src/test/java/udmi/lib/ListPublisherTest.java
similarity index 97%
rename from pubber/src/test/java/daq/pubber/ListPublisherTest.java
rename to pubber/src/test/java/udmi/lib/ListPublisherTest.java
index b299981b7f..337995d197 100644
--- a/pubber/src/test/java/daq/pubber/ListPublisherTest.java
+++ b/pubber/src/test/java/udmi/lib/ListPublisherTest.java
@@ -1,4 +1,4 @@
-package daq.pubber;
+package udmi.lib;
 
 import static org.junit.Assert.assertEquals;
 
@@ -26,4 +26,4 @@ public void testPublish() throws InterruptedException {
     String publishedMessage = messages.get(0);
     assertEquals("published message", EXPECTED_MESSAGE_STRING, publishedMessage);
   }
-}
\ No newline at end of file
+}
diff --git a/pubber/src/test/java/daq/pubber/MqttDeviceTest.java b/pubber/src/test/java/udmi/lib/MqttDeviceTest.java
similarity index 98%
rename from pubber/src/test/java/daq/pubber/MqttDeviceTest.java
rename to pubber/src/test/java/udmi/lib/MqttDeviceTest.java
index 3dc9019524..e7815d3882 100644
--- a/pubber/src/test/java/daq/pubber/MqttDeviceTest.java
+++ b/pubber/src/test/java/udmi/lib/MqttDeviceTest.java
@@ -1,4 +1,4 @@
-package daq.pubber;
+package udmi.lib;
 
 import static org.junit.Assert.assertEquals;
 
@@ -33,4 +33,4 @@ public void publishTopicPrefix() throws InterruptedException {
     assertEquals("published message", EXPECTED_MESSAGE_STRING, publishedMessage);
   }
 
-}
\ No newline at end of file
+}
diff --git a/pubber/src/test/java/udmi/lib/base/MqttPublisherTest.java b/pubber/src/test/java/udmi/lib/MqttPublisherTest.java
similarity index 97%
rename from pubber/src/test/java/udmi/lib/base/MqttPublisherTest.java
rename to pubber/src/test/java/udmi/lib/MqttPublisherTest.java
index d15cf5f7c8..6c2ca4439c 100644
--- a/pubber/src/test/java/udmi/lib/base/MqttPublisherTest.java
+++ b/pubber/src/test/java/udmi/lib/MqttPublisherTest.java
@@ -1,14 +1,14 @@
-package udmi.lib.base;
+package udmi.lib;
 
 import static org.junit.Assert.assertEquals;
 
-import daq.pubber.TestBase;
 import java.util.concurrent.CountDownLatch;
 import java.util.function.Consumer;
 import org.eclipse.paho.client.mqttv3.MqttClient;
 import org.eclipse.paho.client.mqttv3.MqttException;
 import org.junit.Test;
 import org.mockito.Mockito;
+import udmi.lib.base.MqttPublisher;
 import udmi.schema.Auth_provider;
 import udmi.schema.Basic;
 import udmi.schema.EndpointConfiguration;
@@ -69,4 +69,4 @@ protected MqttClient getMqttClient(String clientId, String brokerUrl) throws Mqt
       return mocked;
     }
   }
-}
\ No newline at end of file
+}
diff --git a/pubber/src/test/java/daq/pubber/TestBase.java b/pubber/src/test/java/udmi/lib/TestBase.java
similarity index 98%
rename from pubber/src/test/java/daq/pubber/TestBase.java
rename to pubber/src/test/java/udmi/lib/TestBase.java
index 712be38b0a..61c079550b 100644
--- a/pubber/src/test/java/daq/pubber/TestBase.java
+++ b/pubber/src/test/java/udmi/lib/TestBase.java
@@ -1,4 +1,4 @@
-package daq.pubber;
+package udmi.lib;
 
 import static udmi.lib.base.ListPublisher.getMessageString;