Skip to content

Commit

Permalink
Merge branch 'master' into reply-update
Browse files Browse the repository at this point in the history
khyatimahendru committed Jan 27, 2025

Verified

This commit was signed with the committer’s verified signature. The key has expired.
Czaki Grzegorz Bokota
2 parents 48336d4 + 0c8962b commit ad99288
Showing 60 changed files with 2,750 additions and 2,396 deletions.
33 changes: 29 additions & 4 deletions .github/workflows/integration.yaml
Original file line number Diff line number Diff line change
@@ -22,7 +22,6 @@ jobs:
container: [ "udmis", "validator", "pubber", "misc" ]
env:
PUSH_REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
CONTAINER: ${{ matrix.container }}
REF_NAME: ${{ github.ref_name }}
steps:
@@ -40,11 +39,13 @@ jobs:
registry: ${{ env.PUSH_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Convert repository name to lowercase
run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Container build and push
run: |
revhash=$(git log -n 1 --pretty=format:"%h")
IMAGE_HASH=g${revhash:0:8}
PUSH_REPO=$PUSH_REGISTRY/${IMAGE_NAME,,}
PUSH_REPO=$PUSH_REGISTRY/$IMAGE_NAME
TAG_BASE=$PUSH_REPO:$CONTAINER
PUSH_TAG=${TAG_BASE}-$IMAGE_HASH
@@ -66,6 +67,28 @@ jobs:
echo
docker history $PUSH_TAG
maven:
name: Publish maven package (pubber)
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
timeout-minutes: 5
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
- name: Publish maven package
run: pubber/bin/publish

simple:
name: Simple sequence test
runs-on: ubuntu-latest
@@ -76,7 +99,6 @@ jobs:
matrix:
device_id: [ "AHU-1", "AHU-22", "GAT-123" ]
env:
IMAGE_NAME: ${{ github.repository }}
REF_NAME: ${{ github.ref_name }}
DEVICE_ID: ${{ matrix.device_id }}
steps:
@@ -89,6 +111,8 @@ jobs:
jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json
jq . site_model/cloud_iot_config.json
docker network create udminet --subnet 192.168.99.0/24
- name: Convert repository name to lowercase
run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Start UDMIS container
run: |
export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME
@@ -148,7 +172,6 @@ jobs:
timeout-minutes: 5
needs: images
env:
IMAGE_NAME: ${{ github.repository }}
REF_NAME: ${{ github.ref_name }}
DEVICE_ID: ${{'AHU-1'}}
steps:
@@ -161,6 +184,8 @@ jobs:
jq ".device_id = \"$DEVICE_ID\"" site_model/cloud_iot_config.json | sponge site_model/cloud_iot_config.json
jq . site_model/cloud_iot_config.json
docker network create udminet --subnet 192.168.99.0/24
- name: Convert repository name to lowercase
run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Start UDMIS container
run: |
export IMAGE_TAG=ghcr.io/$IMAGE_NAME:udmis-$REF_NAME
40 changes: 36 additions & 4 deletions common/src/main/java/com/google/udmi/util/CertManager.java
Original file line number Diff line number Diff line change
@@ -5,9 +5,13 @@
import static java.lang.String.format;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.Security;
@@ -49,6 +53,9 @@ public class CertManager {
private final File crtFile;
private final char[] password;
private final Transport transport;
private final String caCertificate;
private final String clientCertificate;
private final String clientPrivateKey;

{
Security.addProvider(new BouncyCastleProvider());
@@ -61,6 +68,9 @@ public CertManager(File caCrtFile, File clientDir, Transport transport,
String passString, Consumer<String> logging) {
this.caCrtFile = caCrtFile;
this.transport = transport;
this.caCertificate = null;
this.clientCertificate = null;
this.clientPrivateKey = null;

if (Transport.SSL.equals(transport)) {
String prefix = keyPrefix(clientDir);
@@ -78,6 +88,18 @@ public CertManager(File caCrtFile, File clientDir, Transport transport,
}
}

public CertManager(String caCertificate, String clientCertificate, String clientPrivateKey,
Transport transport, String passString) {
caCrtFile = null;
crtFile = null;
keyFile = null;
this.transport = transport;
this.password = passString.toCharArray();
this.caCertificate = caCertificate;
this.clientCertificate = clientCertificate;
this.clientPrivateKey = clientPrivateKey;
}

private String keyPrefix(File clientDir) {
File rsaCrtFile = new File(clientDir, "rsa_private.crt");
File ecCrtFile = new File(clientDir, "ec_private.crt");
@@ -92,18 +114,27 @@ private String keyPrefix(File clientDir) {
public SSLSocketFactory getCertSocketFactory() throws Exception {
CertificateFactory certFactory = CertificateFactory.getInstance(X509_FACTORY);

InputStream caCertStream = caCrtFile != null
? new FileInputStream(caCrtFile)
: new ByteArrayInputStream(caCertificate.getBytes());
final X509Certificate caCert;
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(caCrtFile))) {
try (BufferedInputStream bis = new BufferedInputStream(caCertStream)) {
caCert = (X509Certificate) certFactory.generateCertificate(bis);
}

InputStream clientCertStream = crtFile != null
? new FileInputStream(crtFile)
: new ByteArrayInputStream(clientCertificate.getBytes());
final X509Certificate clientCert;
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(crtFile))) {
try (BufferedInputStream bis = new BufferedInputStream(clientCertStream)) {
clientCert = (X509Certificate) certFactory.generateCertificate(bis);
}

Reader keyReader = keyFile != null
? new FileReader(keyFile)
: new StringReader(clientPrivateKey);
final PrivateKey privateKey;
try (PEMParser pemParser = new PEMParser(new FileReader(keyFile))) {
try (PEMParser pemParser = new PEMParser(keyReader)) {
Object pemObject = pemParser.readObject();
if (pemObject instanceof PEMEncryptedKeyPair keyPair) {
PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password);
@@ -114,7 +145,8 @@ public SSLSocketFactory getCertSocketFactory() throws Exception {
privateKey = converter.getPrivateKey(keyPair);
} else {
throw new RuntimeException(format("Unknown pem file type %s from %s",
pemObject.getClass().getSimpleName(), keyFile.getAbsolutePath()));
pemObject.getClass().getSimpleName(),
keyFile == null ? "" : keyFile.getAbsolutePath()));
}
}

Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@ public class CommandLineProcessor {
private static final int LINUX_ERROR_CODE = -1;
private final Object target;
private final Method showHelpMethod;
private List<String> usageForms;

Map<CommandLineOption, Method> optionMap = new TreeMap<>(
(a, b) -> CASE_INSENSITIVE_ORDER.compare(getSortArg(a), getSortArg(b)));
@@ -69,6 +70,13 @@ public CommandLineProcessor(Object target) {
checkState(duplicateLong.isEmpty(), "duplicate short form command line option");
}

public CommandLineProcessor(Object target, List<String> usageForms) {
this(target);
if (usageForms != null && !usageForms.isEmpty()) {
this.usageForms = usageForms;
}
}

private Method getShowHelpMethod() {
try {
return CommandLineProcessor.class.getDeclaredMethod("showUsage");
@@ -91,6 +99,10 @@ private void showUsage() {
*/
public void showUsage(String message) {
ifNotNullThen(message, m -> System.err.println(m));
ifNotNullThen(usageForms, () -> {
System.err.println("Usage forms:");
usageForms.forEach(form -> System.err.printf(" %s%n", form));
});
System.err.println("Options supported:");
optionMap.forEach((option, method) -> System.err.printf(" %s %12s %s%n",
option.short_form(), option.arg_name(), option.description()));
1 change: 0 additions & 1 deletion docs/cloud/gcp/readme.md
Original file line number Diff line number Diff line change
@@ -4,5 +4,4 @@

- [UDMI terraform configuration files](terraform.md): Automate the creation of UDMI related infrastructure on GCP
- [UDMIS Infrastructure Setup](udmis.md): UDMIS Infrastructure (cloud functions)
- [UDMIF Deployment](../../../udmif/deploy.md): UDMIF Dashboard deployment instructions
- [Pubber Swarm](swarm.md): Simulate 1000's of devices connecting into IoT Core
2 changes: 1 addition & 1 deletion docs/cloud/gcp/udmis.md
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@

# UDMIS

UDMIS (Universal Device Management Interface Services) is a cloud infrastructure which comprises cloud functions. These process messages according to the [sub-blocks API](../../specs/subblocks.md). This is required for the enablement of other applications and tooling e.g. [UDMIF](UDMIF Deployment](../../../../../udmif/deploy.md) and [validation tooling](../../tools/readme.md)
UDMIS (Universal Device Management Interface Services) is a cloud infrastructure which comprises cloud functions. These process messages according to the [sub-blocks API](../../specs/subblocks.md). This is required for the enablement of other applications and tooling e.g. [validation tooling](../../tools/readme.md).

# Deployment

27 changes: 19 additions & 8 deletions docs/tools/registrar.md
Original file line number Diff line number Diff line change
@@ -39,22 +39,33 @@ be used to specific specific device(s) to register (rather than all).
```
Usage:
bin/registrar site_path [project_id] [options] [devices...]
bin/registrar site_model project_spec [options] [devices...]
bin/registrar config_file
bin/registrar site_spec [options] [devices...]
```

* `config_file`: Path to a configuration file which contains configuration options;
* `site_path`: The _directory_ containing the site model, or a model-with-project _file_ directly.
* `project_id`: The project ID that contains the target registry. The project ID can be prepended with iot_provider:
* `site_model`: The path to the _directory_ containing the site model, or a model-with-project _file_ directly.
* `project_spec`: The project ID that contains the target registry. The project ID can be prepended with iot_provider:
* `//clearblade/PROJECT_ID` for a public ClearBlade project.
* `//gbos/PROJECT_ID` for a Google operated ClearBlade project.
* `site_spec`: Path to a configuration file which contains configuration options;
* `options`: Various options to impact behavior:
* `-u` Update.
* `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry)
* `-a` Set alternate registry
* `-b` Block unknown devices.
* `-x` Delete unknown devices from the registry.
* `-c` Count of registries to be created
* `-d` Delete all device in the site model from the registry (combine with `-x` to delete all devices from the registry)
* `-e` Set registry suffix
* `-f` Set PubSub feed topic
* `-h` Show help and exit
* `-l` Set idle limit
* `-m` Initial metadata model out
* `-n` Number of thread counts.
* `-p` Set Project ID
* `-q` Query only, registry to not be updated
* `-r` Set tool root path
* `-s` Set site path
* `-t` Do not validate metadata
* `-x` Delete unknown devices from the registry.
* `devices`: Multiple device entries for limited registration. Can be just the device name
(`AHU-12`), or path to device (`site/devices/AHU-12`) for use with file-name glob.

30 changes: 15 additions & 15 deletions etc/validator.out
Original file line number Diff line number Diff line change
@@ -339,15 +339,15 @@ sites/udmi_site_model/out/devices/AHU-22/state.out
"sub_folder" : "update",
"sub_type" : "state",
"status" : {
"message" : "1 schema violations found",
"detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
},
"errors" : [ {
"message" : "1 schema violations found",
"detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
@@ -382,15 +382,15 @@ sites/udmi_site_model/out/devices/AHU-22/state_system.out
"sub_folder" : "system",
"sub_type" : "state",
"status" : {
"message" : "1 schema violations found",
"detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
},
"errors" : [ {
"message" : "1 schema violations found",
"detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
@@ -795,14 +795,14 @@ sites/udmi_site_model/out/devices/SNS-4/state.out
"sub_type" : "state",
"status" : {
"message" : "Multiple validation errors",
"detail" : "1 schema violations found; Device has missing points: split_threshold, triangulating_axis",
"detail" : "2 schema violations found; Device has missing points: split_threshold, triangulating_axis",
"category" : "validation.device.multiple",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
},
"errors" : [ {
"message" : "1 schema violations found",
"detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_update: 2 schema violations found; /system: object has missing required properties ([\"serial_no\",\"software\"]); /system/hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
@@ -850,15 +850,15 @@ sites/udmi_site_model/out/devices/SNS-4/state_system.out
"sub_folder" : "system",
"sub_type" : "state",
"status" : {
"message" : "1 schema violations found",
"detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
},
"errors" : [ {
"message" : "1 schema violations found",
"detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])",
"message" : "2 schema violations found",
"detail" : "state_system: 2 schema violations found; object has missing required properties ([\"serial_no\",\"software\"]); /hardware: object has missing required properties ([\"make\",\"model\"])",
"category" : "validation.device.schema",
"timestamp" : "REDACTED_TIMESTAMP",
"level" : 500
17 changes: 0 additions & 17 deletions pubber/.idea/runConfigurations/Swarm.xml

This file was deleted.

24 changes: 24 additions & 0 deletions pubber/bin/publish
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash -e

UDMI_ROOT=$(realpath $(dirname $0)/../..)
cd $UDMI_ROOT

DEFAULT_TASK="publish"
task=${1:-$DEFAULT_TASK}

# Validate the task
VALID_TASKS=("publish" "publishToMavenLocal")
if [[ ! " ${VALID_TASKS[@]} " =~ " $task " ]]; then
echo "Usage: $0 [publishTask]"
echo " publishTask: Task to execute. Default is '$DEFAULT_TASK'."
echo " Valid tasks are: ${VALID_TASKS[*]}"
echo
echo "Error: Invalid task '$task'"
exit 1
fi

source $UDMI_ROOT/etc/shell_common.sh

echo pubber/gradlew -p pubber $task
echo $UDMI_VERSION
$UDMI_ROOT/pubber/gradlew -p pubber $task
Loading

0 comments on commit ad99288

Please sign in to comment.