split centre-id from topic prefix environment variables

ECCC-MSC · Apr 4, 2024 · c2e9cab · c2e9cab
1 parent 606c970
commit c2e9cab
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 5 deletions.
diff --git a/msc-wis2node.env b/msc-wis2node.env
@@ -5,4 +5,5 @@ export MSC_WIS2NODE_BROKER_PASSWORD=password
 export MSC_WIS2NODE_MSC_DATAMART_AMQP=amqps://dd.weather.gc.ca
 export MSC_WIS2NODE_DATASET_CONFIG=/opt/msc-wis2node/conf/datasets.yaml
 export MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL=https://example.org/discovery-metadata.zip
-export MSC_WIS2NODE_TOPIC_PREFIX=origin/a/wis2/ca-eccc-msc
+export MSC_WIS2NODE_CENTRE_ID=ca-eccc-msc
+export MSC_WIS2NODE_TOPIC_PREFIX=origin/a/wis2
diff --git a/msc_wis2node/dataset.py b/msc_wis2node/dataset.py
@@ -76,7 +76,7 @@ def create_datasets_conf(metadata_zipfile: Union[Path, None],
             z.extractall(td)
 
             path = Path(td)
-            mcfs_to_process = path.rglob('*/*.yml')
+            mcfs_to_process = path.rglob('mcf/**/*.yml')
 
             for path_object in mcfs_to_process:
                 LOGGER.debug(f'Path: {path_object}')

diff --git a/msc_wis2node/env.py b/msc_wis2node/env.py
@@ -31,6 +31,7 @@
 DATASET_CONFIG = os.environ.get('MSC_WIS2NODE_DATASET_CONFIG')
 TOPIC_PREFIX = os.environ.get('MSC_WIS2NODE_TOPIC_PREFIX', 'origin/a/wis2')
 DISCOVERY_METADATA_ZIP_URL = os.environ.get('MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL')  # noqa
+CENTRE_ID = os.environ.get('MSC_WIS2NODE_CENTRE_ID')
 
 if None in [BROKER_HOSTNAME, BROKER_PORT, BROKER_USERNAME, BROKER_PASSWORD,
             MSC_DATAMART_AMQP, DATASET_CONFIG, TOPIC_PREFIX]:

diff --git a/msc_wis2node/publisher.py b/msc_wis2node/publisher.py
@@ -34,7 +34,8 @@
 import yaml
 
 from msc_wis2node.env import (BROKER_HOSTNAME, BROKER_PORT, BROKER_USERNAME,
-                              BROKER_PASSWORD, DATASET_CONFIG, TOPIC_PREFIX)
+                              BROKER_PASSWORD, CENTRE_ID, DATASET_CONFIG,
+                              TOPIC_PREFIX)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -164,15 +165,17 @@ def publish_to_wis2(self, dataset: dict, url: str) -> None:
         :returns: `bool` of dispatch result
         """
 
-        topic = f"{TOPIC_PREFIX}/{dataset['wis2-topic']}"
+        topic = f"{TOPIC_PREFIX}/{CENTRE_ID}/{dataset['wis2-topic']}"
         LOGGER.info(f'URL: {url}')
 
         datetime_ = self._topic_regex2datetime(
             url, dataset.get('msc-filename-datetime-regex'))
 
+        metadata_id = f"urn:wmo:md:{CENTRE_ID}:{dataset['metadata-id']}"
+
         message = create_message(
             identifier=str(uuid.uuid4()),
-            # metadata_id=dataset['metadata-id'],
+            metadata_id=metadata_id,
             datetime_=datetime_,
             topic=topic,
             content_type=dataset['media-type'],