diff --git a/bin/pps_l1c_collector.py b/bin/pps_l1c_collector.py new file mode 100644 index 0000000..9510a86 --- /dev/null +++ b/bin/pps_l1c_collector.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019 - 2021 Pytroll + +# Author(s): + +# Erik Johansson +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Collector to add PPS level1c file to message with PPS files.""" + +import argparse +import logging + +from nwcsafpps_runner.logger import setup_logging +from nwcsafpps_runner.pps_collector_lib import pps_collector_runner + + +LOOP = True + +LOG = logging.getLogger('pps-l1c-collector') + + +def get_arguments(): + """Get command line arguments.""" + parser = argparse.ArgumentParser() + + parser.add_argument("-l", "--log-config", + help="Log config file to use instead of the standard logging.") + parser.add_argument('-c', '--config_file', + type=str, + dest='config_file', + default='config.yaml', + help="The file containing " + + "configuration parameters, \n" + + "default = ./config.yaml", + required=True) + parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, + help="Verbosity (between 1 and 2 occurrences with more leading to more " + "verbose logging). WARN=0, INFO=1, " + "DEBUG=2. This is overridden by the log config file if specified.") + + args = parser.parse_args() + setup_logging(args) + + if 'template' in args.config_file: + raise IOError("Template file given as master config, aborting!") + + return args.config_file + + +if __name__ == '__main__': + + config_file = get_arguments() + pps_collector_runner(config_file) diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index 6eeaec5..4586a59 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -26,10 +26,65 @@ import os from posttroll.message import Message +from nwcsafpps_runner.utils import create_pps_file_from_lvl1c + LOG = logging.getLogger(__name__) +def remove_non_pps_products(msg_data): + """ Remove non-PPS files from datasetlis.t""" + msg_data["dataset"] = [item for item in msg_data["dataset"] if "S_NWC" in item["uid"]] + + +def get_pps_sensor_from_msg(sensor_msg): + """ Get pps sensor from msg sensor.""" + sensor = None + if type(sensor_msg) is list and len(sensor_msg) == 1: + sensor = sensor_msg[0] + if sensor is None: + for pps_sensor in ['viirs', 'avhrr', 'modis', 'mersi2', 'metimage', 'slstr']: + if pps_sensor in sensor_msg: + sensor = pps_sensor + if "avhrr/3" in sensor_msg: + sensor = "avhrr" + return sensor + + +def add_lvl1c_to_msg(msg_data, options): + """Add PPS lvl1c file to a collection of PPS products.""" + level1c_path = os.environ.get('SM_IMAGER_DIR', options.get('pps_lvl1c_dir', './')) + sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"])) + num_files = len(msg_data['dataset']) + to_add = {} + for item in msg_data['dataset']: + lvl1c_file = create_pps_file_from_lvl1c(item["uri"], level1c_path, + name_tag=sensor, file_type='.nc') + to_add[lvl1c_file] = { + "uri": lvl1c_file, + "uid": os.path.basename(lvl1c_file)} + msg_data['dataset'].extend(to_add.values()) + + +def flatten_collection(msg_data): + """Flatten collection msg to dataset msg.""" + if "collection" in msg_data: + collection = msg_data.pop("collection") + msg_data["dataset"] = [] + for ind in range(0, len(collection)): + for item in collection[ind]["dataset"]: + if type(item) == dict: + msg_data["dataset"].append(item) + + +def prepare_pps_collector_message(msg, options): + to_send = msg.data.copy() + flatten_collection(to_send) + remove_non_pps_products(to_send) + add_lvl1c_to_msg(to_send, options) + return to_send + + def prepare_nwp_message(result_file, publish_topic): """Prepare message for NWP files.""" to_send = {} @@ -73,10 +128,10 @@ def prepare_l1c_message(result_file, mda, **kwargs): return to_send -def publish_l1c(publisher, publish_msg, publish_topic): +def publish_l1c(publisher, publish_msg, publish_topic, msg_type="file"): """Publish the messages that l1c files are ready.""" LOG.debug('Publish topic = %s', publish_topic) for topic in publish_topic: - msg = Message(topic, "file", publish_msg).encode() + msg = Message(topic, msg_type, publish_msg).encode() LOG.debug("sending: %s", str(msg)) publisher.send(msg) diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py new file mode 100644 index 0000000..a516146 --- /dev/null +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018 - 2021 Pytroll Developers + +# Author(s): + +# Nina Hakansson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import signal +import logging + +from contextlib import closing, suppress +from posttroll.publisher import create_publisher_from_dict_config +from posttroll.subscriber import Subscribe +from nwcsafpps_runner.config import get_config + +from nwcsafpps_runner.message_utils import (publish_l1c, + prepare_pps_collector_message) + +LOG = logging.getLogger(__name__) +LOOP = True + + +def _run_subscribe_publisher(subscriber, publisher, options): + """The posttroll subscribe/publisher runner.""" + def signal_handler(sig, frame): + LOG.warning('You pressed Ctrl+C!') + global LOOP + LOOP = False + + signal.signal(signal.SIGINT, signal_handler) + + while LOOP: + for msg in subscriber.recv(): + LOG.debug( + "Received message data = %s", msg) + pub_msg = prepare_pps_collector_message(msg, options) + publish_l1c(publisher, pub_msg, publish_topic=[options["publish_topic"]], msg_type="dataset") + LOG.info("L1c and PPS products collected.") + +def pps_collector_runner(config_file): + """The live runner for collecting the NWCSAF/PPS l1c and lvl2 products.""" + LOG.info("Start the NWCSAF/PPS products and level-1c collector runner") + + options = get_config(config_file) + settings = {"name": 'pps-collector-runner', + "nameservers": False, + "port": options.get("publish_port", 3002)} + with Subscribe('', options["subscribe_topics"], True) as sub: + with closing(create_publisher_from_dict_config(settings)) as pub: + pub.start() + _run_subscribe_publisher(sub, pub, options) diff --git a/nwcsafpps_runner/pps_posttroll_hook.py b/nwcsafpps_runner/pps_posttroll_hook.py index b757822..a7cd6d2 100644 --- a/nwcsafpps_runner/pps_posttroll_hook.py +++ b/nwcsafpps_runner/pps_posttroll_hook.py @@ -73,6 +73,7 @@ 'ppsCtype': 'CT', 'ppsCpp': 'CPP', 'ppsCmic': 'CMIC', + 'ppsHrw': 'HRW', 'ppsPrecip': 'PC', 'ppsPrecipPrepare': 'PC-PRE'} @@ -247,6 +248,8 @@ def send(self): # Error # pubmsg = self.create_message("FAILED", self.metadata) LOG.warning("Module %s failed, so no message sent", self.metadata.get('module', 'unknown')) + elif self.metadata["filename"] == "" or self.metadata["filename"] == []: + LOG.info("Module %s did not create any files, so no message sent", self.metadata.get('module', 'unknown')) else: # Ok pubmsg = self.create_message("OK") @@ -287,8 +290,10 @@ def create_message(self, status): self.clean_unused_keys_in_message() publish_topic = self._create_message_topic() - - return {'header': publish_topic, 'type': 'file', 'content': self._to_send} + msg_type = 'file' + if "dataset" in self._to_send: + msg_type = 'dataset' + return {'header': publish_topic, 'type': msg_type, 'content': self._to_send} def _create_message_topic(self): """Create the publish topic from yaml file items and PPS metadata.""" @@ -349,7 +354,10 @@ def get_message_with_uri_and_uid(self): LOG.debug("Servername = %s", str(servername)) msg = {} - if isinstance(self.metadata['filename'], list): + + if self.metadata['filename'] == '': + return {} + elif isinstance(self.metadata['filename'], list): dataset = [] for filename in self.metadata['filename']: uri = os.path.abspath(filename) diff --git a/nwcsafpps_runner/tests/test_pps_hook.py b/nwcsafpps_runner/tests/test_pps_hook.py index 6daaa17..cb34c19 100644 --- a/nwcsafpps_runner/tests/test_pps_hook.py +++ b/nwcsafpps_runner/tests/test_pps_hook.py @@ -222,17 +222,19 @@ def test_send_method(self, mandatory_param, filename): mandatory_param.return_value = True filename.return_value = True + my_metadata = self.metadata.copy() + my_metadata["filename"] = "dummy" with patch.object(PostTrollMessage, 'publish_message', return_value=None) as mock_method_publish: with patch.object(PostTrollMessage, 'create_message', return_value=None) as mock_method_create: - posttroll_message = PostTrollMessage(0, self.metadata) + posttroll_message = PostTrollMessage(0, my_metadata) posttroll_message.send() self.assertEqual(mock_method_publish.call_count, 1) self.assertEqual(mock_method_create.call_count, 1) with patch.object(PostTrollMessage, 'publish_message', return_value=None) as mock_method_publish: with patch.object(PostTrollMessage, 'create_message', return_value=None) as mock_method_create: - posttroll_message = PostTrollMessage(1, self.metadata) + posttroll_message = PostTrollMessage(1, my_metadata) posttroll_message.send() self.assertEqual(mock_method_publish.call_count, 0) self.assertEqual(mock_method_create.call_count, 0) @@ -344,6 +346,30 @@ def test_create_message_with_topic(self, socket_gethostname): expected_message_header = "/my/pps/publish/topic/UNKNOWN/" self.assertEqual(expected_message_header, result_message['header']) + @patch('socket.gethostname') + def test_create_message_without_filename(self, socket_gethostname): + """Test creating a message with header/topic, type and content.""" + from nwcsafpps_runner.pps_posttroll_hook import PostTrollMessage + + socket_gethostname.return_value = 'TEST_SERVERNAME' + + metadata = {'publish_topic': '/my/pps/publish/topic/{pps_product}/', + 'output_format': 'CF', + 'level': '2', + 'variant': 'DR', + 'geo_or_polar': 'polar', + 'software': 'NWCSAF-PPSv2018', + 'start_time': START_TIME1, 'end_time': END_TIME1, + 'sensor': 'viirs', + 'filename': '', + 'platform_name': 'npp'} + + posttroll_message = PostTrollMessage(0, metadata) + with patch.object(PostTrollMessage, 'is_segment', return_value=False): + result_message = posttroll_message.create_message('OK') + self.assertFalse("uri" in result_message["content"]) + + @patch('socket.gethostname') def test_create_message_with_topic_pattern(self, socket_gethostname): """Test creating a message with header/topic that is a pattern, type and content.""" diff --git a/nwcsafpps_runner/tests/test_pps_l1c_collector.py b/nwcsafpps_runner/tests/test_pps_l1c_collector.py new file mode 100644 index 0000000..b7624c7 --- /dev/null +++ b/nwcsafpps_runner/tests/test_pps_l1c_collector.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Pytroll Community + +# Author(s): + +# Adam.Dybbroe +# Nina.Hakansson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test the nwp_prepare runner code.""" +import unittest + +import pytest +from posttroll.message import Message +# from posttroll.testing import patched_subscriber_recv +# from nwcsafpps_runner.pps_collector_lib import pps_collector_runner +from nwcsafpps_runner.message_utils import prepare_pps_collector_message +from nwcsafpps_runner.config import get_config +TEST_INPUT_MSG_DATASET = ( + """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + + """2023-05-15T04:30:21.034050 v1.01 application/json """ + + """{"start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:15:38.900000",""" + + """ "orbit_number": 2637,""" + + """ "platform_name": "NOAA-21",""" + + """ "format": "SDR",""" + + """ "type": "HDF5",""" + + """ "data_processing_level": "1B",""" + + """ "variant": "DR",""" + + """ "orig_orbit_number": 2636,""" + + """ "sensor": ["viirs"],""" + + """ "dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"}]}""") + +TEST_INPUT_MSG = ( + """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + + """2023-05-15T04:30:21.034050 v1.01 application/json """ + + """{"start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:15:38.900000",""" + + """ "orbit_number": 2637,""" + + """ "platform_name": "NOAA-21",""" + + """ "format": "SDR",""" + + """ "type": "HDF5",""" + + """ "data_processing_level": "1B",""" + + """ "variant": "DR",""" + + """ "orig_orbit_number": 2636,""" + + """ "sensor": ["viirs"],""" + + """ "collection_area_id": "euron1",""" + + """ "collection": [ """ + + """{"dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"}],""" + + """ "start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:04:15.200000"},""" + + """ {"dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0404164_e0405411_b02637_c20230515041125283206_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0404164_e0405411_b02637_c20230515041125283206_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0404164_e0405411_b02637_c20230515041125323663_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0404164_e0405411_b02637_c20230515041125323663_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125368728_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125368728_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125417060_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125417060_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125464882_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125464882_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0404164_e0405411_b02637_c20230515041125507702_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0404164_e0405411_b02637_c20230515041125507702_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0404164_e0405411_b02637_c20230515041125550239_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0404164_e0405411_b02637_c20230515041125550239_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0404164_e0405411_b02637_c20230515041125593334_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0404164_e0405411_b02637_c20230515041125593334_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0404164_e0405411_b02637_c20230515041125633718_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0404164_e0405411_b02637_c20230515041125633718_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0404164_e0405411_b02637_c20230515041125677125_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0404164_e0405411_b02637_c20230515041125677125_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0404164_e0405411_b02637_c20230515041125718392_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0404164_e0405411_b02637_c20230515041125718392_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0404164_e0405411_b02637_c20230515041125757076_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0404164_e0405411_b02637_c20230515041125757076_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0404164_e0405411_b02637_c20230515041125781624_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0404164_e0405411_b02637_c20230515041125781624_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0404164_e0405411_b02637_c20230515041125826785_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0404164_e0405411_b02637_c20230515041125826785_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0404164_e0405411_b02637_c20230515041125868575_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0404164_e0405411_b02637_c20230515041125868575_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0404164_e0405411_b02637_c20230515041125908912_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0404164_e0405411_b02637_c20230515041125908912_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053201130_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053201130_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053020517_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053020517_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0404164_e0405411_b02637_c20230515041124875834_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0404164_e0405411_b02637_c20230515041124875834_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0404164_e0405411_b02637_c20230515041124975813_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0404164_e0405411_b02637_c20230515041124975813_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125069217_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125069217_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125155142_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125155142_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125244732_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125244732_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0404164_e0405411_b02637_c20230515041052831937_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0404164_e0405411_b02637_c20230515041052831937_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124725653_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124725653_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124790139_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124790139_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc", """ + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"}], """ + + """ "start_time": "2023-05-15T04:04:16.400000", "end_time": "2023-05-15T04:05:41.100000"}]}""") + + +TEST_PPS_COLLECTOR_OK = """ + +subscribe_topics: [/collection/SDR+CF/1+2/CloudProducts] +publish_topic: NWCSAFPPS/2+1C/collection +pps_lvl1c_dir: my_test_dir + +""" + + +@pytest.fixture +def fake_file(tmp_path): + """Create directory with test files.""" + file_cfg = tmp_path / 'pps_collector_config.yaml' + file_h = open(file_cfg, 'w') + file_h.write(TEST_PPS_COLLECTOR_OK.replace("my_test_dir", str(tmp_path))) + file_h.close() + return str(file_cfg) + + +class TestPpsCollector: + """Test the pps collector.""" + + # def test_pps_collector_runner(self, fake_file): + # myconfig_filename = fake_file + # input_msg = Message.decode(rawstr=TEST_INPUT_MSG) + # messages = [input_msg] + # subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) + # with patched_subscriber_recv(messages): + # pps_collector_runner(myconfig_filename) + + def test_prepare_pps_collector_message(self, fake_file): + """Test that meesage is prepared correctly.""" + myconfig_filename = fake_file + options = get_config(myconfig_filename) + input_msg = Message.decode(rawstr=TEST_INPUT_MSG) + output_msg = prepare_pps_collector_message(input_msg, options) + n_level1c_file_included = 0 + for item in output_msg['dataset']: + assert "S_NWC" in item["uid"] + if "S_NWC_viirs" in item["uid"]: + n_level1c_file_included += 1 + assert n_level1c_file_included == 2 + + input_msg = Message.decode(rawstr=TEST_INPUT_MSG_DATASET) + output_msg = prepare_pps_collector_message(input_msg, options) + n_level1c_file_included = 0 + for item in output_msg['dataset']: + assert "S_NWC" in item["uid"] + if "S_NWC_viirs" in item["uid"]: + n_level1c_file_included += 1 + assert n_level1c_file_included == 1 diff --git a/setup.py b/setup.py index e7121b4..5f31624 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ packages=find_packages(), scripts=['bin/pps_runner.py', 'bin/run_nwp_preparation.py', + 'bin/pps_l1c_collector.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', 'level1c4pps'],