From 8edb97fea004122be825492b3d2c878cdb7ceb6b Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 25 Apr 2024 13:13:51 +0200 Subject: [PATCH 01/12] Adding a pps-l1c-collector --- bin/pps_l1c_collector.py | 68 ++++++ nwcsafpps_runner/message_utils.py | 48 +++- nwcsafpps_runner/pps_collector_lib.py | 63 +++++ .../tests/test_pps_l1c_collector.py | 229 ++++++++++++++++++ setup.py | 1 + 5 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 bin/pps_l1c_collector.py create mode 100644 nwcsafpps_runner/pps_collector_lib.py create mode 100644 nwcsafpps_runner/tests/test_pps_l1c_collector.py diff --git a/bin/pps_l1c_collector.py b/bin/pps_l1c_collector.py new file mode 100644 index 0000000..542a716 --- /dev/null +++ b/bin/pps_l1c_collector.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019 - 2021 Pytroll + +# Author(s): + +# Erik Johansson +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Collector to add PPS level1c file to message with PPS files.""" + +import argparse +import logging + +from nwcsafpps_runner.logger import setup_logging +from nwcsatppsrunner.pps_collector_lib import pps_collector_runner + + +LOOP = True + +LOG = logging.getLogger('pps-l1c-collector') + + +def get_arguments(): + """Get command line arguments.""" + parser = argparse.ArgumentParser() + + parser.add_argument("-l", "--log-config", + help="Log config file to use instead of the standard logging.") + parser.add_argument('-c', '--config_file', + type=str, + dest='config_file', + default='l1c_config.yaml', + help="The file containing " + + "configuration parameters e.g. product_filter_config.yaml, \n" + + "default = ./l1c_config.yaml", + required=True) + parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, + help="Verbosity (between 1 and 2 occurrences with more leading to more " + "verbose logging). WARN=0, INFO=1, " + "DEBUG=2. This is overridden by the log config file if specified.") + + args = parser.parse_args() + setup_logging(args) + + if 'template' in args.config_file: + raise IOError("Template file given as master config, aborting!") + + return args + + +if __name__ == '__main__': + + config_file = get_arguments() + pps_collector_runner(config_file) diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index 6eeaec5..a00835a 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -26,10 +26,56 @@ import os from posttroll.message import Message +from nwcsafpps_runner.utils import create_pps_file_from_lvl1c -LOG = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) + +def remove_non_pps_products(msg_data): + for ind in range(len(msg_data["collection"])): + list_of_files_as_dicts = msg_data["collection"][ind]['dataset'] + list_of_files_to_keep = [] + for item in list_of_files_as_dicts: + if "S_NWC" in item["uid"]: + list_of_files_to_keep.append(item) + msg_data["collection"][ind]['dataset'] = list_of_files_to_keep + + +def get_pps_sensor_from_msg(sensor_msg): + """ Get pps sensor from msg sensor.""" + sensor = None + if type(sensor_msg) is list and len(sensor_msg) == 1: + sensor = sensor_msg[0] + if sensor is None: + for pps_sensor in ['viirs', 'avhrr', 'modis', 'mersi2', 'metimage', 'slstr']: + if pps_sensor in sensor_msg: + sensor = pps_sensor + if "avhrr/3" in sensor_msg: + sensor = "avhrr" + return sensor + + +def add_lvl1c_to_msg(msg_data, options): + """Add PPS lvl1c file to a collection of PPS products.""" + level1c_path = os.environ.get('SM_IMAGER_DIR', options.get('pps_lvl1c_dir', './')) + sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"])) + for ind in range(len(msg_data["collection"])): + msg_data["collection"][ind]['dataset'] + pps_file = msg_data["collection"][ind]['dataset'][0]["uri"] + lvl1c_file = create_pps_file_from_lvl1c(pps_file, level1c_path, + name_tag=sensor, file_type='nc') + msg_data["collection"][ind]['dataset'].append({ + "uri": lvl1c_file, + "uid": os.path.basename(lvl1c_file)}) + + +def prepare_pps_collector_message(msg, options): + to_send = msg.data.copy() + remove_non_pps_products(to_send) + add_lvl1c_to_msg(to_send, options) + return to_send + def prepare_nwp_message(result_file, publish_topic): """Prepare message for NWP files.""" to_send = {} diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py new file mode 100644 index 0000000..7dd37d9 --- /dev/null +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018 - 2021 Pytroll Developers + +# Author(s): + +# Nina Hakansson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import signal +import logging + +from posttroll.publisher import Publish +from posttroll.subscriber import Subscribe +from nwcsafpps_runner.config import get_config + +from nwcsafpps_runner.message_utils import (publish_l1c, + prepare_pps_collector_message) + +LOG = logging.getLogger(__name__) +LOOP = True + + +def _run_subscribe_publisher(subscriber, publisher, options): + """The posttroll subscribe/publisher runner.""" + def signal_handler(sig, frame): + LOG.warning('You pressed Ctrl+C!') + global LOOP + LOOP = False + + signal.signal(signal.SIGINT, signal_handler) + + while LOOP: + for msg in subscriber.recv(): + LOG.debug( + "Received message data = %s", msg) + pub_msg = prepare_pps_collector_message(msg, options) + publish_l1c(publisher, pub_msg, publish_topic=[options["publish_topic"]]) + LOG.info("L1c and PPS products collected.") + +def pps_collector_runner(config_file): + """The live runner for collecting the NWCSAF/PPS l1c and lvl2 products.""" + LOG.info("Start the NWCSAF/PPS products and level-1c collector runner") + + options = get_config(config_file) + publish_name = 'pps-collector-runner' + with Subscribe('', options["subscribe_topics"], True) as sub: + with Publish(publish_name, 0) as pub: + _run_subscribe_publisher(sub, pub, options) diff --git a/nwcsafpps_runner/tests/test_pps_l1c_collector.py b/nwcsafpps_runner/tests/test_pps_l1c_collector.py new file mode 100644 index 0000000..12ec122 --- /dev/null +++ b/nwcsafpps_runner/tests/test_pps_l1c_collector.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Pytroll Community + +# Author(s): + +# Adam.Dybbroe +# Nina.Hakansson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test the nwp_prepare runner code.""" +import logging +import os +import unittest +from datetime import datetime, timedelta, timezone + +import pytest +from posttroll.message import Message +from posttroll.testing import patched_subscriber_recv +from nwcsafpps_runner.pps_collector_lib import pps_collector_runner +from nwcsafpps_runner.message_utils import prepare_pps_collector_message +from nwcsafpps_runner.config import get_config + +TEST_INPUT_MSG = ( + """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + + """2023-05-15T04:30:21.034050 v1.01 application/json """ + + """{"start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:15:38.900000",""" + + """ "orbit_number": 2637,""" + + """ "platform_name": "NOAA-21",""" + + """ "format": "SDR",""" + + """ "type": "HDF5",""" + + """ "data_processing_level": "1B",""" + + """ "variant": "DR",""" + + """ "orig_orbit_number": 2636,""" + + """ "sensor": ["viirs"],""" + + """ "collection_area_id": "euron1",""" + + """ "collection": [ """ + + """{"dataset": [{"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"}],""" + + """ "start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:04:15.200000"},""" + + """ {"dataset": [{"uri": "/my_dir/GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0404164_e0405411_b02637_c20230515041125283206_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0404164_e0405411_b02637_c20230515041125283206_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0404164_e0405411_b02637_c20230515041125323663_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0404164_e0405411_b02637_c20230515041125323663_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125368728_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125368728_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125417060_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125417060_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125464882_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125464882_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0404164_e0405411_b02637_c20230515041125507702_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0404164_e0405411_b02637_c20230515041125507702_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0404164_e0405411_b02637_c20230515041125550239_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0404164_e0405411_b02637_c20230515041125550239_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0404164_e0405411_b02637_c20230515041125593334_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0404164_e0405411_b02637_c20230515041125593334_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0404164_e0405411_b02637_c20230515041125633718_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0404164_e0405411_b02637_c20230515041125633718_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0404164_e0405411_b02637_c20230515041125677125_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0404164_e0405411_b02637_c20230515041125677125_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0404164_e0405411_b02637_c20230515041125718392_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0404164_e0405411_b02637_c20230515041125718392_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0404164_e0405411_b02637_c20230515041125757076_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0404164_e0405411_b02637_c20230515041125757076_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0404164_e0405411_b02637_c20230515041125781624_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0404164_e0405411_b02637_c20230515041125781624_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0404164_e0405411_b02637_c20230515041125826785_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0404164_e0405411_b02637_c20230515041125826785_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0404164_e0405411_b02637_c20230515041125868575_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0404164_e0405411_b02637_c20230515041125868575_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0404164_e0405411_b02637_c20230515041125908912_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0404164_e0405411_b02637_c20230515041125908912_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053201130_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053201130_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053020517_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053020517_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0404164_e0405411_b02637_c20230515041124875834_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0404164_e0405411_b02637_c20230515041124875834_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0404164_e0405411_b02637_c20230515041124975813_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0404164_e0405411_b02637_c20230515041124975813_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125069217_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0404164_e0405411_b02637_c20230515041125069217_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125155142_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0404164_e0405411_b02637_c20230515041125155142_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125244732_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0404164_e0405411_b02637_c20230515041125244732_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0404164_e0405411_b02637_c20230515041052831937_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0404164_e0405411_b02637_c20230515041052831937_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124725653_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124725653_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124790139_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0404164_e0405411_b02637_c20230515041124790139_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc", """ + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0404164Z_20230515T0405411Z.nc"}], """ + + """ "start_time": "2023-05-15T04:04:16.400000", "end_time": "2023-05-15T04:05:41.100000"}]}""") + + +TEST_PPS_COLLECTOR_OK = """ + +subscribe_topics: [/collection/SDR+CF/1+2/CloudProducts] +publish_topic: NWCSAFPPS/2+1C/collection +pps_lvl1c_dir: my_test_dir + +""" + +@pytest.fixture +def fake_file(tmp_path): + """Create directory with test files.""" + file_cfg = tmp_path / 'pps_collector_config.yaml' + file_h = open(file_cfg, 'w') + file_h.write(TEST_PPS_COLLECTOR_OK.replace("my_test_dir", str(tmp_path))) + file_h.close() + return str(file_cfg) + +class TestPpsCollector: + """Test the pps collector""" + + # def test_pps_collector_runner(self, fake_file): + # myconfig_filename = fake_file + # input_msg = Message.decode(rawstr=TEST_INPUT_MSG) + # messages = [input_msg] + # subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) + # with patched_subscriber_recv(messages): + # pps_collector_runner(myconfig_filename) + + def test_prepare_pps_collector_message(self, fake_file): + myconfig_filename = fake_file + options = get_config(myconfig_filename) + input_msg = Message.decode(rawstr=TEST_INPUT_MSG) + output_msg = prepare_pps_collector_message(input_msg, options) + level1c_file_included = False + for index in [0, 1]: + level1c_file_included = False + for item in output_msg["collection"][index]['dataset']: + assert "S_NWC" in item["uid"] + if "S_NWC_viirs" in item["uid"]: + level1c_file_included = True + assert level1c_file_included diff --git a/setup.py b/setup.py index e7121b4..5f31624 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ packages=find_packages(), scripts=['bin/pps_runner.py', 'bin/run_nwp_preparation.py', + 'bin/pps_l1c_collector.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', 'level1c4pps'], From 2f413de1da24cfae0a9ebfc9ff28416d3339738c Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 25 Apr 2024 13:19:02 +0200 Subject: [PATCH 02/12] flake8 --- .../tests/test_pps_l1c_collector.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/nwcsafpps_runner/tests/test_pps_l1c_collector.py b/nwcsafpps_runner/tests/test_pps_l1c_collector.py index 12ec122..2bbb390 100644 --- a/nwcsafpps_runner/tests/test_pps_l1c_collector.py +++ b/nwcsafpps_runner/tests/test_pps_l1c_collector.py @@ -22,21 +22,18 @@ # along with this program. If not, see . """Test the nwp_prepare runner code.""" -import logging -import os import unittest -from datetime import datetime, timedelta, timezone import pytest from posttroll.message import Message -from posttroll.testing import patched_subscriber_recv -from nwcsafpps_runner.pps_collector_lib import pps_collector_runner +# from posttroll.testing import patched_subscriber_recv +# from nwcsafpps_runner.pps_collector_lib import pps_collector_runner from nwcsafpps_runner.message_utils import prepare_pps_collector_message from nwcsafpps_runner.config import get_config TEST_INPUT_MSG = ( """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + - """2023-05-15T04:30:21.034050 v1.01 application/json """ + + """2023-05-15T04:30:21.034050 v1.01 application/json """ + """{"start_time": "2023-05-15T04:02:52.300000",""" + """ "end_time": "2023-05-15T04:15:38.900000",""" + """ "orbit_number": 2637,""" + @@ -49,7 +46,8 @@ """ "sensor": ["viirs"],""" + """ "collection_area_id": "euron1",""" + """ "collection": [ """ + - """{"dataset": [{"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + + """{"dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + """ "uid": "GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5"},""" + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5",""" + """ "uid": "GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5"},""" + @@ -117,7 +115,8 @@ """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"}],""" + """ "start_time": "2023-05-15T04:02:52.300000",""" + """ "end_time": "2023-05-15T04:04:15.200000"},""" + - """ {"dataset": [{"uri": "/my_dir/GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5",""" + + """ {"dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5",""" + """ "uid": "GMODO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053656421_cspp_dev.h5"},""" + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5",""" + """ "uid": "GMTCO_j02_d20230515_t0404164_e0405411_b02637_c20230515041053605930_cspp_dev.h5"},""" + @@ -193,7 +192,8 @@ pps_lvl1c_dir: my_test_dir """ - + + @pytest.fixture def fake_file(tmp_path): """Create directory with test files.""" @@ -203,9 +203,10 @@ def fake_file(tmp_path): file_h.close() return str(file_cfg) + class TestPpsCollector: - """Test the pps collector""" - + """Test the pps collector.""" + # def test_pps_collector_runner(self, fake_file): # myconfig_filename = fake_file # input_msg = Message.decode(rawstr=TEST_INPUT_MSG) @@ -215,6 +216,7 @@ class TestPpsCollector: # pps_collector_runner(myconfig_filename) def test_prepare_pps_collector_message(self, fake_file): + """Test that meesage is prepared correctly.""" myconfig_filename = fake_file options = get_config(myconfig_filename) input_msg = Message.decode(rawstr=TEST_INPUT_MSG) From 1a622e43818c5be49843b16a754443942256fc01 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 25 Apr 2024 14:55:23 +0200 Subject: [PATCH 03/12] fix syntax errors --- bin/pps_l1c_collector.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/pps_l1c_collector.py b/bin/pps_l1c_collector.py index 542a716..100d93a 100644 --- a/bin/pps_l1c_collector.py +++ b/bin/pps_l1c_collector.py @@ -26,7 +26,7 @@ import logging from nwcsafpps_runner.logger import setup_logging -from nwcsatppsrunner.pps_collector_lib import pps_collector_runner +from nwcsafpps_runner.pps_collector_lib import pps_collector_runner LOOP = True @@ -43,10 +43,10 @@ def get_arguments(): parser.add_argument('-c', '--config_file', type=str, dest='config_file', - default='l1c_config.yaml', + default='config.yaml', help="The file containing " + - "configuration parameters e.g. product_filter_config.yaml, \n" + - "default = ./l1c_config.yaml", + "configuration parameters, \n" + + "default = ./config.yaml", required=True) parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, help="Verbosity (between 1 and 2 occurrences with more leading to more " From 707d6238cbb4f9c928f4650ae7803638ac966964 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 25 Apr 2024 15:09:13 +0200 Subject: [PATCH 04/12] Bugfix --- bin/pps_l1c_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pps_l1c_collector.py b/bin/pps_l1c_collector.py index 100d93a..9510a86 100644 --- a/bin/pps_l1c_collector.py +++ b/bin/pps_l1c_collector.py @@ -59,7 +59,7 @@ def get_arguments(): if 'template' in args.config_file: raise IOError("Template file given as master config, aborting!") - return args + return args.config_file if __name__ == '__main__': From cf086166f78d517e52dda7471b1fb477ec1f1bd6 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Mon, 29 Apr 2024 13:44:43 +0200 Subject: [PATCH 05/12] Flatten collection into dataset --- nwcsafpps_runner/message_utils.py | 49 +++++---- nwcsafpps_runner/pps_collector_lib.py | 2 +- .../tests/test_pps_l1c_collector.py | 103 ++++++++++++++++-- 3 files changed, 125 insertions(+), 29 deletions(-) diff --git a/nwcsafpps_runner/message_utils.py b/nwcsafpps_runner/message_utils.py index a00835a..4586a59 100644 --- a/nwcsafpps_runner/message_utils.py +++ b/nwcsafpps_runner/message_utils.py @@ -31,15 +31,10 @@ LOG = logging.getLogger(__name__) - + def remove_non_pps_products(msg_data): - for ind in range(len(msg_data["collection"])): - list_of_files_as_dicts = msg_data["collection"][ind]['dataset'] - list_of_files_to_keep = [] - for item in list_of_files_as_dicts: - if "S_NWC" in item["uid"]: - list_of_files_to_keep.append(item) - msg_data["collection"][ind]['dataset'] = list_of_files_to_keep + """ Remove non-PPS files from datasetlis.t""" + msg_data["dataset"] = [item for item in msg_data["dataset"] if "S_NWC" in item["uid"]] def get_pps_sensor_from_msg(sensor_msg): @@ -47,7 +42,7 @@ def get_pps_sensor_from_msg(sensor_msg): sensor = None if type(sensor_msg) is list and len(sensor_msg) == 1: sensor = sensor_msg[0] - if sensor is None: + if sensor is None: for pps_sensor in ['viirs', 'avhrr', 'modis', 'mersi2', 'metimage', 'slstr']: if pps_sensor in sensor_msg: sensor = pps_sensor @@ -59,23 +54,37 @@ def get_pps_sensor_from_msg(sensor_msg): def add_lvl1c_to_msg(msg_data, options): """Add PPS lvl1c file to a collection of PPS products.""" level1c_path = os.environ.get('SM_IMAGER_DIR', options.get('pps_lvl1c_dir', './')) - sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"])) - for ind in range(len(msg_data["collection"])): - msg_data["collection"][ind]['dataset'] - pps_file = msg_data["collection"][ind]['dataset'][0]["uri"] - lvl1c_file = create_pps_file_from_lvl1c(pps_file, level1c_path, - name_tag=sensor, file_type='nc') - msg_data["collection"][ind]['dataset'].append({ + sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"])) + num_files = len(msg_data['dataset']) + to_add = {} + for item in msg_data['dataset']: + lvl1c_file = create_pps_file_from_lvl1c(item["uri"], level1c_path, + name_tag=sensor, file_type='.nc') + to_add[lvl1c_file] = { "uri": lvl1c_file, - "uid": os.path.basename(lvl1c_file)}) + "uid": os.path.basename(lvl1c_file)} + msg_data['dataset'].extend(to_add.values()) + + +def flatten_collection(msg_data): + """Flatten collection msg to dataset msg.""" + if "collection" in msg_data: + collection = msg_data.pop("collection") + msg_data["dataset"] = [] + for ind in range(0, len(collection)): + for item in collection[ind]["dataset"]: + if type(item) == dict: + msg_data["dataset"].append(item) def prepare_pps_collector_message(msg, options): to_send = msg.data.copy() + flatten_collection(to_send) remove_non_pps_products(to_send) add_lvl1c_to_msg(to_send, options) return to_send - + + def prepare_nwp_message(result_file, publish_topic): """Prepare message for NWP files.""" to_send = {} @@ -119,10 +128,10 @@ def prepare_l1c_message(result_file, mda, **kwargs): return to_send -def publish_l1c(publisher, publish_msg, publish_topic): +def publish_l1c(publisher, publish_msg, publish_topic, msg_type="file"): """Publish the messages that l1c files are ready.""" LOG.debug('Publish topic = %s', publish_topic) for topic in publish_topic: - msg = Message(topic, "file", publish_msg).encode() + msg = Message(topic, msg_type, publish_msg).encode() LOG.debug("sending: %s", str(msg)) publisher.send(msg) diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py index 7dd37d9..590d87d 100644 --- a/nwcsafpps_runner/pps_collector_lib.py +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -49,7 +49,7 @@ def signal_handler(sig, frame): LOG.debug( "Received message data = %s", msg) pub_msg = prepare_pps_collector_message(msg, options) - publish_l1c(publisher, pub_msg, publish_topic=[options["publish_topic"]]) + publish_l1c(publisher, pub_msg, publish_topic=[options["publish_topic"]], msg_type="dataset") LOG.info("L1c and PPS products collected.") def pps_collector_runner(config_file): diff --git a/nwcsafpps_runner/tests/test_pps_l1c_collector.py b/nwcsafpps_runner/tests/test_pps_l1c_collector.py index 2bbb390..b7624c7 100644 --- a/nwcsafpps_runner/tests/test_pps_l1c_collector.py +++ b/nwcsafpps_runner/tests/test_pps_l1c_collector.py @@ -30,6 +30,86 @@ # from nwcsafpps_runner.pps_collector_lib import pps_collector_runner from nwcsafpps_runner.message_utils import prepare_pps_collector_message from nwcsafpps_runner.config import get_config +TEST_INPUT_MSG_DATASET = ( + """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + + """2023-05-15T04:30:21.034050 v1.01 application/json """ + + """{"start_time": "2023-05-15T04:02:52.300000",""" + + """ "end_time": "2023-05-15T04:15:38.900000",""" + + """ "orbit_number": 2637,""" + + """ "platform_name": "NOAA-21",""" + + """ "format": "SDR",""" + + """ "type": "HDF5",""" + + """ "data_processing_level": "1B",""" + + """ "variant": "DR",""" + + """ "orig_orbit_number": 2636,""" + + """ "sensor": ["viirs"],""" + + """ "dataset": [""" + + """ {"uri": "/my_dir/GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5",""" + + """ "uid": "GMODO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842931901_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5",""" + + """ "uid": "GMTCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842847426_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5",""" + + """ "uid": "SVM01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918683116_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5",""" + + """ "uid": "SVM02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918729002_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5",""" + + """ "uid": "SVM03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918778479_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5",""" + + """ "uid": "SVM04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918824679_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5",""" + + """ "uid": "SVM05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918871757_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5",""" + + """ "uid": "SVM06_j02_d20230515_t0402523_e0404152_b02637_c20230515040918926725_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5",""" + + """ "uid": "SVM07_j02_d20230515_t0402523_e0404152_b02637_c20230515040918982899_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5",""" + + """ "uid": "SVM08_j02_d20230515_t0402523_e0404152_b02637_c20230515040919028526_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5",""" + + """ "uid": "SVM09_j02_d20230515_t0402523_e0404152_b02637_c20230515040919069935_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5",""" + + """ "uid": "SVM10_j02_d20230515_t0402523_e0404152_b02637_c20230515040919110030_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5",""" + + """ "uid": "SVM11_j02_d20230515_t0402523_e0404152_b02637_c20230515040919155907_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5",""" + + """ "uid": "SVM12_j02_d20230515_t0402523_e0404152_b02637_c20230515040919206051_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5",""" + + """ "uid": "SVM13_j02_d20230515_t0402523_e0404152_b02637_c20230515040919232307_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5",""" + + """ "uid": "SVM14_j02_d20230515_t0402523_e0404152_b02637_c20230515040919281872_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5",""" + + """ "uid": "SVM15_j02_d20230515_t0402523_e0404152_b02637_c20230515040919325359_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5",""" + + """ "uid": "SVM16_j02_d20230515_t0402523_e0404152_b02637_c20230515040919379332_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5",""" + + """ "uid": "GIMGO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842366314_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5",""" + + """ "uid": "GITCO_j02_d20230515_t0402523_e0404152_b02637_c20230515040842104190_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5",""" + + """ "uid": "SVI01_j02_d20230515_t0402523_e0404152_b02637_c20230515040918210080_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5",""" + + """ "uid": "SVI02_j02_d20230515_t0402523_e0404152_b02637_c20230515040918311250_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5",""" + + """ "uid": "SVI03_j02_d20230515_t0402523_e0404152_b02637_c20230515040918417310_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5",""" + + """ "uid": "SVI04_j02_d20230515_t0402523_e0404152_b02637_c20230515040918522149_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5",""" + + """ "uid": "SVI05_j02_d20230515_t0402523_e0404152_b02637_c20230515040918632921_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5",""" + + """ "uid": "GDNBO_j02_d20230515_t0402523_e0404152_b02637_c20230515040841929317_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5",""" + + """ "uid": "SVDNB_j02_d20230515_t0402523_e0404152_b02637_c20230515040917932681_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5",""" + + """ "uid": "IVCDB_j02_d20230515_t0402523_e0404152_b02637_c20230515040918055123_cspp_dev.h5"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMA_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CTTH_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CT_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMIC_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"},""" + + """ {"uri": "/my_dir/lvl2/S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc",""" + + """ "uid": "S_NWC_CMAPROB_noaa21_00000_20230515T0402523Z_20230515T0404152Z.nc"}]}""") TEST_INPUT_MSG = ( """pytroll://collection/SDR+CF/1+2/CloudProducts/ collection auser@some.server.se """ + @@ -221,11 +301,18 @@ def test_prepare_pps_collector_message(self, fake_file): options = get_config(myconfig_filename) input_msg = Message.decode(rawstr=TEST_INPUT_MSG) output_msg = prepare_pps_collector_message(input_msg, options) - level1c_file_included = False - for index in [0, 1]: - level1c_file_included = False - for item in output_msg["collection"][index]['dataset']: - assert "S_NWC" in item["uid"] - if "S_NWC_viirs" in item["uid"]: - level1c_file_included = True - assert level1c_file_included + n_level1c_file_included = 0 + for item in output_msg['dataset']: + assert "S_NWC" in item["uid"] + if "S_NWC_viirs" in item["uid"]: + n_level1c_file_included += 1 + assert n_level1c_file_included == 2 + + input_msg = Message.decode(rawstr=TEST_INPUT_MSG_DATASET) + output_msg = prepare_pps_collector_message(input_msg, options) + n_level1c_file_included = 0 + for item in output_msg['dataset']: + assert "S_NWC" in item["uid"] + if "S_NWC_viirs" in item["uid"]: + n_level1c_file_included += 1 + assert n_level1c_file_included == 1 From 3b51746d73aacdfcae11130512981ad559e617f3 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 15 May 2024 14:47:42 +0200 Subject: [PATCH 06/12] Use specific port from config file --- nwcsafpps_runner/pps_collector_lib.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py index 590d87d..b0fd0da 100644 --- a/nwcsafpps_runner/pps_collector_lib.py +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -24,7 +24,7 @@ import signal import logging -from posttroll.publisher import Publish +from posttroll.publisher import create_publisher_from_dict_config from posttroll.subscriber import Subscribe from nwcsafpps_runner.config import get_config @@ -57,7 +57,9 @@ def pps_collector_runner(config_file): LOG.info("Start the NWCSAF/PPS products and level-1c collector runner") options = get_config(config_file) - publish_name = 'pps-collector-runner' + settings = {"name": 'pps-collector-runner', + "nameservers": False, + "port": options.get("publish_port", 0)} with Subscribe('', options["subscribe_topics"], True) as sub: - with Publish(publish_name, 0) as pub: - _run_subscribe_publisher(sub, pub, options) + pub = create_publisher_from_dict_config(settings): + _run_subscribe_publisher(sub, pub, options) From e67878125246da9a83f8c8427ca52f0915bc6a3c Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 15 May 2024 14:58:23 +0200 Subject: [PATCH 07/12] syntax fix --- nwcsafpps_runner/pps_collector_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py index b0fd0da..81838a9 100644 --- a/nwcsafpps_runner/pps_collector_lib.py +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -61,5 +61,5 @@ def pps_collector_runner(config_file): "nameservers": False, "port": options.get("publish_port", 0)} with Subscribe('', options["subscribe_topics"], True) as sub: - pub = create_publisher_from_dict_config(settings): + pub = create_publisher_from_dict_config(settings) _run_subscribe_publisher(sub, pub, options) From 0fe3526aa87fb24627095c04b1d5201292549fd5 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Thu, 16 May 2024 11:01:03 +0200 Subject: [PATCH 08/12] Start the publisher --- nwcsafpps_runner/pps_collector_lib.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nwcsafpps_runner/pps_collector_lib.py b/nwcsafpps_runner/pps_collector_lib.py index 81838a9..a516146 100644 --- a/nwcsafpps_runner/pps_collector_lib.py +++ b/nwcsafpps_runner/pps_collector_lib.py @@ -24,6 +24,7 @@ import signal import logging +from contextlib import closing, suppress from posttroll.publisher import create_publisher_from_dict_config from posttroll.subscriber import Subscribe from nwcsafpps_runner.config import get_config @@ -59,7 +60,8 @@ def pps_collector_runner(config_file): options = get_config(config_file) settings = {"name": 'pps-collector-runner', "nameservers": False, - "port": options.get("publish_port", 0)} + "port": options.get("publish_port", 3002)} with Subscribe('', options["subscribe_topics"], True) as sub: - pub = create_publisher_from_dict_config(settings) - _run_subscribe_publisher(sub, pub, options) + with closing(create_publisher_from_dict_config(settings)) as pub: + pub.start() + _run_subscribe_publisher(sub, pub, options) From a2d62fbfdf38941a9ca1027336dcb9c6b029e1aa Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 29 May 2024 08:47:51 +0200 Subject: [PATCH 09/12] Fix posttroll hook for HRW --- nwcsafpps_runner/pps_posttroll_hook.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nwcsafpps_runner/pps_posttroll_hook.py b/nwcsafpps_runner/pps_posttroll_hook.py index b757822..f9759ec 100644 --- a/nwcsafpps_runner/pps_posttroll_hook.py +++ b/nwcsafpps_runner/pps_posttroll_hook.py @@ -73,6 +73,7 @@ 'ppsCtype': 'CT', 'ppsCpp': 'CPP', 'ppsCmic': 'CMIC', + 'ppsHrw': 'HRW', 'ppsPrecip': 'PC', 'ppsPrecipPrepare': 'PC-PRE'} @@ -287,8 +288,10 @@ def create_message(self, status): self.clean_unused_keys_in_message() publish_topic = self._create_message_topic() - - return {'header': publish_topic, 'type': 'file', 'content': self._to_send} + msg_type = 'file' + if "dataset" in self._to_send: + msg_type = 'dataset' + return {'header': publish_topic, 'type': msg_type, 'content': self._to_send} def _create_message_topic(self): """Create the publish topic from yaml file items and PPS metadata.""" From e7a399b311ec7ae564ef44b43220936263268fe8 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Wed, 29 May 2024 10:13:40 +0200 Subject: [PATCH 10/12] Add option to produce time control from xml separately --- bin/pps_runner.py | 2 +- bin/produce_pps_time_xml_stats.py | 37 ++++++++++++++++++++++++++++ nwcsafpps_runner/tests/test_utils.py | 23 +++++++++++------ nwcsafpps_runner/utils.py | 29 ++++++++++++++-------- setup.py | 1 + 5 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 bin/produce_pps_time_xml_stats.py diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 81d313a..5eca8ca 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -165,7 +165,7 @@ def pps_worker(scene, publish_q, input_msg, options): err_reader2.join() pps_control_path = my_env.get('SM_STATISTICS_DIR', options.get('pps_statistics_dir', './')) - xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) + xml_files = create_xml_timestat_from_lvl1c(scene['file4pps'], pps_control_path) xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) LOG.info("PPS summary statistics files: %s", str(xml_files)) diff --git a/bin/produce_pps_time_xml_stats.py b/bin/produce_pps_time_xml_stats.py new file mode 100644 index 0000000..f27818a --- /dev/null +++ b/bin/produce_pps_time_xml_stats.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018 - 2022 Pytroll Developers + +# Author(s): + +# Nina.Hakansson + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +from nwcsafpps_runner.utils import process_timectrl_xml_from_pps_result_file_with_extension + +if __name__ == "__main__": + """From a PPS-file create the time-control xml file.""" + parser = argparse.ArgumentParser( + description=('Script to produce a timectrl xml file from a PPS-file')) + parser.add_argument('-f', '--pps_file', required=True, type=str, help='A PPS file, for which timectrl in xml format is needed') + parser.add_argument('--extension', required=False, default="", + help="Output filename will be {pps-timectrl-file-name-prefix}{extension}.xml") + parser.add_argument('--pps_statistics_dir', type=str, nargs='?', + required=False, default=None, + help="Output directory where to store the xml file") + options = parser.parse_args() + process_timectrl_xml_from_pps_result_file_with_extension(options.pps_file, options.pps_statistics_dir, options.extension) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index f3c7ebe..0b1da55 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -27,6 +27,7 @@ from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, + process_timectrl_xml_from_pps_result_file_with_extension, get_lvl1c_file_from_msg, publish_pps_files, ready2run) @@ -85,18 +86,30 @@ def test_xml_for_timectrl(self, fake_file_dir): mymodule = MagicMock() import sys sys.modules["pps_time_control"] = mymodule - res = create_xml_timestat_from_lvl1c(self.scene, mydir_out) + res = create_xml_timestat_from_lvl1c(self.scene['file4pps'], mydir_out) expected = [os.path.join(mydir_out, "S_NWC_timectrl_npp_12345_19810305T0715000Z_19810305T0730000Z.xml")] assert len(res) == len(set(expected)) assert set(res) == set(expected) + def test_xml_for_timectrl_from_product_with_extension(self, fake_file_dir): + """Test xml files for timectrl.""" + mydir, mydir_out = fake_file_dir + mymodule = MagicMock() + import sys + sys.modules["pps_time_control"] = mymodule + res = process_timectrl_xml_from_pps_result_file_with_extension( + "S_NWC_HRWbs_npp_12345_19810305T0715000Z_19810305T0730000Z.pdf", mydir_out, extension="_hrw") + expected = [os.path.join(mydir_out, "S_NWC_timectrl_npp_12345_19810305T0715000Z_19810305T0730000Z_hrw.xml")] + assert len(res) == len(set(expected)) + assert set(res) == set(expected) + def test_xml_for_timectrl_files_missing(self, fake_file_dir): """Test xml files for timectrl.""" mydir, mydir_out = fake_file_dir mymodule = MagicMock() import sys sys.modules["pps_time_control"] = mymodule - res = create_xml_timestat_from_lvl1c(self.scene, mydir) # Look in wrong place + res = create_xml_timestat_from_lvl1c(self.scene['file4pps'], mydir) # Look in wrong place expected = [] assert res == expected @@ -111,12 +124,6 @@ def test_xml_for_products(self, fake_file_dir): assert len(res) == len(set(expected)) assert set(res) == set(expected) - def test_xml_for_timectrl_no_file4pps(self, fake_file_dir): - """Test xml files for timectrl without file4pps attribute.""" - mydir, mydir_out = fake_file_dir - res = create_xml_timestat_from_lvl1c(self.empty_scene, mydir_out) - assert res == [] - def test_xml_for_products_no_file4pps(self, fake_file_dir): """Test xml files for products without file4pps attribute.""" mydir, mydir_out = fake_file_dir diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 87b76b9..78bf307 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -234,14 +234,11 @@ def create_pps_call_command(python_exec, pps_script_name, scene): return cmdstr -def create_xml_timestat_from_lvl1c(scene, pps_control_path): +def create_xml_timestat_from_lvl1c(pps_file, pps_control_path): """From lvl1c file create XML file and return a file list.""" - try: - txt_time_control = create_pps_file_from_lvl1c(scene['file4pps'], pps_control_path, "timectrl", ".txt") - except KeyError: - return [] + txt_time_control = create_pps_file_from_lvl1c(pps_file, pps_control_path, "timectrl", ".txt") if os.path.exists(txt_time_control): - return create_xml_timestat_from_ascii(txt_time_control, pps_control_path) + return create_xml_timestat_from_ascii(txt_time_control) else: LOG.warning('No XML Time statistics file created!') return [] @@ -256,6 +253,17 @@ def find_product_statistics_from_lvl1c(scene, pps_control_path): return [] +def process_timectrl_xml_from_pps_result_file_with_extension(filename, export_path, extension): + if export_path is None: + export_path = os.path.dirname(filename) + print(export_path) + timectrl_file = create_pps_file_from_lvl1c(filename, export_path, "timectrl", ".txt") + timectrl_xmlfile = timectrl_file.replace(".txt", extension + ".xml") + print(timectrl_file) + result_file = create_xml_timestat_from_ascii(timectrl_file, outfile=timectrl_xmlfile) + return result_file + + def create_pps_file_from_lvl1c(l1c_file_name, pps_control_path, name_tag, file_type): """From lvl1c file create name_tag-file of type file_type.""" from trollsift import compose, parse @@ -267,8 +275,10 @@ def create_pps_file_from_lvl1c(l1c_file_name, pps_control_path, name_tag, file_t return os.path.join(pps_control_path, compose(f_pattern, data)) -def create_xml_timestat_from_ascii(infile, pps_control_path): +def create_xml_timestat_from_ascii(infile, outfile=None): """From ascii file(s) with PPS time statistics create XML file(s) and return a file list.""" + if outfile is None: + outfile = infile.replace('.txt', '.xml') try: from pps_time_control import PPSTimeControl except ImportError: @@ -279,13 +289,12 @@ def create_xml_timestat_from_ascii(infile, pps_control_path): ppstime_con = PPSTimeControl(infile) ppstime_con.sum_up_processing_times() try: - ppstime_con.write_xml() + ppstime_con.write_xml(filename=outfile) except Exception as e: # TypeError as e: LOG.warning('Not able to write time control xml file') LOG.warning(e) - # There should always be only one xml file for each ascii file found above! - return [infile.replace('.txt', '.xml')] + return [outfile] def publish_pps_files(input_msg, publish_q, scene, result_files, **kwargs): diff --git a/setup.py b/setup.py index 5f31624..deebf2d 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ scripts=['bin/pps_runner.py', 'bin/run_nwp_preparation.py', 'bin/pps_l1c_collector.py', + 'bin/produce_pps_time_xml_stats.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', 'level1c4pps'], From 92e49dcb6a49439edb7a69383e4b91bb86c81fa8 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 13 Aug 2024 14:03:15 +0200 Subject: [PATCH 11/12] Revert "Add option to produce time control from xml separately" This reverts commit e7a399b311ec7ae564ef44b43220936263268fe8. --- bin/pps_runner.py | 2 +- bin/produce_pps_time_xml_stats.py | 37 ---------------------------- nwcsafpps_runner/tests/test_utils.py | 23 ++++++----------- nwcsafpps_runner/utils.py | 29 ++++++++-------------- setup.py | 1 - 5 files changed, 19 insertions(+), 73 deletions(-) delete mode 100644 bin/produce_pps_time_xml_stats.py diff --git a/bin/pps_runner.py b/bin/pps_runner.py index 5eca8ca..81d313a 100644 --- a/bin/pps_runner.py +++ b/bin/pps_runner.py @@ -165,7 +165,7 @@ def pps_worker(scene, publish_q, input_msg, options): err_reader2.join() pps_control_path = my_env.get('SM_STATISTICS_DIR', options.get('pps_statistics_dir', './')) - xml_files = create_xml_timestat_from_lvl1c(scene['file4pps'], pps_control_path) + xml_files = create_xml_timestat_from_lvl1c(scene, pps_control_path) xml_files += find_product_statistics_from_lvl1c(scene, pps_control_path) LOG.info("PPS summary statistics files: %s", str(xml_files)) diff --git a/bin/produce_pps_time_xml_stats.py b/bin/produce_pps_time_xml_stats.py deleted file mode 100644 index f27818a..0000000 --- a/bin/produce_pps_time_xml_stats.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright (c) 2018 - 2022 Pytroll Developers - -# Author(s): - -# Nina.Hakansson - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -from nwcsafpps_runner.utils import process_timectrl_xml_from_pps_result_file_with_extension - -if __name__ == "__main__": - """From a PPS-file create the time-control xml file.""" - parser = argparse.ArgumentParser( - description=('Script to produce a timectrl xml file from a PPS-file')) - parser.add_argument('-f', '--pps_file', required=True, type=str, help='A PPS file, for which timectrl in xml format is needed') - parser.add_argument('--extension', required=False, default="", - help="Output filename will be {pps-timectrl-file-name-prefix}{extension}.xml") - parser.add_argument('--pps_statistics_dir', type=str, nargs='?', - required=False, default=None, - help="Output directory where to store the xml file") - options = parser.parse_args() - process_timectrl_xml_from_pps_result_file_with_extension(options.pps_file, options.pps_statistics_dir, options.extension) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 0b1da55..f3c7ebe 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -27,7 +27,6 @@ from nwcsafpps_runner.utils import (create_xml_timestat_from_lvl1c, find_product_statistics_from_lvl1c, - process_timectrl_xml_from_pps_result_file_with_extension, get_lvl1c_file_from_msg, publish_pps_files, ready2run) @@ -86,30 +85,18 @@ def test_xml_for_timectrl(self, fake_file_dir): mymodule = MagicMock() import sys sys.modules["pps_time_control"] = mymodule - res = create_xml_timestat_from_lvl1c(self.scene['file4pps'], mydir_out) + res = create_xml_timestat_from_lvl1c(self.scene, mydir_out) expected = [os.path.join(mydir_out, "S_NWC_timectrl_npp_12345_19810305T0715000Z_19810305T0730000Z.xml")] assert len(res) == len(set(expected)) assert set(res) == set(expected) - def test_xml_for_timectrl_from_product_with_extension(self, fake_file_dir): - """Test xml files for timectrl.""" - mydir, mydir_out = fake_file_dir - mymodule = MagicMock() - import sys - sys.modules["pps_time_control"] = mymodule - res = process_timectrl_xml_from_pps_result_file_with_extension( - "S_NWC_HRWbs_npp_12345_19810305T0715000Z_19810305T0730000Z.pdf", mydir_out, extension="_hrw") - expected = [os.path.join(mydir_out, "S_NWC_timectrl_npp_12345_19810305T0715000Z_19810305T0730000Z_hrw.xml")] - assert len(res) == len(set(expected)) - assert set(res) == set(expected) - def test_xml_for_timectrl_files_missing(self, fake_file_dir): """Test xml files for timectrl.""" mydir, mydir_out = fake_file_dir mymodule = MagicMock() import sys sys.modules["pps_time_control"] = mymodule - res = create_xml_timestat_from_lvl1c(self.scene['file4pps'], mydir) # Look in wrong place + res = create_xml_timestat_from_lvl1c(self.scene, mydir) # Look in wrong place expected = [] assert res == expected @@ -124,6 +111,12 @@ def test_xml_for_products(self, fake_file_dir): assert len(res) == len(set(expected)) assert set(res) == set(expected) + def test_xml_for_timectrl_no_file4pps(self, fake_file_dir): + """Test xml files for timectrl without file4pps attribute.""" + mydir, mydir_out = fake_file_dir + res = create_xml_timestat_from_lvl1c(self.empty_scene, mydir_out) + assert res == [] + def test_xml_for_products_no_file4pps(self, fake_file_dir): """Test xml files for products without file4pps attribute.""" mydir, mydir_out = fake_file_dir diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 78bf307..87b76b9 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -234,11 +234,14 @@ def create_pps_call_command(python_exec, pps_script_name, scene): return cmdstr -def create_xml_timestat_from_lvl1c(pps_file, pps_control_path): +def create_xml_timestat_from_lvl1c(scene, pps_control_path): """From lvl1c file create XML file and return a file list.""" - txt_time_control = create_pps_file_from_lvl1c(pps_file, pps_control_path, "timectrl", ".txt") + try: + txt_time_control = create_pps_file_from_lvl1c(scene['file4pps'], pps_control_path, "timectrl", ".txt") + except KeyError: + return [] if os.path.exists(txt_time_control): - return create_xml_timestat_from_ascii(txt_time_control) + return create_xml_timestat_from_ascii(txt_time_control, pps_control_path) else: LOG.warning('No XML Time statistics file created!') return [] @@ -253,17 +256,6 @@ def find_product_statistics_from_lvl1c(scene, pps_control_path): return [] -def process_timectrl_xml_from_pps_result_file_with_extension(filename, export_path, extension): - if export_path is None: - export_path = os.path.dirname(filename) - print(export_path) - timectrl_file = create_pps_file_from_lvl1c(filename, export_path, "timectrl", ".txt") - timectrl_xmlfile = timectrl_file.replace(".txt", extension + ".xml") - print(timectrl_file) - result_file = create_xml_timestat_from_ascii(timectrl_file, outfile=timectrl_xmlfile) - return result_file - - def create_pps_file_from_lvl1c(l1c_file_name, pps_control_path, name_tag, file_type): """From lvl1c file create name_tag-file of type file_type.""" from trollsift import compose, parse @@ -275,10 +267,8 @@ def create_pps_file_from_lvl1c(l1c_file_name, pps_control_path, name_tag, file_t return os.path.join(pps_control_path, compose(f_pattern, data)) -def create_xml_timestat_from_ascii(infile, outfile=None): +def create_xml_timestat_from_ascii(infile, pps_control_path): """From ascii file(s) with PPS time statistics create XML file(s) and return a file list.""" - if outfile is None: - outfile = infile.replace('.txt', '.xml') try: from pps_time_control import PPSTimeControl except ImportError: @@ -289,12 +279,13 @@ def create_xml_timestat_from_ascii(infile, outfile=None): ppstime_con = PPSTimeControl(infile) ppstime_con.sum_up_processing_times() try: - ppstime_con.write_xml(filename=outfile) + ppstime_con.write_xml() except Exception as e: # TypeError as e: LOG.warning('Not able to write time control xml file') LOG.warning(e) + # There should always be only one xml file for each ascii file found above! - return [outfile] + return [infile.replace('.txt', '.xml')] def publish_pps_files(input_msg, publish_q, scene, result_files, **kwargs): diff --git a/setup.py b/setup.py index deebf2d..5f31624 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,6 @@ scripts=['bin/pps_runner.py', 'bin/run_nwp_preparation.py', 'bin/pps_l1c_collector.py', - 'bin/produce_pps_time_xml_stats.py', 'bin/level1c_runner.py', ], data_files=[], install_requires=['posttroll', 'trollsift', 'pygrib', 'level1c4pps'], From 113862d94a80522bfcb8016ab98bb1cc8f53f15a Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" Date: Tue, 13 Aug 2024 15:23:26 +0200 Subject: [PATCH 12/12] Handle empty list or string of filenames from PPS --- nwcsafpps_runner/pps_posttroll_hook.py | 7 +++++- nwcsafpps_runner/tests/test_pps_hook.py | 30 +++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/nwcsafpps_runner/pps_posttroll_hook.py b/nwcsafpps_runner/pps_posttroll_hook.py index f9759ec..a7cd6d2 100644 --- a/nwcsafpps_runner/pps_posttroll_hook.py +++ b/nwcsafpps_runner/pps_posttroll_hook.py @@ -248,6 +248,8 @@ def send(self): # Error # pubmsg = self.create_message("FAILED", self.metadata) LOG.warning("Module %s failed, so no message sent", self.metadata.get('module', 'unknown')) + elif self.metadata["filename"] == "" or self.metadata["filename"] == []: + LOG.info("Module %s did not create any files, so no message sent", self.metadata.get('module', 'unknown')) else: # Ok pubmsg = self.create_message("OK") @@ -352,7 +354,10 @@ def get_message_with_uri_and_uid(self): LOG.debug("Servername = %s", str(servername)) msg = {} - if isinstance(self.metadata['filename'], list): + + if self.metadata['filename'] == '': + return {} + elif isinstance(self.metadata['filename'], list): dataset = [] for filename in self.metadata['filename']: uri = os.path.abspath(filename) diff --git a/nwcsafpps_runner/tests/test_pps_hook.py b/nwcsafpps_runner/tests/test_pps_hook.py index 6daaa17..cb34c19 100644 --- a/nwcsafpps_runner/tests/test_pps_hook.py +++ b/nwcsafpps_runner/tests/test_pps_hook.py @@ -222,17 +222,19 @@ def test_send_method(self, mandatory_param, filename): mandatory_param.return_value = True filename.return_value = True + my_metadata = self.metadata.copy() + my_metadata["filename"] = "dummy" with patch.object(PostTrollMessage, 'publish_message', return_value=None) as mock_method_publish: with patch.object(PostTrollMessage, 'create_message', return_value=None) as mock_method_create: - posttroll_message = PostTrollMessage(0, self.metadata) + posttroll_message = PostTrollMessage(0, my_metadata) posttroll_message.send() self.assertEqual(mock_method_publish.call_count, 1) self.assertEqual(mock_method_create.call_count, 1) with patch.object(PostTrollMessage, 'publish_message', return_value=None) as mock_method_publish: with patch.object(PostTrollMessage, 'create_message', return_value=None) as mock_method_create: - posttroll_message = PostTrollMessage(1, self.metadata) + posttroll_message = PostTrollMessage(1, my_metadata) posttroll_message.send() self.assertEqual(mock_method_publish.call_count, 0) self.assertEqual(mock_method_create.call_count, 0) @@ -344,6 +346,30 @@ def test_create_message_with_topic(self, socket_gethostname): expected_message_header = "/my/pps/publish/topic/UNKNOWN/" self.assertEqual(expected_message_header, result_message['header']) + @patch('socket.gethostname') + def test_create_message_without_filename(self, socket_gethostname): + """Test creating a message with header/topic, type and content.""" + from nwcsafpps_runner.pps_posttroll_hook import PostTrollMessage + + socket_gethostname.return_value = 'TEST_SERVERNAME' + + metadata = {'publish_topic': '/my/pps/publish/topic/{pps_product}/', + 'output_format': 'CF', + 'level': '2', + 'variant': 'DR', + 'geo_or_polar': 'polar', + 'software': 'NWCSAF-PPSv2018', + 'start_time': START_TIME1, 'end_time': END_TIME1, + 'sensor': 'viirs', + 'filename': '', + 'platform_name': 'npp'} + + posttroll_message = PostTrollMessage(0, metadata) + with patch.object(PostTrollMessage, 'is_segment', return_value=False): + result_message = posttroll_message.create_message('OK') + self.assertFalse("uri" in result_message["content"]) + + @patch('socket.gethostname') def test_create_message_with_topic_pattern(self, socket_gethostname): """Test creating a message with header/topic that is a pattern, type and content."""