New version

.github/workflows/python-publish.yml

@@ -51,6 +51,8 @@ jobs:
 
       - name: Tests
         run: pytest
+        env:
+          ECMWF_OPENDATA_URLS: ${{ secrets.ECMWF_OPENDATA_URLS }}
 
   deploy:
 

README.md

@@ -21,7 +21,7 @@ client.retrieve(
 
 ## Download examples
 
-### Download a single surface parameter at a single forecast step from ECMWF's 00UTC HRES forecast (Product Set I-i)
+### Download a single surface parameter at a single forecast step from ECMWF's 00UTC HRES forecast
 
 ```python
 from ecmwf.opendata import Client
@@ -66,7 +66,7 @@ client.retrieve(
 ...
 ```
 
-### Download a single surface parameter at a single forecast step for all ensemble members from ECMWF's 12UTC 00UTC ENS forecast (Set III-i)
+### Download a single surface parameter at a single forecast step for all ensemble members from ECMWF's 12UTC 00UTC ENS forecast
 
 ```python
 from ecmwf.opendata import Client
@@ -85,7 +85,7 @@ client.retrieve(
 - All of the odd numbered ensemble members use `number = [num for num in range(1,51,2)]`.
 - To download the control member, use `type = "cf"`.
 
-### Download the Tropical Cyclone tracks from ECMWF's 00UTC ENS forecast (Set I-iii)
+### Download the Tropical Cyclone tracks from ECMWF's 00UTC ENS forecast
 
 The Tropical Cyclone tracks are identified by the keyword `type = "tf"`.
 
@@ -106,7 +106,7 @@ client.retrieve(
 - The downloaded data are encoded in BUFR edition 4
 - For the ENS Tropical Cyclone tracks at time=06 and time=18 replace `step = [240,]` with `step = [144,]`.
 
-### Download the ensemble mean and standard deviation for all parameters at a single forecast step from ECMWF's 00UTC ENS forecast (Set III-iv)
+### Download the ensemble mean and standard deviation for all parameters at a single forecast step from ECMWF's 00UTC ENS forecast
 
 The ensemble mean and standard deviation are identified by the keywords `type = "em"`:
 
@@ -140,13 +140,13 @@ client.retrieve(
 
 ```
 
-### Download the ensemble probability products (Set III-iii-c)
+### Download the ensemble probability products
 
 The ensemble probability products are identified by the keyword`type = "ep"`.  The probability products are available only for time=00 and time=12.
 
 Two different productsa are available.
 
-#### Probabilities - Instantaneous weather events - Pressure levels (Set III-iii-c)
+#### Probabilities - Instantaneous weather events - Pressure levels
 
 The probability of temperature standardized anomalies at a constant pressure level of 850hPa are available at 12 hourly forecast steps.
 
@@ -174,7 +174,7 @@ client.retrieve(
 )
 
 ```
-#### Probabilities - Daily weather events - Single level (Set III-iii-e)
+#### Probabilities - Daily weather events - Single level
 
 The probabilities of total precipitation and wind gusts exceeding specified thresholds in a 24 hour period are available for step ranges 0-24 to 336-360 by 12​​.  These are specified in the retrieval request using, e.g.: `step = ["0-24", "12-36", "24-48"]`.
 

ecmwf/opendata/__init__.py

@@ -10,6 +10,6 @@
 
 from .client import Client
 
-__version__ = "0.0.13"
+__version__ = "0.0.14"
 
 __all__ = ["Client"]

ecmwf/opendata/client.py

@@ -13,7 +13,6 @@
 import json
 import logging
 import os
-import re
 from collections import defaultdict
 
 import requests
@@ -45,15 +44,16 @@ def __init__(
         url=None,
         beta=True,
         preserve_request_order=False,
+        auto_stream=True,
     ):
         self._url = url
         self.source = source
         self.beta = beta
         self.preserve_request_order = preserve_request_order
+        self.auto_stream = auto_stream
 
     @property
     def url(self):
-        global URLS
 
         if self._url is None:
 
@@ -291,6 +291,18 @@ def patch_stream(self, args):
             ("oper", "18"): "scda",
             ("wave", "06"): "scwv",
             ("wave", "18"): "scwv",
+            #
+            ("oper", "ef"): "enfo",
+            ("wave", "ef"): "waef",
+            ("oper", "ep"): "enfo",
+            ("wave", "ep"): "waef",
         }
-        stream, time = args["stream"], args["_H"]
-        return URL_STREAM_MAPPING.get((stream, time), stream)
+        stream, time, type = args["stream"], args["_H"], args["type"]
+
+        if not self.auto_stream:
+            return stream
+
+        stream = URL_STREAM_MAPPING.get((stream, time), stream)
+        stream = URL_STREAM_MAPPING.get((stream, type), stream)
+
+        return stream

ecmwf/opendata/urls.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# (C) Copyright 2021 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import json
+import os
+
+DOT_ECMWF_OPENDATA = os.path.expanduser("~/.ecmwf-opendata")
+
+URLS = {}
+
+if os.path.exists(DOT_ECMWF_OPENDATA):
+    with open(DOT_ECMWF_OPENDATA) as f:
+        URLS = json.load(f)
+
+if "ECMWF_OPENDATA_URLS" in os.environ:
+    URLS = json.loads(os.environ["ECMWF_OPENDATA_URLS"])

tests/test_date.py

@@ -88,3 +88,9 @@ def test_time():
     assert canonical_time("0600") == 6
     assert canonical_time("1200") == 12
     assert canonical_time("1800") == 18
+
+
+def test_step():
+    assert end_step(24) == 24
+    assert end_step("24") == 24
+    assert end_step("12-24") == 24

tests/test_examples.py

@@ -2,8 +2,6 @@
 
 import pytest
 
-from ecmwf.opendata import Client
-
 README = os.path.realpath(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
 )

tests/test_opendata.py

@@ -1,27 +1,50 @@
-import eccodes
 from freezegun import freeze_time
 
 from ecmwf.opendata import Client
-from ecmwf.opendata.grib import count_gribs, grib_index
+from ecmwf.opendata.bufr import count_bufrs
+from ecmwf.opendata.grib import count_gribs
 
 TEST_URL = "https://get.ecmwf.int/repository/ecmwf-opendata/testing"
 
+"""
+format=bufr, type=tf
+   HH=00/12
+      stream=enfo/oper, step=240h
+   HH=06/18
+      stream=enfo, step=144h
+      stream=scda, step=90h
+
+format=grib2
+   HH=00/12
+       stream=enfo/waef
+          type=ef, step=0h to 144h by 3h, 144h to 360h by 6h
+          type=ep, step=240h/360h
+  stream=oper, wave
+       type=fc, step=0h to 144h by 3h, 144h to 360h by 6h
+     HH=06/18
+       stream=enfo/waef
+          type=ef, step=0h to 144h by 3h
+       stream= scda /scwv
+          type=fc, step=0h to 90h by 3h
+     HH=00
+       stream=mmsf, type=fc, u=m, step=1m to 7m
+"""
+
 
 @freeze_time("2022-01-21t13:21:34z")
-def xx_test_opendata_1():
+def xxx_test_opendata_1():
     client = Client(TEST_URL)
     client.retrieve(
-        date=-1,
         time=0,
-        step="144-168",
-        stream="enfo",
-        type="ep",
+        step=240,
+        stream="oper",
+        type="tf",
         levtype="sfc",
         param="tpg100",
-        target="data.grib",
+        target="data.bufr",
     )
 
-    assert count_gribs("data.grib") == 1
+    assert count_bufrs("data.bufr") > 0
 
 
 @freeze_time("2022-01-21t13:21:34z")

tests/test_sources.py

@@ -2,9 +2,11 @@
 
 from ecmwf.opendata import Client
 from ecmwf.opendata.bufr import count_bufrs
-from ecmwf.opendata.grib import count_gribs, grib_index
+from ecmwf.opendata.grib import count_gribs
 
-SOURCES = ["ecmwf", "azure"]
+SOURCES = [
+    "ecmwf",
+]  # "azure"]
 
 
 @pytest.mark.parametrize("source", SOURCES)