Merge branch 'main' into main

.github/workflows/ci.yml

@@ -56,12 +56,15 @@ jobs:
         channels: conda-forge
         environment-file: ci${{ matrix.python-version}}.yml
         activate-environment: weather-tools
+        miniforge-variant: Mambaforge
+        miniforge-version: latest
+        use-mamba: true
     - name: Check MetView's installation
       shell: bash -l {0}
       run: python -m metview selfcheck
     - name: Run unit tests
       shell: bash -l {0}
-      run: pytest --memray
+      run: pytest --memray --ignore=weather_dl_v2 # Ignoring dl-v2 as it only supports py3.10
   lint:
     runs-on: ubuntu-latest
     strategy:
@@ -84,42 +87,43 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
       - name: Install linter
         run: |
-          pip install ruff
+          pip install ruff==0.1.2
       - name: Lint project
         run: ruff check .
   type-check:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        python-version: ["3.8"]
     steps:
       - name: Cancel previous
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
         if: ${{github.ref != 'refs/head/main'}}
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: conda cache
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
         with:
-          python-version: "3.8"
-      - name: Setup conda
-        uses: s-weigand/setup-conda@v1
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ matrix.python-version }}-${{ hashFiles('ci3.8.yml') }}
+      - name: Setup conda environment
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          update-conda: true
-          python-version: "3.8"
-          conda-channels: anaconda, conda-forge
-      - name: Install ecCodes
-        run: |
-          conda install -y eccodes>=2.21.0 -c conda-forge
-          conda install -y pyproj -c conda-forge
-          conda install -y gdal -c conda-forge
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          python -m pip install --upgrade pip wheel
-          echo "::set-output name=dir::$(pip cache dir)"
-      - name: Install weather-tools
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          environment-file: ci${{ matrix.python-version}}.yml
+          activate-environment: weather-tools
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+      - name: Install weather-tools[test]
         run: |
-          pip install -e .[test] --use-deprecated=legacy-resolver
+          conda run -n weather-tools pip install -e .[test] --use-deprecated=legacy-resolver
       - name: Run type checker
-        run: pytype
+        run: conda run -n weather-tools pytype

Dockerfile

@@ -29,6 +29,7 @@ ARG weather_tools_git_rev=main
 RUN git clone https://github.com/google/weather-tools.git /weather
 WORKDIR /weather
 RUN git checkout "${weather_tools_git_rev}"
+RUN rm -r /weather/weather_*/test_data/
 RUN conda env create -f environment.yml --debug
 
 # Activate the conda env and update the PATH

ci3.8.yml

@@ -16,7 +16,7 @@ dependencies:
   - requests=2.28.1
   - netcdf4=1.6.1
   - rioxarray=0.13.4
-  - xarray-beam=0.3.1
+  - xarray-beam=0.6.2
   - ecmwf-api-client=1.6.3
   - fsspec=2022.11.0
   - gcsfs=2022.11.0
@@ -30,9 +30,11 @@ dependencies:
   - pip=22.3
   - pygrib=2.1.4
   - xarray==2023.1.0
-  - ruff==0.0.260
+  - ruff==0.1.2
   - google-cloud-sdk=410.0.0
   - aria2=1.36.0
+  - zarr=2.15.0
   - pip:
+    - cython==0.29.34
     - earthengine-api==0.1.329
     - .[test]

ci3.9.yml

@@ -16,7 +16,7 @@ dependencies:
   - requests=2.28.1
   - netcdf4=1.6.1
   - rioxarray=0.13.4
-  - xarray-beam=0.3.1
+  - xarray-beam=0.6.2
   - ecmwf-api-client=1.6.3
   - fsspec=2022.11.0
   - gcsfs=2022.11.0
@@ -32,7 +32,9 @@ dependencies:
   - google-cloud-sdk=410.0.0
   - aria2=1.36.0
   - xarray==2023.1.0
-  - ruff==0.0.260
+  - ruff==0.1.2
+  - zarr=2.15.0
   - pip:
+    - cython==0.29.34
     - earthengine-api==0.1.329
     - .[test]

environment.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python=3.8.13
   - apache-beam=2.40.0
-  - xarray-beam=0.3.1
+  - xarray-beam=0.6.2
   - xarray=2023.1.0
   - fsspec=2022.11.0
   - gcsfs=2022.11.0
@@ -25,7 +25,9 @@ dependencies:
   - google-cloud-sdk=410.0.0
   - aria2=1.36.0
   - pip=22.3
+  - zarr=2.15.0
   - pip:
+    - cython==0.29.34
     - earthengine-api==0.1.329
     - firebase-admin==6.0.1
     - .

pyproject.toml

@@ -42,4 +42,4 @@ target-version = "py310"
 
 [tool.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
-max-complexity = 10
+max-complexity = 10

setup.py

@@ -57,8 +57,9 @@
     "earthengine-api>=0.1.263",
     "pyproj",  # requires separate binary installation!
     "gdal",  # requires separate binary installation!
-    "xarray-beam==0.3.1",
+    "xarray-beam==0.6.2",
     "gcsfs==2022.11.0",
+    "zarr==2.15.0",
 ]
 
 weather_sp_requirements = [
@@ -70,7 +71,7 @@
 
 test_requirements = [
     "pytype==2021.11.29",
-    "ruff",
+    "ruff==0.1.2",
     "pytest",
     "pytest-subtests",
     "netcdf4",
@@ -82,6 +83,7 @@
     "memray",
     "pytest-memray",
     "h5py",
+    "pooch",
 ]
 
 all_test_requirements = beam_gcp_requirements + weather_dl_requirements + \
@@ -115,7 +117,7 @@
 
     ],
     python_requires='>=3.8, <3.10',
-    install_requires=['apache-beam[gcp]==2.40.0'],
+    install_requires=['apache-beam[gcp]==2.40.0', 'gcsfs==2022.11.0'],
     use_scm_version=True,
     setup_requires=['setuptools_scm'],
     scripts=['weather_dl/weather-dl', 'weather_mv/weather-mv', 'weather_sp/weather-sp'],

weather_dl/download_pipeline/util.py

@@ -102,10 +102,10 @@ def to_json_serializable_type(value: t.Any) -> t.Any:
         return None
     elif np.issubdtype(type(value), np.floating):
         return float(value)
-    elif type(value) == np.ndarray:
+    elif isinstance(value, np.ndarray):
         # Will return a scaler if array is of size 1, else will return a list.
         return value.tolist()
-    elif type(value) == datetime.datetime or type(value) == str or type(value) == np.datetime64:
+    elif isinstance(value, datetime.datetime) or isinstance(value, str) or isinstance(value, np.datetime64):
         # Assume strings are ISO format timestamps...
         try:
             value = datetime.datetime.fromisoformat(value)
@@ -126,7 +126,7 @@ def to_json_serializable_type(value: t.Any) -> t.Any:
 
         # We assume here that naive timestamps are in UTC timezone.
         return value.replace(tzinfo=datetime.timezone.utc).isoformat()
-    elif type(value) == np.timedelta64:
+    elif isinstance(value, np.timedelta64):
         # Return time delta in seconds.
         return float(value / np.timedelta64(1, 's'))
     # This check must happen after processing np.timedelta64 and np.datetime64.

weather_dl_v2/README.md

@@ -0,0 +1,12 @@
+## weather-dl-v2
+
+<!-- TODO: Create a setup script to ease deployment process. -->
+
+> **_NOTE:_**   weather-dl-v2 only supports python 3.10
+
+### Sequence of steps:
+1) Refer to downloader_kubernetes/README.md
+2) Refer to license_deployment/README.md
+3) Refer to fastapi-server/README.md
+4) Refer to cli/README.md
+

weather_dl_v2/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.