diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64a2cb20..b8689978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,8 @@ on: workflow_dispatch: env: - CDSAPI_URL: https://cds.climate.copernicus.eu/api/v2 - CDSAPI_KEY: 12345:1234567-ab12-34cd-9876-4o4fake90909 # A fake key for testing - + CDSAPI_URL: https://cds-beta.climate.copernicus.eu/api + CDSAPI_KEY: 1234567-ab12-34cd-9876-4o4fake90909 # A fake key for testing jobs: build: runs-on: ubuntu-latest diff --git a/ci3.8.yml b/ci3.8.yml index 211d36be..ab040765 100644 --- a/ci3.8.yml +++ b/ci3.8.yml @@ -5,7 +5,6 @@ channels: dependencies: - python=3.8.13 - apache-beam=2.40.0 - - cdsapi=0.5.1 - pytest=7.2.0 - pytest-subtests=0.8.0 - cfgrib=0.9.10.2 @@ -37,4 +36,5 @@ dependencies: - pip: - cython==0.29.34 - earthengine-api==0.1.329 + - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl. - .[test] diff --git a/ci3.9.yml b/ci3.9.yml index 86f0968d..85cb9cd4 100644 --- a/ci3.9.yml +++ b/ci3.9.yml @@ -5,7 +5,6 @@ channels: dependencies: - python=3.9.13 - apache-beam=2.40.0 - - cdsapi=0.5.1 - pytest=7.2.0 - pytest-subtests=0.8.0 - cfgrib=0.9.10.2 @@ -37,4 +36,5 @@ dependencies: - pip: - cython==0.29.34 - earthengine-api==0.1.329 + - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl. - .[test] diff --git a/environment.yml b/environment.yml index 0b043980..a892cd28 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - rioxarray=0.13.4 - gdal=3.5.1 - pyproj=3.4.0 - - cdsapi=0.5.1 - ecmwf-api-client=1.6.3 - eccodes=2.27.0 - cfgrib=0.9.10.2 @@ -30,6 +29,7 @@ dependencies: - cython==0.29.34 - earthengine-api==0.1.329 - firebase-admin==6.0.1 + - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl. - . - ./weather_dl - ./weather_mv diff --git a/setup.py b/setup.py index 233e5193..64324a70 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ ] weather_dl_requirements = [ - "cdsapi", "ecmwf-api-client", "numpy>=1.19.1", "pandas", @@ -118,6 +117,8 @@ ], python_requires='>=3.8, <3.10', install_requires=['apache-beam[gcp]==2.40.0', 'gcsfs==2022.11.0'], + dependency_links=['https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi'], # TODO([ + #474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl. use_scm_version=True, setup_requires=['setuptools_scm'], scripts=['weather_dl/weather-dl', 'weather_mv/weather-mv', 'weather_sp/weather-sp'], diff --git a/weather_dl/download_pipeline/clients.py b/weather_dl/download_pipeline/clients.py index cb551880..099674b4 100644 --- a/weather_dl/download_pipeline/clients.py +++ b/weather_dl/download_pipeline/clients.py @@ -27,6 +27,7 @@ from urllib.parse import urljoin from cdsapi import api as cds_api +from cads_api_client import legacy_api_client import urllib3 from ecmwfapi import api @@ -73,7 +74,7 @@ def license_url(self): pass -class SplitCDSRequest(cds_api.Client): +class SplitCDSRequest(legacy_api_client.LegacyApiClient): """Extended CDS class that separates fetch and download stage.""" @retry_with_exponential_backoff def _download(self, url, path: str, size: int) -> None: diff --git a/weather_dl/setup.py b/weather_dl/setup.py index b96466fd..266f8115 100644 --- a/weather_dl/setup.py +++ b/weather_dl/setup.py @@ -32,7 +32,6 @@ ] base_requirements = [ - "cdsapi==0.5.1", "ecmwf-api-client==1.6.3", "numpy>=1.19.1", "pandas==1.5.1", @@ -48,10 +47,12 @@ setup( name='download_pipeline', packages=find_packages(), - version='0.1.20', + version='0.1.21', author='Anthromets', author_email='anthromets-ecmwf@google.com', url='https://weather-tools.readthedocs.io/en/latest/weather_dl/', description='A tool to download weather data.', install_requires=beam_gcp_requirements + base_requirements, + dependency_links=['https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi'], # TODO([ + #474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl. ) diff --git a/weather_dl_v2/license_deployment/clients.py b/weather_dl_v2/license_deployment/clients.py index 331888ea..776ac874 100644 --- a/weather_dl_v2/license_deployment/clients.py +++ b/weather_dl_v2/license_deployment/clients.py @@ -28,6 +28,7 @@ from urllib.parse import urljoin from cdsapi import api as cds_api +from cads_api_client import legacy_api_client import urllib3 from ecmwfapi import api @@ -75,7 +76,7 @@ def license_url(self): pass -class SplitCDSRequest(cds_api.Client): +class SplitCDSRequest(legacy_api_client.LegacyApiClient): """Extended CDS class that separates fetch and download stage.""" @retry_with_exponential_backoff diff --git a/weather_dl_v2/license_deployment/environment.yml b/weather_dl_v2/license_deployment/environment.yml index 683cbf90..0c22d69e 100644 --- a/weather_dl_v2/license_deployment/environment.yml +++ b/weather_dl_v2/license_deployment/environment.yml @@ -4,7 +4,6 @@ channels: dependencies: - python=3.10 - geojson - - cdsapi=0.5.1 - ecmwf-api-client=1.6.3 - pip=22.3 - pip: @@ -16,3 +15,4 @@ dependencies: - xarray - apache-beam[gcp] - firebase-admin + - git+https://github.com/dabhicusp/cdsapi-beta-google-weather-tools.git@master#egg=cdsapi # TODO([#474](https://github.com/google/weather-tools/issues/474)): Compatible cdsapi with weather-dl.