From 849aee9d7707b01a20ce700197064f671e4a4cfa Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 14:33:38 -0400 Subject: [PATCH 01/13] initial guess at fms_yaml_tools conda environment --- environment.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 environment.yaml diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..fa0f3bc --- /dev/null +++ b/environment.yaml @@ -0,0 +1,10 @@ +name: fms_yaml_tools +channels: + - defaults + - conda-forge + - noaa-gfdl +dependencies: + - python + - pip + - click + - pyyaml \ No newline at end of file From 6b901b07b62c2398ecef1aaa24ac4dae33664da6 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 14:34:30 -0400 Subject: [PATCH 02/13] lets be a tad liberal... --- environment.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index fa0f3bc..d1dc90d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,4 +7,7 @@ dependencies: - python - pip - click - - pyyaml \ No newline at end of file + - pyyaml + - pylint + - pytest + - jsonschema \ No newline at end of file From d712c0b1df65f6b8ceba8ce4fd71a1c4b6d1b5b5 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 14:55:46 -0400 Subject: [PATCH 03/13] to run on push: workflow to create conda env, pip install to conda env, test in conda env (create_test_conda_env). also to run on push: workflow to build conda package but not publish the package (build_conda). to run on merge to main branch: workflow to publish conda package (publish_conda). environment.yaml builds usable environment with `conda env create -f environment.yaml`, which can then be activated. the python within that environment may be used to pip install the `fms_yaml_tools` package into the conda env. meta.yaml provides info for `conda build`, this is a first-pass ultimate goal is a conda package fms_yaml_tools to link fre-cli's fre.yamltools into, ala fre.catalog into catalogbuilder --- .github/workflows/build_conda.yml | 22 ++++++ .github/workflows/create_test_conda_env.yml | 86 +++++++++++++++++++++ .github/workflows/publish_conda.yml | 21 +++++ environment.yaml | 1 - meta.yaml | 37 +++++++++ 5 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_conda.yml create mode 100644 .github/workflows/create_test_conda_env.yml create mode 100644 .github/workflows/publish_conda.yml create mode 100644 meta.yaml diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml new file mode 100644 index 0000000..8194f73 --- /dev/null +++ b/.github/workflows/build_conda.yml @@ -0,0 +1,22 @@ +name: build_conda +on: + [push] +# pull_request: +# branches: +# - main + +jobs: + build: + runs-on: ubuntu-latest + container: + image: continuumio/miniconda3:latest + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build conda-verify + conda build . + ls . diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml new file mode 100644 index 0000000..33eab15 --- /dev/null +++ b/.github/workflows/create_test_conda_env.yml @@ -0,0 +1,86 @@ +name: create_test_conda_env + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '>=3.9' + + - name: Add conda to system path + run: | + # $CONDA is an env var pointing to root of miniconda dir + echo $CONDA/bin >> $GITHUB_PATH + + - name: Create fms_yaml_tools environment + run: | + # create environment containing all dependencies + # the env cannot be explicitly activated in github CI/CD + conda env create -f environment.yml --name fms_yaml_tools + + # add conda env's executables to github's PATH equiv. + echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH + + # use *conda environment's pip* to install fms_yaml_tools + # called w/ full path to conda's python for explicitness + # called as a module (-m pip) for explicitness + $CONDA/envs/fms_yaml_tools/bin/python -m pip install --prefix $CONDA/envs/fms_yaml_tools . + + - name: Run unittest in fms_yaml_tools environment + run: | + # try to make sure the right things are in GITHUB_PATH + echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH + python -m unittest discover -s test + +# - name: Run pytest in fms_yaml_tools environment +# run: | +# # try to make sure the right things are in GITHUB_PATH +# echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH +# +# # are we talking to the right python? +# which python +# python --version +# $CONDA/envs/fms_yaml_tools/bin/python --version +# +# # run pytest +# pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ +# +# # install genbadge to generate coverage badge based on xml +# pip install genbadge +# genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg +# genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg +# +# - name: Run pylint in fms_yaml_tools environment +# run: | +# # try to make sure the right things are in GITHUB_PATH +# echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH +# +# # are we talking to the right python? +# which python +# python --version +# $CONDA/envs/fms_yaml_tools/bin/python --version +# +# # run pylint +# pylint -ry --ignored-modules netCDF4 fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." + +# - name: Install Sphinx and Build Documentation +# run: | +# pip install sphinx renku-sphinx-theme sphinx-rtd-theme +# pip install --upgrade sphinx-rtd-theme +# sphinx-apidoc --output-dir docs fre/ --separate +# sphinx-build docs build +# +# - name: Deploy to GitHub Pages +# uses: peaceiris/actions-gh-pages@v3 +# if: ${{ github.ref == 'refs/heads/main' }} +# with: +# publish_branch: gh-pages +# github_token: ${{ secrets.GITHUB_TOKEN }} +# publish_dir: build/ +# force_orphan: true +# diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml new file mode 100644 index 0000000..e36a72e --- /dev/null +++ b/.github/workflows/publish_conda.yml @@ -0,0 +1,21 @@ +name: publish_conda +on: + push: + branches: + - main +jobs: + publish: + runs-on: ubuntu-latest + container: + image: continuumio/miniconda3:latest + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + conda build . diff --git a/environment.yaml b/environment.yaml index d1dc90d..7bf8f6d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -9,5 +9,4 @@ dependencies: - click - pyyaml - pylint - - pytest - jsonschema \ No newline at end of file diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..64915d7 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,37 @@ +package: + name: fms_yaml_tools + +source: + path: . + +build: + script: + - {{ PYTHON }} -m pip install . -vv + number: 1 + noarch: python + +channels: + - defaults + - conda-forge + - noaa-gfdl + +requirements: + host: + - python + - pip + run: + - python + - pip + - click + - pyyaml + - pylint + - jsonschema + +test: + source_files: + - fms_yaml_tools/ + commands: + - python -m unittest discover -s test + +about: + home: https://github.com/NOAA-GFDL/fms_yaml_tools \ No newline at end of file From faddccba35d54e499fbbeeb5c1ebd2fc9460e918 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 15:00:58 -0400 Subject: [PATCH 04/13] fix whitespace error (tab in workflow files) --- .github/workflows/build_conda.yml | 2 +- .github/workflows/create_test_conda_env.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 8194f73..b4c87db 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -19,4 +19,4 @@ jobs: conda config --append channels noaa-gfdl conda install conda-build conda-verify conda build . - ls . + ls . diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 33eab15..d5b7477 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -35,7 +35,7 @@ jobs: run: | # try to make sure the right things are in GITHUB_PATH echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH - python -m unittest discover -s test + python -m unittest discover -s test # - name: Run pytest in fms_yaml_tools environment # run: | From 48425be0d1d0d50f81b7c8bfbc269adc94e2b230 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 15:03:47 -0400 Subject: [PATCH 05/13] rename environment.yaml --> environment.yml --- environment.yaml => environment.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename environment.yaml => environment.yml (100%) diff --git a/environment.yaml b/environment.yml similarity index 100% rename from environment.yaml rename to environment.yml From 6acdfd8737042b373024234f51931a8e6a11a0eb Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 15:08:26 -0400 Subject: [PATCH 06/13] disable conda_build workflow for now. recursively update submodule for gfdl_msd_schemas/FMS folder to execute final test --- .github/workflows/build_conda.yml | 8 ++++---- .github/workflows/create_test_conda_env.yml | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index b4c87db..c80b81d 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,9 +1,9 @@ name: build_conda on: - [push] -# pull_request: -# branches: -# - main + pull_request: + branches: + - FOOmainBAR +# [push] jobs: build: diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index d5b7477..bc8751e 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -25,6 +25,9 @@ jobs: # add conda env's executables to github's PATH equiv. echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH + + # get fms_yaml_tools/schema/gfdl_msd_schemas/FMS for unittest + git submodule update --init --recursive --remote # use *conda environment's pip* to install fms_yaml_tools # called w/ full path to conda's python for explicitness From 1ec037bb7b085778778cf225e63dfad054ea104c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 18:53:12 -0400 Subject: [PATCH 07/13] try a github CI/CD conda build. give version alpha, fix whitespace tab problems --- .github/workflows/build_conda.yml | 9 +++++---- meta.yaml | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index c80b81d..301860e 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -1,9 +1,10 @@ name: build_conda on: - pull_request: - branches: - - FOOmainBAR -# [push] + [push] +# pull_request: +# branches: +# - FOOmainBAR + jobs: build: diff --git a/meta.yaml b/meta.yaml index 64915d7..a7bd9ac 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,12 +1,14 @@ package: name: fms_yaml_tools + version: 'alpha' +# version: '{{ environ.get("GIT_DESCRIBE_TAG", data.get("version")) }}' source: path: . build: script: - - {{ PYTHON }} -m pip install . -vv + - python3 -m pip install . -vv number: 1 noarch: python @@ -31,7 +33,7 @@ test: source_files: - fms_yaml_tools/ commands: - - python -m unittest discover -s test + - python -m unittest discover -s test about: home: https://github.com/NOAA-GFDL/fms_yaml_tools \ No newline at end of file From e31cf401b9572cc160d7b2fde4e32b5f28c583be Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 18:58:27 -0400 Subject: [PATCH 08/13] add submodule recursive update line to conda build recipe so it finds gfdl_msd_schemas/MSD --- .github/workflows/build_conda.yml | 1 + .github/workflows/create_test_conda_env.yml | 48 --------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 301860e..6be979a 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -19,5 +19,6 @@ jobs: conda config --append channels conda-forge conda config --append channels noaa-gfdl conda install conda-build conda-verify + git submodule update --init --recursive --remote conda build . ls . diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index bc8751e..7785e5c 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -39,51 +39,3 @@ jobs: # try to make sure the right things are in GITHUB_PATH echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH python -m unittest discover -s test - -# - name: Run pytest in fms_yaml_tools environment -# run: | -# # try to make sure the right things are in GITHUB_PATH -# echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH -# -# # are we talking to the right python? -# which python -# python --version -# $CONDA/envs/fms_yaml_tools/bin/python --version -# -# # run pytest -# pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ -# -# # install genbadge to generate coverage badge based on xml -# pip install genbadge -# genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg -# genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg -# -# - name: Run pylint in fms_yaml_tools environment -# run: | -# # try to make sure the right things are in GITHUB_PATH -# echo $CONDA/envs/fms_yaml_tools/bin >> $GITHUB_PATH -# -# # are we talking to the right python? -# which python -# python --version -# $CONDA/envs/fms_yaml_tools/bin/python --version -# -# # run pylint -# pylint -ry --ignored-modules netCDF4 fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." - -# - name: Install Sphinx and Build Documentation -# run: | -# pip install sphinx renku-sphinx-theme sphinx-rtd-theme -# pip install --upgrade sphinx-rtd-theme -# sphinx-apidoc --output-dir docs fre/ --separate -# sphinx-build docs build -# -# - name: Deploy to GitHub Pages -# uses: peaceiris/actions-gh-pages@v3 -# if: ${{ github.ref == 'refs/heads/main' }} -# with: -# publish_branch: gh-pages -# github_token: ${{ secrets.GITHUB_TOKEN }} -# publish_dir: build/ -# force_orphan: true -# From 7b31a206cb0d141938a9d0b506aa6807c6ea5ddf Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 3 Oct 2024 19:00:19 -0400 Subject: [PATCH 09/13] whitespace etc and submodule update to publish recipe --- .github/workflows/build_conda.yml | 28 ++++++++++++------------ .github/workflows/publish_conda.yml | 33 +++++++++++++++-------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 6be979a..3b54b8e 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -8,17 +8,17 @@ on: jobs: build: - runs-on: ubuntu-latest - container: - image: continuumio/miniconda3:latest - steps: - - name: Checkout Files - uses: actions/checkout@v4 - - name: Run Conda to Build - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build conda-verify - git submodule update --init --recursive --remote - conda build . - ls . + runs-on: ubuntu-latest + container: + image: continuumio/miniconda3:latest + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build conda-verify + git submodule update --init --recursive --remote + conda build . + ls . diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index e36a72e..2d4859a 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -1,21 +1,22 @@ name: publish_conda on: push: - branches: - - main + branches: + - main jobs: publish: - runs-on: ubuntu-latest - container: - image: continuumio/miniconda3:latest - steps: - - name: Checkout Files - uses: actions/checkout@v4 - - name: Run Conda to Build and Publish - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build anaconda-client conda-verify - export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} - conda config --set anaconda_upload yes - conda build . + runs-on: ubuntu-latest + container: + image: continuumio/miniconda3:latest + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + git submodule update --init --recursive --remote + conda build . From f18372581b63989a76582c343edae8f7ad236c8f Mon Sep 17 00:00:00 2001 From: Ian <6273252+ilaflott@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:02:57 -0400 Subject: [PATCH 10/13] Update build_conda.yml --- .github/workflows/build_conda.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 3b54b8e..74a2126 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -8,17 +8,17 @@ on: jobs: build: - runs-on: ubuntu-latest - container: - image: continuumio/miniconda3:latest - steps: - - name: Checkout Files - uses: actions/checkout@v4 - - name: Run Conda to Build - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build conda-verify - git submodule update --init --recursive --remote - conda build . - ls . + runs-on: ubuntu-latest + container: + image: continuumio/miniconda3:latest + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build conda-verify + git submodule update --init --recursive --remote + conda build . + ls . From 68328bd84f54c553e31593c46e3ed15deb17ea53 Mon Sep 17 00:00:00 2001 From: Ian <6273252+ilaflott@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:03:53 -0400 Subject: [PATCH 11/13] Update build_conda.yml whitespace change please workflow run --- .github/workflows/build_conda.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 74a2126..6a6c682 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -5,7 +5,6 @@ on: # branches: # - FOOmainBAR - jobs: build: runs-on: ubuntu-latest From b66b0c7f7f0579b0481cace6ea79a069f9ee51f2 Mon Sep 17 00:00:00 2001 From: Ian <6273252+ilaflott@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:40:06 -0400 Subject: [PATCH 12/13] Update build_conda.yml see if i can't convince conda build to use the gfdl_msd_schemas repo code anyways --- .github/workflows/build_conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 6a6c682..ae7fd27 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -19,5 +19,6 @@ jobs: conda config --append channels noaa-gfdl conda install conda-build conda-verify git submodule update --init --recursive --remote + git config --global --add safe.directory fms_yaml_tools/schema/gfdl_msd_schemas conda build . ls . From 9c80f999db6847e3f8793b3044018514b913d814 Mon Sep 17 00:00:00 2001 From: Ian <6273252+ilaflott@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:46:03 -0400 Subject: [PATCH 13/13] Update build_conda.yml probably not gonna work. gotta try it juuuuust one more thing though... --- .github/workflows/build_conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index ae7fd27..b7dcede 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -20,5 +20,6 @@ jobs: conda install conda-build conda-verify git submodule update --init --recursive --remote git config --global --add safe.directory fms_yaml_tools/schema/gfdl_msd_schemas + rm -rf fms_yaml_tools/schema/gfdl_msd_schemas/.git conda build . ls .