From 9ffbb9e4035835d01407297a582935de20af5817 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Fri, 19 Mar 2021 16:25:14 -0400 Subject: [PATCH 1/4] Create main.yml quick test to see if our docker image works --- .github/workflows/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..8f4804bbe --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,10 @@ +on: [push] + +jobs: + mdtf_docker_test: + runs-on: ubuntu-latest + name: A job to print mdtf version + steps: + - name: Hello world action step + id: hello + uses: aradhakrishnanGFDL/MDTF-diagnostics@dev From 4660afc3d091e064dffec547e979df7c4d162e08 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Fri, 19 Mar 2021 16:27:48 -0400 Subject: [PATCH 2/4] Update main.yml --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8f4804bbe..6f795dbdb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,5 @@ jobs: runs-on: ubuntu-latest name: A job to print mdtf version steps: - - name: Hello world action step - id: hello + - name: build image uses: aradhakrishnanGFDL/MDTF-diagnostics@dev From 417fe24f0f6e4739e9685bd72003361eb6b046ac Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Wed, 15 Sep 2021 12:52:56 -0400 Subject: [PATCH 3/4] Update mdtf_tests.yml test to use aws --- .github/workflows/mdtf_tests.yml | 49 +++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mdtf_tests.yml b/.github/workflows/mdtf_tests.yml index 898440a6c..d740712f9 100644 --- a/.github/workflows/mdtf_tests.yml +++ b/.github/workflows/mdtf_tests.yml @@ -1,7 +1,7 @@ # This file builds and runs a lightweight version of the MDTF test suite. # Note that the tests assess functionality of the diagnostics, # and do not evaluate data or scientific content. -name: MDTF_test +name: MDTF_test_with_aws on: push: @@ -16,6 +16,31 @@ defaults: run: shell: bash -l {0} jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-092a9e946def5be32 + ec2-instance-type: t2.micro + subnet-id: subnet-003faaf9f1e386979 + security-group-id: sg-016e543200ed56a0f + #iam-role-name: github-actions-role-a # optional, requires additional permissions + build: runs-on: ${{matrix.os}} continue-on-error: ${{ matrix.experimental }} @@ -157,3 +182,25 @@ jobs: echo "${PWD}" conda activate _MDTF_base python -m unittest discover + +stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - do-the-job # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} From 548a97a9393d89110a1ba7dfde58692b34599299 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Wed, 15 Sep 2021 13:04:41 -0400 Subject: [PATCH 4/4] Update mdtf_tests.yml --- .github/workflows/mdtf_tests.yml | 95 +++++++++++++------------------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/.github/workflows/mdtf_tests.yml b/.github/workflows/mdtf_tests.yml index d740712f9..8c403065c 100644 --- a/.github/workflows/mdtf_tests.yml +++ b/.github/workflows/mdtf_tests.yml @@ -1,7 +1,7 @@ # This file builds and runs a lightweight version of the MDTF test suite. # Note that the tests assess functionality of the diagnostics, # and do not evaluate data or scientific content. -name: MDTF_test_with_aws +name: MDTF_test on: push: @@ -16,31 +16,6 @@ defaults: run: shell: bash -l {0} jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-092a9e946def5be32 - ec2-instance-type: t2.micro - subnet-id: subnet-003faaf9f1e386979 - security-group-id: sg-016e543200ed56a0f - #iam-role-name: github-actions-role-a # optional, requires additional permissions - build: runs-on: ${{matrix.os}} continue-on-error: ${{ matrix.experimental }} @@ -50,6 +25,7 @@ jobs: conda-root: [/usr/share/miniconda3, /Users/runner/miniconda3] json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"] json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"] + json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"] # if experimental is true, other jobs to run if one fails experimental: [false] exclude: @@ -59,12 +35,16 @@ jobs: json-file: "tests/github_actions_test_macos_set1.jsonc" - os: ubuntu-latest json-file-set2: "tests/github_actions_test_macos_set2.jsonc" + - os: ubuntu-latest + json-file-set3: "tests/github_actions_test_macos_set3.jsonc" - os: macos-latest conda-root: /usr/share/miniconda3 - os: macos-latest json-file: "tests/github_actions_test_ubuntu_set1.jsonc" - os: macos-latest json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc" + - os: macos-latest + json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc" - conda-root: /usr/share/miniconda3 json-file: "tests/github_actions_test_macos_set1.jsonc" - conda-root: /Users/runner/minconda3 @@ -73,6 +53,10 @@ jobs: json-file-set2: "tests/github_actions_test_macos_set2.jsonc" - conda-root: /Users/runner/minconda3 json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc" + - conda-root: /usr/share/miniconda3 + json-file-set3: "tests/github_actions_test_macos_set3.jsonc" + - conda-root: /Users/runner/minconda3 + json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc" max-parallel: 2 steps: - uses: actions/checkout@v2 @@ -94,15 +78,15 @@ jobs: echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV - name: Install Conda Environments run: | - # create the synthetic data environment - echo "Creating the _MDTF_synthetic_data environment" - conda env create --force -q -f ./src/conda/_env_synthetic_data.yml # install mamba (https://github.com/mamba-org/mamba) for faster dependency resolution echo "Installing Mamba" conda install mamba -n base -c conda-forge + # create the synthetic data environment + echo "Creating the _MDTF_synthetic_data environment" + mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml echo "Installing Conda Environments" # MDTF-specific setup: install all conda envs - ./src/conda/conda_env_setup.sh --all --mamba --conda_root ${{matrix.conda-root}} + ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.conda-root}} - name: Generate Model Data run: | cd ../ @@ -112,6 +96,7 @@ jobs: # generate the data and run unit tests mdtf_synthetic.py -c GFDL --startyear 1 --nyears 10 --unittest mdtf_synthetic.py -c NCAR --startyear 1975 --nyears 7 + mdtf_synthetic.py -c CMIP --startyear 1 --nyears 10 cd ../ mkdir wkdir - name: Get Observational Data for Set 1 @@ -172,35 +157,33 @@ jobs: - name: Run diagnostic tests set 2 run: | conda activate _MDTF_base - cat ./mdtf - # trivial check that install script worked - ./mdtf --version # run the test PODs ./mdtf -v -f ${{matrix.json-file-set2}} - - name: Run unit tests + - name: Get observational data for set 3 run: | echo "${PWD}" + # remove data from previous run + # Actions moves you to the root repo directory in every step, so need to cd again + cd ../inputdata/obs_data + echo "deleting obs data from set 2" + rm -rf * + cd ../../ + echo "Available Space" + df -h + # attempt FTP data fetch + # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual + # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos. + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/temp_extremes_distshape_obs_data.tar --output temp_extremes_distshape_obs_data.tar + echo "Untarring set 3 test files" + tar -xvf temp_extremes_distshape_obs_data.tar + # clean up tarballs + rm -f *.tar + - name: Run diagnostic tests set 3 + run: | + conda activate _MDTF_base + # run the test PODs + ./mdtf -v -f ${{matrix.json-file-set3}} + - name: Run unit tests + run: | conda activate _MDTF_base python -m unittest discover - -stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}