Merge remote-tracking branch 'upstream/master'

.github/workflows/codeql-analysis.yml.disabled

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '25 22 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dailyhospital_scraper.yml

@@ -0,0 +1,60 @@
+name: Scrape provincial hospitalization data from nicd.ac.za
+# Unsure exactly when this data is released.  Appears to be similar to the daily COVID stats, 
+# This scraper runs only once every night
+# Adjusted to GMT+0 times
+# This runner takes some time to setup.  
+
+# test this locally by using the "act" tool:  https://github.com/nektos/act
+# need to download the "full" version of the image, as R is not installed in the medium image.
+# docker pull catthehacker/ubuntu:full-20.04
+# act -j dailystats_scrape
+# check the ~/.actrc file for mappings between ubuntu-20.04 and the full or medium images....
+
+# regular schedule - every 10 mins, starting 2 over the hour; or also when the script changes, or when it is triggered manually.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '23 8,19 * * *'
+  push:
+    branches:
+      - master
+    paths:
+      - scripts/daily_nicd_datcov.R
+
+
+jobs:
+  datcov_scrape:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@master
+      - name: OS dependencies
+        run: |
+          # remove mongo - otherwise the config below fails
+          sudo apt-get remove mongodb-org-server -y
+          # some of the R packages below has issues automatically installing, and requires this explicitly
+          sudo apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update || echo "Done"
+          # for package level CRAN, we need this repo added
+          sudo add-apt-repository ppa:c2d4u.team/c2d4u4.0+ -y || echo "Done"
+          # some of the R packages below has issues automatically installing, and requires this explicitly
+          sudo apt-get install libcurl4-openssl-dev libmagick++-dev libpoppler-cpp-dev -y --fix-missing
+          # now install the R-cran based binary repos
+          sudo apt-get install r-cran-httr r-cran-xml2 r-cran-xml r-cran-data.table r-cran-git2r r-cran-magrittr r-cran-rcpp r-cran-magick r-cran-qpdf r-cran-pdftools r-cran-rappdirs r-cran-lubridate r-cran-zoo r-cran-reshape2 -y --fix-missing
+      - name: Scrape nicd.ac.za data 
+        run: Rscript --vanilla 'scripts/daily_nicd_datcov.R'
+      - name: Push changes
+        run:  |
+          git status --porcelain
+          git rev-list --left-right --count origin/master...master
+          # only push if there has been any changes
+          # if [[ "$(git rev-list --left-right --count origin/master...master)" != "0\t0" ]]; then
+            echo "We have new commits!"
+            # pull first, there might have been other commits since this job started.... Especially when more and more actions are added to the repo 
+            git config pull.rebase false
+            git pull origin ${{ github.ref }}
+            # lets hope there were no merge conflicts.... 
+            git push origin ${{ github.ref }}
+          #fi
+        if: ${{ always() }}
+        env:
+          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
+          username: krokkie

.github/workflows/dailystats_scraper.yml

@@ -0,0 +1,68 @@
+name: Scrape prov cases,deaths,recov from sacoronavirus.co.za and/or nicd.ac.za
+# The NICD data is release typically between 18:00 and 20:00.  Run this every 5 mins between those times   
+# Adjusted to GMT+0 times
+# This runner takes some time to setup.  
+
+# test this locally by using the "act" tool:  https://github.com/nektos/act
+# need to download the "full" version of the image, as R is not installed in the medium image.
+# docker pull catthehacker/ubuntu:full-20.04
+# act -j dailystats_scrape
+# check the ~/.actrc file for mappings between ubuntu-20.04 and the full or medium images....
+
+# regular schedule - every 10 mins, starting 2 over the hour; or also when the script changes, or when it is triggered manually.
+on:
+  workflow_dispatch:
+#  schedule:
+#    - cron: '32 16-20 * * *'
+  push:
+    branches:
+      - master
+    paths:
+      - scripts/daily_nicd_scraper.R
+      - scripts/daily_sacoronovirus_scraper.R
+
+
+jobs:
+  dailystats_scrape:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@master
+      - name: OS dependencies
+        run: |
+          # remove mongo - otherwise the config below fails
+          sudo apt-get remove mongodb-org-server -y
+          # some of the R packages below has issues automatically installing, and requires this explicitly
+          sudo apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update || echo "Done"
+          # for package level CRAN, we need this repo added
+          sudo add-apt-repository ppa:c2d4u.team/c2d4u4.0+ -y || echo "Done"
+          # some of the R packages below has issues automatically installing, and requires this explicitly
+          sudo apt-get install libcurl4-openssl-dev libmagick++-dev libpoppler-cpp-dev libtesseract-dev tesseract-ocr-eng -y --fix-missing
+          # now install the R-cran based binary repos
+          sudo apt-get install r-cran-httr r-cran-xml2 r-cran-xml r-cran-data.table r-cran-git2r r-cran-magrittr r-cran-rcpp r-cran-magick r-cran-qpdf r-cran-pdftools r-cran-rappdirs -y --fix-missing
+      - name: Install dependencies via Rscript not available via deb packages
+        id: environmentsetup
+        run: |
+          print(.libPaths())
+          install.packages(c("tesseract"), repos="https://cran.r-project.org/", lib="/usr/lib/R/library")
+          # these packages are not available via the apt-get installation method....
+        shell: sudo Rscript {0}
+      - name: Scrape nicd.ac.za data 
+        run: Rscript --vanilla 'scripts/daily_nicd_scraper.R'
+      - name: Scrape sacoronavirus.co.za data 
+        run: sudo Rscript --vanilla 'scripts/daily_sacoronovirus_scraper.R'
+        if: steps.environmentsetup.conclusion == 'success'
+      - name: Push changes
+        run:  |
+          git status --porcelain
+          # only push if there has been any changes
+          if [[ "$(git status --porcelain)" != "" ]]; then
+            # pull first, there might have been other commits since this job started.... Especially when more and more actions are added to the repo 
+            git config pull.rebase false
+            git pull origin ${{ github.ref }}
+            # lets hope there were no merge conflicts.... 
+            git push origin ${{ github.ref }}
+          fi
+        if: ${{ always() }}
+        env:
+          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
+          username: krokkie

.github/workflows/realtime_rt_action.yaml → .github/workflows/realtime_rt_action.yaml.disabled

@@ -14,28 +14,39 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.6'
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.8'
 
-    - name: Ensure Data Directory Exists
-      run: |
-        mkdir -p data/calc
-        ls
+      - name: Ensure Data Directory Exists
+        run: |
+          mkdir -p data/calc
+          ls
 
-    - name: Setup Python Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install pandas numpy matplotlib scipy
-    #IPython
+      - name: Setup Python Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pandas numpy matplotlib scipy
+        #IPython
 
-    - name: Run Python Script
-      run: |
-        cd scripts
-        python realtime_r0.py
+      - name: Run Python Script
+        run: |
+          cd scripts
+          python realtime_r0.py
+
+      - name: Commit changes
+        run: |
+          git config --local user.name github-actions
+          git config --local user.email "[email protected]"
+          git add data/calc/*.csv
+          if [[ "$(git status --porcelain)" != "" ]]; then
+              git pull origin master
+              git commit -m "Rt model 1 action run" -a
+              git push origin master
+          fi
 
     #- name: Run Jupyter Notebook and Export csv
     #  uses: yaananth/run-notebook@v1
@@ -49,22 +60,22 @@ jobs:
     #    isReport: False
     #    poll: True
 
-    - uses: stefanzweifel/git-auto-commit-action@v4
-      with:
+    #- uses: stefanzweifel/git-auto-commit-action@v4
+    #  with:
         # Required
-        commit_message: Rt model 1 action run
+    #    commit_message: Rt model 1 action run
         # Optional branch to push to, defaults to the current branch
         #branch: master
         # Optional options appended to `git-commit`
         #commit_options: '--no-verify --signoff'
         # Optional glob pattern of files which should be added to the commit
-        file_pattern: data/calc/*.csv
+    #    file_pattern: data/calc/*.csv
         # Optional local file path to the repository
         #repository: .
         # Optional commit user and author settings
         # Optional tag message
         # Action will create and push a new tag to the remote repository and the defined branch
         #tagging_message: 'v1.0.0'
         # Optional options appended to `git-push`
-        push_options: '--force'
-        commit_author: github-actions <[email protected]>
+    #    push_options: '--force'
+    #    commit_author: github-actions <[email protected]>

.github/workflows/rtlive-model-za.yml

@@ -9,44 +9,55 @@ on:
       - master
     paths:
       - data/covid19za_provincial_cumulative_timeline_confirmed.csv
+      - notebooks/covid-model/*
 
 jobs:
-  run:
+  runmcmc:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
 
-    - name: Setup Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.6'
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.8'
 
-    - name: Setup Python Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r notebooks/covid-model/requirements.txt 
+      - name: Setup Python Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r notebooks/covid-model/requirements.txt
 
-    - name: Run Python Script
-      run: |
-        cd notebooks/covid-model
-        python rtlive-model-za.py
+      - name: Run Python Script
+        run: |
+          cd notebooks/covid-model
+          python rtlive-model-za.py
 
-    - uses: stefanzweifel/git-auto-commit-action@v4
-      with:
-        # Required
-        commit_message: Rt model 2 action run
-        # Optional branch to push to, defaults to the current branch
-        #branch: master
-        # Optional options appended to `git-commit`
-        #commit_options: '--no-verify --signoff'
-        # Optional glob pattern of files which should be added to the commit
-        file_pattern: data/calc/*.csv
-        # Optional local file path to the repository
-        #repository: .
-        # Optional commit user and author settings
-        # Optional tag message
-        # Action will create and push a new tag to the remote repository and the defined branch
-        #tagging_message: 'v1.0.0'
-        # Optional options appended to `git-push`
-        push_options: '--force'
-        commit_author: github-actions <[email protected]>
+#      - name: Commit changes
+#        run: |
+#          git config --local user.name github-actions
+#          git config --local user.email "[email protected]"
+#          git add data/calc/*.csv
+#          if [[ "$(git status --porcelain)" != "" ]]; then
+#              git commit -m "Rt model 2 action run" -a
+#              git push origin master
+#          fi
+
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          # Required
+          commit_message: Rt model 2 action run
+          # Optional branch to push to, defaults to the current branch
+          #branch: master
+          # Optional options appended to `git-commit`
+          #commit_options: '--no-verify --signoff'
+          # Optional glob pattern of files which should be added to the commit
+          file_pattern: data/calc/*.csv
+          # Optional local file path to the repository
+          #repository: .
+          # Optional commit user and author settings
+          # Optional tag message
+          # Action will create and push a new tag to the remote repository and the defined branch
+          #tagging_message: 'v1.0.0'
+          # Optional options appended to `git-push`
+#           push_options: '--force'
+          commit_author: github-actions <[email protected]>