initial commit - contains python config

.gitignore

@@ -0,0 +1,7 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Jupyter Notebook
+.ipynb_checkpoints

0_config/README.md

@@ -0,0 +1,4 @@
+# Configuration file
+The `config.txt` file in this folder contains high-level workflow decisions. This prevents needing to hard-code such decisions (like, for example, file paths) in the remainder of the code and this in turns leads to more efficient reproducibility.
+
+The file contains a brief explanation of its own structure. 

0_config/config.txt

@@ -0,0 +1,21 @@
+## Contains settings for CAMELS-spat processing, for transparency and reproducibility
+
+# Overview:
+# [setting] | [value] | [type] # [description]
+#
+# [setting]     : Scripts look for this string to identify the line in this file that contains the information they need.
+# [value]       : Value of the setting used by the scripts. 
+# [type]        : Data type the value is converted to in the script. 
+# [description] : Description for user's benefit
+
+
+# Main settings
+code_path | C:\Git_repos\camels_spat\       | string  # Root path of processing code
+data_path | C:\Globus endpoint\CAMELS_spat\ | string  # Root path of data location
+
+# 0. Python setup
+venv_path | C:\Globus endpoint\CAMELS_spat\ # Path where the Python virtual environment will be created
+venv_name | camels-spat-env                 # Virtual environment name
+reqs_path | 1_Python_setup\                 # Sub-folders in root that contain requirements
+reqs_file | 0_requirements.txt              # Python environment requirements for `pip install -r reqs.txt`
+

1_Python_setup/0_requirements.txt

@@ -0,0 +1 @@
+cdsapi==0.2.4

1_Python_setup/1_make_venv.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Creates a virtual environment for CAMELS-spat computations.
+# Uses pip and the `0_requirements.txt` file in this folder.
+
+# To run Bash files under Windows, use a dedicated terminal such as:
+# - Windows Subsystem for Linux: https://docs.microsoft.com/en-us/windows/wsl/install
+# - Git Bash (part of Git for Windows): https://gitforwindows.org/
+# - Cygwin: https://www.cygwin.com/
+
+# Define the location of the config file
+# --------------------------------------
+config="../0_config/config.txt"
+
+# Get all the relevant info from the config file
+# ----------------------------------------------
+# Config file always has the setting we want as the 2nd item in a row, separated by \ or #
+while IFS='|#' read -ra LINE; do venv_path=$(echo ${LINE[1]}); done <<< $(grep -m 1 "^venv_path" $config) # Where the venv should go
+while IFS='|#' read -ra LINE; do venv_name=$(echo ${LINE[1]}); done <<< $(grep -m 1 "^venv_name" $config) # What the venv will be called
+while IFS='|#' read -ra LINE; do code_path=$(echo ${LINE[1]}); done <<< $(grep -m 1 "^code_path" $config) # Root path to code
+while IFS='|#' read -ra LINE; do reqs_path=$(echo ${LINE[1]}); done <<< $(grep -m 1 "^reqs_path" $config) # Requirements file sub-folder(s)
+while IFS='|#' read -ra LINE; do reqs_file=$(echo ${LINE[1]}); done <<< $(grep -m 1 "^reqs_file" $config) # Requirements filename
+
+# Ensure a clean install
+# ----------------------
+echo "Attempting to install virtualenv ${venv_path}/${venv_name}. Will remove venv folder if it exists to ensure a clean install."
+mkdir -p "${venv_path}/${venv_name}" # Ensure the destination directory exists
+rm -rf "${venv_path}/${venv_name}" # Remove the virtualenv if it exists
+
+# Make the virtualenv
+# -------------------
+# Source: https://docs.python.org/3/library/venv.html
+# Note: command should work on both Windows and Unix(-like) OS
+# Note: assumes Python 3 is available on the system and added to the PATH
+python -m venv "${venv_path}/${venv_name}"
+
+# Figure out what OS we are on
+# ----------------------------
+# Source: https://stackoverflow.com/a/3466183
+# Note: just adding a few common ones. More options in table here: https://en.wikipedia.org/wiki/Uname
+# Implicit assumption: follow-up commands are identical for Cygwin and Mingw. Tested only on Mingw.
+# Implicit assumption: follow-up commands are identical for Linux and MacOS. Not tested.
+uname_out="$(uname -s)"
+case "${uname_out}" in
+    Linux*)     machine=Unix;;
+    Darwin*)    machine=Unix;;
+    CYGWIN*)    machine=Windows;; 
+    MINGW*)     machine=Windows;;
+    *)          machine="UNKNOWN:${uname_out}" 
+esac
+
+# Check if we're ready for the OS we're on
+# ----------------------------------------
+if [[ "UNKNOWN" == *${machine}* ]]; then
+    echo "OS ${machine}"
+	exit 1 # Catch-all error code: https://tldp.org/LDP/abs/html/exitcodes.html
+fi
+
+# Activate the virtualenv
+# -----------------------
+# Source: https://docs.python.org/3/library/venv.html
+echo "Attempting to activate ${venv_name} using ${machine} command."
+if [[ "Unix" == *${machine}* ]]; then
+    source "${venv_path}/${venv_name}/bin/activate"
+elif [[ "Windows" == *${machine}* ]]; then
+    # Source: https://medium.com/@presh_onyee/activating-virtualenv-on-windows-using-git-bash-python-3-7-1-6b4b21640368
+    chmod +x "${venv_path}/${venv_name}/Scripts/activate.bat"
+    . "${venv_path}/${venv_name}/Scripts/activate"
+fi
+
+# Check if we activated the virtualenv
+# ------------------------------------
+# Source: https://stackoverflow.com/a/3063887
+if [[ -z "$VIRTUAL_ENV" ]]; then
+    echo "Virtual environment not successfully activated."
+	exit 1
+else
+    echo "Virtual environment successfully activated in: ${VIRTUAL_ENV}"
+	echo # empty line for clarity on the terminal - ugly, but no idea how to do better
+fi
+
+# Install the required packages
+# -----------------------------
+python -m pip install --upgrade pip # Ensure we have the latest pip version
+python -m pip install -r "${code_path}/${reqs_path}/${reqs_file}" # Install everything else
+
+# Run test to check if we have everything we need
+# -----------------------------------------------
+# Source: https://stackoverflow.com/a/45474387
+echo # empty line for clarity on the terminal
+echo "Checking packages installed in: ${venv_name}"
+cd $code_path/python_tests # needed because using relative paths to a parent directory doesn't play nice with next line
+python -m unittest test_requirements
+
+# Clean-up
+# --------
+# Note: virtualenv will be automatically deactivated upon script exit, 
+# because executing a script opens a new virtual terminal. Everything
+# happens in there and hence in the current terminal (from which we
+# activate this script) no virtualenv is activated at all.

README.md

@@ -0,0 +1,16 @@
+# CAMELS-spat
+Repository that contains processing code used to generate the CAMELS-spat (Catchment Attributes and MEteorology for Large-Sample studies - SPATially distributed) data set.
+
+## Repository description
+
+This repository contains the following sub-folders:
+1. **0_config** - contains a configuration file with all high-level decisions. Prevents needing to hard-code paths in the remainder of the code and this in turns leads to more efficient reproducibility.
+2. **1_Python_setup** - contains:
+	- **0_tools**: folder with shared functions.
+	- **1_make_venv**: requirements of and code to configure a Python virtual environment.
+
+## Reproducibility
+
+To reproduce the data processing steps, execute scripts in order, starting at folder `1_Python_setup` and its sub-folders, before moving on to main folder `2_` and its sub-folders, main folder `3_` and its sub-folders, etc. Folder names starting `0_` do not contain anything that needs to be executed manually. 
+
+Further instructions and descriptions are found in the Readme's contain in sub-folders.

python_cs_functions/__init__.py

@@ -0,0 +1 @@
+from python_cs_functions.config import *

python_cs_functions/config.py

@@ -0,0 +1,25 @@
+'''Contains functions to interact with the configuration file.'''
+
+def my_tester():
+    print('Successfully executed test function.')
+
+def read_from_config( file, setting ):
+
+    # Open 'config.txt' and ...
+    with open(file) as contents:
+        for line in contents:
+
+            # ... find the line with the requested setting
+            if setting in line:
+                break
+
+    # Line comes out as: [setting] | [value] | [type] # [comment]
+    # Harmonize the delimiters
+    line = line.replace('#','|')
+
+    # Extract the setting's value and type
+    value = line.split('|')[1].strip()
+    type = line.split('|')[2].strip()
+
+    # Return this value    
+    return value,type

python_tests/test_requirements.py

@@ -0,0 +1,27 @@
+"""Test availability of required packages."""
+"""Source: https://stackoverflow.com/a/45474387"""
+
+import os,sys
+from pathlib import Path
+import unittest
+import pkg_resources
+sys.path.append(str(Path().absolute().parent))
+import python_cs_functions as cs
+
+# Get the location of the requirements file from the config file
+code_path,_ = cs.read_from_config('../0_config/config.txt','code_path')
+reqs_path,_ = cs.read_from_config('../0_config/config.txt','reqs_path')
+reqs_file,_ = cs.read_from_config('../0_config/config.txt','reqs_file')
+_REQUIREMENTS_PATH = Path(os.path.join(code_path,reqs_path,reqs_file))
+
+class TestRequirements(unittest.TestCase):
+    """Test availability of required packages."""
+
+    def test_requirements(self):
+        """Test that each required package is available."""
+        requirements = pkg_resources.parse_requirements(_REQUIREMENTS_PATH.open())
+        for requirement in requirements:
+            requirement = str(requirement)
+            with self.subTest(requirement=requirement):
+                pkg_resources.require(requirement)
+