Merge pull request #88 from hed-standard/develop

Implemented the HED remodeling tools in the web interface

.github/workflows/ci.yaml

@@ -6,9 +6,11 @@ on:
   pull_request:
     branches: ["*"]
 
+env:
+  BRANCH_NAME: ${{ github.event.pull_request.base.ref || github.ref_name }}
+
 jobs:
   build:
-
     strategy:
       matrix:
         platform: [ubuntu-latest]
@@ -17,19 +19,33 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}-${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }}
 
+      - name: Install dependencies(using github hed-python)
+        if: ${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }}
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+          pip install coverage
+          pip install git+https://github.com/hed-standard/hed-python/@${{env.BRANCH_NAME}}
+          pip install -r requirements.txt
+          pip install -r docs/requirements.txt
+
+      - name: Install dependencies(using pip)
+        if: ${{ env.BRANCH_NAME != 'develop' && env.BRANCH_NAME != 'master' }}
         run: |
           python -m pip install --upgrade pip
           pip install flake8
           pip install coverage
-          echo Using ${CI_COMMIT_BRANCH}
           pip install hedtools
           pip install -r requirements.txt
           pip install -r docs/requirements.txt
@@ -38,19 +54,18 @@ jobs:
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 
 
       - name: Test with unittest
         run: |
           cp -f ./config_template.py ./config.py
           coverage run -m unittest
 
-
       - name: publish-coverages
-        uses: paambaati/[email protected]
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        if: ${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }}
+        continue-on-error: true
         with:
-            coverageCommand: coverage xml
-            debug: true
+          coverageCommand: coverage xml
+          debug: true
+        uses: paambaati/[email protected]
+        env:
+          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID_WEB }}

README.md

@@ -72,4 +72,30 @@ to your `deploy_hed` directory.
 ```
 
 The `deploy.sh` script will download the latest versions of the `hed-python`
-and the `hed-web` repositories and deploy.
+and the `hed-web` repositories and deploy.
+
+### Branches and versions
+
+The web tools are built on the `hedtools` package housed in the `hed-python`
+GitHub repository.
+The tools are related to the `hed-specification` and `hed-schemas` repositories.
+The branches correspond as follows:
+
+| Branch |  Meaning | Synchronized with |
+| ------ | -------- | ------------------ |
+| stable | Tagged as a released version - will not change. | `stable@hed-python`<br/>`stable@hed-specification`<br/>`stable@hed-examples` |
+| master | Most recent usable version.<br/>[https://hedtools/edu/hed](https://hedtools/edu/hed). | `master@hed-python`<br/>`master@hed-specification`<br/>`main@hed-examples` |
+| develop | Experimental and evolving.<br/>[https://hedtools/edu/hed_dev](https://hedtools/edu/hed_dev). | `develop@hed-python`<br/>`develop@hed-specification`<br/>`develop@hed-examples` |
+
+As features are integrated, they first appear in the `develop` branches of the
+repositories.
+The `develop` branches of the repositories will be kept in sync as much as possible
+If an interface change in `hed-python` triggers a change in `hed-web` or `hed-examples`,
+every effort will be made to get the `master`/`main` branches of the respective repositories in
+sync.
+The `stable` version refers to the last officially released version.
+It generally refers to a version without the latest features.
+
+API documentation is generated on ReadTheDocs when a new version is
+pushed on any of the three branches. For example, the API documentation for the
+`latest` branch can be found on [hed-python.readthedocs.io/en/latest/](hed-python.readthedocs.io/en/latest/).

deploy_hed/Dockerfile

@@ -10,7 +10,7 @@ apache2 \
 apache2-dev && \
 pip3 install --upgrade pip && \
 pip3 install --no-cache-dir -r requirements.txt && \
-pip3 install hedtools && \
+pip3 install git+https://github.com/hed-standard/hed-python/@master && \
 mkdir -p /var/www/localhost/htdocs && \
 cp /etc/mime.types /var/www/mime.types && \
 mkdir -p /var/log/hedtools && \

deploy_hed/base_config.py

@@ -26,6 +26,8 @@ class Config(object):
 class DevelopmentConfig(Config):
     DEBUG = False
     TESTING = False
+    URL_PREFIX = '/heddev'
+    STATIC_URL_PATH = '/heddev/hedweb/static'
 
 
 class ProductionConfig(Config):

deploy_hed/deploy.sh

@@ -77,7 +77,7 @@ docker rm -f $CONTAINER_NAME
 run_new_container()
 {
 echo "Running new container $CONTAINER_NAME ..."
-docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME
+docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME --log-opt max-size=50m
 }
 
 cleanup_directory()

deploy_hed/httpd.conf

@@ -13,3 +13,4 @@ LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
 #LoadModule unixd_module /usr/lib/apache2/modules/mod_unixd.so
 LoadModule wsgi_module /usr/local/lib/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so
 WSGIScriptAlias / /var/www/hedtools/web.wsgi
+WSGIApplicationGroup %{GLOBAL}

deploy_hed_dev/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.9-slim-buster
+COPY requirements.txt /root/
+WORKDIR /root
+RUN apt-get update && apt-get install -y gcc \
+git \
+musl-dev \
+openrc \
+libxslt-dev \
+libxml2-dev \
+apache2 \
+apache2-dev && \
+pip3 install --upgrade pip && \
+pip3 install --no-cache-dir -r requirements.txt && \
+pip3 install git+https://github.com/hed-standard/hed-python/@develop && \
+mkdir -p /var/www/localhost/htdocs && \
+cp /etc/mime.types /var/www/mime.types && \
+mkdir -p /var/log/hedtools && \
+chown -R www-data:www-data /var/log/hedtools && \
+mkdir -p /var/cache/schema_cache && \
+chown -R www-data:www-data /var/cache/schema_cache
+COPY httpd.conf /etc/apache2/apache2.conf
+COPY ./hedtools /var/www/hedtools/
+COPY ./hedtools/hedweb /var/www/hedtools/hedweb/
+ENTRYPOINT /usr/sbin/apache2 -D FOREGROUND -f /etc/apache2/apache2.conf
+ENV HEDTOOLS_CONFIG_CLASS=config.ProductionConfig

deploy_hed_dev/base_config.py

@@ -0,0 +1,45 @@
+"""
+This module contains the configurations for the HEDTools application.
+"""
+
+import os
+import tempfile
+
+
+class Config(object):
+    LOG_DIRECTORY = '/var/log/hedtools'
+    LOG_FILE = os.path.join(LOG_DIRECTORY, 'error.log')
+    if not os.path.exists('/var/log/hedtools/tmp.txt'):
+        f = open('/var/log/hedtools/tmp.txt', 'w+')
+        f.write(str(os.urandom(24)))
+        f.close()
+    f = open('/var/log/hedtools/tmp.txt', 'r')
+    SECRET_KEY = f.read()  # os.getenv('SECRET_KEY') # os.urandom(24)
+    f.close()
+    STATIC_URL_PATH = None
+    STATIC_URL_PATH_ATTRIBUTE_NAME = 'STATIC_URL_PATH'
+    UPLOAD_FOLDER = os.path.join(tempfile.gettempdir(), 'hedtools_uploads')
+    URL_PREFIX = None
+    HED_CACHE_FOLDER = '/var/cache/schema_cache'
+
+
+class DevelopmentConfig(Config):
+    DEBUG = False
+    TESTING = False
+
+
+class ProductionConfig(Config):
+    DEBUG = False
+    TESTING = False
+    URL_PREFIX = '/hed_dev'
+    STATIC_URL_PATH = '/hed_dev/hedweb/static'
+
+
+class TestConfig(Config):
+    DEBUG = False
+    TESTING = True
+
+
+class DebugConfig(Config):
+    DEBUG = True
+    TESTING = False

deploy_hed_dev/deploy.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# deploy.sh - A script used to _build and deploy a docker container for the HEDTools online validator
+
+if [ $# -eq 0 ]; then
+  BRANCH="master"
+else
+  BRANCH="$1"
+fi
+##### Constants
+
+DEPLOY_DIR=${PWD}
+IMAGE_NAME="hedtools_dev:latest"
+CONTAINER_NAME="hedtools_dev"
+GIT_WEB_REPO_URL="https://github.com/hed-standard/hed-web"
+GIT_HED_WEB_DIR="${DEPLOY_DIR}/hed-web"
+GIT_WEB_REPO_BRANCH=${BRANCH}
+HOST_PORT=33004
+CONTAINER_PORT=80
+
+CODE_DEPLOY_DIR="${DEPLOY_DIR}/hedtools"
+SOURCE_DEPLOY_DIR="${DEPLOY_DIR}/hed-web/deploy_hed_dev"
+BASE_CONFIG_FILE="${SOURCE_DEPLOY_DIR}/base_config.py"
+CONFIG_FILE="${CODE_DEPLOY_DIR}/config.py"
+SOURCE_WSGI_FILE="${SOURCE_DEPLOY_DIR}/web.wsgi"
+SOURCE_DOCKERFILE="${SOURCE_DEPLOY_DIR}/Dockerfile"
+SOURCE_REQUIREMENTS_FILE="${SOURCE_DEPLOY_DIR}/requirements.txt"
+SOURCE_HTTPD_CONF="${SOURCE_DEPLOY_DIR}/httpd.conf"
+WEB_CODE_DIR="${DEPLOY_DIR}/hed-web/hedweb"
+
+##### Functions
+
+clone_github_repos(){
+echo "Deploy dir: ${DEPLOY_DIR}"
+cd "${DEPLOY_DIR}" || exit_error
+echo "Cloning repo ${GIT_WEB_REPO_URL} in ${DEPLOY_DIR} using ${GIT_WEB_REPO_BRANCH} branch"
+git clone "${GIT_WEB_REPO_URL}" -b "${GIT_WEB_REPO_BRANCH}"
+}
+
+create_web_directory()
+{
+echo Creating hedweb directory...
+echo "Make ${CODE_DEPLOY_DIR}"
+mkdir "${CODE_DEPLOY_DIR}"
+echo "Copy ${BASE_CONFIG_FILE} to ${CONFIG_FILE}"
+cp "${BASE_CONFIG_FILE}" "${CONFIG_FILE}"
+echo "Copy ${SOURCE_WSGI_FILE} to ${CODE_DEPLOY_DIR}"
+cp "${SOURCE_WSGI_FILE}" "${CODE_DEPLOY_DIR}/."
+echo "Copy ${SOURCE_DOCKERFILE} to ${DEPLOY_DIR}"
+cp "${SOURCE_DOCKERFILE}" "${DEPLOY_DIR}/."
+echo "Copy ${SOURCE_REQUIREMENTS_FILE} to ${DEPLOY_DIR}"
+cp "${SOURCE_REQUIREMENTS_FILE}" "${DEPLOY_DIR}/."
+echo "Copy ${SOURCE_HTTPD_CONF} to ${DEPLOY_DIR}"
+cp "${SOURCE_HTTPD_CONF}" "${DEPLOY_DIR}/."
+echo "Copy ${WEB_CODE_DIR} directory to ${CODE_DEPLOY_DIR}"
+cp -r "${WEB_CODE_DIR}" "${CODE_DEPLOY_DIR}"
+}
+
+switch_to_web_directory()
+{
+echo Switching to hedweb directory...
+cd "${DEPLOY_DIR}" || error_exit "Cannot access $DEPLOY_DIR"
+}
+
+build_new_container()
+{
+echo "Building new container ${IMAGE_NAME} ..."
+docker build -t $IMAGE_NAME .
+}
+
+delete_old_container()
+{
+echo "Deleting old container ${CONTAINER_NAME} ..."
+docker rm -f $CONTAINER_NAME
+}
+
+run_new_container()
+{
+echo "Running new container $CONTAINER_NAME ..."
+docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME --log-opt max-size=50m
+}
+
+cleanup_directory()
+{
+echo "Cleaning up directory ${GIT_HED_WEB_DIR} ..."
+rm -rf "${GIT_HED_WEB_DIR}"
+echo "Cleaning up ${CODE_DEPLOY_DIR}"
+rm -rf "${CODE_DEPLOY_DIR}"
+}
+
+error_exit()
+{
+	echo "$1" 1>&2
+	exit 1
+}
+
+output_paths()
+{
+echo "The relevant deployment information is:"
+echo "Deploy directory: ${DEPLOY_DIR}"
+echo "Docker image name: ${IMAGE_NAME}"
+echo "Docker container name: ${CONTAINER_NAME}"
+echo "Git tools repo: ${GIT_TOOLS_REPO_URL}"
+echo "Git web repo: ${GIT_WEB_REPO_URL}"
+echo "Git web repo branch: ${GIT_WEB_REPO_BRANCH}"
+echo "Git hed web dir: ${GIT_HED_WEB_DIR}"
+echo "Host port: ${HOST_PORT}"
+echo "Container port: ${CONTAINER_PORT}"
+echo "Local deployment directory: ${DEPLOY_DIR}"
+echo "Local deploy code dir: ${DEPLOY_CODE_DIR}"
+echo "Local code deployment directory: ${CODE_DEPLOY_DIR}"
+echo "Configuration file: ${CONFIG_FILE}"
+echo "Base configuration file: ${BASE_CONFIG_FILE}"
+echo "Source WSGI file: ${SOURCE_WSGI_FILE}"
+echo "Source web code directory: ${WEB_CODE_DIR}"
+}
+
+##### Main
+echo "Starting...."
+output_paths
+echo "....."
+echo "Cleaning up directories before deploying..."
+cleanup_directory
+clone_github_repos || error_exit "Cannot clone repo ${GIT_WEB_REPO_URL}"
+create_web_directory
+switch_to_web_directory
+build_new_container
+delete_old_container
+run_new_container
+cleanup_directory

deploy_hed_dev/httpd.conf

@@ -0,0 +1,16 @@
+DocumentRoot /var/www/localhost/htdocs
+ErrorLog /dev/stderr
+Transferlog /dev/stdout
+Listen 80
+ServerName localhost
+ServerRoot /var/www
+User www-data
+Group www-data
+LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
+LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
+LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
+LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
+#LoadModule unixd_module /usr/lib/apache2/modules/mod_unixd.so
+LoadModule wsgi_module /usr/local/lib/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so
+WSGIScriptAlias / /var/www/hedtools/web.wsgi
+WSGIApplicationGroup %{GLOBAL}