build(dev): option for setting service image and volume names (#2288)

* custom db preparation
  - add optional var RAKE_DB_MIGRATE to run db:migrate instead of db:setup
  - also prevent multiple `yarn install`

* devcontainer - avoid overwriting .env

copy compose files and env variable to .devcontainer
and keep .env if present untouched as it was originaly
for the rails app (dotenv)

* env var to set volume names, app or db images:

   set env variables in the docker-compose.dev.yml to allow using
      - prebuild images for the app and skip the building process
      - distinct postgres images
      - distinct named volume for the db and homedir(asdf, gems, etc)

  remove image tags from dev docker compose file by yml overwrite for devcontainer build to avoid
retagging the images

* set a .dockerenv.example
that can be used instead of .env

to have a better separation of variable for the configuring
the docker env and variables to be used insides the services

* vscode tasks to up/down with .dockerenv

.devcontainer/devcontainer.json

@@ -2,7 +2,8 @@
 {
   "name": "Chemotion Dockerfile",
   "dockerComposeFile": [
-    "../docker-compose.dev.yml"
+    "docker-compose.dev.yml",
+    "docker-compose.vs.yml"
   ],
   "service": "app",
   "workspaceFolder": "/home/chemotion-dev/app",
@@ -61,4 +62,4 @@
   "containerEnv": {
     "RAILS_ENV": "development"
   },
-}
+}

.devcontainer/docker-compose.vs.yml

@@ -0,0 +1,28 @@
+x-common-volumes:
+  - &vhome homedir:/home/chemotion-dev
+  - &vapp ..:/home/chemotion-dev/app
+
+
+services:
+  app:
+    build:
+      context: '.'
+      dockerfile: 'Dockerfile.chemotion-dev'
+      # args:
+        # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy}  # Build ARG for base image
+        # FULL_BUILD: ${FULL_BUILD:-false}
+    image: ""
+    volumes:
+      - *vhome
+      - *vapp
+  webpacker:
+    build:
+      context: '.'
+      dockerfile: 'Dockerfile.chemotion-dev'
+      # args:
+        # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy}  # Build ARG for base image
+        # FULL_BUILD: ${FULL_BUILD:-false}
+    image: ""
+    volumes:
+      - *vhome
+      - *vapp

.devcontainer/pre_create.sh

@@ -1,7 +1,55 @@
 #!/bin/bash
 
+# set the .env for the root directory
+# that will be used for the app and worker services
+# - use .env if it exists
+# - otherwise use .env.example if it exists
+# - otherwise create an empty .env file
+if [ -f .env ]; then
+  echo ".env already exists"
+else
+  if [ -f .env.example ]; then
+    cp .env.example .env
+  else
+    echo "No .env.example file found"
+    touch .env
+  fi
+fi
+
+# set the devcontainer/.env to be used by the devcontainer docker-compose
+# in order - the last one wins
+# - use .dockerenv if it exists
+# - otherwise use .dockerenv.example
+# - append the contents of .env
+# - append the contents of .env.development (this should be removed and kept for the service)
+
+if [ -f .dockerenv ]; then
+  echo "Using .dockerenv to create .devcontainer/.env"
+  cp .dockerenv .devcontainer/.env
+elif [ -f ./.dockerenv.example ]; then
+  echo "Using .dockerenv.example to create .devcontainer/.env"
+  cp .dockerenv.example .devcontainer/.env
+else
+  echo "Neither .dockerenv nor .dockerenv.example found. Exiting."
+  exit 1
+fi
+
+if [ -f .env ]; then
+  echo "Appending .env contents to .devcontainer/.env"
+  cat .env >> .devcontainer/.env
+else
+  echo "No .env file found in the current directory."
+fi
+
+cat .env.development >> .devcontainer/.env
+
+echo ".devcontainer/.env created successfully."
+
+# make copies of the docker-compose and Dockerfile for the devcontainer
+cp docker-compose.dev.yml .devcontainer/docker-compose.dev.yml
+cp Dockerfile.chemotion-dev .devcontainer/Dockerfile.chemotion-dev
+
 # enable configuration files
-cp .env.development .env
 cp public/welcome-message-sample.md public/welcome-message.md
 cp config/datacollectors.yml.example config/datacollectors.yml
 cp config/storage.yml.example config/storage.yml

.dockerenv.example

@@ -0,0 +1,28 @@
+## Environment variables for docker-compose.dev.yml
+## Copy this file to .dockerenv and adjust the values
+## Do not commit .dockerenv to the repository
+## `docker compose --env-file .dockerenv -f docker-compose.dev.yml config`
+##  will use the values from this file
+## - DOCKER_PG_IMAGE: the image to use as base for the db container
+## - DOCKER_DEV_IMAGE: the image to use as base for the app and webpacker containers
+##  overwriten to '' when using vs decontainer.json to avoid tag conflicts
+## Latest available version: https://hub.docker.com/u/complat/dev/tags
+## app image with preinstalled asdf plugins(ruby, nodejs), gems and nodejs packages
+
+DOCKER_DEV_IMAGE=complat/dev:v1.10.3-37-ga95534401
+#DOCKER_PG_IMAGE=postgres:16
+
+## - VOLUME_NAME_HOMEDIR: Use another named volume for homedir (asdf, gems, etc)
+## - VOLUME_NAME_DB: or database
+
+#VOLUME_NAME_HOMEDIR=chemotion_eln_homedir2
+#VOLUME_NAME_DB=chemotion_eln_database2
+
+## ENV for the app container
+## - RAKE_DB_MIGRATE: use by prepare sh to run db migration (rake db:migrate)
+## when starting the app container {always, once, never}
+## always: run db migration on every start
+## once: run db migration only once after the db is created
+## never: never run db migration on start
+RAKE_DB_MIGRATE=once
+

.gitignore

@@ -32,6 +32,9 @@
 
 .env
 .env.test
+.dockerenv
+/.devcontainer/Dockerfile*
+/.devcontainer/docker-compose.dev*
 
 /config/matrices.json
 /config/mailcollector.yml

.vscode/tasks.json

@@ -53,6 +53,18 @@
       "label": "mocha - unit tests current file",
       "type": "shell",
       "command": "NODE_PATH=./spec/javascripts:./app/javascript yarn mocha --watch --exit --require '@babel/register' './spec/javascripts/helper/setup.js' '${file}'"
+    },
+    {
+      "label": "Dev Docker Compose Up",
+      "type": "shell",
+      "command": "docker",
+      "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "up", "-d"]
+    },
+    {
+      "label": "Dev Docker Compose Down",
+      "type": "shell",
+      "command": "docker",
+      "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "down", "--remove-orphans"]
     }
   ]
 }

docker-compose.dev.yml

@@ -9,55 +9,74 @@
 # - Run tests:
 #     bundle exec rspec
 #
+# - see .dockerenv.example for environment variables usage
+
+x-common-volumes:
+  - &vhome homedir:/home/chemotion-dev
+  - &vapp .:/home/chemotion-dev/app
+
 services:
   postgres:
-    image: 'postgres:14'
+    image: ${DOCKER_PG_IMAGE:-postgres:14}
     environment:
       - 'POSTGRES_HOST_AUTH_METHOD=trust'
     expose: # expose port to app container
       - '5432'
     ports: # expose port to host machine in case we want to use external db gui tools
       - '5432:5432'
     volumes:
-      - 'database:/var/lib/postgresql/data'
+      - database:/var/lib/postgresql/data
 
   app:
     build:
       context: '.'
       dockerfile: 'Dockerfile.chemotion-dev'
+      # args:
+        # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy}  # Build ARG for base image
+        # FULL_BUILD: ${FULL_BUILD:-}
+    image: ${DOCKER_DEV_IMAGE:-}
     depends_on:
       - 'postgres'
     healthcheck:
       test: ["CMD", "bundle", "check"] # exit 0 if all gems from Gemfile are installed, otherwise exit 1
       interval: 30s
       timeout: 10s
+    env_file:
+      - ./.env
     environment:
       - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker'
       - 'SHAKAPACKER_DEV_SERVER_PORT=3035'
       - 'THOR_SILENCE_DEPRECATION=true'
+      - RAKE_DB_MIGRATE=${RAKE_DB_MIGRATE:-never}
     ports: # expose default rails port to host machine
       - "3000:3000"
     volumes:
-      - 'homedir:/home/chemotion-dev/'
-      - '.:/home/chemotion-dev/app'
+      - *vhome
+      - *vapp
     working_dir: "/home/chemotion-dev/app"
     command: "./run-ruby-dev.sh"
 
   webpacker:
     build:
       context: '.'
       dockerfile: 'Dockerfile.chemotion-dev'
+      # args:
+        # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy}  # Build ARG for base image
+        # FULL_BUILD: ${FULL_BUILD:-}
+    image: ${DOCKER_DEV_IMAGE:-}
     depends_on:
       app:
         condition: service_healthy
     environment:
       - 'NODE_ENV=development'
       - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker'
       - 'SHAKAPACKER_DEV_SERVER_PORT=3035'
-    env_file: ./.env
+    env_file:
+      - ./.env
+      #- ./.env.development
     volumes:
-      - 'homedir:/home/chemotion-dev/'
-      - '.:/home/chemotion-dev/app'
+      - *vhome
+      - *vapp
     ports: # expose webpacker dev server port to app container
       - '3035:3035'
     expose:
@@ -67,4 +86,6 @@ services:
 
 volumes:
   database:
+    name: ${VOLUME_NAME_DB:-chemotion_eln_database}
   homedir:
+    name: ${VOLUME_NAME_HOMEDIR:-chemotion_eln_homedir}

prepare-nodejs.sh

@@ -7,6 +7,7 @@
 
 set -e
 
+echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
 # Get the currently installed Node.js version using asdf
 CURRENT_NODE_VERSION=$(asdf current nodejs 2>/dev/null | awk '{print $2}')
 

prepare-nodejspkg.sh

@@ -24,5 +24,4 @@ fi
 
 echo '>>> Installing JS packages...'
 yarn install --production=false
-yarn install
 

prepare-ruby-dev.sh

@@ -7,29 +7,69 @@ export ASDF_BRANCH=v0.14.0
 echo '>>> checking asdf installation'
 ./prepare-asdf.sh
 
-# check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
-echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
+# nodejs installation
 ./prepare-nodejs.sh
 
 # ruby gems installation
 ./prepare-rubygems.sh
 
-# node packages installation
-./prepare-nodejspkg.sh
-
 # prepare rails server
 rm -f tmp/pids/server.pid
 
-if [ "$( psql -h postgres -U postgres -XtAc "SELECT 1 FROM pg_database WHERE datname='chemotion_dev'" )" = '1' ]
+# prepare rails database
+# assume default database configuration
+DATABASE_NAME=${DATABASE_NAME:-chemotion_dev}
+DATABASE_USER=${DATABASE_USER:-postgres}
+DATABASE_HOST=${DATABASE_HOST:-postgres}
+DATABASE_PORT=${DATABASE_PORT:-5432}
+
+# check if yq is installed
+# if yq is installed parse config/database.yml file for the actual values
+# if yq is not installed, then keep the set values default values
+if command -v yq &> /dev/null
 then
+  DATABASE_NAME=$(yq -r .development.database config/database.yml)
+  DATABASE_USER=$(yq -r .development.username config/database.yml)
+  DATABASE_HOST=$(yq -r .development.host config/database.yml)
+  DATABASE_PORT=$(yq -r .development.port config/database.yml)
+fi
+echo "DATABASE_NAME: $DATABASE_NAME"
+echo "DATABASE_USER: $DATABASE_USER"
+echo "DATABASE_HOST: $DATABASE_HOST"
+echo "DATABASE_PORT: $DATABASE_PORT"
+
+# check if the database for the given environment configuration exists
+db_exists=$( psql -h $DATABASE_HOST -U $DATABASE_USER -p $DATABASE_PORT -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" )
+echo "Database exists: $db_exists"
+if [ "$db_exists" = '1' ]
+then
+  echo "==================================================="
+  echo "Database already exists, skipping Database creation"
+  echo "==================================================="
+  if [ "$RAKE_DB_MIGRATE" = "always" ]
+  then
     echo "================================================"
-    echo "Database already exists, skipping Database setup"
+    echo "Running 'rake db:migrate'"
     echo "================================================"
+      bundle exec rake db:migrate
+  fi
 else
+  # if RAKE_DB_MIGRATE is set to always or once, run rake db:setup
+  if [ "$RAKE_DB_MIGRATE" = "always" ] || [ "$RAKE_DB_MIGRATE" = "once" ]
+  then
+    echo "================================================"
+    echo "Database does not exist"
+    echo "running 'rake db:create/migrate/seed'"
+    echo "================================================"
+    bundle exec rake db:create
+    bundle exec rake db:migrate
+    bundle exec rake db:seed
+  else
     echo "================================================"
     echo "Database does not exist, running 'rake db:setup'"
     echo "================================================"
     bundle exec rake db:setup
+  fi
 fi
 
 

run-js-dev.sh

@@ -1,14 +1,8 @@
 #!/bin/bash
 
-if command -v yarn; then
-    echo '>>> yarn is installed -> continue'
-else
-    echo '>>> Missing yarn. Installing...'
-    npm install -g yarn
-fi
-
-echo '>>> Installing JS packages...'
-yarn install
+## check yarn installation and install nodejs packages
+## assume nodejs is installed (through ./run-ruby-dev.sh)
+./prepare-nodejspkg.sh
 
 echo "=========================================================================================================="
 echo "THIS WILL FAIL UNTIL THE RUBY GEMS ARE INSTALLED BY run-ruby-dev.sh. JUST TRY AGAIN AFTER INSTALLING THEM."