From ded6bad7e35b521cc89b81d126fba4b0994cc9c6 Mon Sep 17 00:00:00 2001 From: PiTrem Date: Wed, 5 Feb 2025 13:36:34 +0100 Subject: [PATCH] build(dev): option for setting service image and volume names (#2288) * custom db preparation - add optional var RAKE_DB_MIGRATE to run db:migrate instead of db:setup - also prevent multiple `yarn install` * devcontainer - avoid overwriting .env copy compose files and env variable to .devcontainer and keep .env if present untouched as it was originaly for the rails app (dotenv) * env var to set volume names, app or db images: set env variables in the docker-compose.dev.yml to allow using - prebuild images for the app and skip the building process - distinct postgres images - distinct named volume for the db and homedir(asdf, gems, etc) remove image tags from dev docker compose file by yml overwrite for devcontainer build to avoid retagging the images * set a .dockerenv.example that can be used instead of .env to have a better separation of variable for the configuring the docker env and variables to be used insides the services * vscode tasks to up/down with .dockerenv --- .devcontainer/devcontainer.json | 5 +-- .devcontainer/docker-compose.vs.yml | 28 +++++++++++++++ .devcontainer/pre_create.sh | 50 +++++++++++++++++++++++++- .dockerenv.example | 28 +++++++++++++++ .gitignore | 3 ++ .vscode/tasks.json | 12 +++++++ docker-compose.dev.yml | 35 +++++++++++++++---- prepare-nodejs.sh | 1 + prepare-nodejspkg.sh | 1 - prepare-ruby-dev.sh | 54 +++++++++++++++++++++++++---- run-js-dev.sh | 12 ++----- 11 files changed, 202 insertions(+), 27 deletions(-) create mode 100644 .devcontainer/docker-compose.vs.yml create mode 100644 .dockerenv.example diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f5f1360d44..b8679f3b51 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,8 @@ { "name": "Chemotion Dockerfile", "dockerComposeFile": [ - "../docker-compose.dev.yml" + "docker-compose.dev.yml", + "docker-compose.vs.yml" ], "service": "app", "workspaceFolder": "/home/chemotion-dev/app", @@ -61,4 +62,4 @@ "containerEnv": { "RAILS_ENV": "development" }, -} \ No newline at end of file +} diff --git a/.devcontainer/docker-compose.vs.yml b/.devcontainer/docker-compose.vs.yml new file mode 100644 index 0000000000..4bf6927aa9 --- /dev/null +++ b/.devcontainer/docker-compose.vs.yml @@ -0,0 +1,28 @@ +x-common-volumes: + - &vhome homedir:/home/chemotion-dev + - &vapp ..:/home/chemotion-dev/app + + +services: + app: + build: + context: '.' + dockerfile: 'Dockerfile.chemotion-dev' + # args: + # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy} # Build ARG for base image + # FULL_BUILD: ${FULL_BUILD:-false} + image: "" + volumes: + - *vhome + - *vapp + webpacker: + build: + context: '.' + dockerfile: 'Dockerfile.chemotion-dev' + # args: + # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy} # Build ARG for base image + # FULL_BUILD: ${FULL_BUILD:-false} + image: "" + volumes: + - *vhome + - *vapp diff --git a/.devcontainer/pre_create.sh b/.devcontainer/pre_create.sh index 968595f728..ad96bdd9f5 100644 --- a/.devcontainer/pre_create.sh +++ b/.devcontainer/pre_create.sh @@ -1,7 +1,55 @@ #!/bin/bash +# set the .env for the root directory +# that will be used for the app and worker services +# - use .env if it exists +# - otherwise use .env.example if it exists +# - otherwise create an empty .env file +if [ -f .env ]; then + echo ".env already exists" +else + if [ -f .env.example ]; then + cp .env.example .env + else + echo "No .env.example file found" + touch .env + fi +fi + +# set the devcontainer/.env to be used by the devcontainer docker-compose +# in order - the last one wins +# - use .dockerenv if it exists +# - otherwise use .dockerenv.example +# - append the contents of .env +# - append the contents of .env.development (this should be removed and kept for the service) + +if [ -f .dockerenv ]; then + echo "Using .dockerenv to create .devcontainer/.env" + cp .dockerenv .devcontainer/.env +elif [ -f ./.dockerenv.example ]; then + echo "Using .dockerenv.example to create .devcontainer/.env" + cp .dockerenv.example .devcontainer/.env +else + echo "Neither .dockerenv nor .dockerenv.example found. Exiting." + exit 1 +fi + +if [ -f .env ]; then + echo "Appending .env contents to .devcontainer/.env" + cat .env >> .devcontainer/.env +else + echo "No .env file found in the current directory." +fi + +cat .env.development >> .devcontainer/.env + +echo ".devcontainer/.env created successfully." + +# make copies of the docker-compose and Dockerfile for the devcontainer +cp docker-compose.dev.yml .devcontainer/docker-compose.dev.yml +cp Dockerfile.chemotion-dev .devcontainer/Dockerfile.chemotion-dev + # enable configuration files -cp .env.development .env cp public/welcome-message-sample.md public/welcome-message.md cp config/datacollectors.yml.example config/datacollectors.yml cp config/storage.yml.example config/storage.yml diff --git a/.dockerenv.example b/.dockerenv.example new file mode 100644 index 0000000000..37f7e01452 --- /dev/null +++ b/.dockerenv.example @@ -0,0 +1,28 @@ +## Environment variables for docker-compose.dev.yml +## Copy this file to .dockerenv and adjust the values +## Do not commit .dockerenv to the repository +## `docker compose --env-file .dockerenv -f docker-compose.dev.yml config` +## will use the values from this file +## - DOCKER_PG_IMAGE: the image to use as base for the db container +## - DOCKER_DEV_IMAGE: the image to use as base for the app and webpacker containers +## overwriten to '' when using vs decontainer.json to avoid tag conflicts +## Latest available version: https://hub.docker.com/u/complat/dev/tags +## app image with preinstalled asdf plugins(ruby, nodejs), gems and nodejs packages + +DOCKER_DEV_IMAGE=complat/dev:v1.10.3-37-ga95534401 +#DOCKER_PG_IMAGE=postgres:16 + +## - VOLUME_NAME_HOMEDIR: Use another named volume for homedir (asdf, gems, etc) +## - VOLUME_NAME_DB: or database + +#VOLUME_NAME_HOMEDIR=chemotion_eln_homedir2 +#VOLUME_NAME_DB=chemotion_eln_database2 + +## ENV for the app container +## - RAKE_DB_MIGRATE: use by prepare sh to run db migration (rake db:migrate) +## when starting the app container {always, once, never} +## always: run db migration on every start +## once: run db migration only once after the db is created +## never: never run db migration on start +RAKE_DB_MIGRATE=once + diff --git a/.gitignore b/.gitignore index 166edb19da..b3963972da 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ .env .env.test +.dockerenv +/.devcontainer/Dockerfile* +/.devcontainer/docker-compose.dev* /config/matrices.json /config/mailcollector.yml diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9c0b82451a..fc4339ca49 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -53,6 +53,18 @@ "label": "mocha - unit tests current file", "type": "shell", "command": "NODE_PATH=./spec/javascripts:./app/javascript yarn mocha --watch --exit --require '@babel/register' './spec/javascripts/helper/setup.js' '${file}'" + }, + { + "label": "Dev Docker Compose Up", + "type": "shell", + "command": "docker", + "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "up", "-d"] + }, + { + "label": "Dev Docker Compose Down", + "type": "shell", + "command": "docker", + "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "down", "--remove-orphans"] } ] } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 54e5e8d198..ce7277ac9d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -9,9 +9,15 @@ # - Run tests: # bundle exec rspec # +# - see .dockerenv.example for environment variables usage + +x-common-volumes: + - &vhome homedir:/home/chemotion-dev + - &vapp .:/home/chemotion-dev/app + services: postgres: - image: 'postgres:14' + image: ${DOCKER_PG_IMAGE:-postgres:14} environment: - 'POSTGRES_HOST_AUTH_METHOD=trust' expose: # expose port to app container @@ -19,27 +25,34 @@ services: ports: # expose port to host machine in case we want to use external db gui tools - '5432:5432' volumes: - - 'database:/var/lib/postgresql/data' + - database:/var/lib/postgresql/data app: build: context: '.' dockerfile: 'Dockerfile.chemotion-dev' + # args: + # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy} # Build ARG for base image + # FULL_BUILD: ${FULL_BUILD:-} + image: ${DOCKER_DEV_IMAGE:-} depends_on: - 'postgres' healthcheck: test: ["CMD", "bundle", "check"] # exit 0 if all gems from Gemfile are installed, otherwise exit 1 interval: 30s timeout: 10s + env_file: + - ./.env environment: - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker' - 'SHAKAPACKER_DEV_SERVER_PORT=3035' - 'THOR_SILENCE_DEPRECATION=true' + - RAKE_DB_MIGRATE=${RAKE_DB_MIGRATE:-never} ports: # expose default rails port to host machine - "3000:3000" volumes: - - 'homedir:/home/chemotion-dev/' - - '.:/home/chemotion-dev/app' + - *vhome + - *vapp working_dir: "/home/chemotion-dev/app" command: "./run-ruby-dev.sh" @@ -47,6 +60,10 @@ services: build: context: '.' dockerfile: 'Dockerfile.chemotion-dev' + # args: + # source_image: ${DOCKER_DEV_IMAGE:-ubuntu:jammy} # Build ARG for base image + # FULL_BUILD: ${FULL_BUILD:-} + image: ${DOCKER_DEV_IMAGE:-} depends_on: app: condition: service_healthy @@ -54,10 +71,12 @@ services: - 'NODE_ENV=development' - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker' - 'SHAKAPACKER_DEV_SERVER_PORT=3035' - env_file: ./.env + env_file: + - ./.env + #- ./.env.development volumes: - - 'homedir:/home/chemotion-dev/' - - '.:/home/chemotion-dev/app' + - *vhome + - *vapp ports: # expose webpacker dev server port to app container - '3035:3035' expose: @@ -67,4 +86,6 @@ services: volumes: database: + name: ${VOLUME_NAME_DB:-chemotion_eln_database} homedir: + name: ${VOLUME_NAME_HOMEDIR:-chemotion_eln_homedir} diff --git a/prepare-nodejs.sh b/prepare-nodejs.sh index 85f85087e5..d71ea0df67 100755 --- a/prepare-nodejs.sh +++ b/prepare-nodejs.sh @@ -7,6 +7,7 @@ set -e +echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions' # Get the currently installed Node.js version using asdf CURRENT_NODE_VERSION=$(asdf current nodejs 2>/dev/null | awk '{print $2}') diff --git a/prepare-nodejspkg.sh b/prepare-nodejspkg.sh index 3811d13989..9d4f285a11 100755 --- a/prepare-nodejspkg.sh +++ b/prepare-nodejspkg.sh @@ -24,5 +24,4 @@ fi echo '>>> Installing JS packages...' yarn install --production=false -yarn install diff --git a/prepare-ruby-dev.sh b/prepare-ruby-dev.sh index 280b54ed73..2aa3c86e05 100755 --- a/prepare-ruby-dev.sh +++ b/prepare-ruby-dev.sh @@ -7,29 +7,69 @@ export ASDF_BRANCH=v0.14.0 echo '>>> checking asdf installation' ./prepare-asdf.sh -# check nodejs version as set in package.json: install if mismatch, and correct .tool-versions' -echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions' +# nodejs installation ./prepare-nodejs.sh # ruby gems installation ./prepare-rubygems.sh -# node packages installation -./prepare-nodejspkg.sh - # prepare rails server rm -f tmp/pids/server.pid -if [ "$( psql -h postgres -U postgres -XtAc "SELECT 1 FROM pg_database WHERE datname='chemotion_dev'" )" = '1' ] +# prepare rails database +# assume default database configuration +DATABASE_NAME=${DATABASE_NAME:-chemotion_dev} +DATABASE_USER=${DATABASE_USER:-postgres} +DATABASE_HOST=${DATABASE_HOST:-postgres} +DATABASE_PORT=${DATABASE_PORT:-5432} + +# check if yq is installed +# if yq is installed parse config/database.yml file for the actual values +# if yq is not installed, then keep the set values default values +if command -v yq &> /dev/null then + DATABASE_NAME=$(yq -r .development.database config/database.yml) + DATABASE_USER=$(yq -r .development.username config/database.yml) + DATABASE_HOST=$(yq -r .development.host config/database.yml) + DATABASE_PORT=$(yq -r .development.port config/database.yml) +fi +echo "DATABASE_NAME: $DATABASE_NAME" +echo "DATABASE_USER: $DATABASE_USER" +echo "DATABASE_HOST: $DATABASE_HOST" +echo "DATABASE_PORT: $DATABASE_PORT" + +# check if the database for the given environment configuration exists +db_exists=$( psql -h $DATABASE_HOST -U $DATABASE_USER -p $DATABASE_PORT -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" ) +echo "Database exists: $db_exists" +if [ "$db_exists" = '1' ] +then + echo "===================================================" + echo "Database already exists, skipping Database creation" + echo "===================================================" + if [ "$RAKE_DB_MIGRATE" = "always" ] + then echo "================================================" - echo "Database already exists, skipping Database setup" + echo "Running 'rake db:migrate'" echo "================================================" + bundle exec rake db:migrate + fi else + # if RAKE_DB_MIGRATE is set to always or once, run rake db:setup + if [ "$RAKE_DB_MIGRATE" = "always" ] || [ "$RAKE_DB_MIGRATE" = "once" ] + then + echo "================================================" + echo "Database does not exist" + echo "running 'rake db:create/migrate/seed'" + echo "================================================" + bundle exec rake db:create + bundle exec rake db:migrate + bundle exec rake db:seed + else echo "================================================" echo "Database does not exist, running 'rake db:setup'" echo "================================================" bundle exec rake db:setup + fi fi diff --git a/run-js-dev.sh b/run-js-dev.sh index a7d4005bf7..e39499256f 100755 --- a/run-js-dev.sh +++ b/run-js-dev.sh @@ -1,14 +1,8 @@ #!/bin/bash -if command -v yarn; then - echo '>>> yarn is installed -> continue' -else - echo '>>> Missing yarn. Installing...' - npm install -g yarn -fi - -echo '>>> Installing JS packages...' -yarn install +## check yarn installation and install nodejs packages +## assume nodejs is installed (through ./run-ruby-dev.sh) +./prepare-nodejspkg.sh echo "==========================================================================================================" echo "THIS WILL FAIL UNTIL THE RUBY GEMS ARE INSTALLED BY run-ruby-dev.sh. JUST TRY AGAIN AFTER INSTALLING THEM."