Bb dev docker dev environment

.gitignore

@@ -8,5 +8,6 @@ docker/.tmp/*
 **node_modules**
 
 .env-secrets.env
+.env-ckan.override.env
 
-**/*build.log
+**/*build.log

Makefile

@@ -26,6 +26,9 @@ ecrLogin:
 build: ecrLogin
 	@docker build ./docker/ckan -t 29_ckan:default --build-arg GH_TOKEN=${GH_TOKEN} --progress plain --no-cache 2>&1 | tee build.log
 
+build-dev: ecrLogin
+	@docker build ./docker/ckan -t 29_ckan:default --build-arg GH_TOKEN=${GH_TOKEN} --build-arg TWDH_BUILD=dev --progress plain --no-cache 2>&1 | tee build.log
+
 tag: build
 	docker tag 29_ckan:default ${ECR_URL}/29_ckan:${TAG}
 
@@ -34,4 +37,4 @@ ecrBuild: tag
 ecrPush:
 	docker push ${ECR_URL}/29_ckan:${TAG}
 
-ecrBuildPush: ecrBuild ecrPush
+ecrBuildPush: ecrBuild ecrPush

docker/ckan/Dockerfile

@@ -80,6 +80,7 @@ RUN pip wheel --wheel-dir=${VIRTUAL_ENV}/src -r ckan/requirements.txt
 FROM ubuntu:focal-20210827
 
 ARG GH_TOKEN=default
+ARG TWDH_BUILD=prod
 
 LABEL maintainer="Texas Geographic Information Office"
 LABEL org.opencontainers.image.source https://github.com/tnris/twdh_ckan_docker
@@ -215,6 +216,9 @@ COPY --from=ckanbuild ${WHEEL_DIR} /srv/app/wheels
 COPY --from=ckanbuild ${CKAN_DIR} ${CKAN_DIR}
 COPY ./plugins ${SRC_DIR}/plugins
 
+# Copy pre-built binaries into place
+COPY bin/qsvdp /usr/local/bin/qsvdp
+
 RUN ls ${SRC_DIR}/plugins
 
 # Additional install steps for build stages artifacts
@@ -241,6 +245,11 @@ RUN ckan generate config ${APP_DIR}/production.ini
 # Install plugins
 RUN python ${SRC_DIR}/plugins/plugins.py
 
+# LOCAL DEV Setup
+RUN if [ "$TWDH_BUILD" = "dev" ]; then pip install flask-debugtoolbar==0.14.1; fi
+RUN if [ "$TWDH_BUILD" = "dev" ]; then apt-get update && apt-get install --no-install-recommends -y inotify-tools; fi
+RUN if [ "$TWDH_BUILD" = "dev" ]; then ckan config-tool ${APP_DIR}/production.ini "debug = true"; fi
+
 # Configure plugins
 RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
     # Create the data directory
@@ -273,19 +282,6 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
 COPY ./supervisord.conf /etc/supervisor/
 COPY ./supervisor-ckan-worker.conf /etc/supervisor/conf.d/
 
-# install Rust, then qsvdp
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
-    # source the cargo env to setup the $PATH for rust
-    . $HOME/.cargo/env && \
-    # install qsvdp from source
-    git clone https://github.com/dathere/qsv.git && \
-    cd qsv && \
-    export CARGO_BUILD_RUSTFLAGS='-C target-cpu=native' && \
-    cargo build --release --locked --bin qsvdp -F datapusher_plus,luau,polars && \
-    # mv the qsvdp binary so that ckan user can run it
-    mv target/release/qsvdp /usr/local/bin/ && \
-    cargo clean
-
 # Remove wheels
 RUN rm -rf /srv/app/wheels
 

docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh → docker/ckan/afterinit.d/00_ckan_db_upgrade.sh

@@ -4,7 +4,7 @@ echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 UPGRADE=0
 
 if [ "$UPGRADE" -ne 1 ]; then
-    echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
+    echo "Warning: CKAN DB UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
 else 
   ckan -c /srv/app/production.ini db upgrade 
   ckan -c /srv/app/production.ini db pending-migrations --apply 

docker/ckan/afterinit.d/02_datatables.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-echo "@@@@@@ INIT SECURITY @@@@@@"
+echo "@@@@@@ INIT CKANEXT-DATATABLESVIEW-PLUS @@@@@@"
 ckan -c /srv/app/production.ini datatablesview-plus migrate

docker/ckan/afterinit.d/03_checklink.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "@@@@@@ INIT CKANEXT-CHECK_LINK @@@@@@"
+ckan -c /srv/app/production.ini db upgrade -p check_link

docker/ckan/afterinit.d/04_datapusherplus.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "@@@@@@ INIT DATAPUSHER_PLUS @@@@@@"
+ckan -c /srv/app/production.ini db upgrade -p datapusher_plus

docker/ckan/afterinit.d/05_pages.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "@@@@@@ INIT CKANEXT-PAGES @@@@@@"
+ckan -c /srv/app/production.ini db upgrade -p pages

docker/ckan/setup/app/start_ckan.sh

@@ -13,7 +13,7 @@ then
 fi
 
 # Set the common uwsgi options
-UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p 2 -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application --single-interpreter --need-app --disable-logging --log-4xx --log-5xx --log-slow 5000"
+UWSGI_OPTS="--socket /tmp/uwsgi.sock --pidfile=/tmp/uwsgi.pid --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p 2 -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application --single-interpreter --need-app --disable-logging --log-4xx --log-5xx --log-slow 5000"
 
 # Run the prerun script to init CKAN and create the default admin user
 python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }

docker/db/docker-entrypoint-initdb.d/02_create_datastore.sql

@@ -1,2 +1,2 @@
-CREATE ROLE datastore WITH LOGIN CREATEDB CREATEROLE REPLICATION SUPERUSER ENCRYPTED PASSWORD 'datastore';
-CREATE DATABASE datastore WITH OWNER datastore;
+CREATE ROLE datastore WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD 'datastore';
+CREATE DATABASE datastore WITH OWNER datastore;

docker/docker-compose_local.yml

@@ -26,23 +26,14 @@ services:
       - "0.0.0.0:5000:5000"
     volumes:
       - ckan_data:/srv/app/data
-
-  datapusher:
-    container_name: datapusher
-    hostname: datapusher
-    image: 746466009731.dkr.ecr.us-east-1.amazonaws.com/datapusher-plus:0.7.0
-    depends_on:
-      - db
-      - solr
-      - ckan
-    networks:
-      - frontend
-      - backend
-    ports:
-      - "8800:8800"
-    env_file:
-      - ./.env-ckan.env
-      - ./.env-datapusher.env
+    develop:
+      watch:
+        - action: sync
+          path: /usr/lib/ckan/default/src/ckanext-twdh_theme
+          target: /srv/app/src/ckan/ckanext-twdh_theme
+        - action: sync
+          path: /usr/lib/ckan/default/src/ckan
+          target: /srv/app/src/ckan
 
   db:
     container_name: db