Consolidate legacy and docker jetty deployments which required upgrad…

…ing legacy dependencies to jetty version 9.4.6.v20170531 and switching to start.d module configurations. Move server configuration files from logs directory to resources directory. Add render-server.properties file to store default view, maxTileSpecsToRender, and maxImageProcessorCacheGb values for web server. This fixes saalfeldlab#81 and saalfeldlab#78.  Expose render-server properties, logging properties, and database connection properties (including connection string) as docker environment variables so they can be easily changed for different docker containers.  Consolidate  common components between docker build and non-container install.sh to reuse as much as possible.  Update docker instructions.

.dockerignore

@@ -1,5 +1,6 @@
 deploy
 htmlcov
+**/site
 test-reports
 Dockerfile
 **/target

Dockerfile

@@ -1,98 +1,88 @@
+# ======================================================================================
+# Stage 0: builder
+
 FROM openjdk:8-jdk as builder
-MAINTAINER Forrest Collman ([email protected])
+LABEL maintainer="Forrest Collman <[email protected]>, Eric Trautman <[email protected]>"
 
 RUN apt-get update && apt-get install -y maven
 
+# ---------------------------------
+# Install library dependencies before actually building source.
+# This caches libraries into an image layer that can be reused when only source code has changed.
+
 WORKDIR /var/www/render/
-ADD pom.xml .
-ADD render-ws/pom.xml render-ws/pom.xml
-ADD render-ws-java-client/pom.xml render-ws-java-client/pom.xml
-ADD render-ws-spark-client/pom.xml render-ws-spark-client/pom.xml
-ADD render-app/pom.xml render-app/pom.xml
-ADD trakem2-scripts/pom.xml trakem2-scripts/pom.xml
-ADD docs/pom.xml docs/pom.xml
-RUN mvn verify clean --fail-never
+COPY pom.xml .
+COPY render-ws/pom.xml render-ws/pom.xml
+COPY render-ws-java-client/pom.xml render-ws-java-client/pom.xml
+COPY render-ws-spark-client/pom.xml render-ws-spark-client/pom.xml
+COPY render-app/pom.xml render-app/pom.xml
+COPY trakem2-scripts/pom.xml trakem2-scripts/pom.xml
+COPY docs/pom.xml docs/pom.xml
+RUN mvn -T 1C verify clean --fail-never
+
+# ---------------------------------
+# Build the source code, save resulting jar and war files, and remove everything else
+
 COPY . /var/www/render/
 RUN mvn clean
-RUN mvn -Dproject.build.sourceEncoding=UTF-8 package && \
- rm -rf /tmp/* && \
- rm -rf render-ws/target/test-classes && \
- rm -rf render-app/target/test-classes && \
- rm -rf render-ws/target/test-classes && \
- rm -rf render-ws-java-client/target/test-classes && \
- rm -rf render-ws-spark-client/target/test-classes && \
- rm -rf /root/.embedmongo
-
-FROM jetty:9.4.6-jre8-alpine as render-ws
-# see https://github.com/saalfeldlab/render/blob/master/docs/src/site/markdown/render-ws.md
-
-#RUN apt-get update
-ENV MONGO_HOST="localhost"
-ENV MONGO_PORT="27107"
-ENV LOGBACK_VERSION="1.1.5"
-ENV SLF4J_VERSION="1.7.16"
-ENV SWAGGER_UI_VERSION="2.1.4"
-ENV SLF4J_URL="https://www.slf4j.org/dist/slf4j-${SLF4J_VERSION}.tar.gz"
-ENV LOGBACK_URL="https://logback.qos.ch/dist/logback-${LOGBACK_VERSION}.tar.gz"
-ENV SWAGGER_UI_URL="https://github.com/swagger-api/swagger-ui/archive/v${SWAGGER_UI_VERSION}.tar.gz"
-#RUN echo y | java -jar "$JETTY_HOME/start.jar" --add-to-startd=slf4j-logback
-#RUN echo y | java -jar "$JETTY_HOME/start.jar" --add-to-startd=logging-slf4j 
-WORKDIR /root
-ENV JETTY_LIB_LOGGING="${JETTY_BASE}/lib/ext"
-RUN mkdir -p $JETTY_LIB_LOGGING
-ENV SWAGGER_UI_DEPLOY_DIR="${JETTY_BASE}/webapps/swagger-ui"
-RUN apk add --update curl && \
-    rm -rf /var/cache/apk/*
-RUN  curl ${SLF4J_URL} | tar xz && \
- curl ${LOGBACK_URL} | tar xz && \
- curl -L ${SWAGGER_UI_URL} | tar xz && \
- cp logback-${LOGBACK_VERSION}/logback-access-${LOGBACK_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp logback-${LOGBACK_VERSION}/logback-classic-${LOGBACK_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp logback-${LOGBACK_VERSION}/logback-core-${LOGBACK_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp slf4j-${SLF4J_VERSION}/jcl-over-slf4j-${SLF4J_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp slf4j-${SLF4J_VERSION}/jul-to-slf4j-${SLF4J_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp slf4j-${SLF4J_VERSION}/log4j-over-slf4j-${SLF4J_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp slf4j-${SLF4J_VERSION}/slf4j-api-${SLF4J_VERSION}.jar ${JETTY_LIB_LOGGING} &&\
- cp -r swagger-ui-${SWAGGER_UI_VERSION}/dist ${SWAGGER_UI_DEPLOY_DIR} &&\
- rm -rf slf4j-${SLF4J_VERSION} &&\
- rm -rf logback-${LOGBACK_VERSION} &&\
- rm -rf v${SWAGGER_UI_VERSION} 
-
-COPY render-ws/src/main/scripts/fix_swagger.sh ${SWAGGER_UI_DEPLOY_DIR}
-RUN cd ${SWAGGER_UI_DEPLOY_DIR} && ./fix_swagger.sh
-EXPOSE 8080
 
-#ENV PATH $JAVA_HOME/bin:$PATH
-#ENV JAVA $JAVA_HOME/bin/java
+# use -T 1C maven option to multi-thread process
+RUN mvn -T 1C -Dproject.build.sourceEncoding=UTF-8 package && \
+    mkdir -p /root/render-lib && \
+    mv */target/*.*ar /root/render-lib && \
+    printf "\nsaved the following build artifacts:\n\n" && \
+    ls -alh /root/render-lib/* && \
+    printf "\nremoving everything else ...\n\n" && \
+    rm -rf /var/www/render/* && \
+    rm -rf /root/.m2 && \
+    rm -rf /root/.embedmongo
 
-# small 4GB server:
+# ======================================================================================
+# Stage 1: render-ws
 
-ENV JAVA_OPTIONS="-Xms3g -Xmx3g -server -Djava.awt.headless=true"
+# NOTE: jetty version should be kept in sync with values in render/render-ws/pom.xml and render/render-ws/src/main/scripts/install.sh
+FROM jetty:9.4.6-jre8-alpine as render-ws
 
-# larger 16GB server
-#ENV JAVA_OPTIONS="-Xms5g -Xmx5g -server -Djava.awt.headless=true"
+RUN apk add --update curl && \
+    rm -rf /var/cache/apk/*
 
-# super 500GB server
-#ENV JAVA_OPTIONS="-Xms400g -Xmx400g -server -Djava.awt.headless=true"
+WORKDIR $JETTY_BASE
 
-# setup supervisor
-#RUN mkdir -p /var/log/supervisor
+COPY render-ws/src/main/scripts/jetty/ .
+RUN ls -al $JETTY_BASE/* && \
+    chmod 755 ./configure_web_server.sh && \
+    ./configure_web_server.sh
 
-#COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-#CMD ["/usr/bin/supervisord"]
+COPY --from=builder /root/render-lib/render-ws-*.war webapps/render-ws.war
+COPY render-ws/src/main/scripts/docker/render-config-entrypoint.sh /render-config-entrypoint.sh
+RUN chown -R jetty:jetty $JETTY_BASE 
 
-# clone render repo
-WORKDIR $JETTY_BASE
-RUN echo --module=rewrite>>$JETTY_BASE/start.d/start.ini
-RUN mkdir -p $JETTY_BASE/logs && chown -R jetty:jetty $JETTY_BASE/logs
+EXPOSE 8080
 
-COPY render-ws/docker_jetty/ .
-COPY render-ws/docker_jetty/lib/logging/*.jar $JETTY_LIB_LOGGING
-COPY render-ws/docker_jetty/resources/root-context.xml $JETTY_BASE/webapps/root-context.xml
+ENV JAVA_OPTIONS="-Xms3g -Xmx3g -server -Djava.awt.headless=true" \
+    LOG_ACCESS_ROOT_APPENDER="STDOUT" \
+    LOG_JETTY_ROOT_APPENDER="STDOUT" \
+    LOG_JETTY_ROOT_LEVEL="WARN" \
+    LOG_JETTY_JANELIA_LEVEL="WARN" \
+    MONGO_HOST="" \
+    MONGO_PORT="" \
+    MONGO_USERNAME="" \
+    MONGO_PASSWORD="" \
+    MONGO_AUTH_DB="" \
+    MONGO_CONNECTION_STRING="" \
+    MONGO_CONNECTION_STRING_USES_AUTH="" \
+    NDVIZHOST="" \
+    NDVIZPORT="" \
+    NDVIZ_URL="" \
+    VIEW_CATMAID_HOST_AND_PORT="" \
+    VIEW_DYNAMIC_RENDER_HOST_AND_PORT="" \
+    VIEW_RENDER_STACK_OWNER="" \
+    VIEW_RENDER_STACK_PROJECT="" \
+    VIEW_RENDER_STACK="" \
+    VIEW_MATCH_OWNER="" \
+    VIEW_MATCH_COLLECTION="" \
+    WEB_SERVICE_MAX_TILE_SPECS_TO_RENDER="20" \
+    WEB_SERVICE_MAX_IMAGE_PROCESSOR_GB=""
 
-COPY --from=0 /var/www/render/render-ws/target/render-ws-*.war webapps/render-ws.war
-#RUN echo y | java -jar "$JETTY_HOME/start.jar" --add-to-startd=logging-log4j 
-COPY render-ws/render-config-entrypoint.sh /render-config-entrypoint.sh
-RUN chown -R jetty:jetty $JETTY_BASE 
 USER jetty
 ENTRYPOINT ["/render-config-entrypoint.sh"]

README.md

@@ -8,7 +8,7 @@ A collection of Java tools and HTTP services (APIs) for rendering transformed im
     persisted collections of tile and transform specifications, and 
   - a set of [Java clients] for processing/rendering data that utilize the [render web services] 
     to retrieve and/or store persisted specifications.
-  - a [render-ws docker] for building a jetty based web-service  
+  - [Docker packaging] for building the libraries and deploying the [render web services]  
 
   ![Render Components Diagram](docs/src/site/resources/image/render-components.png)
 
@@ -18,4 +18,4 @@ A collection of Java tools and HTTP services (APIs) for rendering transformed im
   [render application]: <docs/src/site/markdown/render-app.md>
   [render web services]: <docs/src/site/markdown/render-ws.md>
   [RESTful APIs]: <docs/src/site/markdown/render-ws-api/render-ws-api.md>
-  [render-ws docker]: <docs/src/site/markdown/render-ws-docker.md>
+  [Docker packaging]: <docs/src/site/markdown/render-ws-docker.md>

docs/src/site/markdown/render-ws-docker.md

@@ -1,35 +1,136 @@
 # Render Web Services Docker Instructions
 
+# Building Images
 
+To build the full render-ws image: 
 
-# Build
+```bash
+# cd to root directory of render repo (where Dockerfile is located) 
+docker build -t render-ws:latest --target render-ws .
+```
 
-to build the render-ws, you should do 
+To build just the web service JARs without deploying into a Jetty container:
 
 ```bash
-docker build -t IMAGE_NAME --target render-ws .
+# cd to root directory of render repo (where Dockerfile is located) 
+# saves JAR and WAR files in /root/render-lib 
+docker build -t render-ws:latest-build --target builder .
 ```
 
-to compile the JARs you can target the first builder stage with
+# Running Images
+
+To run the full render-ws image:
 
 ```bash
-docker build -t IMAGE_NAME --target builder .
+# with database running on same host at default port
+docker run -p 8080:8080 -e "MONGO_HOST=localhost" render-ws:latest
+
+# with customized environment variables in an file
+docker run -p 8080:8080 --env-file ./env.janelia.mongo render-ws:latest
 ```
 
-# Running render-ws
-The image paramaterizing many of the configuration variables via runtime environment variables that should be set via docker run or docker-compose
+You must explicitly specify either the MONGO_HOST or MONGO_CONNECTION_STRING environment variables
+when running a container based upon the full render-ws image.  
+
+The full render-ws image supports the following set of environment variables (specified values are defaults):
+
+### Environment Variables 
+
+```bash
+# ---------------------------------
+# database connection parameters 
+
+# if a connection string is specified, other mongo connection variables are ignored
+# format details are here: https://docs.mongodb.com/manual/reference/connection-string
+MONGO_CONNECTION_STRING=""  
+
+# should be 'y' if you are using a connection string that includes username and password 
+MONGO_CONNECTION_STRING_USES_AUTH="n"
+
+MONGO_HOST=""
+MONGO_PORT=""
+
+# if authentication is not needed, leave these empty
+MONGO_USERNAME=""                            
+MONGO_PASSWORD=""
+MONGO_AUTH_DB=""
+
+# ---------------------------------
+# web service JVM parameters
+
+JAVA_OPTIONS="-Xms3g -Xmx3g -server -Djava.awt.headless=true"
+
+# ---------------------------------
+# web service logging parameters
+
+# appender options are 'STDOUT', 'FILE', or 'NONE'
+LOG_ACCESS_ROOT_APPENDER="STDOUT"
+LOG_JETTY_ROOT_APPENDER="STDOUT"
+
+# log level options are: 'OFF', 'ERROR', 'WARN', 'INFO', 'DEBUG', or 'TRACE'
+LOG_JETTY_ROOT_LEVEL="WARN" 
+LOG_JETTY_JANELIA_LEVEL="WARN" 
+
+# ---------------------------------
+# web service rendering parameters
+
+# use this to improve dynamic rendering speed for zoomed-out views
+# views that contain more than this number of tiles will have bounding boxes rendered instead of actual tile content 
+WEB_SERVICE_MAX_TILE_SPECS_TO_RENDER="20"          
+
+# if left empty, the image processor cache will be sized at half of the memory allocated to the JVM
+WEB_SERVICE_MAX_IMAGE_PROCESSOR_GB="" 
+
+# ---------------------------------
+# viewing tools parameters
+
+NDVIZHOST=""                                
+NDVIZPORT=""
 
-Here are the present environment variables and their default values
+# use the url parameter if you need https (overrides host and port parameters)
+NDVIZ_URL=""
 
-* MONGO_HOST=localhost
-* MONGO_PORT=27017
-* MONGO_USERNAME= (default is no authentication)
-* MONGO_PASSWORD= (default is no authenticaiton)
-* NDVIZHOST= (default blank)
-* NDVIZPORT= (default blank)
-* NDVIZ_URL= (default blank, use either NDVIZHOST:NDVIZPORT or NDVIZ_URL if you want to specify https)
-* JAVA_OPTIONS="-Xms3g -Xmx3g -server -Djava.awt.headless=true"
+# specify without protocol (assumes http) like: 'renderer-catmaid:8080'
+VIEW_CATMAID_HOST_AND_PORT=""                
+VIEW_DYNAMIC_RENDER_HOST_AND_PORT=""
 
+VIEW_RENDER_STACK_OWNER=""
+VIEW_RENDER_STACK_PROJECT=""
+VIEW_RENDER_STACK=""
 
+VIEW_MATCH_OWNER=""
+VIEW_MATCH_COLLECTION=""
+```
+
+### Entrypoint
+
+The default render-ws image entrypoint script is here:
+[render-config-entrypoint.sh](../../../../render-ws/src/main/scripts/docker/render-config-entrypoint.sh)
+
+# Other Useful Docker Commands
+
+To open a Bourne shell on the latest running render-ws container: 
+
+```bash
+docker exec -it $(docker ps --latest --quiet) /bin/sh
+```
+
+To open a Bourne shell on a new render-ws container without running jetty: 
+
+```bash
+docker run -it --entrypoint /bin/sh render-ws:latest
+```
+
+To remove all stopped containers: 
+
+```bash
+docker rm $(docker ps --all --quiet)
+```
+
+To remove all untagged \(\<none\>\) images \(including the large build artifacts image if you did not tag it\): 
+
+```bash
+docker image rm $(docker images --filter "dangling=true" -q --no-trunc)
+```
 
 

docs/src/site/markdown/render-ws.md

@@ -14,7 +14,9 @@ machines that are tuned for their specific purpose.
 
 ## Basic Installation
 These installation instructions cover setup of an evaluation instance with a collocated web server and 
-database running on Ubuntu 16.04.  
+database running on Ubuntu 16.04.
+
+> Note: In lieu of running these steps manually, you may prefer to use a [Docker deployment] instead. 
 
 ### 1. Install Git and Maven
 ```bash
@@ -105,6 +107,7 @@ small example project.
 
 
   [data model]: <data-model.md>
+  [Docker deployment]: <render-ws-docker.md>
   [Jetty]: <https://eclipse.org/jetty/>
   [level 2 REST APIs]: <http://martinfowler.com/articles/richardsonMaturityModel.html>
   [MongoDB]: <https://www.mongodb.org/>

render-app/src/main/java/org/janelia/alignment/filter/FilterFactory.java

@@ -168,6 +168,6 @@ public static List<Filter> buildInstanceList(final List<FilterSpec> specList) {
             new JsonUtils.Helper<>(FilterFactory.class);
 
     private static File getSystemConfigurationFile() {
-        return new File("logs/filter_lists.json").getAbsoluteFile();
+        return new File("resources/filter_lists.json").getAbsoluteFile();
     }
 }