Ngap dit test deploy

.travis.yml

@@ -12,7 +12,9 @@ services:
 
 branches:
   only:
-    - master
+  - master
+  - main
+  - /^(.*-test-deploy)$/
 
 notifications:
   email:
@@ -57,7 +59,8 @@ env:
 
 before_install:
   - pip3 install awscli
-  - if test "${TRAVIS_PULL_REQUEST}" = "false"; then export DEPLOY_ME="true"; else export DEPLOY_ME="false"; fi
+  - if test "$TRAVIS_PULL_REQUEST" = "false" || [[ "$TRAVIS_PULL_REQUEST_BRANCH" =~ ^(.*-test-deploy)$ ]]; then export DEPLOY_ME="true"; else export DEPLOY_ME="false"; fi
+  - echo "TRAVIS_PULL_REQUEST is $TRAVIS_PULL_REQUEST, DEPLOY_ME is ${DEPLOY_ME}, TRAVIS_PULL_REQUEST_BRANCH is $TRAVIS_PULL_REQUEST_BRANCH"
 
 install:
   - echo "print env:"
@@ -284,7 +287,7 @@ jobs:
         - export TEST_IMAGE=opendap/hyrax:${HYRAX_VERSION}
         - echo "Running Hyrax Regression Test Script on ${TEST_IMAGE}"
         - ./regression_test_script -i "${TEST_IMAGE}"
-        - export DEPLOY_ME="false"
+        - export DEPLOY_ME="true"
 
 before_deploy:
   - export DOCKER_LOGIN=`aws ecr get-login --region us-east-1 | sed "s/-e none //g"`;

hyrax-builds/build_dmrpp/tomcat9-server.xml

@@ -74,13 +74,15 @@
                connectionTimeout="20000"
                redirectPort="8443" / -->
 
+    <!--
     <Connector port="8080" protocol="HTTP/1.1"
                connectionTimeout="20000"
                redirectPort="8443"
                compression="force"
                compressionMinSize="2048"
                compressableMimeType="text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/octet-stream,application/vnd.opendap.dap4.dataset-services+xml,application/vnd.opendap.dap4.dataset-metadata+xml,application/vnd.opendap.dap4.data,application/vnd.opendap.dap4.error+xml,application/json,application/prs.coverage+json,application/rdf+xml,application/x-netcdf,image/tiff;application=geotiff"
     />
+    -->
 
     <!-- A "Connector" using the shared thread pool-->
     <!--
@@ -111,18 +113,17 @@
          Either JSSE or OpenSSL style configuration may be used. OpenSSL style
          configuration is used below.
     -->
-    <!--
+
     <Connector port="8443" protocol="org.apache.coyote.http11.Http11AprProtocol"
                maxThreads="150" SSLEnabled="true" >
-        <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
-        <SSLHostConfig>
-            <Certificate certificateKeyFile="conf/localhost-rsa-key.pem"
-                         certificateFile="conf/localhost-rsa-cert.pem"
-                         certificateChainFile="conf/localhost-rsa-chain.pem"
-                         type="RSA" />
-        </SSLHostConfig>
+      <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
+      <SSLHostConfig>
+        <Certificate certificateKeyFile="conf/NGAP-CA-certificate.key"
+                     certificateFile="conf/NGAP-CA-certificate.crt"
+                     certificateChainFile="conf/NGAP-CA-certificate-chain.crt"
+                     type="RSA" />
+      </SSLHostConfig>
     </Connector>
-    -->
 
     <!-- Define an AJP 1.3 Connector on port 8009 -->
     <!--

hyrax-builds/ngap/Dockerfile

@@ -101,9 +101,15 @@ RUN set -e && dnf install -y make curl unzip which autoconf automake emacs vim
 RUN set -e && dnf install -y diffutils
 RUN set -e && dnf install -y procps
 RUN set -e && dnf install -y python3 git bc valgrind gdb
+RUN set -e && dnf install -y java-11-openjdk
 RUN set -e && dnf install -y java-11-openjdk-devel
 # Temporary Hack to fix broken OpenJDK-11.0.20.0.8-2
 RUN set -e && dnf install -y tzdata-java
+# Temporary Hack to add missing Apache Portable Runtime library for Tomcat
+RUN set -e && dnf install -y ant
+RUN set -e && dnf install -y redhat-rpm-config
+RUN set -e && dnf install -y apr-devel
+RUN set -e && dnf install -y openssl-devel
 #
 RUN set -e && dnf clean all
 #
@@ -166,6 +172,24 @@ RUN set -e \
     && echo "Cleaning up Tomcat distribution files..." >&2 \
     && rm -fv "/${TOMCAT_DISTRO}.tar.gz"  >&2
 
+# Install the Tomcat Native-APR library.
+#
+RUN set -e \
+    && export JAVA_HOME="$(dirname $(dirname $(readlink -f $(which javac))))" >&2 \
+    && echo $JAVA_HOME >&2 \
+    && tar -xvf "/${CATALINA_HOME}/bin/tomcat-native.tar.gz" -C ${CATALINA_HOME}/bin >&2 \
+    && cd ${CATALINA_HOME}/bin/tomcat-native-1.3.0-src/native >&2 \
+    && ./configure >&2 \
+    && make >&2 \
+    && make install >&2 \
+    && echo "SUCCESS: Tomcat Native-APR Library installed"
+
+ENV CATALINA_OPTS "-Djava.library.path=/usr/local/apr/lib"
+
+# Uninstall Java-1.8.0 installed by Ant
+RUN set -e && yum remove -y java-1.8.0-openjdk >&2
+RUN set -e && which java && java -version
+
 ################################################################
 # Retrieve, verify, and install the NGAP/OLFS web application
 #  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -

hyrax-builds/ngap/beslog2json.awk

@@ -2,6 +2,7 @@
 # Translates OPeNDAP bes.logs into JSON
 #
 BEGIN {
+
     FS="\\|&\\|";
 
     if(debug!="true"){
@@ -29,13 +30,13 @@ BEGIN {
     # Log line base.
 
     # The time that the log message was written.
-    log_line_base[1]="time";
+    log_line_base[1]="hyrax-time";
 
     # The PID of the beslistener that wrote the log entry.
-    log_line_base[2]="pid";
+    log_line_base[2]="hyrax-pid";
 
     # The message associate with the log entry
-    log_line_base[3]="log_name";
+    log_line_base[3]="hyrax-log_name";
 
 
 
@@ -46,96 +47,66 @@ BEGIN {
 # 1601646465|&|2122|&|request|&|OLFS|&|0:0:0:0:0:0:0:1|&|USER_AGENT|&|92F3C71F959B56515C98A09088CA2A8E|&|-|&|1601646465304|&|18|&|HTTP-GET|&|/opendap/hyrax/data/nc/fnoc1.nc.dds|&|u|&|BES|&|get.dds|&|dap2|&|/Users/ndp/OPeNDAP/hyrax/build/share/hyrax/data/nc/fnoc1.nc|&|u
 
     # OLFS Tag.
-    request_log_fields[4]="OLFS";
+    request_log_fields[4]="hyrax-OLFS";
 
     # ip-address of requesting client's system.
-    request_log_fields[5]="client_ip";
+    request_log_fields[5]="hyrax-client_ip";
 
     # The value of the User-Agent request header sent from the client.
-    request_log_fields[6]="user_agent";
+    request_log_fields[6]="hyrax-user_agent";
 
     # The session id, if present.
-    request_log_fields[7]="session_id";
+    request_log_fields[7]="hyrax-session_id";
 
     # The user's user id, if a user is logged in.
-    request_log_fields[8]="user_id";
+    request_log_fields[8]="hyrax-user_id";
 
     # The time the the request was received.
-    request_log_fields[9]="start_time";
+    request_log_fields[9]="hyrax-start_time";
 
     # We are not so sure what this number is...
-    request_log_fields[10]="duration";
+    request_log_fields[10]="hyrax-duration";
 
     # The HTTP verb of the request (GET, POST, etc)
-    request_log_fields[11]="http_verb";
+    request_log_fields[11]="hyrax-http_verb";
 
     # The path component of the requested resource.
-    request_log_fields[12]="url_path";
+    request_log_fields[12]="hyrax-url_path";
 
     # The query string, if any, submitted with the request.
-    request_log_fields[13]="query_string";
+    request_log_fields[13]="hyrax-query_string";
 
     # Field 13 is a field that indicates the following fields orginated
     # in the BES, it is not semantically important to NGAP
-    request_log_fields[14]="bes";
+    request_log_fields[14]="hyrax-bes";
 
     # The type of BES action/request/command invoked by the request
-    request_log_fields[15]="bes_request";
+    request_log_fields[15]="hyrax-bes_request";
 
     # The DAP protocl
-    request_log_fields[16]="dap_version";
+    request_log_fields[16]="hyrax-dap_version";
 
     # The local file path to the resource.
-    request_log_fields[17]="local_path";
+    request_log_fields[17]="hyrax-local_path";
 
     # Field 18 is a duplicate of field 13 and if the query string is absent
     # then field 18 will be missing entirely.
-    request_log_fields[18]="constraint_expression";
+    request_log_fields[18]="hyrax-constraint_expression";
 
     # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
     # Error/Verbose/Info Fields
 
     # The message associate with the log entry
-    msg_fields[4]="message";
-
+    msg_fields[4]="hyrax-message";
 
     # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
     # Start Timing Fields
     # 1601642669|&|2122|&|timing|&|start_us|&|1601642669943035|&|ReqId|&|TIMER_NAME
 
-    # The message associate with the log entry
-    start_time_fields[4]="start_us";
-
-    # The message associate with the log entry
-    start_time_fields[5]="start_time_us";
-
-    # The message associate with the log entry
-    start_time_fields[6]="ReqId";
-
-    # The message associate with the log entry
-    start_time_fields[7]="Name";
-
     # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
     # Stop Timing Fields
     # 1601642669|&|2122|&|timing|&|stop_us|&|1601642669944587|&|elapsed_us|&|1552|&ReqId|&|TIMER_NAME
     #     1          2      3         4            5               6           7     8        9
-    # The message associate with the log entry
-    stop_time_fields[4]="stop_us";
-
-    # The message associate with the log entry
-    stop_time_fields[5]="stop_time_us";
-
-    # The message associate with the log entry
-    stop_time_fields[6]="elapsed_us";
-
-    # The message associate with the log entry
-    stop_time_fields[6]="elapsed_time_us";
-
-    # The message associate with the log entry
-    stop_time_fields[7]="ReqId";
-
-    # The message associate with the log entry
-    stop_time_fields[4]="Name";
 
 
 }
@@ -152,9 +123,9 @@ BEGIN {
     pid=$2;
     log_name=$3;
     printf("{%s", n);
-    printf("%s\"time\": %s, %s", indent, ltime, n);
-    printf("%s\"pid\": \"%s\", %s", indent, pid, n);
-    printf("%s\"type\": \"%s\"", indent, log_name);
+    printf("%s\"hyrax-time\": %s, %s", indent, ltime, n);
+    printf("%s\"hyrax-pid\": \"%s\", %s", indent, pid, n);
+    printf("%s\"hyrax-type\": \"%s\"", indent, log_name);
 
     if(log_name=="request"){
         for(i=4; i<=NF  ; i++){
@@ -180,18 +151,18 @@ BEGIN {
         if(time_type=="start_us"){
             # 1601642669|&|2122|&|timing|&|start_us|&|1601642669945133|&|-|&|TIMER_NAME
             #      1         2      3        4             5             6      7
-            printf(", %s%s\"%s\": %s", n, indent, "start_time_us", $5);
-            printf(", %s%s\"%s\": \"%s\"", n, indent, "req_id", $6);
-            printf(", %s%s\"%s\": \"%s\"", n, indent, "name:", $7);
+            printf(", %s%s\"%s\": %s", n, indent, "hyrax-start_time_us", $5);
+            printf(", %s%s\"%s\": \"%s\"", n, indent, "hyrax-req_id", $6);
+            printf(", %s%s\"%s\": \"%s\"", n, indent, "hyrax-name:", $7);
         }
         else if(time_type=="elapsed_us"){
             #1601653546|&|7096|&|timing|&|elapsed_us|&|2169|&|start_us|&|1601653546269617|&|stop_us|&|1601653546271786|&|ReqId|&|TIMER_NAME
             #     1          2      3         4          5        6            7                8           9              10       11
-            printf(", %s%s\"%s\": %s", n, indent, "elapsed_time_us", $5);
-            printf(", %s%s\"%s\": %s", n, indent, "start_time_us", $7);
-            printf(", %s%s\"%s\": %s", n, indent, "stop_time_us", $9);
-            printf(", %s%s\"%s\": \"%s\"", n, indent, "req_id", $10);
-            printf(", %s%s\"%s\": \"%s\"", n, indent,  "name:", $11);
+            printf(", %s%s\"%s\": %s", n, indent, "hyrax-elapsed_time_us", $5);
+            printf(", %s%s\"%s\": %s", n, indent, "hyrax-start_time_us", $7);
+            printf(", %s%s\"%s\": %s", n, indent, "hyrax-stop_time_us", $9);
+            printf(", %s%s\"%s\": \"%s\"", n, indent, "hyrax-req_id", $10);
+            printf(", %s%s\"%s\": \"%s\"", n, indent,  "hyrax-name:", $11);
 
         }
         else {

hyrax-builds/ngap/entrypoint.sh

@@ -62,6 +62,17 @@ echo "#     AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" >&2
 export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-"<not set>"}
 echo "#    AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION}" >&2
 
+################################################################################
+echo "${HR2}" >&2
+export NGAP_CERTIFICATE_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate.crt"
+echo "#   NGAP_CERTIFICATE_FILE: ${NGAP_CERTIFICATE_FILE}" >&2
+
+export NGAP_CERTIFICATE_CHAIN_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate-chain.crt"
+echo "#   NGAP_CERTIFICATE_CHAIN_FILE: ${NGAP_CERTIFICATE_CHAIN_FILE}" >&2
+
+export NGAP_CERTIFICATE_KEY_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate.key"
+echo "#   NGAP_CERTIFICATE_KEY: ${NGAP_CERTIFICATE_KEY_FILE}" >&2
+
 ################################################################################
 echo "${HR2}" >&2
 export NETRC_FILE="/etc/bes/ngap_netrc"
@@ -173,6 +184,48 @@ fi
 ################################################################################
 
 
+################################################################################
+# Inject Tomcat's NGAP[CA] certificate document to configure the Tomcat to
+# utilize SSL/TLS Data-In-Transit in the NGAP environment.
+#
+# Test if the bes.conf env variable is set (by way of not unset) and not empty
+if test -n "${NGAP_CERTIFICATE}" ; then
+    echo "${HR2}" >&2
+    echo "# Tomcat  file: ${NGAP_CERTIFICATE_FILE}" >&2
+    echo "${NGAP_CERTIFICATE}" > ${NGAP_CERTIFICATE_FILE}
+    cat "${NGAP_CERTIFICATE_FILE}" | awk '{print "##    "$0;}' >&2
+    echo "#" >&2
+fi
+################################################################################
+
+################################################################################
+# Inject Tomcat's NGAP[CA] certificate-chain to configure the Tomcat to
+# utilize SSL/TLS Data-In-Transit in the NGAP environment.
+#
+# Test if the bes.conf env variable is set (by way of not unset) and not empty
+if test -n "${NGAP_CERTIFICATE_CHAIN}" ; then
+    echo "${HR2}" >&2
+    echo "# Tomcat  file: ${NGAP_CREDENTIALS_CHAIN_FILE}" >&2
+    echo "${NGAP_CERTIFICATE_CHAIN}" > ${NGAP_CERTIFICATE_CHAIN_FILE}
+    cat "${NGAP_CERTIFICATE_CHAIN_FILE}" | awk '{print "##    "$0;}' >&2
+    echo "#" >&2
+fi
+################################################################################
+
+################################################################################
+# Inject Tomcat's NGAP[CA] certificate key to configure the Tomcat to
+# utilize SSL/TLS Data-In-Transit in the NGAP environment.
+#
+# Test if the bes.conf env variable is set (by way of not unset) and not empty
+if test -n "${NGAP_CERTIFICATE_KEY}" ; then
+    echo "${HR2}" >&2
+    echo "# Tomcat  file: ${NGAP_CERTIFICATE_KEY_FILE}" >&2
+    echo "${NGAP_CERTIFICATE_KEY}" > ${NGAP_CERTIFICATE_KEY_FILE}
+    cat "${NGAP_CERTIFICATE_KEY_FILE}" | awk '{print "##    "$0;}' >&2
+    echo "#" >&2
+fi
+################################################################################
+
 ################################################################################
 # Inject an NGAP Cumulus Configuration
 # Only amend the /etc/bes/site.conf file if all the necessary environment

hyrax-builds/ngap/tomcat9-server.xml

@@ -74,13 +74,15 @@
                connectionTimeout="20000"
                redirectPort="8443" / -->
 
+    <!--
     <Connector port="8080" protocol="HTTP/1.1"
                connectionTimeout="1000000"
                redirectPort="8443"
                compression="force"
                compressionMinSize="2048"
                compressableMimeType="text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/octet-stream,application/vnd.opendap.dap4.dataset-services+xml,application/vnd.opendap.dap4.dataset-metadata+xml,application/vnd.opendap.dap4.data,application/vnd.opendap.dap4.error+xml,application/json,application/prs.coverage+json,application/rdf+xml,application/x-netcdf,image/tiff;application=geotiff"
     />
+    -->
 
     <!-- A "Connector" using the shared thread pool-->
     <!--
@@ -111,18 +113,17 @@
          Either JSSE or OpenSSL style configuration may be used. OpenSSL style
          configuration is used below.
     -->
-    <!--
+
     <Connector port="8443" protocol="org.apache.coyote.http11.Http11AprProtocol"
                maxThreads="150" SSLEnabled="true" >
         <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
         <SSLHostConfig>
-            <Certificate certificateKeyFile="conf/localhost-rsa-key.pem"
-                         certificateFile="conf/localhost-rsa-cert.pem"
-                         certificateChainFile="conf/localhost-rsa-chain.pem"
+            <Certificate certificateKeyFile="/usr/share/tomcat/conf/NGAP-CA-certificate.key"
+                         certificateFile="/usr/share/tomcat/conf/NGAP-CA-certificate.crt"
+                         certificateChainFile="/usr/share/tomcat/conf/NGAP-CA-certificate-chain.crt"
                          type="RSA" />
         </SSLHostConfig>
     </Connector>
-    -->
 
     <!-- Define an AJP 1.3 Connector on port 8009 -->
     <!-- Connector port="8009" protocol="AJP/1.3" redirectPort="8443" / -->