#379 - adding ability to not fail batches and keep track of what failed.

fixed some things on search view

.dockerignore

@@ -0,0 +1,13 @@
+.dockerignore
+
+.DS_Store
+
+**/node_modules
+.gradle
+**/.gradle
+!**/build.gradle
+!**/settings.gradle
+
+.idea
+
+# **/build/*

.gitignore

@@ -5,8 +5,9 @@
 .idea/
 .gradletasknamecache
 npm-debug.log
-*/bin/
+marklogic-data-hub/bin/
 marklogic-data-hub/src/main/resources/ml-modules/root/trace-ui
+quick-start/bin/
 build/
 releases/
 .classpath

.travis.yml

@@ -1,21 +1,11 @@
-language: java
-jdk:
-  - oraclejdk8
-env:
- - NODE_VERSION=6.1
-sudo: true
+dist: trusty
+sudo: required
+services:
+  - docker
+
 before_install:
-  - echo 'America/Los_Angeles' | sudo tee /etc/timezone
-  - sudo dpkg-reconfigure --frontend noninteractive tzdata
-install:
-  - nvm install $NODE_VERSION
-  - nvm use $NODE_VERSION
-  - npm install -g typings
-  - ./shared/dev-tasks/install-dependencies.sh
+ - ./.travis/download.sh
+
 script:
-  - nvm use $NODE_VERSION
-  - if [[ $TRAVIS_SECURE_ENV_VARS == true ]]; then
-      ./gradlew check;
-    else
-      ./gradlew jshint;
-    fi
+  - docker-compose build
+  - docker-compose run marklogic

.travis/Dockerfile-java

@@ -0,0 +1,5 @@
+FROM openjdk:8
+VOLUME /code
+ADD ./ /code
+WORKDIR /code
+ENTRYPOINT [ "sh", "-c", "./gradlew check" ]

.travis/Dockerfile-ml

@@ -0,0 +1,35 @@
+FROM centos:centos7
+
+ENV LANG C.UTF-8
+
+RUN yum -y install glibc.i686 \
+    gdb.x86_64 redhat-lsb.x86_64 vim \
+    bzip2 \
+    unzip \
+    xz-utils \
+    java-1.8.0-openjdk-devel 2>&1 > /dev/null
+
+# Install MarkLogic
+COPY ./MarkLogic.rpm /tmp/MarkLogic.rpm
+
+RUN yum -y install /tmp/MarkLogic.rpm 2>&1 > /dev/null
+
+# Expose MarkLogic Server ports - add additional ones for your REST, etc
+# endpoints
+EXPOSE 7997-8002
+
+# init
+COPY .travis/startml.sh /tmp/startml.sh
+RUN chmod 755 /tmp/startml.sh
+COPY .travis/setup.sh /tmp/setup.sh
+RUN chmod 755 /tmp/setup.sh
+RUN /tmp/setup.sh
+
+VOLUME /marklogic-data-hub
+ADD ./ /marklogic-data-hub
+WORKDIR /marklogic-data-hub
+
+RUN ./gradlew tasks 2>&1 > /dev/null
+
+# Define default command (which avoids immediate shutdown)
+CMD /tmp/startml.sh && ./gradlew test

.travis/download.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# runs command from parameters and exits with the eoror code of the command
+# if it fails
+function successOrExit {
+    "$@"
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "$1 exited with error: $status"
+        exit $status
+    fi
+}
+
+set | grep TRAVIS
+
+test $1 && arg1=$1
+
+ver=${ML_VERSION}
+fname=MarkLogic-RHEL7-${ver}.x86_64.rpm
+fnamedeb="marklogic_"
+fnamedeb=$fnamedeb$ver
+suff="_amd64.deb"
+fnamedeb=$fnamedeb$suff
+
+echo "Logging in for Download"
+curl -s -c cookies.txt --data "email=${MLBUILD_USER}&password=${MLBUILD_PASSWORD}" https://developer.marklogic.com/login > /dev/null 2>&1
+
+echo
+echo "Getting Download Link"
+dl_link=$(curl -s -b cookies.txt --data "download=/download/binaries/8.0/${fname}" https://developer.marklogic.com/get-download-url | perl -pe 's/.*"path":"([^"]+).*/\1/')
+url="https://developer.marklogic.com${dl_link}"
+
+echo "********* Downloading MarkLogic $ver"
+successOrExit curl -s -k -o ./MarkLogic.rpm $url

.travis/setup.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+#
+# Initializes a virgin MarkLogic installation.
+
+HOST=localhost
+ADMINUSER=admin
+ADMINPASSWORD=admin
+
+/tmp/startml.sh
+
+until curl -fsS --head --digest --user "$ADMINUSER":"$ADMINPASSWORD" http://"$HOST":8001/admin/v1/timestamp # &>/dev/null
+do
+  echo "MarkLogic hasn't started yet. Retrying in 3 seconds…"
+  sleep 3
+done
+
+# curl -X POST --data "" http://"$HOST":8001/admin/v1/init
+echo "Initializing…"
+curl --fail --show-error --silent -X POST --data "" http://"$HOST":8001/admin/v1/init 1>/dev/null
+if [[ $? != 0 ]] ; then
+    echo "error on init"
+    exit 1
+fi
+echo "Completed initialization. Waiting for restart…"
+sleep 20
+
+# curl -fsS --head --digest --user admin:"$ADMINPASSWORD" http://"$HOST":8001/admin/v1/timestamp
+# One liner: until curl -fsS --head http://192.168.56.101:8001/admin/v1/timestamp --digest --user admin:admin; do sleep 5; done
+
+
+until curl -fsS --head --digest --user "$ADMINUSER":"$ADMINPASSWORD" http://"$HOST":8001/admin/v1/timestamp &>/dev/null
+do
+  echo "Restart hasn't completed. Retrying in 3 seconds…"
+  sleep 3
+done
+
+# curl -X POST -H "Content-type: application/x-www-form-urlencoded" --data "admin-username=admin" --data "admin-password=********" http://localhost:8001/admin/v1/instance-admin
+echo "Starting instance administration…"
+curl --fail --show-error --silent \
+  -X POST -H "Content-type: application/x-www-form-urlencoded" \
+  --data "admin-username=${ADMINUSER}" --data "admin-password=${ADMINPASSWORD}" --data "realm=public" \
+  http://"$HOST":8001/admin/v1/instance-admin 1>/dev/null
+if [[ $? != 0 ]] ; then
+    echo "Error on instance-admin"
+    exit 1
+fi
+
+echo "Completed instance administration. Waiting for restart…"
+sleep 10
+until curl -fsS --head --digest --user admin:"$ADMINPASSWORD" http://"$HOST":8001/admin/v1/timestamp &>/dev/null
+do
+  echo "Restart hasn't completed. Retrying in 5 seconds…"
+  sleep 5
+done
+
+echo "Turning off log file rotation"
+curl --fail --show-error --silent \
+  -X PUT --digest -u "$ADMINUSER":"$ADMINPASSWORD" -H "Content-type: application/json" \
+  -d '{"rotate-log-files":"never"}' http://"$HOST":8002/manage/v2/groups/Default/properties &>/dev/null
+
+echo "Done!"
+exit 0

.travis/startml.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export MARKLOGIC_INSTALL_DIR=/opt/MarkLogic
+export MARKLOGIC_DATA_DIR=/data
+
+export MARKLOGIC_FSTYPE=ext4
+export MARKLOGIC_USER=daemon
+export MARKLOGIC_PID_FILE=/var/run/MarkLogic.pid
+export MARKLOGIC_MLCMD_PID_FILE=/var/run/mlcmd.pid
+export MARKLOGIC_UMASK=022
+
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/MarkLogic/mlcmd/bin
+export LD_PRELOAD=/opt/MarkLogic/lib/libjemalloc.so.1
+export LD_LIBRARY_PATH=/opt/MarkLogic/lib:/data/Lib
+
+echo "STARTING MARKLOGIC"
+
+/opt/MarkLogic/bin/MarkLogic

CONTRIBUTING.md

@@ -36,7 +36,7 @@ To build the entire DHF (marklogic-data-hub.jar, quickstart.war, and ml-data-hub
 
 ```bash
 cd /path/to/data-hub-project/
-gradle build -x test
+./gradlew build -x test
 ```
 
 #### Making Changes to the Hub Gradle Plugin
@@ -47,9 +47,9 @@ Still here? Seems you really want to modify the Gradle Plugin. Here's how to tel
 
 ```bash
 cd /path/to/data-hub-project/
-gradle publishToMavenLocal
+./gradlew publishToMavenLocal
 cd /path/to/data-hub-project/ml-data-hub-plugin
-gradle publishToMavenLocal
+./gradlew publishToMavenLocal
 ```
 
 Then in your build.gradle file you will need to use the local version:
@@ -60,10 +60,10 @@ buildscript {
     jcenter()
   } 
   dependencies {
-    classpath "com.marklogic:ml-data-hub-plugin:(the version number you chose)"
+    classpath "com.marklogic:ml-data-hub:(the version number you chose)"
   }
 }
-apply plugin: "com.marklogic.ml-data-hub-plugin"
+apply plugin: "com.marklogic.ml-data-hub"
 ```
 
 #### Running the QuickStart UI from source
@@ -74,7 +74,7 @@ You will need to open two terminal windows.
 **Terminal window 1** - This runs the webapp.
 ```bash
 cd /path/to/data-hub-project
-gradle bootrun
+./gradlew bootrun
 ```
 
 **Terminal window 2** - This runs the Quickstart UI
@@ -179,7 +179,7 @@ $ git rebase upstream/2.0-develop
 Make sure the JUnit tests pass.
 
 ```sh
-$ gradle test
+$ ./gradlew test
 ```
 
 Make sure that all tests pass. Please, do not submit patches that fail.

README.md

@@ -1,12 +1,11 @@
-<p style="font-style: italic; font-size:12px;">The MarkLogic Data Hub Framework is a data integration framework and tool-set to quickly and efficiently integrate data from many sources into a single MarkLogic database, and expose that data.</p>
+_The MarkLogic Data Hub Framework is a data integration framework and tool-set to quickly and efficiently integrate data from many sources into a single MarkLogic database, and expose that data._
 
-<p style="font-style: italic; font-size:12px;">The Data Hub Framework is free and open source under the <a href="https://github.com/marklogic/marklogic-data-hub/blob/1.0-master/LICENSE">Apache 2 License</a> and is supported by the community of developers who build and contribute to it. Please note that this open source project and its code and functionality is not representative of MarkLogic Server and is not supported by MarkLogic.
-</p>
+_The Data Hub Framework is free and open source under the [Apache 2 License](https://github.com/marklogic/marklogic-data-hub/blob/1.0-master/LICENSE) and is supported by the community of developers who build and contribute to it. Please note that this open source project and its code and functionality is not representative of MarkLogic Server and is not supported by MarkLogic._
 
-OS | Status
---- | --- | ---
-Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=2.0-develop)](https://travis-ci.org/marklogic/marklogic-data-hub)
-Windows | [![Windows Build status](https://ci.appveyor.com/api/projects/status/kgj0k5na59uhkvbv?svg=true)](https://ci.appveyor.com/project/paxtonhare/marklogic-data-hub)
+| OS | Status |
+| --- | --- |
+| Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=2.0-develop)](https://travis-ci.org/marklogic/marklogic-data-hub) |
+| Windows | [![Windows Build status](https://ci.appveyor.com/api/projects/status/kgj0k5na59uhkvbv?svg=true)](https://ci.appveyor.com/project/paxtonhare/marklogic-data-hub) |
 
 # MarkLogic Data Hub
 
@@ -16,17 +15,23 @@ Go from nothing to Enterprise Data Hub in a matter of minutes.
 
 This project allows you to deploy a skeleton Data Hub into MarkLogic. With some basic configuration you will be running an Enterprise Data Hub in no time.
 
+# Getting Started
 
+### Prerequisites
 
-# Getting Started
+You need these to get started
+
+- Java 8 JDK
+- MarkLogic 9.0 or greater
+- Gradle 3.1 or greater **(Optional)**
 
-###TL;DR
+### TL;DR
 
 Head over to our [Getting Started Tutorial](https://marklogic.github.io/marklogic-data-hub/) to get up and running with the Data Hub.
 
 Or watch the [MarkLogic University - Data Hub Framework On Demand Video Courses](http://mlu.marklogic.com/ondemand/index.xqy?q=Series%3A%22Operational%20Data%20Hubs%22)
 
-###The Easiest Way
+### The Easiest Way
 
 To use the Data Hub Framework you should download the quickstart.war file from the [releases page](https://github.com/marklogic/marklogic-data-hub/releases).
 
@@ -36,7 +41,7 @@ Then Run the war like so:
 java -jar quickstart.war
 ```
 
-###Using the Hub in your existing Java project
+### Using the Hub in your existing Java project
 
 Alternatively you can include the jar file as a build dependency in your Java project. Make sure you reference the latest version.
 
@@ -78,7 +83,7 @@ plugins {
 Now you have full access to the Data Hub tasks. To see all available tasks run:
 
 ```bash
-gradle tasks
+./gradlew tasks
 ```
 
 # Building From Source

build.gradle

@@ -18,6 +18,7 @@ subprojects {
     test{
         testLogging{
             events 'started','passed', 'skipped', 'failed'
+            exceptionFormat 'full'
         }
     }
 

docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2'
+services:
+  marklogic:
+    build:
+      context: ./
+      dockerfile: .travis/Dockerfile-ml
+    ports:
+      - "8000-8020"

gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Wed Dec 14 14:06:48 EST 2016
+#Mon Mar 13 12:57:35 EDT 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -154,16 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"