Merge branch 'develop'

README.md

@@ -18,17 +18,12 @@ These charts can be accessed by cloning our repository or by using our Helm repo
 
 To use this repo, run:
 ```bash
-helm repo add gaffer https://gchq.github.io/gaffer-docker
+helm repo add gaffer-docker https://gchq.github.io/gaffer-docker
 ```
 
-There are guides on how to deploy the charts on:
-* a local Kubernetes cluster, [using kind (Kubernetes IN Docker)](kubernetes/kind-deployment.md)
-* an [AWS EKS cluster](kubernetes/aws-eks-deployment.md)
+## Kubernetes How-to Guides
 
-These guides assume you've cloned the repository and are using the Helm charts contained within. We do also host our Helm Charts index on our Github pages site. You can get access to them by running:
-```bash
-helm repo add gaffer https://gchq.github.io/gaffer-docker
-```
+We have a number of [guides](./kubernetes/docs/guides.md) to help you deploy Gaffer on Kubernetes.
 
 ## Versioning
 Each of our images will be tagged in DockerHub with the version of the software they represent. Every release,

app_version

@@ -1 +1 @@
-0.6.2
+0.7.0

cd/install.sh

@@ -50,10 +50,10 @@ buildImages
 
 # Deploy Images to Kind
 kind load docker-image gchq/hdfs:3.2.1
-kind load docker-image gchq/gaffer:1.12.0
-kind load docker-image gchq/gaffer-rest:1.12.0
-kind load docker-image gchq/gaffer-road-traffic-loader:1.12.0
-kind load docker-image gchq/gaffer-operation-runner:1.12.0
+kind load docker-image gchq/gaffer:1.13.4
+kind load docker-image gchq/gaffer-rest:1.13.4
+kind load docker-image gchq/gaffer-road-traffic-loader:1.13.4
+kind load docker-image gchq/gaffer-operation-runner:1.13.4
 
 # Deploy containers onto Kind
 # Travis needs this setting to avoid reverse dns lookup errors

docker/gaffer-operation-runner/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.12.0
-GAFFER_TOOLS_VERSION=1.12.0
+GAFFER_VERSION=1.13.4
+GAFFER_TOOLS_VERSION=1.13.1
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-operation-runner/Dockerfile

@@ -16,7 +16,7 @@ ARG BUILDER_IMAGE_NAME=maven
 ARG BUILDER_IMAGE_TAG=3.6.3-jdk-8
 
 ARG BASE_IMAGE_NAME=gchq/gaffer
-ARG BASE_IMAGE_TAG=1.12.0
+ARG BASE_IMAGE_TAG=1.13.4
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 

docker/gaffer-operation-runner/operation-runner/pom.xml

@@ -35,7 +35,7 @@
     <dependency>
       <groupId>uk.gov.gchq.gaffer</groupId>
       <artifactId>graph</artifactId>
-      <version>1.12.0</version>
+      <version>1.13.4</version>
     </dependency>
   </dependencies>
 

docker/gaffer-rest/.env

@@ -1,2 +1,2 @@
-GAFFER_VERSION=1.12.0
-GAFFER_TOOLS_VERSION=1.12.0
+GAFFER_VERSION=1.13.4
+GAFFER_TOOLS_VERSION=1.13.1

docker/gaffer-rest/Dockerfile

@@ -20,8 +20,8 @@ ARG BASE_IMAGE_TAG=10.1.0.Final
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.12.0
-ARG GAFFER_TOOLS_VERSION=${GAFFER_VERSION}
+ARG GAFFER_VERSION=1.13.4
+ARG GAFFER_TOOLS_VERSION=1.13.1
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_TOOLS_GIT_REPO=https://github.com/gchq/gaffer-tools.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2

docker/gaffer-road-traffic-loader/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.12.0
-GAFFER_TOOLS_VERSION=1.12.0
+GAFFER_VERSION=1.13.4
+GAFFER_TOOLS_VERSION=1.13.1
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-road-traffic-loader/Dockerfile

@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=8u242-jre-slim-buster
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.12.0
+ARG GAFFER_VERSION=1.13.4
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

docker/gaffer/.env

@@ -1,5 +1,5 @@
-GAFFER_VERSION=1.12.0
-GAFFER_TOOLS_VERSION=1.12.0
+GAFFER_VERSION=1.13.4
+GAFFER_TOOLS_VERSION=1.13.1
 ACCUMULO_VERSION=1.9.3
 HADOOP_VERSION=3.2.1
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer/Dockerfile

@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=1.9.3
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=1.12.0
+ARG GAFFER_VERSION=1.13.4
 ARG GAFFER_LIBS=bitmap-library,sketches-library,time-library
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git

kubernetes/docs/add-libraries.md

@@ -0,0 +1,62 @@
+Adding your own libraries and functions
+=======================================
+By default with the Gaffer deployment you get access to the:
+* Sketches library
+* JCS cache library
+
+If you want more libraries than this (either one of ours of one of your own) you'll need to customise the docker images and use them in place of the defaults.
+
+You'll need a basic Gaffer instance deployed on Kubernetes. Here's [how to do that](./deploy-empty-graph.md).
+
+### Overwrite the REST war file
+At the moment, Gaffer uses a WAR file with all the dependencies bundled in. You'll need to extend the WAR file using [these instructions](https://gchq.github.io/gaffer-doc/components/rest-api.html#how-to-modify-the-rest-api-for-your-project). Once you have a custom war file, you'll need to create a new image based on the `gaffer-rest` one. To do that you'll need a `Dockerfile` like this one:
+```Dockerfile
+FROM gchq/gaffer-rest:latest
+COPY ./my-custom-rest:1.0-SNAPSHOT.war /opt/jboss/wildfly/standalone/deployments/rest.war
+```
+
+Build the image using:
+```bash
+docker build -t custom-rest:latest .
+```
+
+### Add the extra libraries to the Accumulo image
+Gaffer's accumulo image includes support for the following gaffer libraries:
+* The Bitmap Library
+* The Sketches Library
+* The Time Library
+
+In order to push down any extra value objects and filters to Accumulo that aren't in those libraries, we have to add the jars to the accumulo /lib/ext directory. Here's an example `Dockerfile`:
+```Dockerfile
+FROM gchq/gaffer:latest
+COPY ./my-library-1.0-SNAPSHOT.jar /opt/accumulo/lib/ext
+```
+Then build the image
+```bash
+docker build -t custom-gaffer-accumulo:latest .
+```
+
+### Switch the images in the deployment
+
+You'll need a way of making the custom images visible to the kubernetes cluster. With EKS, you can do this by uploading the images to ECR. There's an example for how to do that in one of our [other guides](./aws-eks-deployment.md#Container+Images). With KinD, you just run `kind load docker-image <image:tag>`.
+
+Once visible you can switch them out. Create a `custom-images.yaml` file with the following contents:
+```yaml
+api:
+  image:
+    repository: custom-rest
+    tag: latest
+
+accumulo:
+  image:
+    repository: custom-gaffer-accumulo
+    tag: latest
+```
+
+To switch them run:
+```bash
+helm upgrade my-graph gaffer-docker/gaffer -f custom-images.yaml --reuse-values
+```
+
+### What next?
+See our [guides](./guides.md) for other things you can do with Gaffer on Kubernetes.

kubernetes/aws-eks-deployment.md → kubernetes/docs/aws-eks-deployment.md

@@ -21,8 +21,8 @@ The following instructions build all the container images and host them in AWS E
 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
-export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
-export GAFFER_TOOLS_VERSION=${GAFFER_TOOLS_VERSION:-$GAFFER_VERSION}
+export GAFFER_VERSION=${GAFFER_VERSION:-1.13.4}
+export GAFFER_TOOLS_VERSION=${GAFFER_TOOLS_VERSION:-1.13.1}
 
 docker-compose --project-directory ../docker/accumulo/ -f ../docker/accumulo/docker-compose.yaml build
 docker-compose --project-directory ../docker/gaffer-operation-runner/ -f ../docker/gaffer-operation-runner/docker-compose.yaml build 
@@ -52,7 +52,6 @@ for repo in ${GAFFER_IMAGES}; do
 done
 ```
 
-
 ## EKS Cluster
 
 There are a number of ways to provision an AWS EKS cluster. This guide uses a cli tool called `eksctl`. Documentation is available at https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html for some of the other methods.
@@ -135,9 +134,9 @@ curl https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controlle
 
 ## Deploy Helm Charts
 
-* [HDFS](hdfs/docs/aws-eks-deployment.md)
-* [Gaffer](gaffer/docs/aws-eks-deployment.md)
-* [Example Gaffer Graph containing Road Traffic Dataset](gaffer-road-traffic/docs/aws-eks-deployment.md)
+* [HDFS](../hdfs/docs/aws-eks-deployment.md)
+* [Gaffer](../gaffer/docs/aws-eks-deployment.md)
+* [Example Gaffer Graph containing Road Traffic Dataset](../gaffer-road-traffic/docs/aws-eks-deployment.md)
 
 
 ## Access Web UIs

kubernetes/docs/change-graph-metadata.md

@@ -0,0 +1,57 @@
+Changing the Graph Id and Description
+=======================================
+By default, the default Gaffer deployment ships with the Graph name "simpleGraph" and description "A  graph for demo purposes" These are just placeholders and can be overwritten. This guide will show you how.
+
+The first thing you'll need to do is [deploy a simple graph](./deploy-empty-graph.md).
+
+### Changing the description
+Create a file called `graph-meta.yaml`. We will use this file to add our description and graph Id.
+Changing the description is as easy as changing the `graph.config.description` value.
+```yaml
+graph:
+  config:
+    description: "My graph description"
+```
+Feel free to be a bit more imaginative.
+
+### Deploy the new description
+Upgrade your deployment using helm:
+
+```bash
+helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values
+```
+
+The `--reuse-values` argument means we don't override any passwords that we set in the initial construction.
+
+You can see your new description if you go to the Swagger UI and call the /graph/config/description endpoint.
+
+### Updating the graph Id
+Updating the graph Id is a little more complicated since the Graph Id corresponds to an Accumulo table. We have to change the gaffer user's permissions to read and write to that table. To do that update the `graph-meta.yaml` file with the following contents:
+```yaml
+graph:
+  config:
+    graphId: "MyGraph"
+    description: "My Graph description"
+
+accumulo:
+  userManagement:
+    gaffer:
+      permissions:
+        table:
+          MyGraph:
+          - READ
+          - WRITE
+          - BULK_IMPORT
+          - ALTER_TABLE
+```
+
+### Deploy your changes
+Upgrade your deployment using Helm.
+```bash
+helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values
+```
+
+If you take a look at the Accumulo monitor, you will see your new Accumulo table
+
+### What next?
+See our [guides](./guides.md) for other things you can do with Gaffer on Kubernetes.

kubernetes/docs/deploy-empty-graph.md

@@ -0,0 +1,44 @@
+How to deploy a simple graph
+==================================
+This guide will describe how to deploy a simple empty graph with the minimum configuration.
+
+You will need:
+1. Helm
+2. Kubectl
+3. A Kubernetes cluster (local or remote)
+
+### Add the Gaffer Docker repo
+To start with, you should add the Gaffer Docker repo to your helm repos. This will save the need for
+cloning this Git repository. If you've already done this, you can skip this step.
+```bash
+helm repo add gaffer-docker https://gchq.github.io/gaffer-docker
+```
+
+### Set the Accumulo passwords
+By default, we don't set the Accumulo passwords for you. To set these create a file called `password-values.yaml` and add the following:
+
+```yaml
+accumulo:
+  config:
+    accumuloSite:
+      instance.secret: "DEFAULT"
+    userManagement:
+      rootPassword: "root"
+      users:
+        gaffer:
+          password: "gaffer"
+        tracer:
+          password: "tracer"
+```
+Replace the default values with your own. Once this is deployed you cannot change it.
+
+### Deploy the default Gaffer Graph
+You can now deploy the default Gaffer Graph with a default schema. The first time you deploy
+Gaffer it can take around 5 minutes for everything to start so it may be worth adding `--timeout 10m0s` to this command.
+```bash
+helm install my-graph gaffer-docker/gaffer -f password-values.yaml
+```
+Feel free to change "my-graph" to something more interesting.
+
+### What next?
+See our [guides](./guides.md) for other things you can do with Gaffer on Kubernetes.

kubernetes/docs/guides.md

@@ -0,0 +1,10 @@
+Guides
+========
+Here you'll find all our guides for deploying Gaffer on Kubernetes.
+
+1. [How to deploy on a KinD cluster locally](./kind-deployment.md)
+2. [How to deploy to EKS](./aws-eks-deployment.md)
+3. [How to deploy a simple empty graph](./deploy-empty-graph.md)
+4. [How to add your schema](./schema.md)
+5. [How to change the Graph Id and Description](./change-graph-metadata.md)
+6. [Adding your own functions and libraries](./add-libraries.md)

kubernetes/kind-deployment.md → kubernetes/docs/kind-deployment.md

@@ -27,8 +27,8 @@ To import the images, run this from the kubernetes directory:
 
 ```bash
 export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1}
-export GAFFER_VERSION=${GAFFER_VERSION:-1.12.0}
-export GAFFER_TOOLS_VERSION=${GAFFER_TOOLS_VERSION:-$GAFFER_VERSION}
+export GAFFER_VERSION=${GAFFER_VERSION:-1.13.4}
+export GAFFER_TOOLS_VERSION=${GAFFER_TOOLS_VERSION:-1.13.1}
 
 docker-compose --project-directory ../docker/accumulo/ -f ../docker/accumulo/docker-compose.yaml build
 docker-compose --project-directory ../docker/gaffer-operation-runner/ -f ../docker/gaffer-operation-runner/docker-compose.yaml build
@@ -51,9 +51,9 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/${IN
 
 ## Deploy Helm Charts
 
-* [HDFS](hdfs/docs/kind-deployment.md)
-* [Gaffer](gaffer/docs/kind-deployment.md)
-* [Example Gaffer Graph containing Road Traffic Dataset](gaffer-road-traffic/docs/kind-deployment.md)
+* [HDFS](../hdfs/docs/kind-deployment.md)
+* [Gaffer](../gaffer/docs/kind-deployment.md)
+* [Example Gaffer Graph containing Road Traffic Dataset](../gaffer-road-traffic/docs/kind-deployment.md)
 
 
 ## Uninstall