redis-cluster-new.yml

#
# Redis Cluster service
#
apiVersion: v1
kind: Service
metadata:
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
  name: redis-cluster
  namespace: dipper
  labels:
    app: redis-cluster
spec:
  ports:
  - port: 6379
    targetPort: 6379
    name: client
  - port: 16379
    targetPort: 16379
    name: gossip
  clusterIP: None
  #type: ClusterIP
  selector:
    app: redis-cluster
---
#
# Redis configuration file for clustered mode
#
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster-config
  namespace: dipper
data:
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file nodes.conf
    cluster-migration-barrier 1 
    activerehashing yes
    appendonly yes
    appendfsync everysec
    hash-max-ziplist-entries 128
    hash-max-ziplist-value 1024
    slave-read-only yes
    stop-writes-on-bgsave-error no
    loglevel notice
    protected-mode no
    databases 16
    maxmemory-policy volatile-lru
    maxmemory-samples 5
    dbfilename dump.rdb
    dir /var/lib/redis
  bootstrap-pod.sh: |+
    #!/bin/bash
    set -e

    PORT=6379
    CLUSTER_IPS=""


    redis-server /conf/redis.conf &
    chmod u+x /bin/peer-finder
    chmod u+x /usr/local/bin/redis-trib.rb
    chmod u+x /conf/meet-cluster.sh
    peer-finder -on-start '/conf/meet-cluster.sh' -service redis-cluster -ns $POD_NAMESPACE

    sleep 1
    wait
  meet-cluster.sh: |+
    #!/bin/bash
    
    set -e
    
    PORT=6379
    CLUSTER_IPS=""

    pod_name=$(cat /etc/podinfo/pod_name)
    ns=$(cat /etc/podinfo/pod_namespace)
    podIP=$(getent hosts $(pod_name).redis-cluster.$(ns).svc.cluster.local|awk {'print $1'})
    PET_ORDINAL=$(cat /etc/podinfo/pod_name | rev | cut -d- -f1)

    sleep 10
    if [[ -f /var/lib/redis/nodes.conf ]]; then
      echo "nodes.conf exists"
      LC=$(cat /var/lib/redis/nodes.conf | wc -l)
      echo "lines of nodes.conf is "$lc
      if [[ $LC <= 2 && $PET_ORDINAL == 7 ]]; then
        echo "creating cluster..."
        while read -ra LINE; do
          CLUSTER_IPS="${CLUSTER_IPS} $(getent hosts ${LINE} | cut -d ' ' -f1):${PORT}"
        done
        until redis-cli -h 127.0.0.1 ping; do sleep 1; done
        echo yes | /usr/local/bin/redis-trib.rb create --replicas 1 ${CLUSTER_IPS}
      elif [[ $lc > 2 ]]; then
        redis-cli -c cluster meet $(podIP) $(PORT)
      fi
    fi
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: redis-cluster
  namespace: dipper
spec:
  serviceName: redis-cluster
  replicas: 8
  template:
    metadata:
      labels:
        app: redis-cluster
      annotations:
        # NOTE: Init container must be idempotent
        # Add a baked-in Redis config file that enables cluster mode
        #pod.alpha.kubernetes.io/init-containers: '[
        #]'
    spec:
      terminationGracePeriodSeconds: 10
      #hostNetwork: true
      containers:
      - name: redis-cluster
        image: hub.bonc:5000/redis:3.2.8-rb
        ports:
        - containerPort: 6379
          name: client
        - containerPort: 16379
          name: gossip
        command:
        - sh
        args:
        - /conf/bootstrap-pod.sh
        # Ensure that Redis is online before initializing the next node.
        # TODO: Test that the cluster node is init'd properly.
        #readinessProbe:
        #  exec:
        #    command:
        #    - sh
        #    - -c
        #    - "redis-cli -h $(hostname) ping"
        #  initialDelaySeconds: 30
        #  timeoutSeconds: 10

        # Mark a node as down if Redis server stops running
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - "redis-cli -h $(hostname) ping"
          initialDelaySeconds: 15
          periodSeconds: 5
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: data
          mountPath: /var/lib/redis
          readOnly: false
        - name: conf
          mountPath: /conf
          readOnly: false
        - name: podinfo
          mountPath: /etc/podinfo
          readOnly: false
      volumes:
      - name: "data"
        persistentVolumeClaim:
          claimName: "data"
      # Insert our pre-baked Redis configuration file into /conf/redis.conf
      - name: conf
        configMap:
          name: redis-cluster-config
          items: 
          - key: redis.conf
            path: redis.conf
          - key: bootstrap-pod.sh  # TODO: Move this or extract it into its own Docker image
            path: bootstrap-pod.sh
          - key: meet-cluster.sh
            path: meet-cluster.sh
      # The init container will use this info to find cluster peers
      - name: podinfo
        downwardAPI:
          items:
            - path: "labels"
              fieldRef:
                fieldPath: metadata.labels
            - path: "annotations"
              fieldRef:
                fieldPath: metadata.annotations
            - path: "pod_name"
              fieldRef:
                fieldPath: metadata.name
            - path: "pod_namespace"
              fieldRef:
                fieldPath: metadata.namespace
  volumeClaimTemplates:
  - metadata:
      name: data
      namespace: dipper
    spec:
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi
      storageClassName: ceph-rbd