redis-cluster-debug.yml

#
# Redis Cluster service
#
apiVersion: v1
kind: Service
metadata:
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
  name: redis-cluster
  namespace: dipper
  labels:
    app: redis-cluster
spec:
  ports:
  - port: 6379
    targetPort: 6379
    name: client
  - port: 16379
    targetPort: 16379
    name: gossip
  clusterIP: None
  #type: ClusterIP
  selector:
    app: redis-cluster
---
#
# Redis configuration file for clustered mode
#
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster-config
  namespace: dipper
data:
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file nodes.conf
    cluster-migration-barrier 1 
    activerehashing yes
    appendonly yes
    appendfsync everysec
    hash-max-ziplist-entries 128
    hash-max-ziplist-value 1024
    slave-read-only yes
    stop-writes-on-bgsave-error no
    loglevel notice
    protected-mode no
    databases 16
    maxmemory-policy volatile-lru
    maxmemory-samples 5
    dbfilename dump.rdb
    dir /var/lib/redis
  bootstrap-pod.sh: |+
    #!/bin/bash
    set -e

    PORT=6379
    CLUSTER_IPS=""

    mkdir -p /var/lib/redis

    redis-server /conf/redis.conf &

    sleep 1
    wait

    #chmod u+x /bin/peer-finder
    #chmod u+x /usr/local/bin/redis-trib.rb
    #chmod u+x /conf/meet-cluster.sh
    #peer-finder -on-start '/conf/meet-cluster.sh' -service redis-cluster -ns $POD_NAMESPACE

    sleep 1
    wait
  meet-cluster.sh: |+
    #!/bin/bash
    
    set -e
    
    PORT=6379
    CLUSTER_IPS=""

    CLUSTER_BUILDER=$(getent hosts redis-cluster-5.redis-cluster.dipper.svc.cluster.local|awk {'print $1'})

    PET_ORDINAL=$(cat /etc/podinfo/pod_name | rev | cut -d- -f1)
    sleep 10
    if [[ $PET_ORDINAL == 5 ]]; then
      echo "creating cluster..."
      sleep 10
      while read -ra LINE; do
        CLUSTER_IPS="${CLUSTER_IPS} $(getent hosts ${LINE} | cut -d ' ' -f1):${PORT}"
      done
      until redis-cli -h 127.0.0.1 ping; do sleep 1; done
      echo yes | /usr/local/bin/redis-trib.rb create --replicas 1 ${CLUSTER_IPS}
    elif [[ $PET_ORDINAL -gt 5 ]]; then
      echo "add new node..."
      sleep 10
      if [[ $(expr $PET_ORDINAL % 2) == 0 ]]; then
        echo "add as a master node..."
        /usr/local/bin/redis-trib.rb add-node $(getent hosts $(hostname)|awk {'print $1'}):$PORT $CLUSTER_BUILDER:$PORT
      else
        echo "add slave for the new master..."
        /usr/local/bin/redis-trib.rb add-node --slave $(getent hosts $(hostname)|awk {'print $1'}):$PORT $CLUSTER_BUILDER:$PORT
        echo $CLUSTER_BUILDER
        /usr/local/bin/redis-trib.rb rebalance --use-empty-masters $CLUSTER_BUILDER:$PORT
      fi
    else
      echo "cluster too small"
    fi
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: redis-cluster
  namespace: dipper
spec:
  serviceName: redis-cluster
  replicas: 8
  template:
    metadata:
      labels:
        app: redis-cluster
      annotations:
        # NOTE: Init container must be idempotent
        # Add a baked-in Redis config file that enables cluster mode
        #pod.alpha.kubernetes.io/init-containers: '[
        #]'
    spec:
      terminationGracePeriodSeconds: 10
      #hostNetwork: true
      containers:
      - name: redis-cluster
        image: dipperroy/redis:3.2.8-rb
        ports:
        - containerPort: 6379
          name: client
        - containerPort: 16379
          name: gossip
        command:
        - sh
        args:
        - /conf/bootstrap-pod.sh
        # Ensure that Redis is online before initializing the next node.
        # TODO: Test that the cluster node is init'd properly.
        readinessProbe:
          exec:
            command:
            - sh
            - -c
            - "redis-cli -h $(hostname) ping"
          initialDelaySeconds: 15
          timeoutSeconds: 5

        # Mark a node as down if Redis server stops running
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - "redis-cli -h $(hostname) ping"
          initialDelaySeconds: 20
          periodSeconds: 3
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        #- name: data
        #  mountPath: /var/lib/redis
        #  readOnly: false
        - name: conf
          mountPath: /conf
          readOnly: false
        - name: podinfo
          mountPath: /etc/podinfo
          readOnly: false
      volumes:
      # Mount the PVC to storage redis data and info of cluster
#      - name: "data"
#        persistentVolumeClaim:
#          claimName: "data"
      # Insert our pre-baked Redis configuration file into /conf/redis.conf
      - name: conf
        configMap:
          name: redis-cluster-config
          items: 
          - key: redis.conf
            path: redis.conf
          - key: bootstrap-pod.sh
            path: bootstrap-pod.sh
          - key: meet-cluster.sh
            path: meet-cluster.sh
      # The init container will use this info to find cluster peers
      - name: podinfo
        downwardAPI:
          items:
            - path: "labels"
              fieldRef:
                fieldPath: metadata.labels
            - path: "annotations"
              fieldRef:
                fieldPath: metadata.annotations
            - path: "pod_name"
              fieldRef:
                fieldPath: metadata.name
            - path: "pod_namespace"
              fieldRef:
                fieldPath: metadata.namespace
#  volumeClaimTemplates:
#  - metadata:
#      name: data
#      namespace: dipper
#    spec:
#      accessModes:
#        - ReadWriteOnce
#      resources:
#        requests:
#          storage: 1Gi
#      storageClassName: ceph-rbd