redis-cluster.yml

#
# Redis Cluster service
#
apiVersion: v1
kind: Service
metadata:
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
  name: redis-cluster
  namespace: dipper
  labels:
    app: redis-cluster
spec:
  ports:
  - port: 6379
    targetPort: 6379
    name: client
  - port: 16379
    targetPort: 16379
    name: gossip
  clusterIP: None
  #type: ClusterIP
  selector:
    app: redis-cluster
---
#
# Redis configuration file for clustered mode
#
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster-config
  namespace: dipper
data:
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file nodes.conf
    cluster-migration-barrier 1 
    activerehashing yes
    appendonly yes
    appendfsync everysec
    hash-max-ziplist-entries 128
    hash-max-ziplist-value 1024
    slave-read-only yes
    stop-writes-on-bgsave-error no
    loglevel notice
    protected-mode no
    databases 16
    maxmemory-policy volatile-lru
    maxmemory-samples 5
    dbfilename dump.rdb
    dir /var/lib/redis
  bootstrap-pod.sh: |+
    #!/bin/bash
    set -e

    PORT=6379
    CLUSTER_IPS=""


    redis-server /conf/redis.conf &
    chmod u+x /bin/peer-finder
    chmod u+x /usr/local/bin/redis-trib.rb
    chmod u+x /conf/meet-cluster.sh
    peer-finder -on-start '/conf/meet-cluster.sh' -service redis-cluster -ns $POD_NAMESPACE

    sleep 1
    wait
  meet-cluster.sh: |+
    #!/bin/bash
    
    set -e
    
    PORT=6379
    CLUSTER_IPS=""
    PET_ORDINAL=$(cat /etc/podinfo/pod_name | rev | cut -d- -f1)

    REBOOT_FLAG=$(cat /var/lib/redis/nodes.conf 2>/dev/null | wc -l)
    POD_IP=$(getent hosts $HOSTNAME.$SERVICE_NAME.$POD_NAMESPACE.svc.cluster.local|awk {'print $1'})

    sleep 10

    if [[ $REBOOT_FLAG == 2 && $PET_ORDINAL == 7 ]]; then #a new pod_7 has been created, then create cluster
      echo "here is a new redis cluster node"
      echo "creating cluster..."
      while read -ra LINE
      do
        CLUSTER_IPS="${CLUSTER_IPS} $(getent hosts ${LINE} | cut -d ' ' -f1):${PORT}"
      done
      echo "check cluster node ..."
      for PEER_IP in $(getent hosts ${LINE} | cut -d ' ' -f1)
      do
        until redis-cli -h $PEER_IP ping; do sleep 1; done
      done
      echo yes | /usr/local/bin/redis-trib.rb create --replicas 1 ${CLUSTER_IPS}
    elif [[ $REBOOT_FLAG > 2 ]];then #a used pod has been found,and meet into the cluster
      echo "here is a used node from other cluster"
      redis-cli -c cluster meet $(POD_IP) $(PORT)

    #elif [[ $PET_ORDINAL -gt 5 ]]; then
    #  echo "add new node..."
    #  sleep 10
    #  if [[ $(expr $PET_ORDINAL % 2) == 0 ]]; then
    #    echo "add as a master node..."
    #    /usr/local/bin/redis-trib.rb add-node $(getent hosts $(hostname)|awk {'print $1'}):$PORT $CLUSTER_BUILDER:$PORT
    #  else
    #    echo "add slave for the new master..."
    #    /usr/local/bin/redis-trib.rb add-node --slave $(getent hosts $(hostname)|awk {'print $1'}):$PORT $CLUSTER_BUILDER:$PORT
    #    echo $CLUSTER_BUILDER
    #    /usr/local/bin/redis-trib.rb rebalance --use-empty-masters $CLUSTER_BUILDER:$PORT
    #  fi

    elif [[ $REBOOT_FLAG == 2 && $PET_ORDINAL < 7 ]];then 
      echo "cluster too small"
    else 
      echo "there is an unknow error "
    fi
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: redis-cluster
  namespace: dipper
spec:
  serviceName: redis-cluster
  replicas: 8
  template:
    metadata:
      labels:
        app: redis-cluster
      annotations:
    spec:
      terminationGracePeriodSeconds: 10
      #hostNetwork: true
      containers:
      - name: redis-cluster
        image: hub.bonc:5000/redis:3.2.8-rb
        ports:
        - containerPort: 6379
          name: client
        - containerPort: 16379
          name: gossip
        command:
        - sh
        args:
        - /conf/bootstrap-pod.sh
        # the health check of redis-cluster
         # readinessProbe:
         #   exec:
         #     command:
         #     - sh
         #     - -c
         #     - "redis-cli -h $(hostname) ping"
         #   initialDelaySeconds: 15
         #   timeoutSeconds: 5

         # livenessProbe:
         #   exec:
         #     command:
         #     - sh
         #     - -c
         #     - "redis-cli -h $(hostname) ping"
         #   initialDelaySeconds: 20
         #   periodSeconds: 3
        # set some ENV from the yaml of redis-cluster
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: SERVICE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.servicename
        # mount all things on the redis container's filesystem
        volumeMounts:
        - name: data
          mountPath: /var/lib/redis
          readOnly: false
        - name: conf
          mountPath: /conf
          readOnly: false
        - name: podinfo
          mountPath: /etc/podinfo
          readOnly: false
      # create PVC and volumes from storageclass 
      volumes:
      - name: "data"
        persistentVolumeClaim:
          claimName: "data"
      # Insert our pre-baked Redis configuration file into /conf/redis.conf
      - name: conf
        configMap:
          name: redis-cluster-config
          items: 
          - key: redis.conf
            path: redis.conf
          - key: bootstrap-pod.sh  
            path: bootstrap-pod.sh
          - key: meet-cluster.sh
            path: meet-cluster.sh
      # The init container will use this info to find cluster peers
      - name: podinfo
        downwardAPI:
          items:
            - path: "labels"
              fieldRef:
                fieldPath: metadata.labels
            - path: "annotations"
              fieldRef:
                fieldPath: metadata.annotations
            - path: "pod_name"
              fieldRef:
                fieldPath: metadata.name
            - path: "pod_namespace"
              fieldRef:
                fieldPath: metadata.namespace
  volumeClaimTemplates:
  - metadata:
      name: data
      namespace: dipper
    spec:
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi
      storageClassName: ceph-rbd