Merge pull request #2548 from kuzzleio/elasticsearch-8

Elasticsearch 8

.ci/scripts/run-monkey-tests.sh

@@ -1,32 +1,30 @@
-#!/bin/bash
-
-set -ex
+echo "Testing Kuzzle against node v$NODE_VERSION"
 
-if [ -z "$NODE_VERSION" ];
-then
-  echo "Missing NODE_VERSION, use default NODE_20_VERSION"
-  export NODE_VERSION=$NODE_20_VERSION
+if [ "$ES_VERSION" == "7" ]; then
+    YML_FILE='./.ci/test-cluster-7.yml'
+elif [ "$ES_VERSION" == "8" ]; then
+    YML_FILE='./.ci/test-cluster-8.yml'
+else
+    echo "Invalid ES_VERSION. It should be either '7' or '8'."
+    exit 1 
 fi
 
-echo "Testing Kuzzle against node v$NODE_VERSION"
-
-docker compose -f ./.ci/test-cluster.yml down -v
+docker compose -f $YML_FILE down -v
 
 echo "Installing dependencies..."
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm ci
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm ci
 
-if [ "$REBUILD" == "true" ];
-then
-  docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm rebuild
+if [ "$REBUILD" == "true" ]; then
+    docker compose -f $YML_FILE run --rm kuzzle_node_1 npm rebuild
 fi
 
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm run build
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm run build
 
 echo "[$(date)] - Starting Kuzzle Cluster..."
 
-trap 'docker compose -f ./.ci/test-cluster.yml logs' err
+trap 'docker compose -f $YML_FILE logs' err
 
-docker compose -f ./.ci/test-cluster.yml up -d
+docker compose -f $YML_FILE up -d
 
 # don't wait on 7512: nginx will accept connections far before Kuzzle does
 KUZZLE_PORT=17510 ./bin/wait-kuzzle
@@ -39,4 +37,4 @@ echo "Installing Kuzzle Monkey Tester..."
 
 cd kuzzle-monkey-tests
 npm ci
-node index.js
+node index.js

.ci/scripts/run-test-cluster.sh

@@ -2,7 +2,6 @@
 
 set -ex
 
-
 if [ -z "$NODE_VERSION" ];
 then
   echo "Missing NODE_VERSION, use default NODE_20_VERSION"
@@ -11,23 +10,32 @@ fi
 
 echo "Testing Kuzzle against node v$NODE_VERSION"
 
-docker compose -f ./.ci/test-cluster.yml down -v
+if [ "$ES_VERSION" == "7" ]; then
+    YML_FILE='./.ci/test-cluster-7.yml'
+elif [ "$ES_VERSION" == "8" ]; then
+    YML_FILE='./.ci/test-cluster-8.yml'
+else
+    echo "Invalid ES_VERSION. It should be either '7' or '8'."
+    exit 1 
+fi
+
+docker compose -f $YML_FILE down -v
 
 echo "Installing dependencies..."
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm ci
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm ci
 
 if [ "$REBUILD" == "true" ];
 then
-  docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm rebuild
+  docker compose -f $YML_FILE run --rm kuzzle_node_1 npm rebuild
 fi
 
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm run build
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm run build
 
 echo "[$(date)] - Starting Kuzzle Cluster..."
 
-trap 'docker compose -f ./.ci/test-cluster.yml logs' err
+trap 'docker compose -f $YML_FILE logs' err
 
-docker compose -f ./.ci/test-cluster.yml up -d
+docker compose -f $YML_FILE up -d
 
 # don't wait on 7512: nginx will accept connections far before Kuzzle does
 KUZZLE_PORT=17510 ./bin/wait-kuzzle
@@ -36,4 +44,4 @@ KUZZLE_PORT=17512 ./bin/wait-kuzzle
 
 trap - err
 
-npm run $KUZZLE_FUNCTIONAL_TESTS
+npm run $KUZZLE_FUNCTIONAL_TESTS

.ci/services.yml → .ci/services-7.yml

@@ -13,6 +13,7 @@ services:
     sysctls:
       - net.core.somaxconn=8192
     environment:
+      - kuzzle_services__storageEngine__majorVersion=7
       - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
       - kuzzle_services__internalCache__node__host=redis
       - kuzzle_services__memoryStorage__node__host=redis

.ci/services-8.yml

@@ -0,0 +1,40 @@
+version: '3'
+
+services:
+  kuzzle:
+    build:
+      dockerfile: ./docker/images/${KUZZLE_IMAGE}/Dockerfile
+      context: ..
+    command: ["kuzzle", "start"]
+    cap_add:
+      - SYS_PTRACE
+    ulimits:
+      nofile: 65536
+    sysctls:
+      - net.core.somaxconn=8192
+    environment:
+      - kuzzle_services__storageEngine__majorVersion=8
+      - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
+      - kuzzle_services__internalCache__node__host=redis
+      - kuzzle_services__memoryStorage__node__host=redis
+    depends_on:
+      - redis
+      - elasticsearch
+    ports:
+      - "7512:7512"
+
+  redis:
+    image: redis:6
+
+  elasticsearch:
+    image: elasticsearch:8.11.3
+    environment:
+      - xpack.security.enabled=false
+      - action.destructive_requires_name=false
+      - cluster.name=kuzzle
+      - node.name=alyx
+      - discovery.type=single-node
+      - ingest.geoip.downloader.enabled=false
+      - indices.id_field_data.enabled=true
+    ports:
+      - '9200:9200'

.ci/test-cluster.yml → .ci/test-cluster-7.yml

@@ -18,6 +18,7 @@ x-kuzzle-config: &kuzzle-config
     - redis
     - elasticsearch
   environment:
+    - kuzzle_services__storageEngine__majorVersion=7
     - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
     - kuzzle_services__storageEngine__commonMapping__dynamic=true
     - kuzzle_services__internalCache__node__host=redis

.ci/test-cluster-8.yml

@@ -0,0 +1,81 @@
+version: '3.4'
+
+x-kuzzle-config: &kuzzle-config
+  image: kuzzleio/kuzzle-runner:${NODE_VERSION:-20}
+  command: > 
+    bash -c "
+      node ./docker/scripts/start-kuzzle-test.js --enable-plugins kuzzle-plugin-cluster,functional-test-plugin
+    "
+  volumes:
+    - "..:/var/app"
+  cap_add:
+    - SYS_PTRACE
+  ulimits:
+    nofile: 65536
+  sysctls:
+    - net.core.somaxconn=8192
+  depends_on:
+    - redis
+    - elasticsearch
+  environment:
+    - kuzzle_services__storageEngine__majorVersion=8
+    - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
+    - kuzzle_services__storageEngine__commonMapping__dynamic=true
+    - kuzzle_services__internalCache__node__host=redis
+    - kuzzle_services__memoryStorage__node__host=redis
+    - kuzzle_server__protocols__mqtt__enabled=true
+    - kuzzle_server__protocols__mqtt__developmentMode=false
+    - kuzzle_http__accessControlAllowOrigin=localhost
+    - kuzzle_limits__loginsPerSecond=50
+    - kuzzle_server__protocols__http__additionalContentTypes=*json:["application/x-yaml"]
+    - NODE_ENV=development
+    - NODE_VERSION=${NODE_VERSION:-20}
+    - DEBUG=none
+    - SECRETS_FILE_PREFIX=/var/app/
+    # cluster
+    - kuzzle_plugins__cluster__privileged=true
+    - kuzzle_plugins__cluster__minimumNodes=3
+
+services:
+  nginx:
+    image: nginx:1.19-alpine
+    depends_on:
+      - kuzzle_node_1
+      - kuzzle_node_2
+      - kuzzle_node_3
+    ports:
+      - "7512:7512"
+    volumes:
+      - ../docker/nginx-dev:/etc/nginx/conf.d
+
+  kuzzle_node_1:
+    <<: *kuzzle-config
+    ports:
+      - "17510:7512"
+      - "1883:1883"
+
+  kuzzle_node_2:
+    <<: *kuzzle-config
+    ports:
+      - "17511:7512"
+
+  kuzzle_node_3:
+    <<: *kuzzle-config
+    ports:
+      - "17512:7512"
+
+  redis:
+    image: redis:6
+
+  elasticsearch:
+    image: elasticsearch:8.11.3
+    environment:
+      - xpack.security.enabled=false
+      - action.destructive_requires_name=false
+      - cluster.name=kuzzle
+      - node.name=alyx
+      - discovery.type=single-node
+      - ingest.geoip.downloader.enabled=false
+      - indices.id_field_data.enabled=true
+    ports:
+      - '9200:9200'

.eslintignore

@@ -58,7 +58,8 @@ lib/types/Token.js
 lib/types/User.js
 lib/types/Global.d.ts
 lib/types/Global.js
-lib/types/storage/Elasticsearch.js
+lib/types/storage/7/Elasticsearch.js
+lib/types/storage/8/Elasticsearch.js
 lib/util/interfaces.js
 lib/util/mutex.js
 lib/util/Inflector.js
@@ -126,7 +127,10 @@ lib/util/dump-collection.js
 lib/api/controllers/debugController.js
 lib/api/openapi/components/security/index.js
 lib/config/documentEventAliases.js
-lib/service/storage/elasticsearch.js
+lib/service/storage/Elasticsearch.js
+lib/service/storage/7/elasticsearch.js
+lib/service/storage/8/elasticsearch.js
+lib/service/storage/commons/queryTranslator.js
 lib/types/DebugModule.js
 lib/util/time.js
 lib/util/async.js

.github/actions/build-and-run-kuzzle/action.yml

@@ -2,9 +2,12 @@ name: Build and Run Kuzzle image
 description: Build and Run Kuzzle image
 
 inputs:
-  KUZZLE_IMAGE:
+  kuzzle-image:
     description: Kuzzle image target
     required: true
+  es-version:
+    description: Elasticsearch version to use
+    required: true
 
 runs:
   using: "composite"
@@ -19,18 +22,54 @@ runs:
       shell: bash
 
     - run: |
-        export KUZZLE_IMAGE=${{ inputs.KUZZLE_IMAGE }}
-        docker compose -f ./.ci/services.yml up -d
+        export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+        export ES_VERSION=${{ inputs.es-version }}
+
+        if [ "$ES_VERSION" == "7" ]; then
+            YML_FILE='./.ci/services-7.yml'
+        elif [ "$ES_VERSION" == "8" ]; then
+            YML_FILE='./.ci/services-8.yml'
+        else
+            echo "Invalid ES_VERSION. It should be either '7' or '8'."
+            exit 1 
+        fi
+
+        docker compose -f $YML_FILE up -d
       shell: bash
 
     - run: |
-        trap 'docker compose -f ./.ci/services.yml logs' err
+        export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+        export ES_VERSION=${{ inputs.es-version }}
+
+        if [ "$ES_VERSION" == "7" ]; then
+            YML_FILE='./.ci/services-7.yml'
+        elif [ "$ES_VERSION" == "8" ]; then
+            YML_FILE='./.ci/services-8.yml'
+        else
+            echo "Invalid ES_VERSION. It should be either '7' or '8'."
+            exit 1 
+        fi
+
+        trap 'docker compose -f $YML_FILE logs' err
 
         export MAX_TRIES=60
         ./bin/wait-kuzzle
 
         trap - err
       shell: bash
 
-    - run: docker compose -f ./.ci/services.yml down
+    - run: |
+        export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+        export ES_VERSION=${{ inputs.es-version }}
+
+        if [ "$ES_VERSION" == "7" ]; then
+            YML_FILE='./.ci/services-7.yml'
+        elif [ "$ES_VERSION" == "8" ]; then
+            YML_FILE='./.ci/services-8.yml'
+        else
+            echo "Invalid ES_VERSION. It should be either '7' or '8'."
+            exit 1 
+        fi
+
+        docker compose -f $YML_FILE down -v
       shell: bash

.github/actions/functional-tests/action.yml

@@ -10,6 +10,9 @@ inputs:
   node-version:
     description: Node version to use in the Kuzzle Docker image
     required: true
+  es-version:
+    description: Elasticsearch version to use
+    required: true
 
 runs:
   using: "composite"
@@ -18,4 +21,5 @@ runs:
       env:
         KUZZLE_FUNCTIONAL_TESTS: ${{ inputs.test-set }}
         NODE_VERSION: ${{ inputs.node-version }}
+        ES_VERSION: ${{ inputs.es-version }}
       shell: bash

.github/actions/monkey-tests/action.yml

@@ -11,4 +11,5 @@ runs:
     - run: ./.ci/scripts/run-monkey-tests.sh
       env:
         NODE_VERSION: ${{ inputs.node-version }}
+        ES_VERSION: ${{ inputs.es-version }}
       shell: bash

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/core-dev.yml

@@ -13,7 +13,7 @@ jobs:
     name: core-dev image
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

.github/workflows/elasticsearch.yml

@@ -14,7 +14,7 @@ jobs:
     name: Elasticsearch image
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3