Merge branch 'main' into dependabot/pip/contrib/config/marketplace/aw…

…s/tests/idna-3.7
dgraph-io · Aug 3, 2024 · 4e80fcf · 4e80fcf
2 parents dde5916 + 63186e8
commit 4e80fcf
Show file tree

Hide file tree

Showing 188 changed files with 6,664 additions and 1,450 deletions.
diff --git a/.github/workflows/cd-dgraph.yml b/.github/workflows/cd-dgraph.yml
@@ -10,6 +10,10 @@ on:
         description: releasetag
         required: true
         type: string
+      custom-build:
+        type: boolean
+        default: false
+        description: if checked, images will be pushed to dgraph-custom repo in Dockerhub 
 jobs:
   dgraph-build-amd64:
     runs-on: warp-ubuntu-latest-x64-16x
@@ -85,7 +89,10 @@ jobs:
         run: |
           make docker-image DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-amd64
           [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 dgraph/dgraph:latest-amd64 || true
+          [[ "${{ inputs.custom-build }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-amd64  || true
       - name: Make Dgraph Standalone Docker Image with Version
+        #No need to build and push Standalone Image when its a custom build
+        if: inputs.custom-build == false
         run: |
           make docker-image-standalone DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-amd64
           [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 dgraph/standalone:latest-amd64 || true
@@ -96,8 +103,12 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
       - name: Push Images to DockerHub
         run: |
-          docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
-          docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
+          if [ "${{ inputs.custom-build }}" == "true" ]; then
+            docker push dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
+          else
+            docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
+            docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
+          fi
 
   dgraph-build-arm64:
     runs-on: warp-ubuntu-latest-arm64-16x
@@ -173,7 +184,10 @@ jobs:
         run: |
           make docker-image DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-arm64
           [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 dgraph/dgraph:latest-arm64 || true
+          [[ "${{ inputs.custom-build }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-arm64  || true
       - name: Make Dgraph Standalone Docker Image with Version
+         #No need to build and push Standalone Image when its a custom build
+        if: inputs.custom-build == false
         run: |
           make docker-image-standalone DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-arm64
           [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 dgraph/standalone:latest-arm64 || true
@@ -184,8 +198,12 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
       - name: Push Images to DockerHub
         run: |
-          docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+          if [ "${{ inputs.custom-build }}" == "true" ]; then
+            docker push dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+          else
+            docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+            docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+          fi
 
   dgraph-docker-manifest:
     needs: [dgraph-build-amd64, dgraph-build-arm64]
@@ -215,13 +233,23 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
       - name: Docker Manifest
         run: |
-          # standalone
-          docker manifest create dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker manifest push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest create dgraph/standalone:latest --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest push dgraph/standalone:latest || true
-          # dgraph
-          docker manifest create dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker manifest push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest create dgraph/dgraph:latest --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 || true
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest push dgraph/dgraph:latest || true
+            if [ "${{ github.event.inputs.custom-build }}" == "true" ]; then
+              docker manifest create dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+              docker manifest push dgraph/dgraph-custom:${{ env.DGRAPH_RELEASE_VERSION }}
+            else
+              # standalone
+              docker manifest create dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64      
+              docker manifest push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}
+              if [ "${{ github.event.inputs.latest }}" == "true" ]; then
+                 docker manifest create dgraph/standalone:latest --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+                 docker manifest push dgraph/standalone:latest
+              fi
+              # dgraph
+              docker manifest create dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+              docker manifest push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}
+              if [ "${{ github.event.inputs.latest }}" == "true" ]; then
+                 docker manifest create dgraph/dgraph:latest --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
+                 docker manifest push dgraph/dgraph:latest
+              fi
+            fi
+
diff --git a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml
@@ -1,7 +1,7 @@
 name: ci-dgraph-upgrade-fixed-versions-tests
 on:
   schedule:
-    - cron: "0 3 * * *" # 1 run per day
+    - cron: "00 20 * * *" # 1 run per day
 jobs:
   dgraph-upgrade-fixed-versions-tests:
     runs-on: warp-ubuntu-latest-x64-16x 

diff --git a/.github/workflows/ci-dgraph-upgrade-tests.yml b/.github/workflows/ci-dgraph-upgrade-tests.yml
@@ -9,6 +9,8 @@ on:
     branches:
       - main
       - 'release/**'
+  schedule:
+    - cron: "00 20 * * *" # 1 run per day
 jobs:
   dgraph-upgrade-tests:
     if: github.event.pull_request.draft == false

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -0,0 +1,53 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+    branches:
+      - main
+      - release/**
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: warp-ubuntu-latest-x64-16x
+    timeout-minutes: 360
+    permissions:
+      security-events: write
+      packages: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: go
+          build-mode: autobuild
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -0,0 +1,18 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '00 02,14 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: 'This issue has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          stale-pr-message: 'This PR has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          operations-per-run: 250
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,110 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project will adhere to [Semantic Versioning](https://semver.org) starting `v22.0.0`.
 
+## [v24.0.1] - 2024-07-30
+[v24.0.1]: https://github.com/dgraph-io/dgraph/compare/v24.0.0...v24.0.1
+
+> **Warning**
+> After upgrading to v24.0.1, vector index needs to be rebuilt as underlying data has changed.
+
+- **Fixed**
+
+	- fix(core): Fix regression in parsing json empty string #9108
+	- fix(upgrade): fix failing upgrade tests #9042
+	- fix(ci): fixing health endpoint issue #9116
+	- Fix(graphql): issue with local variable squashing intended JWK index by @matthewmcneely in #9114
+
+- **Chore**
+	- chore(deps): bump urllib3 from 1.26.18 to 1.26.19 /contrib/config/marketplace/aws/tests #9103
+	- chore(deps): bump requests from 2.31.0 to 2.32.0 /contrib/config/marketplace/aws/tests #9090
+
+
+- **Perf**
+	- perf(vector): updated marshalling of vector #9109
+
+
+## [v24.0.0] - 2024-06-06
+[v24.0.0]: https://github.com/dgraph-io/dgraph/compare/v24.0.0...v23.1.0
+
+> **Warning**
+> This will be a breaking change for anyone moving from to `v.24.0.0`.
+> If you have any duplicate users or groups in Dgraph ACL, they would not be accessible from now on. Please delete any 
+> duplicate users and groups before you upgrade. File format is the same, so binary can be directly replaced after.
+> deleting duplicate users and groups.
+
+### Added
+
+- **Vector**
+	- feat(graphql): Add vector support to graphql (#9074)
+	- feat(vector): add vector to schema in #9060
+	- feat(vector): Added similar_to in vector in #9062
+	- feat(vector): Add vector type to Dgraph in #9050
+	- feat(vector): fix live loader and add tests for dropall, drop namespace, live load in #9063
+	- fix(vector): show error is invalid input is provided to vector predicate in #9064
+	- fix(vector):fix similar_to() error return when data is not present in #9084
+	- fix(vector): Update query_rewriter to fix dotproduct and cosine query conversion in #9083
+
+- **Core**
+	- feat(core): Add cache to dgraph.type predicate in #9068
+	- [BREAKING]feat(core): add unique constraint support in schema for new predicates in #8827
+	- feat(debug): add parse_key to debug tool in #7640
+	- feat(acl): support more JWT algorithms for ACL by in #8912
+	- feat(restore): add support for namespace aware restore by in #8968
+
+- **GraphQL**
+	- feat(vector): Added lang support by in #8924
+	- feat(graphql): allow updatable and nullable id fields. (#7736) in #9020
+### Fixed
+
+- **Core**
+	- Fix(debug): Close file correctly before exiting on error in #9076
+	- fix(restore): fix incr restore and normal restore for vector predicates in #9078
+	- Fix(core): Fix deadlock in runMutation and error handling in #9085
+	- fix(core): Fixed deadlock that happens due to timeout in #9007
+	- fix(core): limit #edges using LimitMutationsNquad config and add logs in #9010
+	- fix(core): Update math parsing function by in #9053
+	- fix(restore): use different map directory for each group (#8047) in #8972
+	- fix(export): Support for any s3 endpoint by in #8978
+	- fix(restore): use custom type for sensitive fields by in #8969
+	- fix(export): Escape MySQL column names in #8961
+	- fix(debug): fix debug tool for schema keys in #7939
+	- fix(restore): allow incrementalFrom to be 1 in restore API by in #8988
+	- fix(raft):alpha leader fails to stream snapshot to new alpha nodes in #9022
+	- fix(query): fix has function in filter in #9043
+	- fix(core):Reduce x.ParsedKey memory allocation from 72 to 56 bytes by optimizing struct memory alignment in #9047
+	- fix(restore): do not retry restore proposal (#8058) in #9017
+
+- **Perf**
+	- perf(core): Fix performance issue in type filter (#9065) in #9089
+	- perf(core): Update postinglistCountAndLength function to improve performance in #9088
+	- perf(query): use quickselect instead of sorting while pagination by in #8995
+	- perf(query): Update CompressedBin IntersectionAlgo by in #9000
+
+- **Chore**
+	- chore(upgrade): run tests from v23.1.0 -> main in #9097
+	- chore(deps): upgrade etcd/raft to v3 by in #7688
+	- chore(restore): add more logs for restore request (#8050) in #8975
+	- upgrade(go): update go version to 1.22 in #9058
+	- chore(deps): bump github.com/apache/thrift from 0.12.0 to 0.13.0 by in #8982
+	- chore(deps): bump golang.org/x/net from 0.14.0 to 0.17.0 in #9015
+	- chore(deps): use bleve 2.3.10 for more languages in full text search in #9030
+	- chore(deps): bump golang.org/x/crypto from 0.12.0 to 0.17.0 in #9032
+	- chore(deps): bump urllib3 from 1.26.5 to 1.26.18 in /contrib/config/marketplace/aws/tests in #9018
+	- chore(deps): bump google.golang.org/grpc from 1.56.2 to 1.56.3 in #9024
+	- chore(deps): bump google.golang.org/protobuf from 1.31.0 to 1.33.0in #9051[
+
+## [23.1.1] - 2024-04-26
+[v23.1.1]: https://github.com/dgraph-io/dgraph/compare/v23.1.0...v23.1.1
+
+### Fixed
+
+- **Core Dgraph**
+    - perf(core): Fix performance issue in type filter (#9065)
+
+
+- **CI & Testing**
+    - ci/cd optimizations (#9069)
+
 ## [v23.1.0] - 2023-08-17
 [v23.1.0]: https://github.com/dgraph-io/dgraph/compare/v23.0.1...v23.1.0