Merge branch 'main' into main

.github/workflows/stale.yml

@@ -0,0 +1,18 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '00 02,14 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: 'This issue has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          stale-pr-message: 'This PR has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          operations-per-run: 250

CHANGELOG.md

@@ -4,6 +4,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project will adhere to [Semantic Versioning](https://semver.org) starting `v22.0.0`.
 
+## [v24.0.1] - 2024-07-30
+[v24.0.1]: https://github.com/dgraph-io/dgraph/compare/v24.0.0...v24.0.1
+
+> **Warning**
+> After upgrading to v24.0.1, vector index needs to be rebuilt as underlying data has changed.
+
+- **Fixed**
+
+	- fix(core): Fix regression in parsing json empty string #9108
+	- fix(upgrade): fix failing upgrade tests #9042
+	- fix(ci): fixing health endpoint issue #9116
+	- Fix(graphql): issue with local variable squashing intended JWK index by @matthewmcneely in #9114
+
+- **Chore**
+	- chore(deps): bump urllib3 from 1.26.18 to 1.26.19 /contrib/config/marketplace/aws/tests #9103
+	- chore(deps): bump requests from 2.31.0 to 2.32.0 /contrib/config/marketplace/aws/tests #9090
+
+
+- **Perf**
+	- perf(vector): updated marshalling of vector #9109
+
+
 ## [v24.0.0] - 2024-06-06
 [v24.0.0]: https://github.com/dgraph-io/dgraph/compare/v24.0.0...v23.1.0
 
@@ -74,6 +96,18 @@ and this project will adhere to [Semantic Versioning](https://semver.org) starti
 	- chore(deps): bump google.golang.org/grpc from 1.56.2 to 1.56.3 in #9024
 	- chore(deps): bump google.golang.org/protobuf from 1.31.0 to 1.33.0in #9051[
 
+## [23.1.1] - 2024-04-26
+[v23.1.1]: https://github.com/dgraph-io/dgraph/compare/v23.1.0...v23.1.1
+
+### Fixed
+
+- **Core Dgraph**
+    - perf(core): Fix performance issue in type filter (#9065)
+
+
+- **CI & Testing**
+    - ci/cd optimizations (#9069)
+
 ## [v23.1.0] - 2023-08-17
 [v23.1.0]: https://github.com/dgraph-io/dgraph/compare/v23.0.1...v23.1.0
 

chunker/json_parser.go

@@ -230,7 +230,7 @@ func handleBasicType(k string, v interface{}, op int, nq *api.NQuad) error {
 			return nil
 		}
 
-		if vf, err := types.ParseVFloat(v); err == nil {
+		if vf, err := types.ParseVFloat(v); err == nil && len(vf) != 0 {
 			nq.ObjectValue = &api.Value{Val: &api.Value_Vfloat32Val{Vfloat32Val: types.FloatArrayAsBytes(vf)}}
 			return nil
 		}

chunker/json_parser_test.go

@@ -74,6 +74,13 @@ type Person struct {
 	School    *School    `json:"school,omitempty"`
 }
 
+type Product struct {
+	Uid           string `json:"uid,omitempty"`
+	Name          string `json:"name"`
+	Discription   string `json:"discription"`
+	Discription_v string `json:"discription_v"`
+}
+
 func Parse(b []byte, op int) ([]*api.NQuad, error) {
 	nqs := NewNQuadBuffer(1000)
 	err := nqs.ParseJSON(b, op)
@@ -1380,3 +1387,128 @@ func BenchmarkNoFacetsFast(b *testing.B) {
 		_, _ = FastParse(json, SetNquads)
 	}
 }
+
+func TestNquadsEmptyStringFromJson(t *testing.T) {
+	json := `[{"name":""}]`
+
+	nq, err := Parse([]byte(json), SetNquads)
+	require.NoError(t, err)
+
+	fastNQ, err := FastParse([]byte(json), SetNquads)
+	require.NoError(t, err)
+
+	// The string value should be empty.
+	require.Equal(t, nq[0].ObjectValue.GetStrVal(), "")
+	require.Equal(t, fastNQ[0].ObjectValue.GetStrVal(), "")
+}
+
+func TestNquadsJsonEmptyStringVectorPred(t *testing.T) {
+	p := Product{
+		Uid:           "1",
+		Name:          "",
+		Discription_v: "",
+	}
+
+	b, err := json.Marshal([]Product{p})
+	require.NoError(t, err)
+
+	nq, err := Parse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(nq))
+
+	fastNQ, err := FastParse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(fastNQ))
+
+	// predicate Name should be empty and edge for Discription_v should not be there
+	// we do not create edge for "" in float32vector.
+	exp := &Experiment{
+		t:   t,
+		nqs: nq,
+		schema: `name: string @index(exact) .
+		discription_v: float32vector .`,
+		query: `{product(func: uid(1)) {
+			name
+			discription_v
+		}}`,
+		expected: `{"product":[{
+			"name":""}]}`,
+	}
+	exp.verify()
+
+	exp.nqs = fastNQ
+	exp.verify()
+}
+
+func TestNquadsJsonEmptySquareBracketVectorPred(t *testing.T) {
+	p := Product{
+		Name:          "ipad",
+		Discription_v: "[]",
+	}
+
+	b, err := json.Marshal(p)
+	require.NoError(t, err)
+
+	nq, err := Parse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(nq))
+
+	fastNQ, err := FastParse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(fastNQ))
+
+	// predicate Name should have value "ipad" and edge for Discription_v should not be there
+	// we do not create edge for [] in float32vector.
+	exp := &Experiment{
+		t:   t,
+		nqs: nq,
+		schema: `name: string @index(exact) .
+		discription_v: float32vector .`,
+		query: `{product(func: eq(name, "ipad")) {
+			name
+			discription_v
+		}}`,
+		expected: `{"product":[{
+			"name":"ipad"}]}`,
+	}
+	exp.verify()
+
+	exp.nqs = fastNQ
+	exp.verify()
+}
+
+func TestNquadsJsonValidVector(t *testing.T) {
+	p := Product{
+		Name:          "ipad",
+		Discription_v: "[1.1, 2.2, 3.3]",
+	}
+
+	b, err := json.Marshal(p)
+	require.NoError(t, err)
+
+	nq, err := Parse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(nq))
+
+	fastNQ, err := FastParse(b, SetNquads)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(fastNQ))
+
+	exp := &Experiment{
+		t:   t,
+		nqs: nq,
+		schema: `name: string @index(exact) .
+		discription_v: float32vector .`,
+		query: `{product(func: eq(name, "ipad")) {
+			name
+			discription_v
+		}}`,
+		expected: `{"product":[{
+			"name":"ipad",
+			"discription_v":[1.1, 2.2, 3.3]}]}`,
+	}
+	exp.verify()
+
+	exp.nqs = fastNQ
+	exp.verify()
+}

contrib/config/marketplace/aws/tests/requirements.txt

@@ -4,7 +4,7 @@ awscli==1.18.40
 backports.shutil-get-terminal-size==1.0.0
 boto3==1.12.40
 botocore==1.15.40
-certifi==2023.7.22
+certifi==2024.7.4
 cfn-lint==0.29.5
 chardet==3.0.4
 colorama==0.4.3
@@ -13,7 +13,7 @@ decorator==4.4.2
 docker==4.2.0
 docutils==0.15.2
 dulwich==0.19.15
-idna==2.9
+idna==3.7
 Jinja2==3.1.4
 jmespath==0.9.5
 jsonpatch==1.25
@@ -29,13 +29,13 @@ pyrsistent==0.16.0
 python-dateutil==2.8.1
 PyYAML==5.4
 reprint==0.5.2
-requests==2.31.0
+requests==2.32.0
 rsa==4.7
 s3transfer==0.3.3
 six==1.14.0
 tabulate==0.8.7
 taskcat==0.9.17
 typing-extensions==3.7.4.2
-urllib3==1.26.18
+urllib3==1.26.19
 websocket-client==0.57.0
 yattag==1.13.2

dgraphtest/config.go

@@ -147,6 +147,15 @@ func NewClusterConfig() ClusterConfig {
 	}
 }
 
+//func newClusterConfigFrom(cc ClusterConfig) ClusterConfig {
+//	prefix := fmt.Sprintf("dgraphtest-%d", rand.NewSource(time.Now().UnixNano()).Int63()%1000000)
+//	defaultBackupVol := fmt.Sprintf("%v_backup", prefix)
+//	defaultExportVol := fmt.Sprintf("%v_export", prefix)
+//	cc.prefix = prefix
+//	cc.volumes = map[string]string{DefaultBackupDir: defaultBackupVol, DefaultExportDir: defaultExportVol}
+//	return cc
+//}
+
 // WithNAlphas sets the number of alphas in the cluster
 func (cc ClusterConfig) WithNumAlphas(n int) ClusterConfig {
 	cc.numAlphas = n

dgraphtest/dgraph.go

@@ -172,7 +172,7 @@ func (z *zero) healthURL(c *LocalCluster) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return "http://localhost:" + publicPort + "/health", nil
+	return "http://0.0.0.0:" + publicPort + "/health", nil
 }
 
 func (z *zero) changeStatus(isRunning bool) {
@@ -184,7 +184,7 @@ func (z *zero) assignURL(c *LocalCluster) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return "http://localhost:" + publicPort + "/assign", nil
+	return "http://0.0.0.0:" + publicPort + "/assign", nil
 }
 
 func (z *zero) alphaURL(c *LocalCluster) (string, error) {
@@ -196,7 +196,7 @@ func (z *zero) zeroURL(c *LocalCluster) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return "localhost:" + publicPort + "", nil
+	return "0.0.0.0:" + publicPort + "", nil
 }
 
 type alpha struct {
@@ -351,7 +351,7 @@ func (a *alpha) healthURL(c *LocalCluster) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return "http://localhost:" + publicPort + "/health", nil
+	return "http://0.0.0.0:" + publicPort + "/health", nil
 }
 
 func (a *alpha) assignURL(c *LocalCluster) (string, error) {
@@ -363,7 +363,7 @@ func (a *alpha) alphaURL(c *LocalCluster) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return "localhost:" + publicPort + "", nil
+	return "0.0.0.0:" + publicPort + "", nil
 }
 
 func (a *alpha) changeStatus(isRunning bool) {
@@ -388,8 +388,14 @@ func publicPort(dcli *docker.Client, dc dnode, privatePort string) (string, erro
 		if len(bindings) == 0 {
 			continue
 		}
-		if port.Port() == privatePort {
-			return bindings[0].HostPort, nil
+		if port.Port() != privatePort {
+			continue
+		}
+
+		for _, binding := range bindings {
+			if binding.HostIP == "0.0.0.0" {
+				return binding.HostPort, nil
+			}
 		}
 	}
 

dgraphtest/image.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"golang.org/x/mod/modfile"
 )
 
 func (c *LocalCluster) dgraphImage() string {
@@ -151,6 +152,10 @@ func getHash(ref string) (string, error) {
 func buildDgraphBinary(dir, binaryDir, version string) error {
 	log.Printf("[INFO] building dgraph binary for version [%v]", version)
 
+	if err := fixGoModIfNeeded(); err != nil {
+		return err
+	}
+
 	cmd := exec.Command("make", "dgraph")
 	cmd.Dir = filepath.Join(dir, "dgraph")
 	if out, err := cmd.CombinedOutput(); err != nil {
@@ -229,3 +234,28 @@ func IsHigherVersion(higher, lower string) (bool, error) {
 
 	return true, nil
 }
+
+func fixGoModIfNeeded() error {
+	repoModFilePath := filepath.Join(repoDir, "go.mod")
+	repoModFile, err := modfile.Parse(repoModFilePath, nil, nil)
+	if err != nil {
+		return errors.Wrapf(err, "error parsing mod file in repoDir [%v]", repoDir)
+	}
+
+	modFile, err := modfile.Parse("go.mod", nil, nil)
+	if err != nil {
+		return errors.Wrapf(err, "error while parsing go.mod file")
+	}
+
+	if len(modFile.Replace) == len(repoModFile.Replace) {
+		return nil
+	}
+
+	repoModFile.Replace = modFile.Replace
+	if data, err := repoModFile.Format(); err != nil {
+		return errors.Wrapf(err, "error while formatting mod file")
+	} else if err := os.WriteFile(repoModFilePath, data, 0644); err != nil {
+		return errors.Wrapf(err, "error while writing to go.mod file")
+	}
+	return nil
+}