From f1a5348fe1fa969996b09a1db8df428d3cba2f9e Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Wed, 5 Jul 2023 19:45:40 +0530 Subject: [PATCH 1/7] Changes to make plugin_test work with dgraphtest --- dgraphtest/image.go | 6 ++ systest/plugin/integration_test.go | 61 +++++++++++++++ systest/plugin/plugin_test.go | 118 +++++++++++++++++------------ systest/plugin/upgrade_test.go | 83 ++++++++++++++++++++ 4 files changed, 220 insertions(+), 48 deletions(-) create mode 100644 systest/plugin/integration_test.go create mode 100644 systest/plugin/upgrade_test.go diff --git a/dgraphtest/image.go b/dgraphtest/image.go index a2fc0e587a6..24898105116 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strings" + "github.com/dgraph-io/dgraph/testutil" "github.com/pkg/errors" ) @@ -52,6 +53,11 @@ func (c *LocalCluster) setupBinary() error { if err := buildDgraphBinary(repoDir, binariesPath, c.conf.version); err != nil { return err } + + // explicit var declaration to avoid confusion on the next line + race := false + testutil.GeneratePlugins(race) + return copyBinary(binariesPath, c.tempBinDir, c.conf.version) } diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go new file mode 100644 index 00000000000..be67995e126 --- /dev/null +++ b/systest/plugin/integration_test.go @@ -0,0 +1,61 @@ +//go:build integration + +/* + * Copyright 2023 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/dgraph-io/dgo/v230/protos/api" + "github.com/dgraph-io/dgraph/dgraphtest" + "github.com/dgraph-io/dgraph/x" +) + +type PluginTestSuite struct { + suite.Suite + dc dgraphtest.Cluster +} + +func (psuite *PluginTestSuite) SetupTest() { + psuite.dc = dgraphtest.NewComposeCluster() +} + +func (psuite *PluginTestSuite) TearDownTest() { + t := psuite.T() + gcli, cleanup, err := psuite.dc.Client() + defer cleanup() + require.NoError(t, err) + require.NoError(t, gcli.LoginIntoNamespace(context.Background(), + dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) + require.NoError(t, gcli.Alter(context.Background(), &api.Operation{DropAll: true})) +} + +func (psuite *PluginTestSuite) Upgrade() { + // Not implemented for integration tests +} + +func TestPluginTestSuite(t *testing.T) { + for _, e := range pfiArray { + psuite.pfiEntry := e + suite.Run(t, new(PluginTestSuite)) + } +} diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index b09cf8692a9..57ccaea7c58 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -1,4 +1,4 @@ -//go:build integration +//go:build (!oss && integration) || upgrade /* * Copyright 2017-2023 Dgraph Labs, Inc. and Contributors @@ -26,46 +26,24 @@ import ( "github.com/stretchr/testify/require" "github.com/dgraph-io/dgo/v230/protos/api" - "github.com/dgraph-io/dgraph/testutil" + "github.com/dgraph-io/dgraph/dgraphtest" + "github.com/dgraph-io/dgraph/x" ) -func TestPlugins(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode") - } - - type testCase struct { - query string - wantResult string - } - - client, err := testutil.DgraphClient(testutil.SockAddr) - require.NoError(t, err) - suite := func(initialSchema string, setJSON string, cases []testCase) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - require.NoError(t, client.Alter(ctx, &api.Operation{ - DropAll: true, - })) - require.NoError(t, client.Alter(ctx, &api.Operation{ - Schema: initialSchema, - })) - - txn := client.NewTxn() - _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(setJSON)}) - require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) +type testCase struct { + query string + wantResult string +} - for _, test := range cases { - txn := client.NewTxn() - reply, err := txn.Query(ctx, test.query) - require.NoError(t, err) - testutil.CompareJSON(t, test.wantResult, string(reply.GetJson())) - } - } +type PluginFuncInp struct { + initialSchema string + setJSON string + cases []testCase +} - suite( +var pfiArray = []PluginFuncInp{ + { "word: string @index(anagram) .", `[ { "word": "airmen" }, @@ -104,9 +82,8 @@ func TestPlugins(t *testing.T) { ]}`, }, }, - ) - - suite( + }, + { "word: string @index(anagram) @lang .", `[ { "word@en": "airmen", "word@fr": "no match" }, @@ -122,9 +99,8 @@ func TestPlugins(t *testing.T) { ]}`, }, }, - ) - - suite( + }, + { "ip: string @index(cidr) .", `[ { "ip": "100.55.22.11/32" }, @@ -173,9 +149,8 @@ func TestPlugins(t *testing.T) { ]}`, }, }, - ) - - suite( + }, + { "name: string @index(rune) .", `[ { "name": "Adam" }, @@ -247,8 +222,8 @@ func TestPlugins(t *testing.T) { ]}`, }, }, - ) - suite( + }, + { "num: int @index(factor) .", `[ { "num": 2 }, @@ -332,5 +307,52 @@ func TestPlugins(t *testing.T) { ]}`, }, }, - ) + }, +} + +func (psuite *PluginTestSuite) TestPlugins() { + t := psuite.T() + + if testing.Short() { + t.Skip("skipping test in short mode") + } + + pluginFn := func(initialSchema string, setJSON string, cases []testCase) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + gcli, cleanup, err := psuite.dc.Client() + defer cleanup() + require.NoError(t, err) + require.NoError(t, gcli.LoginIntoNamespace(context.Background(), + dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) + require.NoError(t, gcli.Alter(ctx, &api.Operation{ + DropAll: true, + })) + require.NoError(t, gcli.Alter(ctx, &api.Operation{ + Schema: initialSchema, + })) + + txn := gcli.NewTxn() + _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(setJSON)}) + require.NoError(t, err) + require.NoError(t, txn.Commit(ctx)) + + // Upgrade + psuite.Upgrade() + + for _, test := range cases { + txn := gcli.NewTxn() + reply, err := txn.Query(ctx, test.query) + require.NoError(t, err) + dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) + } + } + + initialSchema string + setJSON string + cases []testCase + + arg := psuite.pfiEntry + pluginFn(arg.initialSchema, arg.setJSON, arg.cases) } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go new file mode 100644 index 00000000000..cbad1a71bad --- /dev/null +++ b/systest/plugin/upgrade_test.go @@ -0,0 +1,83 @@ +//go:build upgrade + +/* + * Copyright 2023 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/dgraph-io/dgraph/dgraphtest" + "github.com/dgraph-io/dgraph/x" +) + +type PluginTestSuite struct { + suite.Suite + dc dgraphtest.Cluster + lc *dgraphtest.LocalCluster + uc dgraphtest.UpgradeCombo + pfiEntry PluginFuncInp +} + +func (psuite *PluginTestSuite) SetupTest() { + conf := dgraphtest.NewClusterConfig().WithNumAlphas(1).WithNumZeros(1).WithReplicas(1). + WithACL(20 * time.Second).WithEncryption().WithVersion(psuite.uc.Before) + c, err := dgraphtest.NewLocalCluster(conf) + x.Panic(err) + if err := c.Start(); err != nil { + c.Cleanup(true) + panic(err) + } + + psuite.dc = c + psuite.lc = c +} + +func (psuite *PluginTestSuite) TearDownTest() { + psuite.lc.Cleanup(psuite.T().Failed()) +} + +func (psuite *PluginTestSuite) Upgrade() { + t := psuite.T() + + if err := psuite.lc.Upgrade(psuite.uc.After, psuite.uc.Strategy); err != nil { + t.Fatal(err) + } +} + +// For testing purpose +var comboEntry = []dgraphtest.UpgradeCombo{ + {"v21.03.0", "v23.0.0", dgraphtest.BackupRestore}, +} + +func TestPluginTestSuite(t *testing.T) { + //for _, uc := range dgraphtest.AllUpgradeCombos { + //} + for _, uc := range comboEntry { + log.Printf("running: backup in [%v], restore in [%v]", uc.Before, uc.After) + var psuite PluginTestSuite + psuite.uc = uc + for _, e := range pfiArray { + psuite.pfiEntry := e + suite.Run(t, &psuite) + } + } +} From e813702c2598f2da7eb2c7540692d5a8979c2fd5 Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Wed, 5 Jul 2023 19:54:48 +0530 Subject: [PATCH 2/7] Compile time error fix --- systest/plugin/integration_test.go | 4 +++- systest/plugin/plugin_test.go | 4 ---- systest/plugin/upgrade_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go index be67995e126..82c4f6d7e77 100644 --- a/systest/plugin/integration_test.go +++ b/systest/plugin/integration_test.go @@ -33,6 +33,7 @@ import ( type PluginTestSuite struct { suite.Suite dc dgraphtest.Cluster + pfiEntry PluginFuncInp } func (psuite *PluginTestSuite) SetupTest() { @@ -54,8 +55,9 @@ func (psuite *PluginTestSuite) Upgrade() { } func TestPluginTestSuite(t *testing.T) { + var psuite PluginTestSuite for _, e := range pfiArray { - psuite.pfiEntry := e + psuite.pfiEntry = e suite.Run(t, new(PluginTestSuite)) } } diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index 57ccaea7c58..bf140880524 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -349,10 +349,6 @@ func (psuite *PluginTestSuite) TestPlugins() { } } - initialSchema string - setJSON string - cases []testCase - arg := psuite.pfiEntry pluginFn(arg.initialSchema, arg.setJSON, arg.cases) } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index cbad1a71bad..5e75a2583c1 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -76,7 +76,7 @@ func TestPluginTestSuite(t *testing.T) { var psuite PluginTestSuite psuite.uc = uc for _, e := range pfiArray { - psuite.pfiEntry := e + psuite.pfiEntry = e suite.Run(t, &psuite) } } From e9407993ea18cf31177aeb8b775614a7929dd029 Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Fri, 7 Jul 2023 18:54:59 +0530 Subject: [PATCH 3/7] integration_test.go file logical error fix. --- systest/plugin/integration_test.go | 7 +++---- systest/plugin/plugin_test.go | 11 +++++------ systest/plugin/upgrade_test.go | 9 +-------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go index 82c4f6d7e77..e59999e1705 100644 --- a/systest/plugin/integration_test.go +++ b/systest/plugin/integration_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/dgraph-io/dgo/v230/protos/api" "github.com/dgraph-io/dgraph/dgraphtest" "github.com/dgraph-io/dgraph/x" ) @@ -43,11 +42,11 @@ func (psuite *PluginTestSuite) SetupTest() { func (psuite *PluginTestSuite) TearDownTest() { t := psuite.T() gcli, cleanup, err := psuite.dc.Client() - defer cleanup() require.NoError(t, err) + defer cleanup() require.NoError(t, gcli.LoginIntoNamespace(context.Background(), dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) - require.NoError(t, gcli.Alter(context.Background(), &api.Operation{DropAll: true})) + require.NoError(t, gcli.DropAll()) } func (psuite *PluginTestSuite) Upgrade() { @@ -58,6 +57,6 @@ func TestPluginTestSuite(t *testing.T) { var psuite PluginTestSuite for _, e := range pfiArray { psuite.pfiEntry = e - suite.Run(t, new(PluginTestSuite)) + suite.Run(t, &psuite) } } diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index bf140880524..489f56b143e 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -1,4 +1,4 @@ -//go:build (!oss && integration) || upgrade +//go:build integration || upgrade /* * Copyright 2017-2023 Dgraph Labs, Inc. and Contributors @@ -30,11 +30,12 @@ import ( "github.com/dgraph-io/dgraph/x" ) - +/* type testCase struct { query string wantResult string } +*/ type PluginFuncInp struct { initialSchema string @@ -322,13 +323,11 @@ func (psuite *PluginTestSuite) TestPlugins() { defer cancel() gcli, cleanup, err := psuite.dc.Client() - defer cleanup() require.NoError(t, err) + defer cleanup() require.NoError(t, gcli.LoginIntoNamespace(context.Background(), dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) - require.NoError(t, gcli.Alter(ctx, &api.Operation{ - DropAll: true, - })) + require.NoError(t, gcli.DropAll()) require.NoError(t, gcli.Alter(ctx, &api.Operation{ Schema: initialSchema, })) diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index 5e75a2583c1..7086f40a56c 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -63,15 +63,8 @@ func (psuite *PluginTestSuite) Upgrade() { } } -// For testing purpose -var comboEntry = []dgraphtest.UpgradeCombo{ - {"v21.03.0", "v23.0.0", dgraphtest.BackupRestore}, -} - func TestPluginTestSuite(t *testing.T) { - //for _, uc := range dgraphtest.AllUpgradeCombos { - //} - for _, uc := range comboEntry { + for _, uc := range dgraphtest.AllUpgradeCombos { log.Printf("running: backup in [%v], restore in [%v]", uc.Before, uc.After) var psuite PluginTestSuite psuite.uc = uc From e8f197f78da1c3d203729f9c4c690c6cdf9950e0 Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Fri, 21 Jul 2023 17:22:27 +0530 Subject: [PATCH 4/7] conflict resolution after git rebase --- dgraphtest/config.go | 8 ++++ dgraphtest/image.go | 9 ++--- systest/plugin/integration_test.go | 10 ++--- systest/plugin/plugin_test.go | 64 +++++++++++++++--------------- systest/plugin/upgrade_test.go | 24 ++++++----- 5 files changed, 61 insertions(+), 54 deletions(-) diff --git a/dgraphtest/config.go b/dgraphtest/config.go index 73dd6e04bd4..7abb283071a 100644 --- a/dgraphtest/config.go +++ b/dgraphtest/config.go @@ -116,6 +116,7 @@ type ClusterConfig struct { portOffset int // exposed port offset for grpc/http port for both alpha/zero bulkOutDir string featureFlags []string + customPlugins bool } // NewClusterConfig generates a default ClusterConfig @@ -134,6 +135,7 @@ func NewClusterConfig() ClusterConfig { refillInterval: 20 * time.Second, uidLease: 50, portOffset: -1, + customPlugins: false, } } @@ -225,3 +227,9 @@ func (cc ClusterConfig) WithNormalizeCompatibilityMode(mode string) ClusterConfi cc.featureFlags = append(cc.featureFlags, fmt.Sprintf("normalize-compatibility-mode=%v", mode)) return cc } + +// WithCustomPlugins enables generation of custom plugins in testutil/custom_plugins +func (cc ClusterConfig) WithCustomPlugins() ClusterConfig { + cc.customPlugins = true + return cc +} diff --git a/dgraphtest/image.go b/dgraphtest/image.go index 24898105116..b4821c776e3 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -53,11 +53,10 @@ func (c *LocalCluster) setupBinary() error { if err := buildDgraphBinary(repoDir, binariesPath, c.conf.version); err != nil { return err } - - // explicit var declaration to avoid confusion on the next line - race := false - testutil.GeneratePlugins(race) - + if c.conf.customPlugins { + race := false // Explicit var declaration to avoid confusion on the next line + testutil.GeneratePlugins(race) + } return copyBinary(binariesPath, c.tempBinDir, c.conf.version) } diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go index e59999e1705..843dea293f9 100644 --- a/systest/plugin/integration_test.go +++ b/systest/plugin/integration_test.go @@ -31,8 +31,8 @@ import ( type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - pfiEntry PluginFuncInp + dc dgraphtest.Cluster + dataSetNdx int } func (psuite *PluginTestSuite) SetupTest() { @@ -54,9 +54,5 @@ func (psuite *PluginTestSuite) Upgrade() { } func TestPluginTestSuite(t *testing.T) { - var psuite PluginTestSuite - for _, e := range pfiArray { - psuite.pfiEntry = e - suite.Run(t, &psuite) - } + suite.Run(t, new(PluginTestSuite)) } diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index 489f56b143e..48c6d4a24f3 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -20,6 +20,7 @@ package main import ( "context" + "fmt" "testing" "time" @@ -30,20 +31,16 @@ import ( "github.com/dgraph-io/dgraph/x" ) -/* type testCase struct { query string wantResult string } -*/ -type PluginFuncInp struct { +var testInp = []struct { initialSchema string setJSON string cases []testCase -} - -var pfiArray = []PluginFuncInp{ +}{ { "word: string @index(anagram) .", `[ @@ -318,36 +315,37 @@ func (psuite *PluginTestSuite) TestPlugins() { t.Skip("skipping test in short mode") } - pluginFn := func(initialSchema string, setJSON string, cases []testCase) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() + for i := 0; i < len(testInp); i++ { + psuite.dataSetNdx = i + psuite.Run(fmt.Sprintf("test case %d", i+1), psuite.pluginFn) + } +} - gcli, cleanup, err := psuite.dc.Client() - require.NoError(t, err) - defer cleanup() - require.NoError(t, gcli.LoginIntoNamespace(context.Background(), - dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) - require.NoError(t, gcli.DropAll()) - require.NoError(t, gcli.Alter(ctx, &api.Operation{ - Schema: initialSchema, - })) +func (psuite *PluginTestSuite) pluginFn() { + t := psuite.T() + gcli, cleanup, err := psuite.dc.Client() + require.NoError(t, err) + defer cleanup() + require.NoError(t, gcli.LoginIntoNamespace(context.Background(), + dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) + require.NoError(t, gcli.DropAll()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + fmt.Printf("\ndataSetNdx = %d\n", psuite.dataSetNdx) + fmt.Printf("\ntestInp[psuite.dataSetNdx].initialSchema = [%s]\n", testInp[psuite.dataSetNdx].initialSchema) + require.NoError(t, gcli.Alter(ctx, &api.Operation{ + Schema: testInp[psuite.dataSetNdx].initialSchema, + })) + + txn := gcli.NewTxn() + _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[psuite.dataSetNdx].setJSON)}) + require.NoError(t, err) + require.NoError(t, txn.Commit(ctx)) + for _, test := range testInp[psuite.dataSetNdx].cases { txn := gcli.NewTxn() - _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(setJSON)}) + reply, err := txn.Query(ctx, test.query) require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) - - // Upgrade - psuite.Upgrade() - - for _, test := range cases { - txn := gcli.NewTxn() - reply, err := txn.Query(ctx, test.query) - require.NoError(t, err) - dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) - } + dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) } - - arg := psuite.pfiEntry - pluginFn(arg.initialSchema, arg.setJSON, arg.cases) } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index 7086f40a56c..281ce78e68c 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -31,15 +31,21 @@ import ( type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - lc *dgraphtest.LocalCluster - uc dgraphtest.UpgradeCombo - pfiEntry PluginFuncInp + dc dgraphtest.Cluster + lc *dgraphtest.LocalCluster + uc dgraphtest.UpgradeCombo + dataSetNdx int } func (psuite *PluginTestSuite) SetupTest() { +} + +func (psuite *PluginTestSuite) SetupSubTest() { + psuite.lc.Cleanup(psuite.T().Failed()) + conf := dgraphtest.NewClusterConfig().WithNumAlphas(1).WithNumZeros(1).WithReplicas(1). - WithACL(20 * time.Second).WithEncryption().WithVersion(psuite.uc.Before) + WithACL(20 * time.Second).WithEncryption().WithVersion(psuite.uc.Before). + WithCustomPlugins() c, err := dgraphtest.NewLocalCluster(conf) x.Panic(err) if err := c.Start(); err != nil { @@ -64,13 +70,13 @@ func (psuite *PluginTestSuite) Upgrade() { } func TestPluginTestSuite(t *testing.T) { - for _, uc := range dgraphtest.AllUpgradeCombos { + for _, uc := range dgraphtest.AllUpgradeCombos() { log.Printf("running: backup in [%v], restore in [%v]", uc.Before, uc.After) var psuite PluginTestSuite psuite.uc = uc - for _, e := range pfiArray { - psuite.pfiEntry = e - suite.Run(t, &psuite) + suite.Run(t, &psuite) + if t.Failed() { + t.Fatal("TestPluginTestSuite tests failed") } } } From 77a3e3d0ca59199cf9731d5f92ff9c6dc8948690 Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Wed, 26 Jul 2023 17:22:01 +0530 Subject: [PATCH 5/7] Review comment changes --- dgraphtest/config.go | 2 +- dgraphtest/dgraph.go | 7 +++- dgraphtest/image.go | 11 +++--- dgraphtest/local_cluster.go | 48 ++++++++++++++++++++++++- systest/plugin/integration_test.go | 7 +--- systest/plugin/plugin_test.go | 57 +++++++++++++++--------------- systest/plugin/upgrade_test.go | 23 +++++------- 7 files changed, 97 insertions(+), 58 deletions(-) diff --git a/dgraphtest/config.go b/dgraphtest/config.go index 7abb283071a..57a1af0957b 100644 --- a/dgraphtest/config.go +++ b/dgraphtest/config.go @@ -228,7 +228,7 @@ func (cc ClusterConfig) WithNormalizeCompatibilityMode(mode string) ClusterConfi return cc } -// WithCustomPlugins enables generation of custom plugins in testutil/custom_plugins +// Enables generation of the custom_plugins in testutil/custom_plugins func (cc ClusterConfig) WithCustomPlugins() ClusterConfig { cc.customPlugins = true return cc diff --git a/dgraphtest/dgraph.go b/dgraphtest/dgraph.go index 3dfb4aa30f4..80d615e4043 100644 --- a/dgraphtest/dgraph.go +++ b/dgraphtest/dgraph.go @@ -60,6 +60,7 @@ const ( DefaultUser = "groot" DefaultPassword = "password" + goBinMountPath = "/gobin" localVersion = "local" waitDurBeforeRetry = time.Second requestTimeout = 120 * time.Second @@ -278,6 +279,10 @@ func (a *alpha) cmd(c *LocalCluster) []string { acmd = append(acmd, fmt.Sprintf("--feature-flags=%v", strings.Join(c.conf.featureFlags, ";"))) } + if c.conf.customPlugins { + acmd = append(acmd, fmt.Sprintf("--custom_tokenizers=%s", c.customTokenizers)) + } + return acmd } @@ -385,7 +390,7 @@ func mountBinary(c *LocalCluster) (mount.Mount, error) { return mount.Mount{ Type: mount.TypeBind, Source: c.tempBinDir, - Target: "/gobin", + Target: goBinMountPath, ReadOnly: true, }, nil } diff --git a/dgraphtest/image.go b/dgraphtest/image.go index b4821c776e3..06261fe8404 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -25,7 +25,6 @@ import ( "path/filepath" "strings" - "github.com/dgraph-io/dgraph/testutil" "github.com/pkg/errors" ) @@ -34,6 +33,12 @@ func (c *LocalCluster) dgraphImage() string { } func (c *LocalCluster) setupBinary() error { + if c.conf.customPlugins { + race := false // Explicit var declaration to avoid confusion on the next line + if err := c.GeneratePlugins(race); err != nil { + return err + } + } if c.conf.version == localVersion { fromDir := filepath.Join(os.Getenv("GOPATH"), "bin") return copyBinary(fromDir, c.tempBinDir, c.conf.version) @@ -53,10 +58,6 @@ func (c *LocalCluster) setupBinary() error { if err := buildDgraphBinary(repoDir, binariesPath, c.conf.version); err != nil { return err } - if c.conf.customPlugins { - race := false // Explicit var declaration to avoid confusion on the next line - testutil.GeneratePlugins(race) - } return copyBinary(binariesPath, c.tempBinDir, c.conf.version) } diff --git a/dgraphtest/local_cluster.go b/dgraphtest/local_cluster.go index c0859796d91..082191ce1f2 100644 --- a/dgraphtest/local_cluster.go +++ b/dgraphtest/local_cluster.go @@ -26,6 +26,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -57,7 +58,9 @@ type LocalCluster struct { tempBinDir string tempSecretsDir string encKeyPath string - lowerThanV21 bool + + lowerThanV21 bool + customTokenizers string // resources dcli *docker.Client @@ -976,3 +979,46 @@ func runOpennssl(args ...string) error { } return nil } + +func (c *LocalCluster) GeneratePlugins(raceEnabled bool) error { + _, curr, _, ok := runtime.Caller(0) + if !ok { + return errors.New("error while getting current file") + } + var soFiles []string + for i, src := range []string{ + "../testutil/custom_plugins/anagram/main.go", + "../testutil/custom_plugins/cidr/main.go", + "../testutil/custom_plugins/factor/main.go", + "../testutil/custom_plugins/rune/main.go", + } { + so := c.tempBinDir + "/plugins/" + strconv.Itoa(i) + ".so" + log.Printf("compiling plugin: src=%q so=%q\n", src, so) + opts := []string{"build"} + if raceEnabled { + opts = append(opts, "-race") + } + opts = append(opts, "-buildmode=plugin", "-o", so, src) + os.Setenv("GOOS", "linux") + os.Setenv("GOARCH", "amd64") + cmd := exec.Command("go", opts...) + cmd.Dir = filepath.Dir(curr) + if out, err := cmd.CombinedOutput(); err != nil { + log.Printf("Error: %v\n", err) + log.Printf("Output: %v\n", string(out)) + return err + } + absSO, err := filepath.Abs(so) + if err != nil { + log.Printf("Error: %v\n", err) + return err + } + soFiles = append(soFiles, absSO) + } + + sofiles := strings.Join(soFiles, ",") + c.customTokenizers = strings.ReplaceAll(sofiles, c.tempBinDir, goBinMountPath) + log.Printf("plugin build completed. Files are: %s\n", sofiles) + + return nil +} diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go index 843dea293f9..267ac4c00bc 100644 --- a/systest/plugin/integration_test.go +++ b/systest/plugin/integration_test.go @@ -19,20 +19,17 @@ package main import ( - "context" "testing" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dgraph-io/dgraph/dgraphtest" - "github.com/dgraph-io/dgraph/x" ) type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - dataSetNdx int + dc dgraphtest.Cluster } func (psuite *PluginTestSuite) SetupTest() { @@ -44,8 +41,6 @@ func (psuite *PluginTestSuite) TearDownTest() { gcli, cleanup, err := psuite.dc.Client() require.NoError(t, err) defer cleanup() - require.NoError(t, gcli.LoginIntoNamespace(context.Background(), - dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) require.NoError(t, gcli.DropAll()) } diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index 48c6d4a24f3..8b207d33893 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -28,7 +28,6 @@ import ( "github.com/dgraph-io/dgo/v230/protos/api" "github.com/dgraph-io/dgraph/dgraphtest" - "github.com/dgraph-io/dgraph/x" ) type testCase struct { @@ -316,36 +315,36 @@ func (psuite *PluginTestSuite) TestPlugins() { } for i := 0; i < len(testInp); i++ { - psuite.dataSetNdx = i - psuite.Run(fmt.Sprintf("test case %d", i+1), psuite.pluginFn) - } -} + psuite.Run(fmt.Sprintf("test case %d", i+1), func() { + t := psuite.T() + gcli, cleanup, err := psuite.dc.Client() + require.NoError(t, err) + defer cleanup() + require.NoError(t, gcli.DropAll()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + require.NoError(t, gcli.Alter(ctx, &api.Operation{ + Schema: testInp[i].initialSchema, + })) -func (psuite *PluginTestSuite) pluginFn() { - t := psuite.T() - gcli, cleanup, err := psuite.dc.Client() - require.NoError(t, err) - defer cleanup() - require.NoError(t, gcli.LoginIntoNamespace(context.Background(), - dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) - require.NoError(t, gcli.DropAll()) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - fmt.Printf("\ndataSetNdx = %d\n", psuite.dataSetNdx) - fmt.Printf("\ntestInp[psuite.dataSetNdx].initialSchema = [%s]\n", testInp[psuite.dataSetNdx].initialSchema) - require.NoError(t, gcli.Alter(ctx, &api.Operation{ - Schema: testInp[psuite.dataSetNdx].initialSchema, - })) + txn := gcli.NewTxn() + _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[i].setJSON)}) + require.NoError(t, err) + require.NoError(t, txn.Commit(ctx)) + + // Upgrade + psuite.Upgrade() - txn := gcli.NewTxn() - _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[psuite.dataSetNdx].setJSON)}) - require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) + gcli, cleanup, err = psuite.dc.Client() + require.NoError(t, err) + defer cleanup() - for _, test := range testInp[psuite.dataSetNdx].cases { - txn := gcli.NewTxn() - reply, err := txn.Query(ctx, test.query) - require.NoError(t, err) - dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) + for _, test := range testInp[i].cases { + txn := gcli.NewTxn() + reply, err := txn.Query(ctx, test.query) + require.NoError(t, err) + dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) + } + }) } } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index 281ce78e68c..f2ed4a498a0 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -21,8 +21,8 @@ package main import ( "log" "testing" - "time" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dgraph-io/dgraph/dgraphtest" @@ -31,21 +31,18 @@ import ( type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - lc *dgraphtest.LocalCluster - uc dgraphtest.UpgradeCombo - dataSetNdx int -} - -func (psuite *PluginTestSuite) SetupTest() { + dc dgraphtest.Cluster + lc *dgraphtest.LocalCluster + uc dgraphtest.UpgradeCombo } func (psuite *PluginTestSuite) SetupSubTest() { + // The TestPlugins() invokes subtest function, hence using + // SetupSubTest() instead of SetupTest(). psuite.lc.Cleanup(psuite.T().Failed()) conf := dgraphtest.NewClusterConfig().WithNumAlphas(1).WithNumZeros(1).WithReplicas(1). - WithACL(20 * time.Second).WithEncryption().WithVersion(psuite.uc.Before). - WithCustomPlugins() + WithVersion(psuite.uc.Before).WithCustomPlugins() c, err := dgraphtest.NewLocalCluster(conf) x.Panic(err) if err := c.Start(); err != nil { @@ -62,11 +59,7 @@ func (psuite *PluginTestSuite) TearDownTest() { } func (psuite *PluginTestSuite) Upgrade() { - t := psuite.T() - - if err := psuite.lc.Upgrade(psuite.uc.After, psuite.uc.Strategy); err != nil { - t.Fatal(err) - } + require.NoError(psuite.T(), psuite.lc.Upgrade(psuite.uc.After, psuite.uc.Strategy)) } func TestPluginTestSuite(t *testing.T) { From 7c3b33cd2cf925edc24f074e5ce5ff5f35e524f6 Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Sun, 20 Aug 2023 11:54:39 +0530 Subject: [PATCH 6/7] Added alpha container kill and more cleanup logic. --- dgraphtest/local_cluster.go | 8 +++++++- systest/plugin/plugin_test.go | 26 ++++++-------------------- systest/plugin/upgrade_test.go | 5 ++--- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/dgraphtest/local_cluster.go b/dgraphtest/local_cluster.go index 082191ce1f2..80188de1ed3 100644 --- a/dgraphtest/local_cluster.go +++ b/dgraphtest/local_cluster.go @@ -387,9 +387,15 @@ func (c *LocalCluster) StopAlpha(id int) error { func (c *LocalCluster) stopContainer(dc dnode) error { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) defer cancel() - stopTimeout := 30 // in seconds + + stopTimeout := 30 o := container.StopOptions{Timeout: &stopTimeout} if err := c.dcli.ContainerStop(ctx, dc.cid(), o); err != nil { + // Force kill the container if timeout exceeded + if strings.Contains(err.Error(), "context deadline exceeded") { + _ = c.dcli.ContainerKill(ctx, dc.cid(), "KILL") + return nil + } return errors.Wrapf(err, "error stopping container [%v]", dc.cname()) } return nil diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index 8b207d33893..cfc1891e625 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -19,10 +19,7 @@ package main import ( - "context" "fmt" - "testing" - "time" "github.com/stretchr/testify/require" @@ -308,12 +305,6 @@ var testInp = []struct { } func (psuite *PluginTestSuite) TestPlugins() { - t := psuite.T() - - if testing.Short() { - t.Skip("skipping test in short mode") - } - for i := 0; i < len(testInp); i++ { psuite.Run(fmt.Sprintf("test case %d", i+1), func() { t := psuite.T() @@ -321,16 +312,12 @@ func (psuite *PluginTestSuite) TestPlugins() { require.NoError(t, err) defer cleanup() require.NoError(t, gcli.DropAll()) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - require.NoError(t, gcli.Alter(ctx, &api.Operation{ - Schema: testInp[i].initialSchema, - })) - - txn := gcli.NewTxn() - _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[i].setJSON)}) + require.NoError(t, gcli.SetupSchema(testInp[i].initialSchema)) + _, err = gcli.Mutate(&api.Mutation{ + SetJson: []byte(testInp[i].setJSON), + CommitNow: true, + }) require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) // Upgrade psuite.Upgrade() @@ -340,8 +327,7 @@ func (psuite *PluginTestSuite) TestPlugins() { defer cleanup() for _, test := range testInp[i].cases { - txn := gcli.NewTxn() - reply, err := txn.Query(ctx, test.query) + reply, err := gcli.Query(test.query) require.NoError(t, err) dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index f2ed4a498a0..14b23c539ea 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -19,6 +19,7 @@ package main import ( + "errors" "log" "testing" @@ -39,8 +40,6 @@ type PluginTestSuite struct { func (psuite *PluginTestSuite) SetupSubTest() { // The TestPlugins() invokes subtest function, hence using // SetupSubTest() instead of SetupTest(). - psuite.lc.Cleanup(psuite.T().Failed()) - conf := dgraphtest.NewClusterConfig().WithNumAlphas(1).WithNumZeros(1).WithReplicas(1). WithVersion(psuite.uc.Before).WithCustomPlugins() c, err := dgraphtest.NewLocalCluster(conf) @@ -69,7 +68,7 @@ func TestPluginTestSuite(t *testing.T) { psuite.uc = uc suite.Run(t, &psuite) if t.Failed() { - t.Fatal("TestPluginTestSuite tests failed") + x.Panic(errors.New("TestPluginTestSuite tests failed")) } } } From 20936aa51800a3e70a73cd66117e47d9cc7118fc Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Wed, 23 Aug 2023 13:34:02 +0530 Subject: [PATCH 7/7] decreased timeout value from 120sec to 30sec for ContainerStop() and ContainerKill() called from it --- systest/plugin/upgrade_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index 14b23c539ea..4ff2d59ec0a 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -62,7 +62,7 @@ func (psuite *PluginTestSuite) Upgrade() { } func TestPluginTestSuite(t *testing.T) { - for _, uc := range dgraphtest.AllUpgradeCombos() { + for _, uc := range dgraphtest.AllUpgradeCombos(false) { log.Printf("running: backup in [%v], restore in [%v]", uc.Before, uc.After) var psuite PluginTestSuite psuite.uc = uc