From a25dd3f8c14483640ea37509d42e5f1c180d77fa Mon Sep 17 00:00:00 2001 From: JASWINDER BHAMRA Date: Wed, 26 Jul 2023 17:22:01 +0530 Subject: [PATCH] Review comment changes --- dgraphtest/config.go | 2 +- dgraphtest/dgraph.go | 7 +++- dgraphtest/image.go | 7 +++- dgraphtest/local_cluster.go | 50 ++++++++++++++++++++++++-- systest/plugin/integration_test.go | 7 +--- systest/plugin/plugin_test.go | 57 +++++++++++++++--------------- systest/plugin/upgrade_test.go | 23 +++++------- 7 files changed, 98 insertions(+), 55 deletions(-) diff --git a/dgraphtest/config.go b/dgraphtest/config.go index 8a70a8e7c6b..33efaa78454 100644 --- a/dgraphtest/config.go +++ b/dgraphtest/config.go @@ -197,7 +197,7 @@ func (cc ClusterConfig) WithNormalizeCompatibilityMode(mode string) ClusterConfi return cc } -// WithCustomPlugins enables generation of custom plugins in testutil/custom_plugins +// Enables generation of the custom_plugins in testutil/custom_plugins func (cc ClusterConfig) WithCustomPlugins() ClusterConfig { cc.customPlugins = true return cc diff --git a/dgraphtest/dgraph.go b/dgraphtest/dgraph.go index 68c616915fb..58246f37c22 100644 --- a/dgraphtest/dgraph.go +++ b/dgraphtest/dgraph.go @@ -51,6 +51,7 @@ const ( DefaultBackupDir = "/data/backups" DefaultExportDir = "/data/exports" + goBinMountPath = "/gobin" aclSecretMountPath = "/dgraph-acl/hmac-secret" encKeyMountPath = "/dgraph-enc/enc-key" @@ -251,6 +252,10 @@ func (a *alpha) cmd(c *LocalCluster) []string { acmd = append(acmd, fmt.Sprintf("--feature-flags=%v", strings.Join(c.conf.featureFlags, ";"))) } + if c.conf.customPlugins { + acmd = append(acmd, fmt.Sprintf("--custom_tokenizers=%s", c.customTokenizers)) + } + return acmd } @@ -361,7 +366,7 @@ func mountBinary(c *LocalCluster) (mount.Mount, error) { return mount.Mount{ Type: mount.TypeBind, Source: c.tempBinDir, - Target: "/gobin", + Target: goBinMountPath, ReadOnly: true, }, nil } diff --git a/dgraphtest/image.go b/dgraphtest/image.go index 8ca9e00b771..1a4f832812f 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -25,8 +25,9 @@ import ( "path/filepath" "strings" - "github.com/dgraph-io/dgraph/testutil" "github.com/pkg/errors" + + "github.com/dgraph-io/dgraph/testutil" ) func (c *LocalCluster) dgraphImage() string { @@ -34,6 +35,10 @@ func (c *LocalCluster) dgraphImage() string { } func (c *LocalCluster) setupBinary() error { + if c.conf.customPlugins { + race := false // Explicit var declaration to avoid confusion on the next line + c.GeneratePlugins(race) + } if c.conf.version == localVersion { fromDir := filepath.Join(os.Getenv("GOPATH"), "bin") return copyBinary(fromDir, c.tempBinDir, c.conf.version) diff --git a/dgraphtest/local_cluster.go b/dgraphtest/local_cluster.go index 70797afc2c3..a0c6993082c 100644 --- a/dgraphtest/local_cluster.go +++ b/dgraphtest/local_cluster.go @@ -24,6 +24,10 @@ import ( "log" "net/http" "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" "strings" "time" @@ -49,8 +53,9 @@ type cnet struct { // LocalCluster is a local dgraph cluster type LocalCluster struct { - conf ClusterConfig - tempBinDir string + conf ClusterConfig + tempBinDir string + customTokenizers string // resources dcli *docker.Client @@ -864,3 +869,44 @@ func (c *LocalCluster) inspectContainer(containerID string) (string, error) { } return string(raw), nil } + +func (c *LocalCluster) GeneratePlugins(raceEnabled bool) { + _, curr, _, ok := runtime.Caller(0) + if !ok { + fmt.Print("error while getting current file") + return + } + var soFiles []string + for i, src := range []string{ + "../testutil/custom_plugins/anagram/main.go", + "../testutil/custom_plugins/cidr/main.go", + "../testutil/custom_plugins/factor/main.go", + "../testutil/custom_plugins/rune/main.go", + } { + so := c.tempBinDir + "/plugins/" + strconv.Itoa(i) + ".so" + fmt.Printf("compiling plugin: src=%q so=%q\n", src, so) + opts := []string{"build"} + if raceEnabled { + opts = append(opts, "-race") + } + opts = append(opts, "-buildmode=plugin", "-o", so, src) + os.Setenv("GOOS", "linux") + os.Setenv("GOARCH", "amd64") + cmd := exec.Command("go", opts...) + cmd.Dir = filepath.Dir(curr) + if out, err := cmd.CombinedOutput(); err != nil { + fmt.Printf("Error: %v\n", err) + fmt.Printf("Output: %v\n", string(out)) + return + } + absSO, err := filepath.Abs(so) + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + soFiles = append(soFiles, absSO) + } + + c.customTokenizers = strings.ReplaceAll(strings.Join(soFiles, ","), c.tempBinDir, goBinMountPath) + fmt.Printf("plugin build completed. Files are: %s\n", c.customTokenizers) +} diff --git a/systest/plugin/integration_test.go b/systest/plugin/integration_test.go index 843dea293f9..267ac4c00bc 100644 --- a/systest/plugin/integration_test.go +++ b/systest/plugin/integration_test.go @@ -19,20 +19,17 @@ package main import ( - "context" "testing" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dgraph-io/dgraph/dgraphtest" - "github.com/dgraph-io/dgraph/x" ) type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - dataSetNdx int + dc dgraphtest.Cluster } func (psuite *PluginTestSuite) SetupTest() { @@ -44,8 +41,6 @@ func (psuite *PluginTestSuite) TearDownTest() { gcli, cleanup, err := psuite.dc.Client() require.NoError(t, err) defer cleanup() - require.NoError(t, gcli.LoginIntoNamespace(context.Background(), - dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) require.NoError(t, gcli.DropAll()) } diff --git a/systest/plugin/plugin_test.go b/systest/plugin/plugin_test.go index 48c6d4a24f3..8b207d33893 100644 --- a/systest/plugin/plugin_test.go +++ b/systest/plugin/plugin_test.go @@ -28,7 +28,6 @@ import ( "github.com/dgraph-io/dgo/v230/protos/api" "github.com/dgraph-io/dgraph/dgraphtest" - "github.com/dgraph-io/dgraph/x" ) type testCase struct { @@ -316,36 +315,36 @@ func (psuite *PluginTestSuite) TestPlugins() { } for i := 0; i < len(testInp); i++ { - psuite.dataSetNdx = i - psuite.Run(fmt.Sprintf("test case %d", i+1), psuite.pluginFn) - } -} + psuite.Run(fmt.Sprintf("test case %d", i+1), func() { + t := psuite.T() + gcli, cleanup, err := psuite.dc.Client() + require.NoError(t, err) + defer cleanup() + require.NoError(t, gcli.DropAll()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + require.NoError(t, gcli.Alter(ctx, &api.Operation{ + Schema: testInp[i].initialSchema, + })) -func (psuite *PluginTestSuite) pluginFn() { - t := psuite.T() - gcli, cleanup, err := psuite.dc.Client() - require.NoError(t, err) - defer cleanup() - require.NoError(t, gcli.LoginIntoNamespace(context.Background(), - dgraphtest.DefaultUser, dgraphtest.DefaultPassword, x.GalaxyNamespace)) - require.NoError(t, gcli.DropAll()) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - fmt.Printf("\ndataSetNdx = %d\n", psuite.dataSetNdx) - fmt.Printf("\ntestInp[psuite.dataSetNdx].initialSchema = [%s]\n", testInp[psuite.dataSetNdx].initialSchema) - require.NoError(t, gcli.Alter(ctx, &api.Operation{ - Schema: testInp[psuite.dataSetNdx].initialSchema, - })) + txn := gcli.NewTxn() + _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[i].setJSON)}) + require.NoError(t, err) + require.NoError(t, txn.Commit(ctx)) + + // Upgrade + psuite.Upgrade() - txn := gcli.NewTxn() - _, err = txn.Mutate(ctx, &api.Mutation{SetJson: []byte(testInp[psuite.dataSetNdx].setJSON)}) - require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) + gcli, cleanup, err = psuite.dc.Client() + require.NoError(t, err) + defer cleanup() - for _, test := range testInp[psuite.dataSetNdx].cases { - txn := gcli.NewTxn() - reply, err := txn.Query(ctx, test.query) - require.NoError(t, err) - dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) + for _, test := range testInp[i].cases { + txn := gcli.NewTxn() + reply, err := txn.Query(ctx, test.query) + require.NoError(t, err) + dgraphtest.CompareJSON(test.wantResult, string(reply.GetJson())) + } + }) } } diff --git a/systest/plugin/upgrade_test.go b/systest/plugin/upgrade_test.go index 281ce78e68c..f2ed4a498a0 100644 --- a/systest/plugin/upgrade_test.go +++ b/systest/plugin/upgrade_test.go @@ -21,8 +21,8 @@ package main import ( "log" "testing" - "time" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dgraph-io/dgraph/dgraphtest" @@ -31,21 +31,18 @@ import ( type PluginTestSuite struct { suite.Suite - dc dgraphtest.Cluster - lc *dgraphtest.LocalCluster - uc dgraphtest.UpgradeCombo - dataSetNdx int -} - -func (psuite *PluginTestSuite) SetupTest() { + dc dgraphtest.Cluster + lc *dgraphtest.LocalCluster + uc dgraphtest.UpgradeCombo } func (psuite *PluginTestSuite) SetupSubTest() { + // The TestPlugins() invokes subtest function, hence using + // SetupSubTest() instead of SetupTest(). psuite.lc.Cleanup(psuite.T().Failed()) conf := dgraphtest.NewClusterConfig().WithNumAlphas(1).WithNumZeros(1).WithReplicas(1). - WithACL(20 * time.Second).WithEncryption().WithVersion(psuite.uc.Before). - WithCustomPlugins() + WithVersion(psuite.uc.Before).WithCustomPlugins() c, err := dgraphtest.NewLocalCluster(conf) x.Panic(err) if err := c.Start(); err != nil { @@ -62,11 +59,7 @@ func (psuite *PluginTestSuite) TearDownTest() { } func (psuite *PluginTestSuite) Upgrade() { - t := psuite.T() - - if err := psuite.lc.Upgrade(psuite.uc.After, psuite.uc.Strategy); err != nil { - t.Fatal(err) - } + require.NoError(psuite.T(), psuite.lc.Upgrade(psuite.uc.After, psuite.uc.Strategy)) } func TestPluginTestSuite(t *testing.T) {