test(cli): validate unit test requests against schema

cli/go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/rs/cors v1.10.1
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
+	github.com/vektah/gqlparser v1.3.1
 	github.com/vektah/gqlparser/v2 v2.5.11
 	github.com/zalando/go-keyring v0.2.3
 	gorm.io/gorm v1.25.6

cli/go.sum

@@ -12,6 +12,7 @@ github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VP
 github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
@@ -113,6 +114,7 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
 github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
 github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sosodev/duration v1.2.0 h1:pqK/FLSjsAADWY74SyWDCjOcd5l7H8GSnnOGEB9A1Us=
@@ -126,12 +128,15 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU=
+github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74=
 github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8=
 github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -184,12 +189,14 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

cli/internal/gql/app/create_test.go

@@ -28,7 +28,7 @@ func TestCreate(t *testing.T) {
 			CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
 		}
 		response := test.AppToQueryResult("toolCreate", expectedApp)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualApp, err := Create(testApp, c)
 
@@ -38,7 +38,7 @@ func TestCreate(t *testing.T) {
 
 	t.Run("can return error on AppCreate mutation", func(t *testing.T) {
 		appNotFoundResponse := `{"errors":[{"message":"Something went wrong","path":["toolCreate"]}],"data":null}`
-		c := test.CreateTestGqlClient(appNotFoundResponse)
+		c := test.CreateTestGqlClient(t, appNotFoundResponse)
 
 		actualApp, err := Create(testApp, c)
 

cli/internal/gql/app/delete_test.go

@@ -11,7 +11,7 @@ import (
 func TestDelete(t *testing.T) {
 	t.Run("can return status on AppDelete mutation", func(t *testing.T) {
 		response, expectedType := test.DeleteSuccessQueryResult()
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualStatus, err := Delete("id", c)
 
@@ -22,7 +22,7 @@ func TestDelete(t *testing.T) {
 	t.Run("can return error on AppDelete mutation", func(t *testing.T) {
 		appNotFoundResponse := "record not found"
 		response, expectedType := test.DeleteFailureQueryResult(appNotFoundResponse)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 		actualStatus, _ := Delete("id", c)
 		assert.Equal(t, expectedType, actualStatus.ToolDelete.Typename)
 		assert.Equal(t, appNotFoundResponse, actualStatus.ToolDelete.Result)

cli/internal/gql/app/publish_test.go

@@ -19,7 +19,7 @@ func TestPublish(t *testing.T) {
 			CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
 		}
 		response := test.AppToQueryResult("toolPublish", expectedApp)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualApp, err := Publish(expectedApp.ID, c)
 
@@ -29,7 +29,7 @@ func TestPublish(t *testing.T) {
 
 	t.Run("can return error on AppPublish mutation", func(t *testing.T) {
 		appNotFoundResponse := `{"errors":[{"message":"record not found","path":["tool"]}],"data":null}`
-		c := test.CreateTestGqlClient(appNotFoundResponse)
+		c := test.CreateTestGqlClient(t, appNotFoundResponse)
 
 		actualApp, err := Publish("id", c)
 

cli/internal/gql/app/query_list_test.go

@@ -33,7 +33,7 @@ func TestQueryList(t *testing.T) {
 				"tools": [` + strings.Join(appsAsStrings, ", ") + `]
 			}
 		}`
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualApps, err := QueryList(c)
 
@@ -43,7 +43,7 @@ func TestQueryList(t *testing.T) {
 
 	t.Run("can return permission denied error", func(t *testing.T) {
 		appNotFoundResponse := `{"errors":[{"message":"permission denied","path":["tools"]}],"data":null}`
-		c := test.CreateTestGqlClient(appNotFoundResponse)
+		c := test.CreateTestGqlClient(t, appNotFoundResponse)
 
 		actualApps, err := QueryList(c)
 

cli/internal/gql/app/query_test.go

@@ -19,7 +19,7 @@ func TestQuery(t *testing.T) {
 			CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
 		}
 		response := test.AppToQueryResult("tool", expectedApp)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualApp, err := Query("id", c)
 
@@ -29,7 +29,7 @@ func TestQuery(t *testing.T) {
 
 	t.Run("can return error on app query", func(t *testing.T) {
 		appNotFoundResponse := `{"errors":[{"message":"record not found","path":["tool"]}],"data":null}`
-		c := test.CreateTestGqlClient(appNotFoundResponse)
+		c := test.CreateTestGqlClient(t, appNotFoundResponse)
 
 		actualApp, err := Query("non-existing-id", c)
 

cli/internal/gql/app/unpublish_test.go

@@ -19,7 +19,7 @@ func TestUnpublish(t *testing.T) {
 			CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
 		}
 		response := test.AppToQueryResult("toolUnpublish", expectedApp)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualApp, err := Unpublish(expectedApp.ID, c)
 
@@ -29,7 +29,7 @@ func TestUnpublish(t *testing.T) {
 
 	t.Run("can return error on AppUnpublish mutation", func(t *testing.T) {
 		appNotFoundResponse := `{"errors":[{"message":"record not found","path":["toolUnpublish"]}],"data":null}`
-		c := test.CreateTestGqlClient(appNotFoundResponse)
+		c := test.CreateTestGqlClient(t, appNotFoundResponse)
 
 		actualApp, err := Unpublish("id", c)
 

cli/internal/gql/organization/create_test.go

@@ -17,7 +17,7 @@ func TestCreate(t *testing.T) {
 			Slug: "slug",
 		}
 		response := organizationToQueryResult("organizationCreate", expectedOrganization)
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualOrganization, err := Create(expectedOrganization.Name, c)
 
@@ -27,7 +27,7 @@ func TestCreate(t *testing.T) {
 
 	t.Run("can return error on OrganizationCreate mutation if name contains invalid character", func(t *testing.T) {
 		organizationInvalidCharactersResponse := `{"errors":[{"message":"organization name contains invalid characters","path":["organizationCreate"]}],"data":null}`
-		c := test.CreateTestGqlClient(organizationInvalidCharactersResponse)
+		c := test.CreateTestGqlClient(t, organizationInvalidCharactersResponse)
 
 		actualOrganization, err := Create("!", c)
 
@@ -38,7 +38,7 @@ func TestCreate(t *testing.T) {
 
 	t.Run("can return error on OrganizationCreate mutation if unknown error", func(t *testing.T) {
 		organizationErrorResponse := `{"errors":[{"message":"unknown error","path":["organizationCreate"]}],"data":null}`
-		c := test.CreateTestGqlClient(organizationErrorResponse)
+		c := test.CreateTestGqlClient(t, organizationErrorResponse)
 
 		actualOrganization, err := Create("", c)
 

cli/internal/gql/user/query_user_test.go

@@ -1,6 +1,7 @@
 package user
 
 import (
+	"os"
 	"testing"
 
 	"numerous/cli/internal/gql/organization"
@@ -11,6 +12,8 @@ import (
 
 func TestQueryUser(t *testing.T) {
 	t.Run("can return user on user query", func(t *testing.T) {
+		wd, err := os.Getwd()
+		println(wd, err)
 		membership := organization.OrganizationMembership{
 			Role: organization.Admin,
 			Organization: organization.Organization{
@@ -43,7 +46,7 @@ func TestQueryUser(t *testing.T) {
 				}
 			}
 		}`
-		c := test.CreateTestGqlClient(response)
+		c := test.CreateTestGqlClient(t, response)
 
 		actualUser, err := QueryUser(c)
 
@@ -53,7 +56,7 @@ func TestQueryUser(t *testing.T) {
 
 	t.Run("can return permission denied error", func(t *testing.T) {
 		userNotFoundResponse := `{"errors":[{"message":"permission denied","path":["me"]}],"data":null}`
-		c := test.CreateTestGqlClient(userNotFoundResponse)
+		c := test.CreateTestGqlClient(t, userNotFoundResponse)
 
 		actualUser, err := QueryUser(c)
 

cli/test/gql_client.go

@@ -2,14 +2,30 @@ package test
 
 import (
 	"bytes"
+	"encoding/json"
 	"io"
 	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
 
 	"git.sr.ht/~emersion/gqlclient"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	"github.com/vektah/gqlparser/v2"
+	"github.com/vektah/gqlparser/v2/ast"
+	"github.com/vektah/gqlparser/v2/gqlerror"
+	"github.com/vektah/gqlparser/v2/parser"
+	"github.com/vektah/gqlparser/v2/validator"
+
+	_ "embed"
 )
 
-func CreateTestGqlClient(response string) *gqlclient.Client {
+func CreateTestGqlClient(t *testing.T, response string) *gqlclient.Client {
+	t.Helper()
+
+	schema := loadSchema(t)
 	h := http.Header{}
 	h.Add("Content-Type", "application/json")
 
@@ -19,6 +35,17 @@ func CreateTestGqlClient(response string) *gqlclient.Client {
 			StatusCode: http.StatusOK,
 			Body:       io.NopCloser(bytes.NewReader([]byte(response))),
 		},
+		Handler: func(r *http.Request) *struct {
+			Response *http.Response
+			Error    error
+		} {
+			query, err := io.ReadAll(r.Body)
+			require.NoError(t, err)
+			doc := parseQuery(t, string(query))
+			validateQuery(t, schema, doc)
+
+			return nil
+		},
 	}
 
 	return gqlclient.New("http://localhost:8080", &http.Client{Transport: &ts})
@@ -47,3 +74,72 @@ func AddResponseToMockGqlClient(response string, ts *MockTransport) {
 		nil,
 	)
 }
+
+func loadSchema(t *testing.T) *ast.Schema {
+	t.Helper()
+
+	dots := findSchemaFilePath(t)
+
+	content, err := os.ReadFile(dots + "shared/schema.gql")
+	require.NoError(t, err)
+
+	schema, err := gqlparser.LoadSchema(&ast.Source{
+		Name:  "schema.gql",
+		Input: string(content),
+	})
+	require.NoError(t, err)
+
+	return schema
+}
+
+var schemaRelative = "/shared/schema.gql"
+
+func findSchemaFilePath(t *testing.T) string {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+	dots := ""
+	for {
+		if _, err := os.Stat(wd + schemaRelative); err == nil {
+			break
+		}
+
+		dir, _ := filepath.Split(wd)
+		wd = filepath.Clean(dir)
+		require.NotEmpty(t, wd)
+		require.NotEqual(t, "/", wd)
+
+		dots += "../"
+	}
+
+	return dots
+}
+
+func validateQuery(t *testing.T, schema *ast.Schema, doc *ast.QueryDocument) {
+	t.Helper()
+
+	require.NotEqual(t, []ast.Operation{}, doc.Operations)
+	listErr := validator.Validate(schema, doc)
+	require.Equal(t, []error{}, listErr.Unwrap())
+}
+
+func parseQuery(t *testing.T, query string) *ast.QueryDocument {
+	t.Helper()
+
+	var queryObj struct {
+		Query     string
+		Variables map[string]any
+	}
+
+	err := json.NewDecoder(strings.NewReader(query)).Decode(&queryObj)
+	require.NoError(t, err, "error decoding query")
+
+	doc, err := parser.ParseQuery(&ast.Source{Input: queryObj.Query})
+	if err != nil {
+		_, ok := err.(*gqlerror.Error)
+		require.False(t, ok, "error parsing query")
+	}
+
+	return doc
+}

cli/test/http_transport.go

@@ -11,11 +11,21 @@ type TestTransport struct {
 	WithResponse *http.Response
 	WithError    error
 	Requests     []*http.Request
+	Handler      func(*http.Request) *struct {
+		Response *http.Response
+		Error    error
+	}
 }
 
 func (t *TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	t.Requests = append(t.Requests, req)
 
+	if t.Handler != nil {
+		if res := t.Handler(req); res != nil {
+			return res.Response, res.Error
+		}
+	}
+
 	return t.WithResponse, t.WithError
 }