Support distributed execution strategies

.github/workflows/test.yaml

@@ -13,7 +13,7 @@ on:
 jobs:
   test-go:
     name: Run Go lint and unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       pull-requests: write
     steps:
@@ -27,7 +27,7 @@ jobs:
       - name: Vet
         run: go vet ./...
       - name: Lint
-        uses: golangci/golangci-lint-action@v4
+        uses: golangci/golangci-lint-action@v6
         with:
           version: latest
           only-new-issues: true
@@ -36,6 +36,9 @@ jobs:
         run: |
           go test -v -coverpkg=./... -race -timeout 3m -coverprofile=coverage.out.tmp ./...
           cat coverage.out.tmp | grep -v "main.go" > coverage.out
+      - name: Run Go unit tests
+        run: |
+          ./scripts/test.sh
       - name: Go coverage format
         if: ${{ github.event_name == 'pull_request' }}
         run: |

.gitignore

@@ -16,6 +16,7 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+tmp/
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

README.md

@@ -24,20 +24,23 @@ The connector reads `config.{json,yaml}` file in the configuration folder. The f
 
 ```yaml
 files:
-  - path: swagger.json
+  - file: swagger.json
     spec: openapi2
-  - path: openapi.yaml
+  - file: openapi.yaml
     spec: openapi3
     trimPrefix: /v1
     envPrefix: PET_STORE
     methodAlias:
       post: create
       put: update
-  - path: schema.json
+  - file: schema.json
     spec: ndc
 ```
 
-`trimPrefix`, `envPrefix` and `methodAlias` options are used to convert OpenAPI by [ndc-rest-schema](https://github.com/hasura/ndc-rest-schema#openapi).
+The config of each element follows the [config schema](https://github.com/hasura/ndc-rest-schema/blob/main/config.example.yaml) of `ndc-rest-schema`.
+
+> [!IMPORTANT]
+> Conflicted object and scalar types will be ignored. Only the type of the first file is kept in the schema.
 
 #### Supported specs
 
@@ -85,3 +88,129 @@ settings:
 The current version supports API key and Auth token authentication schemes. The configuration is inspired from `securitySchemes` [with env variables](https://github.com/hasura/ndc-rest-schema#authentication)
 
 See [this example](rest/testdata/auth/schema.yaml) for more context.
+
+## Distributed execution
+
+Imagine that your backend have many server replications, or multiple applications with different credentials. You want to:
+
+- Specify the server where the request will be executed to.
+- Execute an operation to all servers.
+
+For example, with below server settings, the connector will replicate existing operations with `Distributed` suffixes:
+
+```yaml
+settings:
+  servers:
+    - id: dog
+      url: "http://localhost:3000"
+      securitySchemes:
+        api_key:
+          type: apiKey
+          value: "dog-secret"
+          in: header
+          name: api_key
+    - id: cat
+      url: "http://localhost:3001"
+      securitySchemes:
+        api_key:
+          type: apiKey
+          value: "cat-secret"
+          in: header
+          name: api_key
+```
+
+```json
+{
+  "functions": [
+    {
+      "arguments": {
+        "restOptions": {
+          "type": {
+            "type": "nullable",
+            "underlying_type": {
+              "name": "RestSingleOptions",
+              "type": "named"
+            }
+          }
+        }
+      },
+      "name": "findPets",
+      "result_type": {
+        "element_type": {
+          "name": "Pet",
+          "type": "named"
+        },
+        "type": "array"
+      }
+    },
+    {
+      "arguments": {
+        "restOptions": {
+          "type": {
+            "type": "nullable",
+            "underlying_type": {
+              "name": "RestDistributedOptions",
+              "type": "named"
+            }
+          }
+        }
+      },
+      "name": "findPetsDistributed",
+      "result_type": {
+        "name": "FindPetsDistributedResult",
+        "type": "named"
+      }
+    }
+  ],
+  "object_types": {
+    "RestDistributedOptions": {
+      "description": "Distributed execution options for REST requests to multiple servers",
+      "fields": {
+        "parallel": {
+          "description": "Execute requests to remote servers in parallel",
+          "type": {
+            "type": "nullable",
+            "underlying_type": {
+              "name": "Boolean",
+              "type": "named"
+            }
+          }
+        },
+        "servers": {
+          "description": "Specify remote servers to receive the request",
+          "type": {
+            "type": "nullable",
+            "underlying_type": {
+              "element_type": {
+                "name": "RestServerId",
+                "type": "named"
+              },
+              "type": "array"
+            }
+          }
+        }
+      }
+    },
+    "RestSingleOptions": {
+      "description": "Execution options for REST requests to a single server",
+      "fields": {
+        "servers": {
+          "description": "Specify remote servers to receive the request. If there are many server IDs the server is selected randomly",
+          "type": {
+            "type": "nullable",
+            "underlying_type": {
+              "element_type": {
+                "name": "RestServerId",
+                "type": "named"
+              },
+              "type": "array"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+`RestSingleOptions` object type is added to existing operations (findPets). API consumers can specify the server to be executed. If you want to execute all remote servers in sequence or parallel, `findPetsDistributed` function should be used.

connector-definition/config/config.yaml

@@ -1,3 +1,3 @@
 files:
-  - path: https://raw.githubusercontent.com/hasura/ndc-rest/main/rest/testdata/jsonplaceholder/swagger.json
+  - file: https://raw.githubusercontent.com/hasura/ndc-rest/main/rest/testdata/jsonplaceholder/swagger.json
     spec: openapi2

go.mod

@@ -3,11 +3,13 @@ module github.com/hasura/ndc-rest
 go 1.21
 
 require (
-	github.com/hasura/ndc-rest-schema v0.1.3
-	github.com/hasura/ndc-sdk-go v1.1.3
+	github.com/hasura/ndc-rest-schema v0.2.0
+	github.com/hasura/ndc-sdk-go v1.2.0
 	github.com/lmittmann/tint v1.0.4
-	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/otel v1.27.0
+	go.opentelemetry.io/otel/trace v1.27.0
 	gopkg.in/yaml.v3 v3.0.1
+	gotest.tools/v3 v3.5.1
 )
 
 require (
@@ -17,42 +19,41 @@ require (
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/evanphx/json-patch v0.5.2 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/pb33f/libopenapi v0.16.5 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pb33f/libopenapi v0.16.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.53.0 // indirect
-	github.com/prometheus/procfs v0.14.0 // indirect
+	github.com/prometheus/common v0.54.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
-	go.opentelemetry.io/contrib/propagators/b3 v1.26.0 // indirect
-	go.opentelemetry.io/otel v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect
-	go.opentelemetry.io/otel/exporters/prometheus v0.48.0 // indirect
-	go.opentelemetry.io/otel/metric v1.26.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.26.0 // indirect
-	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.27.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
+	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240509183442-62759503f434 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 // indirect
-	google.golang.org/grpc v1.63.2 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
+	google.golang.org/grpc v1.64.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 )