Merge branch 'main' of github.com:goto/compass into feat-create-delet…

…e-assets-api

.github/workflows/lint.yml

@@ -12,13 +12,13 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           cache: false
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.53.3
+          version: v1.59.1
           args: --config=".golangci-prod.toml" --new-from-rev=HEAD~1 --max-same-issues=0 --max-issues-per-linter=0

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.20'
+          go-version: '^1.21'
       - name: Checkout
         uses: actions/checkout@v2
         with:

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Install dependencies
         run: sudo apt-get install build-essential
       - name: Install packages

.golangci-prod.toml

@@ -298,8 +298,6 @@
       "returnAfterHttpError",
       # Detects suspicious/confusing re-assignments.
       "sloppyReassign",
-      # Detects unsupported test and benchmark funcs.
-      "sloppyTestFuncName",
       # Detects redundant type assertions.
       "sloppyTypeAssert",
       # Detects “%s” formatting directives that can be replaced with %q.

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20 as base
+FROM golang:1.21 as base
 WORKDIR /build/
 COPY . .
 RUN ["make"]

README.md

@@ -126,7 +126,7 @@ Compass provides a fully-featured GRPC and HTTP API to interact with Compass ser
 <details>
   <summary>Dependencies:</summary>
 
-- Compass is written in Golang, and requires go version &gt;= 1.20. Please make sure that the go toolchain is available on your machine. See Golang’s [documentation](https://golang.org/) for installation instructions. Alternatively, you can use docker to build Compass as a docker image. More on this in the next section.
+- Compass is written in Golang, and requires go version &gt;= 1.21. Please make sure that the go toolchain is available on your machine. See Golang’s [documentation](https://golang.org/) for installation instructions. Alternatively, you can use docker to build Compass as a docker image. More on this in the next section.
 - Compass uses PostgreSQL 13 as its main storage and Elasticsearch v7 as the secondary storage to power the search. In order to run compass locally, you’ll need to have an instance of postgres and elasticsearch running. You can either download them and run it manually, or you can run them inside docker by using `docker-compose` with `docker-compose.yaml` provided in the root of this project.
 - PostgreSQL details and Elasticsearch brokers can alternatively be specified via the environment variable, `ELASTICSEARCH_BROKERS` for elasticsearch and `DB_HOST`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` for postgres.
 - If you use Docker to build compass, then configuring networking requires extra steps. Following is one of doing it by running postgres and elasticsearch inside with `docker-compose` first.
@@ -191,6 +191,8 @@ If you are using Compass binary, you can run this command.
 
 ## Running tests
 
+Before running unit test, be warned that it will spin up docker container behind the scene. If the engine being used is Lima-based (like Colima), ensure that the `DOCKER_HOST` environment variable is set to avoid spin up failure.
+
 Running all unit tests
 
 ```

cli/config.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/MakeNowJust/heredoc"
+	"github.com/goto/compass/core/asset"
 	"github.com/goto/compass/internal/client"
 	"github.com/goto/compass/internal/server"
 	esStore "github.com/goto/compass/internal/store/elasticsearch"
@@ -102,6 +103,12 @@ type Config struct {
 
 	// Column search excluded keyword list
 	ColSearchExclusionKeywords string `yaml:"col_search_excluded_keywords" mapstructure:"col_search_excluded_keywords"`
+
+	Asset Asset `mapstructure:"asset"`
+}
+
+type Asset struct {
+	AdditionalTypes []string `mapstructure:"additional_types"`
 }
 
 func LoadConfig() (*Config, error) {
@@ -148,3 +155,12 @@ func LoadConfigFromFlag(cfgFile string, cfg *Config) error {
 
 	return config.NewLoader(opts...).Load(cfg)
 }
+
+func registerAdditionalAssetTypes(additionalTypes []string) error {
+	transformedTypes := make([]asset.Type, len(additionalTypes))
+	for i := range additionalTypes {
+		transformedTypes[i] = asset.Type(additionalTypes[i])
+	}
+
+	return asset.RegisterSupportedTypes(transformedTypes...)
+}

cli/root.go

@@ -1,6 +1,8 @@
 package cli
 
 import (
+	"fmt"
+
 	"github.com/MakeNowJust/heredoc"
 	"github.com/goto/salt/cmdx"
 	"github.com/spf13/cobra"
@@ -43,6 +45,11 @@ func New(cliConfig *Config) *cobra.Command {
 				return err
 			}
 		}
+
+		if err := registerAdditionalAssetTypes(cliConfig.Asset.AdditionalTypes); err != nil {
+			return fmt.Errorf("error registering additional asset types: %w", err)
+		}
+
 		return nil
 	}
 

compass.yaml.example

@@ -74,3 +74,7 @@ client:
     host: localhost:8081
     serverheaderkey_uuid: Compass-User-UUID // if ommited, will use value on service.identity.headerkey_uuid
     serverheadervalue_uuid: [email protected]
+
+asset:
+    additional_types:
+        - fact_source

core/asset/asset_test.go

@@ -273,7 +273,7 @@ func TestAssetPatch(t *testing.T) {
 			description: "should patch all allowed fields",
 			asset: asset.Asset{
 				URN:         "some-urn",
-				Type:        asset.TypeJob,
+				Type:        asset.Type("job"),
 				Service:     "optimus",
 				Description: "sample-description",
 				Name:        "old-name",
@@ -302,7 +302,7 @@ func TestAssetPatch(t *testing.T) {
 			},
 			expected: asset.Asset{
 				URN:         "new-urn",
-				Type:        asset.TypeTable,
+				Type:        asset.Type("table"),
 				Service:     "firehose",
 				Description: "new-description",
 				Name:        "new-name",
@@ -321,7 +321,7 @@ func TestAssetPatch(t *testing.T) {
 			description: "should patch all allowed fields without JSON",
 			asset: asset.Asset{
 				URN:         "some-urn",
-				Type:        asset.TypeJob,
+				Type:        asset.Type("job"),
 				Service:     "optimus",
 				Description: "sample-description",
 				Name:        "old-name",
@@ -351,7 +351,7 @@ func TestAssetPatch(t *testing.T) {
 			},
 			expected: asset.Asset{
 				URN:         "new-urn",
-				Type:        asset.TypeTable,
+				Type:        asset.Type("table"),
 				Service:     "firehose",
 				Description: "new-description",
 				Name:        "new-name",
@@ -370,7 +370,7 @@ func TestAssetPatch(t *testing.T) {
 			description: "should patch all allowed fields without labels and owners",
 			asset: asset.Asset{
 				URN:         "some-urn",
-				Type:        asset.TypeJob,
+				Type:        asset.Type("job"),
 				Service:     "optimus",
 				Description: "sample-description",
 				Name:        "old-name",
@@ -392,7 +392,7 @@ func TestAssetPatch(t *testing.T) {
 			},
 			expected: asset.Asset{
 				URN:         "new-urn",
-				Type:        asset.TypeTable,
+				Type:        asset.Type("table"),
 				Service:     "firehose",
 				Description: "new-description",
 				Name:        "new-name",

core/asset/discovery_test.go

@@ -32,7 +32,7 @@ func TestToAsset(t *testing.T) {
 				ID:          "an-id",
 				URN:         "an-urn",
 				Name:        "a-title",
-				Type:        asset.TypeTable,
+				Type:        asset.Type("table"),
 				Service:     "a-service",
 				Description: "a-description",
 				Labels: map[string]string{

core/asset/service_test.go

@@ -120,6 +120,12 @@ func TestService_GetTypes(t *testing.T) {
 		Setup       func(context.Context, *mocks.AssetRepository)
 	}
 
+	const (
+		typeJob   = asset.Type("job")
+		typeTable = asset.Type("table")
+		typeTopic = asset.Type("topic")
+	)
+
 	testCases := []testCase{
 		{
 			Description: `should return error if asset repository get types return error`,
@@ -133,15 +139,15 @@ func TestService_GetTypes(t *testing.T) {
 			Description: `should return map types if asset repository get types return no error`,
 			Setup: func(ctx context.Context, ar *mocks.AssetRepository) {
 				ar.EXPECT().GetTypes(ctx, asset.Filter{}).Return(map[asset.Type]int{
-					asset.TypeJob:   1,
-					asset.TypeTable: 1,
-					asset.TypeTopic: 1,
+					typeJob:   1,
+					typeTable: 1,
+					typeTopic: 1,
 				}, nil)
 			},
 			Result: map[asset.Type]int{
-				asset.TypeJob:   1,
-				asset.TypeTable: 1,
-				asset.TypeTopic: 1,
+				typeJob:   1,
+				typeTable: 1,
+				typeTopic: 1,
 			},
 			Err: nil,
 		},
@@ -170,7 +176,7 @@ func TestService_GetTypes(t *testing.T) {
 }
 
 func TestService_UpsertAsset(t *testing.T) {
-	sampleAsset := &asset.Asset{ID: "some-id", URN: "some-urn", Type: asset.TypeDashboard, Service: "some-service"}
+	sampleAsset := &asset.Asset{ID: "some-id", URN: "some-urn", Type: asset.Type("dashboard"), Service: "some-service"}
 	sampleNodes1 := []string{"1-urn-1", "1-urn-2"}
 	sampleNodes2 := []string{"2-urn-1", "2-urn-2"}
 	type testCase struct {
@@ -261,7 +267,7 @@ func TestService_UpsertAsset(t *testing.T) {
 }
 
 func TestService_UpsertAssetWithoutLineage(t *testing.T) {
-	sampleAsset := &asset.Asset{ID: "some-id", URN: "some-urn", Type: asset.TypeDashboard, Service: "some-service"}
+	sampleAsset := &asset.Asset{ID: "some-id", URN: "some-urn", Type: asset.Type("dashboard"), Service: "some-service"}
 	testCases := []struct {
 		Description string
 		Asset       *asset.Asset

core/asset/type.go

@@ -1,28 +1,68 @@
 package asset
 
+import (
+	"errors"
+	"regexp"
+)
+
+var (
+	errTypeInvalidLength    = errors.New("type length must be 3 to 16 inclusive")
+	errTypeInvalidCharacter = errors.New("type must be combination of alphanumeric and underscores")
+)
+
+var invalidTypePattern = regexp.MustCompile(`[^a-z0-9_]`)
+
 const (
-	TypeTable        Type = "table"
-	TypeJob          Type = "job"
-	TypeDashboard    Type = "dashboard"
-	TypeTopic        Type = "topic"
-	TypeFeatureTable Type = "feature_table"
-	TypeApplication  Type = "application"
-	TypeModel        Type = "model"
-	TypeQuery        Type = "query"
-	TypeMetric       Type = "metric"
+	typeMinLength = 3
+	typeMaxLength = 16
 )
 
-// AllSupportedTypes holds a list of all supported types struct
-var AllSupportedTypes = []Type{
-	TypeTable,
-	TypeJob,
-	TypeDashboard,
-	TypeTopic,
-	TypeFeatureTable,
-	TypeApplication,
-	TypeModel,
-	TypeQuery,
-	TypeMetric,
+const (
+	typeTable        Type = "table"
+	typeJob          Type = "job"
+	typeDashboard    Type = "dashboard"
+	typeTopic        Type = "topic"
+	typeFeatureTable Type = "feature_table"
+	typeApplication  Type = "application"
+	typeModel        Type = "model"
+	typeQuery        Type = "query"
+	typeMetric       Type = "metric"
+)
+
+var supportedTypeMap = map[Type]bool{
+	typeTable:        true,
+	typeJob:          true,
+	typeDashboard:    true,
+	typeTopic:        true,
+	typeFeatureTable: true,
+	typeApplication:  true,
+	typeModel:        true,
+	typeQuery:        true,
+	typeMetric:       true,
+}
+
+func GetSupportedTypes() []Type {
+	output := make([]Type, 0, len(supportedTypeMap))
+	for _type := range supportedTypeMap {
+		output = append(output, _type)
+	}
+	return output
+}
+
+func RegisterSupportedTypes(types ...Type) error {
+	for _, t := range types {
+		if err := t.validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, t := range types {
+		if supported := supportedTypeMap[t]; !supported {
+			supportedTypeMap[t] = true
+		}
+	}
+
+	return nil
 }
 
 // Type specifies a supported type name
@@ -35,10 +75,17 @@ func (t Type) String() string {
 
 // IsValid will validate whether the typename is valid or not
 func (t Type) IsValid() bool {
-	for _, supportedType := range AllSupportedTypes {
-		if t == supportedType {
-			return true
-		}
+	return supportedTypeMap[t]
+}
+
+func (t Type) validate() error {
+	if l := len(t.String()); l < typeMinLength || l > typeMaxLength {
+		return errTypeInvalidLength
 	}
-	return false
+
+	if invalidTypePattern.FindStringSubmatch(t.String()) != nil {
+		return errTypeInvalidCharacter
+	}
+
+	return nil
 }