From c93997ddfd5c691f2b4f82fd3ed466ee48f10af5 Mon Sep 17 00:00:00 2001 From: zongz Date: Fri, 24 Nov 2023 15:42:10 +0800 Subject: [PATCH 1/4] feat: add api to 'GetFullSchemaTypeMappingWithFilters' Signed-off-by: zongz --- go.mod | 4 +- go.sum | 8 +- pkg/api/kpm_pkg.go | 77 +++++++++++++++++++ pkg/api/kpm_pkg_test.go | 21 +++++ .../get_schema_ty/aaa/kcl.mod | 7 ++ .../get_schema_ty/aaa/kcl.mod.lock | 7 ++ .../test_kpm_package/get_schema_ty/aaa/main.k | 5 ++ .../get_schema_ty/bbb/kcl.mod | 5 ++ .../get_schema_ty/bbb/kcl.mod.lock | 0 .../test_kpm_package/get_schema_ty/bbb/main.k | 2 + pkg/constants/constants.go | 3 + 11 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod.lock create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/main.k create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod.lock create mode 100644 pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/main.k diff --git a/go.mod b/go.mod index 493a0223..4479e251 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/urfave/cli/v2 v2.25.0 gotest.tools/v3 v3.4.0 - kcl-lang.io/kcl-go v0.7.0-alpha.2 + kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09 ) require ( @@ -91,7 +91,7 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - kcl-lang.io/kcl-artifact-go v0.7.0-alpha.2 // indirect + kcl-lang.io/kcl-artifact-go v0.7.0-beta.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 7fd9df40..d6d4fc8f 100644 --- a/go.sum +++ b/go.sum @@ -763,10 +763,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -kcl-lang.io/kcl-artifact-go v0.7.0-alpha.2 h1:ct6HPq+ssN4kNwMnyqyJAJ35YGiWPMgV4vWyyqzqpYo= -kcl-lang.io/kcl-artifact-go v0.7.0-alpha.2/go.mod h1:c07mqi9Hu2UjPW7lYfHhAAWOlZiB7lo7Vkr4jL5ov/M= -kcl-lang.io/kcl-go v0.7.0-alpha.2 h1:UjKyGx4OSKUnpHTTNJC6q/qYHcexA0nEBLdd/ZdxNnA= -kcl-lang.io/kcl-go v0.7.0-alpha.2/go.mod h1:GnXGUfNp+72g4ichID2VkfI679SCLEnmSuTPF7+N1H0= +kcl-lang.io/kcl-artifact-go v0.7.0-beta.1 h1:u1l/OXWblOiDHabN+6ApSHTZwNyOskuOCf90fXreGu4= +kcl-lang.io/kcl-artifact-go v0.7.0-beta.1/go.mod h1:c07mqi9Hu2UjPW7lYfHhAAWOlZiB7lo7Vkr4jL5ov/M= +kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09 h1:66i5Gf9S5UC0lDKalYTzv8b03WreLe/yuoEIfYZHbo0= +kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09/go.mod h1:yf8pMkic+1m4wedG9MLxrkeVoqFhIrMbY+x4nPPg2O8= oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY= oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg= oras.land/oras-go/v2 v2.3.0 h1:lqX1aXdN+DAmDTKjiDyvq85cIaI4RkIKp/PghWlAGIU= diff --git a/pkg/api/kpm_pkg.go b/pkg/api/kpm_pkg.go index d4ff4da0..53e9f982 100644 --- a/pkg/api/kpm_pkg.go +++ b/pkg/api/kpm_pkg.go @@ -1,12 +1,14 @@ package api import ( + "fmt" "os" "path/filepath" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/errors" pkg "kcl-lang.io/kpm/pkg/package" ) @@ -114,6 +116,81 @@ func (pkg *KclPackage) GetSchemaTypeMappingNamed(schemaName string) (map[string] return pkg.GetSchemaTypeMappingWithFilters([]KclTypeFilterFunc{IsSchemaType, namedFilterFunc}) } +// GetAllFullSchemaTypeMapping returns all the full schema types of the package. +func (pkg *KclPackage) GetAllFullSchemaTypeMapping() (map[string]map[string]*KclType, error) { + cli, err := client.NewKpmClient() + if err != nil { + return nil, err + } + return pkg.GetFullSchemaTypeMappingWithFilters(cli, []KclTypeFilterFunc{IsSchemaType}) +} + +// GetFullSchemaTypeMappingWithFilters returns the full schema type filtered by the filter functions. +func (pkg *KclPackage) GetFullSchemaTypeMappingWithFilters(kpmcli *client.KpmClient, fileterFuncs []KclTypeFilterFunc) (map[string]map[string]*KclType, error) { + schemaTypes := make(map[string]map[string]*KclType) + err := filepath.Walk(pkg.GetPkgHomePath(), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + fileteredKtypeMap := make(map[string]*KclType) + + depsMap, err := kpmcli.ResolveDepsIntoMap(pkg.pkg) + if err != nil { + return err + } + + opts := kcl.NewOption() + opts.Merge(kcl.WithKFilenames(path)) + for depName, depPath := range depsMap { + opts.Merge(kcl.WithExternalPkgs(fmt.Sprintf(constants.EXTERNAL_PKGS_ARG_PATTERN, depName, depPath))) + } + + schemaTypeList, err := kcl.GetFullSchemaType([]string{path}, "", *opts) + if err != nil && err.Error() != errors.NoKclFiles.Error() { + return err + } + + schemaTypeMap := make(map[string]*gpyrpc.KclType) + for _, schemaType := range schemaTypeList { + schemaTypeMap[schemaType.SchemaName] = schemaType + } + + relPath, err := filepath.Rel(pkg.GetPkgHomePath(), path) + if err != nil { + return err + } + if len(schemaTypeMap) != 0 && schemaTypeMap != nil { + for kName, kType := range schemaTypeMap { + kTy := NewKclTypes(kName, relPath, kType) + filterPassed := true + for _, filterFunc := range fileterFuncs { + if !filterFunc(kTy) { + filterPassed = false + break + } + } + if filterPassed { + fileteredKtypeMap[kName] = kTy + } + } + if len(fileteredKtypeMap) > 0 { + schemaTypes[relPath] = fileteredKtypeMap + } + } + } + + return nil + }) + + if err != nil { + return nil, err + } + + return schemaTypes, nil +} + // GetSchemaTypeMappingWithFilters returns the schema type filtered by the filter functions. func (pkg *KclPackage) GetSchemaTypeMappingWithFilters(fileterFuncs []KclTypeFilterFunc) (map[string]map[string]*KclType, error) { schemaTypes := make(map[string]map[string]*KclType) diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go index 99bd70e9..fe37dbb7 100644 --- a/pkg/api/kpm_pkg_test.go +++ b/pkg/api/kpm_pkg_test.go @@ -134,6 +134,27 @@ func TestGetSchemaTypeMappingWithFilters(t *testing.T) { assert.Equal(t, schemas[filepath.Join("sub", "sub1")]["SchemaWithSameName"].RelPath, filepath.Join("sub", "sub1")) } +func TestGetFullSchemaTypeMappingWithFilters(t *testing.T) { + pkg_path := filepath.Join(getTestDir("test_kpm_package"), "get_schema_ty", "aaa") + pkg, err := GetKclPackage(pkg_path) + assert.Equal(t, err, nil) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.ResolvePkgDepsMetadata(pkg.pkg, true) + assert.Equal(t, err, nil) + + filterFunc := func(kt *KclType) bool { + return kt.Type == "schema" + } + + schemas, err := pkg.GetFullSchemaTypeMappingWithFilters(kpmcli, []KclTypeFilterFunc{filterFunc}) + assert.Equal(t, err, nil) + assert.Equal(t, len(schemas), 1) + + assert.Equal(t, schemas[filepath.Join(".")]["B"].Type, "schema") + assert.Equal(t, schemas[filepath.Join(".")]["B"].SchemaName, "B") +} + func TestGetSchemaTypeUnderEmptyDir(t *testing.T) { pkg_path := filepath.Join(getTestDir("test_kpm_package"), "no_kcl_files") pkg, err := GetKclPackage(pkg_path) diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod new file mode 100644 index 00000000..79bceba5 --- /dev/null +++ b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "aaa" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +bbb = { path = "../bbb" } diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod.lock b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod.lock new file mode 100644 index 00000000..ba2265e8 --- /dev/null +++ b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/kcl.mod.lock @@ -0,0 +1,7 @@ +[dependencies] + [dependencies.bbb] + name = "bbb" + full_name = "bbb_0.0.1" + version = "0.0.1" + sum = "/PGJJE6Rp0lAsq3n9RUZ34jPGfwzYIZESs17L6kSG/Y=" + path = "../bbb" diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/main.k b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/main.k new file mode 100644 index 00000000..e2c85898 --- /dev/null +++ b/pkg/api/test_data/test_kpm_package/get_schema_ty/aaa/main.k @@ -0,0 +1,5 @@ +import bbb as b + +a = b.B { + name: "b instance in a" +} \ No newline at end of file diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod b/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod new file mode 100644 index 00000000..e9ea10a5 --- /dev/null +++ b/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "bbb" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod.lock b/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/main.k b/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/main.k new file mode 100644 index 00000000..21dad4f9 --- /dev/null +++ b/pkg/api/test_data/test_kpm_package/get_schema_ty/bbb/main.k @@ -0,0 +1,2 @@ +schema B: + name: str \ No newline at end of file diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index c8cd3fcd..621e5ea1 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -19,4 +19,7 @@ const ( DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION = "org.kcllang.package.description" DEFAULT_KCL_OCI_MANIFEST_SUM = "org.kcllang.package.sum" DEFAULT_CREATE_OCI_MANIFEST_TIME = "org.opencontainers.image.created" + + // The pattern of the external package argument. + EXTERNAL_PKGS_ARG_PATTERN = "%s=%s" ) From 9caa2dba2f06c85f932b68ca4b36a2861cc949f3 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 30 Nov 2023 15:24:34 +0800 Subject: [PATCH 2/4] chore: bump kcl-go version Signed-off-by: zongz --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4479e251..87adf728 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/urfave/cli/v2 v2.25.0 gotest.tools/v3 v3.4.0 - kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09 + kcl-lang.io/kcl-go v0.7.0-beta.1.0.20231130071901-a0ae3a253d62 ) require ( @@ -91,7 +91,7 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - kcl-lang.io/kcl-artifact-go v0.7.0-beta.1 // indirect + kcl-lang.io/kcl-artifact-go v0.7.0-beta.2 // indirect ) require ( diff --git a/go.sum b/go.sum index d6d4fc8f..d50ab880 100644 --- a/go.sum +++ b/go.sum @@ -763,10 +763,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -kcl-lang.io/kcl-artifact-go v0.7.0-beta.1 h1:u1l/OXWblOiDHabN+6ApSHTZwNyOskuOCf90fXreGu4= -kcl-lang.io/kcl-artifact-go v0.7.0-beta.1/go.mod h1:c07mqi9Hu2UjPW7lYfHhAAWOlZiB7lo7Vkr4jL5ov/M= -kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09 h1:66i5Gf9S5UC0lDKalYTzv8b03WreLe/yuoEIfYZHbo0= -kcl-lang.io/kcl-go v0.7.0-alpha.2.0.20231124021812-a15c657f6c09/go.mod h1:yf8pMkic+1m4wedG9MLxrkeVoqFhIrMbY+x4nPPg2O8= +kcl-lang.io/kcl-artifact-go v0.7.0-beta.2 h1:7L5qbEqD8y672XNTk2snv8/UXScABFdM3g9Bu2Ama1Q= +kcl-lang.io/kcl-artifact-go v0.7.0-beta.2/go.mod h1:c07mqi9Hu2UjPW7lYfHhAAWOlZiB7lo7Vkr4jL5ov/M= +kcl-lang.io/kcl-go v0.7.0-beta.1.0.20231130071901-a0ae3a253d62 h1:6FQgAZyAMXUUNEF9pY/LC+doA6I2A0Pwv5iR294Hw+s= +kcl-lang.io/kcl-go v0.7.0-beta.1.0.20231130071901-a0ae3a253d62/go.mod h1:dxF3hghkxoJKOkBrXwrqc8nRT0HylrOvsI83FxdQT6M= oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY= oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg= oras.land/oras-go/v2 v2.3.0 h1:lqX1aXdN+DAmDTKjiDyvq85cIaI4RkIKp/PghWlAGIU= From 16b1abe7d8e76c91a84045ee9dcadd377174a5f9 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 30 Nov 2023 16:15:31 +0800 Subject: [PATCH 3/4] feat: remove useless code Signed-off-by: zongz --- pkg/api/kpm_pkg.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/api/kpm_pkg.go b/pkg/api/kpm_pkg.go index 53e9f982..b67dd2b0 100644 --- a/pkg/api/kpm_pkg.go +++ b/pkg/api/kpm_pkg.go @@ -103,7 +103,11 @@ func (pkg *KclPackage) GetPkgProfile() pkg.Profile { // // And, the value is a map of schema types, the key is the schema name, the value is the schema type. func (pkg *KclPackage) GetAllSchemaTypeMapping() (map[string]map[string]*KclType, error) { - return pkg.GetSchemaTypeMappingWithFilters([]KclTypeFilterFunc{IsSchemaType}) + cli, err := client.NewKpmClient() + if err != nil { + return nil, err + } + return pkg.GetFullSchemaTypeMappingWithFilters(cli, []KclTypeFilterFunc{IsSchemaType}) } // GetSchemaTypeMappingNamed returns the schema type filtered by schema name. @@ -116,15 +120,6 @@ func (pkg *KclPackage) GetSchemaTypeMappingNamed(schemaName string) (map[string] return pkg.GetSchemaTypeMappingWithFilters([]KclTypeFilterFunc{IsSchemaType, namedFilterFunc}) } -// GetAllFullSchemaTypeMapping returns all the full schema types of the package. -func (pkg *KclPackage) GetAllFullSchemaTypeMapping() (map[string]map[string]*KclType, error) { - cli, err := client.NewKpmClient() - if err != nil { - return nil, err - } - return pkg.GetFullSchemaTypeMappingWithFilters(cli, []KclTypeFilterFunc{IsSchemaType}) -} - // GetFullSchemaTypeMappingWithFilters returns the full schema type filtered by the filter functions. func (pkg *KclPackage) GetFullSchemaTypeMappingWithFilters(kpmcli *client.KpmClient, fileterFuncs []KclTypeFilterFunc) (map[string]map[string]*KclType, error) { schemaTypes := make(map[string]map[string]*KclType) @@ -142,7 +137,6 @@ func (pkg *KclPackage) GetFullSchemaTypeMappingWithFilters(kpmcli *client.KpmCli } opts := kcl.NewOption() - opts.Merge(kcl.WithKFilenames(path)) for depName, depPath := range depsMap { opts.Merge(kcl.WithExternalPkgs(fmt.Sprintf(constants.EXTERNAL_PKGS_ARG_PATTERN, depName, depPath))) } From 98ee68a122aaafecba68f51e6c81ad9b6a582473 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 30 Nov 2023 16:50:57 +0800 Subject: [PATCH 4/4] fix: fix failed test case Signed-off-by: zongz --- pkg/api/kpm_pkg_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go index fe37dbb7..f77cd6a8 100644 --- a/pkg/api/kpm_pkg_test.go +++ b/pkg/api/kpm_pkg_test.go @@ -37,7 +37,7 @@ func TestPackageApi(t *testing.T) { schemas, err := pkg.GetAllSchemaTypeMapping() assert.Equal(t, err, nil) assert.Equal(t, len(schemas), 3) - assert.Equal(t, len(schemas["."]), 2) + assert.Equal(t, len(schemas["."]), 4) assert.Equal(t, len(schemas[filepath.Join("sub")]), 1) assert.Equal(t, len(schemas[filepath.Join("sub", "sub1")]), 2)