Merge remote-tracking branch 'databricks/main' into faster-dev-mode-k…

…ill-kill

.codegen/_openapi_sha

@@ -1 +1 @@
-3821dc51952c5cf1c276dd84967da011b191e64a
+93763b0d7ae908520c229c786fff28b8fd623261

.github/workflows/push.yml

@@ -89,3 +89,29 @@ jobs:
         run: |
           # Exit with status code 1 if there are differences (i.e. unformatted files)
           git diff --exit-code
+
+  validate-bundle-schema:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.x
+
+      # Github repo: https://github.com/ajv-validator/ajv-cli
+      - name: Install ajv-cli
+        run: npm install -g [email protected]
+
+      # Assert that the generated bundle schema is a valid JSON schema by using
+      # ajv-cli to validate it against a sample configuration file.
+      # By default the ajv-cli runs in strict mode which will fail if the schema
+      # itself is not valid. Strict mode is more strict than the JSON schema
+      # specification. See for details: https://ajv.js.org/options.html#strict-mode-options
+      - name: Validate bundle schema
+        run: |
+          go run main.go bundle schema > schema.json
+          ajv -s schema.json -d ./bundle/tests/basic/databricks.yml

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Version changelog
 
+## 0.216.0
+
+CLI:
+ * Propagate correct `User-Agent` for CLI during OAuth flow ([#1264](https://github.com/databricks/cli/pull/1264)).
+ * Add usage string when command fails with incorrect arguments ([#1276](https://github.com/databricks/cli/pull/1276)).
+
+Bundles:
+ * Include `dyn.Path` as argument to the visit callback function ([#1260](https://github.com/databricks/cli/pull/1260)).
+ * Inline logic to set a value in `dyn.SetByPath` ([#1261](https://github.com/databricks/cli/pull/1261)).
+ * Add assertions for the `dyn.Path` argument to the visit callback ([#1265](https://github.com/databricks/cli/pull/1265)).
+ * Add `dyn.MapByPattern` to map a function to values with matching paths ([#1266](https://github.com/databricks/cli/pull/1266)).
+ * Filter current user from resource permissions ([#1262](https://github.com/databricks/cli/pull/1262)).
+ * Retain location annotation when expanding globs for pipeline libraries ([#1274](https://github.com/databricks/cli/pull/1274)).
+ * Added deployment state for bundles ([#1267](https://github.com/databricks/cli/pull/1267)).
+ * Do CheckRunningResource only after terraform.Write ([#1292](https://github.com/databricks/cli/pull/1292)).
+ * Rewrite relative paths using `dyn.Location` of the underlying value ([#1273](https://github.com/databricks/cli/pull/1273)).
+ * Push deployment state right after files upload ([#1293](https://github.com/databricks/cli/pull/1293)).
+ * Make `Append` function to `dyn.Path` return independent slice ([#1295](https://github.com/databricks/cli/pull/1295)).
+ * Move bundle tests into bundle/tests ([#1299](https://github.com/databricks/cli/pull/1299)).
+ * Upgrade Terraform provider to 1.38.0 ([#1308](https://github.com/databricks/cli/pull/1308)).
+
+Internal:
+ * Add integration test for mlops-stacks initialization ([#1155](https://github.com/databricks/cli/pull/1155)).
+ * Update actions/setup-python to v5 ([#1290](https://github.com/databricks/cli/pull/1290)).
+ * Update codecov/codecov-action to v4 ([#1291](https://github.com/databricks/cli/pull/1291)).
+
+API Changes:
+ * Changed `databricks catalogs list` command.
+ * Changed `databricks online-tables create` command.
+ * Changed `databricks lakeview publish` command.
+ * Added `databricks lakeview create` command.
+ * Added `databricks lakeview get` command.
+ * Added `databricks lakeview get-published` command.
+ * Added `databricks lakeview trash` command.
+ * Added `databricks lakeview update` command.
+ * Moved settings related commands to `databricks settings` and `databricks account settings`.
+
+OpenAPI commit 93763b0d7ae908520c229c786fff28b8fd623261 (2024-03-20)
+
+Dependency updates:
+ * Bump golang.org/x/oauth2 from 0.17.0 to 0.18.0 ([#1270](https://github.com/databricks/cli/pull/1270)).
+ * Bump golang.org/x/mod from 0.15.0 to 0.16.0 ([#1271](https://github.com/databricks/cli/pull/1271)).
+ * Update Go SDK to v0.35.0 ([#1300](https://github.com/databricks/cli/pull/1300)).
+ * Update Go SDK to v0.36.0 ([#1304](https://github.com/databricks/cli/pull/1304)).
+
 ## 0.215.0
 
 CLI:

bundle/artifacts/all.go

@@ -7,6 +7,7 @@ import (
 	"slices"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
 	"golang.org/x/exp/maps"
 )
 
@@ -21,7 +22,7 @@ func (m *all) Name() string {
 	return fmt.Sprintf("artifacts.%sAll", m.name)
 }
 
-func (m *all) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *all) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	var out []bundle.Mutator
 
 	// Iterate with stable ordering.
@@ -31,7 +32,7 @@ func (m *all) Apply(ctx context.Context, b *bundle.Bundle) error {
 	for _, name := range keys {
 		m, err := m.fn(name)
 		if err != nil {
-			return err
+			return diag.FromErr(err)
 		}
 		if m != nil {
 			out = append(out, m)

bundle/artifacts/artifacts.go

@@ -14,6 +14,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/log"
 )
@@ -57,17 +58,17 @@ func (m *basicBuild) Name() string {
 	return fmt.Sprintf("artifacts.Build(%s)", m.name)
 }
 
-func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	artifact, ok := b.Config.Artifacts[m.name]
 	if !ok {
-		return fmt.Errorf("artifact doesn't exist: %s", m.name)
+		return diag.Errorf("artifact doesn't exist: %s", m.name)
 	}
 
 	cmdio.LogString(ctx, fmt.Sprintf("Building %s...", m.name))
 
 	out, err := artifact.Build(ctx)
 	if err != nil {
-		return fmt.Errorf("build for %s failed, error: %w, output: %s", m.name, err, out)
+		return diag.Errorf("build for %s failed, error: %v, output: %s", m.name, err, out)
 	}
 	log.Infof(ctx, "Build succeeded")
 
@@ -87,29 +88,29 @@ func (m *basicUpload) Name() string {
 	return fmt.Sprintf("artifacts.Upload(%s)", m.name)
 }
 
-func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	artifact, ok := b.Config.Artifacts[m.name]
 	if !ok {
-		return fmt.Errorf("artifact doesn't exist: %s", m.name)
+		return diag.Errorf("artifact doesn't exist: %s", m.name)
 	}
 
 	if len(artifact.Files) == 0 {
-		return fmt.Errorf("artifact source is not configured: %s", m.name)
+		return diag.Errorf("artifact source is not configured: %s", m.name)
 	}
 
 	uploadPath, err := getUploadBasePath(b)
 	if err != nil {
-		return err
+		return diag.FromErr(err)
 	}
 
 	client, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath)
 	if err != nil {
-		return err
+		return diag.FromErr(err)
 	}
 
 	err = uploadArtifact(ctx, b, artifact, uploadPath, client)
 	if err != nil {
-		return fmt.Errorf("upload for %s failed, error: %w", m.name, err)
+		return diag.Errorf("upload for %s failed, error: %v", m.name, err)
 	}
 
 	return nil

bundle/artifacts/autodetect.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts/whl"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/log"
 )
 
@@ -19,7 +20,7 @@ func (m *autodetect) Name() string {
 	return "artifacts.DetectPackages"
 }
 
-func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// If artifacts section explicitly defined, do not try to auto detect packages
 	if b.Config.Artifacts != nil {
 		log.Debugf(ctx, "artifacts block is defined, skipping auto-detecting")

bundle/artifacts/build.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
 )
 
 func BuildAll() bundle.Mutator {
@@ -27,25 +28,25 @@ func (m *build) Name() string {
 	return fmt.Sprintf("artifacts.Build(%s)", m.name)
 }
 
-func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	artifact, ok := b.Config.Artifacts[m.name]
 	if !ok {
-		return fmt.Errorf("artifact doesn't exist: %s", m.name)
+		return diag.Errorf("artifact doesn't exist: %s", m.name)
 	}
 
 	// Skip building if build command is not specified or infered
 	if artifact.BuildCommand == "" {
 		// If no build command was specified or infered and there is no
 		// artifact output files specified, artifact is misconfigured
 		if len(artifact.Files) == 0 {
-			return fmt.Errorf("misconfigured artifact: please specify 'build' or 'files' property")
+			return diag.Errorf("misconfigured artifact: please specify 'build' or 'files' property")
 		}
 		return nil
 	}
 
 	// If artifact path is not provided, use bundle root dir
 	if artifact.Path == "" {
-		artifact.Path = b.Config.Path
+		artifact.Path = b.RootPath
 	}
 
 	if !filepath.IsAbs(artifact.Path) {

bundle/artifacts/infer.go

@@ -7,6 +7,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts/whl"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/diag"
 )
 
 var inferMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{
@@ -41,10 +42,10 @@ func (m *infer) Name() string {
 	return fmt.Sprintf("artifacts.Infer(%s)", m.name)
 }
 
-func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	artifact, ok := b.Config.Artifacts[m.name]
 	if !ok {
-		return fmt.Errorf("artifact doesn't exist: %s", m.name)
+		return diag.Errorf("artifact doesn't exist: %s", m.name)
 	}
 
 	// only try to infer command if it's not already defined

bundle/artifacts/upload.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
@@ -33,14 +34,14 @@ func (m *upload) Name() string {
 	return fmt.Sprintf("artifacts.Upload(%s)", m.name)
 }
 
-func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	artifact, ok := b.Config.Artifacts[m.name]
 	if !ok {
-		return fmt.Errorf("artifact doesn't exist: %s", m.name)
+		return diag.Errorf("artifact doesn't exist: %s", m.name)
 	}
 
 	if len(artifact.Files) == 0 {
-		return fmt.Errorf("artifact source is not configured: %s", m.name)
+		return diag.Errorf("artifact source is not configured: %s", m.name)
 	}
 
 	// Check if source paths are absolute, if not, make them absolute
@@ -57,11 +58,11 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error {
 	for _, f := range artifact.Files {
 		matches, err := filepath.Glob(f.Source)
 		if err != nil {
-			return fmt.Errorf("unable to find files for %s: %w", f.Source, err)
+			return diag.Errorf("unable to find files for %s: %v", f.Source, err)
 		}
 
 		if len(matches) == 0 {
-			return fmt.Errorf("no files found for %s", f.Source)
+			return diag.Errorf("no files found for %s", f.Source)
 		}
 
 		for _, match := range matches {
@@ -81,10 +82,10 @@ func (m *cleanUp) Name() string {
 	return "artifacts.CleanUp"
 }
 
-func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) error {
+func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	uploadPath, err := getUploadBasePath(b)
 	if err != nil {
-		return err
+		return diag.FromErr(err)
 	}
 
 	b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{
@@ -94,7 +95,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) error {
 
 	err = b.WorkspaceClient().Workspace.MkdirsByPath(ctx, uploadPath)
 	if err != nil {
-		return fmt.Errorf("unable to create directory for %s: %w", uploadPath, err)
+		return diag.Errorf("unable to create directory for %s: %v", uploadPath, err)
 	}
 
 	return nil

bundle/artifacts/upload_test.go

@@ -9,13 +9,14 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/testfile"
 	"github.com/stretchr/testify/require"
 )
 
 type noop struct{}
 
-func (n *noop) Apply(context.Context, *bundle.Bundle) error {
+func (n *noop) Apply(context.Context, *bundle.Bundle) diag.Diagnostics {
 	return nil
 }
 
@@ -35,8 +36,8 @@ func TestExpandGlobFilesSource(t *testing.T) {
 	t2.Close(t)
 
 	b := &bundle.Bundle{
+		RootPath: rootPath,
 		Config: config.Root{
-			Path: rootPath,
 			Artifacts: map[string]*config.Artifact{
 				"test": {
 					Type: "custom",
@@ -57,8 +58,8 @@ func TestExpandGlobFilesSource(t *testing.T) {
 		return &noop{}
 	}
 
-	err = bundle.Apply(context.Background(), b, u)
-	require.NoError(t, err)
+	diags := bundle.Apply(context.Background(), b, u)
+	require.NoError(t, diags.Error())
 
 	require.Equal(t, 2, len(b.Config.Artifacts["test"].Files))
 	require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source)
@@ -71,8 +72,8 @@ func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) {
 	require.NoError(t, err)
 
 	b := &bundle.Bundle{
+		RootPath: rootPath,
 		Config: config.Root{
-			Path: rootPath,
 			Artifacts: map[string]*config.Artifact{
 				"test": {
 					Type: "custom",
@@ -93,6 +94,6 @@ func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) {
 		return &noop{}
 	}
 
-	err = bundle.Apply(context.Background(), b, u)
-	require.ErrorContains(t, err, "no files found for")
+	diags := bundle.Apply(context.Background(), b, u)
+	require.ErrorContains(t, diags.Error(), "no files found for")
 }