Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls

.codegen/_openapi_sha

@@ -1 +1 @@
-d05898328669a3f8ab0c2ecee37db2673d3ea3f7
+6f6b1371e640f2dfeba72d365ac566368656f6b6

.gitattributes

@@ -6,6 +6,7 @@ cmd/account/cmd.go linguist-generated=true
 cmd/account/credentials/credentials.go linguist-generated=true
 cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true
 cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true
+cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
 cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
 cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
 cmd/account/groups/groups.go linguist-generated=true
@@ -52,6 +53,7 @@ cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true
 cmd/workspace/dashboards/dashboards.go linguist-generated=true
 cmd/workspace/data-sources/data-sources.go linguist-generated=true
 cmd/workspace/default-namespace/default-namespace.go linguist-generated=true
+cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true
 cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true
 cmd/workspace/experiments/experiments.go linguist-generated=true
 cmd/workspace/external-locations/external-locations.go linguist-generated=true
@@ -108,6 +110,7 @@ cmd/workspace/storage-credentials/storage-credentials.go linguist-generated=true
 cmd/workspace/system-schemas/system-schemas.go linguist-generated=true
 cmd/workspace/table-constraints/table-constraints.go linguist-generated=true
 cmd/workspace/tables/tables.go linguist-generated=true
+cmd/workspace/temporary-table-credentials/temporary-table-credentials.go linguist-generated=true
 cmd/workspace/token-management/token-management.go linguist-generated=true
 cmd/workspace/tokens/tokens.go linguist-generated=true
 cmd/workspace/users/users.go linguist-generated=true

.github/workflows/push.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -68,7 +68,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # No need to download cached dependencies when running gofmt.
           cache: false
@@ -100,7 +100,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
       # Github repo: https://github.com/ajv-validator/ajv-cli
       - name: Install ajv-cli

.github/workflows/release-snapshot.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action

CHANGELOG.md

@@ -1,5 +1,72 @@
 # Version changelog
 
+## [Release] Release v0.229.0
+
+Bundles:
+ * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)).
+ * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)).
+ * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)).
+ * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)).
+ * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)).
+ * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)).
+ * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)).
+
+Internal:
+ * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)).
+ * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)).
+ * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)).
+ * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)).
+ * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)).
+ * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)).
+ * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)).
+ * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)).
+ * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)).
+ * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)).
+
+API Changes:
+ * Changed `databricks apps delete` command to return .
+ * Changed `databricks apps deploy` command with new required argument order.
+ * Changed `databricks apps start` command to return .
+ * Changed `databricks apps stop` command to return .
+ * Added `databricks temporary-table-credentials` command group.
+ * Added `databricks serving-endpoints put-ai-gateway` command.
+ * Added `databricks disable-legacy-access` command group.
+ * Added `databricks account disable-legacy-features` command group.
+
+OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19)
+Dependency updates:
+ * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)).
+ * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)).
+ * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)).
+ * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)).
+
+## [Release] Release v0.228.1
+
+Bundles:
+ * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)).
+ * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)).
+ * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)).
+ * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)).
+ * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)).
+ * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)).
+
+Internal:
+ * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)).
+ * Make bundle JSON schema modular with `` ([#1700](https://github.com/databricks/cli/pull/1700)).
+ * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)).
+ * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)).
+ * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)).
+
+API Changes:
+ * Added `databricks quality-monitors regenerate-dashboard` command.
+
+OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04)
+Dependency updates:
+ * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)).
+ * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)).
+ * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)).
+
 ## [Release] Release v0.228.0
 
 CLI:

bundle/artifacts/expand_globs_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +24,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
 	testutil.Touch(t, tmpDir, "bc.txt")
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -36,7 +37,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(
@@ -62,7 +63,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -77,7 +78,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(
@@ -110,7 +111,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
 	testutil.Touch(t, tmpDir, "b2.txt")
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -125,7 +126,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(

bundle/artifacts/prepare.go

@@ -47,7 +47,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 
 	// If artifact path is not provided, use bundle root dir
 	if artifact.Path == "" {
-		artifact.Path = b.RootPath
+		artifact.Path = b.BundleRootPath
 	}
 
 	if !filepath.IsAbs(artifact.Path) {

bundle/artifacts/whl/autodetect.go

@@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	log.Infof(ctx, "Detecting Python wheel project...")
 
 	// checking if there is setup.py in the bundle root
-	setupPy := filepath.Join(b.RootPath, "setup.py")
+	setupPy := filepath.Join(b.BundleRootPath, "setup.py")
 	_, err := os.Stat(setupPy)
 	if err != nil {
 		log.Infof(ctx, "No Python wheel project found at bundle root folder")
 		return nil
 	}
 
-	log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath))
+	log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath))
 	module := extractModuleName(setupPy)
 
 	if b.Config.Artifacts == nil {
 		b.Config.Artifacts = make(map[string]*config.Artifact)
 	}
 
-	pkgPath, err := filepath.Abs(b.RootPath)
+	pkgPath, err := filepath.Abs(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/bundle.go

@@ -31,22 +31,26 @@ import (
 const internalFolder = ".internal"
 
 type Bundle struct {
-	// RootPath contains the directory path to the root of the bundle.
+	// BundleRootPath is the local path to the root directory of the bundle.
 	// It is set when we instantiate a new bundle instance.
-	RootPath string
+	BundleRootPath string
 
-	// BundleRoot is a virtual filesystem path to the root of the bundle.
+	// BundleRoot is a virtual filesystem path to [BundleRootPath].
 	// Exclusively use this field for filesystem operations.
 	BundleRoot vfs.Path
 
-	// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
-	// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
-	SyncRoot vfs.Path
-
 	// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
-	// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
+	// By default, it is the same as [BundleRootPath].
+	// If it is different, it must be an ancestor to [BundleRootPath].
+	// That is, [SyncRootPath] must contain [BundleRootPath].
 	SyncRootPath string
 
+	// SyncRoot is a virtual filesystem path to [SyncRootPath].
+	// Exclusively use this field for filesystem operations.
+	SyncRoot vfs.Path
+
+	// Config contains the bundle configuration.
+	// It is loaded from the bundle configuration files and mutators may update it.
 	Config config.Root
 
 	// Metadata about the bundle deployment. This is the interface Databricks services
@@ -84,14 +88,14 @@ type Bundle struct {
 
 func Load(ctx context.Context, path string) (*Bundle, error) {
 	b := &Bundle{
-		RootPath:   filepath.Clean(path),
-		BundleRoot: vfs.MustNew(path),
+		BundleRootPath: filepath.Clean(path),
+		BundleRoot:     vfs.MustNew(path),
 	}
 	configFile, err := config.FileNames.FindInPath(path)
 	if err != nil {
 		return nil, err
 	}
-	log.Debugf(ctx, "Found bundle root at %s (file %s)", b.RootPath, configFile)
+	log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile)
 	return b, nil
 }
 
@@ -160,7 +164,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
 	if !exists || cacheDirName == "" {
 		cacheDirName = filepath.Join(
 			// Anchor at bundle root directory.
-			b.RootPath,
+			b.BundleRootPath,
 			// Static cache directory.
 			".databricks",
 			"bundle",
@@ -212,7 +216,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
+	internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir)
 	if err != nil {
 		return nil, err
 	}

bundle/bundle_read_only.go

@@ -21,7 +21,7 @@ func (r ReadOnlyBundle) Config() config.Root {
 }
 
 func (r ReadOnlyBundle) RootPath() string {
-	return r.b.RootPath
+	return r.b.BundleRootPath
 }
 
 func (r ReadOnlyBundle) BundleRoot() vfs.Path {

bundle/bundle_test.go

@@ -79,7 +79,7 @@ func TestBundleMustLoadSuccess(t *testing.T) {
 	t.Setenv(env.RootVariable, "./tests/basic")
 	b, err := MustLoad(context.Background())
 	require.NoError(t, err)
-	assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
+	assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
 }
 
 func TestBundleMustLoadFailureWithEnv(t *testing.T) {
@@ -98,7 +98,7 @@ func TestBundleTryLoadSuccess(t *testing.T) {
 	t.Setenv(env.RootVariable, "./tests/basic")
 	b, err := TryLoad(context.Background())
 	require.NoError(t, err)
-	assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
+	assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
 }
 
 func TestBundleTryLoadFailureWithEnv(t *testing.T) {

bundle/config/bundle.go

@@ -38,8 +38,11 @@ type Bundle struct {
 	// Annotated readonly as this should be set at the target level.
 	Mode Mode `json:"mode,omitempty" bundle:"readonly"`
 
-	// Overrides the compute used for jobs and other supported assets.
-	ComputeID string `json:"compute_id,omitempty"`
+	// DEPRECATED: Overrides the compute used for jobs and other supported assets.
+	ComputeId string `json:"compute_id,omitempty"`
+
+	// Overrides the cluster used for jobs and other supported assets.
+	ClusterId string `json:"cluster_id,omitempty"`
 
 	// Deployment section specifies deployment related configuration for bundle
 	Deployment Deployment `json:"deployment,omitempty"`

bundle/config/loader/entry_point.go

@@ -20,7 +20,7 @@ func (m *entryPoint) Name() string {
 }
 
 func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
-	path, err := config.FileNames.FindInPath(b.RootPath)
+	path, err := config.FileNames.FindInPath(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/config/loader/entry_point_test.go

@@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) {
 
 func TestEntryPoint(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "testdata",
+		BundleRootPath: "testdata",
 	}
 	diags := bundle.Apply(context.Background(), b, loader.EntryPoint())
 	require.NoError(t, diags.Error())

bundle/config/loader/process_include_test.go

@@ -14,15 +14,15 @@ import (
 
 func TestProcessInclude(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "testdata",
+		BundleRootPath: "testdata",
 		Config: config.Root{
 			Workspace: config.Workspace{
 				Host: "foo",
 			},
 		},
 	}
 
-	m := loader.ProcessInclude(filepath.Join(b.RootPath, "host.yml"), "host.yml")
+	m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, "host.yml"), "host.yml")
 	assert.Equal(t, "ProcessInclude(host.yml)", m.Name())
 
 	// Assert the host value prior to applying the mutator