Merge branch 'main' of github.com:databricks/cli into fix/source-link…

…ed-file-path-fixes

.codegen/service.go.tmpl

@@ -140,9 +140,9 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- end}}
     {{$method := .}}
 	{{ if not .IsJsonOnly }}
-	{{range $request.Fields -}}
+	{{range .AllFields -}}
 		{{- if not .Required -}}
-			{{if .Entity.IsObject }}// TODO: complex arg: {{.Name}}
+			{{if .Entity.IsObject}}{{if not (eq . $method.RequestBodyField) }}// TODO: complex arg: {{.Name}}{{end}}
 			{{else if .Entity.IsAny }}// TODO: any: {{.Name}}
 			{{else if .Entity.ArrayValue }}// TODO: array: {{.Name}}
 			{{else if .Entity.MapValue }}// TODO: map via StringToStringVar: {{.Name}}

.github/workflows/push.yml

@@ -60,6 +60,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
 
+      - name: Run ruff
+        uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
+        with:
+          version: "0.9.1"
+          args: "format --check"
+
       - name: Set go env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV

.gitignore

@@ -32,3 +32,4 @@ __pycache__
 .vscode/tasks.json
 
 .databricks
+.ruff_cache

.golangci.yaml

@@ -22,6 +22,13 @@ linters-settings:
     disable:
       - fieldalignment
       - shadow
+    settings:
+      printf:
+        funcs:
+          - (github.com/databricks/cli/internal/testutil.TestingT).Infof
+          - (github.com/databricks/cli/internal/testutil.TestingT).Errorf
+          - (github.com/databricks/cli/internal/testutil.TestingT).Fatalf
+          - (github.com/databricks/cli/internal/testutil.TestingT).Skipf
   gofmt:
     rewrite-rules:
       - pattern: 'a[b:len(a)]'
@@ -42,6 +49,8 @@ linters-settings:
     disable:
       # good check, but we have too many assert.(No)?Errorf? so excluding for now
       - require-error
+  copyloopvar:
+    check-alias: true
 issues:
   exclude-dirs-use-default: false  # recommended by docs https://golangci-lint.run/usage/false-positives/
   max-issues-per-linter: 1000

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Version changelog
 
+## [Release] Release v0.239.0
+
+### New feature announcement
+
+#### Databricks Apps support
+
+You can now manage Databricks Apps using DABs by defining an `app` resource in your bundle configuration.
+For more information see Databricks documentation https://docs.databricks.com/en/dev-tools/bundles/resources.html#app
+
+#### Referencing complex variables in complex variables
+
+You can now reference complex variables within other complex variables.
+For more details see https://github.com/databricks/cli/pull/2157
+
+CLI:
+ * Filter out system clusters in cluster picker ([#2131](https://github.com/databricks/cli/pull/2131)).
+ * Add command line flags for fields that are not in the API request body ([#2155](https://github.com/databricks/cli/pull/2155)).
+
+Bundles:
+ * Added support for Databricks Apps in DABs ([#1928](https://github.com/databricks/cli/pull/1928)).
+ * Allow artifact path to be located outside the sync root ([#2128](https://github.com/databricks/cli/pull/2128)).
+ * Retry app deployment if there is an active deployment in progress ([#2153](https://github.com/databricks/cli/pull/2153)).
+ * Resolve variables in a loop ([#2164](https://github.com/databricks/cli/pull/2164)).
+ * Improve resolution of complex variables within complex variables ([#2157](https://github.com/databricks/cli/pull/2157)).
+ * Added output message to warn about slower deployments with apps ([#2161](https://github.com/databricks/cli/pull/2161)).
+ * Patch references to UC schemas to capture dependencies automatically ([#1989](https://github.com/databricks/cli/pull/1989)).
+ * Format default-python template ([#2110](https://github.com/databricks/cli/pull/2110)).
+ * Encourage the use of root_path in production to ensure single deployment ([#1712](https://github.com/databricks/cli/pull/1712)).
+ * Log warnings to stderr for "bundle validate -o json" ([#2109](https://github.com/databricks/cli/pull/2109)).
+
+API Changes:
+ * Changed `databricks account federation-policy update` command with new required argument order.
+ * Changed `databricks account service-principal-federation-policy update` command with new required argument order.
+
+OpenAPI commit 779817ed8d63031f5ea761fbd25ee84f38feec0d (2025-01-08)
+Dependency updates:
+ * Upgrade TF provider to 1.63.0 ([#2162](https://github.com/databricks/cli/pull/2162)).
+ * Bump golangci-lint version to v1.63.4 from v1.63.1 ([#2114](https://github.com/databricks/cli/pull/2114)).
+ * Bump astral-sh/setup-uv from 4 to 5 ([#2116](https://github.com/databricks/cli/pull/2116)).
+ * Bump golang.org/x/oauth2 from 0.24.0 to 0.25.0 ([#2080](https://github.com/databricks/cli/pull/2080)).
+ * Bump github.com/hashicorp/hc-install from 0.9.0 to 0.9.1 ([#2079](https://github.com/databricks/cli/pull/2079)).
+ * Bump golang.org/x/term from 0.27.0 to 0.28.0 ([#2078](https://github.com/databricks/cli/pull/2078)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.54.0 to 0.55.0 ([#2126](https://github.com/databricks/cli/pull/2126)).
+
 ## [Release] Release v0.238.0
 
 Bundles:

Makefile

@@ -14,6 +14,7 @@ lintcheck:
 # formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
 # If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
 fmt:
+	ruff format -q
 	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
 
 test:
@@ -44,7 +45,7 @@ snapshot:
 
 vendor:
 	go mod vendor
-  
+
 schema:
 	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 

acceptance/acceptance_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"slices"
 	"sort"
@@ -89,6 +90,7 @@ func TestAccept(t *testing.T) {
 	require.NotNil(t, user)
 	testdiff.PrepareReplacementsUser(t, &repls, *user)
 	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
+	testdiff.PrepareReplacementsUUID(t, &repls)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
@@ -154,70 +156,86 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		require.NoError(t, err)
 		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
 	}
+
+	// Write combined output to a file
+	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
+	require.NoError(t, err)
+	cmd.Stdout = out
+	cmd.Stderr = out
 	cmd.Dir = tmpDir
-	outB, err := cmd.CombinedOutput()
+	err = cmd.Run()
 
-	out := formatOutput(string(outB), err)
-	out = repls.Replace(out)
-	doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
+	// Include exit code in output (if non-zero)
+	formatOutput(out, err)
+	require.NoError(t, out.Close())
 
-	for key := range outputs {
-		if key == "output.txt" {
-			// handled above
-			continue
-		}
-		pathNew := filepath.Join(tmpDir, key)
-		newValBytes, err := os.ReadFile(pathNew)
-		if err != nil {
-			if errors.Is(err, os.ErrNotExist) {
-				t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
-			} else {
-				t.Errorf("%s: could not read: %s", key, err)
-			}
-			continue
-		}
-		pathExpected := filepath.Join(dir, key)
-		newVal := repls.Replace(string(newValBytes))
-		doComparison(t, pathExpected, pathNew, newVal)
+	// Compare expected outputs
+	for relPath := range outputs {
+		doComparison(t, repls, dir, tmpDir, relPath)
 	}
 
 	// Make sure there are not unaccounted for new files
-	files, err := os.ReadDir(tmpDir)
+	files, err := ListDir(t, tmpDir)
 	require.NoError(t, err)
-
-	for _, f := range files {
-		name := f.Name()
-		if _, ok := inputs[name]; ok {
+	for _, relPath := range files {
+		if _, ok := inputs[relPath]; ok {
 			continue
 		}
-		if _, ok := outputs[name]; ok {
+		if _, ok := outputs[relPath]; ok {
 			continue
 		}
-		t.Errorf("Unexpected output: %s", f)
-		if strings.HasPrefix(name, "out") {
+		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
-			pathNew := filepath.Join(tmpDir, name)
-			newVal := testutil.ReadFile(t, pathNew)
-			newVal = repls.Replace(newVal)
-			doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
+			doComparison(t, repls, dir, tmpDir, relPath)
 		}
 	}
 }
 
-func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
-	valueNew = testdiff.NormalizeNewlines(valueNew)
-	valueExpected := string(readIfExists(t, pathExpected))
-	valueExpected = testdiff.NormalizeNewlines(valueExpected)
-	testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
-	if testdiff.OverwriteMode {
-		if valueNew != "" {
-			t.Logf("Overwriting: %s", pathExpected)
-			testutil.WriteFile(t, pathExpected, valueNew)
-		} else {
-			t.Logf("Removing: %s", pathExpected)
-			_ = os.Remove(pathExpected)
+func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string) {
+	pathRef := filepath.Join(dirRef, relPath)
+	pathNew := filepath.Join(dirNew, relPath)
+	bufRef, okRef := readIfExists(t, pathRef)
+	bufNew, okNew := readIfExists(t, pathNew)
+	if !okRef && !okNew {
+		t.Errorf("Both files are missing: %s, %s", pathRef, pathNew)
+		return
+	}
+
+	valueRef := testdiff.NormalizeNewlines(string(bufRef))
+	valueNew := testdiff.NormalizeNewlines(string(bufNew))
+
+	// Apply replacements to the new value only.
+	// The reference value is stored after applying replacements.
+	valueNew = repls.Replace(valueNew)
+
+	// The test did not produce an expected output file.
+	if okRef && !okNew {
+		t.Errorf("Missing output file: %s", relPath)
+		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		if testdiff.OverwriteMode {
+			t.Logf("Removing output file: %s", relPath)
+			require.NoError(t, os.Remove(pathRef))
+		}
+		return
+	}
+
+	// The test produced an unexpected output file.
+	if !okRef && okNew {
+		t.Errorf("Unexpected output file: %s", relPath)
+		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		if testdiff.OverwriteMode {
+			t.Logf("Writing output file: %s", relPath)
+			testutil.WriteFile(t, pathRef, valueNew)
 		}
+		return
+	}
+
+	// Compare the reference and new values.
+	equal := testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+	if !equal && testdiff.OverwriteMode {
+		t.Logf("Overwriting existing output file: %s", relPath)
+		testutil.WriteFile(t, pathRef, valueNew)
 	}
 }
 
@@ -234,13 +252,13 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	cleanups := []string{}
 
 	for {
-		x := readIfExists(t, filepath.Join(dir, CleanupScript))
-		if len(x) > 0 {
+		x, ok := readIfExists(t, filepath.Join(dir, CleanupScript))
+		if ok {
 			cleanups = append(cleanups, string(x))
 		}
 
-		x = readIfExists(t, filepath.Join(dir, PrepareScript))
-		if len(x) > 0 {
+		x, ok = readIfExists(t, filepath.Join(dir, PrepareScript))
+		if ok {
 			prepares = append(prepares, string(x))
 		}
 
@@ -316,29 +334,28 @@ func copyFile(src, dst string) error {
 	return err
 }
 
-func formatOutput(out string, err error) string {
+func formatOutput(w io.Writer, err error) {
 	if err == nil {
-		return out
+		return
 	}
 	if exiterr, ok := err.(*exec.ExitError); ok {
 		exitCode := exiterr.ExitCode()
-		out += fmt.Sprintf("\nExit code: %d\n", exitCode)
+		fmt.Fprintf(w, "\nExit code: %d\n", exitCode)
 	} else {
-		out += fmt.Sprintf("\nError: %s\n", err)
+		fmt.Fprintf(w, "\nError: %s\n", err)
 	}
-	return out
 }
 
-func readIfExists(t *testing.T, path string) []byte {
+func readIfExists(t *testing.T, path string) ([]byte, bool) {
 	data, err := os.ReadFile(path)
 	if err == nil {
-		return data
+		return data, true
 	}
 
 	if !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("%s: %s", path, err)
 	}
-	return []byte{}
+	return []byte{}, false
 }
 
 func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
@@ -353,8 +370,10 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 			return err
 		}
 
-		if strings.HasPrefix(name, "out") {
-			outputs[relPath] = true
+		if strings.HasPrefix(relPath, "out") {
+			if !info.IsDir() {
+				outputs[relPath] = true
+			}
 			return nil
 		} else {
 			inputs[relPath] = true
@@ -373,3 +392,47 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 		return copyFile(path, destPath)
 	})
 }
+
+func ListDir(t *testing.T, src string) ([]string, error) {
+	// exclude folders in .gitignore from comparison
+	ignored := []string{
+		"\\.ruff_cache",
+		"\\.venv",
+		".*\\.egg-info",
+		"__pycache__",
+		// depends on uv version
+		"uv.lock",
+	}
+
+	var files []string
+	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			for _, ignoredFolder := range ignored {
+				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
+					return filepath.SkipDir
+				}
+			}
+
+			return nil
+		} else {
+			for _, ignoredFolder := range ignored {
+				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
+					return nil
+				}
+			}
+		}
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		files = append(files, relPath)
+		return nil
+	})
+	return files, err
+}