Filter out system clusters for --configure-cluster (#1031)

## Changes

Only clusters with their source attribute equal to `UI` or `API` should
be presented in the dropdown.

## Tests

Unit test and manual confirmation.

cmd/configure/configure.go

@@ -50,7 +50,7 @@ func configureInteractive(cmd *cobra.Command, flags *configureFlags, cfg *config
 		if err != nil {
 			return err
 		}
-		clusterID, err := cfgpickers.AskForCluster(cmd.Context(), w)
+		clusterID, err := cfgpickers.AskForCluster(cmd.Context(), w, cfgpickers.WithoutSystemClusters())
 		if err != nil {
 			return err
 		}

libs/databrickscfg/cfgpickers/clusters.go

@@ -118,6 +118,18 @@ func WithDatabricksConnect(minVersion string) func(*compute.ClusterDetails, *iam
 	}
 }
 
+// WithoutSystemClusters removes clusters created for system purposes (e.g. job runs, pipeline maintenance, etc.).
+// It does this by keeping only clusters created through the UI or an API call.
+func WithoutSystemClusters() func(*compute.ClusterDetails, *iam.User) bool {
+	return func(cluster *compute.ClusterDetails, me *iam.User) bool {
+		switch cluster.ClusterSource {
+		case compute.ClusterSourceApi, compute.ClusterSourceUi:
+			return true
+		}
+		return false
+	}
+}
+
 func loadInteractiveClusters(ctx context.Context, w *databricks.WorkspaceClient, filters []clusterFilter) ([]compatibleCluster, error) {
 	promptSpinner := cmdio.Spinner(ctx)
 	promptSpinner <- "Loading list of clusters to select from"

libs/databrickscfg/cfgpickers/clusters_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/qa"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,6 +45,27 @@ func TestIsCompatibleWithSnapshots(t *testing.T) {
 	}, "14.0"))
 }
 
+func TestWithoutSystemClusters(t *testing.T) {
+	fn := WithoutSystemClusters()
+
+	// Sources to exclude.
+	for _, v := range []string{
+		"JOB",
+		"PIPELINE",
+		"SOME_UNKNOWN_VALUE",
+	} {
+		assert.False(t, fn(&compute.ClusterDetails{ClusterSource: compute.ClusterSource(v)}, nil))
+	}
+
+	// Sources to include.
+	for _, v := range []string{
+		"UI",
+		"API",
+	} {
+		assert.True(t, fn(&compute.ClusterDetails{ClusterSource: compute.ClusterSource(v)}, nil))
+	}
+}
+
 func TestFirstCompatibleCluster(t *testing.T) {
 	cfg, server := qa.HTTPFixtures{
 		{