Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add bundle summary to display URLs for deployed resources #1731

Merged
merged 23 commits into from
Oct 18, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
869637d
Add a textual bundle summary command
lennartkats-db Aug 24, 2024
462ee2e
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Aug 29, 2024
1325fd8
Fix typo
lennartkats-db Aug 29, 2024
44110d1
Merge branch 'main' into cp-summary-with-urls
pietern Sep 13, 2024
189d40b
Address reviewer comments
lennartkats-db Oct 5, 2024
8cd03d1
Merge branch 'cp-summary-with-urls' of github.com:lennartkats-db/cli …
lennartkats-db Oct 5, 2024
7911c67
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 5, 2024
65bad56
Fix test failure based on code from main
lennartkats-db Oct 5, 2024
ef2400f
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 11, 2024
d54f641
Fix test name
lennartkats-db Oct 11, 2024
6218539
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 11, 2024
aea4a6e
Styling fix
lennartkats-db Oct 17, 2024
2765a41
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 17, 2024
13049a5
Remove unused name field
pietern Oct 17, 2024
2c8bb75
Add singular/plural titles to config.SupportedResources()
pietern Oct 17, 2024
85bc79f
Use plural title resource header
pietern Oct 17, 2024
dccd70b
require -> assert
pietern Oct 17, 2024
b1dd60c
Include test case for clusters
pietern Oct 17, 2024
7bf2ec3
Always use ID to synthesize URLs
pietern Oct 17, 2024
7658dbd
Use net/url to build resource URLs such that proper escaping is done
pietern Oct 17, 2024
530a819
Merge branch 'main' into cp-summary-with-urls
lennartkats-db Oct 17, 2024
0cd3fcd
Update bundle/config/resources.go
pietern Oct 18, 2024
c45b058
Update bundle/config/mutator/initialize_urls_test.go
pietern Oct 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions bundle/config/mutator/initialize_urls.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package mutator

import (
"context"
"fmt"
"strconv"
"strings"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
)

type initializeURLs struct {
name string
}

// InitializeURLs makes sure the URL field of each resource is configured.
// NOTE: since this depends on an extra API call, this mutator adds some extra
// latency. As such, it should only be used when needed.
// This URL field is used for the output of the 'bundle summary' CLI command.
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
func InitializeURLs() bundle.Mutator {
return &initializeURLs{}
}

func (m *initializeURLs) Name() string {
return fmt.Sprintf("InitializeURLs(%s)", m.name)
}

func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
workspaceId, err := b.WorkspaceClient().CurrentWorkspaceID(ctx)
orgId := strconv.FormatInt(workspaceId, 10)
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return diag.FromErr(err)
}
urlPrefix := b.WorkspaceClient().Config.CanonicalHostName() + "/"
initializeForWorkspace(b, orgId, urlPrefix)
return nil
}

func initializeForWorkspace(b *bundle.Bundle, orgId string, urlPrefix string) {
// Add ?o=<workspace id> only if <workspace id> wasn't in the subdomain already.
// The ?o= is needed when vanity URLs / legacy workspace URLs are used.
// If it's not needed we prefer to leave it out since these URLs are rather
// long for most terminals.
pietern marked this conversation as resolved.
Show resolved Hide resolved
//
// See https://docs.databricks.com/en/workspace/workspace-details.html for
// further reading about the '?o=' suffix.
urlSuffix := ""
if !strings.Contains(urlPrefix, orgId) {
urlSuffix = "?o=" + orgId
}

for _, rs := range b.Config.Resources.AllResources() {
for _, r := range rs {
r.InitializeURL(urlPrefix, urlSuffix)
}
}
}
120 changes: 120 additions & 0 deletions bundle/config/mutator/initialize_urls_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package mutator

import (
"testing"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/serving"
"github.com/stretchr/testify/require"
)

func TestInitializeURLs(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
Host: "https://mycompany.databricks.com/",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
ID: "1",
JobSettings: &jobs.JobSettings{Name: "job1"},
},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {
ID: "3",
PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"},
},
},
Experiments: map[string]*resources.MlflowExperiment{
"experiment1": {
ID: "4",
Experiment: &ml.Experiment{Name: "experiment1"},
},
},
Models: map[string]*resources.MlflowModel{
"model1": {
ID: "6",
Model: &ml.Model{Name: "model1"},
pietern marked this conversation as resolved.
Show resolved Hide resolved
},
},
ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
"servingendpoint1": {
ID: "7",
CreateServingEndpoint: &serving.CreateServingEndpoint{
Name: "my_serving_endpoint",
},
},
},
RegisteredModels: map[string]*resources.RegisteredModel{
"registeredmodel1": {
ID: "8",
CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{
Name: "my_registered_model",
},
},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"qualityMonitor1": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "catalog.schema.qualityMonitor1",
},
},
},
Schemas: map[string]*resources.Schema{
"schema1": {
ID: "catalog.schema",
CreateSchema: &catalog.CreateSchema{
Name: "schema",
},
},
},
},
},
}

expectedURLs := map[string]string{
"job1": "https://mycompany.databricks.com/jobs/1?o=123456",
"pipeline1": "https://mycompany.databricks.com/pipelines/3?o=123456",
"experiment1": "https://mycompany.databricks.com/ml/experiments/4?o=123456",
"model1": "https://mycompany.databricks.com/ml/models/model1?o=123456",
"servingendpoint1": "https://mycompany.databricks.com/ml/endpoints/my_serving_endpoint?o=123456",
"registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456",
"qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456",
"schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456",
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
}

initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/")

for _, rs := range b.Config.Resources.AllResources() {
for key, r := range rs {
require.Equal(t, expectedURLs[key], r.GetURL(), "Unexpected URL for "+key)
}
}
}

func TestInitializeURLsWithoutOrgId(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
ID: "1",
JobSettings: &jobs.JobSettings{Name: "job1"},
},
},
},
},
}

initializeForWorkspace(b, "123456", "https://adb-123456.azuredatabricks.net/")

require.Equal(t, "https://adb-123456.azuredatabricks.net/jobs/1", b.Config.Resources.Jobs["job1"].URL)
}
69 changes: 69 additions & 0 deletions bundle/config/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,75 @@ type ConfigResource interface {
// Terraform equivalent name of the resource. For example "databricks_job"
// for jobs and "databricks_pipeline" for pipelines.
TerraformResourceName() string

// GetName returns the in-product name of the resource.
GetName() string

// GetURL returns the URL of the resource.
GetURL() string

// InitializeURL initializes the URL field of the resource.
InitializeURL(urlPrefix string, urlSuffix string)
}

func (r *Resources) AllResources() map[string]map[string]ConfigResource {
result := make(map[string]map[string]ConfigResource)

jobResources := make(map[string]ConfigResource)
for key, job := range r.Jobs {
jobResources[key] = job
}
result["jobs"] = jobResources

pipelineResources := make(map[string]ConfigResource)
for key, pipeline := range r.Pipelines {
pipelineResources[key] = pipeline
}
result["pipelines"] = pipelineResources

modelResources := make(map[string]ConfigResource)
for key, model := range r.Models {
modelResources[key] = model
}
result["models"] = modelResources

experimentResources := make(map[string]ConfigResource)
for key, experiment := range r.Experiments {
experimentResources[key] = experiment
}
result["experiments"] = experimentResources

modelServingEndpointResources := make(map[string]ConfigResource)
for key, endpoint := range r.ModelServingEndpoints {
modelServingEndpointResources[key] = endpoint
}
result["model_serving_endpoints"] = modelServingEndpointResources

registeredModelResources := make(map[string]ConfigResource)
for key, registeredModel := range r.RegisteredModels {
registeredModelResources[key] = registeredModel
}
result["registered_models"] = registeredModelResources

qualityMonitorResources := make(map[string]ConfigResource)
for key, qualityMonitor := range r.QualityMonitors {
qualityMonitorResources[key] = qualityMonitor
}
result["quality_monitors"] = qualityMonitorResources

schemaResources := make(map[string]ConfigResource)
for key, schema := range r.Schemas {
schemaResources[key] = schema
}
result["schemas"] = schemaResources

clusterResources := make(map[string]ConfigResource)
for key, schema := range r.Clusters {
clusterResources[key] = schema
}
result["clusters"] = clusterResources

return result
}

func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) {
Expand Down
16 changes: 16 additions & 0 deletions bundle/config/resources/clusters.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type Cluster struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`

*compute.ClusterSpec
}
Expand All @@ -37,3 +38,18 @@ func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id
func (s *Cluster) TerraformResourceName() string {
return "databricks_cluster"
}

func (s *Cluster) InitializeURL(urlPrefix string, urlSuffix string) {
if s.ID == "" {
return
}
s.URL = urlPrefix + "compute/clusters/" + s.ID + urlSuffix
}

func (s *Cluster) GetName() string {
return s.ClusterName
}

func (s *Cluster) GetURL() string {
return s.URL
}
16 changes: 16 additions & 0 deletions bundle/config/resources/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type Job struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved

*jobs.JobSettings
}
Expand Down Expand Up @@ -44,3 +45,18 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri
func (j *Job) TerraformResourceName() string {
return "databricks_job"
}

func (j *Job) InitializeURL(urlPrefix string, urlSuffix string) {
if j.ID == "" {
return
}
j.URL = urlPrefix + "jobs/" + j.ID + urlSuffix
}

func (j *Job) GetName() string {
return j.Name
}

func (j *Job) GetURL() string {
return j.URL
}
16 changes: 16 additions & 0 deletions bundle/config/resources/mlflow_experiment.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type MlflowExperiment struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`

*ml.Experiment
}
Expand All @@ -39,3 +40,18 @@ func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceCl
func (s *MlflowExperiment) TerraformResourceName() string {
return "databricks_mlflow_experiment"
}

func (s *MlflowExperiment) InitializeURL(urlPrefix string, urlSuffix string) {
if s.ID == "" {
return
}
s.URL = urlPrefix + "ml/experiments/" + s.ID + urlSuffix
}

func (s *MlflowExperiment) GetName() string {
return s.Name
}

func (s *MlflowExperiment) GetURL() string {
return s.URL
}
16 changes: 16 additions & 0 deletions bundle/config/resources/mlflow_model.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type MlflowModel struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`

*ml.Model
}
Expand All @@ -39,3 +40,18 @@ func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient,
func (s *MlflowModel) TerraformResourceName() string {
return "databricks_mlflow_model"
}

func (s *MlflowModel) InitializeURL(urlPrefix string, urlSuffix string) {
if s.ID == "" {
return
}
s.URL = urlPrefix + "ml/models/" + s.Name + urlSuffix
}

func (s *MlflowModel) GetName() string {
return s.Name
}

func (s *MlflowModel) GetURL() string {
return s.URL
}
16 changes: 16 additions & 0 deletions bundle/config/resources/model_serving_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type ModelServingEndpoint struct {
Permissions []Permission `json:"permissions,omitempty"`

ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`
}

func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error {
Expand All @@ -47,3 +48,18 @@ func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.Workspa
func (s *ModelServingEndpoint) TerraformResourceName() string {
return "databricks_model_serving"
}

func (s *ModelServingEndpoint) InitializeURL(urlPrefix string, urlSuffix string) {
if s.ID == "" {
return
}
s.URL = urlPrefix + "ml/endpoints/" + s.Name + urlSuffix
}

func (s *ModelServingEndpoint) GetName() string {
return s.Name
}

func (s *ModelServingEndpoint) GetURL() string {
return s.URL
}
Loading