diff --git a/api/v1/component_types.go b/api/v1/component_types.go index b70233293..0b626dbdc 100644 --- a/api/v1/component_types.go +++ b/api/v1/component_types.go @@ -16,13 +16,14 @@ type Component struct { } type ComponentSpec struct { - Name string `json:"name,omitempty"` - Tooltip string `json:"tooltip,omitempty"` - Icon string `json:"icon,omitempty"` - Owner string `json:"owner,omitempty"` - Id *Template `json:"id,omitempty"` //nolint - Order int `json:"order,omitempty"` - Labels map[string]string `json:"labels,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Tooltip string `json:"tooltip,omitempty"` + Icon string `json:"icon,omitempty"` + Owner string `json:"owner,omitempty"` + Id *Template `json:"id,omitempty"` //nolint + Order int `json:"order,omitempty"` + Labels map[string]string `json:"labels,omitempty"` // The type of component, e.g. service, API, website, library, database, etc. Type string `json:"type,omitempty"` // The lifecycle state of the component e.g. production, staging, dev, etc. @@ -51,6 +52,9 @@ type ComponentSpec struct { ForEach *ForEach `json:"forEach,omitempty"` // Logs is a list of logs selector for apm-hub. LogSelectors types.LogSelectors `json:"logs,omitempty"` + + // Reference to populate parent_id + ParentLookup *ParentLookup `json:"parentLookup,omitempty"` } // +kubebuilder:validation:Type=object @@ -84,6 +88,12 @@ func (f *ForEach) String() string { return fmt.Sprintf("ForEach(components=%d, properties=%d)", len(f.Components), len(f.Properties)) } +type ParentLookup struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Type string `json:"type,omitempty"` +} + type ComponentStatus struct { Status types.ComponentStatus `json:"status,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 744d4e0d8..4f2dd6f9a 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1032,6 +1032,11 @@ func (in *ComponentSpec) DeepCopyInto(out *ComponentSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ParentLookup != nil { + in, out := &in.ParentLookup, &out.ParentLookup + *out = new(ParentLookup) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentSpec. @@ -1127,6 +1132,11 @@ func (in *ComponentSpecObject) DeepCopyInto(out *ComponentSpecObject) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ParentLookup != nil { + in, out := &in.ParentLookup, &out.ParentLookup + *out = new(ParentLookup) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentSpecObject. @@ -2447,6 +2457,21 @@ func (in *OpenSearchCheck) DeepCopy() *OpenSearchCheck { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParentLookup) DeepCopyInto(out *ParentLookup) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParentLookup. +func (in *ParentLookup) DeepCopy() *ParentLookup { + if in == nil { + return nil + } + out := new(ParentLookup) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in diff --git a/config/deploy/crd.yaml b/config/deploy/crd.yaml index b717ee733..20586be5f 100644 --- a/config/deploy/crd.yaml +++ b/config/deploy/crd.yaml @@ -6900,10 +6900,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: @@ -7208,10 +7220,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: diff --git a/config/deploy/manifests.yaml b/config/deploy/manifests.yaml index 9b6020e03..fdecab482 100644 --- a/config/deploy/manifests.yaml +++ b/config/deploy/manifests.yaml @@ -7168,10 +7168,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: @@ -7476,10 +7488,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: diff --git a/config/schemas/component.schema.json b/config/schemas/component.schema.json index e80921d9b..773328100 100644 --- a/config/schemas/component.schema.json +++ b/config/schemas/component.schema.json @@ -789,6 +789,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -856,6 +859,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -866,6 +872,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -933,6 +942,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -3048,6 +3060,21 @@ "uid" ] }, + "ParentLookup": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "PodCheck": { "properties": { "description": { diff --git a/config/schemas/topology.schema.json b/config/schemas/topology.schema.json index 3eeec23c8..a3e7d6ae0 100644 --- a/config/schemas/topology.schema.json +++ b/config/schemas/topology.schema.json @@ -768,6 +768,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -835,6 +838,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -845,6 +851,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -912,6 +921,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -3018,6 +3030,21 @@ "uid" ] }, + "ParentLookup": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "PodCheck": { "properties": { "description": { diff --git a/fixtures/topology/component-with-parent-lookup.yml b/fixtures/topology/component-with-parent-lookup.yml new file mode 100644 index 000000000..1f6bd6f64 --- /dev/null +++ b/fixtures/topology/component-with-parent-lookup.yml @@ -0,0 +1,31 @@ +apiVersion: canaries.flanksource.com/v1 +kind: Topology +metadata: + name: test-topology-with-parent-lookup +spec: + schedule: "@every 10m" + components: + - name: Parent-1 + type: Type1 + components: + - name: Child-1A + - name: Child-1B + - name: Child-1C + parentLookup: + name: Parent-2 + type: Type2 + - name: Child-1D + parentLookup: + name: Parent-3 + type: Type3 + namespace: parent3-namespace + + - name: Parent-2 + type: Type2 + components: + - name: Child-2A + - name: Child-2B + + - name: Parent-3 + type: Type3 + namespace: parent3-namespace diff --git a/pkg/system_api.go b/pkg/system_api.go index f022234b6..b9b113ed4 100644 --- a/pkg/system_api.go +++ b/pkg/system_api.go @@ -134,7 +134,7 @@ type Component struct { ComponentChecks v1.ComponentChecks `json:"-" gorm:"componentChecks" swaggerignore:"true"` Checks Checks `json:"checks,omitempty" gorm:"-"` Configs dutyTypes.ConfigQueries `json:"configs,omitempty" gorm:"type:configs"` - TopologyID *uuid.UUID `json:"topology_id,omitempty"` //nolint + TopologyID uuid.UUID `json:"topology_id,omitempty"` //nolint CreatedAt time.Time `json:"created_at,omitempty" time_format:"postgres_timestamp"` UpdatedAt time.Time `json:"updated_at,omitempty" time_format:"postgres_timestamp"` DeletedAt *time.Time `json:"deleted_at,omitempty" time_format:"postgres_timestamp" swaggerignore:"true"` @@ -148,6 +148,7 @@ type Component struct { CostTotal7d float64 `json:"cost_total_7d,omitempty" gorm:"column:cost_total_7d"` CostTotal30d float64 `json:"cost_total_30d,omitempty" gorm:"column:cost_total_30d"` LogSelectors dutyTypes.LogSelectors `json:"logs,omitempty" gorm:"column:log_selectors"` + ParentLookup *v1.ParentLookup `json:"parentLookup,omitempty" gorm:"-"` } func (component *Component) FindExisting(db *gorm.DB) (*models.Component, error) { @@ -245,6 +246,7 @@ func (component Component) GetAsEnvironment() map[string]interface{} { func NewComponent(c v1.ComponentSpec) *Component { _c := Component{ Name: c.Name, + Namespace: c.Namespace, Owner: c.Owner, Type: c.Type, Order: c.Order, @@ -256,6 +258,7 @@ func NewComponent(c v1.ComponentSpec) *Component { Labels: c.Labels, Configs: c.Configs, LogSelectors: c.LogSelectors, + ParentLookup: c.ParentLookup, } if c.Summary != nil { _c.Summary = *c.Summary diff --git a/pkg/topology/component_check_test.go b/pkg/topology/component_check_test.go index aeafdcf8b..6ef6c7d10 100644 --- a/pkg/topology/component_check_test.go +++ b/pkg/topology/component_check_test.go @@ -11,6 +11,7 @@ import ( ) var _ = ginkgo.Describe("Topology checks", ginkgo.Ordered, func() { + topology := pkg.Topology{Name: "Topology ComponentCheck"} component := pkg.Component{ Name: "Component", ComponentChecks: []v1.ComponentCheck{{ @@ -27,7 +28,11 @@ var _ = ginkgo.Describe("Topology checks", ginkgo.Ordered, func() { } ginkgo.BeforeAll(func() { - err := DefaultContext.DB().Create(&component).Error + err := DefaultContext.DB().Create(&topology).Error + Expect(err).To(BeNil()) + + component.TopologyID = topology.ID + err = DefaultContext.DB().Create(&component).Error Expect(err).To(BeNil()) err = DefaultContext.DB().Create(&canary).Error diff --git a/pkg/topology/component_config_test.go b/pkg/topology/component_config_test.go index 65df728d9..fe1d441e6 100644 --- a/pkg/topology/component_config_test.go +++ b/pkg/topology/component_config_test.go @@ -9,6 +9,7 @@ import ( ) var _ = ginkgo.Describe("Topology configs", ginkgo.Ordered, func() { + topology := pkg.Topology{Name: "Topology ComponentConfig"} component := pkg.Component{ Name: "Component with configs", Configs: types.ConfigQueries{ @@ -21,7 +22,11 @@ var _ = ginkgo.Describe("Topology configs", ginkgo.Ordered, func() { } ginkgo.BeforeAll(func() { - err := DefaultContext.DB().Save(&component).Error + err := DefaultContext.DB().Save(&topology).Error + Expect(err).To(BeNil()) + + component.TopologyID = topology.ID + err = DefaultContext.DB().Save(&component).Error Expect(err).To(BeNil()) }) diff --git a/pkg/topology/component_relationship_test.go b/pkg/topology/component_relationship_test.go index c97fe6888..b381517e2 100644 --- a/pkg/topology/component_relationship_test.go +++ b/pkg/topology/component_relationship_test.go @@ -8,6 +8,7 @@ import ( ) var _ = ginkgo.Describe("Topology relationships", ginkgo.Ordered, func() { + topology := pkg.Topology{Name: "Topology ComponentRelationship"} parentComponent := pkg.Component{ Name: "Component", Selectors: []types.ResourceSelector{ @@ -31,10 +32,17 @@ var _ = ginkgo.Describe("Topology relationships", ginkgo.Ordered, func() { } ginkgo.BeforeAll(func() { + err := DefaultContext.DB().Create(&topology).Error + Expect(err).To(BeNil()) + + parentComponent.TopologyID = topology.ID ComponentRelationshipSync.Context = DefaultContext - err := DefaultContext.DB().Create(&parentComponent).Error + err = DefaultContext.DB().Create(&parentComponent).Error Expect(err).To(BeNil()) + childComponent1.TopologyID = topology.ID + childComponent2.TopologyID = topology.ID + childComponent3.TopologyID = topology.ID err = DefaultContext.DB().Create(pkg.Components{&childComponent1, &childComponent2, &childComponent3}).Error Expect(err).To(BeNil()) }) diff --git a/pkg/topology/run.go b/pkg/topology/run.go index dad3bf43c..75b494e86 100644 --- a/pkg/topology/run.go +++ b/pkg/topology/run.go @@ -336,6 +336,37 @@ func mergeComponentProperties(components pkg.Components, propertiesRaw []byte) e return nil } +func populateParentRefMap(c *pkg.Component, parentRefMap map[string]*pkg.Component) { + parentRefMap[genParentKey(c.Name, c.Type, c.Namespace)] = c + for _, child := range c.Components { + populateParentRefMap(child, parentRefMap) + } +} + +func changeComponentParents(c *pkg.Component, parentRefMap map[string]*pkg.Component) { + var children pkg.Components + for _, child := range c.Components { + if child.ParentLookup == nil { + children = append(children, child) + continue + } + + key := genParentKey(child.ParentLookup.Name, child.ParentLookup.Type, child.ParentLookup.Namespace) + if parentComp, exists := parentRefMap[key]; exists { + // Set nil to prevent processing again + child.ParentLookup = nil + parentComp.Components = append(parentComp.Components, child) + } else { + children = append(children, child) + } + } + c.Components = children + + for _, child := range c.Components { + changeComponentParents(child, parentRefMap) + } +} + type TopologyRunOptions struct { dutyContext.Context Depth int @@ -410,6 +441,11 @@ func Run(opts TopologyRunOptions, t v1.Topology) ([]*pkg.Component, models.JobHi } } + // Update component parents based on ParentLookup + parentRefMap := make(map[string]*pkg.Component) + populateParentRefMap(rootComponent, parentRefMap) + changeComponentParents(rootComponent, parentRefMap) + if len(rootComponent.Components) == 1 && rootComponent.Components[0].Type == "virtual" { // if there is only one component and it is virtual, then we don't need to show it ctx.Components = &rootComponent.Components[0].Components @@ -489,10 +525,12 @@ func SyncComponents(opts TopologyRunOptions, topology v1.Topology) (int, error) var compIDs []uuid.UUID for _, component := range components { + // Is this step required ever ? component.Name = topology.Name component.Namespace = topology.Namespace component.Labels = topology.Labels - component.TopologyID = &topologyID + component.TopologyID = topologyID + componentsIDs, err := db.PersistComponent(opts.Context, component) if err != nil { return 0, fmt.Errorf("failed to persist component(id=%s, name=%s): %w", component.ID, component.Name, err) diff --git a/pkg/topology/run_test.go b/pkg/topology/run_test.go index 07d1868df..adf575404 100644 --- a/pkg/topology/run_test.go +++ b/pkg/topology/run_test.go @@ -105,6 +105,41 @@ var _ = ginkgo.Describe("Topology run", ginkgo.Ordered, func() { Expect(componentC.Configs[0].Name).To(Equal(componentC.Name)) Expect(componentC.Configs[0].Type).To(Equal("Service")) }) + + ginkgo.It("should update component's parents", func() { + t, err := yamlFileToTopology("../../fixtures/topology/component-with-parent-lookup.yml") + if err != nil { + ginkgo.Fail("Error converting yaml to v1.Topology:" + err.Error()) + } + + rootComponent, history := Run(TopologyRunOptions{ + Context: DefaultContext, + Depth: 10, + Namespace: "default", + }, t) + + Expect(history.Errors).To(HaveLen(0)) + + Expect(len(rootComponent[0].Components)).To(Equal(3)) + + parent1 := rootComponent[0].Components[0] + parent2 := rootComponent[0].Components[1] + parent3 := rootComponent[0].Components[2] + + Expect(len(parent1.Components)).To(Equal(2)) + Expect(len(parent2.Components)).To(Equal(3)) + Expect(len(parent3.Components)).To(Equal(1)) + + Expect(parent1.Components[0].Name).To(Equal("Child-1A")) + Expect(parent1.Components[1].Name).To(Equal("Child-1B")) + + Expect(parent2.Components[0].Name).To(Equal("Child-2A")) + Expect(parent2.Components[1].Name).To(Equal("Child-2B")) + Expect(parent2.Components[2].Name).To(Equal("Child-1C")) + + Expect(parent3.Components[0].Name).To(Equal("Child-1D")) + }) + }) func yamlFileToTopology(file string) (t v1.Topology, err error) { diff --git a/pkg/topology/utils.go b/pkg/topology/utils.go index e81ae0890..936ebe90a 100644 --- a/pkg/topology/utils.go +++ b/pkg/topology/utils.go @@ -1,5 +1,7 @@ package topology +import "strings" + func isComponent(s map[string]interface{}) bool { _, name := s["name"] _, properties := s["properties"] @@ -33,3 +35,7 @@ func isComponentList(data []byte) bool { } return isComponent(s[0]) } + +func genParentKey(name, _type, namespace string) string { + return strings.Join([]string{name, _type, namespace}, "/") +}