diff --git a/api/v1/component_types.go b/api/v1/component_types.go index 97addef29..0b626dbdc 100644 --- a/api/v1/component_types.go +++ b/api/v1/component_types.go @@ -2,10 +2,8 @@ package v1 import ( "fmt" - "strings" "github.com/flanksource/duty/types" - "github.com/google/uuid" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -18,13 +16,14 @@ type Component struct { } type ComponentSpec struct { - Name string `json:"name,omitempty"` - Tooltip string `json:"tooltip,omitempty"` - Icon string `json:"icon,omitempty"` - Owner string `json:"owner,omitempty"` - Id *Template `json:"id,omitempty"` //nolint - Order int `json:"order,omitempty"` - Labels map[string]string `json:"labels,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Tooltip string `json:"tooltip,omitempty"` + Icon string `json:"icon,omitempty"` + Owner string `json:"owner,omitempty"` + Id *Template `json:"id,omitempty"` //nolint + Order int `json:"order,omitempty"` + Labels map[string]string `json:"labels,omitempty"` // The type of component, e.g. service, API, website, library, database, etc. Type string `json:"type,omitempty"` // The lifecycle state of the component e.g. production, staging, dev, etc. @@ -61,16 +60,6 @@ type ComponentSpec struct { // +kubebuilder:validation:Type=object type ComponentSpecObject ComponentSpec -type ParentLookup struct { - Name string `json:"name,omitempty"` - Namespace string `json:"namespace,omitempty"` - Type string `json:"type,omitempty"` -} - -func (p ParentLookup) CacheKey(topologyID uuid.UUID) string { - return strings.Join([]string{topologyID.String(), p.Name, p.Namespace, p.Type}, "/") -} - func (c ComponentSpec) String() string { if c.Name != "" { return c.Name @@ -99,6 +88,12 @@ func (f *ForEach) String() string { return fmt.Sprintf("ForEach(components=%d, properties=%d)", len(f.Components), len(f.Properties)) } +type ParentLookup struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Type string `json:"type,omitempty"` +} + type ComponentStatus struct { Status types.ComponentStatus `json:"status,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 744d4e0d8..4f2dd6f9a 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1032,6 +1032,11 @@ func (in *ComponentSpec) DeepCopyInto(out *ComponentSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ParentLookup != nil { + in, out := &in.ParentLookup, &out.ParentLookup + *out = new(ParentLookup) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentSpec. @@ -1127,6 +1132,11 @@ func (in *ComponentSpecObject) DeepCopyInto(out *ComponentSpecObject) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ParentLookup != nil { + in, out := &in.ParentLookup, &out.ParentLookup + *out = new(ParentLookup) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentSpecObject. @@ -2447,6 +2457,21 @@ func (in *OpenSearchCheck) DeepCopy() *OpenSearchCheck { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParentLookup) DeepCopyInto(out *ParentLookup) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParentLookup. +func (in *ParentLookup) DeepCopy() *ParentLookup { + if in == nil { + return nil + } + out := new(ParentLookup) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in diff --git a/config/deploy/crd.yaml b/config/deploy/crd.yaml index b717ee733..20586be5f 100644 --- a/config/deploy/crd.yaml +++ b/config/deploy/crd.yaml @@ -6900,10 +6900,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: @@ -7208,10 +7220,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: diff --git a/config/deploy/manifests.yaml b/config/deploy/manifests.yaml index 9b6020e03..fdecab482 100644 --- a/config/deploy/manifests.yaml +++ b/config/deploy/manifests.yaml @@ -7168,10 +7168,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: @@ -7476,10 +7488,22 @@ spec: x-kubernetes-preserve-unknown-fields: true name: type: string + namespace: + type: string order: type: integer owner: type: string + parentLookup: + description: Reference to populate parent_id + properties: + name: + type: string + namespace: + type: string + type: + type: string + type: object properties: items: properties: diff --git a/config/schemas/component.schema.json b/config/schemas/component.schema.json index e80921d9b..773328100 100644 --- a/config/schemas/component.schema.json +++ b/config/schemas/component.schema.json @@ -789,6 +789,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -856,6 +859,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -866,6 +872,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -933,6 +942,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -3048,6 +3060,21 @@ "uid" ] }, + "ParentLookup": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "PodCheck": { "properties": { "description": { diff --git a/config/schemas/topology.schema.json b/config/schemas/topology.schema.json index 3eeec23c8..a3e7d6ae0 100644 --- a/config/schemas/topology.schema.json +++ b/config/schemas/topology.schema.json @@ -768,6 +768,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -835,6 +838,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -845,6 +851,9 @@ "name": { "type": "string" }, + "namespace": { + "type": "string" + }, "tooltip": { "type": "string" }, @@ -912,6 +921,9 @@ }, "logs": { "$ref": "#/$defs/LogSelectors" + }, + "parentLookup": { + "$ref": "#/$defs/ParentLookup" } }, "additionalProperties": false, @@ -3018,6 +3030,21 @@ "uid" ] }, + "ParentLookup": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "PodCheck": { "properties": { "description": { diff --git a/fixtures/topology/component-with-parent-lookup.yml b/fixtures/topology/component-with-parent-lookup.yml new file mode 100644 index 000000000..1f6bd6f64 --- /dev/null +++ b/fixtures/topology/component-with-parent-lookup.yml @@ -0,0 +1,31 @@ +apiVersion: canaries.flanksource.com/v1 +kind: Topology +metadata: + name: test-topology-with-parent-lookup +spec: + schedule: "@every 10m" + components: + - name: Parent-1 + type: Type1 + components: + - name: Child-1A + - name: Child-1B + - name: Child-1C + parentLookup: + name: Parent-2 + type: Type2 + - name: Child-1D + parentLookup: + name: Parent-3 + type: Type3 + namespace: parent3-namespace + + - name: Parent-2 + type: Type2 + components: + - name: Child-2A + - name: Child-2B + + - name: Parent-3 + type: Type3 + namespace: parent3-namespace diff --git a/pkg/db/topology.go b/pkg/db/topology.go index 90e2599f6..1b90b797b 100644 --- a/pkg/db/topology.go +++ b/pkg/db/topology.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "time" v1 "github.com/flanksource/canary-checker/api/v1" "github.com/flanksource/canary-checker/pkg" @@ -12,7 +11,6 @@ import ( "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/google/uuid" - "github.com/patrickmn/go-cache" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -109,7 +107,7 @@ func PersistComponent(ctx context.Context, component *pkg.Component) ([]uuid.UUI child.Path = component.ID.String() } - child.ParentId = getComponentParent(ctx, child, component) + child.ParentId = &component.ID if childIDs, err := PersistComponent(ctx, child); err != nil { logger.Errorf("Error persisting child component of %v, :v", component.ID, err) } else { @@ -120,53 +118,6 @@ func PersistComponent(ctx context.Context, component *pkg.Component) ([]uuid.UUI return persisted, tx.Error } -// Component parent can either be a lookup for the direct ID of the component if ParentLookup is nil -func getComponentParent(ctx context.Context, child, component *pkg.Component) *uuid.UUID { - if child.ParentLookup == nil { - return &component.ID - } - - parentID, err := lookupComponentParent(ctx, *child.ParentLookup, component.TopologyID) - if err != nil { - logger.Errorf("Error looking up component parent with lookup spec %v of topology[%s]: %v", *child.ParentLookup, component.TopologyID, err) - return nil - } - return parentID -} - -var componentParentCache = cache.New(3*24*time.Hour, 3*24*time.Hour) - -func lookupComponentParent(ctx context.Context, parentLookup v1.ParentLookup, topologyID uuid.UUID) (*uuid.UUID, error) { - if parentLookup.Name == "" || parentLookup.Type == "" { - return nil, fmt.Errorf("name or type field missing from spec") - } - - // Check cache - cacheKey := parentLookup.CacheKey(topologyID) - if parentID, exists := componentParentCache.Get(cacheKey); exists { - return parentID.(*uuid.UUID), nil - } - - var parentID *uuid.UUID - query := ctx.DB().Table("components"). - Select("id"). - Where(duty.LocalFilter). - Where("topology_id = ?", topologyID). - Where("name = ?", parentLookup.Name). - Where("type = ?", parentLookup.Type) - - if parentLookup.Namespace != "" { - query = query.Where("namespace = ?", parentLookup.Namespace) - } - - if err := query.First(&parentID).Error; err != nil { - return nil, fmt.Errorf("error querying parent_id from components table: %w", err) - } - - componentParentCache.SetDefault(cacheKey, parentID) - return parentID, nil -} - func UpdateStatusAndSummaryForComponent(db *gorm.DB, id uuid.UUID, status types.ComponentStatus, summary types.Summary) (int64, error) { tx := db.Table("components").Where("id = ? and (status != ? or summary != ?)", id, status, summary). UpdateColumns(models.Component{Status: status, Summary: summary}) diff --git a/pkg/system_api.go b/pkg/system_api.go index 08f71f6e4..b9b113ed4 100644 --- a/pkg/system_api.go +++ b/pkg/system_api.go @@ -246,6 +246,7 @@ func (component Component) GetAsEnvironment() map[string]interface{} { func NewComponent(c v1.ComponentSpec) *Component { _c := Component{ Name: c.Name, + Namespace: c.Namespace, Owner: c.Owner, Type: c.Type, Order: c.Order, diff --git a/pkg/topology/run.go b/pkg/topology/run.go index 77d6355db..cf2efc173 100644 --- a/pkg/topology/run.go +++ b/pkg/topology/run.go @@ -11,6 +11,7 @@ import ( "github.com/flanksource/canary-checker/pkg/db" "github.com/flanksource/canary-checker/pkg/utils" "github.com/flanksource/commons/collections" + "github.com/flanksource/commons/logger" dutyContext "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" "github.com/flanksource/duty/query" @@ -336,6 +337,47 @@ func mergeComponentProperties(components pkg.Components, propertiesRaw []byte) e return nil } +func populateParentRefMap(c *pkg.Component, parentRefMap map[string]*pkg.Component) { + parentRefMap[genParentKey(c.Name, c.Type, c.Namespace)] = c + logger.Infof("YASH KEY SET %s", genParentKey(c.Name, c.Type, c.Namespace)) + for _, child := range c.Components { + populateParentRefMap(child, parentRefMap) + } +} + +func changeParents(c *pkg.Component, parentRefMap map[string]*pkg.Component) { + var children pkg.Components + for _, child := range c.Components { + if child.ParentLookup == nil { + children = append(children, child) + continue + } + + key := genParentKey(child.ParentLookup.Name, child.ParentLookup.Type, child.ParentLookup.Namespace) + logger.Infof("YASH KEY IS %s", key) + if parentComp, exists := parentRefMap[key]; exists { + // Set nil to prevent again + child.ParentLookup = nil + logger.Infof("YASH KEY FOUND IS %s", key) + parentComp.Components = append(parentComp.Components, child) + } else { + logger.Errorf("YASH ERROR") + children = append(children, child) + } + } + c.Components = children + + for _, child := range c.Components { + changeParents(child, parentRefMap) + } +} + +func parentRefs(rootComponent *pkg.Component) { + parentRefMap := make(map[string]*pkg.Component) + populateParentRefMap(rootComponent, parentRefMap) + changeParents(rootComponent, parentRefMap) +} + type TopologyRunOptions struct { dutyContext.Context Depth int @@ -410,6 +452,11 @@ func Run(opts TopologyRunOptions, t v1.Topology) ([]*pkg.Component, models.JobHi } } + // MAP + parentRefMap := make(map[string]*pkg.Component) + populateParentRefMap(rootComponent, parentRefMap) + changeParents(rootComponent, parentRefMap) + if len(rootComponent.Components) == 1 && rootComponent.Components[0].Type == "virtual" { // if there is only one component and it is virtual, then we don't need to show it ctx.Components = &rootComponent.Components[0].Components diff --git a/pkg/topology/run_test.go b/pkg/topology/run_test.go index 07d1868df..adf575404 100644 --- a/pkg/topology/run_test.go +++ b/pkg/topology/run_test.go @@ -105,6 +105,41 @@ var _ = ginkgo.Describe("Topology run", ginkgo.Ordered, func() { Expect(componentC.Configs[0].Name).To(Equal(componentC.Name)) Expect(componentC.Configs[0].Type).To(Equal("Service")) }) + + ginkgo.It("should update component's parents", func() { + t, err := yamlFileToTopology("../../fixtures/topology/component-with-parent-lookup.yml") + if err != nil { + ginkgo.Fail("Error converting yaml to v1.Topology:" + err.Error()) + } + + rootComponent, history := Run(TopologyRunOptions{ + Context: DefaultContext, + Depth: 10, + Namespace: "default", + }, t) + + Expect(history.Errors).To(HaveLen(0)) + + Expect(len(rootComponent[0].Components)).To(Equal(3)) + + parent1 := rootComponent[0].Components[0] + parent2 := rootComponent[0].Components[1] + parent3 := rootComponent[0].Components[2] + + Expect(len(parent1.Components)).To(Equal(2)) + Expect(len(parent2.Components)).To(Equal(3)) + Expect(len(parent3.Components)).To(Equal(1)) + + Expect(parent1.Components[0].Name).To(Equal("Child-1A")) + Expect(parent1.Components[1].Name).To(Equal("Child-1B")) + + Expect(parent2.Components[0].Name).To(Equal("Child-2A")) + Expect(parent2.Components[1].Name).To(Equal("Child-2B")) + Expect(parent2.Components[2].Name).To(Equal("Child-1C")) + + Expect(parent3.Components[0].Name).To(Equal("Child-1D")) + }) + }) func yamlFileToTopology(file string) (t v1.Topology, err error) { diff --git a/pkg/topology/utils.go b/pkg/topology/utils.go index e81ae0890..936ebe90a 100644 --- a/pkg/topology/utils.go +++ b/pkg/topology/utils.go @@ -1,5 +1,7 @@ package topology +import "strings" + func isComponent(s map[string]interface{}) bool { _, name := s["name"] _, properties := s["properties"] @@ -33,3 +35,7 @@ func isComponentList(data []byte) bool { } return isComponent(s[0]) } + +func genParentKey(name, _type, namespace string) string { + return strings.Join([]string{name, _type, namespace}, "/") +}