diff --git a/policy/resolved_policy_builder.go b/policy/resolved_policy_builder.go new file mode 100644 index 00000000..1d8b01e5 --- /dev/null +++ b/policy/resolved_policy_builder.go @@ -0,0 +1,822 @@ +package policy + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "go.mondoo.com/cnquery/v11/explorer" + "go.mondoo.com/cnquery/v11/llx" + "go.mondoo.com/cnquery/v11/mqlc" + "go.mondoo.com/cnquery/v11/mrn" +) + +type edgeImpact struct { + edge string + impact *explorer.Impact +} + +type resolvedPolicyBuilder struct { + bundleMrn string + bundleMap *PolicyBundleMap + assetFilters map[string]struct{} + nodes map[string]rpBuilderNode + reportsToEdges map[string][]string + reportsFromEdges map[string][]edgeImpact + policyScoringSystems map[string]explorer.ScoringSystem + actionOverrides map[string]explorer.Action + impactOverrides map[string]*explorer.Impact + propsCache explorer.PropsCache + now time.Time +} + +type rpBuilderNodeType int + +const ( + // rpBuilderNodeTypeRoot is the root node. This will represent the asset + rpBuilderNodeTypeRoot rpBuilderNodeType = iota + // rpBuilderNodeTypePolicy is a policy node. Checks and data queries report to this + rpBuilderNodeTypePolicy + // rpBuilderNodeTypeFramework is a framework node. Controls report to this + rpBuilderNodeTypeFramework + // rpBuilderNodeTypeControl is a control node. Checks, data queries, and policies report to this + rpBuilderNodeTypeControl + // rpBuilderNodeTypeQuery is a check and/or a data query. Execution queries report to this + rpBuilderNodeTypeQuery + rpBuilderNodeTypeExecutionQuery +) + +// rpBuilderData is the data that is used to build the resolved policy +type rpBuilderData struct { + baseChecksum string + impactOverrides map[string]*explorer.Impact + propsCache explorer.PropsCache + compilerConf mqlc.CompilerConfig +} + +func (d *rpBuilderData) relativeChecksum(s string) string { + return checksumStrings(d.baseChecksum, s) +} + +type rpBuilderNode interface { + getType() rpBuilderNodeType + getId() string + isLeaf() bool + build(*ResolvedPolicy, *rpBuilderData) error +} + +type rpBuilderPolicyNode struct { + policy *Policy + scoringSystem explorer.ScoringSystem + isRoot bool +} + +func (n *rpBuilderPolicyNode) getType() rpBuilderNodeType { + return rpBuilderNodeTypePolicy +} + +func (n *rpBuilderPolicyNode) getId() string { + return n.policy.Mrn +} + +func (n *rpBuilderPolicyNode) isLeaf() bool { + return false +} + +func (n *rpBuilderPolicyNode) build(rp *ResolvedPolicy, data *rpBuilderData) error { + if n.isRoot { + addReportingJob(n.policy.Mrn, data.relativeChecksum(n.policy.GraphExecutionChecksum), ReportingJob_POLICY, rp) + } else { + // TODO: the uuid used to be a checksum of the policy mrn, impact, and action + // I don't think this can be correct in all cases as you could at some point + // have a policy report to multiple other policies with different impacts + // (we don't have that case right now) + // These checksum changes should be accounted for in the root + addReportingJob(n.policy.Mrn, data.relativeChecksum(n.policy.Mrn), ReportingJob_POLICY, rp) + } + + return nil +} + +type rpBuilderControlNode struct { + controlMrn string +} + +func (n *rpBuilderControlNode) getType() rpBuilderNodeType { + return rpBuilderNodeTypeControl +} + +func (n *rpBuilderControlNode) getId() string { + return n.controlMrn +} + +func (n *rpBuilderControlNode) isLeaf() bool { + return false +} + +func (n *rpBuilderControlNode) build(rp *ResolvedPolicy, data *rpBuilderData) error { + addReportingJob(n.controlMrn, data.relativeChecksum(n.controlMrn), ReportingJob_CONTROL, rp) + return nil +} + +type rpBuilderFrameworkNode struct { + frameworkMrn string +} + +func (n *rpBuilderFrameworkNode) getType() rpBuilderNodeType { + return rpBuilderNodeTypeFramework +} + +func (n *rpBuilderFrameworkNode) getId() string { + return n.frameworkMrn +} + +func (n *rpBuilderFrameworkNode) isLeaf() bool { + return false +} + +func (n *rpBuilderFrameworkNode) build(rp *ResolvedPolicy, data *rpBuilderData) error { + addReportingJob(n.frameworkMrn, data.relativeChecksum(n.frameworkMrn), ReportingJob_FRAMEWORK, rp) + return nil +} + +func addReportingJob(qrId string, uuid string, typ ReportingJob_Type, rp *ResolvedPolicy) *ReportingJob { + if _, ok := rp.CollectorJob.ReportingJobs[uuid]; !ok { + rp.CollectorJob.ReportingJobs[uuid] = &ReportingJob{ + QrId: qrId, + Uuid: uuid, + ChildJobs: map[string]*explorer.Impact{}, + Datapoints: map[string]bool{}, + Type: typ, + } + } + return rp.CollectorJob.ReportingJobs[uuid] +} + +type rpBuilderGenericQueryNode struct { + query *explorer.Mquery + selectedVariant *explorer.Mquery + isDataQuery bool + selectedCodeId string +} + +func (n *rpBuilderGenericQueryNode) getType() rpBuilderNodeType { + return rpBuilderNodeTypeQuery +} + +func (n *rpBuilderGenericQueryNode) getId() string { + return n.query.Mrn +} + +func (n *rpBuilderGenericQueryNode) isLeaf() bool { + return true +} + +func compileProps(query *explorer.Mquery, rp *ResolvedPolicy, data *rpBuilderData) (map[string]*llx.Primitive, map[string]string, error) { + var propTypes map[string]*llx.Primitive + var propToChecksums map[string]string + if len(query.Props) != 0 { + propTypes = make(map[string]*llx.Primitive, len(query.Props)) + propToChecksums = make(map[string]string, len(query.Props)) + for j := range query.Props { + prop := query.Props[j] + + // we only get this if there is an override higher up in the policy + override, name, _ := data.propsCache.Get(prop.Mrn) + if override != nil { + prop = override + } + if name == "" { + var err error + name, err = mrn.GetResource(prop.Mrn, MRN_RESOURCE_QUERY) + if err != nil { + return nil, nil, errors.New("failed to get property name") + } + } + + executionQuery, dataChecksum, err := mquery2executionQuery(prop, nil, map[string]string{}, rp.CollectorJob, false, data.compilerConf) + if err != nil { + return nil, nil, errors.New("resolver> failed to compile query for MRN " + prop.Mrn + ": " + err.Error()) + } + if dataChecksum == "" { + return nil, nil, errors.New("property returns too many value, cannot determine entrypoint checksum: '" + prop.Mql + "'") + } + rp.ExecutionJob.Queries[prop.CodeId] = executionQuery + + propTypes[name] = &llx.Primitive{Type: prop.Type} + propToChecksums[name] = dataChecksum + } + } + return propTypes, propToChecksums, nil +} + +func (n *rpBuilderGenericQueryNode) build(rp *ResolvedPolicy, data *rpBuilderData) error { + if n.selectedVariant == nil { + // If there are no variants, we need to add an execution query + // for the check, as well as a reporting job for the check by code + // id and a reporting job for the check by mrn + mrnReportingJobUUID := data.relativeChecksum(n.query.Mrn) + propTypes, propToChecksums, err := compileProps(n.query, rp, data) + if err != nil { + return err + } + if rp.ExecutionJob.Queries[n.query.CodeId] == nil { + eq, _, err := mquery2executionQuery(n.query, propTypes, propToChecksums, rp.CollectorJob, false, data.compilerConf) + if err != nil { + return err + } + rp.ExecutionJob.Queries[n.query.CodeId] = eq + } + + executionQuery := rp.ExecutionJob.Queries[n.query.CodeId] + + codeIdReportingJobUUID := data.relativeChecksum(n.query.CodeId) + + var codeIdReportingJob *ReportingJob + if _, ok := rp.CollectorJob.ReportingJobs[codeIdReportingJobUUID]; !ok { + codeIdReportingJob = addReportingJob(n.query.CodeId, codeIdReportingJobUUID, ReportingJob_CHECK, rp) + connectDatapointsToReportingJob(executionQuery, codeIdReportingJob, rp.CollectorJob.Datapoints) + } else { + codeIdReportingJob = rp.CollectorJob.ReportingJobs[codeIdReportingJobUUID] + } + + impact := data.impactOverrides[n.query.Mrn] + var mrnReportingJob *ReportingJob + if _, ok := rp.CollectorJob.ReportingJobs[mrnReportingJobUUID]; !ok { + mrnReportingJob = addReportingJob(n.query.Mrn, mrnReportingJobUUID, ReportingJob_CHECK, rp) + } else { + mrnReportingJob = rp.CollectorJob.ReportingJobs[mrnReportingJobUUID] + } + + connectReportingJobNotifies(codeIdReportingJob, mrnReportingJob, impact) + } else { + reportingJobUUID := data.relativeChecksum(n.query.Mrn) + + if _, ok := rp.CollectorJob.ReportingJobs[reportingJobUUID]; !ok { + addReportingJob(n.query.Mrn, reportingJobUUID, ReportingJob_CHECK, rp) + } + } + return nil +} + +func connectReportingJobNotifies(child *ReportingJob, parent *ReportingJob, impact *explorer.Impact) { + for _, n := range child.Notify { + if n == parent.Uuid { + fmt.Println("already connected") + } + } + child.Notify = append(child.Notify, parent.Uuid) + parent.ChildJobs[child.Uuid] = impact +} + +// normalizeAction normalizes the action based on the group type and impact. We need to do this because +// we've had different ways of representing actions in the past and we need to normalize them to the current +func normalizeAction(groupType GroupType, action explorer.Action, impact *explorer.Impact) explorer.Action { + switch groupType { + case GroupType_DISABLE: + return explorer.Action_DEACTIVATE + case GroupType_OUT_OF_SCOPE: + return explorer.Action_OUT_OF_SCOPE + case GroupType_IGNORED: + return explorer.Action_IGNORE + default: + if impact != nil && impact.Scoring == explorer.ScoringSystem_IGNORE_SCORE { + return explorer.Action_IGNORE + } + return action + } +} + +func (b *resolvedPolicyBuilder) addEdge(from, to string, impact *explorer.Impact) { + if _, ok := b.reportsToEdges[from]; !ok { + b.reportsToEdges[from] = make([]string, 0, 1) + } + for _, e := range b.reportsToEdges[from] { + // If the edge already exists, don't add it + if e == to { + return + } + } + b.reportsToEdges[from] = append(b.reportsToEdges[from], to) + + if _, ok := b.reportsFromEdges[to]; !ok { + b.reportsFromEdges[to] = make([]edgeImpact, 0, 1) + } + if impact == nil { + impact = b.impactOverrides[from] + } + b.reportsFromEdges[to] = append(b.reportsFromEdges[to], edgeImpact{edge: from, impact: impact}) +} + +func (b *resolvedPolicyBuilder) addNode(node rpBuilderNode) { + b.nodes[node.getId()] = node +} + +func (b *resolvedPolicyBuilder) gatherOverridesFromPolicy(policy *Policy) (map[string]explorer.Action, map[string]*explorer.Impact, map[string]explorer.ScoringSystem) { + actions := make(map[string]explorer.Action) + impacts := make(map[string]*explorer.Impact) + scoringSystems := make(map[string]explorer.ScoringSystem) + + for _, g := range policy.Groups { + if !b.isPolicyGroupMatching(g) { + continue + } + for _, pRef := range g.Policies { + p := b.bundleMap.Policies[pRef.Mrn] + + a, i, s := b.gatherOverridesFromPolicy(p) + for k, v := range a { + actions[k] = v + } + + for k, v := range i { + impacts[k] = v + } + + for k, v := range s { + scoringSystems[k] = v + } + + action := normalizeAction(g.Type, pRef.Action, pRef.Impact) + actions[pRef.Mrn] = action + impacts[pRef.Mrn] = pRef.Impact + scoringSystem := pRef.ScoringSystem + + if scoringSystem != explorer.ScoringSystem_SCORING_UNSPECIFIED { + scoringSystems[pRef.Mrn] = pRef.ScoringSystem + } + } + + for _, c := range g.Checks { + impact := c.Impact + action := normalizeAction(g.Type, c.Action, impact) + if action == explorer.Action_IGNORE { + impact = &explorer.Impact{ + Scoring: explorer.ScoringSystem_IGNORE_SCORE, + } + } + if action != explorer.Action_UNSPECIFIED { + actions[c.Mrn] = action + } + if (action == explorer.Action_MODIFY || action == explorer.Action_IGNORE) && c.Impact != nil && c.Impact.Value != nil { + if existingImpact, ok := impacts[c.Mrn]; !ok || existingImpact.Value.GetValue() < impact.Value.GetValue() { + // If the impact is higher than the existing impact, we override it + impacts[c.Mrn] = c.Impact + } + } + } + + for _, q := range g.Queries { + if q.Action != explorer.Action_UNSPECIFIED { + a := normalizeAction(g.Type, q.Action, q.Impact) + switch a { + case explorer.Action_IGNORE, explorer.Action_OUT_OF_SCOPE, explorer.Action_DEACTIVATE: + actions[q.Mrn] = a + default: + log.Warn().Str("mrn", q.Mrn).Msg("Invalid action for data query") + } + } + } + } + + return actions, impacts, scoringSystems +} + +func canRun(action explorer.Action) bool { + return !(action == explorer.Action_DEACTIVATE || action == explorer.Action_OUT_OF_SCOPE) +} + +func (b *resolvedPolicyBuilder) isPolicyGroupMatching(group *PolicyGroup) bool { + if group.ReviewStatus == ReviewStatus_REJECTED { + return false + } + + if group.EndDate != 0 { + // TODO: we also need to check if the group is accepted or rejected + endDate := time.Unix(group.EndDate, 0) + if endDate.Before(b.now) { + return false + } + } + + if group.Filters == nil || len(group.Filters.Items) == 0 { + return true + } + + for _, filter := range group.Filters.Items { + if _, ok := b.assetFilters[filter.CodeId]; ok { + return true + } + } + + return false +} + +func isOverride(action explorer.Action) bool { + return action != explorer.Action_UNSPECIFIED +} + +func (b *resolvedPolicyBuilder) addPolicy(policy *Policy) bool { + action := b.actionOverrides[policy.Mrn] + + if !canRun(action) { + return false + } + + if !b.anyFilterMatches(policy.ComputedFilters) { + return false + } + + b.propsCache.Add(policy.Props...) + + // Add node for policy + scoringSystem := b.policyScoringSystems[policy.Mrn] + b.addNode(&rpBuilderPolicyNode{policy: policy, scoringSystem: scoringSystem, isRoot: b.bundleMrn == policy.Mrn}) + hasMatchingGroup := false + for _, g := range policy.Groups { + if !b.isPolicyGroupMatching(g) { + continue + } + hasMatchingGroup = true + for _, pRef := range g.Policies { + p := b.bundleMap.Policies[pRef.Mrn] + if b.addPolicy(p) { + b.addEdge(pRef.Mrn, policy.Mrn, nil) + } + } + + for _, c := range g.Checks { + // Check the action. If its an override, we don't need to add the check + // because it will get included in a policy that wants it run. + // This will prevent the check from being connected to the policy that + // overrides its action + if isOverride(c.Action) { + b.propsCache.Add(c.Props...) + continue + } + + c, ok := b.bundleMap.Queries[c.Mrn] + if !ok { + log.Warn().Str("mrn", c.Mrn).Msg("check not found in bundle") + continue + } + + if b.addCheck(c) { + b.addEdge(c.Mrn, policy.Mrn, nil) + } + } + + for _, q := range g.Queries { + // Check the action. If its an override, we don't need to add the query + // because it will get included in a policy that wants it run. + // This will prevent the query from being connected to the policy that + // overrides its action + if isOverride(q.Action) { + b.propsCache.Add(q.Props...) + continue + } + + q, ok := b.bundleMap.Queries[q.Mrn] + if !ok { + log.Warn().Str("mrn", q.Mrn).Msg("query not found in bundle") + continue + } + + if b.addDataQuery(q) { + b.addEdge(q.Mrn, policy.Mrn, &explorer.Impact{ + Scoring: explorer.ScoringSystem_IGNORE_SCORE, + Value: &explorer.ImpactValue{Value: 0}, + }) + } + } + } + + return hasMatchingGroup +} + +func (b *resolvedPolicyBuilder) anyFilterMatches(f *explorer.Filters) bool { + return f.Supports(b.assetFilters) +} + +func (b *resolvedPolicyBuilder) addFramework(framework *Framework) bool { + action := b.actionOverrides[framework.Mrn] + if !canRun(action) { + return false + } + + // Create a node for the framework, but only if its a valid framework mrn + // Otherwise, we have the asset / space policies which we will connect + // to. We need to do this because we cannot have a space frame and space + // policy reporting job because they would have the same qr id. + // TODO: we should create a new reporting job type for asset and space + // reporting jobs so its cleare that we can connect both frameworks and + // policies to them + // If the node already exists, its represented by the asset or space policy + // and is not a valid framework mrn + if _, ok := b.nodes[framework.Mrn]; !ok { + b.addNode(&rpBuilderFrameworkNode{frameworkMrn: framework.Mrn}) + } + + for _, fmap := range framework.FrameworkMaps { + for _, control := range fmap.Controls { + b.addControl(control) + } + } + + for _, fdep := range framework.Dependencies { + f, ok := b.bundleMap.Frameworks[fdep.Mrn] + if !ok { + log.Warn().Str("mrn", fdep.Mrn).Msg("framework not found in bundle") + continue + } + if b.addFramework(f) { + b.addEdge(fdep.Mrn, framework.Mrn, nil) + } + } + + return true +} + +func (b *resolvedPolicyBuilder) addControl(control *ControlMap) bool { + action := b.actionOverrides[control.Mrn] + if !canRun(action) { + return false + } + + hasChild := false + + for _, q := range control.Checks { + if _, ok := b.nodes[q.Mrn]; ok { + n := b.nodes[q.Mrn] + if n == nil { + continue + } + qNode, ok := n.(*rpBuilderGenericQueryNode) + if ok { + b.addEdge(qNode.selectedCodeId, control.Mrn, nil) + hasChild = true + } + } + } + + for _, q := range control.Queries { + if _, ok := b.nodes[q.Mrn]; ok { + n := b.nodes[q.Mrn] + if n == nil { + continue + } + qNode, ok := n.(*rpBuilderGenericQueryNode) + if ok { + b.addEdge(qNode.selectedCodeId, control.Mrn, nil) + hasChild = true + } + } + } + + for _, p := range control.Policies { + if _, ok := b.nodes[p.Mrn]; ok { + // Add the edge from the control to the policy + b.addEdge(p.Mrn, control.Mrn, nil) + hasChild = true + } + } + + for _, c := range control.Controls { + // We will just assume that the control is in the graph + // If its not, it will get filtered out later when we build + // the resolved policy + // Doing this so we don't need to topologically sort the dependency + // tree for the controls + b.addEdge(c.Mrn, control.Mrn, nil) + hasChild = true + } + + if hasChild { + // Add node for control + b.addNode(&rpBuilderControlNode{controlMrn: control.Mrn}) + } + + return true +} + +func (b *resolvedPolicyBuilder) addCheck(query *explorer.Mquery) bool { + _, added := b.addQuery(query, false) + return added +} +func (b *resolvedPolicyBuilder) addDataQuery(query *explorer.Mquery) bool { + _, added := b.addQuery(query, true) + return added +} + +func (b *resolvedPolicyBuilder) addQuery(query *explorer.Mquery, isDataQuery bool) (string, bool) { + action := b.actionOverrides[query.Mrn] + + if !canRun(action) { + return "", false + } + + if len(query.Variants) != 0 { + var matchingVariant *explorer.Mquery + var selectedCodeId string + for _, v := range query.Variants { + q, ok := b.bundleMap.Queries[v.Mrn] + if !ok { + log.Warn().Str("mrn", v.Mrn).Msg("variant not found in bundle") + continue + } + if codeId, added := b.addQuery(q, isDataQuery); added { + // The first matching variant is selected + matchingVariant = q + selectedCodeId = codeId + break + } + } + + if matchingVariant == nil { + return "", false + } + + b.propsCache.Add(query.Props...) + b.propsCache.Add(matchingVariant.Props...) + + // Add node for query + b.addNode(&rpBuilderGenericQueryNode{query: query, selectedVariant: matchingVariant, selectedCodeId: selectedCodeId, isDataQuery: isDataQuery}) + + // Add edge from variant to query + b.addEdge(matchingVariant.Mrn, query.Mrn, nil) + + return selectedCodeId, true + } else { + if !b.anyFilterMatches(query.Filters) { + return "", false + } + + b.propsCache.Add(query.Props...) + + // Add node for query + b.addNode(&rpBuilderGenericQueryNode{query: query, selectedCodeId: query.CodeId, isDataQuery: isDataQuery}) + + return query.CodeId, true + } +} + +func buildResolvedPolicy(bundleMrn string, bundle *Bundle, assetFilters []*explorer.Mquery, now time.Time, compilerConf mqlc.CompilerConfig) (*ResolvedPolicy, error) { + bundleMap := bundle.ToMap() + assetFilterMap := make(map[string]struct{}, len(assetFilters)) + for _, f := range assetFilters { + assetFilterMap[f.CodeId] = struct{}{} + } + + policyObj := bundleMap.Policies[bundleMrn] + frameworkObj := bundleMap.Frameworks[bundleMrn] + + builder := &resolvedPolicyBuilder{ + bundleMrn: bundleMrn, + bundleMap: bundleMap, + assetFilters: assetFilterMap, + nodes: map[string]rpBuilderNode{}, + reportsToEdges: map[string][]string{}, + reportsFromEdges: map[string][]edgeImpact{}, + policyScoringSystems: map[string]explorer.ScoringSystem{}, + actionOverrides: map[string]explorer.Action{}, + impactOverrides: map[string]*explorer.Impact{}, + propsCache: explorer.NewPropsCache(), + now: now, + } + + actions, impacts, scoringSystems := builder.gatherOverridesFromPolicy(policyObj) + builder.actionOverrides = actions + builder.impactOverrides = impacts + builder.policyScoringSystems = scoringSystems + + builder.addPolicy(policyObj) + + if frameworkObj != nil { + builder.addFramework(frameworkObj) + } + + resolvedPolicyExecutionChecksum := BundleExecutionChecksum(policyObj, frameworkObj) + assetFiltersChecksum, err := ChecksumAssetFilters(assetFilters, compilerConf) + if err != nil { + return nil, err + } + + builderData := &rpBuilderData{ + baseChecksum: checksumStrings(resolvedPolicyExecutionChecksum, assetFiltersChecksum, "v2"), + impactOverrides: impacts, + propsCache: builder.propsCache, + compilerConf: compilerConf, + } + + resolvedPolicy := &ResolvedPolicy{ + ExecutionJob: &ExecutionJob{ + Checksum: "", + Queries: map[string]*ExecutionQuery{}, + }, + CollectorJob: &CollectorJob{ + Checksum: "", + ReportingJobs: map[string]*ReportingJob{}, + ReportingQueries: map[string]*StringArray{}, + Datapoints: map[string]*DataQueryInfo{}, + RiskMrns: map[string]*StringArray{}, + RiskFactors: map[string]*RiskFactor{}, + }, + Filters: assetFilters, + GraphExecutionChecksum: resolvedPolicyExecutionChecksum, + FiltersChecksum: assetFiltersChecksum, + } + + // We will build from the leaf nodes out. This means that if something is not connected + // to a leaf node, it will not be included in the resolved policy + leafNodes := make([]rpBuilderNode, 0, len(builder.nodes)) + + for _, n := range builder.nodes { + if n.isLeaf() { + leafNodes = append(leafNodes, n) + } + } + + visited := make(map[string]struct{}, len(builder.nodes)) + var walk func(node rpBuilderNode) error + walk = func(node rpBuilderNode) error { + // Check if we've already visited this node + if _, ok := visited[node.getId()]; ok { + return nil + } + visited[node.getId()] = struct{}{} + + // Build the necessary parts of the resolved policy for each node + if err := node.build(resolvedPolicy, builderData); err != nil { + log.Error().Err(err).Str("node", node.getId()).Msg("error building node") + return err + } + // Walk to each parent node and recurse + for _, edge := range builder.reportsToEdges[node.getId()] { + if edgeNode, ok := builder.nodes[edge]; ok { + if err := walk(edgeNode); err != nil { + return err + } + + } else { + log.Debug().Str("from", node.getId()).Str("to", edge).Msg("edge not found") + } + } + return nil + } + + for _, n := range leafNodes { + if err := walk(n); err != nil { + return nil, err + } + } + + // We need to connect the reporting jobs. We've stored them by uuid in the collector job. However, + // our graph uses the qr id to connect them. + reportingJobsByQrId := make(map[string]*ReportingJob, len(resolvedPolicy.CollectorJob.ReportingJobs)) + for _, rj := range resolvedPolicy.CollectorJob.ReportingJobs { + if _, ok := reportingJobsByQrId[rj.QrId]; ok { + // We should never have multiple reporting jobs with the same qr id. Scores are stored + // by qr id, not by uuid. This would cause issues where scores would flop around + log.Error().Str("qr_id", rj.QrId).Msg("multipe reporting jobs with the same qr id") + return nil, errors.New("multiple reporting jobs with the same qr id") + } + reportingJobsByQrId[rj.QrId] = rj + } + + // For each parent qr id, we need to connect the child reporting jobs with the impact. + // connectReportingJobNotifies will add the link from the child to the parent, and + // the parent to the child with the impact + for parentQrId, edges := range builder.reportsFromEdges { + for _, edge := range edges { + parent := reportingJobsByQrId[parentQrId] + if parent == nil { + // It's possible that the parent reporting job was not included in the resolved policy + // because it was not connected to a leaf node (e.g. a policy that was not connected to + // any check or data query). In this case, we can just skip it + log.Debug().Str("parent", parentQrId).Msg("reporting job not found") + continue + } + + if child, ok := reportingJobsByQrId[edge.edge]; ok { + // Also possible a child was not included in the resolved policy + connectReportingJobNotifies(child, parent, edge.impact) + } + } + } + + rootReportingJob := reportingJobsByQrId[bundleMrn] + if rootReportingJob == nil { + return nil, explorer.NewAssetMatchError(bundleMrn, "policies", "no-matching-policy", assetFilters, policyObj.ComputedFilters) + } + rootReportingJob.QrId = "root" + + resolvedPolicy.ReportingJobUuid = rootReportingJob.Uuid + + refreshChecksums(resolvedPolicy.ExecutionJob, resolvedPolicy.CollectorJob) + for _, rj := range resolvedPolicy.CollectorJob.ReportingJobs { + rj.RefreshChecksum() + } + + return resolvedPolicy, nil +} diff --git a/policy/resolver.go b/policy/resolver.go index bc12c8a3..d554d058 100644 --- a/policy/resolver.go +++ b/policy/resolver.go @@ -477,6 +477,11 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF if err != nil { return nil, err } + + if true { + return buildResolvedPolicy(bundleMrn, bundle, assetFilters, time.Now(), conf) + } + bundleMap := bundle.ToMap() frameworkObj := bundleMap.Frameworks[bundleMrn] @@ -655,7 +660,7 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF Msg("resolver> phase 5: resolve controls [ok]") // phase 6: refresh all checksums - s.refreshChecksums(executionJob, collectorJob) + refreshChecksums(executionJob, collectorJob) // the final phases are done in the DataLake for _, rj := range collectorJob.ReportingJobs { @@ -679,7 +684,7 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF return &resolvedPolicy, nil } -func (s *LocalServices) refreshChecksums(executionJob *ExecutionJob, collectorJob *CollectorJob) { +func refreshChecksums(executionJob *ExecutionJob, collectorJob *CollectorJob) { // execution job { queryKeys := sortx.Keys(executionJob.Queries) diff --git a/policy/resolver_v2_test.go b/policy/resolver_v2_test.go new file mode 100644 index 00000000..0648ec6e --- /dev/null +++ b/policy/resolver_v2_test.go @@ -0,0 +1,327 @@ +package policy_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v11/explorer" + "go.mondoo.com/cnquery/v11/mqlc" + "go.mondoo.com/cnspec/v11/policy" +) + +func newResolvedPolicyTester(bundle *policy.Bundle, resolvedPolicy *policy.ResolvedPolicy, conf mqlc.CompilerConfig) *resolvedPolicyTester { + return &resolvedPolicyTester{ + bundleMap: bundle.ToMap(), + items: []resolvedPolicyTesterItem{}, + conf: conf, + } +} + +type resolvedPolicyTesterItem interface { + testIt(t *testing.T, resolvedPolicy *policy.ResolvedPolicy) +} + +type resolvedPolicyTester struct { + bundleMap *policy.PolicyBundleMap + items []resolvedPolicyTesterItem + conf mqlc.CompilerConfig +} + +func (r *resolvedPolicyTester) doTest(t *testing.T, rp *policy.ResolvedPolicy) { + for _, item := range r.items { + item.testIt(t, rp) + } +} + +func (r *resolvedPolicyTester) ExecutesQuery(mrn string) *resolvedPolicyTesterExecutesQueryBuilder { + item := &resolvedPolicyTesterExecutesQueryBuilder{tester: r, mrn: mrn} + r.items = append(r.items, item) + return item +} + +type resolvedPolicyTesterExecutesQueryBuilder struct { + tester *resolvedPolicyTester + mrn string + datapoints *[]string + props *map[string]string +} + +func (r *resolvedPolicyTesterExecutesQueryBuilder) WithProps(props map[string]string) *resolvedPolicyTesterExecutesQueryBuilder { + r.props = &props + return r +} + +func (r *resolvedPolicyTesterExecutesQueryBuilder) testIt(t *testing.T, rp *policy.ResolvedPolicy) { + q := r.tester.bundleMap.Queries[r.mrn] + require.NotNilf(t, q, "query not found in bundle: %s", r.mrn) + codeId := q.CodeId + require.NotEmptyf(t, codeId, "query %s doesn't have code id", r.mrn) + + eq := rp.ExecutionJob.Queries[codeId] + require.NotNilf(t, eq, "query %s not found in ExecutionJob", r.mrn) + + if r.datapoints != nil { + require.ElementsMatchf(t, *r.datapoints, eq.Datapoints, "datapoints mismatch for query %q", r.mrn) + } + + if r.props != nil { + require.Lenf(t, eq.Properties, len(*r.props), "properties mismatch for query %q", r.mrn) + for propName, mql := range *r.props { + // Compile the property + codeBundle, err := mqlc.Compile(mql, nil, r.tester.conf) + require.NoErrorf(t, err, "failed to compile property %q for query %q", propName, r.mrn) + propCodeId := codeBundle.CodeV2.Id + require.NotEmptyf(t, propCodeId, "property %s doesn't have code id", propName) + propEq := rp.ExecutionJob.Queries[propCodeId] + require.NotNilf(t, propEq, "property %q not found in ExecutionJob with code id %q", propName, propCodeId) + require.Lenf(t, propEq.Datapoints, 1, "property %q should have exactly one datapoint", propName) + propDatapoint := propEq.Datapoints[0] + require.Equalf(t, eq.Properties[propName], propDatapoint, "property %q value mismatch", propName) + } + } +} + +type resolvedPolicyTesterReportingJobNotifiesBuilder struct { + rjTester *resolvedPolicyTesterReportingJobBuilder + childMrn string + childMrnForCodeId string + parent string + impact *explorer.Impact + impactSet bool +} + +func (r *resolvedPolicyTesterReportingJobNotifiesBuilder) WithImpact(impact *explorer.Impact) *resolvedPolicyTesterReportingJobNotifiesBuilder { + r.impact = impact + r.impactSet = true + return r +} + +func findReportingJobByQrId(rp *policy.ResolvedPolicy, qrId string) *policy.ReportingJob { + for _, rj := range rp.CollectorJob.ReportingJobs { + if rj.QrId == qrId { + return rj + } + } + return nil +} + +func (r *resolvedPolicyTesterReportingJobNotifiesBuilder) testIt(t *testing.T, rp *policy.ResolvedPolicy) { + var qrId string + var extraInfo string + if r.childMrn != "" { + qrId = r.childMrn + } else { + q := r.rjTester.tester.bundleMap.Queries[r.childMrnForCodeId] + require.NotNilf(t, q, "query not found in bundle: %s", r.childMrnForCodeId) + require.NotEmptyf(t, q.CodeId, "query %s doesn't have code id", r.childMrnForCodeId) + qrId = q.CodeId + extraInfo = " (" + r.childMrnForCodeId + ")" + } + childRj := findReportingJobByQrId(rp, qrId) + require.NotNilf(t, childRj, "child reporting job %s%s not found", qrId, extraInfo) + + parentRj := findReportingJobByQrId(rp, r.parent) + require.NotNilf(t, parentRj, "parent reporting job %s not found", r.parent) + require.Containsf(t, childRj.Notify, parentRj.Uuid, "child reporting job %s%s doesn't notify parent reporting job %s", qrId, extraInfo, r.parent) + + require.Containsf(t, parentRj.ChildJobs, childRj.Uuid, "parent reporting job %s doesn't have child reporting job %s%s", r.parent, qrId, extraInfo) + if r.impactSet { + require.Equalf(t, r.impact, parentRj.ChildJobs[childRj.Uuid], "impact mismatch for child reporting job %s%s", qrId, extraInfo) + } +} + +type resolvedPolicyTesterReportingJobBuilder struct { + tester *resolvedPolicyTester + mrn string + mrnForCodeId string + typ *policy.ReportingJob_Type + notifies []*resolvedPolicyTesterReportingJobNotifiesBuilder + notifiesSet bool +} + +func (r *resolvedPolicyTester) CodeIdReportingJobForMrn(mrn string) *resolvedPolicyTesterReportingJobBuilder { + item := &resolvedPolicyTesterReportingJobBuilder{tester: r, mrnForCodeId: mrn} + r.items = append(r.items, item) + return item +} + +func (r *resolvedPolicyTester) ReportingJobByMrn(mrn string) *resolvedPolicyTesterReportingJobBuilder { + item := &resolvedPolicyTesterReportingJobBuilder{tester: r, mrn: mrn} + r.items = append(r.items, item) + return item +} + +func (r *resolvedPolicyTesterReportingJobBuilder) WithType(typ policy.ReportingJob_Type) *resolvedPolicyTesterReportingJobBuilder { + r.typ = &typ + return r +} + +func (r *resolvedPolicyTesterReportingJobBuilder) Notifies(qrId string) *resolvedPolicyTesterReportingJobNotifiesBuilder { + n := &resolvedPolicyTesterReportingJobNotifiesBuilder{rjTester: r, childMrnForCodeId: r.mrnForCodeId, childMrn: r.mrn, parent: qrId} + r.notifies = append(r.notifies, n) + r.notifiesSet = true + return n +} + +func (r *resolvedPolicyTesterReportingJobBuilder) testIt(t *testing.T, rp *policy.ResolvedPolicy) { + var qrId string + var extraInfo string + if r.mrn != "" { + qrId = r.mrn + } else { + q := r.tester.bundleMap.Queries[r.mrnForCodeId] + require.NotNilf(t, q, "query not found in bundle: %s", r.mrnForCodeId) + require.NotEmptyf(t, q.CodeId, "query %s doesn't have code id", r.mrnForCodeId) + qrId = q.CodeId + extraInfo = " (" + r.mrnForCodeId + ")" + } + rj := findReportingJobByQrId(rp, qrId) + require.NotNilf(t, rj, "reporting job %s%s not found", qrId, extraInfo) + + if r.typ != nil { + require.Equalf(t, *r.typ, rj.Type, "reporting job %s%s type mismatch", qrId, extraInfo) + } + + if r.notifiesSet { + for _, n := range r.notifies { + n.testIt(t, rp) + } + require.Len(t, rj.Notify, len(r.notifies), "reporting job %s%s notify mismatch", qrId, extraInfo) + } +} + +func contextResolverV2() context.Context { + return context.Background() +} + +func TestResolveV2_EmptyPolicy(t *testing.T) { + ctx := contextResolverV2() + b := parseBundle(t, ` +owner_mrn: //test.sth +policies: +- uid: policy1 +`) + + srv := initResolver(t, []*testAsset{ + {asset: "asset1", policies: []string{policyMrn("policy1")}}, + }, []*policy.Bundle{b}) + + t.Run("resolve w/o filters", func(t *testing.T) { + _, err := srv.Resolve(ctx, &policy.ResolveReq{ + PolicyMrn: policyMrn("policy1"), + }) + assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = asset doesn't support any policies") + }) + + t.Run("resolve with empty filters", func(t *testing.T) { + _, err := srv.Resolve(ctx, &policy.ResolveReq{ + PolicyMrn: policyMrn("policy1"), + AssetFilters: []*explorer.Mquery{{}}, + }) + assert.EqualError(t, err, "failed to compile query: failed to compile query '': query is not implemented ''") + }) + + t.Run("resolve with random filters", func(t *testing.T) { + _, err := srv.Resolve(ctx, &policy.ResolveReq{ + PolicyMrn: policyMrn("policy1"), + AssetFilters: []*explorer.Mquery{{Mql: "true"}}, + }) + assert.EqualError(t, err, + "rpc error: code = InvalidArgument desc = asset isn't supported by any policies\n"+ + "policies didn't provide any filters\n"+ + "asset supports: true\n") + }) +} + +func TestResolveV2_SimplePolicy(t *testing.T) { + b := parseBundle(t, ` +owner_mrn: //test.sth +policies: +- uid: policy1 + groups: + - type: chapter + filters: "true" + checks: + - uid: check1 + mql: asset.name == props.name + props: + - uid: name + mql: return "definitely not the asset name" + queries: + - uid: query1 + mql: asset{*} +`) + + srv := initResolver(t, []*testAsset{ + {asset: "asset1", policies: []string{policyMrn("policy1")}}, + }, []*policy.Bundle{b}) + + t.Run("resolve with correct filters", func(t *testing.T) { + rp, err := srv.Resolve(context.Background(), &policy.ResolveReq{ + PolicyMrn: policyMrn("policy1"), + AssetFilters: []*explorer.Mquery{{Mql: "true"}}, + }) + require.NoError(t, err) + require.NotNil(t, rp) + require.Len(t, rp.ExecutionJob.Queries, 3) + require.Len(t, rp.Filters, 1) + require.Len(t, rp.CollectorJob.ReportingJobs, 5) + + qrIdToRj := map[string]*policy.ReportingJob{} + for _, rj := range rp.CollectorJob.ReportingJobs { + qrIdToRj[rj.QrId] = rj + } + // scoring queries report by code id + require.NotNil(t, qrIdToRj[b.Queries[1].CodeId]) + // require.Len(t, qrIdToRj[b.Queries[1].CodeId].Mrns, 1) + // require.Equal(t, queryMrn("check1"), qrIdToRj[b.Queries[1].CodeId].Mrns[0]) + // data queries report by mrn + require.NotNil(t, qrIdToRj[queryMrn("query1")]) + require.NotNil(t, qrIdToRj[queryMrn("check1")]) + + require.Len(t, qrIdToRj[b.Queries[1].CodeId].Datapoints, 3) + require.Len(t, qrIdToRj[b.Queries[0].CodeId].Datapoints, 1) + + rpTester := newResolvedPolicyTester(b, rp, srv.NewCompilerConfig()) + rpTester.ExecutesQuery(queryMrn("query1")) + rpTester. + ExecutesQuery(queryMrn("check1")). + WithProps(map[string]string{"name": `return "definitely not the asset name"`}) + rpTester.CodeIdReportingJobForMrn(queryMrn("check1")).Notifies(queryMrn("check1")) + rpTester.CodeIdReportingJobForMrn(queryMrn("query1")).Notifies(queryMrn("query1")) + rpTester.ReportingJobByMrn(queryMrn("check1")).Notifies("root") + rpTester.ReportingJobByMrn(queryMrn("query1")).Notifies("root") + + rpTester.doTest(t, rp) + }) + + // t.Run("resolve with many filters (one is correct)", func(t *testing.T) { + // rp, err := srv.Resolve(context.Background(), &policy.ResolveReq{ + // PolicyMrn: policyMrn("policy1"), + // AssetFilters: []*explorer.Mquery{ + // {Mql: "asset.family.contains(\"linux\")"}, + // {Mql: "true"}, + // {Mql: "asset.family.contains(\"windows\")"}, + // }, + // }) + // require.NoError(t, err) + // require.NotNil(t, rp) + // }) + + // t.Run("resolve with incorrect filters", func(t *testing.T) { + // _, err := srv.Resolve(context.Background(), &policy.ResolveReq{ + // PolicyMrn: policyMrn("policy1"), + // AssetFilters: []*explorer.Mquery{ + // {Mql: "asset.family.contains(\"linux\")"}, + // {Mql: "false"}, + // {Mql: "asset.family.contains(\"windows\")"}, + // }, + // }) + // assert.EqualError(t, err, + // "rpc error: code = InvalidArgument desc = asset isn't supported by any policies\n"+ + // "policies support: true\n"+ + // "asset supports: asset.family.contains(\"linux\"), asset.family.contains(\"windows\"), false\n") + // }) +}