From 50e4098c72eae472223c5e17af102868825bba39 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 7 Mar 2024 17:36:17 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix=20incognito=20scans=20(#3518?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix incognito scans Signed-off-by: Ivan Milchev * set bundle in reporter Signed-off-by: Ivan Milchev * remove todo Signed-off-by: Ivan Milchev --------- Signed-off-by: Ivan Milchev --- apps/cnquery/cmd/scan.go | 37 ++++++----------- explorer/scan/local_scanner.go | 72 ++++++++++++++++++++++++---------- explorer/scan/reporter.go | 7 +++- 3 files changed, 70 insertions(+), 46 deletions(-) diff --git a/apps/cnquery/cmd/scan.go b/apps/cnquery/cmd/scan.go index e995179b0b..5895051671 100644 --- a/apps/cnquery/cmd/scan.go +++ b/apps/cnquery/cmd/scan.go @@ -237,31 +237,20 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes * conf.Inventory.ApplyCategory(inventory.AssetCategory_CATEGORY_CICD) } - var serviceAccount *upstream.ServiceAccountCredentials - if !conf.IsIncognito { - serviceAccount = opts.GetServiceCredential() - if serviceAccount != nil { - // TODO: determine if this needs migrating - // // determine information about the client - // sysInfo, err := sysinfo.GatherSystemInfo() - // if err != nil { - // log.Warn().Err(err).Msg("could not gather client information") - // } - // plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...) - - log.Info().Msg("using service account credentials") - conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{ - SpaceMrn: opts.GetParentMrn(), - ApiEndpoint: opts.UpstreamApiEndpoint(), - ApiProxy: opts.APIProxy, - Incognito: conf.IsIncognito, - Creds: serviceAccount, - } - providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig - } else { - log.Warn().Msg("No credentials provided. Switching to --incognito mode.") - conf.IsIncognito = true + serviceAccount := opts.GetServiceCredential() + if serviceAccount != nil { + log.Info().Msg("using service account credentials") + conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{ + SpaceMrn: opts.GetParentMrn(), + ApiEndpoint: opts.UpstreamApiEndpoint(), + ApiProxy: opts.APIProxy, + Incognito: conf.IsIncognito, + Creds: serviceAccount, } + providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig + } else { + log.Warn().Msg("No credentials provided. Switching to --incognito mode.") + conf.IsIncognito = true } if len(conf.QueryPackPaths) > 0 && !conf.IsIncognito { diff --git a/explorer/scan/local_scanner.go b/explorer/scan/local_scanner.go index 3c1e28f71e..8af7b2399b 100644 --- a/explorer/scan/local_scanner.go +++ b/explorer/scan/local_scanner.go @@ -115,18 +115,21 @@ func (s *LocalScanner) Run(ctx context.Context, job *Job) (*explorer.ReportColle // returns the upstream config for the job. If the job has a specified config, it has precedence // over the automatically detected one func (s *LocalScanner) getUpstreamConfig(inv *inventory.Inventory, incognito bool) (*upstream.UpstreamConfig, error) { - jobCreds := inv.GetSpec().GetUpstreamCredentials() - if s.upstream == nil && jobCreds == nil { - return nil, errors.New("no default or job upstream config provided") + var res *upstream.UpstreamConfig + if s.upstream != nil { + res = proto.Clone(s.upstream).(*upstream.UpstreamConfig) + } else { + res = &upstream.UpstreamConfig{} } - u := proto.Clone(s.upstream).(*upstream.UpstreamConfig) - u.Incognito = incognito + + jobCreds := inv.GetSpec().GetUpstreamCredentials() + res.Incognito = incognito if jobCreds != nil { - u.ApiEndpoint = jobCreds.GetApiEndpoint() - u.Creds = jobCreds - u.SpaceMrn = jobCreds.GetParentMrn() + res.ApiEndpoint = jobCreds.GetApiEndpoint() + res.Creds = jobCreds + res.SpaceMrn = jobCreds.GetParentMrn() } - return u, nil + return res, nil } func (s *LocalScanner) RunIncognito(ctx context.Context, job *Job) (*explorer.ReportCollection, error) { @@ -223,6 +226,25 @@ func (s *LocalScanner) distributeJob(job *Job, ctx context.Context, upstream *up // plan scan jobs reporter := NewAggregateReporter() + if job.Bundle == nil && upstream != nil && upstream.Creds != nil { + client, err := upstream.InitClient() + if err != nil { + return nil, err + } + + services, err := explorer.NewRemoteServices(client.ApiEndpoint, client.Plugins, client.HttpClient) + if err != nil { + return nil, err + } + + bundle, err := services.GetBundle(ctx, &explorer.Mrn{Mrn: upstream.Creds.ParentMrn}) + if err != nil { + return nil, err + } + job.Bundle = bundle + reporter.AddBundle(bundle) + } + // if we had asset errors we want to place them into the reporter for i := range discoveredAssets.Errors { reporter.AddScanError(discoveredAssets.Errors[i].Asset, discoveredAssets.Errors[i].Err) @@ -478,13 +500,21 @@ func (s *localAssetScanner) prepareAsset() error { var hub explorer.QueryHub = s.services var conductor explorer.QueryConductor = s.services - // if we are using upstream we get the bundle from there - if s.job.UpstreamConfig != nil && !s.job.UpstreamConfig.Incognito { + // if we are using upstream we get the bundle from there. If we are in incognito mode, + // we should still use the upstream bundle but without reporting the results back + if !s.job.UpstreamConfig.Incognito { return nil } - if err := s.ensureBundle(); err != nil { - return err + if s.job.Bundle == nil { + if err := s.ensureBundle(); err != nil { + return err + } + + // add asset bundle to the reporter + if s.job.Reporter != nil && s.job.Bundle != nil { + s.job.Reporter.AddBundle(s.job.Bundle) + } } if s.job.Bundle == nil { @@ -609,13 +639,16 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) { var conductor explorer.QueryConductor = s.services log.Debug().Str("asset", s.job.Asset.Mrn).Msg("client> request bundle for asset") - assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) - if err != nil { - return nil, err + // If we run in debug mode, download the asset bundle and dump it to disk + if val, ok := os.LookupEnv("DEBUG"); ok && (val == "1" || val == "true") { + assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) + if err != nil { + return nil, err + } + log.Debug().Msg("client> got bundle") + logger.TraceJSON(assetBundle) + logger.DebugDumpJSON("assetBundle", assetBundle) } - log.Debug().Msg("client> got bundle") - logger.TraceJSON(assetBundle) - logger.DebugDumpJSON("assetBundle", assetBundle) rawFilters, err := hub.GetFilters(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) if err != nil { @@ -676,7 +709,6 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) { ar := &AssetReport{ Mrn: s.job.Asset.Mrn, - Bundle: assetBundle, Resolved: resolvedPack, } diff --git a/explorer/scan/reporter.go b/explorer/scan/reporter.go index ff90e38222..a9244ddc1c 100644 --- a/explorer/scan/reporter.go +++ b/explorer/scan/reporter.go @@ -11,12 +11,12 @@ import ( type Reporter interface { AddReport(asset *inventory.Asset, results *AssetReport) + AddBundle(bundle *explorer.Bundle) AddScanError(asset *inventory.Asset, err error) } type AssetReport struct { Mrn string - Bundle *explorer.Bundle Report *explorer.Report Resolved *explorer.ResolvedPack } @@ -42,7 +42,10 @@ func (r *AggregateReporter) AddReport(asset *inventory.Asset, results *AssetRepo r.assets[asset.Mrn] = &explorer.Asset{Name: asset.Name, Mrn: asset.Mrn} r.assetReports[asset.Mrn] = results.Report r.resolved[asset.Mrn] = results.Resolved - r.bundle = results.Bundle +} + +func (r *AggregateReporter) AddBundle(bundle *explorer.Bundle) { + r.bundle = bundle } func (r *AggregateReporter) AddScanError(asset *inventory.Asset, err error) {