enh: use iter.Map for better concurrency; fix: cmd flags

Makefile

@@ -64,6 +64,6 @@ stop-containers:
 	@docker compose down -v
 	@docker compose rm -fv
 
-.PHONY: tests
-tests: start-containers build
-	cd ./assets/ && $(MAKE) -C tests all
+.PHONY: test
+test: start-containers build
+	cd ./assets/ && $(MAKE) -C tests all

assets/tests/tests.go

@@ -3,7 +3,7 @@ package tests
 import (
 	"context"
 
-	awsconnector "github.com/primait/nuvola/connector/services/aws"
+	awsconnector "github.com/primait/nuvola/pkg/connector/services/aws"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/retry"

cmd/assess.go

@@ -8,7 +8,7 @@ import (
 	"log"
 	"strings"
 
-	"github.com/primait/nuvola/connector"
+	"github.com/primait/nuvola/pkg/connector"
 	"github.com/primait/nuvola/pkg/io/logging"
 	"github.com/primait/nuvola/tools/filesystem/files"
 	unzip "github.com/primait/nuvola/tools/filesystem/zip"
@@ -29,7 +29,7 @@ var assessCmd = &cobra.Command{
 
 		connector.SetActions()
 		storageConnector := connector.NewStorageConnector()
-		if importFile != "" && !noImport {
+		if importFile != "" {
 			logger.Info("Flushing database")
 			logger.Info(fmt.Sprintf("Importing %s", importFile))
 			importZipFile(storageConnector, importFile)

cmd/dump.go

@@ -6,8 +6,7 @@ import (
 	"reflect"
 	"time"
 
-	"github.com/primait/nuvola/connector"
-	"github.com/primait/nuvola/pkg/io/logging"
+	"github.com/primait/nuvola/pkg/connector"
 	"github.com/primait/nuvola/tools/filesystem/files"
 	"github.com/primait/nuvola/tools/filesystem/zip"
 	"github.com/spf13/cobra"
@@ -33,10 +32,6 @@ var dumpCmd = &cobra.Command{
 	Short: "Dump AWS resources and policies information and store them in Neo4j",
 	Run: func(cmd *cobra.Command, args []string) {
 		startTime := time.Now()
-		markAsRequired("aws-profile")
-		if err := rootCmd.ValidateRequiredFlags(); err != nil {
-			logger.Error("Required flags not provided", "err", err)
-		}
 		if cmd.Flags().Changed(flagVerbose) {
 			logger.SetVerboseLevel()
 		}
@@ -60,7 +55,7 @@ var dumpCmd = &cobra.Command{
 	},
 }
 
-func dumpData(storageConnector *connector.StorageConnector, cloudConnector *connector.CloudConnector) map[string]interface{} {
+func dumpData(storageConnector *connector.StorageConnector, cloudConnector *connector.CloudConnector) {
 	dataChan := make(chan map[string]interface{})
 	go cloudConnector.DumpAll("aws", dataChan)
 	for {
@@ -77,7 +72,6 @@ func dumpData(storageConnector *connector.StorageConnector, cloudConnector *conn
 		storageConnector.ImportResults(mapKey, obj)
 		AWSResults[mapKey] = a[mapKey]
 	}
-	return AWSResults
 }
 
 func saveResults(awsProfile string, outputDir string, outputFormat string) {
@@ -90,8 +84,6 @@ func saveResults(awsProfile string, outputDir string, outputFormat string) {
 
 	today := time.Now().Format("20060102")
 	for key, value := range AWSResults {
-		logger.Info(key, logging.PrettyJSON(value))
-
 		if outputFormat == "json" {
 			files.PrettyJSONToFile(outputDir, fmt.Sprintf("%s_%s.json", key, today), value)
 		}

cmd/root.go

@@ -36,23 +36,19 @@ func init() {
 	logger = logging.GetLogManager()
 	rootCmd.PersistentFlags().BoolP(flagVerbose, "v", false, "Verbose output")
 	rootCmd.PersistentFlags().BoolP(flagDebug, "d", false, "Debug output")
-	dumpCmd.PersistentFlags().StringVarP(&awsProfile, flagAWSProfile, "p", "default", "AWS Profile to use")
-	dumpCmd.PersistentFlags().StringVarP(&outputDirectory, flagOutputDirectory, "o", "", "Output folder where the files will be saved (default: \".\")")
-	dumpCmd.PersistentFlags().StringVarP(&outputFormat, flagOutputFormat, "f", "zip", "Output format: ZIP or json files")
-	dumpCmd.PersistentFlags().BoolVarP(&dumpOnly, flagDumpOnly, "", false, "Flag to prevent loading data into Neo4j (default: \"false\")")
+	dumpCmd.Flags().StringVarP(&awsProfile, flagAWSProfile, "p", "default", "AWS Profile to use")
+	dumpCmd.Flags().StringVarP(&outputDirectory, flagOutputDirectory, "o", "", "Output folder where the files will be saved (default: \".\")")
+	dumpCmd.Flags().StringVarP(&outputFormat, flagOutputFormat, "f", "zip", "Output format: ZIP or json files")
+	dumpCmd.Flags().BoolVarP(&dumpOnly, flagDumpOnly, "", false, "Flag to prevent loading data into Neo4j (default: \"false\")")
+	_ = dumpCmd.MarkFlagRequired(flagAWSProfile)
 
-	assessCmd.PersistentFlags().StringVarP(&importFile, flagImportFile, "i", "", "Input ZIP file to load")
-	assessCmd.PersistentFlags().BoolVarP(&noImport, flagNoImport, "", false, "Use stored data from Neo4j without import (default)")
+	assessCmd.Flags().StringVarP(&importFile, flagImportFile, "i", "", "Input ZIP file to load")
+	assessCmd.Flags().BoolVarP(&noImport, flagNoImport, "", false, "Use stored data from Neo4j without import (default)")
+	assessCmd.MarkFlagsMutuallyExclusive(flagImportFile, flagNoImport)
 }
 
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		logger.Error("Error executing command", "err", err)
 	}
 }
-
-func markAsRequired(flag string) {
-	if err := rootCmd.MarkFlagRequired(flag); err != nil {
-		logger.Error("Required flags not provided", "err", err, "flag", flag)
-	}
-}

go.mod

@@ -24,8 +24,8 @@ require (
 	github.com/notdodo/arner v0.0.0-20230222134658-4fe417a6cc9c
 	github.com/notdodo/goflat v0.0.0-20220904193052-d6f007cccdea
 	github.com/ohler55/ojg v1.21.5
+	github.com/sourcegraph/conc v0.3.0
 	github.com/spf13/cobra v1.8.0
-	golang.org/x/sync v0.7.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -72,7 +72,9 @@ require (
 	github.com/refraction-networking/utls v1.6.4 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/mock v0.4.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect
 	golang.org/x/mod v0.17.0 // indirect

go.sum

@@ -150,16 +150,23 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
 go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc=
@@ -183,8 +190,8 @@ golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

connector/cloud_connector.go → pkg/connector/cloud_connector.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	awsconfig "github.com/primait/nuvola/connector/services/aws"
+	awsconfig "github.com/primait/nuvola/pkg/connector/services/aws"
 )
 
 func NewCloudConnector(profile string, endpointUrl string) (*CloudConnector, error) {

connector/connector_structs.go → pkg/connector/connector_structs.go

@@ -1,8 +1,8 @@
 package connector
 
 import (
-	awsconfig "github.com/primait/nuvola/connector/services/aws"
-	neo4jconnector "github.com/primait/nuvola/connector/services/neo4j"
+	awsconfig "github.com/primait/nuvola/pkg/connector/services/aws"
+	neo4jconnector "github.com/primait/nuvola/pkg/connector/services/neo4j"
 )
 
 type StorageConnector struct {

connector/services/aws/aws.go → pkg/connector/services/aws/aws.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"os"
 
-	"github.com/primait/nuvola/connector/services/aws/database"
-	"github.com/primait/nuvola/connector/services/aws/ec2"
-	"github.com/primait/nuvola/connector/services/aws/iam"
-	"github.com/primait/nuvola/connector/services/aws/lambda"
-	"github.com/primait/nuvola/connector/services/aws/s3"
-	"github.com/primait/nuvola/connector/services/aws/sts"
+	"github.com/primait/nuvola/pkg/connector/services/aws/database"
+	"github.com/primait/nuvola/pkg/connector/services/aws/ec2"
+	"github.com/primait/nuvola/pkg/connector/services/aws/iam"
+	"github.com/primait/nuvola/pkg/connector/services/aws/lambda"
+	"github.com/primait/nuvola/pkg/connector/services/aws/s3"
+	"github.com/primait/nuvola/pkg/connector/services/aws/sts"
 	"github.com/primait/nuvola/pkg/io/logging"
 
 	"github.com/aws/aws-sdk-go-v2/aws"

connector/services/aws/database/dynamodb.go → pkg/connector/services/aws/database/dynamodb.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 
-	"github.com/primait/nuvola/connector/services/aws/ec2"
+	"github.com/primait/nuvola/pkg/connector/services/aws/ec2"
 	"github.com/primait/nuvola/pkg/io/logging"
 
 	"github.com/aws/aws-sdk-go-v2/aws"

connector/services/aws/database/rds.go → pkg/connector/services/aws/database/rds.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 
-	"github.com/primait/nuvola/connector/services/aws/ec2"
+	"github.com/primait/nuvola/pkg/connector/services/aws/ec2"
 	"github.com/primait/nuvola/pkg/io/logging"
 
 	"github.com/aws/aws-sdk-go-v2/aws"

connector/services/aws/database/redshift.go → pkg/connector/services/aws/database/redshift.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 
-	"github.com/primait/nuvola/connector/services/aws/ec2"
+	"github.com/primait/nuvola/pkg/connector/services/aws/ec2"
 	"github.com/primait/nuvola/pkg/io/logging"
 
 	"github.com/aws/aws-sdk-go-v2/aws"