From 3a4d17c83398d120a9c32b63bc258ae9d537f4b4 Mon Sep 17 00:00:00 2001 From: Edoardo Rosa <6991986+notdodo@users.noreply.github.com> Date: Sat, 27 Apr 2024 14:34:13 +0200 Subject: [PATCH] enh: use iter.Map for better concurrency; fix: cmd flags --- Makefile | 6 +- assets/tests/tests.go | 2 +- cmd/assess.go | 4 +- cmd/dump.go | 12 +- cmd/root.go | 20 +-- connector/services/aws/iam/roles.go | 152 ------------------ go.mod | 4 +- go.sum | 11 +- .../connector}/cloud_connector.go | 2 +- .../connector}/connector_structs.go | 4 +- .../connector}/services/aws/aws.go | 12 +- .../services/aws/database/dynamodb.go | 2 +- .../connector}/services/aws/database/rds.go | 2 +- .../services/aws/database/redshift.go | 2 +- .../services/aws/database/structs.go | 0 .../connector}/services/aws/ec2/ec2s.go | 49 +++--- .../connector}/services/aws/ec2/structs.go | 0 .../connector}/services/aws/ec2/vpcs.go | 0 .../connector}/services/aws/iam/groups.go | 39 ++--- .../connector}/services/aws/iam/policies.go | 0 pkg/connector/services/aws/iam/roles.go | 133 +++++++++++++++ .../connector}/services/aws/iam/structs.go | 0 .../connector}/services/aws/iam/users.go | 62 +++---- .../connector}/services/aws/lambda/lambda.go | 37 ++--- .../connector}/services/aws/lambda/structs.go | 0 .../connector}/services/aws/s3/s3.go | 40 ++--- .../connector}/services/aws/s3/structs.go | 0 .../connector}/services/aws/structs.go | 0 .../connector}/services/aws/sts/sts.go | 0 .../connector}/services/aws/tools.go | 0 .../connector}/services/neo4j/neo4j.go | 0 .../connector}/services/neo4j/tools.go | 10 +- .../connector}/services/neo4j/writer.go | 12 +- .../connector}/storage_connector.go | 12 +- pkg/io/logging/logging.go | 2 +- 35 files changed, 260 insertions(+), 371 deletions(-) delete mode 100644 connector/services/aws/iam/roles.go rename {connector => pkg/connector}/cloud_connector.go (96%) rename {connector => pkg/connector}/connector_structs.go (53%) rename {connector => pkg/connector}/services/aws/aws.go (90%) rename {connector => pkg/connector}/services/aws/database/dynamodb.go (94%) rename {connector => pkg/connector}/services/aws/database/rds.go (96%) rename {connector => pkg/connector}/services/aws/database/redshift.go (94%) rename {connector => pkg/connector}/services/aws/database/structs.go (100%) rename {connector => pkg/connector}/services/aws/ec2/ec2s.go (74%) rename {connector => pkg/connector}/services/aws/ec2/structs.go (100%) rename {connector => pkg/connector}/services/aws/ec2/vpcs.go (100%) rename {connector => pkg/connector}/services/aws/iam/groups.go (60%) rename {connector => pkg/connector}/services/aws/iam/policies.go (100%) create mode 100644 pkg/connector/services/aws/iam/roles.go rename {connector => pkg/connector}/services/aws/iam/structs.go (100%) rename {connector => pkg/connector}/services/aws/iam/users.go (75%) rename {connector => pkg/connector}/services/aws/lambda/lambda.go (68%) rename {connector => pkg/connector}/services/aws/lambda/structs.go (100%) rename {connector => pkg/connector}/services/aws/s3/s3.go (80%) rename {connector => pkg/connector}/services/aws/s3/structs.go (100%) rename {connector => pkg/connector}/services/aws/structs.go (100%) rename {connector => pkg/connector}/services/aws/sts/sts.go (100%) rename {connector => pkg/connector}/services/aws/tools.go (100%) rename {connector => pkg/connector}/services/neo4j/neo4j.go (100%) rename {connector => pkg/connector}/services/neo4j/tools.go (92%) rename {connector => pkg/connector}/services/neo4j/writer.go (97%) rename {connector => pkg/connector}/storage_connector.go (90%) diff --git a/Makefile b/Makefile index 94f6af8..38fe1af 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,6 @@ stop-containers: @docker compose down -v @docker compose rm -fv -.PHONY: tests -tests: start-containers build - cd ./assets/ && $(MAKE) -C tests all \ No newline at end of file +.PHONY: test +test: start-containers build + cd ./assets/ && $(MAKE) -C tests all diff --git a/assets/tests/tests.go b/assets/tests/tests.go index f2c71ae..e941047 100644 --- a/assets/tests/tests.go +++ b/assets/tests/tests.go @@ -3,7 +3,7 @@ package tests import ( "context" - awsconnector "github.com/primait/nuvola/connector/services/aws" + awsconnector "github.com/primait/nuvola/pkg/connector/services/aws" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/retry" diff --git a/cmd/assess.go b/cmd/assess.go index c5b5cbd..2f92a3b 100644 --- a/cmd/assess.go +++ b/cmd/assess.go @@ -8,7 +8,7 @@ import ( "log" "strings" - "github.com/primait/nuvola/connector" + "github.com/primait/nuvola/pkg/connector" "github.com/primait/nuvola/pkg/io/logging" "github.com/primait/nuvola/tools/filesystem/files" unzip "github.com/primait/nuvola/tools/filesystem/zip" @@ -29,7 +29,7 @@ var assessCmd = &cobra.Command{ connector.SetActions() storageConnector := connector.NewStorageConnector() - if importFile != "" && !noImport { + if importFile != "" { logger.Info("Flushing database") logger.Info(fmt.Sprintf("Importing %s", importFile)) importZipFile(storageConnector, importFile) diff --git a/cmd/dump.go b/cmd/dump.go index 594c9de..9bee81c 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -6,8 +6,7 @@ import ( "reflect" "time" - "github.com/primait/nuvola/connector" - "github.com/primait/nuvola/pkg/io/logging" + "github.com/primait/nuvola/pkg/connector" "github.com/primait/nuvola/tools/filesystem/files" "github.com/primait/nuvola/tools/filesystem/zip" "github.com/spf13/cobra" @@ -33,10 +32,6 @@ var dumpCmd = &cobra.Command{ Short: "Dump AWS resources and policies information and store them in Neo4j", Run: func(cmd *cobra.Command, args []string) { startTime := time.Now() - markAsRequired("aws-profile") - if err := rootCmd.ValidateRequiredFlags(); err != nil { - logger.Error("Required flags not provided", "err", err) - } if cmd.Flags().Changed(flagVerbose) { logger.SetVerboseLevel() } @@ -60,7 +55,7 @@ var dumpCmd = &cobra.Command{ }, } -func dumpData(storageConnector *connector.StorageConnector, cloudConnector *connector.CloudConnector) map[string]interface{} { +func dumpData(storageConnector *connector.StorageConnector, cloudConnector *connector.CloudConnector) { dataChan := make(chan map[string]interface{}) go cloudConnector.DumpAll("aws", dataChan) for { @@ -77,7 +72,6 @@ func dumpData(storageConnector *connector.StorageConnector, cloudConnector *conn storageConnector.ImportResults(mapKey, obj) AWSResults[mapKey] = a[mapKey] } - return AWSResults } func saveResults(awsProfile string, outputDir string, outputFormat string) { @@ -90,8 +84,6 @@ func saveResults(awsProfile string, outputDir string, outputFormat string) { today := time.Now().Format("20060102") for key, value := range AWSResults { - logger.Info(key, logging.PrettyJSON(value)) - if outputFormat == "json" { files.PrettyJSONToFile(outputDir, fmt.Sprintf("%s_%s.json", key, today), value) } diff --git a/cmd/root.go b/cmd/root.go index 57da535..22d7b21 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,13 +36,15 @@ func init() { logger = logging.GetLogManager() rootCmd.PersistentFlags().BoolP(flagVerbose, "v", false, "Verbose output") rootCmd.PersistentFlags().BoolP(flagDebug, "d", false, "Debug output") - dumpCmd.PersistentFlags().StringVarP(&awsProfile, flagAWSProfile, "p", "default", "AWS Profile to use") - dumpCmd.PersistentFlags().StringVarP(&outputDirectory, flagOutputDirectory, "o", "", "Output folder where the files will be saved (default: \".\")") - dumpCmd.PersistentFlags().StringVarP(&outputFormat, flagOutputFormat, "f", "zip", "Output format: ZIP or json files") - dumpCmd.PersistentFlags().BoolVarP(&dumpOnly, flagDumpOnly, "", false, "Flag to prevent loading data into Neo4j (default: \"false\")") + dumpCmd.Flags().StringVarP(&awsProfile, flagAWSProfile, "p", "default", "AWS Profile to use") + dumpCmd.Flags().StringVarP(&outputDirectory, flagOutputDirectory, "o", "", "Output folder where the files will be saved (default: \".\")") + dumpCmd.Flags().StringVarP(&outputFormat, flagOutputFormat, "f", "zip", "Output format: ZIP or json files") + dumpCmd.Flags().BoolVarP(&dumpOnly, flagDumpOnly, "", false, "Flag to prevent loading data into Neo4j (default: \"false\")") + _ = dumpCmd.MarkFlagRequired(flagAWSProfile) - assessCmd.PersistentFlags().StringVarP(&importFile, flagImportFile, "i", "", "Input ZIP file to load") - assessCmd.PersistentFlags().BoolVarP(&noImport, flagNoImport, "", false, "Use stored data from Neo4j without import (default)") + assessCmd.Flags().StringVarP(&importFile, flagImportFile, "i", "", "Input ZIP file to load") + assessCmd.Flags().BoolVarP(&noImport, flagNoImport, "", false, "Use stored data from Neo4j without import (default)") + assessCmd.MarkFlagsMutuallyExclusive(flagImportFile, flagNoImport) } func Execute() { @@ -50,9 +52,3 @@ func Execute() { logger.Error("Error executing command", "err", err) } } - -func markAsRequired(flag string) { - if err := rootCmd.MarkFlagRequired(flag); err != nil { - logger.Error("Required flags not provided", "err", err, "flag", flag) - } -} diff --git a/connector/services/aws/iam/roles.go b/connector/services/aws/iam/roles.go deleted file mode 100644 index ebd0fe0..0000000 --- a/connector/services/aws/iam/roles.go +++ /dev/null @@ -1,152 +0,0 @@ -package iam - -import ( - "context" - "encoding/json" - "errors" - "net/url" - "reflect" - "sort" - "sync" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/primait/nuvola/pkg/io/logging" - "golang.org/x/sync/semaphore" -) - -// aws iam list-roles and aws iam list-instance-profiles -func ListRoles(cfg aws.Config) (roles []*Role) { - var ( - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(30)) - wg sync.WaitGroup - ) - iamClient = IAMClient{Config: cfg, client: iam.NewFromConfig(cfg)} - - for _, role := range listRoles() { - wg.Add(1) - go func(role types.Role) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "IAM - Roles", "ListRoles - Acquire Semaphore") - } - - defer sem.Release(1) - defer mu.Unlock() - defer wg.Done() - var assumeRoleDocument = PolicyDocument{} - var instanceProfileRef = "" - var instanceProfileArn = "" - decodedValue, _ := url.QueryUnescape(*role.AssumeRolePolicyDocument) - err := json.Unmarshal([]byte(decodedValue), &assumeRoleDocument) - if err != nil { - logging.HandleError(err, "IAM - Roles", "Umarshalling assumeRoleDocument") - } - - // Sort Service object in the AssumeRolePolicyDocument; useful to diff different JSON outputs - assumableBy := []string{} - for i, assStatement := range assumeRoleDocument.Statement { - if reflect.ValueOf(assStatement.Principal.Service).Kind() == reflect.String { - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Service.(string)) - } else if reflect.ValueOf(assStatement.Principal.Service).Kind() == reflect.Slice { - assumeRoleDocument.Statement[i].Principal.Service = sortStringSlice(assStatement.Principal.Service) - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Service.([]string)...) - } - - if reflect.ValueOf(assStatement.Principal.AWS).Kind() == reflect.String { - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.AWS.(string)) - } else if reflect.ValueOf(assStatement.Principal.AWS).Kind() == reflect.Slice { - assumeRoleDocument.Statement[i].Principal.AWS = sortStringSlice(assStatement.Principal.AWS) - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.AWS.([]string)...) - } - - if reflect.ValueOf(assStatement.Principal.Federated).Kind() == reflect.String { - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Federated.(string)) - } else if reflect.ValueOf(assStatement.Principal.Federated).Kind() == reflect.Slice { - assumeRoleDocument.Statement[i].Principal.Federated = sortStringSlice(assStatement.Principal.Federated) - assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Federated.([]string)...) - } - } - sort.Strings(assumableBy) - - for _, instanceProfile := range listInstanceProfiles() { - for _, r := range instanceProfile.Roles { - if aws.ToString(r.RoleId) == aws.ToString(role.RoleId) { - instanceProfileRef = aws.ToString(instanceProfile.InstanceProfileId) - instanceProfileArn = aws.ToString(instanceProfile.Arn) - } - } - } - - inline := iamClient.listInlinePolicies(aws.ToString(role.RoleName), "role") - attached := iamClient.listAttachedPolicies(aws.ToString(role.RoleName), "role") - var item = &Role{ - Role: role, - AssumeRolePolicyDocument: assumeRoleDocument, - AssumableBy: assumableBy, - AttachedPolicies: attached, - InlinePolicies: inline, - InstanceProfileID: instanceProfileRef, - InstanceProfileArn: instanceProfileArn, - } - mu.Lock() - roles = append(roles, item) - }(role) - } - wg.Wait() - - sort.Slice(roles, func(i, j int) bool { - return aws.ToString(roles[i].RoleName) < aws.ToString(roles[j].RoleName) - }) - - return -} - -func listRoles() []types.Role { - var ( - marker *string - collectedRoles []types.Role - ) - - for { - roleOutput, err := iamClient.client.ListRoles(context.TODO(), &iam.ListRolesInput{ - Marker: marker, - MaxItems: aws.Int32(300), - }) - if errors.As(err, &re) { - logging.HandleAWSError(re, "IAM - Roles", "ListRoles") - } - - collectedRoles = append(collectedRoles, roleOutput.Roles...) - if !roleOutput.IsTruncated { - break - } - marker = roleOutput.Marker - } - return collectedRoles -} - -func listInstanceProfiles() []types.InstanceProfile { - var ( - marker *string - collectedInstanceProfiles []types.InstanceProfile - ) - - for { - roleOutput, err := iamClient.client.ListInstanceProfiles(context.TODO(), &iam.ListInstanceProfilesInput{ - Marker: marker, - MaxItems: aws.Int32(300), - }) - if errors.As(err, &re) { - logging.HandleAWSError(re, "IAM - Roles", "ListInstanceProfiles") - } - - collectedInstanceProfiles = append(collectedInstanceProfiles, roleOutput.InstanceProfiles...) - if !roleOutput.IsTruncated { - break - } - marker = roleOutput.Marker - } - return collectedInstanceProfiles -} diff --git a/go.mod b/go.mod index 50504cb..5cc3063 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( github.com/notdodo/arner v0.0.0-20230222134658-4fe417a6cc9c github.com/notdodo/goflat v0.0.0-20220904193052-d6f007cccdea github.com/ohler55/ojg v1.21.5 + github.com/sourcegraph/conc v0.3.0 github.com/spf13/cobra v1.8.0 - golang.org/x/sync v0.7.0 golang.org/x/text v0.14.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -72,7 +72,9 @@ require ( github.com/refraction-networking/utls v1.6.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect go.uber.org/mock v0.4.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect golang.org/x/mod v0.17.0 // indirect diff --git a/go.sum b/go.sum index 1975d90..a0b1b31 100644 --- a/go.sum +++ b/go.sum @@ -150,16 +150,23 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= @@ -183,8 +190,8 @@ golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/connector/cloud_connector.go b/pkg/connector/cloud_connector.go similarity index 96% rename from connector/cloud_connector.go rename to pkg/connector/cloud_connector.go index a13b0e0..d11cca2 100644 --- a/connector/cloud_connector.go +++ b/pkg/connector/cloud_connector.go @@ -4,7 +4,7 @@ import ( "errors" "strings" - awsconfig "github.com/primait/nuvola/connector/services/aws" + awsconfig "github.com/primait/nuvola/pkg/connector/services/aws" ) func NewCloudConnector(profile string, endpointUrl string) (*CloudConnector, error) { diff --git a/connector/connector_structs.go b/pkg/connector/connector_structs.go similarity index 53% rename from connector/connector_structs.go rename to pkg/connector/connector_structs.go index 76ebf57..12c966e 100644 --- a/connector/connector_structs.go +++ b/pkg/connector/connector_structs.go @@ -1,8 +1,8 @@ package connector import ( - awsconfig "github.com/primait/nuvola/connector/services/aws" - neo4jconnector "github.com/primait/nuvola/connector/services/neo4j" + awsconfig "github.com/primait/nuvola/pkg/connector/services/aws" + neo4jconnector "github.com/primait/nuvola/pkg/connector/services/neo4j" ) type StorageConnector struct { diff --git a/connector/services/aws/aws.go b/pkg/connector/services/aws/aws.go similarity index 90% rename from connector/services/aws/aws.go rename to pkg/connector/services/aws/aws.go index e70bf29..1272716 100644 --- a/connector/services/aws/aws.go +++ b/pkg/connector/services/aws/aws.go @@ -4,12 +4,12 @@ import ( "context" "os" - "github.com/primait/nuvola/connector/services/aws/database" - "github.com/primait/nuvola/connector/services/aws/ec2" - "github.com/primait/nuvola/connector/services/aws/iam" - "github.com/primait/nuvola/connector/services/aws/lambda" - "github.com/primait/nuvola/connector/services/aws/s3" - "github.com/primait/nuvola/connector/services/aws/sts" + "github.com/primait/nuvola/pkg/connector/services/aws/database" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/iam" + "github.com/primait/nuvola/pkg/connector/services/aws/lambda" + "github.com/primait/nuvola/pkg/connector/services/aws/s3" + "github.com/primait/nuvola/pkg/connector/services/aws/sts" "github.com/primait/nuvola/pkg/io/logging" "github.com/aws/aws-sdk-go-v2/aws" diff --git a/connector/services/aws/database/dynamodb.go b/pkg/connector/services/aws/database/dynamodb.go similarity index 94% rename from connector/services/aws/database/dynamodb.go rename to pkg/connector/services/aws/database/dynamodb.go index 36f27a1..cdc8cda 100644 --- a/connector/services/aws/database/dynamodb.go +++ b/pkg/connector/services/aws/database/dynamodb.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/primait/nuvola/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" "github.com/primait/nuvola/pkg/io/logging" "github.com/aws/aws-sdk-go-v2/aws" diff --git a/connector/services/aws/database/rds.go b/pkg/connector/services/aws/database/rds.go similarity index 96% rename from connector/services/aws/database/rds.go rename to pkg/connector/services/aws/database/rds.go index 43fd664..515af82 100644 --- a/connector/services/aws/database/rds.go +++ b/pkg/connector/services/aws/database/rds.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/primait/nuvola/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" "github.com/primait/nuvola/pkg/io/logging" "github.com/aws/aws-sdk-go-v2/aws" diff --git a/connector/services/aws/database/redshift.go b/pkg/connector/services/aws/database/redshift.go similarity index 94% rename from connector/services/aws/database/redshift.go rename to pkg/connector/services/aws/database/redshift.go index ee5f58f..1010d74 100644 --- a/connector/services/aws/database/redshift.go +++ b/pkg/connector/services/aws/database/redshift.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/primait/nuvola/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" "github.com/primait/nuvola/pkg/io/logging" "github.com/aws/aws-sdk-go-v2/aws" diff --git a/connector/services/aws/database/structs.go b/pkg/connector/services/aws/database/structs.go similarity index 100% rename from connector/services/aws/database/structs.go rename to pkg/connector/services/aws/database/structs.go diff --git a/connector/services/aws/ec2/ec2s.go b/pkg/connector/services/aws/ec2/ec2s.go similarity index 74% rename from connector/services/aws/ec2/ec2s.go rename to pkg/connector/services/aws/ec2/ec2s.go index 7d8116d..591aa87 100644 --- a/connector/services/aws/ec2/ec2s.go +++ b/pkg/connector/services/aws/ec2/ec2s.go @@ -4,14 +4,13 @@ import ( "context" b64 "encoding/base64" "errors" - "sync" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/primait/nuvola/pkg/io/logging" - "golang.org/x/sync/semaphore" + "github.com/sourcegraph/conc/iter" ) func ListInstances(cfg aws.Config) (ec2s []*Instance, err *awshttp.ResponseError) { @@ -27,12 +26,6 @@ func ListInstances(cfg aws.Config) (ec2s []*Instance, err *awshttp.ResponseError } func (ec *EC2Client) listInstancesForRegion() (ec2s []*Instance) { - var ( - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(15)) - wg sync.WaitGroup - ) - output, err := ec.client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ MaxResults: aws.Int32(1000), Filters: []types.Filter{{ @@ -44,30 +37,24 @@ func (ec *EC2Client) listInstancesForRegion() (ec2s []*Instance) { logging.HandleAWSError(re, "EC2", "listInstancesForRegion") } - for _, instances := range output.Reservations { - wg.Add(1) - go func(instances types.Reservation) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "EC2", "listInstancesForRegion - Acquire Semaphore") - } - defer sem.Release(1) - defer wg.Done() - var instancesSlice []*Instance - for _, instance := range instances.Instances { - userData := ec.getInstanceUserDataAttribute(aws.ToString(instance.InstanceId)) - instancesSlice = append(instancesSlice, &Instance{ - Instance: instance, - UserData: userData, - NetworkInterfaces: ec.getNetworkInterfacesWithGroups(instance.NetworkInterfaces), - InstanceState: ec.getInstanceState(aws.ToString(instance.InstanceId)), - }) - } - mu.Lock() - defer mu.Unlock() - ec2s = append(ec2s, instancesSlice...) - }(instances) + ec2s = make([]*Instance, 0, len(output.Reservations)) + instances := iter.Map(output.Reservations, func(instances *types.Reservation) []*Instance { + var instancesSlice []*Instance + for _, instance := range instances.Instances { + userData := ec.getInstanceUserDataAttribute(aws.ToString(instance.InstanceId)) + instancesSlice = append(instancesSlice, &Instance{ + Instance: instance, + UserData: userData, + NetworkInterfaces: ec.getNetworkInterfacesWithGroups(instance.NetworkInterfaces), + InstanceState: ec.getInstanceState(aws.ToString(instance.InstanceId)), + }) + } + return instancesSlice + }) + + for _, instance := range instances { + ec2s = append(ec2s, instance...) } - wg.Wait() return } diff --git a/connector/services/aws/ec2/structs.go b/pkg/connector/services/aws/ec2/structs.go similarity index 100% rename from connector/services/aws/ec2/structs.go rename to pkg/connector/services/aws/ec2/structs.go diff --git a/connector/services/aws/ec2/vpcs.go b/pkg/connector/services/aws/ec2/vpcs.go similarity index 100% rename from connector/services/aws/ec2/vpcs.go rename to pkg/connector/services/aws/ec2/vpcs.go diff --git a/connector/services/aws/iam/groups.go b/pkg/connector/services/aws/iam/groups.go similarity index 60% rename from connector/services/aws/iam/groups.go rename to pkg/connector/services/aws/iam/groups.go index a21b069..2c718ca 100644 --- a/connector/services/aws/iam/groups.go +++ b/pkg/connector/services/aws/iam/groups.go @@ -4,46 +4,27 @@ import ( "context" "errors" "sort" - "sync" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/primait/nuvola/pkg/io/logging" - "golang.org/x/sync/semaphore" + "github.com/sourcegraph/conc/iter" ) func ListGroups(cfg aws.Config) (groups []*Group) { - var ( - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(30)) - wg sync.WaitGroup - ) - iamClient = IAMClient{client: iam.NewFromConfig(cfg), Config: cfg} - for _, group := range listGroups() { - wg.Add(1) - go func(group types.Group) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "IAM - Groups", "ListGroups - Acquire Semaphore") - } - defer sem.Release(1) - defer mu.Unlock() - defer wg.Done() - inlines := iamClient.listInlinePolicies(aws.ToString(group.GroupName), "group") - attached := iamClient.listAttachedPolicies(aws.ToString(group.GroupName), "group") + groups = iter.Map(listGroups(), func(group *types.Group) *Group { + inlines := iamClient.listInlinePolicies(aws.ToString(group.GroupName), "group") + attached := iamClient.listAttachedPolicies(aws.ToString(group.GroupName), "group") - var item = &Group{ - Group: group, - InlinePolicies: inlines, - AttachedPolicies: attached, - } - mu.Lock() - groups = append(groups, item) - }(group) - } - wg.Wait() + return &Group{ + Group: *group, + InlinePolicies: inlines, + AttachedPolicies: attached, + } + }) sort.Slice(groups, func(i, j int) bool { return aws.ToString(groups[i].GroupName) < aws.ToString(groups[j].GroupName) diff --git a/connector/services/aws/iam/policies.go b/pkg/connector/services/aws/iam/policies.go similarity index 100% rename from connector/services/aws/iam/policies.go rename to pkg/connector/services/aws/iam/policies.go diff --git a/pkg/connector/services/aws/iam/roles.go b/pkg/connector/services/aws/iam/roles.go new file mode 100644 index 0000000..6065ab6 --- /dev/null +++ b/pkg/connector/services/aws/iam/roles.go @@ -0,0 +1,133 @@ +package iam + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "reflect" + "sort" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" + "github.com/primait/nuvola/pkg/io/logging" + "github.com/sourcegraph/conc/iter" +) + +// aws iam list-roles and aws iam list-instance-profiles +func ListRoles(cfg aws.Config) (roles []*Role) { + iamClient = IAMClient{Config: cfg, client: iam.NewFromConfig(cfg)} + + roles = iter.Map(listRoles(), func(role *types.Role) *Role { + var assumeRoleDocument = PolicyDocument{} + var instanceProfileRef = "" + var instanceProfileArn = "" + decodedValue, _ := url.QueryUnescape(*role.AssumeRolePolicyDocument) + err := json.Unmarshal([]byte(decodedValue), &assumeRoleDocument) + if err != nil { + logging.HandleError(err, "IAM - Roles", "Umarshalling assumeRoleDocument") + } + + // Sort Service object in the AssumeRolePolicyDocument; useful to diff different JSON outputs + assumableBy := []string{} + for i, assStatement := range assumeRoleDocument.Statement { + if reflect.ValueOf(assStatement.Principal.Service).Kind() == reflect.String { + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Service.(string)) + } else if reflect.ValueOf(assStatement.Principal.Service).Kind() == reflect.Slice { + assumeRoleDocument.Statement[i].Principal.Service = sortStringSlice(assStatement.Principal.Service) + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Service.([]string)...) + } + + if reflect.ValueOf(assStatement.Principal.AWS).Kind() == reflect.String { + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.AWS.(string)) + } else if reflect.ValueOf(assStatement.Principal.AWS).Kind() == reflect.Slice { + assumeRoleDocument.Statement[i].Principal.AWS = sortStringSlice(assStatement.Principal.AWS) + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.AWS.([]string)...) + } + + if reflect.ValueOf(assStatement.Principal.Federated).Kind() == reflect.String { + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Federated.(string)) + } else if reflect.ValueOf(assStatement.Principal.Federated).Kind() == reflect.Slice { + assumeRoleDocument.Statement[i].Principal.Federated = sortStringSlice(assStatement.Principal.Federated) + assumableBy = append(assumableBy, assumeRoleDocument.Statement[i].Principal.Federated.([]string)...) + } + } + sort.Strings(assumableBy) + + for _, instanceProfile := range listInstanceProfiles() { + for _, r := range instanceProfile.Roles { + if aws.ToString(r.RoleId) == aws.ToString(role.RoleId) { + instanceProfileRef = aws.ToString(instanceProfile.InstanceProfileId) + instanceProfileArn = aws.ToString(instanceProfile.Arn) + } + } + } + + inline := iamClient.listInlinePolicies(aws.ToString(role.RoleName), "role") + attached := iamClient.listAttachedPolicies(aws.ToString(role.RoleName), "role") + return &Role{ + Role: *role, + AssumeRolePolicyDocument: assumeRoleDocument, + AssumableBy: assumableBy, + AttachedPolicies: attached, + InlinePolicies: inline, + InstanceProfileID: instanceProfileRef, + InstanceProfileArn: instanceProfileArn, + } + }) + + sort.Slice(roles, func(i, j int) bool { + return aws.ToString(roles[i].RoleName) < aws.ToString(roles[j].RoleName) + }) + + return +} + +func listRoles() []types.Role { + var ( + marker *string + collectedRoles []types.Role + ) + + for { + roleOutput, err := iamClient.client.ListRoles(context.TODO(), &iam.ListRolesInput{ + Marker: marker, + MaxItems: aws.Int32(300), + }) + if errors.As(err, &re) { + logging.HandleAWSError(re, "IAM - Roles", "ListRoles") + } + + collectedRoles = append(collectedRoles, roleOutput.Roles...) + if !roleOutput.IsTruncated { + break + } + marker = roleOutput.Marker + } + return collectedRoles +} + +func listInstanceProfiles() []types.InstanceProfile { + var ( + marker *string + collectedInstanceProfiles []types.InstanceProfile + ) + + for { + roleOutput, err := iamClient.client.ListInstanceProfiles(context.TODO(), &iam.ListInstanceProfilesInput{ + Marker: marker, + MaxItems: aws.Int32(300), + }) + if errors.As(err, &re) { + logging.HandleAWSError(re, "IAM - Roles", "ListInstanceProfiles") + } + + collectedInstanceProfiles = append(collectedInstanceProfiles, roleOutput.InstanceProfiles...) + if !roleOutput.IsTruncated { + break + } + marker = roleOutput.Marker + } + return collectedInstanceProfiles +} diff --git a/connector/services/aws/iam/structs.go b/pkg/connector/services/aws/iam/structs.go similarity index 100% rename from connector/services/aws/iam/structs.go rename to pkg/connector/services/aws/iam/structs.go diff --git a/connector/services/aws/iam/users.go b/pkg/connector/services/aws/iam/users.go similarity index 75% rename from connector/services/aws/iam/users.go rename to pkg/connector/services/aws/iam/users.go index 954df23..99565a9 100644 --- a/connector/services/aws/iam/users.go +++ b/pkg/connector/services/aws/iam/users.go @@ -6,7 +6,6 @@ import ( "errors" "log" "sort" - "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -14,17 +13,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/gocarina/gocsv" "github.com/primait/nuvola/pkg/io/logging" - "golang.org/x/sync/semaphore" + "github.com/sourcegraph/conc/iter" ) // aws iam list-users func ListUsers(cfg aws.Config, credentialReport map[string]*CredentialReport) (users []*User) { - var ( - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(30)) - wg sync.WaitGroup - ) - if len(credentialReport) > 0 { rootAccount := credentialReport[""] rootDate, _ := time.Parse("2006-01-02T15:04:05+00:00", rootAccount.UserCreation) @@ -43,39 +36,26 @@ func ListUsers(cfg aws.Config, credentialReport map[string]*CredentialReport) (u }) } - for _, user := range listUsers() { - wg.Add(1) - go func(user types.User) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "IAM - Users", "ListUsers - Acquire Semaphore") - } - defer sem.Release(1) - defer mu.Unlock() - defer wg.Done() - - groups := iamClient.listGroupsForUser(aws.ToString(user.UserName)) - inline := iamClient.listInlinePolicies(aws.ToString(user.UserName), "user") - attached := iamClient.listAttachedPolicies(aws.ToString(user.UserName), "user") - accessKeys := iamClient.listAccessKeys(aws.ToString(user.UserName)) - loginProfile := iamClient.listLoginProfile(aws.ToString(user.UserName)) - - userAccount := credentialReport[aws.ToString(user.UserName)] - var item = &User{ - User: user, - Groups: groups, - AccessKeys: accessKeys, - LoginProfile: loginProfile, - InlinePolicies: inline, - AttachedPolicies: attached, - PasswordEnabled: userAccount.PasswordEnabled, - PasswordLastChanged: userAccount.PasswordLastChanged, - MfaActive: userAccount.MfaActive, - } - mu.Lock() - users = append(users, item) - }(user) - } - wg.Wait() + users = append(users, iter.Map(listUsers(), func(user *types.User) *User { + groups := iamClient.listGroupsForUser(aws.ToString(user.UserName)) + inline := iamClient.listInlinePolicies(aws.ToString(user.UserName), "user") + attached := iamClient.listAttachedPolicies(aws.ToString(user.UserName), "user") + accessKeys := iamClient.listAccessKeys(aws.ToString(user.UserName)) + loginProfile := iamClient.listLoginProfile(aws.ToString(user.UserName)) + + userAccount := credentialReport[aws.ToString(user.UserName)] + return &User{ + User: *user, + Groups: groups, + AccessKeys: accessKeys, + LoginProfile: loginProfile, + InlinePolicies: inline, + AttachedPolicies: attached, + PasswordEnabled: userAccount.PasswordEnabled, + PasswordLastChanged: userAccount.PasswordLastChanged, + MfaActive: userAccount.MfaActive, + } + })...) sort.Slice(users, func(i, j int) bool { return aws.ToString(users[i].UserName) < aws.ToString(users[j].UserName) diff --git a/connector/services/aws/lambda/lambda.go b/pkg/connector/services/aws/lambda/lambda.go similarity index 68% rename from connector/services/aws/lambda/lambda.go rename to pkg/connector/services/aws/lambda/lambda.go index 056ae65..c690767 100644 --- a/connector/services/aws/lambda/lambda.go +++ b/pkg/connector/services/aws/lambda/lambda.go @@ -4,16 +4,15 @@ import ( "context" "encoding/json" "errors" - "sync" - "github.com/primait/nuvola/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" "github.com/primait/nuvola/pkg/io/logging" + "github.com/sourcegraph/conc/iter" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "golang.org/x/sync/semaphore" ) // aws iam list-users @@ -30,37 +29,19 @@ func ListFunctions(cfg aws.Config) (lambdas []*Lambda) { } func (lc *LambdaClient) listFunctionsForRegion() (lambdas []*Lambda) { - var ( - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(20)) - wg sync.WaitGroup - ) - output, err := lc.client.ListFunctions(context.TODO(), &lambda.ListFunctionsInput{}) if errors.As(err, &re) { logging.HandleAWSError(re, "Lambda", "ListFunctions") } - for _, lambda := range output.Functions { - wg.Add(1) - go func(lambda types.FunctionConfiguration) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "Lambda", "listFunctionsForRegion - Acquire Semaphore") - } - defer sem.Release(1) - defer wg.Done() - defer mu.Unlock() - var item = &Lambda{ - FunctionConfiguration: lambda, - FunctionCodeLocation: lc.getFunctionCodeLocation(aws.ToString(lambda.FunctionName)), - Policy: lc.getPolicy(aws.ToString(lambda.FunctionName)), - } - mu.Lock() - lambdas = append(lambdas, item) - }(lambda) - } - wg.Wait() + lambdas = iter.Map(output.Functions, func(lambda *types.FunctionConfiguration) *Lambda { + return &Lambda{ + FunctionConfiguration: *lambda, + FunctionCodeLocation: lc.getFunctionCodeLocation(aws.ToString(lambda.FunctionName)), + Policy: lc.getPolicy(aws.ToString(lambda.FunctionName)), + } + }) return } diff --git a/connector/services/aws/lambda/structs.go b/pkg/connector/services/aws/lambda/structs.go similarity index 100% rename from connector/services/aws/lambda/structs.go rename to pkg/connector/services/aws/lambda/structs.go diff --git a/connector/services/aws/s3/s3.go b/pkg/connector/services/aws/s3/s3.go similarity index 80% rename from connector/services/aws/s3/s3.go rename to pkg/connector/services/aws/s3/s3.go index 9ea8509..8907b76 100644 --- a/connector/services/aws/s3/s3.go +++ b/pkg/connector/services/aws/s3/s3.go @@ -6,51 +6,33 @@ import ( "errors" "sort" "strings" - "sync" - "github.com/primait/nuvola/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" "github.com/primait/nuvola/pkg/io/logging" + "github.com/sourcegraph/conc/iter" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "golang.org/x/sync/semaphore" ) func ListBuckets(cfg aws.Config) (buckets []*Bucket) { - var ( - s3Client = S3Client{Config: cfg, client: s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })} - mu = &sync.Mutex{} - sem = semaphore.NewWeighted(int64(15)) - wg sync.WaitGroup - ) + s3Client := S3Client{Config: cfg, client: s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })} output, err := s3Client.client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) if errors.As(err, &re) { logging.HandleAWSError(re, "S3", "ListBuckets") } - for _, bucket := range output.Buckets { - wg.Add(1) - go func(bucket types.Bucket) { - if err := sem.Acquire(context.Background(), 1); err != nil { - logging.HandleError(err, "S3", "ListBuckets - Acquire Semaphore") - } - defer sem.Release(1) - defer mu.Unlock() - defer wg.Done() - var item = &Bucket{ - Bucket: bucket, - Policy: s3Client.getBucketPolicy(bucket.Name), - ACL: s3Client.listBucketACL(bucket.Name), - Encrypted: s3Client.getEncryptionStatus(bucket.Name), - } - mu.Lock() - buckets = append(buckets, item) - }(bucket) - } - wg.Wait() + buckets = iter.Map(output.Buckets, func(bucket *types.Bucket) *Bucket { + return &Bucket{ + Bucket: *bucket, + Policy: s3Client.getBucketPolicy(bucket.Name), + ACL: s3Client.listBucketACL(bucket.Name), + Encrypted: s3Client.getEncryptionStatus(bucket.Name), + } + }) sort.Slice(buckets, func(i, j int) bool { return aws.ToString(buckets[i].Name) < aws.ToString(buckets[j].Name) diff --git a/connector/services/aws/s3/structs.go b/pkg/connector/services/aws/s3/structs.go similarity index 100% rename from connector/services/aws/s3/structs.go rename to pkg/connector/services/aws/s3/structs.go diff --git a/connector/services/aws/structs.go b/pkg/connector/services/aws/structs.go similarity index 100% rename from connector/services/aws/structs.go rename to pkg/connector/services/aws/structs.go diff --git a/connector/services/aws/sts/sts.go b/pkg/connector/services/aws/sts/sts.go similarity index 100% rename from connector/services/aws/sts/sts.go rename to pkg/connector/services/aws/sts/sts.go diff --git a/connector/services/aws/tools.go b/pkg/connector/services/aws/tools.go similarity index 100% rename from connector/services/aws/tools.go rename to pkg/connector/services/aws/tools.go diff --git a/connector/services/neo4j/neo4j.go b/pkg/connector/services/neo4j/neo4j.go similarity index 100% rename from connector/services/neo4j/neo4j.go rename to pkg/connector/services/neo4j/neo4j.go diff --git a/connector/services/neo4j/tools.go b/pkg/connector/services/neo4j/tools.go similarity index 92% rename from connector/services/neo4j/tools.go rename to pkg/connector/services/neo4j/tools.go index 323099d..e2f70b0 100644 --- a/connector/services/neo4j/tools.go +++ b/pkg/connector/services/neo4j/tools.go @@ -7,11 +7,11 @@ import ( "strconv" "strings" - servicesDatabase "github.com/primait/nuvola/connector/services/aws/database" - servicesEC2 "github.com/primait/nuvola/connector/services/aws/ec2" - servicesIAM "github.com/primait/nuvola/connector/services/aws/iam" - servicesLambda "github.com/primait/nuvola/connector/services/aws/lambda" - servicesS3 "github.com/primait/nuvola/connector/services/aws/s3" + servicesDatabase "github.com/primait/nuvola/pkg/connector/services/aws/database" + servicesEC2 "github.com/primait/nuvola/pkg/connector/services/aws/ec2" + servicesIAM "github.com/primait/nuvola/pkg/connector/services/aws/iam" + servicesLambda "github.com/primait/nuvola/pkg/connector/services/aws/lambda" + servicesS3 "github.com/primait/nuvola/pkg/connector/services/aws/s3" "github.com/primait/nuvola/pkg/io/logging" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" diff --git a/connector/services/neo4j/writer.go b/pkg/connector/services/neo4j/writer.go similarity index 97% rename from connector/services/neo4j/writer.go rename to pkg/connector/services/neo4j/writer.go index e458ccd..050c9fa 100644 --- a/connector/services/neo4j/writer.go +++ b/pkg/connector/services/neo4j/writer.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - awsconfig "github.com/primait/nuvola/connector/services/aws" - servicesDatabase "github.com/primait/nuvola/connector/services/aws/database" - servicesEC2 "github.com/primait/nuvola/connector/services/aws/ec2" - servicesIAM "github.com/primait/nuvola/connector/services/aws/iam" - servicesLambda "github.com/primait/nuvola/connector/services/aws/lambda" - servicesS3 "github.com/primait/nuvola/connector/services/aws/s3" + awsconfig "github.com/primait/nuvola/pkg/connector/services/aws" + servicesDatabase "github.com/primait/nuvola/pkg/connector/services/aws/database" + servicesEC2 "github.com/primait/nuvola/pkg/connector/services/aws/ec2" + servicesIAM "github.com/primait/nuvola/pkg/connector/services/aws/iam" + servicesLambda "github.com/primait/nuvola/pkg/connector/services/aws/lambda" + servicesS3 "github.com/primait/nuvola/pkg/connector/services/aws/s3" "github.com/primait/nuvola/pkg/io/logging" "strings" diff --git a/connector/storage_connector.go b/pkg/connector/storage_connector.go similarity index 90% rename from connector/storage_connector.go rename to pkg/connector/storage_connector.go index 529e598..7617d03 100644 --- a/connector/storage_connector.go +++ b/pkg/connector/storage_connector.go @@ -8,12 +8,12 @@ import ( "github.com/primait/nuvola/pkg/io/logging" - "github.com/primait/nuvola/connector/services/aws/database" - "github.com/primait/nuvola/connector/services/aws/ec2" - "github.com/primait/nuvola/connector/services/aws/iam" - "github.com/primait/nuvola/connector/services/aws/lambda" - "github.com/primait/nuvola/connector/services/aws/s3" - neo4jconnector "github.com/primait/nuvola/connector/services/neo4j" + "github.com/primait/nuvola/pkg/connector/services/aws/database" + "github.com/primait/nuvola/pkg/connector/services/aws/ec2" + "github.com/primait/nuvola/pkg/connector/services/aws/iam" + "github.com/primait/nuvola/pkg/connector/services/aws/lambda" + "github.com/primait/nuvola/pkg/connector/services/aws/s3" + neo4jconnector "github.com/primait/nuvola/pkg/connector/services/neo4j" ) func NewStorageConnector() *StorageConnector { diff --git a/pkg/io/logging/logging.go b/pkg/io/logging/logging.go index 5967b87..4eec251 100644 --- a/pkg/io/logging/logging.go +++ b/pkg/io/logging/logging.go @@ -32,7 +32,7 @@ func GetLogManager() LogManager { logger = &logManager{ logger: log.NewWithOptions(os.Stdout, log.Options{ CallerOffset: 1, - Level: log.WarnLevel, + Level: log.InfoLevel, ReportCaller: true, ReportTimestamp: true, TimeFormat: time.RFC1123,