Skip to content

Commit

Permalink
enh: some logging refactoring and general improvs (#31)
Browse files Browse the repository at this point in the history
  • Loading branch information
notdodo authored Jun 7, 2024
1 parent d5f1611 commit c9d444c
Show file tree
Hide file tree
Showing 13 changed files with 247 additions and 278 deletions.
6 changes: 1 addition & 5 deletions assets/tests/Makefile
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
include .env

BUILD_FLAGS=-ldflags="-s -w"
OUT_DIR=dist
OUT_PREFIX=nuvola

all:
docker compose up -d
pipenv install
pipenv run tflocal apply -auto-approve
pipenv run ../../nuvola dump --output-dir ./ --aws-endpoint-url http://localhost:4566
pipenv run ../../nuvola dump --output-dir ./ --aws-endpoint-url http://localhost:4566 --verbose --debug
2 changes: 1 addition & 1 deletion cmd/assess.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func importZipFile(connector *connector.StorageConnector, zipfile string) {
continue
}
if err := processZipFile(connector, f); err != nil {
logging.HandleError(err, "Assess", "Processing ZIP file")
logger.Error("Processing ZIP file", "err", err)
}
}
}
Expand Down
121 changes: 67 additions & 54 deletions cmd/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"encoding/json"
"fmt"
"reflect"
"time"

"github.com/primait/nuvola/pkg/connector"
Expand All @@ -12,80 +11,94 @@ import (
"github.com/spf13/cobra"
)

var AWSResults = map[string]interface{}{
"Whoami": nil,
"CredentialReport": nil,
"Groups": nil,
"Users": nil,
"Roles": nil,
"Buckets": nil,
"EC2s": nil,
"VPCs": nil,
"Lambdas": nil,
"RDS": nil,
"DynamoDBs": nil,
"RedshiftDBs": nil,
}
var (
AWSResults = map[string]interface{}{
"Whoami": nil,
"CredentialReport": nil,
"Groups": nil,
"Users": nil,
"Roles": nil,
"Buckets": nil,
"EC2s": nil,
"VPCs": nil,
"Lambdas": nil,
"RDS": nil,
"DynamoDBs": nil,
"RedshiftDBs": nil,
}
dumpCmd = &cobra.Command{
Use: "dump",
Short: "Dump AWS resources and policies information and store them in Neo4j",
Run: runDumpCmd,
}
)

var dumpCmd = &cobra.Command{
Use: "dump",
Short: "Dump AWS resources and policies information and store them in Neo4j",
Run: func(cmd *cobra.Command, args []string) {
startTime := time.Now()
if cmd.Flags().Changed(flagVerbose) {
logger.SetVerboseLevel()
}
if cmd.Flags().Changed(flagDebug) {
logger.SetDebugLevel()
}
func runDumpCmd(cmd *cobra.Command, args []string) {
startTime := time.Now()

cloudConnector, err := connector.NewCloudConnector(awsProfile, awsEndpointUrl)
if err != nil {
logger.Error(err.Error())
}
if cmd.Flags().Changed(flagVerbose) {
logger.SetVerboseLevel()
}
if cmd.Flags().Changed(flagDebug) {
logger.SetDebugLevel()
}

if dumpOnly {
dumpData(nil, cloudConnector)
} else {
storageConnector := connector.NewStorageConnector().FlushAll()
dumpData(storageConnector, cloudConnector)
}
saveResults(awsProfile, outputDirectory, outputFormat)
logger.Info("Execution Time", "seconds", time.Since(startTime))
},
cloudConnector, err := connector.NewCloudConnector(awsProfile, awsEndpointUrl)
if err != nil {
logger.Error("Failed to create cloud connector", "err", err)
return
}

if dumpOnly {
dumpData(nil, cloudConnector)
} else {
storageConnector := connector.NewStorageConnector().FlushAll()
dumpData(storageConnector, cloudConnector)
}

saveResults(awsProfile, outputDirectory, outputFormat)
logger.Info("Execution Time", "seconds", time.Since(startTime))
}

func dumpData(storageConnector *connector.StorageConnector, cloudConnector *connector.CloudConnector) {
dataChan := make(chan map[string]interface{})
go cloudConnector.DumpAll("aws", dataChan)
for {
a, ok := <-dataChan // receive data step by step and import it to Neo4j
if !ok {
break
}
v := reflect.ValueOf(a)
mapKey := v.MapKeys()[0].Interface().(string)
obj, err := json.Marshal(a[mapKey])
go func() {
cloudConnector.DumpAll("aws", dataChan)
defer close(dataChan)
}()

for data := range dataChan {
processData(data, storageConnector)
}
}

func processData(data map[string]interface{}, storageConnector *connector.StorageConnector) {
for key, value := range data {
obj, err := json.Marshal(value)
if err != nil {
logger.Error("DumpData: error marshalling output", err)
logger.Error("Error marshalling output", "err", err)
continue
}
if storageConnector != nil {
storageConnector.ImportResults(key, obj)
}
storageConnector.ImportResults(mapKey, obj)
AWSResults[mapKey] = a[mapKey]
AWSResults[key] = value
}
}

func saveResults(awsProfile string, outputDir string, outputFormat string) {
func saveResults(awsProfile, outputDir, outputFormat string) {
if awsProfile == "" {
awsProfile = "default"
}
if outputFormat == "zip" {
zip.Zip(outputDir, awsProfile, &AWSResults)
zip.Zip(outputDir, awsProfile, AWSResults)
}

today := time.Now().Format("20060102")
for key, value := range AWSResults {
if outputFormat == "json" {
files.PrettyJSONToFile(outputDir, fmt.Sprintf("%s_%s.json", key, today), value)
filename := fmt.Sprintf("%s_%s.json", key, today)
files.PrettyJSONToFile(outputDir, filename, value)
}
}
}
Expand Down
73 changes: 24 additions & 49 deletions pkg/connector/cloud_connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,56 +26,30 @@ func SetActions() {
func (cc *CloudConnector) DumpAll(cloudprovider string, c chan map[string]interface{}) {
switch strings.ToLower(cloudprovider) {
case "aws":
whoami := cc.AWSConfig.DumpWhoami()
c <- map[string]interface{}{
"Whoami": whoami,
}
credentialReport := cc.AWSConfig.DumpCredentialReport()
c <- map[string]interface{}{
"CredentialReport": credentialReport,
}
groups := cc.AWSConfig.DumpIAMGroups()
c <- map[string]interface{}{
"Groups": groups,
}
users := cc.AWSConfig.DumpIAMUsers()
c <- map[string]interface{}{
"Users": users,
}
roles := cc.AWSConfig.DumpIAMRoles()
c <- map[string]interface{}{
"Roles": roles,
}
buckets := cc.AWSConfig.DumpBuckets()
c <- map[string]interface{}{
"Buckets": buckets,
}
ec2 := cc.AWSConfig.DumpEC2Instances()
c <- map[string]interface{}{
"EC2s": ec2,
}
vpc := cc.AWSConfig.DumpVpcs()
c <- map[string]interface{}{
"VPCs": vpc,
}
lambda := cc.AWSConfig.DumpLambdas()
c <- map[string]interface{}{
"Lambdas": lambda,
}
rds := cc.AWSConfig.DumpRDS()
c <- map[string]interface{}{
"RDS": rds,
}
dynamodb := cc.AWSConfig.DumpDynamoDBs()
c <- map[string]interface{}{
"DynamoDBs": dynamodb,
}
redshift := cc.AWSConfig.DumpRedshiftDBs()
c <- map[string]interface{}{
"RedshiftDBs": redshift,
}
close(c)
cc.dumpAWSData(c)
default:
cc.logger.Error("Unsupported cloud provider", "cloudprovider", cloudprovider)
}
}

func (cc *CloudConnector) dumpAWSData(c chan map[string]interface{}) {
data := map[string]interface{}{
"Whoami": cc.AWSConfig.DumpWhoami(),
"CredentialReport": cc.AWSConfig.DumpCredentialReport(),
"Groups": cc.AWSConfig.DumpIAMGroups(),
"Users": cc.AWSConfig.DumpIAMUsers(),
"Roles": cc.AWSConfig.DumpIAMRoles(),
"Buckets": cc.AWSConfig.DumpBuckets(),
"EC2s": cc.AWSConfig.DumpEC2Instances(),
"VPCs": cc.AWSConfig.DumpVpcs(),
"Lambdas": cc.AWSConfig.DumpLambdas(),
"RDS": cc.AWSConfig.DumpRDS(),
"DynamoDBs": cc.AWSConfig.DumpDynamoDBs(),
"RedshiftDBs": cc.AWSConfig.DumpRedshiftDBs(),
}

for key, value := range data {
c <- map[string]interface{}{key: value}
}
}

Expand All @@ -84,6 +58,7 @@ func (cc *CloudConnector) testConnection(cloudprovider string) bool {
case "aws":
return cc.AWSConfig.TestConnection()
default:
cc.logger.Error("Unsupported cloud provider", "cloudprovider", cloudprovider)
return false
}
}
2 changes: 1 addition & 1 deletion pkg/connector/connector_structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ type StorageConnector struct {
}

type CloudConnector struct {
AWSConfig awsconfig.AWSConfig
AWSConfig *awsconfig.AWSConfig
logger logging.LogManager
}
4 changes: 2 additions & 2 deletions pkg/connector/services/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ var (
countRetries = 100
)

func InitAWSConfiguration(profile string, awsEndpoint string) (awsc AWSConfig) {
func InitAWSConfiguration(profile string, awsEndpoint string) (awsc *AWSConfig) {
// Load the Shared AWS Configuration (~/.aws/config)
cfg, _ := config.LoadDefaultConfig(context.TODO(), config.WithSharedConfigProfile(profile),
config.WithRetryer(func() aws.Retryer {
Expand All @@ -34,7 +34,7 @@ func InitAWSConfiguration(profile string, awsEndpoint string) (awsc AWSConfig) {
if awsEndpoint != "" {
cfg.BaseEndpoint = aws.String(awsEndpoint)
}
awsc = AWSConfig{Profile: profile, Config: cfg}
awsc = &AWSConfig{Profile: profile, Config: cfg}
SetActions()
// Get the available AWS regions dynamically
ec2.ListAndSaveRegions(cfg)
Expand Down
21 changes: 11 additions & 10 deletions pkg/connector/services/aws/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,37 +11,38 @@ import (
)

func SetActions() {
logger := logging.GetLogManager()
URL := "https://awspolicygen.s3.amazonaws.com/js/policies.js"
client := req.C().SetBaseURL(URL).SetTimeout(30 * time.Second).SetUserAgent("Mozilla/5.0 (X11; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0")

response := client.Get().
response, err := client.R().
SetHeader("Connection", "keep-alive").
SetHeader("Pragma", "no-cache").
SetHeader("Cache-Control", "no-cache").
Do()
if response.Err != nil {
logging.HandleError(response.Err, "AWS - SetActions", "Error on calling HTTP endpoint")
Get(URL)
if err != nil {
logger.Error("Error on calling HTTP endpoint", "err", err)
}

resString := strings.Replace(response.String(), "app.PolicyEditorConfig=", "", 1)
obj, err := oj.ParseString(resString)
if err != nil {
logging.HandleError(err, "AWS - SetActions", "Error on parsing output string")
logger.Error("Error on parsing output string", "err", err)
}
query, err := gojq.Parse(`.serviceMap[] | .StringPrefix as $prefix | .Actions[] | "\($prefix):\(.)"`)
if err != nil {
logging.HandleError(err, "AWS - SetActions", "Error on mapping string to object")
logger.Error("Error on mapping string to object", "err", err)
}

iter := query.Run(obj)
ActionsMap = make(map[string][]string, 0)
ActionsMap = make(map[string][]string)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
logging.HandleError(err, "AWS - SetActions", "Error on itering over objects")
logger.Error("Error on iterating over objects", "err", err)
}

ActionsList = append(ActionsList, v.(string))
Expand All @@ -54,9 +55,9 @@ func SetActions() {

func unique(slice []string) []string {
keys := make(map[string]bool)
list := []string{}
var list []string
for _, entry := range slice {
if _, value := keys[entry]; !value {
if !keys[entry] {
keys[entry] = true
list = append(list, entry)
}
Expand Down
3 changes: 1 addition & 2 deletions pkg/connector/storage_connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ import (

func NewStorageConnector() *StorageConnector {
neo4jURL := os.Getenv("NEO4J_URL")
neo4jUsername := "neo4j"
neo4jPassword := os.Getenv("PASSWORD")
logger := logging.GetLogManager()
client, err := neo4j.Connect(neo4jURL, neo4jUsername, neo4jPassword)
client, err := neo4j.Connect(neo4jURL, "neo4j", neo4jPassword)
if err != nil {
logger.Error("Error connecting to database", "err", err)
}
Expand Down
Loading

0 comments on commit c9d444c

Please sign in to comment.