From 8baff71499cd65f2f5664309ba681774c88080e9 Mon Sep 17 00:00:00 2001 From: Edoardo Rosa <6991986+notdodo@users.noreply.github.com> Date: Sun, 12 Nov 2023 16:04:43 +0100 Subject: [PATCH] enh: basic ORM library (#5) * enh: basic ORM library * fix lint * fix lint: consistency * fix: slice lengths --- .github/workflows/gobuild.yml | 4 ++ go.mod | 1 + go.sum | 12 ++---- pkg/app/okta_neo4j.go | 71 +++++------------------------- pkg/infra/neo4j/neo4j_client.go | 22 +++++++++- pkg/infra/neo4j/orm/orm.go | 76 +++++++++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 69 deletions(-) create mode 100644 pkg/infra/neo4j/orm/orm.go diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index 2d83312..894f1cd 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -5,9 +5,13 @@ on: - main paths: - "**.go" + - "go.mod" + - "go.sum" pull_request: paths: - "**.go" + - "go.mod" + - "go.sum" permissions: contents: read diff --git a/go.mod b/go.mod index 1f5acb5..482dc76 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/joho/godotenv v1.5.1 github.com/neo4j/neo4j-go-driver/v5 v5.14.0 + github.com/notdodo/goflat v0.0.0-20231112144802-4032d5f62c2e github.com/okta/okta-sdk-golang/v2 v2.20.0 github.com/spf13/cobra v1.8.0 ) diff --git a/go.sum b/go.sum index 72ecc4c..afc1883 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,10 @@ -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= -github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -28,12 +22,16 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/neo4j/neo4j-go-driver/v5 v5.14.0 h1:5x3vD4HkXQIktlG63jSG8v9iweGjmObIPU7Y9U0ThUI= github.com/neo4j/neo4j-go-driver/v5 v5.14.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= +github.com/notdodo/goflat v0.0.0-20231112144802-4032d5f62c2e h1:6firFyXtaJkFwuw134Xxo2StfKtR3bbIMV+GgHMEmQA= +github.com/notdodo/goflat v0.0.0-20231112144802-4032d5f62c2e/go.mod h1:JmppUbrD+0HrVpftael6rhCkaGbh6OJJcLyze4aypTE= +github.com/ohler55/ojg v1.20.3 h1:Z+fnElsA/GbI5oiT726qJaG4Ca9q5l7UO68Qd0PtkD4= github.com/okta/okta-sdk-golang/v2 v2.20.0 h1:EDKM+uOPfihOMNwgHMdno+NAsIfyXkVnoFAYVPay0YU= github.com/okta/okta-sdk-golang/v2 v2.20.0/go.mod h1:FMy5hN5G8Rd/VoS0XrfyPPhIfOVo78ZK7lvwiQRS2+U= github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFRnjpeojzOpEYs4q7Vditq8fySFG5ap3Y= github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/r3labs/diff v1.1.0 h1:V53xhrbTHrWFWq3gI4b94AjgEJOerO1+1l0xyHOBi8M= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= @@ -44,8 +42,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/app/okta_neo4j.go b/pkg/app/okta_neo4j.go index b4def6f..e5c3312 100644 --- a/pkg/app/okta_neo4j.go +++ b/pkg/app/okta_neo4j.go @@ -1,14 +1,11 @@ package app import ( - "context" "log" - "strings" "github.com/notdodo/IAMme-IAMme/pkg/infra/neo4j" "github.com/notdodo/IAMme-IAMme/pkg/infra/okta" - - neo4jSdk "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "github.com/notdodo/goflat" ) type OktaNeo4jApp interface { @@ -30,67 +27,21 @@ type oktaNeo4jApp struct { func (a *oktaNeo4jApp) Dump() { users, err := a.oktaClient.GetUsers() if err != nil { - log.Println(err.Error()) + log.Println("Error fetching users from Okta:", err) + return } - userParams := make([]map[string]interface{}, 0) + flatUsers := make([]map[string]interface{}, 0, len(users)) for _, user := range users { - userParams = append(userParams, map[string]interface{}{ - "userId": user.Id, - "status": user.Status, - "firstName": (*user.Profile)["firstName"], - "lastName": (*user.Profile)["lastName"], - }) - } - - session := a.neo4jClient.Connect() - ctx := context.TODO() - query := buildDynamicQuery(userParams) - _, err = session.ExecuteWrite(ctx, func(tx neo4jSdk.ManagedTransaction) (interface{}, error) { - _, err := tx.Run(ctx, query, map[string]interface{}{ - "userParams": userParams, + flatUser := goflat.FlatStruct(*user, goflat.FlattenerConfig{ + Separator: "_", + OmitEmpty: true, + OmitNil: true, }) - - if err != nil { - panic(err) - } - return nil, err - }) - if err != nil { - log.Fatalln(err.Error()) - } -} - -// TODO: this is ugly AF -func buildDynamicQuery(userParams []map[string]interface{}) string { - var queryBuilder strings.Builder - - queryBuilder.WriteString("UNWIND $userParams as user\n") - queryBuilder.WriteString("CREATE (u:User {\n") - - fields := userParams[0] - - i := 0 - for field := range fields { - queryBuilder.WriteString(fieldKeyToCypherProperty(field) + ": user." + fieldKeyToCypherProperty(field)) - i++ - if i < len(fields) { - queryBuilder.WriteString(",\n") - } + flatUsers = append(flatUsers, flatUser) } - queryBuilder.WriteString("\n})\n") - queryBuilder.WriteString("RETURN u\n") - - return queryBuilder.String() -} - -// TODO: this is ugly AF -func fieldKeyToCypherProperty(key interface{}) string { - keyStr, ok := key.(string) - if !ok { - panic("Invalid field key") + if _, err = a.neo4jClient.CreateNodes([]string{"User"}, &flatUsers); err != nil { + log.Fatalln(err) } - - return keyStr } diff --git a/pkg/infra/neo4j/neo4j_client.go b/pkg/infra/neo4j/neo4j_client.go index f9302ad..55e8e81 100644 --- a/pkg/infra/neo4j/neo4j_client.go +++ b/pkg/infra/neo4j/neo4j_client.go @@ -5,12 +5,14 @@ import ( "log" "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "github.com/notdodo/IAMme-IAMme/pkg/infra/neo4j/orm" ) // Neo4jClient is an interface for interacting with the Neo4j database. type Neo4jClient interface { Connect() neo4j.SessionWithContext Close() error + CreateNodes([]string, *[]map[string]interface{}) ([]map[string]interface{}, error) } // Session is an interface for a Neo4j database session. @@ -28,14 +30,24 @@ type neo4jClient struct { driver neo4j.DriverWithContext } +/* #nosec */ +//nolint:all +func (c *neo4jClient) setUpDb(session neo4j.SessionWithContext) { + session.Run(context.TODO(), "MATCH (n) DETACH DELETE n;", nil) + session.Run(context.TODO(), "CREATE CONSTRAINT IF NOT EXISTS ON (u:User) ASSERT u.Id IS UNIQUE;", nil) + session.Run(context.TODO(), "CREATE CONSTRAINT IF NOT EXISTS ON (g:Group) ASSERT g.Id IS UNIQUE;", nil) +} + func NewNeo4jClient(dbUri, username, password string) Neo4jClient { driver, err := neo4j.NewDriverWithContext(dbUri, neo4j.BasicAuth(username, password, "")) if err != nil { log.Fatalln("Invalid Neo4j login", err.Error()) } - return &neo4jClient{ + client := &neo4jClient{ driver: driver, } + client.setUpDb(client.Connect()) + return client } func (c *neo4jClient) Connect() neo4j.SessionWithContext { @@ -47,3 +59,11 @@ func (c *neo4jClient) Connect() neo4j.SessionWithContext { func (c *neo4jClient) Close() error { return c.driver.Close(context.TODO()) } + +func (c *neo4jClient) CreateNodes(labels []string, properties *[]map[string]interface{}) ([]map[string]interface{}, error) { + nodeIDs, err := orm.CreateNodes(c.Connect(), []string{"User"}, properties) + if err != nil { + log.Fatalln(err) + } + return nodeIDs, err +} diff --git a/pkg/infra/neo4j/orm/orm.go b/pkg/infra/neo4j/orm/orm.go new file mode 100644 index 0000000..d10e190 --- /dev/null +++ b/pkg/infra/neo4j/orm/orm.go @@ -0,0 +1,76 @@ +package orm + +import ( + "context" + "fmt" + "strings" + + "github.com/neo4j/neo4j-go-driver/v5/neo4j" +) + +// CreateNode creates nodes in Neo4j +func CreateNodes(session neo4j.SessionWithContext, labels []string, properties *[]map[string]interface{}) ([]map[string]interface{}, error) { + ctx := context.TODO() + result, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) { + createNodeQuery := fmt.Sprintf("UNWIND $propsList AS props CREATE (n:%s) SET n += props RETURN id(n) as id", labelString(labels)) + parameters := map[string]interface{}{"propsList": filteredProperties(properties)} + result, err := tx.Run(ctx, createNodeQuery, parameters) + if err != nil { + return nil, err + } + + return collectResults(result, ctx) + }) + + if err != nil { + return nil, err + } + + return result.([]map[string]interface{}), err +} + +func filteredProperties(properties *[]map[string]interface{}) []map[string]interface{} { + filteredProperties := make([]map[string]interface{}, 0, len(*properties)) + for _, props := range *properties { + filteredProp := make(map[string]interface{}, len(props)) + for key, value := range props { + if isPrimitive(value) { + filteredProp[key] = value + } + } + filteredProperties = append(filteredProperties, filteredProp) + } + return filteredProperties +} + +func collectResults(result neo4j.ResultWithContext, ctx context.Context) (results []map[string]interface{}, err error) { + if records, err := result.Collect(ctx); err == nil { + for _, k := range records { + results = append(results, k.AsMap()) + } + } else { + return nil, err + } + return results, result.Err() +} + +// isPrimitive checks if a value is a primitive type +func isPrimitive(value interface{}) bool { + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, + float32, float64, string, bool: + return true + default: + return false + } +} + +// labelString generates a string representation of labels for use in Cypher queries. +func labelString(labels []string) string { + return joinLabels(labels) +} + +// joinLabels joins label strings with ":" separator. +func joinLabels(labels []string) string { + return strings.Join(labels, ":") +}