Skip to content

Commit

Permalink
feat(logger): add logger and pkg directory
Browse files Browse the repository at this point in the history
  • Loading branch information
cobolbaby committed Dec 3, 2021
1 parent 281bace commit 0b8fd1a
Show file tree
Hide file tree
Showing 9 changed files with 174 additions and 32 deletions.
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
config.go
*.py
dist
logs
vendor
.fuse*
__debug_bin
2 changes: 1 addition & 1 deletion example/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"strings"
"time"

"github.com/cobolbaby/lineage/depgraph"
"github.com/cobolbaby/lineage/pkg/depgraph"
"github.com/mitchellh/copystructure"
pg_query "github.com/pganalyze/pg_query_go/v2"
)
Expand Down
16 changes: 11 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,21 @@ module github.com/cobolbaby/lineage

go 1.17

require (
github.com/lib/pq v1.10.4
github.com/mitchellh/copystructure v1.2.0
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/neo4j/neo4j-go-driver/v4 v4.4.0
github.com/pganalyze/pg_query_go/v2 v2.1.0
github.com/sirupsen/logrus v1.8.1
github.com/tidwall/gjson v1.11.0
)

require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/lib/pq v1.10.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/neo4j/neo4j-go-driver/v4 v4.4.0 // indirect
github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
github.com/tidwall/gjson v1.11.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
google.golang.org/protobuf v1.26.0 // indirect
)
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
github.com/neo4j/neo4j-go-driver/v4 v4.4.0 h1:p52GnTFjT/prEHdinfDFCEaQwcIhQ7qpR8axJz8yzuE=
github.com/neo4j/neo4j-go-driver/v4 v4.4.0/go.mod h1:7mOj4ipioP0S07AyC6EPstJysKnVqeJJcK2uJkf8gQg=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
Expand All @@ -39,7 +41,10 @@ github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
github.com/pganalyze/pg_query_go/v2 v2.1.0 h1:donwPZ4G/X+kMs7j5eYtKjdziqyOLVp3pkUrzb9lDl8=
github.com/pganalyze/pg_query_go/v2 v2.1.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4=
github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
Expand Down Expand Up @@ -67,12 +72,14 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
9 changes: 5 additions & 4 deletions graph.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package main

import (
"log"
"strings"

"github.com/cobolbaby/lineage/depgraph"
"github.com/cobolbaby/lineage/pkg/depgraph"
"github.com/cobolbaby/lineage/pkg/log"
"github.com/neo4j/neo4j-go-driver/v4/neo4j"
)

Expand Down Expand Up @@ -33,9 +33,9 @@ func CreateGraph(driver neo4j.Driver, graph *depgraph.Graph, extends *Op) error
session := driver.NewSession(neo4j.SessionConfig{})
defer session.Close()

log.Printf("sqlTree.ShrinkGraph: %+v", graph)
log.Debugf("ShrinkGraph: %+v", graph)
for i, layer := range graph.TopoSortedLayers() {
log.Printf("ShrinkGraph %d: %s\n", i, strings.Join(layer, ", "))
log.Infof("ShrinkGraph %d: %s\n", i, strings.Join(layer, ", "))
}

_, err := session.WriteTransaction(func(tx neo4j.Transaction) (interface{}, error) {
Expand All @@ -47,6 +47,7 @@ func CreateGraph(driver neo4j.Driver, graph *depgraph.Graph, extends *Op) error
return nil, err
}
}
// ? 创建 UDF 点?
// 创建线
for k, v := range graph.GetRelationships() {
for kk := range v {
Expand Down
46 changes: 26 additions & 20 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,16 @@ package main
import (
"database/sql"
"fmt"
"log"
"regexp"
"strings"
"time"

"github.com/cobolbaby/lineage/depgraph"

"github.com/cobolbaby/lineage/pkg/depgraph"
"github.com/cobolbaby/lineage/pkg/log"
_ "github.com/lib/pq"
"github.com/neo4j/neo4j-go-driver/v4/neo4j"
pg_query "github.com/pganalyze/pg_query_go/v2"
"github.com/tidwall/gjson"

_ "github.com/lib/pq"
)

var (
Expand Down Expand Up @@ -109,6 +107,15 @@ func filterUnhandledCommands(content string) string {
return PLPGSQL_UNHANLED_COMMANDS.ReplaceAllString(content, "")
}

func init() {
if err := log.InitLogger(&log.LoggerConfig{
Level: "debug",
Path: "./logs/lineage.log",
}); err != nil {
fmt.Println(err)
}
}

func main() {

// TODO:支持控制台输入
Expand All @@ -131,26 +138,26 @@ func main() {
DB_HOST, DB_PORT, DB_USER, DB_PASSWORD, DB_NAME)
db, err := sql.Open("postgres", psqlInfo)
if err != nil {
log.Fatalln("sql.Open err: ", err)
log.Fatal("sql.Open err: ", err)
}
defer db.Close()

driver, err := neo4j.NewDriver(NEO4J_URL, neo4j.BasicAuth(NEO4J_USER, NEO4J_PASSWORD, ""))
if err != nil {
log.Fatalf("neo4j.NewDriver err: %s", err)
log.Fatal("neo4j.NewDriver err: ", err)
}
// Handle driver lifetime based on your application lifetime requirements driver's lifetime is usually
// bound by the application lifetime, which usually implies one driver instance per application
defer driver.Close()

// 一上来先重置
if err := ResetGraph(driver); err != nil {
log.Fatalln("ResetGraph err: ", err)
log.Fatal("ResetGraph err: ", err)
}

rows, err := db.Query(fmt.Sprintf(PLPGSQL_GET_FUNC_DEFINITION, udf))
if err != nil {
log.Fatalln("db.Query err: ", err)
log.Fatal("db.Query err: ", err)
}
defer rows.Close()

Expand All @@ -162,11 +169,11 @@ func main() {
err := rows.Scan(&nspname, &proname, &definition)
switch err {
case sql.ErrNoRows:
log.Println("No rows were returned")
log.Warn("No rows were returned")
case nil:
log.Printf("Query Data = (%s, %s)\n", nspname, proname)
log.Infof("Query Data = (%s, %s)\n", nspname, proname)
default:
log.Fatalln("rows.Scan err: ", err)
log.Fatal("rows.Scan err: ", err)
}
}

Expand All @@ -176,12 +183,12 @@ func main() {

tree, err := pg_query.ParsePlPgSqlToJSON(plpgsql)
if err != nil {
log.Fatalln("pg_query.ParsePlPgSqlToJSON err: ", err)
log.Fatal("pg_query.ParsePlPgSqlToJSON err: ", err)
}

for _, v := range gjson.Parse(tree).Array() {
// TODO:根据 url 获取 db 别名
sqlTree := depgraph.New("IPT_PG_BDC")
// 重新生成
sqlTree := depgraph.New(DB_ALIAS)

for _, action := range v.Get("PLpgSQL_function.action.PLpgSQL_stmt_block.body").Array() {
// 遍历属性
Expand All @@ -193,24 +200,23 @@ func main() {

// 递归调用 Parse
if err := SQLParser(sqlTree, key.String(), value.String()); err != nil {
log.Printf("pg_query.ParseToJSON err: %s, sql: %s", err, value.String())
log.Errorf("pg_query.ParseToJSON err: %s, sql: %s", err, value.String())
return false
}

return true
})

}

// TODO:完善点的信息

log.Debugf("Graph: %+v", sqlTree)
for i, layer := range sqlTree.TopoSortedLayers() {
log.Printf("Graph %d: %s\n", i, strings.Join(layer, ", "))
log.Infof("Graph %d: %s\n", i, strings.Join(layer, ", "))
}

// log.Printf("%s Parser: %#v\n", key, *sqlTree)
if err := CreateGraph(driver, sqlTree.ShrinkGraph(), op); err != nil {
log.Printf("CreateGraph err: %s", err)
log.Error("CreateGraph err: ", err)
}
}

Expand Down
File renamed without changes.
118 changes: 118 additions & 0 deletions pkg/log/log.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package log

import (
"errors"
"io"
"os"

"github.com/natefinch/lumberjack"
"github.com/sirupsen/logrus"
)

type Logger interface {
Fatalf(string, ...interface{})
Errorf(string, ...interface{})
Warningf(string, ...interface{})
Warnf(string, ...interface{})
Infof(string, ...interface{})
Debugf(string, ...interface{})
Fatal(...interface{})
Error(...interface{})
Warn(...interface{})
Info(...interface{})
Debug(...interface{})
}

type LoggerConfig struct {
Path string `mapstructure:"path"`
Level string `mapstructure:"level"`
}

var logger Logger

func GetLogger() Logger {
return logger
}

func Fatalf(format string, args ...interface{}) {
logger.Fatalf(format, args...)
}

func Errorf(format string, args ...interface{}) {
logger.Errorf(format, args...)
}

func Warningf(format string, args ...interface{}) {
logger.Warningf(format, args...)
}

func Warnf(format string, args ...interface{}) {
logger.Warnf(format, args...)
}

func Infof(format string, args ...interface{}) {
logger.Infof(format, args...)
}

func Debugf(format string, args ...interface{}) {
logger.Debugf(format, args...)
}

func Fatal(args ...interface{}) {
logger.Fatal(args...)
}

func Error(args ...interface{}) {
logger.Error(args...)
}

func Warn(args ...interface{}) {
logger.Warn(args...)
}

func Info(args ...interface{}) {
logger.Info(args...)
}

func Debug(args ...interface{}) {
logger.Debug(args...)
}

func NewLogger(cfg *LoggerConfig) (Logger, error) {

logrusLogger := logrus.New()

// 设置日志格式为json格式
logrusLogger.SetFormatter(&logrus.TextFormatter{})

// 设置将日志输出到标准输出(默认的输出为stderr,标准错误)
if cfg.Path == "" {
return nil, errors.New("no log path")
}
lumberjackLogrotate := &lumberjack.Logger{
Filename: cfg.Path,
MaxSize: 10, // Max megabytes before log is rotated
MaxBackups: 30, // Max number of old log files to keep
MaxAge: 30, // Max number of days to retain log files
Compress: true,
}
logrusLogger.SetOutput(io.MultiWriter(os.Stdout, lumberjackLogrotate))

// 设置日志级别
if cfg.Level == "" {
cfg.Level = "info"
}
logLevel, err := logrus.ParseLevel(cfg.Level)
if err != nil {
logLevel = logrus.InfoLevel
}
logrusLogger.SetLevel(logLevel)

return logrusLogger, nil
}

func InitLogger(cfg *LoggerConfig) error {
var err error
logger, err = NewLogger(cfg)
return err
}
2 changes: 1 addition & 1 deletion sqlparser.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (

"errors"

"github.com/cobolbaby/lineage/depgraph"
"github.com/cobolbaby/lineage/pkg/depgraph"
pg_query "github.com/pganalyze/pg_query_go/v2"
"github.com/tidwall/gjson"
)
Expand Down

0 comments on commit 0b8fd1a

Please sign in to comment.