From 436e8a17326a21479785aa9759c945b948cc9cc3 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 14 Feb 2023 22:00:17 +1000 Subject: [PATCH] updated --- .../pipelines/http/emulator/rod/context.go | 2 + .../pipelines/http/emulator/rod/middleware.go | 1 + .../pipelines/http/emulator/rod/request.go | 1 + .../pipelines/http/emulator/rod/test_test.go | 82 ------------------- extensions/pipelines/http/net-http/body.go | 2 +- extensions/pipelines/http/net-http/context.go | 6 +- .../http/net-http/installation-proxy-old.go | 61 -------------- .../pipelines/http/net-http/middlewares.go | 4 +- extensions/pipelines/http/net-http/proxy.go | 8 +- extensions/pipelines/http/net-http/request.go | 9 +- .../pipelines/http/net-http/stream_test.go | 8 +- .../pipelines/http/net-http/test_test.go | 9 +- extensions/pipelines/http/net-http/utils.go | 3 +- extensions/pipelines/task-pipeline.go | 37 ++++++--- .../service/controllers/check_redis.go | 16 ++-- extensions/servers/gin/server.go | 6 +- integrations/mongo/database.go | 17 ++-- integrations/redis/redis_test.go | 13 +-- interfaces/logger.go | 30 ++++--- profiler.go | 3 +- 20 files changed, 100 insertions(+), 218 deletions(-) delete mode 100644 extensions/pipelines/http/emulator/rod/test_test.go delete mode 100644 extensions/pipelines/http/net-http/installation-proxy-old.go diff --git a/extensions/pipelines/http/emulator/rod/context.go b/extensions/pipelines/http/emulator/rod/context.go index 7bf89bc..10cbb9c 100644 --- a/extensions/pipelines/http/emulator/rod/context.go +++ b/extensions/pipelines/http/emulator/rod/context.go @@ -2,7 +2,9 @@ package rod import ( Context "context" + "github.com/go-rod/rod" + "github.com/vortex14/gotyphoon/ctx" ) diff --git a/extensions/pipelines/http/emulator/rod/middleware.go b/extensions/pipelines/http/emulator/rod/middleware.go index 4a4bebe..f6984f0 100644 --- a/extensions/pipelines/http/emulator/rod/middleware.go +++ b/extensions/pipelines/http/emulator/rod/middleware.go @@ -3,6 +3,7 @@ package rod import ( "context" "fmt" + "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" diff --git a/extensions/pipelines/http/emulator/rod/request.go b/extensions/pipelines/http/emulator/rod/request.go index c2af4c4..120b651 100644 --- a/extensions/pipelines/http/emulator/rod/request.go +++ b/extensions/pipelines/http/emulator/rod/request.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/go-rod/rod" + "github.com/vortex14/gotyphoon/elements/forms" "github.com/vortex14/gotyphoon/elements/models/task" Errors "github.com/vortex14/gotyphoon/errors" diff --git a/extensions/pipelines/http/emulator/rod/test_test.go b/extensions/pipelines/http/emulator/rod/test_test.go deleted file mode 100644 index 49cf9e9..0000000 --- a/extensions/pipelines/http/emulator/rod/test_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package rod - -import ( - "context" - "github.com/PuerkitoBio/goquery" - "github.com/go-rod/rod" - . "github.com/smartystreets/goconvey/convey" - "github.com/vortex14/gotyphoon/elements/forms" - "github.com/vortex14/gotyphoon/elements/models/label" - Task "github.com/vortex14/gotyphoon/elements/models/task" - "github.com/vortex14/gotyphoon/extensions/data/fake" - "github.com/vortex14/gotyphoon/interfaces" - "github.com/vortex14/gotyphoon/log" - "testing" - "time" -) - -func init() { - log.InitD() -} - -func TestHttpRodRequestPipeline_Run(t *testing.T) { - - Convey("Create a rod pipeline", t, func() { - - g1 := forms.PipelineGroup{ - MetaInfo: &label.MetaInfo{ - Name: "Rod group", - }, - Stages: []interfaces.BasePipelineInterface{ - CreateProxyRodRequestPipeline( - forms.GetCustomRetryOptions(1, time.Duration(1)*time.Second), - &DetailsOptions{SleepAfter: 1, MustElement: "#shitcoin > .message"}, - ), - &HttpRodResponsePipeline{ - BasePipeline: &forms.BasePipeline{ - NotIgnorePanic: true, - Options: forms.GetNotRetribleOptions(), - MetaInfo: &label.MetaInfo{ - Name: "http response from rod emulator", - }, - }, - Fn: func(context context.Context, task interfaces.TaskInterface, logger interfaces.LoggerInterface, - browser *rod.Browser, page *rod.Page, body *string, doc *goquery.Document) (error, context.Context) { - - html, err := doc.Find("#shitcoin").Html() - if err != nil { - return err, context - } - logger.Debug(html) - - //logger.Warning(doc.Html()) - - //defer page.MustClose() - //defer browser.MustClose() - - return nil, context - }, - Cn: func(err error, - context context.Context, - task interfaces.TaskInterface, - logger interfaces.LoggerInterface) { - - logger.Error("--- ", err.Error()) - }, - }, - }, - } - - newTask := fake.CreateDefaultTask() - - newTask.SetFetcherUrl("https://honeypot.is/ethereum?address=0x879c61c813147627fe3ddb824f681f65550f2139") - newTask.SetFetcherMethod("GET") - newTask.Fetcher.Timeout = 60 - newTask.SetProxyServerUrl("http://localhost:8987") - ctxGroup := Task.NewTaskCtx(newTask) - - err := g1.Run(ctxGroup) - - So(err, ShouldBeNil) - }) -} diff --git a/extensions/pipelines/http/net-http/body.go b/extensions/pipelines/http/net-http/body.go index 1913396..4d02a2c 100644 --- a/extensions/pipelines/http/net-http/body.go +++ b/extensions/pipelines/http/net-http/body.go @@ -26,7 +26,7 @@ func GetBody(client *http.Client, request *http.Request) (error, *string, *http. var reader io.ReadCloser switch response.Header.Get("Content-Encoding") { - case "gzip": + case GZIP: reader, err = gzip.NewReader(response.Body) if err != nil { return Errors.ResponseHttpGzipDecodeError, nil, nil diff --git a/extensions/pipelines/http/net-http/context.go b/extensions/pipelines/http/net-http/context.go index 423f1f7..0f39bac 100644 --- a/extensions/pipelines/http/net-http/context.go +++ b/extensions/pipelines/http/net-http/context.go @@ -2,11 +2,12 @@ package net_http import ( Context "context" - "github.com/PuerkitoBio/goquery" - "github.com/vortex14/gotyphoon/interfaces" "net/http" + "github.com/PuerkitoBio/goquery" + "github.com/vortex14/gotyphoon/ctx" + "github.com/vortex14/gotyphoon/interfaces" ) const ( @@ -17,6 +18,7 @@ const ( REQUEST = "request" RESPONSE = "response" DATA = "data" + GZIP = "gzip" ) type ValidationCallback func(logger interfaces.LoggerInterface, response *http.Response, doc *goquery.Document) bool diff --git a/extensions/pipelines/http/net-http/installation-proxy-old.go b/extensions/pipelines/http/net-http/installation-proxy-old.go deleted file mode 100644 index fe247df..0000000 --- a/extensions/pipelines/http/net-http/installation-proxy-old.go +++ /dev/null @@ -1,61 +0,0 @@ -package net_http - -import ( - "context" - "fmt" - "net/http" - - "github.com/fatih/color" - - "github.com/vortex14/gotyphoon/elements/forms" - "github.com/vortex14/gotyphoon/elements/models/label" - "github.com/vortex14/gotyphoon/elements/models/task" - Errors "github.com/vortex14/gotyphoon/errors" - "github.com/vortex14/gotyphoon/extensions/models" - "github.com/vortex14/gotyphoon/interfaces" - "github.com/vortex14/gotyphoon/utils" -) - -const ( - NAMEProxyOldMiddleware = "Installation proxy middleware" - DescriptionProxyOldMiddleware = "Setting proxy Middleware" -) - -func ConstructorProxySettingOldMiddleware(required bool) interfaces.MiddlewareInterface { - return &HttpMiddleware{ - Middleware: &forms.Middleware{ - MetaInfo: &label.MetaInfo{ - Required: required, - Name: NAMEProxyOldMiddleware, - Description: DescriptionProxyOldMiddleware, - }, - }, - Fn: func(context context.Context, task *task.TyphoonTask, request *http.Request, logger interfaces.LoggerInterface, reject func(err error), next func(ctx context.Context)) { - - - client := GetHttpClient(task) - - urlOldSupported := fmt.Sprintf("%s/url=%s", task.GetProxyServerUrl(), task.GetFetcherUrl()) - logger.Info(urlOldSupported) - - request, err := http.NewRequest(http.MethodGet, urlOldSupported, nil) - - if err != nil { reject(err); return } - - err, body, _ := GetBody(client, request) - if err != nil || body == nil {reject(Errors.ProxyServerError); return} - logger.Error(string(*body)) - var proxyResponse models.Proxy - err = utils.JsonLoad(&proxyResponse, *body) - if err != nil { - color.Red("JsonLoad has Error: %s", err.Error()) - reject(err) - return - } - proxyFormat := FormattingProxy(proxyResponse.Proxy) - task.SetUserAgent(proxyResponse.Agent) - task.SetProxyAddress(proxyFormat) - - }, - } -} diff --git a/extensions/pipelines/http/net-http/middlewares.go b/extensions/pipelines/http/net-http/middlewares.go index 95fd7fb..0b1b279 100644 --- a/extensions/pipelines/http/net-http/middlewares.go +++ b/extensions/pipelines/http/net-http/middlewares.go @@ -4,12 +4,14 @@ import ( "context" b64 "encoding/base64" "errors" + "net/url" + Gin "github.com/gin-gonic/gin" + "github.com/vortex14/gotyphoon/elements/forms" "github.com/vortex14/gotyphoon/elements/models/label" "github.com/vortex14/gotyphoon/extensions/servers/gin" "github.com/vortex14/gotyphoon/interfaces" - "net/url" ) var ( diff --git a/extensions/pipelines/http/net-http/proxy.go b/extensions/pipelines/http/net-http/proxy.go index 9a413f9..472648a 100644 --- a/extensions/pipelines/http/net-http/proxy.go +++ b/extensions/pipelines/http/net-http/proxy.go @@ -3,12 +3,14 @@ package net_http import ( "context" "fmt" - "github.com/sirupsen/logrus" - "github.com/vortex14/gotyphoon/extensions/middlewares" - "github.com/vortex14/gotyphoon/log" "net/http" "net/url" + "github.com/vortex14/gotyphoon/extensions/middlewares" + "github.com/vortex14/gotyphoon/log" + + "github.com/sirupsen/logrus" + "github.com/vortex14/gotyphoon/elements/forms" "github.com/vortex14/gotyphoon/elements/models/label" "github.com/vortex14/gotyphoon/elements/models/task" diff --git a/extensions/pipelines/http/net-http/request.go b/extensions/pipelines/http/net-http/request.go index 838f753..08af2fb 100644 --- a/extensions/pipelines/http/net-http/request.go +++ b/extensions/pipelines/http/net-http/request.go @@ -3,19 +3,20 @@ package net_http import ( "context" "fmt" - "github.com/avast/retry-go/v4" - "github.com/vortex14/gotyphoon/extensions/models" - "github.com/vortex14/gotyphoon/log" - "github.com/vortex14/gotyphoon/utils" "net/http" "os" + "github.com/avast/retry-go/v4" "github.com/fatih/color" + "github.com/vortex14/gotyphoon/elements/forms" "github.com/vortex14/gotyphoon/elements/models/label" "github.com/vortex14/gotyphoon/elements/models/task" Errors "github.com/vortex14/gotyphoon/errors" + "github.com/vortex14/gotyphoon/extensions/models" "github.com/vortex14/gotyphoon/interfaces" + "github.com/vortex14/gotyphoon/log" + "github.com/vortex14/gotyphoon/utils" ) func Request( diff --git a/extensions/pipelines/http/net-http/stream_test.go b/extensions/pipelines/http/net-http/stream_test.go index 2393e6c..14cbd26 100644 --- a/extensions/pipelines/http/net-http/stream_test.go +++ b/extensions/pipelines/http/net-http/stream_test.go @@ -3,9 +3,6 @@ package net_http import ( "bufio" "fmt" - . "github.com/smartystreets/goconvey/convey" - Fake "github.com/vortex14/gotyphoon/extensions/data/fake" - "github.com/vortex14/gotyphoon/log" "io" "io/ioutil" "os" @@ -13,6 +10,11 @@ import ( "sync" "testing" "time" + + . "github.com/smartystreets/goconvey/convey" + + Fake "github.com/vortex14/gotyphoon/extensions/data/fake" + "github.com/vortex14/gotyphoon/log" ) func init() { diff --git a/extensions/pipelines/http/net-http/test_test.go b/extensions/pipelines/http/net-http/test_test.go index 07e03e5..7083042 100644 --- a/extensions/pipelines/http/net-http/test_test.go +++ b/extensions/pipelines/http/net-http/test_test.go @@ -1,17 +1,18 @@ package net_http import ( + "context" + "net/http" + "testing" + . "github.com/smartystreets/goconvey/convey" + "github.com/vortex14/gotyphoon/elements/forms" "github.com/vortex14/gotyphoon/elements/models/label" "github.com/vortex14/gotyphoon/elements/models/task" "github.com/vortex14/gotyphoon/extensions/data/fake" "github.com/vortex14/gotyphoon/interfaces" "github.com/vortex14/gotyphoon/log" - - "context" - "net/http" - "testing" ) func init() { diff --git a/extensions/pipelines/http/net-http/utils.go b/extensions/pipelines/http/net-http/utils.go index fda12aa..845ae69 100644 --- a/extensions/pipelines/http/net-http/utils.go +++ b/extensions/pipelines/http/net-http/utils.go @@ -2,8 +2,9 @@ package net_http import ( "fmt" - "github.com/vortex14/gotyphoon/extensions/data/fake" "net/http" + + "github.com/vortex14/gotyphoon/extensions/data/fake" ) func FormattingProxy(proxy string) string { diff --git a/extensions/pipelines/task-pipeline.go b/extensions/pipelines/task-pipeline.go index 54ea390..6d9a527 100644 --- a/extensions/pipelines/task-pipeline.go +++ b/extensions/pipelines/task-pipeline.go @@ -17,7 +17,7 @@ type TaskPipeline struct { context context.Context, task interfaces.TaskInterface, logger interfaces.LoggerInterface, - ) (error, context.Context) + ) (error, context.Context) Cn func( err error, @@ -29,7 +29,7 @@ type TaskPipeline struct { func (t *TaskPipeline) UnpackCtx( ctx context.Context, - ) (bool, interfaces.TaskInterface, interfaces.LoggerInterface) { +) (bool, interfaces.TaskInterface, interfaces.LoggerInterface) { okT, taskInstance := task.Get(ctx) okL, logger := log.Get(ctx) @@ -40,15 +40,24 @@ func (t *TaskPipeline) Run( context context.Context, reject func(pipeline interfaces.BasePipelineInterface, err error), next func(ctx context.Context), - ) { +) { - if t.Fn == nil { reject(t, Errors.TaskPipelineRequiredHandler); return } + if t.Fn == nil { + reject(t, Errors.TaskPipelineRequiredHandler) + return + } - ok,taskInstance, logger := t.UnpackCtx(context) - if !ok { reject(t, Errors.PipelineContexFailed); return } + ok, taskInstance, logger := t.UnpackCtx(context) + if !ok { + reject(t, Errors.PipelineContexFailed) + return + } err, newContext := t.Fn(context, taskInstance, logger) - if err != nil { reject(t, err); return } + if err != nil { + reject(t, err) + return + } next(newContext) } @@ -58,11 +67,15 @@ func (t *TaskPipeline) Cancel( err error, ) { - if t.Cn == nil { return } + if t.Cn == nil { + return + } - ok,taskInstance, logger := t.UnpackCtx(context) - if !ok { return } + ok, taskInstance, _logger := t.UnpackCtx(context) + if !ok { + return + } - t.Cn(err, context, taskInstance, logger) + t.Cn(err, context, taskInstance, _logger) -} \ No newline at end of file +} diff --git a/extensions/servers/gin/resources/service/controllers/check_redis.go b/extensions/servers/gin/resources/service/controllers/check_redis.go index b148015..f4fa1c2 100644 --- a/extensions/servers/gin/resources/service/controllers/check_redis.go +++ b/extensions/servers/gin/resources/service/controllers/check_redis.go @@ -10,23 +10,18 @@ import ( ) const ( - CheckRedisPath = "check_redis" + CheckRedisPath = "check_redis" CheckRedisDescription = "Health check controller of redis" LocalhostRedisHost = "localhost" LocalhostRedisPort = 6379 - ) var redisService = &redis.Service{ Project: nil, - Config: &interfaces.ServiceRedis{ + Config: &interfaces.ServiceRedis{ Name: "Test", - Details: struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Password interface{} `yaml:"password"` - }{ + Details: interfaces.RedisDetails{ Host: LocalhostRedisHost, Port: LocalhostRedisPort, }, @@ -40,7 +35,7 @@ var redisService = &redis.Service{ // @Description Health check controller of redis // @Success 200 {object} ServiceResponse // @Router /api/v1/services/check_redis [get] -func RedisHandler (ctx *gin.Context, logger interfaces.LoggerInterface ) { +func RedisHandler(ctx *gin.Context, logger interfaces.LoggerInterface) { ctx.JSON(200, &ServiceResponse{ Status: redisService.Ping(), }) @@ -52,8 +47,7 @@ var RedisController = &GinExtension.Action{ Name: CheckRedisPath, Description: CheckRedisDescription, }, - Methods : []string{interfaces.GET}, + Methods: []string{interfaces.GET}, }, GinController: RedisHandler, } - diff --git a/extensions/servers/gin/server.go b/extensions/servers/gin/server.go index 2fb37e1..d52bf77 100644 --- a/extensions/servers/gin/server.go +++ b/extensions/servers/gin/server.go @@ -174,9 +174,9 @@ func (s *TyphoonGinServer) onInitAction(resource interfaces.ResourceInterface, a func (s *TyphoonGinServer) onInitResource(newResource interfaces.ResourceInterface) { - if _, ok := newResource.(interfaces.ResourceGraphInterface); ok { - //s.LOG.Info("onInitResource, hasGraph: ", graphResource.HasParentGraph()) - } + //if _, ok := newResource.(interfaces.ResourceGraphInterface); ok { + //s.LOG.Info("onInitResource, hasGraph: ", graphResource.HasParentGraph()) + //} } func (s *TyphoonGinServer) onBuildSubResources(subResource interfaces.ResourceInterface) { diff --git a/integrations/mongo/database.go b/integrations/mongo/database.go index da2e8d2..f5b7fd8 100644 --- a/integrations/mongo/database.go +++ b/integrations/mongo/database.go @@ -8,25 +8,24 @@ import ( type DataBase struct { *label.MetaInfo - db *mongo.Database - client *mongo.Client - Collections map[string] *Collection + db *mongo.Database + client *mongo.Client + Collections map[string]*Collection } -func (m *DataBase) GetMongoCollections() { +func (m *DataBase) GetMongoCollections() { } - -func (m *DataBase) GetMongoCollection(collectionName string) *mongo.Collection { - if m.db == nil { } +func (m *DataBase) GetMongoCollection(collectionName string) *mongo.Collection { + //if m.db == nil { } return m.db.Collection(collectionName) } -func (m *DataBase) Export() { +func (m *DataBase) Export() { } -func (m *DataBase) Import() { +func (m *DataBase) Import() { } diff --git a/integrations/redis/redis_test.go b/integrations/redis/redis_test.go index 6b1309f..23a5ac6 100644 --- a/integrations/redis/redis_test.go +++ b/integrations/redis/redis_test.go @@ -22,15 +22,10 @@ func init() { redisService = &Service{ Config: &interfaces.ServiceRedis{ Name: "Redis proxy data", - Details: struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Password interface{} `yaml:"password"` - }(struct { - Host string - Port int - Password interface{} - }{Host: "localhost", Port: 6379}), + Details: interfaces.RedisDetails{ + Host: "localhost", + Port: 6379, + }, }, } redisService.Init() diff --git a/interfaces/logger.go b/interfaces/logger.go index d92dd77..154a150 100644 --- a/interfaces/logger.go +++ b/interfaces/logger.go @@ -3,16 +3,26 @@ package interfaces import "github.com/sirupsen/logrus" const ( - DEBUG = "DEBUG" - WARNING = "WARNING" - INFO = "INFO" - ERROR = "ERROR" + DEBUG = "DEBUG" + WARNING = "WARNING" + INFO = "INFO" + ERROR = "ERROR" ) type LoggerInterface interface { - Debug (args ...interface{}) - Info (args ...interface{}) - Warning (args ...interface{}) - Error (args ...interface{}) - WithFields (fields logrus.Fields) *logrus.Entry -} \ No newline at end of file + Debug(args ...interface{}) + Info(args ...interface{}) + Warning(args ...interface{}) + Error(args ...interface{}) + + Debugf(format string, args ...interface{}) + Infof(format string, args ...interface{}) + Printf(format string, args ...interface{}) + Warnf(format string, args ...interface{}) + Warningf(format string, args ...interface{}) + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Panicf(format string, args ...interface{}) + + WithFields(fields logrus.Fields) *logrus.Entry +} diff --git a/profiler.go b/profiler.go index cf81a60..ca89d30 100644 --- a/profiler.go +++ b/profiler.go @@ -2,7 +2,6 @@ package typhoon import ( "fmt" - "math" "os" "runtime" "runtime/pprof" @@ -35,7 +34,7 @@ func fibo2(n int) int { func N1(n int) bool { - k := math.Floor(float64(n/2 + 1)) + k := float64(n/2 + 1) for i := 2; i < int(k); i++ { if (n % i) == 0 { return false