Skip to content

Commit

Permalink
updated
Browse files Browse the repository at this point in the history
  • Loading branch information
Dmitriy committed Feb 7, 2023
1 parent 1dc4af9 commit 3b67d13
Show file tree
Hide file tree
Showing 6 changed files with 120 additions and 34 deletions.
9 changes: 9 additions & 0 deletions elements/models/task/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,15 @@ func (t *TyphoonTask) AddHeader(key string, value string) {
t.Fetcher.Headers[key] = value
}

func (t *TyphoonTask) SetHeaders(headers map[string]string) {
for key, element := range headers {
t.AddHeader(
key,
element,
)
}
}

func (t *TyphoonTask) SetSaveData(key string, value string) {
t.Processor.Save.Project[key] = value
}
Expand Down
3 changes: 1 addition & 2 deletions extensions/pipelines/http/net-http/test_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@ import (
"github.com/vortex14/gotyphoon/extensions/data/fake"
"github.com/vortex14/gotyphoon/interfaces"
"github.com/vortex14/gotyphoon/log"
"net/http"

"context"

"net/http"
"testing"
)

Expand Down
40 changes: 38 additions & 2 deletions extensions/servers/gin/domains/proxy/models.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
package proxy

import "github.com/vortex14/gotyphoon/interfaces"
import (
"errors"
"net/url"

"github.com/vortex14/gotyphoon/interfaces"
)

type Proxy struct {
Address string
Region string
Provider string
}

type Settings struct {
BlockedTime int
Expand All @@ -22,5 +33,30 @@ type Stats struct {
Locked interface{}
List interface{}
Allowed interface{}
TestEndpoint string
TestEndpoint *UpdateCheckPayload
}

type UpdateCheckPayload struct {
Headers map[string]string `json:"headers"`
UrlSource string `json:"url"`
Url *url.URL
}

func (u *UpdateCheckPayload) ParseUrl() (error, *url.URL) {

var exception error

if len(u.UrlSource) == 0 && u.Url != nil {
u.UrlSource = u.Url.String()
} else if len(u.UrlSource) > 0 {
urlDecoded, e := url.Parse(u.UrlSource)
exception = e
if e == nil {
u.Url = urlDecoded
}
} else {
exception = errors.New("not found source url or *url.URL")
}

return exception, u.Url
}
57 changes: 36 additions & 21 deletions extensions/servers/gin/domains/proxy/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/vortex14/gotyphoon/extensions/data/fake"
net_http "github.com/vortex14/gotyphoon/extensions/pipelines/text/html"
netHttp "github.com/vortex14/gotyphoon/extensions/pipelines/text/html"
"math/rand"
"net/http"
"net/url"
Expand All @@ -27,7 +27,7 @@ func init() {

type Collection struct {
singleton.Singleton
mu sync.Mutex
mu sync.RWMutex
LOG interfaces.LoggerInterface

Settings *Settings
Expand All @@ -41,8 +41,8 @@ type Collection struct {
allowed map[string][]string
banned map[string][]string

availableEndpoints map[string]string // domain key : available endpoint
observableHosts map[string]bool
availableEndpoints map[string]*UpdateCheckPayload // domain key : available endpoint
}

func (c *Collection) GetFullKeyPath(host string, key string) string {
Expand All @@ -57,7 +57,7 @@ func (c *Collection) Clear() error {
c.mu.Lock()
defer c.mu.Unlock()
var err error
for domain, _ := range c.stats {
for domain := range c.stats {
err = c.RemoveBanHistory(domain)
c.LOG.Debug("remove history for ", domain)
err = c.RemoveHistory(domain)
Expand Down Expand Up @@ -113,7 +113,7 @@ func (c *Collection) RemoveHistory(host string) error {

func (c *Collection) RemoveBanHistory(host string) error {
for _, value := range c.list {
err := c.redisService.Remove(c.GetFullBanPathByKey(host, value))
err := c.RemoveProxyBan(host, value)
if err != nil {
return err
}
Expand Down Expand Up @@ -162,17 +162,26 @@ func (c *Collection) unblockingProxy(availableURL *url.URL) {
}
}

func (c *Collection) checkingBlocked(availableURL *url.URL) {
lastIndex := 0
var availableUrl string
host := availableURL.Hostname()
func (c *Collection) getAvailablePayload(availableURL *url.URL) *UpdateCheckPayload {

host := availableURL.Host
if _, ok := c.availableEndpoints[host]; !ok {
availableUrl = availableURL.String()
c.availableEndpoints[host] = availableUrl
} else {
availableUrl = c.availableEndpoints[host]
c.availableEndpoints[host] = &UpdateCheckPayload{Url: availableURL}
_, _ = c.availableEndpoints[host].ParseUrl()
}
return c.availableEndpoints[host]
}

func (c *Collection) setAvailablePayload(payload *UpdateCheckPayload) {
host := payload.Url.Hostname()
_, _ = payload.ParseUrl()
c.availableEndpoints[host] = payload
}

func (c *Collection) checkingBlocked(availableURL *url.URL) {
lastIndex := 0

host := availableURL.Hostname()

for {
c.LOG.Debug(fmt.Sprintf("checking blocked for host: %s ... every %ds Count: %d", host, c.Settings.CheckBlockedTime, len(c.banned[host])))
Expand Down Expand Up @@ -206,20 +215,24 @@ func (c *Collection) checkingBlocked(availableURL *url.URL) {
go func(wg *sync.WaitGroup, proxy string) {
task := fake.CreateDefaultTask()

task.SetFetcherUrl(availableUrl)
payload := c.getAvailablePayload(availableURL)

task.SetFetcherUrl(payload.UrlSource)
task.SetProxyAddress(proxy)
task.SetHeaders(payload.Headers)

c.LOG.Debug("create a new request to ", availableUrl, " through "+proxy)
c.LOG.Debug("create a new request to ", payload.UrlSource, " through "+proxy+
fmt.Sprintf(" Headers keys: %d", len(payload.Headers)))

err := net_http.MakeRequestThroughProxy(task, func(logger interfaces.LoggerInterface,
err := netHttp.MakeRequestThroughProxy(task, func(logger interfaces.LoggerInterface,
response *http.Response, doc *goquery.Document) bool {

return !(response.StatusCode > 400)
})
if err == nil {
ansE := c.RemoveProxyBan(host, proxy)
if ansE == nil {
c.LOG.Debug(fmt.Sprintf("proxy %s %s throught %s", proxy, host, availableUrl))
c.LOG.Debug(fmt.Sprintf("proxy %s %s throught %s is available", proxy, host, payload.UrlSource))
} else {
c.LOG.Error(ansE)
}
Expand Down Expand Up @@ -250,6 +263,8 @@ func (c *Collection) Block(proxy string, u *url.URL) error {
if _, ok := c.observableHosts[host]; !ok {
c.observableHosts[host] = true
go c.unblockingProxy(u)
payload := c.getAvailablePayload(u)
c.setAvailablePayload(payload)
go c.checkingBlocked(u)
}

Expand Down Expand Up @@ -341,12 +356,12 @@ func (c *Collection) Init() *Collection {
c.Construct(func() {
c.LOG = log.New(log.D{"proxy": "rotator"})

c.availableEndpoints = make(map[string]string)
c.observableHosts = make(map[string]bool)
c.stats = make(map[string]map[string]int)
c.allowed = make(map[string][]string)
c.locked = make(map[string][]string)
c.banned = make(map[string][]string)
c.allowed = make(map[string][]string)
c.stats = make(map[string]map[string]int)
c.observableHosts = make(map[string]bool)
c.availableEndpoints = make(map[string]*UpdateCheckPayload)

proxyEnvList := os.Getenv("PROXY_LIST")
if len(proxyEnvList) == 0 {
Expand Down
44 changes: 35 additions & 9 deletions extensions/servers/gin/domains/proxy/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,12 @@ package proxy
import (
Errors "errors"
"fmt"

u_ "github.com/ahl5esoft/golang-underscore"
Gin "github.com/gin-gonic/gin"

"github.com/vortex14/gotyphoon/elements/forms"
"github.com/vortex14/gotyphoon/elements/models/label"
"github.com/vortex14/gotyphoon/extensions/data/fake"
net_http "github.com/vortex14/gotyphoon/extensions/pipelines/http/net-http"
netHttp "github.com/vortex14/gotyphoon/extensions/pipelines/http/net-http"
"github.com/vortex14/gotyphoon/extensions/servers/gin"
"github.com/vortex14/gotyphoon/extensions/servers/gin/controllers/graph"
"github.com/vortex14/gotyphoon/extensions/servers/gin/controllers/ping"
Expand Down Expand Up @@ -53,7 +51,7 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
},
Methods: []string{interfaces.GET},
Middlewares: []interfaces.MiddlewareInterface{
net_http.UrlRequiredMiddleware,
netHttp.UrlRequiredMiddleware,
},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
Expand Down Expand Up @@ -97,7 +95,7 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
},
Methods: []string{interfaces.GET},
Middlewares: []interfaces.MiddlewareInterface{
net_http.UrlRequiredMiddleware,
netHttp.UrlRequiredMiddleware,
},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
Expand Down Expand Up @@ -164,7 +162,7 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
},
Methods: []string{interfaces.GET},
Middlewares: []interfaces.MiddlewareInterface{
net_http.UrlRequiredMiddleware,
netHttp.UrlRequiredMiddleware,
},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
Expand All @@ -184,7 +182,7 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
},
Methods: []string{interfaces.GET},
Middlewares: []interfaces.MiddlewareInterface{
net_http.UrlRequiredMiddleware,
netHttp.UrlRequiredMiddleware,
},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
Expand Down Expand Up @@ -240,22 +238,24 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
},
Methods: []string{interfaces.GET},
Middlewares: []interfaces.MiddlewareInterface{
net_http.UrlRequiredMiddleware,
netHttp.UrlRequiredMiddleware,
},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
logger.Debug("Check stats.")

u := GetUrlParamGin(ctx)

payload := proxyCollection.getAvailablePayload(u)

ctx.JSON(200, &Stats{
Stats: proxyCollection.stats,
Active: append(proxyCollection.locked[u.Hostname()], proxyCollection.allowed[u.Hostname()]...),
Blocked: proxyCollection.banned,
Allowed: proxyCollection.allowed[u.Hostname()],
Locked: proxyCollection.locked,
List: proxyCollection.list,
TestEndpoint: proxyCollection.availableEndpoints[u.Hostname()],
TestEndpoint: payload,
ObservableHosts: proxyCollection.observableHosts,
})
},
Expand All @@ -275,6 +275,32 @@ func Constructor(opts *Settings) interfaces.ServerInterface {
ctx.JSON(200, e)
},
},
"check": &gin.Action{
Action: &forms.Action{
MetaInfo: &label.MetaInfo{
Name: "check",
Path: "check",
Description: "check domain by url",
},
Methods: []string{interfaces.POST},
},
GinController: func(ctx *Gin.Context, logger interfaces.LoggerInterface) {
logger.Debug("change url and headers for check available host through proxy")
payload := &UpdateCheckPayload{}

e := ctx.BindJSON(payload)
if e != nil {
ctx.JSON(500, e)
return
}
_, u := payload.ParseUrl()

proxyCollection.setAvailablePayload(payload)

ctx.JSON(200, proxyCollection.getAvailablePayload(u))

},
},
},
},
)
Expand Down
1 change: 1 addition & 0 deletions interfaces/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type TaskFetcherInterface interface {
SetFetcherMethod(method string)

AddHeader(key string, value string)
SetHeaders(headers map[string]string)

GetFetcherTimeout() int
SetFetcherTimeout(seconds int)
Expand Down

0 comments on commit 3b67d13

Please sign in to comment.