Merge pull request #74 from vortex14/proxy-service-new-features

updated

extensions/pipelines/http/emulator/rod/browser.go

@@ -1,23 +1,42 @@
 package rod
 
 import (
+	"context"
+	uR "github.com/EDDYCJY/fake-useragent"
+
+	"github.com/vortex14/gotyphoon/log"
+	"math/rand"
 	"time"
 
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/devices"
 )
 
+var (
+	D = []devices.Device{
+		devices.IPad,
+		devices.IPadPro,
+		devices.LaptopWithHiDPIScreen,
+		devices.LaptopWithMDPIScreen,
+		devices.Nexus10,
+		devices.KindleFireHDX,
+	}
+)
+
 func getDeviceOfBrowser(options Options) devices.Device {
 	var device devices.Device
 	if len(options.Device.UserAgent) == 0 {
 		device = devices.IPadPro
+	} else if options.Device.UserAgent == "random" {
+		rand.Seed(time.Now().UnixNano())
+		device = D[rand.Intn(len(D))]
 	} else {
 		device = options.Device
 	}
 	return device
 }
 
-func CreateBaseBrowser(options Options) *rod.Browser {
+func CreateBaseBrowser(context context.Context, options Options) (context.Context, *rod.Browser) {
 
 	browser := rod.New().ControlURL(CreateLauncher(options).MustLaunch())
 
@@ -29,5 +48,19 @@ func CreateBaseBrowser(options Options) *rod.Browser {
 		browser = browser.Timeout(options.Timeout)
 	}
 
-	return browser.DefaultDevice(getDeviceOfBrowser(options))
+	device := getDeviceOfBrowser(options)
+
+	logMap := map[string]interface{}{
+		"device":     device.Title,
+		"user-agent": device.UserAgent,
+	}
+
+	if options.RandomAgent {
+		device.UserAgent = uR.Random()
+		logMap["random_agent"] = true
+	}
+
+	context = log.PatchCtx(context, logMap)
+
+	return context, browser.DefaultDevice(device)
 }

extensions/pipelines/http/emulator/rod/options.go

@@ -24,10 +24,11 @@ func (e *EventOptions) Wait() {
 }
 
 type Options struct {
-	Debug   bool
-	Proxy   string
-	Timeout time.Duration
-	Device  devices.Device
+	Debug       bool
+	Proxy       string
+	Timeout     time.Duration
+	Device      devices.Device
+	RandomAgent bool
 }
 
 type DetailsOptions struct {

extensions/pipelines/http/emulator/rod/request.go

@@ -80,11 +80,16 @@ func CreateRodRequestPipeline(
 			detailOptions.Options.Proxy = task.GetProxyAddress()
 
 			if len(task.GetProxyAddress()) > 0 {
-				context = log.PatchCtx(context, map[string]interface{}{"proxy": task.GetProxyAddress()})
-				_, logger = log.Get(context)
+				context = log.PatchCtx(context, map[string]interface{}{
+					"proxy": task.GetProxyAddress(),
+				})
 			}
 
-			browser := CreateBaseBrowser(detailOptions.Options)
+			var browser *rod.Browser
+
+			context, browser = CreateBaseBrowser(context, detailOptions.Options)
+
+			_, logger = log.Get(context)
 
 			var pErr error
 

extensions/pipelines/http/emulator/rod/test_test.go

@@ -44,9 +44,9 @@ func TestCreateBrowser(t *testing.T) {
 	}
 	log.InitD()
 
-	browser := CreateBaseBrowser(opts).MustConnect()
+	_, browser := CreateBaseBrowser(context.Background(), opts)
 
-	_ = browser.MustPage("https://www.wikipedia.org/")
+	_ = browser.MustConnect().MustPage("https://www.wikipedia.org/")
 	time.Sleep(5 * time.Second)
 	_ = browser.Close()
 
@@ -62,6 +62,8 @@ func TestCreateRodPipeline(t *testing.T) {
 
 	Convey("create a new pipeline", t, func() {
 		_task := fake.CreateDefaultTask()
+		d := devices.Device{}
+		d.UserAgent = "random"
 		//_task.SetProxyAddress("http://localhost:8888")
 		//_task.SetProxyAddress("http://ukehiuwv-982:[email protected]")
 		_task.SetProxyServerUrl("http://proxy-manager.typhoon-s1.ru")
@@ -70,9 +72,10 @@ func TestCreateRodPipeline(t *testing.T) {
 			&DetailsOptions{
 				ProxyRequired: true,
 				Options: Options{
-					Debug:   true,
-					Device:  devices.IPadPro,
-					Timeout: 600 * time.Second,
+					RandomAgent: true,
+					Debug:       true,
+					Device:      d,
+					Timeout:     600 * time.Second,
 				},
 				SleepAfter: 10,
 			})