diff --git a/extensions/pipelines/http/emulator/rod/browser.go b/extensions/pipelines/http/emulator/rod/browser.go index c829a9d..6ac3733 100644 --- a/extensions/pipelines/http/emulator/rod/browser.go +++ b/extensions/pipelines/http/emulator/rod/browser.go @@ -1,23 +1,42 @@ package rod import ( + "context" + uR "github.com/EDDYCJY/fake-useragent" + + "github.com/vortex14/gotyphoon/log" + "math/rand" "time" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/devices" ) +var ( + D = []devices.Device{ + devices.IPad, + devices.IPadPro, + devices.LaptopWithHiDPIScreen, + devices.LaptopWithMDPIScreen, + devices.Nexus10, + devices.KindleFireHDX, + } +) + func getDeviceOfBrowser(options Options) devices.Device { var device devices.Device if len(options.Device.UserAgent) == 0 { device = devices.IPadPro + } else if options.Device.UserAgent == "random" { + rand.Seed(time.Now().UnixNano()) + device = D[rand.Intn(len(D))] } else { device = options.Device } return device } -func CreateBaseBrowser(options Options) *rod.Browser { +func CreateBaseBrowser(context context.Context, options Options) (context.Context, *rod.Browser) { browser := rod.New().ControlURL(CreateLauncher(options).MustLaunch()) @@ -29,5 +48,19 @@ func CreateBaseBrowser(options Options) *rod.Browser { browser = browser.Timeout(options.Timeout) } - return browser.DefaultDevice(getDeviceOfBrowser(options)) + device := getDeviceOfBrowser(options) + + logMap := map[string]interface{}{ + "device": device.Title, + "user-agent": device.UserAgent, + } + + if options.RandomAgent { + device.UserAgent = uR.Random() + logMap["random_agent"] = true + } + + context = log.PatchCtx(context, logMap) + + return context, browser.DefaultDevice(device) } diff --git a/extensions/pipelines/http/emulator/rod/options.go b/extensions/pipelines/http/emulator/rod/options.go index d50e637..c84b1c9 100644 --- a/extensions/pipelines/http/emulator/rod/options.go +++ b/extensions/pipelines/http/emulator/rod/options.go @@ -24,10 +24,11 @@ func (e *EventOptions) Wait() { } type Options struct { - Debug bool - Proxy string - Timeout time.Duration - Device devices.Device + Debug bool + Proxy string + Timeout time.Duration + Device devices.Device + RandomAgent bool } type DetailsOptions struct { diff --git a/extensions/pipelines/http/emulator/rod/request.go b/extensions/pipelines/http/emulator/rod/request.go index 0f8431d..c6c92f3 100644 --- a/extensions/pipelines/http/emulator/rod/request.go +++ b/extensions/pipelines/http/emulator/rod/request.go @@ -80,11 +80,16 @@ func CreateRodRequestPipeline( detailOptions.Options.Proxy = task.GetProxyAddress() if len(task.GetProxyAddress()) > 0 { - context = log.PatchCtx(context, map[string]interface{}{"proxy": task.GetProxyAddress()}) - _, logger = log.Get(context) + context = log.PatchCtx(context, map[string]interface{}{ + "proxy": task.GetProxyAddress(), + }) } - browser := CreateBaseBrowser(detailOptions.Options) + var browser *rod.Browser + + context, browser = CreateBaseBrowser(context, detailOptions.Options) + + _, logger = log.Get(context) var pErr error diff --git a/extensions/pipelines/http/emulator/rod/test_test.go b/extensions/pipelines/http/emulator/rod/test_test.go index 8cbdd7a..de4af0d 100644 --- a/extensions/pipelines/http/emulator/rod/test_test.go +++ b/extensions/pipelines/http/emulator/rod/test_test.go @@ -44,9 +44,9 @@ func TestCreateBrowser(t *testing.T) { } log.InitD() - browser := CreateBaseBrowser(opts).MustConnect() + _, browser := CreateBaseBrowser(context.Background(), opts) - _ = browser.MustPage("https://www.wikipedia.org/") + _ = browser.MustConnect().MustPage("https://www.wikipedia.org/") time.Sleep(5 * time.Second) _ = browser.Close() @@ -62,6 +62,8 @@ func TestCreateRodPipeline(t *testing.T) { Convey("create a new pipeline", t, func() { _task := fake.CreateDefaultTask() + d := devices.Device{} + d.UserAgent = "random" //_task.SetProxyAddress("http://localhost:8888") //_task.SetProxyAddress("http://ukehiuwv-982:8htmpmjvdzve@p.webshare.io") _task.SetProxyServerUrl("http://proxy-manager.typhoon-s1.ru") @@ -70,9 +72,10 @@ func TestCreateRodPipeline(t *testing.T) { &DetailsOptions{ ProxyRequired: true, Options: Options{ - Debug: true, - Device: devices.IPadPro, - Timeout: 600 * time.Second, + RandomAgent: true, + Debug: true, + Device: d, + Timeout: 600 * time.Second, }, SleepAfter: 10, })