wip

agent/main/lib/Agent.ts

@@ -52,6 +52,10 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
   private readonly closeBrowserOnClose: boolean = false;
   private isolatedMitm: MitmProxy;
 
+  // We use secretKey all through Agent components to make sure websites can't test if hero is present.
+  // Without this secretKey if would be pretty easy to detect hero.
+  private secretKey = Math.random().toString();
+
   private get proxyConnectionInfo(): IProxyConnectionOptions {
     if (!this.enableMitm) {
       if (this.emulationProfile.upstreamProxyUrl) {
@@ -91,6 +95,7 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
       this.logger,
       this.plugins.profile.upstreamProxyUrl,
       this.plugins.profile.upstreamProxyUseLocalDns,
+      this.secretKey,
     );
     this.enableMitm = !env.disableMitm && !this.plugins.profile.options.disableMitm;
 
@@ -210,6 +215,7 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
       hooks: this.plugins,
       isIncognito: this.isIncognito,
       commandMarker: this.options.commandMarker,
+      secretKey: this.secretKey,
     });
     this.events.once(this.browserContext, 'close', () => this.close());
 

agent/main/lib/Browser.ts

@@ -368,7 +368,7 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
     if (options.proxyPort !== undefined && !launchArgs.some(x => x.startsWith('--proxy-server'))) {
       launchArgs.push(
         // Use proxy for localhost URLs
-        '--proxy-bypass-list=<-loopback>;agent.localhost',
+        '--proxy-bypass-list=<-loopback>',
         `--proxy-server=localhost:${options.proxyPort}`,
       );
     }

agent/main/lib/BrowserContext.ts

@@ -40,6 +40,7 @@ export interface IBrowserContextCreateOptions {
   hooks?: IBrowserContextHooks & IInteractHooks;
   isIncognito?: boolean;
   commandMarker?: ICommandMarker;
+  secretKey?: string,
 }
 
 export default class BrowserContext
@@ -75,6 +76,7 @@ export default class BrowserContext
     frameId: 0,
   };
 
+  public secretKey?: string
   public commandMarker: ICommandMarker;
 
   private attachedTargetIds = new Set<string>();
@@ -94,6 +96,7 @@ export default class BrowserContext
     this.isIncognito = isIncognito;
     this.logger = options?.logger ?? log;
     this.hooks = options?.hooks ?? {};
+    this.secretKey = options?.secretKey;
     this.commandMarker = options?.commandMarker ?? new DefaultCommandMarker(this);
     this.resources = new Resources(this);
     this.websocketMessages = new WebsocketMessages(this.logger);
@@ -111,7 +114,7 @@ export default class BrowserContext
       disposeOnDetach: true,
     };
     if (this.proxy?.address) {
-      createContextOptions.proxyBypassList = '<-loopback>;agent.localhost';
+      createContextOptions.proxyBypassList = '<-loopback>';
       createContextOptions.proxyServer = this.proxy.address;
     }
 

agent/main/lib/Console.ts

@@ -10,15 +10,10 @@ import Protocol from 'devtools-protocol';
 import { IConsoleEvents } from '@ulixee/unblocked-specification/agent/browser/IConsole';
 
 const SCRIPT_PLACEHOLDER = '';
-const { log } = Log(module);
 
 export class Console extends TypedEventEmitter<IConsoleEvents> {
   isReady: Resolvable<void>;
 
-  private readonly host = 'agent.localhost';
-  private port = 9999;
-
-  private readonly secret = Math.random().toString();
   private readonly events = new EventSubscriber();
   // We store resolvable when we received websocket message before, receiving
   // targetId, this way we can await this, and still trigger to get proper ids.
@@ -27,7 +22,7 @@ export class Console extends TypedEventEmitter<IConsoleEvents> {
   private server: Server;
   private intervals = new Set<NodeJS.Timeout>();
 
-  constructor(public devtoolsSession: DevtoolsSession) {
+  constructor(public devtoolsSession: DevtoolsSession, public secretKey?: string) {
     super();
   }
 
@@ -47,21 +42,11 @@ export class Console extends TypedEventEmitter<IConsoleEvents> {
   }
 
   isConsoleRegisterUrl(url: string): boolean {
-    try {
-      const parsed = new URL(url);
-      return (
-        parsed.hostname === this.host
-        // parsed.port === this.port.toString() &&
-        // parsed.searchParams.get('secret') === this.secret
-      );
-    } catch {
-      return false;
-    }
+    return url.includes(`/heroInternalUrl?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=`)
   }
 
   registerFrameId(url: string, frameId: string): void {
     const parsed = new URL(url);
-    if (parsed.searchParams.get('secret') !== this.secret) return;
     const clientId = parsed.searchParams.get('clientId');
     if (!clientId) return;
 
@@ -78,11 +63,7 @@ export class Console extends TypedEventEmitter<IConsoleEvents> {
     const scriptFn = injectedScript
       .toString()
       // eslint-disable-next-line no-template-curly-in-string
-      .replaceAll('${this.host}', this.host)
-      // eslint-disable-next-line no-template-curly-in-string
-      .replaceAll('${this.port}', this.port.toString())
-      // eslint-disable-next-line no-template-curly-in-string
-      .replaceAll('${this.secret}', this.secret)
+      .replaceAll('${this.secretKey}', this.secretKey)
       // Use function otherwise replace will try todo some magic
       .replace('SCRIPT_PLACEHOLDER', () => script);
 
@@ -100,7 +81,7 @@ export class Console extends TypedEventEmitter<IConsoleEvents> {
     try {
       // Doing this is much much cheaper than json parse on everything logged in console debug
       const [secret, maybeClientId, serializedData] = msgAdded.message.text.split('-_-');
-      if (secret !== this.secret) return;
+      if (secret !== this.secretKey) return;
 
       const data = JSON.parse(serializedData);
       name = data.name;
@@ -129,16 +110,20 @@ export class Console extends TypedEventEmitter<IConsoleEvents> {
  * */
 function injectedScript(): void {
   const clientId = Math.random();
-  const url = `${this.host}:${this.port}?secret=${this.secret}&clientId=${clientId}`;
+
+  // By using document.url.origin we avoid all content security problems
+  const url = `${new URL(document.URL).origin}/heroInternalUrl?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=${clientId}`
+
+  // const url = `${this.host}:${this.port}?secret=${this.secret}&clientId=${clientId}`;
   // This will signal to network manager we are trying to make websocket connection
   // This is needed later to map clientId to frameId
-  void fetch(`http://${url}`, { mode: 'no-cors' }).catch(() => undefined);
+  void fetch(url, { mode: 'no-cors' }).catch(() => undefined);
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const callback = (name, payload): void => {
     const serializedData = JSON.stringify({ name, payload });
     // eslint-disable-next-line no-console
-    console.debug(`${this.secret}-_-${clientId}-_-${serializedData}`);
+    console.debug(`${this.secretKey}-_-${clientId}-_-${serializedData}`);
   };
 
   // eslint-disable-next-line @typescript-eslint/no-unused-expressions

agent/main/lib/FrameOutOfProcess.ts

@@ -25,6 +25,7 @@ export default class FrameOutOfProcess {
       this.devtoolsSession,
       frame.logger,
       page.browserContext.proxy,
+      page.browserContext.secretKey,
     );
     this.domStorageTracker = new DomStorageTracker(
       page,

agent/main/lib/FramesManager.ts

@@ -5,7 +5,6 @@ import { IFrame, IFrameManagerEvents } from '@ulixee/unblocked-specification/age
 import { bindFunctions } from '@ulixee/commons/lib/utils';
 import { IBoundLog } from '@ulixee/commons/interfaces/ILog';
 import { CanceledPromiseError } from '@ulixee/commons/interfaces/IPendingWaitEvent';
-import { IWebsocketEvents } from '@ulixee/unblocked-specification/agent/browser/IWebsocketSession';
 import IResourceMeta from '@ulixee/unblocked-specification/agent/net/IResourceMeta';
 import {
   IPageEvents,
@@ -85,7 +84,7 @@ export default class FramesManager extends TypedEventEmitter<IFrameManagerEvents
 
     bindFunctions(this);
 
-    this.console = new Console(devtoolsSession);
+    this.console = new Console(devtoolsSession, this.page.browserContext.secretKey);
 
     this.events.on(page, 'resource-will-be-requested', this.onResourceWillBeRequested);
     this.events.on(page, 'resource-was-requested', this.onResourceWasRequested);

agent/main/lib/NetworkManager.ts

@@ -62,6 +62,7 @@ export default class NetworkManager extends TypedEventEmitter<IBrowserNetworkEve
     devtoolsSession: DevtoolsSession,
     logger: IBoundLog,
     proxyConnectionOptions?: IProxyConnectionOptions,
+    public secretKey?: string,
   ) {
     super();
     this.devtools = devtoolsSession;
@@ -304,7 +305,7 @@ export default class NetworkManager extends TypedEventEmitter<IBrowserNetworkEve
     if (this.requestIdsToIgnore.has(networkRequest.requestId)) return;
 
     const url = networkRequest.request.url;
-    if (url.includes('agent.localhost')) {
+    if (url.includes(`/heroInternalUrl?secretKey=${this.secretKey}`)) {
       this.emit('internal-request', { request: networkRequest });
       this.addRequestIdToIgnore(networkRequest.requestId);
       return;

agent/main/lib/Page.ts

@@ -155,6 +155,7 @@ export default class Page extends TypedEventEmitter<IPageLevelEvents> implements
       devtoolsSession,
       this.logger,
       this.browserContext.proxy,
+      this.browserContext.secretKey,
     );
     this.domStorageTracker = new DomStorageTracker(
       this,

agent/main/lib/Worker.ts

@@ -61,6 +61,7 @@ export class Worker extends TypedEventEmitter<IWorkerEvents> implements IWorker
       devtoolsSession,
       this.logger,
       browserContext.proxy,
+      browserContext.secretKey,
     );
     const session = this.devtoolsSession;
     this.events.on(session, 'Inspector.targetReloadedAfterCrash', () => {

agent/mitm/handlers/RequestSession.ts

@@ -73,6 +73,7 @@ export default class RequestSession
     logger: IBoundLog,
     public upstreamProxyUrl?: string,
     public upstreamProxyUseSystemDns?: boolean,
+    public secretKey?: string,
   ) {
     super();
     this.logger = logger.createChild(module);
@@ -195,6 +196,11 @@ export default class RequestSession
   }
 
   public shouldInterceptRequest(url: string, resourceType?: IResourceType): boolean {
+    // Requests including heroInternalUrl with secret key are considered intenal requests
+    // We use this to cordinate internal frameIds in a way that allows us to do everything
+    // with Runtime domain disabled.
+    if(url.includes(`/heroInternalUrl?secretKey=${this.secretKey}`)) return true;
+
     for (const handler of this.interceptorHandlers) {
       if (handler.types && resourceType) {
         if (handler.types.includes(resourceType)) return true;