Migrate from websocket to deprecated console domain

agent/main/interfaces/IProxyConnectionOptions.ts

@@ -1,4 +1,5 @@
 export default interface IProxyConnectionOptions {
   address: string;
+  username?: string;
   password?: string;
 }

agent/main/lib/Agent.ts

@@ -10,7 +10,10 @@ import { IHooksProvider } from '@ulixee/unblocked-specification/agent/hooks/IHoo
 import IEmulationProfile, {
   IEmulationOptions,
 } from '@ulixee/unblocked-specification/plugin/IEmulationProfile';
-import { IUnblockedPluginClass, PluginConfigs } from '@ulixee/unblocked-specification/plugin/IUnblockedPlugin';
+import {
+  IUnblockedPluginClass,
+  PluginConfigs,
+} from '@ulixee/unblocked-specification/plugin/IUnblockedPlugin';
 import { nanoid } from 'nanoid';
 import env from '../env';
 import ICommandMarker from '../interfaces/ICommandMarker';
@@ -52,12 +55,15 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
   private readonly closeBrowserOnClose: boolean = false;
   private isolatedMitm: MitmProxy;
 
+  // We use secretKey all through Agent components to make sure websites can't test if hero is present.
+  // Without this secretKey if would be pretty easy to detect hero.
+  private secretKey = nanoid();
+
   private get proxyConnectionInfo(): IProxyConnectionOptions {
     if (!this.enableMitm) {
-      if (this.emulationProfile.upstreamProxyUrl) {
-        return { address: this.emulationProfile.upstreamProxyUrl };
-      }
-      return null;
+      if (!this.emulationProfile.upstreamProxyUrl) return null;
+      const url = new URL(this.emulationProfile.upstreamProxyUrl);
+      return { address: url.origin, username: url.username, password: url.password };
     }
     if (this.isolatedMitm) {
       // don't use password for an isolated mitm proxy
@@ -210,6 +216,7 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
       hooks: this.plugins,
       isIncognito: this.isIncognito,
       commandMarker: this.options.commandMarker,
+      secretKey: this.secretKey,
     });
     this.events.once(this.browserContext, 'close', () => this.close());
 

agent/main/lib/Browser.ts

@@ -368,7 +368,7 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
     if (options.proxyPort !== undefined && !launchArgs.some(x => x.startsWith('--proxy-server'))) {
       launchArgs.push(
         // Use proxy for localhost URLs
-        '--proxy-bypass-list=<-loopback>;websocket.localhost',
+        '--proxy-bypass-list=<-loopback>',
         `--proxy-server=localhost:${options.proxyPort}`,
       );
     }
@@ -461,7 +461,6 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
         });
       }
       const context = new BrowserContext(this, false);
-      void context.initialize();
       context.hooks = this.browserContextCreationHooks ?? {};
       context.id = targetInfo.browserContextId;
       context.targetsById.set(targetInfo.targetId, targetInfo);
@@ -573,7 +572,6 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
   private async onNewContext(context: BrowserContext): Promise<void> {
     const id = context.id;
     this.browserContextsById.set(id, context);
-    await context.initialize();
     context.once('close', () => this.browserContextsById.delete(id));
     this.emit('new-context', { context });
     await this.hooks?.onNewBrowserContext?.(context);

agent/main/lib/BrowserContext.ts

@@ -28,7 +28,6 @@ import WebsocketMessages from './WebsocketMessages';
 import { DefaultCommandMarker } from './DefaultCommandMarker';
 import DevtoolsSessionLogger from './DevtoolsSessionLogger';
 import FrameOutOfProcess from './FrameOutOfProcess';
-import { WebsocketSession } from './WebsocketSession';
 import CookieParam = Protocol.Network.CookieParam;
 import TargetInfo = Protocol.Target.TargetInfo;
 import CreateBrowserContextRequest = Protocol.Target.CreateBrowserContextRequest;
@@ -41,6 +40,7 @@ export interface IBrowserContextCreateOptions {
   hooks?: IBrowserContextHooks & IInteractHooks;
   isIncognito?: boolean;
   commandMarker?: ICommandMarker;
+  secretKey?: string,
 }
 
 export default class BrowserContext
@@ -69,14 +69,14 @@ export default class BrowserContext
   }
 
   public isIncognito = true;
-  public readonly websocketSession: WebsocketSession;
 
   public readonly idTracker = {
     navigationId: 0,
     tabId: 0,
     frameId: 0,
   };
 
+  public secretKey?: string
   public commandMarker: ICommandMarker;
 
   private attachedTargetIds = new Set<string>();
@@ -96,6 +96,7 @@ export default class BrowserContext
     this.isIncognito = isIncognito;
     this.logger = options?.logger ?? log;
     this.hooks = options?.hooks ?? {};
+    this.secretKey = options?.secretKey;
     this.commandMarker = options?.commandMarker ?? new DefaultCommandMarker(this);
     this.resources = new Resources(this);
     this.websocketMessages = new WebsocketMessages(this.logger);
@@ -104,11 +105,6 @@ export default class BrowserContext
     this.devtoolsSessionLogger.subscribeToDevtoolsMessages(this.browser.devtoolsSession, {
       sessionType: 'browser',
     });
-    this.websocketSession = new WebsocketSession();
-  }
-
-  public async initialize(): Promise<void> {
-    await this.websocketSession.initialize();
   }
 
   public async open(): Promise<void> {
@@ -118,7 +114,7 @@ export default class BrowserContext
       disposeOnDetach: true,
     };
     if (this.proxy?.address) {
-      createContextOptions.proxyBypassList = '<-loopback>;websocket.localhost';
+      createContextOptions.proxyBypassList = '<-loopback>';
       createContextOptions.proxyServer = this.proxy.address;
     }
 
@@ -351,7 +347,6 @@ export default class BrowserContext
       this.resources.cleanup();
       this.events.close();
       this.emit('close');
-      this.websocketSession.close();
       this.devtoolsSessionLogger.close();
       this.removeAllListeners();
       this.cleanup();

agent/main/lib/Console.ts

@@ -0,0 +1,133 @@
+// Currently this only used to support communication from chrome (injected scripts) to unblocked agent
+import Resolvable from '@ulixee/commons/lib/Resolvable';
+import TypedEventEmitter from '@ulixee/commons/lib/TypedEventEmitter';
+import DevtoolsSession from './DevtoolsSession';
+import EventSubscriber from '@ulixee/commons/lib/EventSubscriber';
+import Protocol from 'devtools-protocol';
+import { IConsoleEvents } from '@ulixee/unblocked-specification/agent/browser/IConsole';
+
+const SCRIPT_PLACEHOLDER = '';
+
+export class Console extends TypedEventEmitter<IConsoleEvents> {
+  isReady: Resolvable<void>;
+
+  private readonly events = new EventSubscriber();
+  // We store resolvable when we received websocket message before, receiving
+  // targetId, this way we can await this, and still trigger to get proper ids.
+  private clientIdToTargetId = new Map<string, Resolvable<string> | string>();
+
+  constructor(
+    public devtoolsSession: DevtoolsSession,
+    public secretKey: string,
+  ) {
+    super();
+  }
+
+  async initialize(): Promise<void> {
+    if (this.isReady) return this.isReady.promise;
+    this.isReady = new Resolvable();
+
+    await this.devtoolsSession.send('Console.enable');
+    this.events.on(
+      this.devtoolsSession,
+      'Console.messageAdded',
+      this.handleConsoleMessage.bind(this),
+    );
+
+    this.isReady.resolve();
+    return this.isReady.promise;
+  }
+
+  isConsoleRegisterUrl(url: string): boolean {
+    return url.includes(
+      `hero.localhost/?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=`,
+    );
+  }
+
+  registerFrameId(url: string, frameId: string): void {
+    const parsed = new URL(url);
+    const clientId = parsed.searchParams.get('clientId');
+    if (!clientId) return;
+
+    const targetId = this.clientIdToTargetId.get(clientId);
+    if (targetId instanceof Resolvable) {
+      targetId.resolve(frameId);
+    }
+    this.clientIdToTargetId.set(clientId, frameId);
+  }
+
+  injectCallbackIntoScript(script: string): string {
+    // We could do this as a simple template script but this logic might get
+    // complex over time and we want typescript to be able to check proxyScript();
+    const scriptFn = injectedScript
+      .toString()
+      // eslint-disable-next-line no-template-curly-in-string
+      .replaceAll('${this.secretKey}', this.secretKey)
+      // Use function otherwise replace will try todo some magic
+      .replace('SCRIPT_PLACEHOLDER', () => script);
+
+    const wsScript = `(${scriptFn})();`;
+    return wsScript;
+  }
+
+  private async handleConsoleMessage(msgAdded: Protocol.Console.MessageAddedEvent): Promise<void> {
+    if (msgAdded.message.source !== 'console-api' || msgAdded.message.level !== 'debug') return;
+
+    let clientId: string;
+    let name: string;
+    let payload: any;
+
+    try {
+      // Doing this is much much cheaper than json parse on everything logged in console debug
+      const text = msgAdded.message.text;
+      const [secret, maybeClientId, serializedData] = [
+        text.slice(6, 27),
+        text.slice(29, 39),
+        text.slice(41),
+      ];
+      if (secret !== this.secretKey) return;
+
+      const data = JSON.parse(serializedData);
+      name = data.name;
+      payload = data.payload;
+      clientId = maybeClientId;
+    } catch {
+      return;
+    }
+
+    let frameId = this.clientIdToTargetId.get(clientId);
+    if (!frameId) {
+      const resolvable = new Resolvable<string>();
+      this.clientIdToTargetId.set(clientId, resolvable);
+      frameId = await resolvable.promise;
+    } else if (frameId instanceof Resolvable) {
+      frameId = await frameId.promise;
+    }
+
+    this.emit('callback-received', { id: frameId, name, payload });
+  }
+}
+
+/** This function will be stringified and inserted as a wrapper script so all injected
+ * scripts have access to a callback function (over a websocket). This function takes
+ * care of setting up that websocket and all other logic it needs as glue to make it all work.
+ * */
+function injectedScript(): void {
+  const clientId = Math.random().toString().slice(2, 12);
+
+  const url = `http://hero.localhost/?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=${clientId}`;
+
+  // This will signal to network manager we are trying to make websocket connection
+  // This is needed later to map clientId to frameId
+  void fetch(url, { mode: 'no-cors' }).catch(() => undefined);
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const callback = (name, payload): void => {
+    const serializedData = JSON.stringify({ name, payload });
+    // eslint-disable-next-line no-console
+    console.debug(`hero: ${this.secretKey}, ${clientId}, ${serializedData}`);
+  };
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+  SCRIPT_PLACEHOLDER;
+}

agent/main/lib/DevtoolsSessionLogger.ts

@@ -141,6 +141,12 @@ export default class DevtoolsSessionLogger extends TypedEventEmitter<IDevtoolsLo
     let frameId = event.frameId;
     let requestId: string;
     let pageId = event.pageTargetId;
+
+    // Filter out internal communication to prevent lots of duplicate events and data
+    if (event.method === 'Console.messageAdded' && params.message.text.startsWith('hero:')) {
+      return
+    }
+
     if (params) {
       frameId = frameId ?? params.frame?.id ?? params.frameId ?? params.context?.auxData?.frameId;
 
@@ -150,12 +156,6 @@ export default class DevtoolsSessionLogger extends TypedEventEmitter<IDevtoolsLo
         params.networkId ??
         params.requestId;
       if (params.networkId) this.fetchRequestIdToNetworkId.set(params.requestId, params.networkId);
-      if (
-        event.method === 'Network.webSocketCreated' &&
-        this.browserContext.websocketSession?.isWebsocketUrl(params.url)
-      ) {
-        this.requestsToSkip.add(requestId);
-      }
 
       if (!pageId && params.targetInfo && params.targetInfo?.type === 'page') {
         pageId = params.targetInfo.targetId;

agent/main/lib/FrameOutOfProcess.ts

@@ -23,9 +23,9 @@ export default class FrameOutOfProcess {
     this.frame = frame;
     this.networkManager = new NetworkManager(
       this.devtoolsSession,
-      this.browserContext.websocketSession,
       frame.logger,
       page.browserContext.proxy,
+      page.browserContext.secretKey,
     );
     this.domStorageTracker = new DomStorageTracker(
       page,