Skip to content

Commit

Permalink
Merge pull request #348 from ulixee/migrate-from-websocket-to-depreca…
Browse files Browse the repository at this point in the history
…ted-console-domain

Migrate from websocket to deprecated console domain
  • Loading branch information
blakebyrnes authored Feb 4, 2025
2 parents dd239ae + 8bf5117 commit 6b46e57
Show file tree
Hide file tree
Showing 17 changed files with 221 additions and 281 deletions.
1 change: 1 addition & 0 deletions agent/main/interfaces/IProxyConnectionOptions.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export default interface IProxyConnectionOptions {
address: string;
username?: string;
password?: string;
}
17 changes: 12 additions & 5 deletions agent/main/lib/Agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ import { IHooksProvider } from '@ulixee/unblocked-specification/agent/hooks/IHoo
import IEmulationProfile, {
IEmulationOptions,
} from '@ulixee/unblocked-specification/plugin/IEmulationProfile';
import { IUnblockedPluginClass, PluginConfigs } from '@ulixee/unblocked-specification/plugin/IUnblockedPlugin';
import {
IUnblockedPluginClass,
PluginConfigs,
} from '@ulixee/unblocked-specification/plugin/IUnblockedPlugin';
import { nanoid } from 'nanoid';
import env from '../env';
import ICommandMarker from '../interfaces/ICommandMarker';
Expand Down Expand Up @@ -52,12 +55,15 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
private readonly closeBrowserOnClose: boolean = false;
private isolatedMitm: MitmProxy;

// We use secretKey all through Agent components to make sure websites can't test if hero is present.
// Without this secretKey if would be pretty easy to detect hero.
private secretKey = nanoid();

private get proxyConnectionInfo(): IProxyConnectionOptions {
if (!this.enableMitm) {
if (this.emulationProfile.upstreamProxyUrl) {
return { address: this.emulationProfile.upstreamProxyUrl };
}
return null;
if (!this.emulationProfile.upstreamProxyUrl) return null;
const url = new URL(this.emulationProfile.upstreamProxyUrl);
return { address: url.origin, username: url.username, password: url.password };
}
if (this.isolatedMitm) {
// don't use password for an isolated mitm proxy
Expand Down Expand Up @@ -210,6 +216,7 @@ export default class Agent extends TypedEventEmitter<{ close: void }> {
hooks: this.plugins,
isIncognito: this.isIncognito,
commandMarker: this.options.commandMarker,
secretKey: this.secretKey,
});
this.events.once(this.browserContext, 'close', () => this.close());

Expand Down
4 changes: 1 addition & 3 deletions agent/main/lib/Browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
if (options.proxyPort !== undefined && !launchArgs.some(x => x.startsWith('--proxy-server'))) {
launchArgs.push(
// Use proxy for localhost URLs
'--proxy-bypass-list=<-loopback>;websocket.localhost',
'--proxy-bypass-list=<-loopback>',
`--proxy-server=localhost:${options.proxyPort}`,
);
}
Expand Down Expand Up @@ -461,7 +461,6 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
});
}
const context = new BrowserContext(this, false);
void context.initialize();
context.hooks = this.browserContextCreationHooks ?? {};
context.id = targetInfo.browserContextId;
context.targetsById.set(targetInfo.targetId, targetInfo);
Expand Down Expand Up @@ -573,7 +572,6 @@ export default class Browser extends TypedEventEmitter<IBrowserEvents> implement
private async onNewContext(context: BrowserContext): Promise<void> {
const id = context.id;
this.browserContextsById.set(id, context);
await context.initialize();
context.once('close', () => this.browserContextsById.delete(id));
this.emit('new-context', { context });
await this.hooks?.onNewBrowserContext?.(context);
Expand Down
13 changes: 4 additions & 9 deletions agent/main/lib/BrowserContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import WebsocketMessages from './WebsocketMessages';
import { DefaultCommandMarker } from './DefaultCommandMarker';
import DevtoolsSessionLogger from './DevtoolsSessionLogger';
import FrameOutOfProcess from './FrameOutOfProcess';
import { WebsocketSession } from './WebsocketSession';
import CookieParam = Protocol.Network.CookieParam;
import TargetInfo = Protocol.Target.TargetInfo;
import CreateBrowserContextRequest = Protocol.Target.CreateBrowserContextRequest;
Expand All @@ -41,6 +40,7 @@ export interface IBrowserContextCreateOptions {
hooks?: IBrowserContextHooks & IInteractHooks;
isIncognito?: boolean;
commandMarker?: ICommandMarker;
secretKey?: string,
}

export default class BrowserContext
Expand Down Expand Up @@ -69,14 +69,14 @@ export default class BrowserContext
}

public isIncognito = true;
public readonly websocketSession: WebsocketSession;

public readonly idTracker = {
navigationId: 0,
tabId: 0,
frameId: 0,
};

public secretKey?: string
public commandMarker: ICommandMarker;

private attachedTargetIds = new Set<string>();
Expand All @@ -96,6 +96,7 @@ export default class BrowserContext
this.isIncognito = isIncognito;
this.logger = options?.logger ?? log;
this.hooks = options?.hooks ?? {};
this.secretKey = options?.secretKey;
this.commandMarker = options?.commandMarker ?? new DefaultCommandMarker(this);
this.resources = new Resources(this);
this.websocketMessages = new WebsocketMessages(this.logger);
Expand All @@ -104,11 +105,6 @@ export default class BrowserContext
this.devtoolsSessionLogger.subscribeToDevtoolsMessages(this.browser.devtoolsSession, {
sessionType: 'browser',
});
this.websocketSession = new WebsocketSession();
}

public async initialize(): Promise<void> {
await this.websocketSession.initialize();
}

public async open(): Promise<void> {
Expand All @@ -118,7 +114,7 @@ export default class BrowserContext
disposeOnDetach: true,
};
if (this.proxy?.address) {
createContextOptions.proxyBypassList = '<-loopback>;websocket.localhost';
createContextOptions.proxyBypassList = '<-loopback>';
createContextOptions.proxyServer = this.proxy.address;
}

Expand Down Expand Up @@ -351,7 +347,6 @@ export default class BrowserContext
this.resources.cleanup();
this.events.close();
this.emit('close');
this.websocketSession.close();
this.devtoolsSessionLogger.close();
this.removeAllListeners();
this.cleanup();
Expand Down
133 changes: 133 additions & 0 deletions agent/main/lib/Console.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Currently this only used to support communication from chrome (injected scripts) to unblocked agent
import Resolvable from '@ulixee/commons/lib/Resolvable';
import TypedEventEmitter from '@ulixee/commons/lib/TypedEventEmitter';
import DevtoolsSession from './DevtoolsSession';
import EventSubscriber from '@ulixee/commons/lib/EventSubscriber';
import Protocol from 'devtools-protocol';
import { IConsoleEvents } from '@ulixee/unblocked-specification/agent/browser/IConsole';

const SCRIPT_PLACEHOLDER = '';

export class Console extends TypedEventEmitter<IConsoleEvents> {
isReady: Resolvable<void>;

private readonly events = new EventSubscriber();
// We store resolvable when we received websocket message before, receiving
// targetId, this way we can await this, and still trigger to get proper ids.
private clientIdToTargetId = new Map<string, Resolvable<string> | string>();

constructor(
public devtoolsSession: DevtoolsSession,
public secretKey: string,
) {
super();
}

async initialize(): Promise<void> {
if (this.isReady) return this.isReady.promise;
this.isReady = new Resolvable();

await this.devtoolsSession.send('Console.enable');
this.events.on(
this.devtoolsSession,
'Console.messageAdded',
this.handleConsoleMessage.bind(this),
);

this.isReady.resolve();
return this.isReady.promise;
}

isConsoleRegisterUrl(url: string): boolean {
return url.includes(
`hero.localhost/?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=`,
);
}

registerFrameId(url: string, frameId: string): void {
const parsed = new URL(url);
const clientId = parsed.searchParams.get('clientId');
if (!clientId) return;

const targetId = this.clientIdToTargetId.get(clientId);
if (targetId instanceof Resolvable) {
targetId.resolve(frameId);
}
this.clientIdToTargetId.set(clientId, frameId);
}

injectCallbackIntoScript(script: string): string {
// We could do this as a simple template script but this logic might get
// complex over time and we want typescript to be able to check proxyScript();
const scriptFn = injectedScript
.toString()
// eslint-disable-next-line no-template-curly-in-string
.replaceAll('${this.secretKey}', this.secretKey)
// Use function otherwise replace will try todo some magic
.replace('SCRIPT_PLACEHOLDER', () => script);

const wsScript = `(${scriptFn})();`;
return wsScript;
}

private async handleConsoleMessage(msgAdded: Protocol.Console.MessageAddedEvent): Promise<void> {
if (msgAdded.message.source !== 'console-api' || msgAdded.message.level !== 'debug') return;

let clientId: string;
let name: string;
let payload: any;

try {
// Doing this is much much cheaper than json parse on everything logged in console debug
const text = msgAdded.message.text;
const [secret, maybeClientId, serializedData] = [
text.slice(6, 27),
text.slice(29, 39),
text.slice(41),
];
if (secret !== this.secretKey) return;

const data = JSON.parse(serializedData);
name = data.name;
payload = data.payload;
clientId = maybeClientId;
} catch {
return;
}

let frameId = this.clientIdToTargetId.get(clientId);
if (!frameId) {
const resolvable = new Resolvable<string>();
this.clientIdToTargetId.set(clientId, resolvable);
frameId = await resolvable.promise;
} else if (frameId instanceof Resolvable) {
frameId = await frameId.promise;
}

this.emit('callback-received', { id: frameId, name, payload });
}
}

/** This function will be stringified and inserted as a wrapper script so all injected
* scripts have access to a callback function (over a websocket). This function takes
* care of setting up that websocket and all other logic it needs as glue to make it all work.
* */
function injectedScript(): void {
const clientId = Math.random().toString().slice(2, 12);

const url = `http://hero.localhost/?secretKey=${this.secretKey}&action=registerConsoleClientId&clientId=${clientId}`;

// This will signal to network manager we are trying to make websocket connection
// This is needed later to map clientId to frameId
void fetch(url, { mode: 'no-cors' }).catch(() => undefined);

// eslint-disable-next-line @typescript-eslint/no-unused-vars
const callback = (name, payload): void => {
const serializedData = JSON.stringify({ name, payload });
// eslint-disable-next-line no-console
console.debug(`hero: ${this.secretKey}, ${clientId}, ${serializedData}`);
};

// eslint-disable-next-line @typescript-eslint/no-unused-expressions
SCRIPT_PLACEHOLDER;
}
12 changes: 6 additions & 6 deletions agent/main/lib/DevtoolsSessionLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,12 @@ export default class DevtoolsSessionLogger extends TypedEventEmitter<IDevtoolsLo
let frameId = event.frameId;
let requestId: string;
let pageId = event.pageTargetId;

// Filter out internal communication to prevent lots of duplicate events and data
if (event.method === 'Console.messageAdded' && params.message.text.startsWith('hero:')) {
return
}

if (params) {
frameId = frameId ?? params.frame?.id ?? params.frameId ?? params.context?.auxData?.frameId;

Expand All @@ -150,12 +156,6 @@ export default class DevtoolsSessionLogger extends TypedEventEmitter<IDevtoolsLo
params.networkId ??
params.requestId;
if (params.networkId) this.fetchRequestIdToNetworkId.set(params.requestId, params.networkId);
if (
event.method === 'Network.webSocketCreated' &&
this.browserContext.websocketSession?.isWebsocketUrl(params.url)
) {
this.requestsToSkip.add(requestId);
}

if (!pageId && params.targetInfo && params.targetInfo?.type === 'page') {
pageId = params.targetInfo.targetId;
Expand Down
2 changes: 1 addition & 1 deletion agent/main/lib/FrameOutOfProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ export default class FrameOutOfProcess {
this.frame = frame;
this.networkManager = new NetworkManager(
this.devtoolsSession,
this.browserContext.websocketSession,
frame.logger,
page.browserContext.proxy,
page.browserContext.secretKey,
);
this.domStorageTracker = new DomStorageTracker(
page,
Expand Down
Loading

0 comments on commit 6b46e57

Please sign in to comment.