diff --git a/docs/api.md b/docs/api.md index aba876f7..543096c5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -6,11 +6,12 @@ Chromeless provides TypeScript typings. - [`end()`](#api-end) **Chrome methods** -- [`goto(url: string)`](#api-goto) +- [`goto(url: string, logRequests?: boolean)`](#api-goto) - [`click(selector: string)`](#api-click) - [`wait(timeout: number)`](#api-wait-timeout) - [`wait(selector: string)`](#api-wait-selector) - [`wait(fn: (...args: any[]) => boolean, ...args: any[])`](#api-wait-fn) +- [`waitForRequest(url: string, fn: Function)`](#api-waitForRequest) - [`focus(selector: string)`](#api-focus) - [`press(keyCode: number, count?: number, modifiers?: any)`](#api-press) - [`type(input: string, selector?: string)`](#api-type) @@ -54,12 +55,13 @@ await chromeless.end() -### goto(url: string): Chromeless +### goto(url: string, logRequests?: boolean): Chromeless Navigate to a URL. __Arguments__ - `url` - URL to navigate to +- `logRequests` - log all requests for this session. only useful for [`waitForRequest(url: string, fn: Function)`](#api-waitForRequest) __Example__ @@ -138,6 +140,30 @@ await chromeless.wait(() => { return console.log('@TODO: put a better example he --------------------------------------- + + +### waitForRequest(url: string, fn: (requests) => requests): Chromeless + +Wait until one or more requests have been sent to the specified URL + +__Arguments__ +- `url` - All requests that include this URL are collected and passed to `fn` +- `fn` - Function that receives an array of all available requests as first parameter. + Waits until this function returns true or the timeout is reached (default: 10s) + +__Example__ + +```js +const result = await chromeless + .goto('https://www.google.com', true) // required to get requests before the load event is fired + .waitForRequest('google.com', (requests) => { + return requests.length > 2; + }) + +console.log(result) +``` + +--------------------------------------- ### focus(selector: string): Chromeless diff --git a/src/api.ts b/src/api.ts index 75d8c184..b15fe933 100644 --- a/src/api.ts +++ b/src/api.ts @@ -60,8 +60,8 @@ export default class Chromeless implements Promise { return this.lastReturnPromise.catch(onrejected) as Promise } - goto(url: string): Chromeless { - this.queue.enqueue({type: 'goto', url}) + goto(url: string, logRequests?: boolean): Chromeless { + this.queue.enqueue({type: 'goto', url, logRequests}) return this } @@ -72,6 +72,12 @@ export default class Chromeless implements Promise { return this } + waitForRequest(url: string, fn: Function): Chromeless { + this.lastReturnPromise = this.queue.process({type: 'wait', url: url, fn: fn}) + + return this + } + wait(timeout: number): Chromeless wait(selector: string): Chromeless wait(fn: (...args: any[]) => boolean, ...args: any[]): Chromeless diff --git a/src/chrome/local-runtime.ts b/src/chrome/local-runtime.ts index 6082c9fe..017aece3 100644 --- a/src/chrome/local-runtime.ts +++ b/src/chrome/local-runtime.ts @@ -2,10 +2,12 @@ import * as AWS from 'aws-sdk' import { Client, Command, ChromelessOptions, Cookie, CookieQuery } from '../types' import * as cuid from 'cuid' import * as fs from 'fs' +import TrafficLog from '../network' import { nodeExists, wait, waitForNode, + waitForRequest, click, evaluate, screenshot, @@ -22,21 +24,25 @@ export default class LocalRuntime { private client: Client private chromlessOptions: ChromelessOptions + private trafficLog: TrafficLog constructor(client: Client, chromlessOptions: ChromelessOptions) { this.client = client this.chromlessOptions = chromlessOptions + this.trafficLog = new(TrafficLog) } async run(command: Command): Promise { switch (command.type) { case 'goto': - return this.goto(command.url) + return this.goto(command.url, command.logRequests) case 'wait': { if (command.timeout) { return this.waitTimeout(command.timeout) } else if (command.selector) { return this.waitSelector(command.selector) + } else if (command.url) { + return this.waitRequest(command.url, command.fn) } else { throw new Error('waitFn not yet implemented') } @@ -70,10 +76,14 @@ export default class LocalRuntime { } } - private async goto(url: string): Promise { + private async goto(url: string, logRequests: boolean): Promise { const {Network, Page} = this.client await Promise.all([Network.enable(), Page.enable()]) await Network.setUserAgentOverride({userAgent: `Chromeless ${version}`}) + if (logRequests) { + this.log(`Logging network requests`) + Network.requestWillBeSent(this.trafficLog.onRequest) + } await Page.navigate({url}) await Page.loadEventFired() this.log(`Navigated to ${url}`) @@ -90,6 +100,13 @@ export default class LocalRuntime { this.log(`Waited for ${selector}`) } + private async waitRequest(url: string, fn: Function): Promise { + this.log(`Waiting for request on url: ${url}`) + const result = await waitForRequest(this.trafficLog, url, fn, this.chromlessOptions.waitTimeout) + this.log(`Waited for request on url: ${url}`) + return result + } + private async click(selector: string): Promise { if (this.chromlessOptions.implicitWait) { this.log(`click(): Waiting for ${selector}`) @@ -214,4 +231,4 @@ export default class LocalRuntime { } } -} \ No newline at end of file +} diff --git a/src/network.ts b/src/network.ts new file mode 100644 index 00000000..db0dbc28 --- /dev/null +++ b/src/network.ts @@ -0,0 +1,48 @@ +import { Chrome, Request, Response, RequestEvent, ResponseEvent } from './types' + +export default class TrafficLog { + + private requests: { + [id: string]: Request + } + + private responses: { + [id: string]: Response[] + } + + constructor() { + this.requests = {} + this.responses = {} + this.onRequest = this.onRequest.bind(this) + this.onResponse = this.onResponse.bind(this) + } + + private addRequest(id: string, request: Request): void { + this.requests[id] = request + } + + private addResponse(id: string, response: Response): void { + this.responses[id].push(response) + } + + onRequest(event: RequestEvent): void { + this.addRequest(event.requestId, event.request) + } + + onResponse(event: ResponseEvent): void { + if (this.requests.hasOwnProperty(event.responseId)) { + this.addResponse(event.responseId, event.response) + } + } + + async getRequests(url: string, fn: Function): Promise { + const result = [] + for (const id of Object.keys(this.requests)) { + if (this.requests[id].url.includes(url)) { + result.push(this.requests[id]) + } + } + const finished = await fn(result) + return {requests: result, finished} + } +} diff --git a/src/types.ts b/src/types.ts index f0bc1ac5..9587b58f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -44,12 +44,14 @@ export type Command = | { type: 'goto' url: string + logRequests?: boolean } | { type: 'wait' timeout?: number selector?: string - fn?: string + fn?: Function + url?: string args?: any[] } | { @@ -130,3 +132,52 @@ export interface CookieQuery { secure?: boolean session?: boolean } + +export interface Request { + url: string + method: string + headers: any + postData?: string + mixedContentType?: string + initialPriority: string + referrerPolicy: string + isLinkPreload?: boolean +} + +export interface Response { + url: string + status: number + statusText: string + headers: any + headersText?: string + mimeType: string + requestHeaders?: any + requestHeadersText?: string + connectionReused: boolean + connectionId: number + fromDiskCache: boolean + fromServiceWorker: boolean + encodedDataLength: number + timing?: any + protocol?: string + securityState: string + securityDetails?: any +} + +export interface RequestEvent { + requestId: string + loaderId: string + documentURL: string + request: Request + timestamp: number + initiator: any + redirectResponse?: Response +} + +export interface ResponseEvent { + responseId: string, + loaderId: string + timestamp: number + type: string + response: Response +} diff --git a/src/util.ts b/src/util.ts index 43af2b23..338b8b22 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,6 +1,7 @@ import * as fs from 'fs' import * as path from 'path' import { Client, Cookie } from './types' +import TrafficLog from './network' export const version: string = ((): string => { if (fs.existsSync(path.join(__dirname, '../package.json'))) { @@ -46,6 +47,31 @@ export async function waitForNode(client: Client, selector: string, waitTimeout: } } +export async function waitForRequest(trafficLog: TrafficLog, url: string, fn: Function, waitTimeout: number): Promise { + const result = await trafficLog.getRequests(url, fn) + + if (!result.finished) { + const start = new Date().getTime() + return new Promise((resolve, reject) => { + const interval = setInterval(async () => { + if (new Date().getTime() - start > waitTimeout) { + clearInterval(interval) + reject(new Error(`waitForRequest("${url}") timed out after ${waitTimeout}ms`)) + } + + const result = await trafficLog.getRequests(url, fn) + + if (result.finished) { + clearInterval(interval) + resolve(result.requests) + } + }, 500) + }) + } else { + return result.requests + } +} + export async function wait(timeout: number): Promise { return new Promise((resolve, reject) => setTimeout(resolve, timeout)) } @@ -311,4 +337,3 @@ export function getDebugOption(): boolean { return false } -