Merge pull request #117 from ulixee/sa2

Migrate to Unblocked internals

.github/workflows/lint-and-test.yml

@@ -13,25 +13,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v2
         with:
           node-version: 14
+          cache: yarn
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: (TEMP) Clone unblocked
+        run: git clone --recurse-submodules -j8 https://github.com/unblockable/unblocked.git
+        working-directory: ../..
 
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - uses: actions/checkout@v2
+      - name: (TEMP) Install unblocked
+        run: yarn build:all
+        working-directory: ../../unblocked
 
       - name: Install Dependencies
         run: yarn install --immutable --network-timeout 1000000
@@ -74,16 +69,27 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
+          cache: yarn
 
       - uses: actions/setup-go@v2
         with:
           go-version: 1.17
 
+      - name: (TEMP) Clone unblocked
+        run: git clone --recurse-submodules -j8 https://github.com/unblockable/unblocked.git
+        working-directory: ../..
+
+      - name: (TEMP) Install unblocked
+        run: yarn build:all
+        working-directory: ../../unblocked
+        env:
+          UBK_MITM_MITM_SOCKET: 1
+
       - name: Download built typescript files
         uses: actions/download-artifact@v2
         with:
@@ -93,44 +99,27 @@ jobs:
       - name: Untar files
         run: tar -xf js_files.tar
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Copy mitm source
-        run: cp -r mitm-socket/go build/mitm-socket/go
-
       - name: Build modules
         run: yarn --network-timeout 1000000 --verbose
         working-directory: ./build
-        env:
-          HERO_REBUILD_MITM_SOCKET: 1
 
       - name: Linux - Add Chrome Env
         if: ${{ matrix.browser }}
-        working-directory: ./build
+        working-directory: ../../unblocked/build
         run: yarn add -W @ulixee/${{ matrix.browser }}
 
       - name: Linux - Apt Install Chrome(s)
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        working-directory: ./build
+        working-directory: ../../unblocked/build
         run: sudo $(npx install-browser-deps)
 
       - name: Run tests
         run: yarn jest --testTimeout=60000 --maxWorkers=2
         working-directory: ./build
         env:
           NODE_ENV: test
-          HERO_DATA_DIR: .data
-          HERO_DEFAULT_BROWSER_ID: ${{ matrix.browser }}
+          ULX_DATA_DIR: .data
+          UBK_DEFAULT_BROWSER_ID: ${{ matrix.browser }}
 
       - name: 'Tar files'
         if: ${{ failure() }}
@@ -143,10 +132,3 @@ jobs:
           name: test-dbs-${{matrix.os}}-${{ matrix.node-version }}
           path: test-dbs.tar
           retention-days: 1
-
-      - name: Coverage
-        run: npm -g install codecov && codecov
-        working-directory: ./build
-        if: ${{ matrix.node-version == '14' && matrix.os == 'ubuntu-latest' }}
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

README.md

@@ -5,21 +5,21 @@ Hero is a web browser built for scraping.
 - [x] **Built for scraping** - it's the first modern headless browsers designed specifically for scraping instead of just automated testing.
 - [x] **Designed for web developers** - We've recreated a fully compliant DOM directly in NodeJS allowing you bypass the headaches of previous scraper tools.
 - [x] **Powered by Chrome** - The powerful Chrome engine sits under the hood, allowing for lightning fast rendering.
-- [x] **Emulates any modern browser** - BrowserEmulators make it easy to disguise your script as practically any browser.
+- [x] **Emulates any modern browser** - Emulators make it easy to disguise your script as practically any browser.
 - [x] **Avoids detection along the entire stack** - Don't be blocked because of TLS fingerprints in your networking stack.
 
 Check out our [website for more details](https://ulixee.org).
 
 ## Installation
 
 ```shell script
-npm i --save ulixee
+npm i --save @ulixee/hero
 ```
 
 or
 
 ```shell script
-yarn add ulixee
+yarn add @ulixee/hero
 ```
 
 ## Usage

client/index.ts

@@ -18,11 +18,11 @@ import {
   INodeList,
 } from 'awaited-dom/base/interfaces/official';
 import { BlockedResourceType } from '@ulixee/hero-interfaces/ITabOptions';
-import { KeyboardKey } from '@ulixee/hero-interfaces/IKeyboardLayoutUS';
-import IResourceType, { ResourceType } from '@ulixee/hero-interfaces/IResourceType';
-import { InteractionCommand, MouseButton } from '@ulixee/hero-interfaces/IInteractions';
+import { KeyboardKey } from '@unblocked-web/specifications/agent/interact/IKeyboardLayoutUS';
+import IResourceType, { ResourceType } from '@unblocked-web/specifications/agent/net/IResourceType';
+import { InteractionCommand, MouseButton } from '@unblocked-web/specifications/agent/interact/IInteractions';
 import { Node, XPathResult } from '@ulixee/hero-interfaces/AwaitedDom';
-import { LoadStatus, LocationStatus, LocationTrigger } from '@ulixee/hero-interfaces/Location';
+import { LoadStatus, LocationStatus, LocationTrigger } from '@unblocked-web/specifications/agent/browser/Location';
 import IHeroCreateOptions from './interfaces/IHeroCreateOptions';
 import IConnectionToCoreOptions from './interfaces/IConnectionToCoreOptions';
 import { Hero, FrameEnvironment, Tab, Resource, WebsocketResource } from './lib/extendables';

client/interfaces/IInteractions.ts

@@ -1,9 +1,9 @@
-import { IKeyboardKeyCode } from '@ulixee/hero-interfaces/IKeyboardLayoutUS';
-import { IKeyboardShortcut } from '@ulixee/hero-interfaces/IKeyboardShortcuts';
+import { IKeyboardKeyCode } from '@unblocked-web/specifications/agent/interact/IKeyboardLayoutUS';
+import { IKeyboardShortcut } from '@unblocked-web/specifications/agent/interact/IKeyboardShortcuts';
 import {
   IElementInteractVerification,
   IMousePositionXY,
-} from '@ulixee/hero-interfaces/IInteractions';
+} from '@unblocked-web/specifications/agent/interact/IInteractions';
 import { ISuperElement } from 'awaited-dom/base/interfaces/super';
 
 export type IInteraction = ICommand | ICommandDetailed;

client/interfaces/IJsPathEventTarget.ts

@@ -1,4 +1,4 @@
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 
 export default interface IJsPathEventTarget {
   addEventListener(

client/interfaces/IWaitForResourceFilter.ts

@@ -1,4 +1,4 @@
-import IResourceType from '@ulixee/hero-interfaces/IResourceType';
+import IResourceType from '@unblocked-web/specifications/agent/net/IResourceType';
 import Resource from '../lib/Resource';
 
 export default interface IWaitForResourceFilter {

client/lib/AwaitedEventTarget.ts

@@ -1,4 +1,4 @@
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 import IAwaitedEventTarget from '../interfaces/IAwaitedEventTarget';
 import IJsPathEventTarget from '../interfaces/IJsPathEventTarget';
 

client/lib/CookieStorage.ts

@@ -1,5 +1,5 @@
 import ISetCookieOptions from '@ulixee/hero-interfaces/ISetCookieOptions';
-import { ICookie } from '@ulixee/hero-interfaces/ICookie';
+import { ICookie } from '@unblocked-web/specifications/agent/net/ICookie';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
 
 export default class CookieStorage {

client/lib/CoreEventHeap.ts

@@ -1,4 +1,4 @@
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 import ISessionMeta from '@ulixee/hero-interfaces/ISessionMeta';
 import Resolvable from '@ulixee/commons/lib/Resolvable';
 import Log from '@ulixee/commons/lib/Logger';

client/lib/CoreFrameEnvironment.ts

@@ -1,21 +1,20 @@
-import { IInteractionGroups, isMousePositionXY } from '@ulixee/hero-interfaces/IInteractions';
+import { IInteractionGroups, isMousePositionXY } from '@unblocked-web/specifications/agent/interact/IInteractions';
 import ISessionMeta from '@ulixee/hero-interfaces/ISessionMeta';
-import { ILoadStatus, ILocationTrigger } from '@ulixee/hero-interfaces/Location';
-import AwaitedPath, { IJsPath } from 'awaited-dom/base/AwaitedPath';
-import { ICookie } from '@ulixee/hero-interfaces/ICookie';
+import { ILoadStatus, ILocationTrigger } from '@unblocked-web/specifications/agent/browser/Location';
+import { IJsPath , INodeVisibility , INodePointer } from '@unblocked-web/js-path';
+import AwaitedPath  from 'awaited-dom/base/AwaitedPath';
+import { ICookie } from '@unblocked-web/specifications/agent/net/ICookie';
 import IWaitForElementOptions from '@ulixee/hero-interfaces/IWaitForElementOptions';
-import IExecJsPathResult from '@ulixee/hero-interfaces/IExecJsPathResult';
+import IExecJsPathResult from '@unblocked-web/specifications/agent/browser/IExecJsPathResult';
 import { IRequestInit } from 'awaited-dom/base/interfaces/official';
-import INodePointer from 'awaited-dom/base/INodePointer';
 import ISetCookieOptions from '@ulixee/hero-interfaces/ISetCookieOptions';
 import {
   getComputedVisibilityFnName,
   isFocusedFnName,
-} from '@ulixee/hero-interfaces/jsPathFnNames';
+} from '@unblocked-web/specifications/agent/browser/IJsPathFunctions';
 import IWaitForOptions from '@ulixee/hero-interfaces/IWaitForOptions';
 import IFrameMeta from '@ulixee/hero-interfaces/IFrameMeta';
-import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta';
-import { INodeVisibility } from '@ulixee/hero-interfaces/INodeVisibility';
+import IResourceMeta from '@unblocked-web/specifications/agent/net/IResourceMeta';
 import StateMachine from 'awaited-dom/base/StateMachine';
 import { IElementIsolate, INodeIsolate } from 'awaited-dom/base/interfaces/isolate';
 import { ISuperElement } from 'awaited-dom/base/interfaces/super';

client/lib/CoreSession.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'events';
 import ISessionMeta from '@ulixee/hero-interfaces/ISessionMeta';
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 import { loggerSessionIdNames } from '@ulixee/commons/lib/Logger';
 import IHeroMeta from '@ulixee/hero-interfaces/IHeroMeta';
 import * as readline from 'readline';
@@ -224,7 +224,7 @@ export default class CoreSession implements IJsPathEventTarget {
   }
 
   private showSessionKeepAlivePrompt(message: string): void {
-    if (/yes|1|true/i.test(process.env.HERO_CLI_NOPROMPT)) return;
+    if (/yes|1|true/i.test(process.env.ULX_CLI_NOPROMPT)) return;
 
     this.cliPrompt = readline.createInterface({
       input: process.stdin,

client/lib/CoreTab.ts

@@ -1,14 +1,14 @@
 import ISessionMeta from '@ulixee/hero-interfaces/ISessionMeta';
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 import IWaitForResourceOptions from '@ulixee/hero-interfaces/IWaitForResourceOptions';
-import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta';
+import IResourceMeta from '@unblocked-web/specifications/agent/net/IResourceMeta';
 import IUserProfile from '@ulixee/hero-interfaces/IUserProfile';
 import IConfigureSessionOptions from '@ulixee/hero-interfaces/IConfigureSessionOptions';
 import IWaitForOptions from '@ulixee/hero-interfaces/IWaitForOptions';
-import IScreenshotOptions from '@ulixee/hero-interfaces/IScreenshotOptions';
+import IScreenshotOptions from '@unblocked-web/specifications/agent/browser/IScreenshotOptions';
 import IFrameMeta from '@ulixee/hero-interfaces/IFrameMeta';
 import TimeoutError from '@ulixee/commons/interfaces/TimeoutError';
-import IFileChooserPrompt from '@ulixee/hero-interfaces/IFileChooserPrompt';
+import IFileChooserPrompt from '@unblocked-web/specifications/agent/browser/IFileChooserPrompt';
 import CoreCommandQueue from './CoreCommandQueue';
 import CoreEventHeap from './CoreEventHeap';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';

client/lib/DomExtender.ts

@@ -10,19 +10,19 @@ import NodeList from 'awaited-dom/impl/official-klasses/NodeList';
 import HTMLCollection from 'awaited-dom/impl/official-klasses/HTMLCollection';
 import HTMLElement from 'awaited-dom/impl/official-klasses/HTMLElement';
 import AwaitedPath from 'awaited-dom/base/AwaitedPath';
-import INodePointer from 'awaited-dom/base/INodePointer';
-import { IElementInteractVerification } from '@ulixee/hero-interfaces/IInteractions';
+import { INodePointer } from '@unblocked-web/js-path';
+import { IElementInteractVerification } from '@unblocked-web/specifications/agent/interact/IInteractions';
 import SuperNodeList from 'awaited-dom/impl/super-klasses/SuperNodeList';
 import SuperHTMLCollection from 'awaited-dom/impl/super-klasses/SuperHTMLCollection';
-import { KeyboardKey } from '@ulixee/hero-interfaces/IKeyboardLayoutUS';
+import { KeyboardKey } from '@unblocked-web/specifications/agent/interact/IKeyboardLayoutUS';
 import { ITypeInteraction } from '../interfaces/IInteractions';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
 import IAwaitedOptions from '../interfaces/IAwaitedOptions';
 import Interactor from './Interactor';
 import XPathResult from 'awaited-dom/impl/official-klasses/XPathResult';
 import { createSuperNode } from 'awaited-dom/impl/create';
 import { getAwaitedPathAsMethodArg } from './SetupAwaitedHandler';
-import { KeyboardShortcuts } from '@ulixee/hero-interfaces/IKeyboardShortcuts';
+import { KeyboardShortcuts } from '@unblocked-web/specifications/agent/interact/IKeyboardShortcuts';
 import { scriptInstance } from './internal';
 
 const awaitedPathState = StateMachine<

client/lib/DomStateHandler.ts

@@ -1,7 +1,7 @@
 import { bindFunctions } from '@ulixee/commons/lib/utils';
 import Timer from '@ulixee/commons/lib/Timer';
 import Resolvable from '@ulixee/commons/lib/Resolvable';
-import { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
 import ISessionMeta from '@ulixee/hero-interfaces/ISessionMeta';
 import IDomStateResult from '@ulixee/hero-interfaces/IDomStateResult';
 import CoreTab from './CoreTab';

client/lib/FileChooser.ts

@@ -1,6 +1,7 @@
-import IFileChooserPrompt from '@ulixee/hero-interfaces/IFileChooserPrompt';
+import IFileChooserPrompt from '@unblocked-web/specifications/agent/browser/IFileChooserPrompt';
 import * as Fs from 'fs';
-import AwaitedPath, { IJsPath } from 'awaited-dom/base/AwaitedPath';
+import { IJsPath } from '@unblocked-web/js-path';
+import AwaitedPath  from 'awaited-dom/base/AwaitedPath';
 import { createHTMLInputElement } from 'awaited-dom/impl/create';
 import { IHTMLInputElement } from 'awaited-dom/base/interfaces/official';
 import * as Path from 'path';

client/lib/FrameEnvironment.ts

@@ -17,7 +17,7 @@ import {
   createSuperNodeList,
 } from 'awaited-dom/impl/create';
 import Request from 'awaited-dom/impl/official-klasses/Request';
-import { ILoadStatus, ILocationTrigger, LocationStatus } from '@ulixee/hero-interfaces/Location';
+import { ILoadStatus, ILocationTrigger, LocationStatus } from '@unblocked-web/specifications/agent/browser/Location';
 import IWaitForElementOptions from '@ulixee/hero-interfaces/IWaitForElementOptions';
 import Response from 'awaited-dom/impl/official-klasses/Response';
 import IWaitForOptions from '@ulixee/hero-interfaces/IWaitForOptions';
@@ -28,7 +28,7 @@ import {
   IHTMLObjectElementIsolate,
   INodeIsolate,
 } from 'awaited-dom/base/interfaces/isolate';
-import { INodeVisibility } from '@ulixee/hero-interfaces/INodeVisibility';
+import { INodeVisibility } from '@unblocked-web/js-path';
 import { INodePointer } from '@ulixee/hero-interfaces/AwaitedDom';
 import IAwaitedOptions from '../interfaces/IAwaitedOptions';
 import RequestGenerator, { getRequestIdOrUrl } from './Request';
@@ -38,7 +38,7 @@ import { getAwaitedPathAsMethodArg } from './SetupAwaitedHandler';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
 import Tab, { getCoreTab } from './Tab';
 import Resource, { createResource } from './Resource';
-import { IMousePositionXY } from '@ulixee/hero-interfaces/IInteractions';
+import { IMousePositionXY } from '@unblocked-web/specifications/agent/interact/IInteractions';
 import { InternalPropertiesSymbol } from './internal';
 
 const awaitedPathState = StateMachine<