Merge pull request #46 from tamarasaurus/setup-e2e-ci

Run example in CI

.github/workflows/main.yml

@@ -13,4 +13,6 @@ jobs:
       - name: Install dependencies
         run: npm install
       - name: Run tests
-        run: npm test
+        run: npm test
+      - name: Run example using Puppeteer
+        run: npm run example

package.json

@@ -15,7 +15,7 @@
     "watch": "tsc -w -p .",
     "lint": "eslint **/*.ts",
     "lint-fix": "eslint --fix **/*.ts",
-    "example": "ts-node ./src/examples/scrape",
+    "example": "ts-node ./src/examples/e2e",
     "example-wait": "ts-node ./src/examples/waitForPageLoadSelector.ts"
   },
   "repository": {
@@ -66,7 +66,6 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
     "globals": "^15.8.0",
-    "sinon": "^18.0.0",
     "source-map-support": "^0.5.21",
     "ts-node": "^10.9.2",
     "tslint-config-airbnb": "^5.11.2",

src/examples/scrape.ts → src/examples/e2e.ts

@@ -21,8 +21,15 @@ const contract = {
   },
 };
 
-const scraper = new Scraper('https://webscraper.io/test-sites/e-commerce/allinone', contract);
+const scraper = new Scraper(
+  'https://webscraper.io/test-sites/e-commerce/allinone',
+  contract,
+);
 
 scraper.scrapePage().then(recipes => {
-  console.log(recipes);
+  if (recipes.length === 0) {
+    process.exit(1);
+  } else {
+    console.log('Scraped successfully');
+  }
 });

test/integration/scrape.test.ts

@@ -1,7 +1,6 @@
 import Scraper from '../../index';
 import { ScrapedPage } from '../../src/fetcher/fetcher';
-import * as sinon from 'sinon';
-import { expect, describe, it } from 'vitest'
+import { expect, it, vi } from 'vitest';
 
 const url = 'http://characters.com';
 
@@ -134,7 +133,7 @@ it('returns scraped data for a url and contract', () => {
   };
 
   const scraper = new Scraper(url, contract);
-  scraper.getFetcher = sinon.stub().returns(new FakeFetcher(url));
+  scraper.getFetcher = vi.fn().mockImplementation(() => new FakeFetcher(url));
 
   const expectedData = [
     {
@@ -227,7 +226,7 @@ it('scrapes a json schema script tag for a url and contract', () => {
     },
   ];
 
-  scraper.getFetcher = sinon.stub().returns(new FakeFetcher(url));
+  scraper.getFetcher = vi.fn().mockImplementation(() => new FakeFetcher(url));
 
   return scraper
     .scrapePage()
@@ -256,7 +255,7 @@ it('scrapes raw html from a script tag', () => {
   const scraper = new Scraper(url, contract);
   const expectedData = [{ names: JSON.stringify({ first: 'Han Solo' }) }];
 
-  scraper.getFetcher = sinon.stub().returns(new FakeFetcher(url));
+  scraper.getFetcher = vi.fn().mockImplementation(() => new FakeFetcher(url));
 
   return scraper
     .scrapePage()
@@ -284,7 +283,7 @@ it('returns the scraped page object', () => {
 
   const scraper = new Scraper(url, contract);
 
-  scraper.getFetcher = sinon.stub().returns(new FakeFetcher(url));
+  scraper.getFetcher = vi.fn().mockImplementation(() => new FakeFetcher(url));
 
   return scraper
     .getPageContents()

test/unit/fetcher/puppeteer.test.ts

@@ -1,6 +1,5 @@
 import PuppeteerFetcher from '../../../src/fetcher/puppeteer';
-import * as sinon from 'sinon';
-import { expect, describe, it } from 'vitest'
+import { expect, describe, it, vi } from 'vitest';
 
 class FakePage {
   setUserAgent() {}
@@ -35,7 +34,9 @@ class FakePuppeteer {
 }
 
 const puppeteerFetcher = new PuppeteerFetcher('http://leboncoin.com');
-puppeteerFetcher.getBrowserType = sinon.stub().returns(new FakePuppeteer());
+puppeteerFetcher.getBrowserType = vi
+  .fn()
+  .mockImplementation(() => new FakePuppeteer());
 
 describe('it fetches data from puppeteer', () => {
   it('sets up the browser', async () => {

test/unit/fetcher/request.test.ts

@@ -1,6 +1,5 @@
-import * as sinon from 'sinon';
 import RequestFetcher from '../../../src/fetcher/request';
-import { expect, describe, it } from 'vitest'
+import { expect, describe, it, vi } from 'vitest';
 
 class FakeRequest {
   get(options: any) {
@@ -18,7 +17,9 @@ class FakeRequest {
 describe('it gets page contents using request', () => {
   it('gets encoded page contents for a given url', () => {
     const request = new RequestFetcher('http://whatever.com');
-    request.getRequestLibrary = sinon.stub().returns(new FakeRequest());
+    request.getRequestLibrary = vi
+      .fn()
+      .mockImplementation(() => new FakeRequest());
 
     const expectedPage = {
       encoding: 'utf-8',

test/unit/index.test.ts

@@ -2,9 +2,8 @@ import Scraper from '../../index';
 import PuppeteerFetcher from '../../src/fetcher/puppeteer';
 import { ScrapedPage } from '../../src/fetcher/fetcher';
 import HTMLProvider from '../../src/provider/html';
-import * as sinon from 'sinon';
 import RequestFetcher from '../../src/fetcher/request';
-import { expect, describe, it, beforeEach } from 'vitest'
+import { expect, describe, it, beforeEach, vi } from 'vitest';
 
 const contract = {
   itemSelector: 'li[itemtype=http://schema.org/Offer]',
@@ -115,10 +114,10 @@ describe('Scrapes a URL based on JSON configuration', () => {
       }
     }
 
-    scraper.getFetcher = sinon
-      .stub()
-      .returns(new FakeFetcher('http://leboncoin.com'));
-    scraper.getProvider = sinon.stub().returns(new ProviderStub());
+    scraper.getFetcher = vi
+      .fn()
+      .mockImplementation(() => new FakeFetcher('http://leboncoin.com'));
+    scraper.getProvider = vi.fn().mockImplementation(() => new ProviderStub());
 
     return scraper
       .scrapePage()

tsconfig.json

@@ -8,8 +8,5 @@
     "resolveJsonModule": true,
     "esModuleInterop": true
   },
-  "include": [
-    "src/**/*.ts",
-    "index.ts"
-  ]
+  "include": ["src/**/*.ts", "index.ts", "test/e2e/scrape.ts"]
 }