Skip to content

Commit

Permalink
feat: scraper & workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
siam-ese committed Dec 4, 2024
1 parent b3edbb1 commit ef97b73
Show file tree
Hide file tree
Showing 173 changed files with 2,787 additions and 2,784 deletions.
3 changes: 0 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,12 @@
"@types/node": "^20.12.12",
"@types/react": "^18.3.2",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react-swc": "^3.6.0",
"@wendellhu/redi": "0.16.1",
"autoprefixer": "^10.4.19",
"cross-env": "^7.0.3",
"eslint": "^8.57.0",
"lint-staged": "^15.2.2",
"postcss": "^8.4.38",
"rimraf": "^5.0.7",
"rxjs": "^7.0.0",
"simple-git-hooks": "^2.11.1",
"tailwindcss": "^3.4.3",
"tslib": "^2.6.2",
Expand Down
3 changes: 3 additions & 0 deletions packages/ajax-intercept/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export { startAjaxIntercept } from '@lib/bootstrap';
export { AJAX_INTERCEPT_MESSAGE_TYPE, type IAjaxInterceptMessage } from '@lib/ajax-intercept.message';
export { interceptRequest } from '@lib/ajax-intercept-script';
65 changes: 65 additions & 0 deletions packages/ajax-intercept/lib/ajax-intercept-script.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import type { IAjaxInterceptMessage } from './ajax-intercept.message';
import { AJAX_INTERCEPT_MESSAGE_TYPE } from './ajax-intercept.message';

function _interceptRequest(onResponse: (response: any) => void) {
const XHR = XMLHttpRequest;
const _fetch = fetch;

const onReadyStateChange = async function (this: XMLHttpRequest) {
if (this.readyState === 4) {
onResponse(this.response);
}
};
// @ts-expect-error
const innerXHR: typeof XMLHttpRequest = function () {
const xhr = new XHR();
xhr.addEventListener('readystatechange', onReadyStateChange.bind(xhr), false);
return xhr;
};
innerXHR.prototype = XHR.prototype;
Object.entries(XHR).forEach(([key, val]) => {
// @ts-ignore
innerXHR[key] = val;
});

const innerFetch: typeof _fetch = async (resource, initOptions) => {
const getOriginalResponse = () => _fetch(resource, initOptions);
const fetchedResponse = getOriginalResponse();

fetchedResponse.then((response) => {
if (response instanceof Response) {
try {
response.clone()
.json()
.then((res) => onResponse(res))
.catch(() => {
// Do nothing
});
} catch (err) {}
}
});
return fetchedResponse;
};
window.XMLHttpRequest = innerXHR;
window.fetch = innerFetch;
}

function serializeToJSON(response: unknown) {
try {
return JSON.parse(JSON.stringify(response));
} catch {
return null;
}
}

export function interceptRequest() {
_interceptRequest((res) => {
const msg: IAjaxInterceptMessage = {
type: AJAX_INTERCEPT_MESSAGE_TYPE,
response: serializeToJSON(res),
};

postMessage(msg);
});
}

8 changes: 8 additions & 0 deletions packages/ajax-intercept/lib/ajax-intercept.message.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@

export const AJAX_INTERCEPT_MESSAGE_TYPE = 'CLIPSHEET_AJAX_INTERCEPT_MESSAGE';

export interface IAjaxInterceptMessage {
type: typeof AJAX_INTERCEPT_MESSAGE_TYPE;
response: any;
}

23 changes: 23 additions & 0 deletions packages/ajax-intercept/lib/bootstrap.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import type { IAjaxInterceptMessage } from './ajax-intercept.message';
import { AJAX_INTERCEPT_MESSAGE_TYPE } from './ajax-intercept.message';

export function startAjaxIntercept(scriptSrc: string, onMessage: (message: any) => void) {
const script = document.createElement('script');
script.setAttribute('type', 'text/javascript');
script.setAttribute('src', scriptSrc);
document.documentElement.appendChild(script);

const listener = (event: MessageEvent) => {
const message = event.data as IAjaxInterceptMessage;
if (message.type === AJAX_INTERCEPT_MESSAGE_TYPE) {
onMessage(message.response);
}
};

window.addEventListener('message', listener);

return () => [
window.removeEventListener('message', listener),
document.documentElement.removeChild(script),
];
}
29 changes: 29 additions & 0 deletions packages/ajax-intercept/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"name": "@univer-clipsheet-core/ajax-intercept",
"version": "0.0.1",
"private": true,
"description": "ajax-intercept code",
"sideEffects": false,
"main": "./dist/index.js",
"module": "./dist/index.mjs",
"types": "./dist/index.d.ts",
"files": [
"dist/**"
],
"scripts": {
"clean": "rimraf ./dist && rimraf .turbo",
"build": "tsup index.ts --format esm,cjs --dts --external react,chrome",
"dev": "pnpm run build --watch",
"type-check": "tsc --noEmit"
},
"peerDependencies": {
"@wendellhu/redi": "0.16.1"
},
"dependencies": {
"@univer-clipsheet-core/shared": "workspace:*"
},
"devDependencies": {
"@univer-clipsheet-core/tailwindcss-config": "workspace:*",
"@univer-clipsheet-core/tsconfig": "workspace:*"
}
}
6 changes: 6 additions & 0 deletions packages/ajax-intercept/postcss.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
};
18 changes: 18 additions & 0 deletions packages/ajax-intercept/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"extends": "@univer-clipsheet-core/tsconfig/utils",
"compilerOptions": {
"jsx": "react-jsx",
"baseUrl": ".",
"paths": {
"@lib/*": [
"lib/*"
]
},
"types": ["chrome"],
"outDir": "dist"
},
"include": [
"index.ts",
"lib"
]
}
8 changes: 8 additions & 0 deletions packages/ajax-intercept/tsup.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { defineConfig } from 'tsup';

export default defineConfig({
treeshake: true,
format: ['cjs', 'esm'],
dts: true,
external: ['chrome'],
});
2 changes: 1 addition & 1 deletion packages/locale/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export function isZhCN() {
return chrome.i18n.getUILanguage() === 'zh-CN';
}

function createTranslator<K extends string = string>(i18mMap: Record<K, string>) {
export function createTranslator<K extends string = string>(i18mMap: Record<K, string>) {
return (key: K, params?: Record<string, string>) => {
const text = i18mMap[key];
if (params) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { Channel } from '@univer-clipsheet-core/shared';

export interface DrillDownTaskChannelRequest {
selectors: string[];
// channelName: string;
}
export interface DrillDownTaskChannelResponse {
items: Array<{
Expand Down
24 changes: 0 additions & 24 deletions packages/scraper/lib/drill-down-service/drill-down.service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
// import type { DrillDownTaskChannelResponse, Message, MessageItem } from '@chrome-extension-boilerplate/shared';
import { requestConnectChannel } from '@univer-clipsheet-core/shared';
import { Inject } from '@wendellhu/redi';
import { drillDownTaskChannel, getDrillDownTaskChannelName } from './drill-down-channel';

import type { DrillDownTaskChannelResponse } from './drill-down-channel';
Expand Down Expand Up @@ -63,7 +61,6 @@ export class DrillDownService {
}

listenMessage() {
// const { _drillDownService } = this;
chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
if (changeInfo.status === 'complete') {
const drillDownContext = this.getContext(tabId);
Expand All @@ -86,26 +83,5 @@ export class DrillDownService {
requestConnectChannel(channelName, tabId);
}
});

// chrome.runtime.onMessage.addListener(async (msg: RequestDrillDownMessage, sender) => {
// const senderTabId = sender.tab?.id;
// if (!senderTabId) {
// return;
// }

// switch (msg.type) {
// case DrillDownMessageTypeEnum.RequestDrillDown: {
// const { payload } = msg;
// const result = await _drillDownService.runDrillDown(payload.url, payload.selectors);

// const response: ResponseDrillDownMessage = {
// type: DrillDownMessageTypeEnum.ResponseDrillDown,
// payload: result.items,
// };

// chrome.tabs.sendMessage(senderTabId, response);
// }
// }
// });
}
}
1 change: 1 addition & 0 deletions packages/scraper/lib/drill-down-service/index.ts
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export { DrillDownService } from './drill-down.service';
export * from './drill-down-channel';
1 change: 1 addition & 0 deletions packages/scraper/lib/scraper-service/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ export {
} from './scraper-threshold';
export * from './scraper.message';
export * from './scraper-data-source';
export * from './scraper-channel';
1 change: 1 addition & 0 deletions packages/scraper/lib/scraper-service/scraper-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ export interface ScraperTaskChannelResponse {
}

export const getScraperTaskChannelName = (id: string) => `ScraperTaskChannel-${id}`;
export const isScraperTaskChannelName = (name: string) => name.startsWith('ScraperTaskChannel-');
export const scraperTaskChannel = new Channel<ScraperTaskChannelRequest, ScraperTaskChannelResponse>();
105 changes: 1 addition & 104 deletions packages/scraper/lib/scraper-service/scraper-data-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,6 @@ import { createIdentifier } from '@wendellhu/redi';
import type { IGetScraperListParams } from './scraper.message';
import { ScraperStorageKeyEnum } from './scraper.message';

// export const pushScraperListDataSource = (value: IScraper[]) => {
// const msg: PushDataSourceMessage = {
// type: ClipsheetMessageTypeEnum.PushDataSource,
// payload: {
// key: ScraperDataSourceKeyEnum.ScraperList,
// value,
// },

// };
// chrome.runtime.sendMessage(msg);
// };
// export const sendScraperListDataSource = (value: IScraper[]) => sendDataSource(DataSourceKeys.ScraperList, value);

// export interface IGetScraperListParams {
// pageSize: number;
// filterRecordIds?: string[];
// }

export interface IScraperDataSource {
getList(params: IGetScraperListParams): Promise<IScraper[]>;
add(scraper: IScraper): Promise<IScraper>;
Expand All @@ -34,7 +16,7 @@ export const IScraperDataSource = createIdentifier('scraper-data-source');

const getStorageScraperList = async () => (await getStorage<IScraper[]>(ScraperStorageKeyEnum.ScraperList)) ?? [];

export class ScraperLocaleDataSource implements IScraperDataSource {
export class LocaleScraperDataSource implements IScraperDataSource {
async getList(params: IGetScraperListParams): Promise<IScraper[]> {
return getStorageScraperList();
}
Expand All @@ -43,15 +25,13 @@ export class ScraperLocaleDataSource implements IScraperDataSource {
const scraperList = await getStorageScraperList();
const newScraperList = [scraper].concat(scraperList);
await setStorage(ScraperStorageKeyEnum.ScraperList, newScraperList);
// injector.get(StorageManager).setStorage(StorageKeys.ScraperList, newScraperList);

return scraper;
}

async delete(id: string): Promise<void> {
const scraperList = await getStorageScraperList();
await setStorage(ScraperStorageKeyEnum.ScraperList, scraperList.filter((s) => s.id !== id));
// injector.get(StorageManager).setStorage(StorageKeys.ScraperList, scraperList.filter((s) => s.id !== id));
}

async update(scraper: IScraper): Promise<void> {
Expand All @@ -63,88 +43,5 @@ export class ScraperLocaleDataSource implements IScraperDataSource {

scraperList[index] = scraper;
await setStorage(ScraperStorageKeyEnum.ScraperList, scraperList.slice());
// injector.get(StorageManager).setStorage(StorageKeys.ScraperList, scraperList.slice());
}
}

// class RemoteScraperDataSource implements IScraperDataSource {
// getScraperList(params: IGetScraperListParams): Promise<IScraper[]> {
// return crxRequest.getScraperList(params).then((res) => {
// return res.records.map((r) => {
// const scraper = r.json;
// scraper.id = r.recordId;
// scraper.createAt = r.createAt;
// scraper.updateAt = r.updateAt;

// return scraper;
// });
// });
// }

// async addScraper(scraper: IScraper): Promise<IScraper> {
// const res = crxRequest.createScraper({
// title: scraper.name,
// json_obj: JSON.stringify(scraper),
// }).then((res) => {
// scraper.id = res.recordId;

// return scraper;
// });

// return res;
// }

// async deleteScraper(id: string) {
// return crxRequest.deleteScraper({ record_id: id });
// }

// async updateScraper(scraper: IScraper) {
// return crxRequest.updateScraper({
// record_id: scraper.id,
// title: scraper.name,
// json_obj: JSON.stringify(scraper),
// });
// }
// }

// export class ScraperDataSource implements IScraperDataSource {
// private _pageSize = 20;

// local$ = new ObservableValue(false);
// private _localDataSource = new LocalScraperDataSource();
// private _remoteDataSource = new RemoteScraperDataSource();

// get currentDataSource(): IScraperDataSource {
// return this.local$.value ? this._localDataSource : this._remoteDataSource;
// }

// private _sendNewScraperList() {
// this.getScraperList({ pageSize: this._pageSize }).then((list) => {
// sendScraperListDataSource(list);
// });
// }

// async getScraperList(params: IGetScraperListParams): Promise<IScraper[]> {
// this._pageSize = params.pageSize;
// return this.currentDataSource.getScraperList(params);
// }

// async addScraper(scraper: IScraper): Promise<IScraper> {
// const res = await this.currentDataSource.addScraper(scraper);
// this._sendNewScraperList();
// return res;
// }

// async deleteScraper(id: string) {
// const res = await this.currentDataSource.deleteScraper(id);
// this._sendNewScraperList();
// return res;
// }

// async updateScraper(scraper: IScraper) {
// const res = await this.currentDataSource.updateScraper(scraper);
// this._sendNewScraperList();

// return res;
// }
// }
Loading

0 comments on commit ef97b73

Please sign in to comment.