From d319f39bb59c3a4f7e1f398e38303c3e727d54ee Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 5 Feb 2025 10:37:06 -0500 Subject: [PATCH] New Components - zenscrape (#15467) * new components * pnpm-lock.yaml * remove source --- .../get-credit-status/get-credit-status.mjs | 19 ++++++ .../get-website-content.mjs | 56 ++++++++++++++++ components/zenscrape/package.json | 8 ++- components/zenscrape/zenscrape.app.mjs | 64 +++++++++++++++++-- pnpm-lock.yaml | 9 ++- 5 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 components/zenscrape/actions/get-credit-status/get-credit-status.mjs create mode 100644 components/zenscrape/actions/get-website-content/get-website-content.mjs diff --git a/components/zenscrape/actions/get-credit-status/get-credit-status.mjs b/components/zenscrape/actions/get-credit-status/get-credit-status.mjs new file mode 100644 index 0000000000000..22ff1bf4b05d8 --- /dev/null +++ b/components/zenscrape/actions/get-credit-status/get-credit-status.mjs @@ -0,0 +1,19 @@ +import zenscrape from "../../zenscrape.app.mjs"; + +export default { + key: "zenscrape-get-credit-status", + name: "Get Credit Status", + description: "Retrieve the number of remaining credits in Zenscrape. [See the documentation](https://app.zenscrape.com/documentation)", + version: "0.0.1", + type: "action", + props: { + zenscrape, + }, + async run({ $ }) { + const response = await this.zenscrape.getStatus({ + $, + }); + $.export("$summary", "Successfully retrieved credit status."); + return response; + }, +}; diff --git a/components/zenscrape/actions/get-website-content/get-website-content.mjs b/components/zenscrape/actions/get-website-content/get-website-content.mjs new file mode 100644 index 0000000000000..dc1195f15d070 --- /dev/null +++ b/components/zenscrape/actions/get-website-content/get-website-content.mjs @@ -0,0 +1,56 @@ +import zenscrape from "../../zenscrape.app.mjs"; + +export default { + key: "zenscrape-get-website-content", + name: "Get Website Content", + description: "Retrieve the content of a website. [See the documentation](https://app.zenscrape.com/documentation)", + version: "0.0.1", + type: "action", + props: { + zenscrape, + url: { + propDefinition: [ + zenscrape, + "url", + ], + }, + premium: { + propDefinition: [ + zenscrape, + "premium", + ], + }, + location: { + propDefinition: [ + zenscrape, + "location", + ], + }, + keepHeaders: { + propDefinition: [ + zenscrape, + "keepHeaders", + ], + }, + render: { + propDefinition: [ + zenscrape, + "render", + ], + }, + }, + async run({ $ }) { + const response = await this.zenscrape.getContent({ + $, + params: { + url: this.url, + premium: this.premium, + location: this.location, + keep_headers: this.keepHeaders, + render: this.render, + }, + }); + $.export("$summary", `Successfully scraped website \`${this.url}.\``); + return response; + }, +}; diff --git a/components/zenscrape/package.json b/components/zenscrape/package.json index 71b652aa0ff86..71477b31cfd7b 100644 --- a/components/zenscrape/package.json +++ b/components/zenscrape/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/zenscrape", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Zenscrape Components", "main": "zenscrape.app.mjs", "keywords": [ @@ -11,5 +11,9 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3", + "md5": "^2.3.0" } -} \ No newline at end of file +} diff --git a/components/zenscrape/zenscrape.app.mjs b/components/zenscrape/zenscrape.app.mjs index 4ac729c665b47..0b59e381202fe 100644 --- a/components/zenscrape/zenscrape.app.mjs +++ b/components/zenscrape/zenscrape.app.mjs @@ -1,11 +1,67 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "zenscrape", - propDefinitions: {}, + propDefinitions: { + url: { + type: "string", + label: "URL", + description: "The target site you want to scrape", + }, + premium: { + type: "boolean", + label: "Premium", + description: "Uses residential proxies, unlocks sites that are hard to scrape. Counts as 20 credits towards your quota.", + optional: true, + }, + location: { + type: "string", + label: "Location", + description: "If premium=`false` possible locations are 'na' (North America) and 'eu' (Europe). If premium=`true` you can choose a location from Zenscrape's [list of 230+ countries](https://app.zenscrape.com/documentation#proxyLocationList)", + optional: true, + }, + keepHeaders: { + type: "boolean", + label: "Keep Headers", + description: "Allow to pass through forward headers (e.g. user agents, cookies)", + optional: true, + }, + render: { + type: "boolean", + label: "Render", + description: "Use a headless browser to fetch content that relies on javascript. Counts as 5 credits towards your quota.", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://app.zenscrape.com/api/v1"; + }, + _makeRequest({ + $ = this, + path, + ...opts + }) { + return axios($, { + url: `${this._baseUrl()}${path}`, + headers: { + apikey: this.$auth.api_key, + }, + ...opts, + }); + }, + getContent(opts = {}) { + return this._makeRequest({ + path: "/get", + ...opts, + }); + }, + getStatus(opts = {}) { + return this._makeRequest({ + path: "/status", + ...opts, + }); }, }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36bc979418a49..be12ad6b860ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12267,7 +12267,14 @@ importers: specifier: ^1.5.1 version: 1.6.6 - components/zenscrape: {} + components/zenscrape: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 + md5: + specifier: ^2.3.0 + version: 2.3.0 components/zenserp: {}