From b5ba73d94b46868233b5b71818e0eff08ee8fc21 Mon Sep 17 00:00:00 2001 From: Raymond Negron Date: Sun, 27 Aug 2023 16:44:37 -0500 Subject: [PATCH 1/5] Adding automatically envvar and variables in job create modal --- .../components/EnvVarsSettingsPage/index.tsx | 4 +- estela-web/src/pages/JobCreateModal/index.tsx | 63 ++++++++++++++----- estela-web/src/pages/JobDetailPage/index.tsx | 14 ++--- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/estela-web/src/components/EnvVarsSettingsPage/index.tsx b/estela-web/src/components/EnvVarsSettingsPage/index.tsx index 7b2c0ba6..61b364d6 100644 --- a/estela-web/src/components/EnvVarsSettingsPage/index.tsx +++ b/estela-web/src/components/EnvVarsSettingsPage/index.tsx @@ -235,7 +235,7 @@ export const EnvVarsSetting: React.FC = ({ projectId, spiderId, e setNewEnvVarValue(""); setNewEnvVarMasked(false); setOpenModal(false); - setActiveUpdateButton(true); + updateEnvVars(); } else { invalidDataNotification("Invalid environment variable name/value pair."); } @@ -394,7 +394,7 @@ export const EnvVarsSetting: React.FC = ({ projectId, spiderId, e onClick={() => updateEnvVars()} className="border-estela bg-estela hover:border-estela hover:text-estela text-white rounded-md text-base h-full" > - Save variables + Save changes diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index f85642c5..ab71a4e1 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -51,11 +51,6 @@ interface TagsData { name: string; } -interface Tags { - name: string; - key: number; -} - interface JobData { args: ArgsData[]; envVars: EnvVarsData[]; @@ -71,7 +66,6 @@ interface Variable { newEnvVarValue: string; newEnvVarMasked: boolean; newTagName: string; - newTags: Tags[]; } interface Request { @@ -119,7 +113,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea newEnvVarValue: "", newEnvVarMasked: false, newTagName: "", - newTags: [], }); const [request, setRequest] = useState({ pid: projectId, @@ -299,51 +292,62 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea setJobData({ ...jobData, tags: [...tags] }); }; - const addArgument = (): void => { + const addArgument = (): ArgsData | undefined => { const args = [...jobData.args]; const newArgName = variable.newArgName.trim(); const newArgValue = variable.newArgValue.trim(); if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - args.push({ name: newArgName, value: newArgValue, key: countKey }); + const arg = { + name: newArgName, + value: newArgValue, + key: countKey, + }; + args.push(arg); setCountKey(countKey + 1); setJobData({ ...jobData, args: [...args] }); setVariable({ ...variable, newArgName: "", newArgValue: "" }); + return arg; } else { invalidDataNotification("Invalid argument name/value pair."); + return; } }; - const addEnvVar = (): void => { + const addEnvVar = (): EnvVarsData | undefined => { const envVars = [...jobData.envVars]; const newEnvVarName = variable.newEnvVarName.trim(); const newEnvVarValue = variable.newEnvVarValue.trim(); if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - envVars.push({ + const newEnvVar = { name: newEnvVarName, value: newEnvVarValue, masked: variable.newEnvVarMasked, key: countKey, - }); + }; + envVars.push(newEnvVar); setCountKey(countKey + 1); setJobData({ ...jobData, envVars: [...envVars] }); setVariable({ ...variable, newEnvVarName: "", newEnvVarValue: "", newEnvVarMasked: false }); + return newEnvVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); + return; } }; - const addTag = (): void => { + const addTag = (): TagsData | undefined => { const tags = [...jobData.tags]; - const newTags = [...variable.newTags]; const newTagName = variable.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: countKey }); + const tag = { name: newTagName, key: countKey }; + tags.push(tag); setCountKey(countKey + 1); - tags.push({ name: newTagName }); setJobData({ ...jobData, tags: [...tags] }); - setVariable({ ...variable, newTags: [...newTags], newTagName: "" }); + setVariable({ ...variable, newTagName: "" }); + return tag; } else { invalidDataNotification("Invalid tag name."); + return; } }; @@ -371,6 +375,31 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const handleSubmit = (): void => { setLoading(true); + const newEnvVarName = variable.newEnvVarName.trim(); + if (newEnvVarName && newEnvVarName.indexOf(" ") == -1) { + const newEnvVar = addEnvVar(); + if (!newEnvVar) { + return; + } + jobData.envVars.push(newEnvVar); + } + const newArgName = variable.newArgName.trim(); + if (newArgName && newArgName.indexOf(" ") == -1) { + const newArg = addArgument(); + if (!newArg) { + return; + } + jobData.args.push(newArg); + } + const newTagName = variable.newTagName.trim(); + if (newTagName && newTagName.indexOf(" ") == -1) { + const newTag = addTag(); + if (!newTag) { + return; + } + jobData.tags.push(newTag); + } + const { args, tags, dataStatus, dataExpiryDays } = jobData; const { pid, sid } = request; diff --git a/estela-web/src/pages/JobDetailPage/index.tsx b/estela-web/src/pages/JobDetailPage/index.tsx index 40106371..55324f42 100644 --- a/estela-web/src/pages/JobDetailPage/index.tsx +++ b/estela-web/src/pages/JobDetailPage/index.tsx @@ -291,7 +291,7 @@ export class JobDetailPage extends Component, J const args = response.args || []; const envVars = response.envVars || []; const tags = response.tags || []; - const lifespan = parseDuration(response.lifespan); + const lifespan = response.lifespan; this.setState({ name: response.name, lifespan: lifespan, @@ -753,7 +753,7 @@ export class JobDetailPage extends Component, J {date} - + Scheduled job @@ -770,7 +770,7 @@ export class JobDetailPage extends Component, J )} - + Tags @@ -790,7 +790,7 @@ export class JobDetailPage extends Component, J - + Environment variables @@ -822,7 +822,7 @@ export class JobDetailPage extends Component, J - + Arguments @@ -855,7 +855,7 @@ export class JobDetailPage extends Component, J )} {status == "WAITING" && ( - + {status} @@ -943,7 +943,7 @@ export class JobDetailPage extends Component, J - {durationToString(lifespan || 0)} + {durationToString(parseDuration(String(lifespan || 0)))} From 9cdd310044f6f8352c8b00f176ae5e7d13fbd4ba Mon Sep 17 00:00:00 2001 From: Raymond Negron Date: Sun, 27 Aug 2023 18:05:26 -0500 Subject: [PATCH 2/5] Adding cautomatically variables in cronjob create modal --- .../src/pages/CronjobCreateModal/index.tsx | 72 +++++++++++++------ estela-web/src/pages/JobCreateModal/index.tsx | 11 +-- .../pages/ProjectCronJobListPage/index.tsx | 23 ++++-- .../src/pages/ProjectJobListPage/index.tsx | 19 +++++ 4 files changed, 89 insertions(+), 36 deletions(-) diff --git a/estela-web/src/pages/CronjobCreateModal/index.tsx b/estela-web/src/pages/CronjobCreateModal/index.tsx index 8fc7f45b..95a2c55a 100644 --- a/estela-web/src/pages/CronjobCreateModal/index.tsx +++ b/estela-web/src/pages/CronjobCreateModal/index.tsx @@ -91,10 +91,6 @@ interface EnvVarsData { interface TagsData { name: string; -} - -interface Tags { - name: string; key: number; } @@ -111,7 +107,6 @@ interface Cronjob { newEnvVarValue: string; newEnvVarMasked: boolean; newTagName: string; - newTags: Tags[]; } interface OptionDataRepeat { @@ -193,7 +188,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro newEnvVarValue: "", newEnvVarMasked: false, newTagName: "", - newTags: [], }); const [crontab, setCrontab] = useState({ expression: "", @@ -365,17 +359,20 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, args: [...args] }); }; - const addArgument = (): void => { + const addArgument = (): ArgsData | undefined => { const args = [...cronjobData.args]; const newArgName = newCronjob.newArgName.trim(); const newArgValue = newCronjob.newArgValue.trim(); if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - args.push({ name: newArgName, value: newArgValue, key: countKey }); + const arg = { name: newArgName, value: newArgValue, key: countKey }; + args.push(arg); setCountKey(countKey + 1); setCronjobData({ ...cronjobData, args: [...args] }); setNewCronjob({ ...newCronjob, newArgName: "", newArgValue: "" }); + return arg; } else { invalidDataNotification("Invalid argument name/value pair."); + return; } }; @@ -414,22 +411,25 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro } }; - const addEnvVar = (): void => { + const addEnvVar = (): EnvVarsData | undefined => { const envVars = [...cronjobData.envVars]; const newEnvVarName = newCronjob.newEnvVarName.trim(); const newEnvVarValue = newCronjob.newEnvVarValue.trim(); if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - envVars.push({ + const envVar = { name: newEnvVarName, value: newEnvVarValue, masked: newCronjob.newEnvVarMasked, key: countKey, - }); + }; + envVars.push(envVar); setCountKey(countKey + 1); setCronjobData({ ...cronjobData, envVars: [...envVars] }); setNewCronjob({ ...newCronjob, newEnvVarName: "", newEnvVarValue: "", newEnvVarMasked: false }); + return envVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); + return; } }; @@ -439,18 +439,21 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, tags: [...tags] }); }; - const addTag = (): void => { + const addTag = (): TagsData | undefined => { const tags = [...cronjobData.tags]; - const newTags = [...newCronjob.newTags]; + // const newTags = [...newCronjob.newTags]; const newTagName = newCronjob.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: countKey }); + // newTags.push({ name: newTagName, key: countKey }); + const tag = { name: newTagName, key: countKey }; setCountKey(countKey + 1); - tags.push({ name: newTagName }); + tags.push(tag); setCronjobData({ ...cronjobData, tags: [...tags] }); - setNewCronjob({ ...newCronjob, newTags: [...newTags], newTagName: "" }); + setNewCronjob({ ...newCronjob, newTagName: "" }); + return tag; } else { invalidDataNotification("Invalid tag name."); + return; } }; @@ -556,9 +559,35 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro expression = getExpression(); } - const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => { - return envVar; - }); + const newEnvVarName = newCronjob.newEnvVarName.trim(); + if (newEnvVarName && newEnvVarName.indexOf(" ") == -1) { + const newEnvVar = addEnvVar(); + if (!newEnvVar) { + return; + } + cronjobData.envVars.push(newEnvVar); + } + const newArgName = newCronjob.newArgName.trim(); + if (newArgName && newArgName.indexOf(" ") == -1) { + const newArg = addArgument(); + if (!newArg) { + return; + } + cronjobData.args.push(newArg); + } + const newTagName = newCronjob.newTagName.trim(); + if (newTagName && newTagName.indexOf(" ") == -1) { + const newTag = addTag(); + if (!newTag) { + return; + } + cronjobData.tags.push(newTag); + } + + // const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => { + // return envVar; + // }); + const envVarsData = [...projectEnvVars]; spiderEnvVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -567,7 +596,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro envVarsData.push(envVar); } }); - cronjobData.envVars.map((envVar: EnvVarsData) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -839,7 +867,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro - +

Tags

{cronjobData.tags.map((tag: TagsData, id) => ( @@ -870,7 +898,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro onClick={addTag} > - +

Select a period

diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index ab71a4e1..1100671e 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -297,11 +297,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const newArgName = variable.newArgName.trim(); const newArgValue = variable.newArgValue.trim(); if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - const arg = { - name: newArgName, - value: newArgValue, - key: countKey, - }; + const arg = { name: newArgName, value: newArgValue, key: countKey }; args.push(arg); setCountKey(countKey + 1); setJobData({ ...jobData, args: [...args] }); @@ -403,9 +399,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const { args, tags, dataStatus, dataExpiryDays } = jobData; const { pid, sid } = request; - const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => { - return envVar; - }); + const envVarsData = [...projectEnvVars]; spiderEnvVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { @@ -414,7 +408,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea envVarsData.push(envVar); } }); - jobData.envVars.map((envVar: EnvVarsData) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { diff --git a/estela-web/src/pages/ProjectCronJobListPage/index.tsx b/estela-web/src/pages/ProjectCronJobListPage/index.tsx index 76f5395c..c0fc376b 100644 --- a/estela-web/src/pages/ProjectCronJobListPage/index.tsx +++ b/estela-web/src/pages/ProjectCronJobListPage/index.tsx @@ -1,5 +1,5 @@ import React, { Component, ReactElement } from "react"; -import { Layout, Pagination, Row, Space, Table, Col, Button, Switch, Tag, message, ConfigProvider } from "antd"; +import { Layout, Pagination, Row, Space, Table, Col, Button, Switch, Tag, message } from "antd"; import { Link, RouteComponentProps } from "react-router-dom"; import "./styles.scss"; import history from "../../history"; @@ -24,6 +24,7 @@ import { RouteParams, } from "../../shared"; import CronjobCreateModal from "../CronjobCreateModal"; +import FolderDotted from "../../assets/icons/folderDotted.svg"; import { convertDateToString } from "../../utils"; const { Content } = Layout; @@ -68,6 +69,7 @@ interface ProjectCronJobListPageState { current: number; loading: boolean; page: number; + isEmpty: boolean; } export class ProjectCronJobListPage extends Component, ProjectCronJobListPageState> { @@ -81,6 +83,7 @@ export class ProjectCronJobListPage extends Component ( + + +

No jobs yet.

+ + ); + render(): JSX.Element { const { loadedCronjobs, cronjobs, selectedRows, count, current } = this.state; return ( @@ -354,8 +365,10 @@ export class ProjectCronJobListPage extends Component - -

No scheduled jobs yet.

}> + {this.state.isEmpty ? ( + {this.emptyText()} + ) : ( + - - + + )} - +

Tags

diff --git a/estela-web/src/pages/JobDetailPage/index.tsx b/estela-web/src/pages/JobDetailPage/index.tsx index 55324f42..2ade46a2 100644 --- a/estela-web/src/pages/JobDetailPage/index.tsx +++ b/estela-web/src/pages/JobDetailPage/index.tsx @@ -1122,16 +1122,16 @@ export class JobDetailPage extends Component, J padding: 0, }} centered - width={681} + width={460} open={modalClone} title={

NEW JOB

} onCancel={() => this.setState({ modalClone: false })} footer={null} > - + -

Spider

+

Spider

, J value={newEnvVarValue} onChange={this.handleInputChange} /> + + Masked + - + - + WELCOME SCRAPER! Start by creating a project to be able to deploy your spiders and start with your scraping. - + Remember to install the  {  to be able to deploy your spiders! - - estela - - {this.getFramework()} - - - - {this.renderNotificationIcon(path === "/notifications/inbox", news)} - - - -
- - -
{this.getUser()} - {this.getUserRole() !== "" && ( - - {this.getUserRole()} - - )} - - - - - - - - - ) : ( - <> - )} - +
+ +
+ + estela + + {this.getFramework()} + + + + {this.renderNotificationIcon(path === "/notifications/inbox", news)} + + + +
+ + +
{this.getUser()} + {this.getUserRole() !== "" && ( + + {this.getUserRole()} + + )} + + + + + + + + ); } } From 2950a45b89301d62552771a42b6ad16a740bc033 Mon Sep 17 00:00:00 2001 From: emegona Date: Tue, 19 Sep 2023 02:20:59 -0500 Subject: [PATCH 4/5] Refactor code --- .../src/pages/CronjobCreateModal/index.tsx | 68 +++++++++++-------- estela-web/src/pages/JobCreateModal/index.tsx | 63 +++++++++-------- 2 files changed, 73 insertions(+), 58 deletions(-) diff --git a/estela-web/src/pages/CronjobCreateModal/index.tsx b/estela-web/src/pages/CronjobCreateModal/index.tsx index 4607bbe2..43cab0bd 100644 --- a/estela-web/src/pages/CronjobCreateModal/index.tsx +++ b/estela-web/src/pages/CronjobCreateModal/index.tsx @@ -359,7 +359,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, args: [...args] }); }; - const addArgument = (): ArgsData | undefined => { + const addArgument = (): ArgsData | null => { const args = [...cronjobData.args]; const newArgName = newCronjob.newArgName.trim(); const newArgValue = newCronjob.newArgValue.trim(); @@ -372,7 +372,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro return arg; } else { invalidDataNotification("Invalid argument name/value pair."); - return; + return null; } }; @@ -411,7 +411,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro } }; - const addEnvVar = (): EnvVarsData | undefined => { + const addEnvVar = (): EnvVarsData | null => { const envVars = [...cronjobData.envVars]; const newEnvVarName = newCronjob.newEnvVarName.trim(); const newEnvVarValue = newCronjob.newEnvVarValue.trim(); @@ -429,7 +429,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro return envVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); - return; + return null; } }; @@ -439,7 +439,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setCronjobData({ ...cronjobData, tags: [...tags] }); }; - const addTag = (): TagsData | undefined => { + const addTag = (): TagsData | null => { const tags = [...cronjobData.tags]; const newTagName = newCronjob.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { @@ -451,7 +451,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro return tag; } else { invalidDataNotification("Invalid tag name."); - return; + return null; } }; @@ -539,8 +539,27 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro setNewCronjob({ ...newCronjob, newEnvVarMasked: checked }); }; + const addPendingFormElement = ( + elementName: string, + elementValue: string, + addElementFunction: callable, + elementList: Array, + ): boolean => { + elementName = elementName.trim(); + elementValue = elementValue.trim(); + if (elementName || elementValue) { + const element = addElementFunction(); + if (!element) { + return false; + } + elementList.push(element); + } + return true; + }; + const handleSubmit = (): void => { setLoading(true); + let expression = ""; if (schedulesFlag[0]) { if (crontab.expression == "") { @@ -557,29 +576,20 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro expression = getExpression(); } - const newEnvVarName = newCronjob.newEnvVarName.trim(); - if (newEnvVarName && newEnvVarName.indexOf(" ") == -1) { - const newEnvVar = addEnvVar(); - if (!newEnvVar) { - return; - } - cronjobData.envVars.push(newEnvVar); - } - const newArgName = newCronjob.newArgName.trim(); - if (newArgName && newArgName.indexOf(" ") == -1) { - const newArg = addArgument(); - if (!newArg) { - return; - } - cronjobData.args.push(newArg); - } - const newTagName = newCronjob.newTagName.trim(); - if (newTagName && newTagName.indexOf(" ") == -1) { - const newTag = addTag(); - if (!newTag) { - return; - } - cronjobData.tags.push(newTag); + if ( + !( + addPendingFormElement(newCronjob.newArgName, newCronjob.newArgValue, addArgument, cronjobData.args) && + addPendingFormElement( + newCronjob.newEnvVarName, + newCronjob.newEnvVarValue, + addEnvVar, + cronjobData.envVars, + ) && + addPendingFormElement(newCronjob.newTagName, "", addTag, cronjobData.tags) + ) + ) { + setLoading(false); + return; } const envVarsData = [...projectEnvVars]; diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index 2e691d89..3f5cf6df 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -293,7 +293,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea setJobData({ ...jobData, tags: [...tags] }); }; - const addArgument = (): ArgsData | undefined => { + const addArgument = (): ArgsData | null => { const args = [...jobData.args]; const newArgName = variable.newArgName.trim(); const newArgValue = variable.newArgValue.trim(); @@ -306,11 +306,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea return arg; } else { invalidDataNotification("Invalid argument name/value pair."); - return; + return null; } }; - const addEnvVar = (): EnvVarsData | undefined => { + const addEnvVar = (): EnvVarsData | null => { const envVars = [...jobData.envVars]; const newEnvVarName = variable.newEnvVarName.trim(); const newEnvVarValue = variable.newEnvVarValue.trim(); @@ -328,11 +328,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea return newEnvVar; } else { invalidDataNotification("Invalid environment variable name/value pair."); - return; + return null; } }; - const addTag = (): TagsData | undefined => { + const addTag = (): TagsData | null => { const tags = [...jobData.tags]; const newTagName = variable.newTagName.trim(); if (newTagName && newTagName.indexOf(" ") == -1) { @@ -344,7 +344,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea return tag; } else { invalidDataNotification("Invalid tag name."); - return; + return null; } }; @@ -370,31 +370,36 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea setVariable({ ...variable, newEnvVarMasked: checked }); }; - const handleSubmit = (): void => { - setLoading(true); - const newEnvVarName = variable.newEnvVarName.trim(); - if (newEnvVarName && newEnvVarName.indexOf(" ") == -1) { - const newEnvVar = addEnvVar(); - if (!newEnvVar) { - return; - } - jobData.envVars.push(newEnvVar); - } - const newArgName = variable.newArgName.trim(); - if (newArgName && newArgName.indexOf(" ") == -1) { - const newArg = addArgument(); - if (!newArg) { - return; + const addPendingFormElement = ( + elementName: string, + elementValue: string, + addElementFunction: callable, + elementList: Array, + ): boolean => { + elementName = elementName.trim(); + elementValue = elementValue.trim(); + if (elementName || elementValue) { + const element = addElementFunction(); + if (!element) { + return false; } - jobData.args.push(newArg); + elementList.push(element); } - const newTagName = variable.newTagName.trim(); - if (newTagName && newTagName.indexOf(" ") == -1) { - const newTag = addTag(); - if (!newTag) { - return; - } - jobData.tags.push(newTag); + return true; + }; + + const handleSubmit = (): void => { + setLoading(true); + + if ( + !( + addPendingFormElement(variable.newArgName, variable.newArgValue, addArgument, jobData.args) && + addPendingFormElement(variable.newEnvVarName, variable.newEnvVarValue, addEnvVar, jobData.envVars) && + addPendingFormElement(variable.newTagName, "", addTag, jobData.tags) + ) + ) { + setLoading(false); + return; } const { args, tags, dataStatus, dataExpiryDays } = jobData; From 739d4430593ef17dc6629d3ce8239ea800568def Mon Sep 17 00:00:00 2001 From: raymond1242 Date: Thu, 21 Sep 2023 21:29:11 -0500 Subject: [PATCH 5/5] Refactor clone modal component --- estela-api/api/serializers/job.py | 3 +- estela-api/api/serializers/job_specific.py | 5 + estela-api/api/utils.py | 2 +- estela-api/docs/api.yaml | 12 +- estela-web/src/pages/JobCreateModal/index.tsx | 54 +- estela-web/src/pages/JobDetailPage/index.tsx | 710 ++---------------- .../src/pages/ProjectJobListPage/index.tsx | 11 +- .../src/pages/SpiderDetailPage/index.tsx | 7 +- .../api/generated-api/models/SpiderJob.ts | 11 +- .../generated-api/models/SpiderJobEnvVar.ts | 4 +- .../generated-api/models/SpiderJobStats.ts | 11 +- 11 files changed, 159 insertions(+), 671 deletions(-) diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py index e7d17929..9cc0986c 100644 --- a/estela-api/api/serializers/job.py +++ b/estela-api/api/serializers/job.py @@ -7,6 +7,7 @@ SpiderJobEnvVarSerializer, SpiderJobTagSerializer, ) +from api.serializers.spider import SpiderSerializer from api.utils import delete_stats_from_redis, update_stats_from_redis from config.job_manager import job_manager from core.models import ( @@ -30,7 +31,7 @@ class SpiderJobSerializer(serializers.ModelSerializer): job_status = serializers.CharField( required=False, read_only=True, help_text="Current job status." ) - spider = serializers.SerializerMethodField("get_spider") + spider = SpiderSerializer(required=True, help_text="Job spider.") class Meta: model = SpiderJob diff --git a/estela-api/api/serializers/job_specific.py b/estela-api/api/serializers/job_specific.py index 415f6068..e84b9fcb 100644 --- a/estela-api/api/serializers/job_specific.py +++ b/estela-api/api/serializers/job_specific.py @@ -10,6 +10,11 @@ class Meta: class SpiderJobEnvVarSerializer(serializers.ModelSerializer): + masked = serializers.BooleanField( + required=True, + help_text="Whether the env variable value is masked.", + ) + class Meta: model = SpiderJobEnvVar fields = ("evid", "name", "value", "masked") diff --git a/estela-api/api/utils.py b/estela-api/api/utils.py index b74a2bb6..bd78ab6b 100644 --- a/estela-api/api/utils.py +++ b/estela-api/api/utils.py @@ -70,5 +70,5 @@ def delete_stats_from_redis(job): redis_conn = redis.from_url(settings.REDIS_URL) try: redis_conn.delete(f"scrapy_stats_{job.key}") - except: + except Exception: pass diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index dcfaef48..89f5524b 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -1709,6 +1709,7 @@ definitions: required: - name - value + - masked type: object properties: evid: @@ -2184,6 +2185,8 @@ definitions: minLength: 1 SpiderJob: description: Project jobs. + required: + - spider type: object properties: jid: @@ -2192,9 +2195,7 @@ definitions: type: integer readOnly: true spider: - title: Spider - type: string - readOnly: true + $ref: '#/definitions/Spider' created: title: Created description: Job creation date. @@ -2791,6 +2792,7 @@ definitions: $ref: '#/definitions/Stats' SpiderJobStats: required: + - spider - stats type: object properties: @@ -2800,9 +2802,7 @@ definitions: type: integer readOnly: true spider: - title: Spider - type: string - readOnly: true + $ref: '#/definitions/Spider' created: title: Created description: Job creation date. diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index 3f5cf6df..5a02932b 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -27,6 +27,10 @@ interface JobCreateModalProps { openModal: boolean; spider: Spider | null; projectId: string; + envVarsProps: SpiderJobEnvVar[]; + argsProps: ArgsData[]; + tagsProps: TagsData[]; + children: React.ReactNode; } interface MaskedTagProps { @@ -41,20 +45,14 @@ interface ArgsData { key: number; } -interface EnvVarsData { - name: string; - value: string; - key: number; - masked: boolean; -} - interface TagsData { name: string; + key: number; } interface JobData { args: ArgsData[]; - envVars: EnvVarsData[]; + envVars: SpiderJobEnvVar[]; tags: TagsData[]; dataStatus: SpiderDataStatusEnum | undefined; dataExpiryDays: number | null | undefined; @@ -90,7 +88,15 @@ const dataPersistenceOptions = [ { label: "Forever", key: 7, value: 720 }, ]; -export default function JobCreateModal({ openModal, spider, projectId }: JobCreateModalProps) { +export default function JobCreateModal({ + openModal, + spider, + projectId, + envVarsProps, + argsProps, + tagsProps, + children, +}: JobCreateModalProps) { const PAGE_SIZE = 15; const apiService = ApiService(); const [open, setOpen] = useState(openModal); @@ -99,9 +105,9 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const [spiders, setSpiders] = useState([]); const [externalComponent, setExternalComponent] = useState(<>); const [jobData, setJobData] = useState({ - args: [], - envVars: [], - tags: [], + args: argsProps, + envVars: envVarsProps, + tags: tagsProps, dataStatus: spider ? spider.dataStatus : undefined, dataExpiryDays: spider ? spider.dataExpiryDays : 1, }); @@ -310,7 +316,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea } }; - const addEnvVar = (): EnvVarsData | null => { + const addEnvVar = (): SpiderJobEnvVar | null => { const envVars = [...jobData.envVars]; const newEnvVarName = variable.newEnvVarName.trim(); const newEnvVarValue = variable.newEnvVarValue.trim(); @@ -319,7 +325,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea name: newEnvVarName, value: newEnvVarValue, masked: variable.newEnvVarMasked, - key: countKey, }; envVars.push(newEnvVar); setCountKey(countKey + 1); @@ -373,8 +378,8 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea const addPendingFormElement = ( elementName: string, elementValue: string, - addElementFunction: callable, - elementList: Array, + addElementFunction: CallableFunction, + elementList: Array, ): boolean => { elementName = elementName.trim(); elementValue = elementValue.trim(); @@ -414,10 +419,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea envVarsData.push(envVar); } }); - jobData.envVars.map((envVar: EnvVarsData) => { + jobData.envVars.map((envVar: SpiderJobEnvVar) => { const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name); if (index != -1) { envVarsData[index] = { + evid: envVar.evid, name: envVar.name, value: envVar.value, masked: envVar.masked, @@ -443,10 +449,12 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea pid: pid, sid: sid, }; + apiService.apiProjectsSpidersJobsCreate(requests).then( (response: SpiderJobCreate) => { setLoading(false); history.push(`/projects/${pid}/spiders/${sid}/jobs/${response.jid}`); + location.reload(); }, async (error) => { setLoading(false); @@ -478,7 +486,7 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea } }} > - Run new job + {children} {externalComponent} {projectEnvVars.length > 0 && ( -

Project

+

Project

{projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( @@ -593,8 +601,8 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea )} {spiderEnvVars.length > 0 && ( - <> -

Spider

+ +

Spider

{spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( @@ -613,10 +621,10 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea ), )}
- + )} - {jobData.envVars.map((envVar: EnvVarsData, id: number) => + {jobData.envVars.map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( , J dataStatus: "", dataExpiryDays: 0, loading_status: false, - modified: false, modalStop: false, modalClone: false, - spiders: [], - loadedSpiders: false, spiderName: "", - newSpiderId: "", - newDataExpireDays: 1, - newDataStatus: "PENDING", newTags: [], - newTagName: "", newArgs: [], - newArgName: "", - newArgValue: "", newEnvVars: [], - newEnvVarName: "", - newEnvVarValue: "", - newEnvVarMasked: false, }; apiService = ApiService(); projectId: string = this.props.match.params.projectId; @@ -289,16 +224,35 @@ export class JobDetailPage extends Component, J this.apiService.apiProjectsSpidersJobsRead(requestParams).then( async (response: SpiderJob) => { const args = response.args || []; + const newArgs = args.map((arg: SpiderJobArg, id: number) => { + return { name: arg.name, value: arg.value, key: id }; + }); const envVars = response.envVars || []; + const newEnvVars = envVars.map((envVar: SpiderJobEnvVar, id: number) => { + return { + evid: envVar.evid, + name: envVar.name, + value: envVar.masked ? "__MASKED__" : envVar.value, + masked: envVar.masked, + key: id, + }; + }); const tags = response.tags || []; + const newTags = tags.map((tag: SpiderJobTag, id: number) => { + return { name: tag.name, key: id }; + }); const lifespan = response.lifespan; this.setState({ name: response.name, lifespan: lifespan, totalResponseBytes: response.totalResponseBytes, args: [...args], + newArgs: [...newArgs], envVars: [...envVars], + newEnvVars: [...newEnvVars], tags: [...tags], + newTags: [...newTags], + spiderName: response.spider.name, date: convertDateToString(response.created), created: `${response.created}`, status: response.jobStatus, @@ -313,244 +267,10 @@ export class JobDetailPage extends Component, J resourceNotAllowedNotification(); }, ); - this.getProjectSpiders(); await this.getItems(1); this.setState({ loadedItemsFirstTime: true }); } - getProjectSpiders = async (): Promise => { - const requestParams: ApiProjectsSpidersListRequest = { pid: this.projectId }; - this.apiService.apiProjectsSpidersList(requestParams).then( - (results) => { - if (results.results.length == 0 || results.results == undefined) { - this.setState({ spiders: [], loadedSpiders: true }); - } else { - const spiderName = results.results.find( - (spider: Spider) => String(spider?.sid) === this.spiderId, - )?.name; - this.setState({ - spiders: [...results.results], - spiderName: spiderName || "", - newSpiderId: String(results.results[0].sid), - loadedSpiders: true, - }); - } - }, - (error: unknown) => { - error; - resourceNotAllowedNotification(); - }, - ); - }; - - getData = async (): Promise => { - const requestParams: ApiProjectsSpidersJobsDataListRequest = { - pid: this.projectId, - sid: this.spiderId, - jid: `${this.jobId}`, - }; - this.apiService.apiProjectsSpidersList(requestParams).then( - (results) => { - if (results.results.length == 0 || results.results == undefined) { - this.setState({ spiders: [], loadedSpiders: true }); - } else { - const spiderName = results.results.find( - (spider: Spider) => String(spider?.sid) === this.spiderId, - )?.name; - this.setState({ - spiders: [...results.results], - spiderName: spiderName || "", - newSpiderId: String(results.results[0].sid), - loadedSpiders: true, - }); - } - }, - (error: unknown) => { - error; - resourceNotAllowedNotification(); - }, - ); - }; - - handlePersistenceChange = (value: number): void => { - if (value == 720) { - this.setState({ newDataStatus: "PERSISTENT" }); - } else { - this.setState({ newDataExpireDays: value }); - } - }; - - handleRemoveTag = (id: number): void => { - const newTags = [...this.state.newTags]; - newTags.splice(id, 1); - this.setState({ newTags: [...newTags] }); - }; - - handleRemoveArg = (id: number): void => { - const newArgs = [...this.state.newArgs]; - newArgs.splice(id, 1); - this.setState({ newArgs: [...newArgs] }); - }; - - handleRemoveEnvVar = (id: number): void => { - const newEnvVars = [...this.state.newEnvVars]; - newEnvVars.splice(id, 1); - this.setState({ newEnvVars: [...newEnvVars] }); - }; - - onChangeEnvVarMasked = (e: CheckboxChangeEvent) => { - this.setState({ newEnvVarMasked: e.target.checked }); - }; - - handleInputChange = (event: React.ChangeEvent): void => { - const { - target: { value, name }, - } = event; - if (name === "newArgName") { - this.setState({ newArgName: value }); - } else if (name === "newArgValue") { - this.setState({ newArgValue: value }); - } else if (name === "newEnvVarName") { - this.setState({ newEnvVarName: value }); - } else if (name === "newEnvVarValue") { - this.setState({ newEnvVarValue: value }); - } else if (name === "newTagName") { - this.setState({ newTagName: value }); - } - }; - - handleSubmit = (): void => { - const futureDate: Date = new Date(); - futureDate.setDate(futureDate.getDate() + this.state.newDataExpireDays); - const requestData = { - args: [...this.state.newArgs], - envVars: [...this.state.newEnvVars], - tags: [...this.state.newTags], - dataStatus: this.state.newDataStatus, - dataExpiryDays: this.state.newDataExpireDays, - }; - const request: ApiProjectsSpidersJobsCreateRequest = { - data: requestData, - pid: this.projectId, - sid: this.state.newSpiderId, - }; - this.apiService.apiProjectsSpidersJobsCreate(request).then( - (response: SpiderJobCreate) => { - history.push(`/projects/${this.projectId}/spiders/${this.state.newSpiderId}/jobs/${response.jid}`); - location.reload(); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - - handleSpiderChange = (value: string): void => { - const spiderId = this.state.spiders.find((spider) => { - return spider.name === value; - }); - this.setState({ newSpiderId: String(spiderId?.sid) }); - }; - - addTag = (): void => { - const newTags = [...this.state.newTags]; - const newTagName = this.state.newTagName.trim(); - if (newTagName && newTagName.indexOf(" ") == -1) { - newTags.push({ name: newTagName, key: this.countKey++ }); - this.setState({ newTags: [...newTags], newTagName: "" }); - } else { - invalidDataNotification("Invalid tag name."); - } - }; - - addArgument = (): void => { - const newArgs = [...this.state.newArgs]; - const newArgName = this.state.newArgName.trim(); - const newArgValue = this.state.newArgValue.trim(); - if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) { - newArgs.push({ name: newArgName, value: newArgValue, key: this.countKey++ }); - this.setState({ newArgs: [...newArgs], newArgName: "", newArgValue: "" }); - } else { - invalidDataNotification("Invalid argument name/value pair."); - } - }; - - addEnvVar = (): void => { - const newEnvVars = [...this.state.newEnvVars]; - const newEnvVarName = this.state.newEnvVarName.trim(); - const newEnvVarValue = this.state.newEnvVarValue.trim(); - const newEnvVarMasked = this.state.newEnvVarMasked; - if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) { - newEnvVars.push({ - name: newEnvVarName, - value: newEnvVarValue, - masked: newEnvVarMasked, - key: this.countKey++, - }); - this.setState({ - newEnvVars: [...newEnvVars], - newEnvVarName: "", - newEnvVarValue: "", - newEnvVarMasked: false, - }); - } else { - invalidDataNotification("Invalid environment variable name/value pair."); - } - }; - - stopJob = (): void => { - const request: ApiProjectsSpidersJobsUpdateRequest = { - jid: this.jobId, - sid: this.spiderId, - pid: this.projectId, - data: { - jid: this.jobId, - status: SpiderJobUpdateStatusEnum.Stopped, - }, - }; - this.apiService.apiProjectsSpidersJobsUpdate(request).then( - (response: SpiderJobUpdate) => { - this.setState({ status: response.status }); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - - updateDataExpiry = (): void => { - this.setState({ loading_status: true }); - const requestData: SpiderJobUpdate = { - dataStatus: - this.state.dataStatus == SpiderJobUpdateDataStatusEnum.Persistent - ? this.state.dataStatus - : SpiderJobUpdateDataStatusEnum.Pending, - dataExpiryDays: this.state.dataExpiryDays, - }; - const request: ApiProjectsSpidersJobsUpdateRequest = { - jid: this.jobId, - pid: this.projectId, - sid: this.spiderId, - data: requestData, - }; - this.apiService.apiProjectsSpidersJobsUpdate(request).then( - (response: SpiderJobUpdate) => { - this.setState({ - dataStatus: response.dataStatus, - dataExpiryDays: response.dataExpiryDays == null ? 1 : response.dataExpiryDays, - modified: false, - loading_status: false, - }); - }, - (error: unknown) => { - error; - incorrectDataNotification(); - }, - ); - }; - updateStatus = (_status: SpiderJobUpdateStatusEnum): void => { this.setState({ loading_status: true }); const request: ApiProjectsSpidersJobsUpdateRequest = { @@ -572,27 +292,6 @@ export class JobDetailPage extends Component, J ); }; - onChangeData = (): void => { - const _dataStatus = - this.state.dataStatus == SpiderJobUpdateDataStatusEnum.Persistent - ? SpiderJobUpdateDataStatusEnum.Pending - : SpiderJobUpdateDataStatusEnum.Persistent; - this.setState({ dataStatus: _dataStatus, modified: true }); - }; - - onChangeDay = (value: number): void => { - this.setState({ dataExpiryDays: value, modified: true }); - }; - - disabledDate: RangePickerProps["disabledDate"] = (current) => { - return current && current < moment().endOf("day"); - }; - - onChangeDate: DatePickerProps["onChange"] = (date) => { - const days = moment.duration(moment(date, "llll").diff(moment(this.state.created, "llll"))).asDays(); - this.setState({ dataExpiryDays: days, modified: true }); - }; - getItems = async (page: number, pageSize?: number): Promise => { const requestParams: ApiProjectsSpidersJobsDataListRequest = { pid: this.projectId, @@ -774,9 +473,9 @@ export class JobDetailPage extends Component, J
Tags - - - {tags.map((tag: TagsData, id) => ( + + + {tags.map((tag: SpiderJobTag, id) => ( , J Arguments - - - {args.map((arg: ArgsData, id) => ( + + + {args.map((arg: SpiderJobArg, id) => ( , J render(): JSX.Element { const { loaded, - newTagName, - status, modalStop, - modalClone, - spiders, activeKey, - loadedSpiders, newTags, newArgs, - newArgName, - newArgValue, newEnvVars, - newEnvVarName, - newEnvVarValue, - newEnvVarMasked, itemsCurrent, loadedItems, loadedItemsFirstTime, + status, } = this.state; return ( - {loaded && loadedSpiders ? ( + {loaded ? ( @@ -1027,42 +717,16 @@ export class JobDetailPage extends Component, J - + - {modalStop && ( - CONFIRM ACTION

- } - onCancel={() => this.setState({ modalStop: false })} - footer={null} - > - - - Are you sure you want to stop this job? - - - - - - - - )} - {modalClone && ( - NEW JOB

} - onCancel={() => this.setState({ modalClone: false })} - footer={null} - > - - - -

Spider

- - - -

Data persistence

- - - -

Tags

- - {newTags.map((tag: Tags, id: number) => { - return ( - this.handleRemoveTag(id)} - > - {tag.name} - - ); - })} - - - - - - - -

Arguments

- - {newArgs.map((arg: Args, id) => ( - this.handleRemoveArg(id)} - > - {arg.name}: {arg.value} - - ))} - - - - - - - - -

Environment Variables

- - {newEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - - this.handleRemoveEnvVar(id)} - className="environment-variables" - key={id} - > - {envVar.name} - - - ) : ( - this.handleRemoveEnvVar(id)} - className="environment-variables" - key={id} - > - {envVar.name}: {envVar.value} - - ), - )} - - - - - - Masked - - - - - - - - - - - - )} + CONFIRM ACTION

} + onCancel={() => this.setState({ modalStop: false })} + footer={null} + > + + + Are you sure you want to stop this job? + + + + + + + diff --git a/estela-web/src/pages/ProjectJobListPage/index.tsx b/estela-web/src/pages/ProjectJobListPage/index.tsx index da0a4210..e93604c7 100644 --- a/estela-web/src/pages/ProjectJobListPage/index.tsx +++ b/estela-web/src/pages/ProjectJobListPage/index.tsx @@ -284,7 +284,16 @@ export class ProjectJobListPage extends Component - + + Run new job + diff --git a/estela-web/src/pages/SpiderDetailPage/index.tsx b/estela-web/src/pages/SpiderDetailPage/index.tsx index fcc8694c..9fd0024a 100644 --- a/estela-web/src/pages/SpiderDetailPage/index.tsx +++ b/estela-web/src/pages/SpiderDetailPage/index.tsx @@ -961,7 +961,12 @@ export class SpiderDetailPage extends Component openModal={false} spider={this.state.spider} projectId={this.projectId} - /> + argsProps={[]} + envVarsProps={[]} + tagsProps={[]} + > + Run this spider +