Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

estela-web: Automatically try to add arguments, env vars, and tags written in creation forms #226

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

estela-web: Automatically try to add arguments, env vars, and tags written in creation forms #226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b5ba73d
Adding automatically envvar and variables in job create modal
Aug 27, 2023
9cdd310
Adding cautomatically variables in cronjob create modal
Aug 27, 2023
7c8fdfd
fix clon job modal, fix style and header no load bug
Aug 28, 2023
2950a45
Refactor code
mgonnav Sep 19, 2023
739d443
Refactor clone modal component
raymond1242 Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion estela-api/api/serializers/job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
SpiderJobEnvVarSerializer,
SpiderJobTagSerializer,
)
from api.serializers.spider import SpiderSerializer
from api.utils import delete_stats_from_redis, update_stats_from_redis
from config.job_manager import job_manager
from core.models import (
Expand All @@ -30,7 +31,7 @@ class SpiderJobSerializer(serializers.ModelSerializer):
job_status = serializers.CharField(
required=False, read_only=True, help_text="Current job status."
)
spider = serializers.SerializerMethodField("get_spider")
spider = SpiderSerializer(required=True, help_text="Job spider.")

class Meta:
model = SpiderJob
Expand Down
5 changes: 5 additions & 0 deletions estela-api/api/serializers/job_specific.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ class Meta:


class SpiderJobEnvVarSerializer(serializers.ModelSerializer):
masked = serializers.BooleanField(
required=True,
help_text="Whether the env variable value is masked.",
)

class Meta:
model = SpiderJobEnvVar
fields = ("evid", "name", "value", "masked")
Expand Down
2 changes: 1 addition & 1 deletion estela-api/api/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,5 @@ def delete_stats_from_redis(job):
redis_conn = redis.from_url(settings.REDIS_URL)
try:
redis_conn.delete(f"scrapy_stats_{job.key}")
except:
except Exception:
pass
12 changes: 6 additions & 6 deletions estela-api/docs/api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1709,6 +1709,7 @@ definitions:
required:
- name
- value
- masked
type: object
properties:
evid:
Expand Down Expand Up @@ -2184,6 +2185,8 @@ definitions:
minLength: 1
SpiderJob:
description: Project jobs.
required:
- spider
type: object
properties:
jid:
Expand All @@ -2192,9 +2195,7 @@ definitions:
type: integer
readOnly: true
spider:
title: Spider
type: string
readOnly: true
$ref: '#/definitions/Spider'
created:
title: Created
description: Job creation date.
Expand Down Expand Up @@ -2791,6 +2792,7 @@ definitions:
$ref: '#/definitions/Stats'
SpiderJobStats:
required:
- spider
- stats
type: object
properties:
Expand All @@ -2800,9 +2802,7 @@ definitions:
type: integer
readOnly: true
spider:
title: Spider
type: string
readOnly: true
$ref: '#/definitions/Spider'
created:
title: Created
description: Job creation date.
Expand Down
4 changes: 2 additions & 2 deletions estela-web/src/components/EnvVarsSettingsPage/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ export const EnvVarsSetting: React.FC<ProjectEnvVar> = ({ projectId, spiderId, e
setNewEnvVarValue("");
setNewEnvVarMasked(false);
setOpenModal(false);
setActiveUpdateButton(true);
updateEnvVars();
} else {
invalidDataNotification("Invalid environment variable name/value pair.");
}
Expand Down Expand Up @@ -394,7 +394,7 @@ export const EnvVarsSetting: React.FC<ProjectEnvVar> = ({ projectId, spiderId, e
onClick={() => updateEnvVars()}
className="border-estela bg-estela hover:border-estela hover:text-estela text-white rounded-md text-base h-full"
>
Save variables
Save changes
</Button>
</div>
</Space>
Expand Down
168 changes: 102 additions & 66 deletions estela-web/src/pages/CronjobCreateModal/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,6 @@ interface EnvVarsData {

interface TagsData {
name: string;
}

interface Tags {
name: string;
key: number;
}

Expand All @@ -111,7 +107,6 @@ interface Cronjob {
newEnvVarValue: string;
newEnvVarMasked: boolean;
newTagName: string;
newTags: Tags[];
}

interface OptionDataRepeat {
Expand Down Expand Up @@ -193,7 +188,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
newEnvVarValue: "",
newEnvVarMasked: false,
newTagName: "",
newTags: [],
});
const [crontab, setCrontab] = useState<Crontab>({
expression: "",
Expand Down Expand Up @@ -365,17 +359,20 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
setCronjobData({ ...cronjobData, args: [...args] });
};

const addArgument = (): void => {
const addArgument = (): ArgsData | null => {
const args = [...cronjobData.args];
const newArgName = newCronjob.newArgName.trim();
const newArgValue = newCronjob.newArgValue.trim();
if (newArgName && newArgValue && newArgName.indexOf(" ") == -1) {
args.push({ name: newArgName, value: newArgValue, key: countKey });
const arg = { name: newArgName, value: newArgValue, key: countKey };
args.push(arg);
setCountKey(countKey + 1);
setCronjobData({ ...cronjobData, args: [...args] });
setNewCronjob({ ...newCronjob, newArgName: "", newArgValue: "" });
return arg;
} else {
invalidDataNotification("Invalid argument name/value pair.");
return null;
}
};

Expand Down Expand Up @@ -414,22 +411,25 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
}
};

const addEnvVar = (): void => {
const addEnvVar = (): EnvVarsData | null => {
const envVars = [...cronjobData.envVars];
const newEnvVarName = newCronjob.newEnvVarName.trim();
const newEnvVarValue = newCronjob.newEnvVarValue.trim();
if (newEnvVarName && newEnvVarValue && newEnvVarName.indexOf(" ") == -1) {
envVars.push({
const envVar = {
name: newEnvVarName,
value: newEnvVarValue,
masked: newCronjob.newEnvVarMasked,
key: countKey,
});
};
envVars.push(envVar);
setCountKey(countKey + 1);
setCronjobData({ ...cronjobData, envVars: [...envVars] });
setNewCronjob({ ...newCronjob, newEnvVarName: "", newEnvVarValue: "", newEnvVarMasked: false });
return envVar;
} else {
invalidDataNotification("Invalid environment variable name/value pair.");
return null;
}
};

Expand All @@ -439,18 +439,19 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
setCronjobData({ ...cronjobData, tags: [...tags] });
};

const addTag = (): void => {
const addTag = (): TagsData | null => {
const tags = [...cronjobData.tags];
const newTags = [...newCronjob.newTags];
const newTagName = newCronjob.newTagName.trim();
if (newTagName && newTagName.indexOf(" ") == -1) {
newTags.push({ name: newTagName, key: countKey });
const tag = { name: newTagName, key: countKey };
setCountKey(countKey + 1);
tags.push({ name: newTagName });
tags.push(tag);
setCronjobData({ ...cronjobData, tags: [...tags] });
setNewCronjob({ ...newCronjob, newTags: [...newTags], newTagName: "" });
setNewCronjob({ ...newCronjob, newTagName: "" });
return tag;
} else {
invalidDataNotification("Invalid tag name.");
return null;
}
};

Expand Down Expand Up @@ -538,8 +539,27 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
setNewCronjob({ ...newCronjob, newEnvVarMasked: checked });
};

const addPendingFormElement = (
elementName: string,
elementValue: string,
addElementFunction: callable,
elementList: Array,
): boolean => {
elementName = elementName.trim();
elementValue = elementValue.trim();
if (elementName || elementValue) {
const element = addElementFunction();
if (!element) {
return false;
}
elementList.push(element);
}
return true;
};

const handleSubmit = (): void => {
setLoading(true);

let expression = "";
if (schedulesFlag[0]) {
if (crontab.expression == "") {
Expand All @@ -556,9 +576,23 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
expression = getExpression();
}

const envVarsData = projectEnvVars.map((envVar: SpiderJobEnvVar) => {
return envVar;
});
if (
!(
addPendingFormElement(newCronjob.newArgName, newCronjob.newArgValue, addArgument, cronjobData.args) &&
addPendingFormElement(
newCronjob.newEnvVarName,
newCronjob.newEnvVarValue,
addEnvVar,
cronjobData.envVars,
) &&
addPendingFormElement(newCronjob.newTagName, "", addTag, cronjobData.tags)
)
) {
setLoading(false);
return;
}

const envVarsData = [...projectEnvVars];
spiderEnvVars.map((envVar: SpiderJobEnvVar) => {
const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name);
if (index != -1) {
Expand All @@ -567,7 +601,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
envVarsData.push(envVar);
}
});

cronjobData.envVars.map((envVar: EnvVarsData) => {
const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name);
if (index != -1) {
Expand All @@ -589,7 +622,6 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
cronjobData.dataStatus == SpiderDataStatusEnum.Persistent
? SpiderCronJobCreateDataStatusEnum.Persistent
: SpiderCronJobCreateDataStatusEnum.Pending;

const requestData = {
cargs: [...cronjobData.args],
cenvVars: [...envVarsData],
Expand Down Expand Up @@ -743,48 +775,52 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
<Content>
<p className="text-base my-2">Environment Variables</p>
<Space direction="vertical" className="flex">
<div className="flex">
{projectEnvVars.length > 0 ? <p className="text-sm mr-2">Project:</p> : <></>}
<div className="flex gap-2">
{projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
envVar.masked ? (
<MaskedTag key={id} id={id} level={true}>
{envVar.name}
</MaskedTag>
) : (
<Tag
className="text-estela-blue-full border-0 bg-estela-blue-low"
closable
key={id}
onClose={() => handleRemoveProjectEnvVar(id, true)}
>
{envVar.name}: {envVar.value}
</Tag>
),
)}
</div>
</div>
<div className="flex">
{spiderEnvVars.length > 0 ? <p className="text-sm mr-2">Spider:</p> : <></>}
<div className="flex gap-2">
{spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
envVar.masked ? (
<MaskedTag key={id} id={id} level={false}>
{envVar.name}
</MaskedTag>
) : (
<Tag
className="text-estela-blue-full border-0 bg-estela-blue-low"
closable
key={id}
onClose={() => handleRemoveProjectEnvVar(id, false)}
>
{envVar.name}: {envVar.value}
</Tag>
),
)}
</div>
</div>
{projectEnvVars.length > 0 && (
<Content className="flex">
<p className="text-sm mr-2">Project:</p>
<div className="flex gap-2">
{projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
envVar.masked ? (
<MaskedTag key={id} id={id} level={true}>
{envVar.name}
</MaskedTag>
) : (
<Tag
className="text-estela-blue-full border-0 bg-estela-blue-low"
closable
key={id}
onClose={() => handleRemoveProjectEnvVar(id, true)}
>
{envVar.name}: {envVar.value}
</Tag>
),
)}
</div>
</Content>
)}
{spiderEnvVars.length > 0 && (
<Content className="flex">
<p className="text-sm mr-2">Spider:</p>
<div className="flex gap-2">
{spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) =>
envVar.masked ? (
<MaskedTag key={id} id={id} level={false}>
{envVar.name}
</MaskedTag>
) : (
<Tag
className="text-estela-blue-full border-0 bg-estela-blue-low"
closable
key={id}
onClose={() => handleRemoveProjectEnvVar(id, false)}
>
{envVar.name}: {envVar.value}
</Tag>
),
)}
</div>
</Content>
)}
<Space direction="horizontal">
{cronjobData.envVars.map((envVar: EnvVarsData, id: number) =>
envVar.masked ? (
Expand Down Expand Up @@ -839,8 +875,8 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
</Space>
</Space>
</Content>
<Content>
<p className="text-base my-2">Tags</p>
<Space direction="vertical" className="my-2">
<p className="text-base">Tags</p>
<Space direction="horizontal">
{cronjobData.tags.map((tag: TagsData, id) => (
<Tag
Expand Down Expand Up @@ -870,7 +906,7 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
onClick={addTag}
></Button>
</Space>
</Content>
</Space>
</Col>
<Col className="schedule mx-4">
<p className="text-base">Select a period</p>
Expand Down
Loading
Loading