From d14d7b37de7d45eab0d14967206d861033c86818 Mon Sep 17 00:00:00 2001 From: Raymond Negron Valqui <33165861+raymond1242@users.noreply.github.com> Date: Tue, 11 Apr 2023 02:23:44 -0500 Subject: [PATCH] BITMAKER-2542 Allow setting DataPersistence field on Project/Spider models (#150) --------- Co-authored-by: Raymond Negron Co-authored-by: emegona --- estela-api/api/serializers/cronjob.py | 21 +- estela-api/api/serializers/deploy.py | 7 +- estela-api/api/serializers/job.py | 18 +- estela-api/api/serializers/project.py | 38 +- estela-api/api/serializers/spider.py | 31 +- estela-api/api/views/cronjob.py | 8 +- estela-api/api/views/job.py | 6 +- estela-api/api/views/project.py | 12 +- estela-api/api/views/spider.py | 31 +- .../migrations/0026_auto_20230411_0631.py | 85 +++ estela-api/core/models.py | 65 ++- estela-api/core/tasks.py | 49 +- estela-api/docs/api.yaml | 111 +++- estela-web/src/components/LoginPage/index.tsx | 13 +- .../ProjectCronJobListPage/index.tsx | 29 +- .../components/ProjectJobListPage/index.tsx | 46 +- .../src/components/ProjectListPage/index.tsx | 2 +- .../components/ProjectSettingsPage/index.tsx | 489 +++++++++--------- .../src/components/SpiderDetailPage/index.tsx | 112 +++- .../src/components/SpiderListPage/index.tsx | 6 +- .../generated-api/.openapi-generator/FILES | 1 + .../services/api/generated-api/apis/ApiApi.ts | 99 ++++ .../api/generated-api/models/Project.ts | 23 + .../api/generated-api/models/ProjectUpdate.ts | 23 + .../api/generated-api/models/Spider.ts | 25 + .../api/generated-api/models/SpiderCronJob.ts | 3 +- .../models/SpiderCronJobUpdate.ts | 3 +- .../api/generated-api/models/SpiderJob.ts | 6 +- .../generated-api/models/SpiderJobUpdate.ts | 6 +- .../api/generated-api/models/SpiderUpdate.ts | 89 ++++ .../api/generated-api/models/index.ts | 1 + 31 files changed, 1064 insertions(+), 394 deletions(-) create mode 100644 estela-api/core/migrations/0026_auto_20230411_0631.py create mode 100644 estela-web/src/services/api/generated-api/models/SpiderUpdate.ts diff --git a/estela-api/api/serializers/cronjob.py b/estela-api/api/serializers/cronjob.py index 4f7a11ef..ae9049d2 100644 --- a/estela-api/api/serializers/cronjob.py +++ b/estela-api/api/serializers/cronjob.py @@ -1,15 +1,20 @@ from croniter import croniter from rest_framework import serializers -from api import errors - -from core.models import SpiderJobArg, SpiderJobEnvVar, SpiderCronJob, SpiderJobTag +from api import errors from api.serializers.job_specific import ( SpiderJobArgSerializer, SpiderJobEnvVarSerializer, SpiderJobTagSerializer, ) -from core.cronjob import enable_cronjob, disable_cronjob, update_schedule +from core.cronjob import disable_cronjob, enable_cronjob, update_schedule +from core.models import ( + DataStatus, + SpiderCronJob, + SpiderJobArg, + SpiderJobEnvVar, + SpiderJobTag, +) class SpiderCronJobSerializer(serializers.ModelSerializer): @@ -144,10 +149,10 @@ def update(self, instance, validated_data): if "unique_collection" in validated_data: instance.unique_collection = unique_collection if "data_status" in validated_data: - if data_status == SpiderCronJob.PERSISTENT_STATUS: - instance.data_status = SpiderCronJob.PERSISTENT_STATUS - elif data_status == SpiderCronJob.PENDING_STATUS: - instance.data_status = SpiderCronJob.PENDING_STATUS + if data_status == DataStatus.PERSISTENT_STATUS: + instance.data_status = DataStatus.PERSISTENT_STATUS + elif data_status == DataStatus.PENDING_STATUS: + instance.data_status = DataStatus.PENDING_STATUS if data_expiry_days < 1: raise serializers.ValidationError( {"error": errors.POSITIVE_SMALL_INTEGER_FIELD} diff --git a/estela-api/api/serializers/deploy.py b/estela-api/api/serializers/deploy.py index de7322be..7ca8bc40 100644 --- a/estela-api/api/serializers/deploy.py +++ b/estela-api/api/serializers/deploy.py @@ -59,7 +59,12 @@ def update(self, instance, validated_data): ) project.spiders.exclude(name__in=spiders_names).update(deleted=True) new_spiders = [ - Spider(name=spider_name, project=project) + Spider( + name=spider_name, + project=project, + data_status=project.data_status, + data_expiry_days=project.data_expiry_days, + ) for spider_name in spiders_names if not project.spiders.filter(name=spider_name).exists() ] diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py index cfda6696..d6facda9 100644 --- a/estela-api/api/serializers/job.py +++ b/estela-api/api/serializers/job.py @@ -7,7 +7,13 @@ SpiderJobTagSerializer, ) from config.job_manager import job_manager -from core.models import SpiderJob, SpiderJobArg, SpiderJobEnvVar, SpiderJobTag +from core.models import ( + DataStatus, + SpiderJob, + SpiderJobArg, + SpiderJobEnvVar, + SpiderJobTag, +) class SpiderJobSerializer(serializers.ModelSerializer): @@ -149,12 +155,12 @@ def update(self, instance, validated_data): if ( "data_status" in validated_data - and instance.data_status != SpiderJob.DELETED_STATUS + and instance.data_status != DataStatus.DELETED_STATUS ): - if data_status == SpiderJob.PERSISTENT_STATUS: - instance.data_status = SpiderJob.PERSISTENT_STATUS - elif data_status == SpiderJob.PENDING_STATUS: - instance.data_status = SpiderJob.PENDING_STATUS + if data_status == DataStatus.PERSISTENT_STATUS: + instance.data_status = DataStatus.PERSISTENT_STATUS + elif data_status == DataStatus.PENDING_STATUS: + instance.data_status = DataStatus.PENDING_STATUS if data_expiry_days < 1: raise serializers.ValidationError( {"error": errors.POSITIVE_SMALL_INTEGER_FIELD} diff --git a/estela-api/api/serializers/project.py b/estela-api/api/serializers/project.py index 77318437..ab0f24e8 100644 --- a/estela-api/api/serializers/project.py +++ b/estela-api/api/serializers/project.py @@ -1,10 +1,7 @@ from django.contrib.auth.models import User -from django.db.models import Sum from rest_framework import serializers -from rest_framework.exceptions import APIException -from config.job_manager import spiderdata_db_client -from core.models import Permission, Project, SpiderJob, UsageRecord +from core.models import DataStatus, Permission, Project, UsageRecord class UserDetailSerializer(serializers.ModelSerializer): @@ -39,7 +36,15 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = Project - fields = ("pid", "name", "category", "container_image", "users") + fields = ( + "pid", + "name", + "category", + "container_image", + "users", + "data_status", + "data_expiry_days", + ) class UsageRecordSerializer(serializers.ModelSerializer): @@ -97,7 +102,6 @@ class ProjectUpdateSerializer(serializers.ModelSerializer): ("DEVELOPER", "Developer"), ("VIEWER", "Viewer"), ] - pid = serializers.UUIDField( read_only=True, help_text="A UUID identifying this project." ) @@ -117,7 +121,27 @@ class ProjectUpdateSerializer(serializers.ModelSerializer): required=False, help_text="New permission.", ) + data_status = serializers.ChoiceField( + write_only=True, + choices=DataStatus.HIGH_LEVEL_OPTIONS, + required=False, + help_text="New data status.", + ) + data_expiry_days = serializers.IntegerField( + write_only=True, + required=False, + help_text="New data expiry days.", + ) class Meta: model = Project - fields = ("pid", "name", "users", "email", "action", "permission") + fields = ( + "pid", + "name", + "users", + "email", + "action", + "permission", + "data_status", + "data_expiry_days", + ) diff --git a/estela-api/api/serializers/spider.py b/estela-api/api/serializers/spider.py index e9d820c5..2a733214 100644 --- a/estela-api/api/serializers/spider.py +++ b/estela-api/api/serializers/spider.py @@ -1,9 +1,36 @@ from rest_framework import serializers -from core.models import Spider +from core.models import DataStatus, Spider class SpiderSerializer(serializers.ModelSerializer): class Meta: model = Spider - fields = ("sid", "name", "project") + fields = ("sid", "name", "project", "data_status", "data_expiry_days") + + +class SpiderUpdateSerializer(serializers.ModelSerializer): + sid = serializers.UUIDField( + read_only=True, help_text="A UUID identifying this spider." + ) + data_status = serializers.ChoiceField( + choices=DataStatus.HIGH_LEVEL_OPTIONS, + required=False, + help_text="New data status.", + ) + data_expiry_days = serializers.IntegerField( + required=False, + help_text="New data expiry days.", + ) + + class Meta: + model = Spider + fields = ("sid", "name", "data_status", "data_expiry_days") + + def update(self, instance, validated_data): + instance.data_status = validated_data.get("data_status", instance.data_status) + instance.data_expiry_days = validated_data.get( + "data_expiry_days", instance.data_expiry_days + ) + instance.save() + return instance diff --git a/estela-api/api/views/cronjob.py b/estela-api/api/views/cronjob.py index 8652e0b9..84ad46ae 100644 --- a/estela-api/api/views/cronjob.py +++ b/estela-api/api/views/cronjob.py @@ -15,7 +15,7 @@ SpiderCronJobUpdateSerializer, ) from core.cronjob import create_cronjob, disable_cronjob, run_cronjob_once -from core.models import Spider, SpiderCronJob +from core.models import DataStatus, Spider, SpiderCronJob class SpiderCronJobViewSet( @@ -68,9 +68,9 @@ def create(self, request, *args, **kwargs): spider = get_object_or_404(Spider, sid=self.kwargs["sid"], deleted=False) serializer = SpiderCronJobCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) - data_status = request.data.pop("data_status", SpiderCronJob.PERSISTENT_STATUS) + data_status = request.data.pop("data_status", DataStatus.PERSISTENT_STATUS) - if data_status == SpiderCronJob.PENDING_STATUS: + if data_status == DataStatus.PENDING_STATUS: date_str = request.data.pop("data_expiry_days", "0/1") data_expiry_days = self.get_days(date_str) if data_expiry_days < 1: @@ -138,7 +138,7 @@ def run_once(self, request, *args, **kwargs): partial = kwargs.pop("partial", False) instance = self.get_object() - cronjob = SpiderCronJobSerializer(instance) + cronjob = SpiderCronJobSerializer(instance, partial=partial) run_cronjob_once(cronjob.data) return Response(cronjob.data, status=status.HTTP_200_OK) diff --git a/estela-api/api/views/job.py b/estela-api/api/views/job.py index fefaa529..14414d4e 100644 --- a/estela-api/api/views/job.py +++ b/estela-api/api/views/job.py @@ -15,7 +15,7 @@ SpiderJobUpdateSerializer, ) from config.job_manager import job_manager -from core.models import Spider, SpiderJob +from core.models import DataStatus, Spider, SpiderJob class SpiderJobViewSet( @@ -101,8 +101,8 @@ def create(self, request, *args, **kwargs): async_param = request.query_params.get("async", False) serializer = SpiderJobCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) - data_status = request.data.pop("data_status", SpiderJob.PERSISTENT_STATUS) - if data_status == SpiderJob.PENDING_STATUS: + data_status = request.data.pop("data_status", DataStatus.PERSISTENT_STATUS) + if data_status == DataStatus.PENDING_STATUS: data_expiry_days = request.data.pop("data_expiry_days", 1) if data_expiry_days < 1: raise ParseError({"error": "Invalid data expiry days value."}) diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index 1dd2b201..c691604a 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -19,6 +19,7 @@ UsageRecordSerializer, ) from core.models import ( + DataStatus, Permission, Project, Spider, @@ -86,8 +87,12 @@ def update(self, request, *args, **kwargs): user_email = serializer.validated_data.pop("email", "") action = serializer.validated_data.pop("action", "") permission = serializer.validated_data.pop("permission", "") + data_status = serializer.validated_data.pop("data_status", "") + data_expiry_days = serializer.validated_data.pop("data_expiry_days", 0) + if name: instance.name = name + if user_email and user_email != request.user.email: if not ( request.user.permission_set.get(project=instance).permission @@ -120,8 +125,13 @@ def update(self, request, *args, **kwargs): instance.users.add(user, through_defaults={"permission": permission}) else: raise ParseError({"error": "Action not supported."}) - serializer.save() + if data_status: + instance.data_status = data_status + if data_status == DataStatus.PENDING_STATUS and data_expiry_days > 0: + instance.data_expiry_days = data_expiry_days + + serializer.save() headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_200_OK, headers=headers) diff --git a/estela-api/api/views/spider.py b/estela-api/api/views/spider.py index 9171bec6..4479374d 100644 --- a/estela-api/api/views/spider.py +++ b/estela-api/api/views/spider.py @@ -1,11 +1,18 @@ -from rest_framework import viewsets +from drf_yasg.utils import swagger_auto_schema +from rest_framework import mixins, status +from rest_framework.response import Response from api.mixins import BaseViewSet -from api.serializers.spider import SpiderSerializer +from api.serializers.spider import SpiderSerializer, SpiderUpdateSerializer from core.models import Spider -class SpiderViewSet(BaseViewSet, viewsets.ReadOnlyModelViewSet): +class SpiderViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + BaseViewSet, +): model_class = Spider serializer_class = SpiderSerializer lookup_field = "sid" @@ -15,3 +22,21 @@ def get_queryset(self): return self.model_class.objects.filter( project__pid=self.kwargs["pid"], deleted=False ) + + @swagger_auto_schema( + request_body=SpiderUpdateSerializer, + responses={status.HTTP_200_OK: SpiderUpdateSerializer()}, + ) + def update(self, request, *args, **kwargs): + partial = kwargs.pop("partial", False) + instance = self.get_object() + serializer = SpiderUpdateSerializer( + instance, data=request.data, partial=partial + ) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + if getattr(instance, "_prefetched_objects_cache", None): + instance._prefetched_objects_cache = {} + + return Response(serializer.data, status=status.HTTP_200_OK) diff --git a/estela-api/core/migrations/0026_auto_20230411_0631.py b/estela-api/core/migrations/0026_auto_20230411_0631.py new file mode 100644 index 00000000..083817ca --- /dev/null +++ b/estela-api/core/migrations/0026_auto_20230411_0631.py @@ -0,0 +1,85 @@ +# Generated by Django 3.1.14 on 2023-04-11 06:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0025_delete_userprofile"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="data_expiry_days", + field=models.PositiveSmallIntegerField( + default=1, help_text="Days before data is deleted." + ), + ), + migrations.AddField( + model_name="project", + name="data_status", + field=models.CharField( + choices=[("PERSISTENT", "Persistent"), ("PENDING", "Pending")], + default="PERSISTENT", + help_text="Data status.", + max_length=20, + ), + ), + migrations.AddField( + model_name="spider", + name="data_expiry_days", + field=models.PositiveSmallIntegerField( + default=1, help_text="Days before data is deleted." + ), + ), + migrations.AddField( + model_name="spider", + name="data_status", + field=models.CharField( + choices=[("PERSISTENT", "Persistent"), ("PENDING", "Pending")], + default="PERSISTENT", + help_text="Data status.", + max_length=20, + ), + ), + migrations.AlterField( + model_name="spidercronjob", + name="data_expiry_days", + field=models.PositiveSmallIntegerField( + help_text="Days before data is deleted.", null=True + ), + ), + migrations.AlterField( + model_name="spidercronjob", + name="data_status", + field=models.CharField( + choices=[("PERSISTENT", "Persistent"), ("PENDING", "Pending")], + default="PERSISTENT", + help_text="Data status.", + max_length=20, + ), + ), + migrations.AlterField( + model_name="spiderjob", + name="data_expiry_days", + field=models.PositiveSmallIntegerField( + help_text="Days before data is deleted.", null=True + ), + ), + migrations.AlterField( + model_name="spiderjob", + name="data_status", + field=models.CharField( + choices=[ + ("PERSISTENT", "Persistent"), + ("PENDING", "Pending"), + ("DELETED", "Deleted"), + ], + default="PERSISTENT", + help_text="Data status.", + max_length=20, + ), + ), + ] diff --git a/estela-api/core/models.py b/estela-api/core/models.py index 5e496368..a14ec476 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -9,6 +9,18 @@ from django.utils import timezone +class DataStatus: + PERSISTENT_STATUS = "PERSISTENT" + DELETED_STATUS = "DELETED" + PENDING_STATUS = "PENDING" + + HIGH_LEVEL_OPTIONS = [ + (PERSISTENT_STATUS, "Persistent"), + (PENDING_STATUS, "Pending"), + ] + JOB_LEVEL_OPTIONS = HIGH_LEVEL_OPTIONS + [(DELETED_STATUS, "Deleted")] + + class Project(models.Model): NOT_ESPECIFIED = "NOT ESPECIFIED" E_COMMERCE = "E-COMMERCE" @@ -26,9 +38,10 @@ class Project(models.Model): (TECHNOLOGY, "Technology"), (OTHER_CATEGORY, "Other category"), ] + PERSISTENT_STATUS = "PERSISTENT" + PENDING_STATUS = "PENDING" pid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000) - users = models.ManyToManyField(User, through="Permission") pid = models.UUIDField( primary_key=True, default=uuid.uuid4, @@ -45,6 +58,15 @@ class Project(models.Model): users = models.ManyToManyField( User, through="Permission", help_text="Users with permissions on this project." ) + data_status = models.CharField( + max_length=20, + choices=DataStatus.HIGH_LEVEL_OPTIONS, + default=DataStatus.PERSISTENT_STATUS, + help_text="Data status.", + ) + data_expiry_days = models.PositiveSmallIntegerField( + default=1, help_text="Days before data is deleted." + ) deleted = models.BooleanField( default=False, help_text="Whether the project was deleted." ) @@ -94,6 +116,15 @@ class Spider(models.Model): related_name="spiders", help_text="Project UUID.", ) + data_status = models.CharField( + max_length=20, + choices=DataStatus.HIGH_LEVEL_OPTIONS, + default=DataStatus.PERSISTENT_STATUS, + help_text="Data status.", + ) + data_expiry_days = models.PositiveSmallIntegerField( + default=1, help_text="Days before data is deleted." + ) deleted = models.BooleanField( default=False, help_text="True if the spider has been deleted." ) @@ -142,16 +173,6 @@ class SpiderCronJob(models.Model): (DISABLED_STATUS, "Disabled"), ] - PERSISTENT_STATUS = "PERSISTENT" - DELETED_STATUS = "DELETED" - PENDING_STATUS = "PENDING" - - DATA_STATUS_OPTIONS = [ - (PERSISTENT_STATUS, "Persistent"), - (DELETED_STATUS, "Deleted"), - (PENDING_STATUS, "Pending"), - ] - cjid = models.AutoField( primary_key=True, help_text="A unique integer value identifying this cron job." ) @@ -179,12 +200,12 @@ class SpiderCronJob(models.Model): ) data_status = models.CharField( max_length=20, - choices=DATA_STATUS_OPTIONS, - default=PERSISTENT_STATUS, + choices=DataStatus.HIGH_LEVEL_OPTIONS, + default=DataStatus.PERSISTENT_STATUS, help_text="Data status.", ) data_expiry_days = models.PositiveSmallIntegerField( - null=True, help_text="Days before data expires." + null=True, help_text="Days before data is deleted." ) deleted = models.BooleanField( default=False, help_text="Whether the Cronjob has been deleted." @@ -224,16 +245,6 @@ class SpiderJob(models.Model): (ERROR_STATUS, "Error"), ] - PERSISTENT_STATUS = "PERSISTENT" - DELETED_STATUS = "DELETED" - PENDING_STATUS = "PENDING" - - DATA_STATUS_OPTIONS = [ - (PERSISTENT_STATUS, "Persistent"), - (DELETED_STATUS, "Deleted"), - (PENDING_STATUS, "Pending"), - ] - jid = models.AutoField( primary_key=True, help_text="A unique integer value identifying this job." ) @@ -255,12 +266,12 @@ class SpiderJob(models.Model): ) data_status = models.CharField( max_length=20, - choices=DATA_STATUS_OPTIONS, - default=PERSISTENT_STATUS, + choices=DataStatus.JOB_LEVEL_OPTIONS, + default=DataStatus.PERSISTENT_STATUS, help_text="Data status.", ) data_expiry_days = models.PositiveSmallIntegerField( - null=True, help_text="Days before data expires." + null=True, help_text="Days before data is deleted." ) created = models.DateTimeField( auto_now_add=True, editable=False, help_text="Job creation date." diff --git a/estela-api/core/tasks.py b/estela-api/core/tasks.py index f9e5bfc5..89f959dd 100644 --- a/estela-api/core/tasks.py +++ b/estela-api/core/tasks.py @@ -10,7 +10,7 @@ from api.serializers.job import SpiderJobCreateSerializer from config.celery import app as celery_app from config.job_manager import job_manager, spiderdata_db_client -from core.models import Project, Spider, SpiderJob, UsageRecord +from core.models import DataStatus, Project, Spider, SpiderJob, UsageRecord def get_default_token(job): @@ -60,35 +60,14 @@ def delete_data(pid, sid, jid, data_type): spiderdata_db_client.delete_collection_data(pid, job_collection_name) -@celery_app.task() -def delete_job_data(job_key): - jid, sid, pid = job_key.split(".") - delete_data(pid, sid, jid, "items") - delete_data(pid, sid, jid, "requests") - delete_data(pid, sid, jid, "logs") - SpiderJob.objects.filter(jid=jid).update(data_status=SpiderJob.DELETED_STATUS) - - -@celery_app.task(name="core.tasks.delete_expired_jobs_data") -def delete_expired_jobs_data(): - pending_data_delete_jobs = SpiderJob.objects.filter( - data_status=SpiderJob.PENDING_STATUS, - status__in=[SpiderJob.COMPLETED_STATUS, SpiderJob.STOPPED_STATUS], - ) - - for job in pending_data_delete_jobs: - if job.created < timezone.now() - timedelta(days=job.data_expiry_days): - delete_job_data.s(job.key).delay() - - @celery_app.task(name="core.tasks.launch_job") def launch_job(sid_, data_, data_expiry_days=None, token=None): spider = Spider.objects.get(sid=sid_) if data_expiry_days is None: - data_["data_status"] = SpiderJob.PERSISTENT_STATUS + data_["data_status"] = DataStatus.PERSISTENT_STATUS else: - data_["data_status"] = SpiderJob.PENDING_STATUS + data_["data_status"] = DataStatus.PENDING_STATUS serializer = SpiderJobCreateSerializer(data=data_) serializer.is_valid(raise_exception=True) @@ -176,6 +155,28 @@ def record_project_usage_after_data_delete(project_id, job_id): ) +@celery_app.task() +def delete_job_data(job_key): + jid, sid, pid = job_key.split(".") + delete_data(pid, sid, jid, "items") + delete_data(pid, sid, jid, "requests") + delete_data(pid, sid, jid, "logs") + SpiderJob.objects.filter(jid=jid).update(data_status=DataStatus.DELETED_STATUS) + record_project_usage_after_data_delete(pid, int(jid)) + + +@celery_app.task(name="core.tasks.delete_expired_jobs_data") +def delete_expired_jobs_data(): + pending_data_delete_jobs = SpiderJob.objects.filter( + data_status=DataStatus.PENDING_STATUS, + status__in=[SpiderJob.COMPLETED_STATUS, SpiderJob.STOPPED_STATUS], + ) + + for job in pending_data_delete_jobs: + if job.created < timezone.now() - timedelta(days=job.data_expiry_days): + delete_job_data.s(job.key).delay() + + @celery_app.task( max_retries=None, autoretry_for=(TaskError,), diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index 443ff0b7..64cb509b 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -639,6 +639,38 @@ paths: $ref: '#/definitions/Spider' tags: - api + put: + operationId: api_projects_spiders_update + description: '' + parameters: + - name: data + in: body + required: true + schema: + $ref: '#/definitions/SpiderUpdate' + responses: + '200': + description: '' + schema: + $ref: '#/definitions/SpiderUpdate' + tags: + - api + patch: + operationId: api_projects_spiders_partial_update + description: '' + parameters: + - name: data + in: body + required: true + schema: + $ref: '#/definitions/Spider' + responses: + '200': + description: '' + schema: + $ref: '#/definitions/Spider' + tags: + - api parameters: - name: pid in: path @@ -1269,6 +1301,19 @@ definitions: type: array items: $ref: '#/definitions/Permission' + data_status: + title: Data status + description: Data status. + type: string + enum: + - PERSISTENT + - PENDING + data_expiry_days: + title: Data expiry days + description: Days before data is deleted. + type: integer + maximum: 65535 + minimum: 0 ProjectUpdate: required: - name @@ -1313,6 +1358,17 @@ definitions: - ADMIN - DEVELOPER - VIEWER + data_status: + title: Data status + description: New data status. + type: string + enum: + - PERSISTENT + - PENDING + data_expiry_days: + title: Data expiry days + description: New data expiry days. + type: integer SpiderJobArg: description: Cron job arguments. required: @@ -1427,11 +1483,10 @@ definitions: type: string enum: - PERSISTENT - - DELETED - PENDING data_expiry_days: title: Data expiry days - description: Days before data expires. + description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 @@ -1532,6 +1587,19 @@ definitions: description: Project UUID. type: string format: uuid + data_status: + title: Data status + description: Data status. + type: string + enum: + - PERSISTENT + - PENDING + data_expiry_days: + title: Data expiry days + description: Days before data is deleted. + type: integer + maximum: 65535 + minimum: 0 Deploy: required: - project @@ -1700,7 +1768,7 @@ definitions: x-nullable: true data_expiry_days: title: Data expiry days - description: Days before data expires. + description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 @@ -1711,8 +1779,8 @@ definitions: type: string enum: - PERSISTENT - - DELETED - PENDING + - DELETED ProjectJob: required: - results @@ -1728,6 +1796,34 @@ definitions: title: Count description: Project jobs count. type: integer + SpiderUpdate: + required: + - name + type: object + properties: + sid: + title: Sid + description: A UUID identifying this spider. + type: string + format: uuid + readOnly: true + name: + title: Name + description: Spider's name. + type: string + maxLength: 1000 + minLength: 1 + data_status: + title: Data status + description: New data status. + type: string + enum: + - PERSISTENT + - PENDING + data_expiry_days: + title: Data expiry days + description: New data expiry days. + type: integer SpiderCronJobCreate: required: - data_status @@ -1808,11 +1904,10 @@ definitions: type: string enum: - PERSISTENT - - DELETED - PENDING data_expiry_days: title: Data expiry days - description: Days before data expires. + description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 @@ -1917,11 +2012,11 @@ definitions: type: string enum: - PERSISTENT - - DELETED - PENDING + - DELETED data_expiry_days: title: Data expiry days - description: Days before data expires. + description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 diff --git a/estela-web/src/components/LoginPage/index.tsx b/estela-web/src/components/LoginPage/index.tsx index 510a7513..24927fe3 100644 --- a/estela-web/src/components/LoginPage/index.tsx +++ b/estela-web/src/components/LoginPage/index.tsx @@ -14,7 +14,14 @@ import { UserContext, UserContextProps } from "../../context"; const { Content } = Layout; const { Text } = Typography; -export class LoginPage extends Component { +interface LoginState { + loading: boolean; +} + +export class LoginPage extends Component { + state: LoginState = { + loading: false, + }; apiService = ApiService(); static contextType = UserContext; @@ -32,6 +39,7 @@ export class LoginPage extends Component { } handleSubmit = (data: { username: string; password: string }): void => { + this.setState({ loading: true }); const request: ApiAuthLoginRequest = { data }; const { updateUsername, updateEmail, updateAccessToken } = this.context as UserContextProps; this.apiService.apiAuthLogin(request).then( @@ -44,6 +52,7 @@ export class LoginPage extends Component { AuthService.setUserEmail(response.user.email ?? ""); updateEmail(response.user.email ?? ""); } + this.setState({ loading: false }); history.push("/projects"); }, (error: unknown) => { @@ -53,6 +62,7 @@ export class LoginPage extends Component { }; render(): JSX.Element { + const { loading } = this.state; return ( @@ -102,6 +112,7 @@ export class LoginPage extends Component { @@ -530,7 +547,7 @@ export class ProjectJobListPage extends Component {this.dataPersistenceOptions.map((option: OptionDataPersistance) => ( ); } diff --git a/estela-web/src/components/SpiderDetailPage/index.tsx b/estela-web/src/components/SpiderDetailPage/index.tsx index f8ca910e..3120596d 100644 --- a/estela-web/src/components/SpiderDetailPage/index.tsx +++ b/estela-web/src/components/SpiderDetailPage/index.tsx @@ -1,5 +1,20 @@ import React, { Component, ReactElement } from "react"; -import { Button, Layout, Pagination, Typography, Row, Table, Col, Tabs, Radio, Checkbox, Space, Tag } from "antd"; +import { + Button, + Layout, + Pagination, + Typography, + message, + Row, + Table, + Col, + Tabs, + Radio, + Checkbox, + Space, + Tag, +} from "antd"; +import type { RadioChangeEvent } from "antd"; import { RouteComponentProps, Link } from "react-router-dom"; import "./styles.scss"; @@ -13,15 +28,19 @@ import Filter from "../../assets/icons/filter.svg"; import { ApiProjectsSpidersReadRequest, ApiProjectsSpidersJobsListRequest, + ApiProjectsSpidersUpdateRequest, ApiProjectsDeploysListRequest, Deploy, Spider, + SpiderDataStatusEnum, + SpiderUpdateDataStatusEnum, + SpiderUpdate, SpiderJob, SpiderJobArg, SpiderJobTag, } from "../../services/api"; import { resourceNotAllowedNotification, Spin, PaginationItem } from "../../shared"; -import { convertDateToString } from "../../utils"; +import { convertDateToString, handleInvalidDataError } from "../../utils"; const { Content } = Layout; const { Text } = Typography; @@ -55,6 +74,10 @@ interface SpiderDetailPageState { errorJobs: SpiderJobData[]; scheduledJobsCount: number; spiderCreationDate: string; + persistenceChanged: boolean; + newDataStatus: SpiderUpdateDataStatusEnum | undefined; + dataStatus: SpiderDataStatusEnum | SpiderUpdateDataStatusEnum | undefined; + dataExpiryDays: number | null | undefined; } interface RouteParams { @@ -62,6 +85,12 @@ interface RouteParams { spiderId: string; } +interface OptionDataPersistance { + label: string; + key: number; + value: number; +} + export class SpiderDetailPage extends Component, SpiderDetailPageState> { PAGE_SIZE = 10; center = "center"; @@ -78,6 +107,10 @@ export class SpiderDetailPage extends Component errorJobs: [], spiderCreationDate: "", scheduledJobsCount: 0, + dataStatus: undefined, + dataExpiryDays: 1, + persistenceChanged: false, + newDataStatus: undefined, tableStatus: new Array(4).fill(true), }; apiService = ApiService(); @@ -188,6 +221,8 @@ export class SpiderDetailPage extends Component const jobs: SpiderJobData[] = data.data; this.setState({ name: response.name, + dataStatus: response.dataStatus, + dataExpiryDays: response.dataExpiryDays, jobs: [...jobs], count: data.count, current: data.current, @@ -249,6 +284,32 @@ export class SpiderDetailPage extends Component return { data, count: response.count, current: page }; }; + changePersistence = (): void => { + const requestData: SpiderUpdate = { + name: this.state.name, + dataStatus: this.state.newDataStatus, + dataExpiryDays: Number(this.state.dataExpiryDays), + }; + const request: ApiProjectsSpidersUpdateRequest = { + pid: this.projectId, + sid: Number(this.spiderId), + data: requestData, + }; + this.apiService.apiProjectsSpidersUpdate(request).then( + (response) => { + message.success("Data persistence changed"); + this.setState({ + dataStatus: response.dataStatus, + dataExpiryDays: response.dataExpiryDays, + persistenceChanged: false, + }); + }, + (error: unknown) => { + handleInvalidDataError(error); + }, + ); + }; + onDetailMenuTabChange = (option: string) => { this.setState({ optionTab: option, @@ -282,6 +343,15 @@ export class SpiderDetailPage extends Component }); }; + onPersistenceChange = (e: RadioChangeEvent): void => { + this.setState({ persistenceChanged: true }); + if (e.target.value === 720) { + this.setState({ newDataStatus: SpiderUpdateDataStatusEnum.Persistent }); + } else { + this.setState({ newDataStatus: SpiderUpdateDataStatusEnum.Pending, dataExpiryDays: e.target.value }); + } + }; + onChangeStatus = (index: number, count: number) => { if (count === 0) { const tableStatus = this.state.tableStatus; @@ -628,13 +698,24 @@ export class SpiderDetailPage extends Component ); }; + dataPersistenceOptions = [ + { label: "1 day", key: 1, value: 1 }, + { label: "1 week", key: 2, value: 7 }, + { label: "1 month", key: 3, value: 30 }, + { label: "3 months", key: 4, value: 90 }, + { label: "6 months", key: 5, value: 180 }, + { label: "1 year", key: 6, value: 365 }, + { label: "Forever", key: 7, value: 720 }, + ]; + settings = (): React.ReactNode => { + const { dataStatus, dataExpiryDays, persistenceChanged } = this.state; return ( - + -
+

Data persistence

Data persistence will be applied to all jobs and scheduled jobs by default. @@ -644,14 +725,18 @@ export class SpiderDetailPage extends Component

General Data Persistent

- - 1 day - 1 week - 1 month - 3 months - 6 months - 1 year - Forever + + {this.dataPersistenceOptions.map((option: OptionDataPersistance) => ( + + {option.label} + + ))} @@ -659,7 +744,8 @@ export class SpiderDetailPage extends Component