Merge branch 'master' into dependabot/github_actions/codecov/codecov-…

…action-5

docs/CHANGELOG.rst

@@ -10,6 +10,42 @@ This project adheres to `Semantic Versioning <http://semver.org/>`_.
 Unreleased
 ==========
 
+Added
+-----
+- Expose ``status`` on ``collection`` and ``entity`` viewset and allow
+  filtering and sorting by it
+Changed
+-------
+- Make processing container startup script ``Python`` 3.12 compatible
+
+
+===================
+42.0.1 - 2024-11-21
+===================
+
+Fixed
+-----
+- Fix crash in ``get_annotation`` when delete marker is set
+
+
+===================
+42.0.0 - 2024-11-21
+===================
+
+Fixed
+-----
+- Filter out deleted annotation values in viewset
+
+Changed
+-------
+- **BACKWARD INCOMPATIBLE:** Return all version of processess on process
+  endpoint
+
+
+===================
+41.0.0 - 2024-11-18
+===================
+
 Changed
 -------
 - **BACKWARD INCOMPATIBLE:** Set ``BaseManager`` as default objects manager.
@@ -28,6 +64,7 @@ Added
 - Add ``contributor`` field to the ``AnnotationValue`` model
 - Add version of base managers without versioning support
 - Add abstract base classes for annotation fields
+- Add ``annotate_status`` to collection and entity queryset
 
 Fixed
 -----

resolwe/flow/executors/startup_processing_container.py

@@ -485,8 +485,12 @@ async def run(self) -> int:
         :raises RuntimeError: in case of failure.
         """
         try:
+            loop = asyncio.get_event_loop()
             await asyncio.wait(
-                [self.protocol_handler.script_started.wait(), self._terminating.wait()],
+                [
+                    loop.create_task(self.protocol_handler.script_started.wait()),
+                    loop.create_task(self._terminating.wait()),
+                ],
                 timeout=SCRIPT_START_TIMEOUT,
                 return_when=asyncio.FIRST_COMPLETED,
             )
@@ -499,10 +503,10 @@ async def run(self) -> int:
                 )
             else:
                 monitored_tasks = (
-                    self.script_server.wait_closed(),
+                    loop.create_task(self.script_server.wait_closed()),
                     self.upload_handler.upload_task,
                     self.communicator_task,
-                    self._terminating.wait(),
+                    loop.create_task(self._terminating.wait()),
                 )
                 # Stop processing when any of the following tasks complete.
                 await asyncio.wait(monitored_tasks, return_when=asyncio.FIRST_COMPLETED)

resolwe/flow/filters.py

@@ -286,6 +286,10 @@ class Meta(BaseResolweFilter.Meta):
         model = DescriptorSchema
 
 
+class CharInFilter(filters.BaseInFilter, filters.CharFilter):
+    """Basic filter for CharField with 'in' lookup."""
+
+
 class BaseCollectionFilter(TextFilterMixin, UserFilterMixin, BaseResolweFilter):
     """Base filter for Collection and Entity endpoints."""
 
@@ -299,6 +303,8 @@ class BaseCollectionFilter(TextFilterMixin, UserFilterMixin, BaseResolweFilter):
     permission = filters.CharFilter(method="filter_for_user")
     tags = TagsFilter()
     text = filters.CharFilter(field_name="search", method="filter_text")
+    status = filters.CharFilter(field_name="status")
+    status__in = CharInFilter(field_name="status", lookup_expr="in")
 
     class Meta(BaseResolweFilter.Meta):
         """Filter configuration."""

resolwe/flow/migrations/0032_alter_process_managers.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.16 on 2024-11-21 16:23
+
+import django.db.models.manager
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("flow", "0031_remove_annotationvalue_uniquetogether_entity_field_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterModelManagers(
+            name="process",
+            managers=[
+                ("unversioned_objects", django.db.models.manager.Manager()),
+            ],
+        ),
+    ]

resolwe/flow/migrations/0033_data_idx_data_status_priority.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.16 on 2024-11-08 09:42
+
+from django.db import migrations, models
+
+import resolwe.flow.models.base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("flow", "0032_alter_process_managers"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="data",
+            index=models.Index(
+                models.Case(
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["ERROR"],
+                        then=models.Value(0),
+                    ),
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["UPLOADING"],
+                        then=models.Value(1),
+                    ),
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["PREPARING"],
+                        then=models.Value(2),
+                    ),
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["WAITING"],
+                        then=models.Value(3),
+                    ),
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["RESOLVING"],
+                        then=models.Value(4),
+                    ),
+                    models.When(
+                        status=resolwe.flow.models.base.DataStatus["DONE"],
+                        then=models.Value(5),
+                    ),
+                ),
+                name="idx_data_status_priority",
+            ),
+        ),
+    ]

resolwe/flow/models/annotations.py

@@ -454,11 +454,14 @@ class AnnotationValueManager(BaseManager["AnnotationValue", PermissionQuerySet])
     version_field = "created"
     QuerySet = PermissionQuerySet
 
-    def get_queryset(self) -> PermissionQuerySet:
-        """Return the latest version for every value."""
-        queryset = super().get_queryset()
-        # The old value is deleted so make sure it is never returned.
-        return queryset.filter(pk__in=queryset).exclude(_value__isnull=True)
+    def remove_delete_markers(self) -> PermissionQuerySet:
+        """Remote delete markers from the queryset.
+
+        Due to the nature of the queryset use the method only on small querysets.
+        """
+        return self.filter(pk__in=list(self.values_list("pk", flat=True))).exclude(
+            _value__isnull=True
+        )
 
 
 def _slug_for_annotation_value(instance: "AnnotationValue") -> str:

resolwe/flow/models/base.py

@@ -1,6 +1,7 @@
 """Base model for all Resolwe models."""
 
 import datetime
+from enum import StrEnum
 from typing import TypeVar
 
 from django.conf import settings
@@ -18,6 +19,42 @@
 MAX_SLUG_LENGTH = 100
 MAX_NAME_LENGTH = 100
 
+# Data status enum.
+
+
+class DataStatus(StrEnum):
+    """Status of the data object."""
+
+    #: data object is uploading
+    UPLOADING = "UP"
+    #: data object is being resolved
+    RESOLVING = "RE"
+    #: data object is waiting
+    WAITING = "WT"
+    #: data object is preparing
+    PREPARING = "PP"
+    #: data object is processing
+    PROCESSING = "PR"
+    #: data object is done
+    DONE = "OK"
+    #: data object is in error state
+    ERROR = "ER"
+    #: data object is in dirty state
+    DIRTY = "DR"
+
+    @classmethod
+    def sort_order(cls):
+        """Return a priority for sorting data statuses in a collection."""
+        return [
+            cls.ERROR,
+            cls.UPLOADING,
+            cls.PREPARING,
+            cls.WAITING,
+            cls.RESOLVING,
+            cls.DONE,
+        ]
+
+
 M = TypeVar("M", bound=models.Model)
 Q = TypeVar("Q", bound=models.QuerySet)
 

resolwe/flow/models/collection.py

@@ -1,6 +1,7 @@
 """Resolwe collection model."""
 
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import ArrayField
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import SearchVectorField
@@ -13,7 +14,7 @@
 from resolwe.permissions.models import PermissionObject
 
 from .annotations import AnnotationField
-from .base import BaseModel
+from .base import BaseModel, DataStatus
 from .history_manager import HistoryMixin
 from .utils import DirtyError, validate_schema
 
@@ -38,7 +39,24 @@ class Meta(BaseModel.Meta):
     search = SearchVectorField(null=True)
 
 
-class CollectionQuerySet(BaseQuerySet):
+class BaseCollectionQuerySet(BaseQuerySet):
+    """A base queryset used by both collection and entity."""
+
+    def annotate_status(self):
+        """Annotate the queryset with status of the collections."""
+
+        priority_order = models.Case(
+            *[
+                models.When(data__status=status, then=models.Value(sort_order))
+                for sort_order, status in enumerate(DataStatus.sort_order())
+            ]
+        )
+        return self.annotate(
+            statuses=ArrayAgg(models.F("data__status"), ordering=priority_order.asc())
+        ).annotate(status=models.F("statuses__0"))
+
+
+class CollectionQuerySet(BaseCollectionQuerySet):
     """Query set for ``Collection`` objects."""
 
     def duplicate(self, request_user) -> BackgroundTask:

resolwe/flow/models/data.py

@@ -31,7 +31,7 @@
 from resolwe.permissions.models import PermissionObject
 from resolwe.permissions.utils import assign_contributor_permissions, copy_permissions
 
-from .base import BaseModel
+from .base import BaseModel, DataStatus
 from .entity import Entity, EntityQuerySet
 from .history_manager import HistoryMixin
 from .secret import Secret
@@ -277,24 +277,33 @@ class Meta(BaseModel.Meta):
             models.Index(name="idx_data_status", fields=["status"]),
             GinIndex(name="idx_data_tags", fields=["tags"]),
             GinIndex(name="idx_data_search", fields=["search"]),
+            models.Index(
+                models.Case(
+                    *[
+                        models.When(status=status, then=models.Value(sort_order))
+                        for sort_order, status in enumerate(DataStatus.sort_order())
+                    ]
+                ),
+                name="idx_data_status_priority",
+            ),
         ]
 
     #: data object is uploading
-    STATUS_UPLOADING = "UP"
+    STATUS_UPLOADING = DataStatus.UPLOADING
     #: data object is being resolved
-    STATUS_RESOLVING = "RE"
+    STATUS_RESOLVING = DataStatus.RESOLVING
     #: data object is waiting
-    STATUS_WAITING = "WT"
+    STATUS_WAITING = DataStatus.WAITING
     #: data object is preparing
-    STATUS_PREPARING = "PP"
+    STATUS_PREPARING = DataStatus.PREPARING
     #: data object is processing
-    STATUS_PROCESSING = "PR"
+    STATUS_PROCESSING = DataStatus.PROCESSING
     #: data object is done
-    STATUS_DONE = "OK"
+    STATUS_DONE = DataStatus.DONE
     #: data object is in error state
-    STATUS_ERROR = "ER"
+    STATUS_ERROR = DataStatus.ERROR
     #: data object is in dirty state
-    STATUS_DIRTY = "DR"
+    STATUS_DIRTY = DataStatus.DIRTY
     # Assumption (in listener): ordered from least to most problematic.
     STATUS_CHOICES = (
         (STATUS_UPLOADING, "Uploading"),

resolwe/flow/models/entity.py

@@ -16,11 +16,11 @@
 from resolwe.permissions.models import PermissionObject
 
 from .base import BaseModel
-from .collection import BaseCollection, Collection
+from .collection import BaseCollection, BaseCollectionQuerySet, Collection
 from .utils import DirtyError, validate_schema
 
 
-class EntityQuerySet(BaseQuerySet):
+class EntityQuerySet(BaseCollectionQuerySet):
     """Query set for ``Entity`` objects."""
 
     def duplicate(self, request_user) -> BackgroundTask:
@@ -210,9 +210,13 @@ def get_annotation(self, path: str, default: Any = None) -> Any:
         :return: value of the annotation or default if not found.
         """
         group_name, field_name = path.split(".", maxsplit=1)
-        annotation: AnnotationValue = self.annotations.filter(
-            field__group__name=group_name, field__name=field_name
-        ).first()
+        annotation: AnnotationValue = (
+            self.annotations.filter(
+                field__group__name=group_name, field__name=field_name
+            )
+            .remove_delete_markers()
+            .first()
+        )
         return annotation.value if annotation else default
 
     def set_annotation(self, path: str, value: Any, contributor=None):

resolwe/flow/models/process.py

@@ -10,7 +10,7 @@
 
 from resolwe.permissions.models import PermissionObject
 
-from .base import BaseModel
+from .base import BaseManagerWithoutVersion, BaseModel
 from .data import Data
 
 
@@ -56,6 +56,9 @@ class Meta(BaseModel.Meta):
         (SCHEDULING_CLASS_BATCH, "Batch"),
     )
 
+    #: unversioned object manager
+    unversioned_objects = BaseManagerWithoutVersion()
+
     #: data type
     type = models.CharField(
         max_length=100,