Add theme column (#890)

Features:
* Add support for filtering by theme, domains, country_codes and asns
* Add support for setting slug of finding
* Add support for retrieving findings by slug

Changed:
* It's now possible to create a finding with the published flag set on creation

Fixes:
* Fix bug in end_time not being returned
* Fix creation to behave like old explorer

Closes:
* api: findings add support for storing and filtering by theme #889
* api: add support for filtering by domain name in findings endpoints #881

ooniapi/common/src/common/alembic/versions/8e7ecea5c2f5_themes_column.py

@@ -0,0 +1,63 @@
+"""themes column
+
+Revision ID: 8e7ecea5c2f5
+Revises: a037e908f3a0
+Create Date: 2024-10-11 12:03:07.662702
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "8e7ecea5c2f5"
+down_revision: Union[str, None] = "a037e908f3a0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def convert_json_to_array(column_name: str):
+    op.add_column("oonifinding", sa.Column(f"{column_name}_tmp", sa.ARRAY(sa.String)))
+    op.execute(
+        f"""
+        UPDATE oonifinding
+        SET {column_name}_tmp = ARRAY(SELECT json_array_elements_text({column_name}))
+    """
+    )
+    op.drop_column("oonifinding", column_name)
+    op.alter_column("oonifinding", f"{column_name}_tmp", new_column_name=column_name)
+
+
+def convert_array_to_json(column_name: str):
+    op.add_column("oonifinding", sa.Column(f"{column_name}_tmp", sa.JSON()))
+    op.execute(
+        f"""
+        UPDATE oonifinding
+        SET {column_name}_tmp = to_json({column_name})
+    """
+    )
+    op.drop_column("oonifinding", column_name)
+    op.alter_column("oonifinding", f"{column_name}_tmp", new_column_name=column_name)
+
+
+def upgrade() -> None:
+    op.add_column("oonifinding", sa.Column("themes", sa.ARRAY(sa.String())))
+    convert_json_to_array("country_codes")
+    convert_json_to_array("asns")
+    convert_json_to_array("domains")
+    convert_json_to_array("tags")
+    convert_json_to_array("links")
+    convert_json_to_array("test_names")
+
+
+def downgrade() -> None:
+    op.drop_column("oonifinding", "themes")
+    convert_array_to_json("country_codes")
+    convert_array_to_json("asns")
+    convert_array_to_json("domains")
+    convert_array_to_json("tags")
+    convert_array_to_json("links")
+    convert_array_to_json("test_names")

ooniapi/common/src/common/alembic/versions/a037e908f3a0_init_oonifindings.py

@@ -5,14 +5,15 @@
 Create Date: 2024-07-17 16:45:25.752551
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision: str = 'a037e908f3a0'
+revision: str = "a037e908f3a0"
 down_revision: Union[str, None] = "c9119c05cf42"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None

ooniapi/services/oonifindings/src/oonifindings/models.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import List
+import sqlalchemy as sa
 from sqlalchemy import String
 from sqlalchemy.orm import Mapped
 from sqlalchemy.orm import mapped_column
@@ -15,7 +16,7 @@ class OONIFinding(Base):
     # TODO(decfox): this is nullable for now. We should
     # make this a non-nullable field eventually and have an endpoint
     # where we can query findings using the finding_slug.
-    finding_slug: Mapped[str] = mapped_column(String, nullable=True)
+    finding_slug: Mapped[str] = mapped_column(String, nullable=False, unique=True)
 
     create_time: Mapped[datetime] = mapped_column(UtcDateTime())
     update_time: Mapped[datetime] = mapped_column(UtcDateTime())
@@ -32,9 +33,12 @@ class OONIFinding(Base):
     published: Mapped[int] = mapped_column()
     deleted: Mapped[int] = mapped_column(default=0)
 
-    country_codes: Mapped[List[str]] = mapped_column(nullable=True)
-    asns: Mapped[List[str]] = mapped_column(nullable=True)
-    domains: Mapped[List[str]] = mapped_column(nullable=True)
-    tags: Mapped[List[str]] = mapped_column(nullable=True)
-    links: Mapped[List[str]] = mapped_column(nullable=True)
-    test_names: Mapped[List[str]] = mapped_column(nullable=True)
+    country_codes: Mapped[List[str]] = mapped_column(
+        sa.ARRAY(sa.String()), nullable=True
+    )
+    asns: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.INT), nullable=True)
+    domains: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.String()), nullable=True)
+    themes: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.String()), nullable=True)
+    tags: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.String()), nullable=True)
+    links: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.String()), nullable=True)
+    test_names: Mapped[List[str]] = mapped_column(sa.ARRAY(sa.String()), nullable=True)