From 185f707ba3487c03c48e546b729178111c115d59 Mon Sep 17 00:00:00 2001 From: Eugen Ciur Date: Sun, 13 Oct 2024 13:11:52 +0200 Subject: [PATCH] playing with custom fields --- papermerge/core/cli/docs.py | 94 +++--------------------- papermerge/core/db/doc.py | 136 +++++++++++------------------------ papermerge/core/db/models.py | 3 - 3 files changed, 53 insertions(+), 180 deletions(-) diff --git a/papermerge/core/cli/docs.py b/papermerge/core/cli/docs.py index d1dc36943..d2c7ee187 100644 --- a/papermerge/core/cli/docs.py +++ b/papermerge/core/cli/docs.py @@ -3,20 +3,10 @@ import typer from rich.console import Console from rich.table import Table -from sqlalchemy import case, desc, select -from sqlalchemy.orm import aliased from papermerge.core import db, schemas from papermerge.core.db.doc import get_documents_by_type from papermerge.core.db.document_types import get_document_types -from papermerge.core.db.models import ( - CustomField, - CustomFieldValue, - Document, - DocumentType, - DocumentTypeCustomField, - Node, -) app = typer.Typer(help="List various entities") session = db.get_session() @@ -37,30 +27,29 @@ def list_documents_by_type(type_id: uuid.UUID): @app.command(name="cfv") -def get_cfv(doc_id: uuid.UUID, cf_names: list[str]): - """Print custom field values for specific document - - Receives as input document ID and list of custom field names - """ - items: list[schemas.CFV] = db.get_doc_cfv( - session, document_id=doc_id, cf_names=cf_names - ) +def get_cfv(doc_id: uuid.UUID): + """Print custom field values for specific document""" + items: list[schemas.CFV] = db.get_doc_cfv(session, document_id=doc_id) table = Table(title="Document's Custom Field Values") + table.add_column("Document ID") table.add_column("CF ID", style="magenta") table.add_column("CF Name") table.add_column("CF Type") - table.add_column("CF Extra Data") table.add_column("CF Value") for item in items: + value = "-" + if item.value: + value = str(item.value) + table.add_row( + str(item.document_id), str(item.custom_field_id), item.name, item.type, - item.extra_data, - str(item.value), + value, ) console = Console() @@ -87,69 +76,6 @@ def update_doc_custom_fields(doc_id: uuid.UUID, custom_fields: list[str]): db.update_document_custom_fields(session, document_id=doc_id, custom_fields=cf) -def get_subq(): - nd = aliased(Node) - cfv = aliased(CustomFieldValue) - cf = aliased(CustomField) - dtcf = aliased(DocumentTypeCustomField) - dt = aliased(DocumentType) - doc = aliased(Document) - - subq = ( - select( - nd.title.label("title"), - doc.id.label("doc_id"), - doc.document_type_id.label("document_type_id"), - ) - .select_from(doc) - .join(nd, nd.id == doc.id) - .join(dt, dt.id == doc.document_type_id) - .join(dtcf, dtcf.document_type_id == dt.id) - .join(cf, cf.id == dtcf.custom_field_id) - .join(cfv, cfv.document_id == doc.id, isouter=True) - .where( - dt.id == uuid.UUID("9f349f14-608d-41f3-9284-dc32449dcf02"), - cf.name == "Total", - nd.parent_id == uuid.UUID("4fdcfbc9-64cb-46d3-bc7e-e1677eaecc70"), - ) - .order_by(desc(cfv.value_monetary)) - ) - - return subq - - -@app.command(name="play") -def play(): - subq = get_subq() - cfv = aliased(CustomFieldValue) - cf = aliased(CustomField) - dtcf = aliased(DocumentTypeCustomField) - dt = aliased(DocumentType) - - stmt = ( - select( - subq.c.title, - subq.c.doc_id.label("doc_id"), - cf.name.label("cf_name"), - case( - (cf.data_type == "monetary", cfv.value_monetary), - (cf.data_type == "string", cfv.value_text), - (cf.data_type == "date", cfv.value_date), - (cf.data_type == "boolean", cfv.value_bool), - (cf.data_type == "url", cfv.value_url), - ).label("cf_value"), - ) - .select_from(subq) - .join(dt, subq.c.document_type_id == dt.id) - .join(dtcf, dtcf.document_type_id == dt.id) - .join(cf, cf.id == dtcf.custom_field_id) - .join(cfv, cfv.document_id == subq.c.doc_id, isouter=True) - ) - - for row in session.execute(stmt): - print(row) - - def print_docs(docs): if len(docs) == 0: print("No entries") diff --git a/papermerge/core/db/doc.py b/papermerge/core/db/doc.py index be9e1782a..c3cd3b7f7 100644 --- a/papermerge/core/db/doc.py +++ b/papermerge/core/db/doc.py @@ -3,7 +3,7 @@ from typing import Optional from uuid import UUID -from sqlalchemy import and_, case, desc, insert, or_, select, update +from sqlalchemy import case, desc, insert, select, text, update from sqlalchemy.orm import Session, aliased from papermerge.core import schemas @@ -102,9 +102,7 @@ def get_page(doc_ver_id): return model_doc -def get_doc_cfv( - session: Session, document_id: UUID, cf_names: list[str] -) -> list[schemas.CFV]: +def get_doc_cfv(session: Session, document_id: UUID) -> list[schemas.CFV]: """ Fetch document's custom field values for each CF name, even if CFV is NULL @@ -119,95 +117,46 @@ def get_doc_cfv( Notice that item 1 and 3 have cf_value=None, which indicates that there is no value for it in `custom_field_values` table. - - The corresponding Sql query is (just an example): - ``` - SELECT doc.basetreenode_ptr_id AS 'Doc ID', - dt.name AS 'Document Type', - cf.cf_name AS 'Custom Field Name', - CASE - WHEN cf.cf_data_type == 'monetary' THEN cf.value_monetary - WHEN cf.cf_data_type == 'date' THEN cf.value_date - WHEN cf.cf_data_type == 'string' THEN cf.value_text - END AS 'CF VALUE', - cf.cfv_id - FROM core_document AS doc - JOIN core_documenttype AS dt ON doc.document_type_id = dt.id - JOIN core_documenttypecustomfield dtcf ON dtcf.document_type_id = dt.id - JOIN ( - SELECT sub_cf.id AS cf_id, - sub_cf.name AS cf_name, - sub_cf.data_type AS cf_data_type, - sub_cfv.id AS cfv_id, - sub_cfv.value_monetary AS value_monetary, - sub_cfv.value_date AS value_date, - sub_cfv.value_text AS value_text - FROM core_customfield sub_cf - LEFT OUTER JOIN core_customfieldvalue sub_cfv ON sub_cfv.field_id = sub_cf.id - WHERE (sub_cfv.document_id = 'b0c90f2f7380404c81179903c55f113b' AND sub_cf.name IN ('Shop', 'Total', 'Date') ) - OR (sub_cfv.document_id IS NULL AND sub_cf.name IN ('Shop', 'Total', 'Date')) - ) cf ON cf.cf_id = dtcf.custom_field_id - WHERE cf.cf_name in ('Total', 'Shop', 'Date') - AND doc.basetreenode_ptr_id = 'b0c90f2f7380404c81179903c55f113b'; - ``` """ - def get_subq(): - cfv = aliased(CustomFieldValue) - cf = aliased(CustomField) - - subq = ( - select( - cf.id.label("cf_id"), - cf.name.label("cf_name"), - cf.data_type.label("cf_data_type"), - cf.extra_data.label("cf_extra_data"), - cfv.id.label("cfv_id"), - cfv.value_monetary, - cfv.value_text, - cfv.value_date, - cfv.value_bool, - ) - .select_from(cf) - .join(cfv, cfv.field_id == cf.id, isouter=True) - .where( - or_( - and_(cfv.document_id == document_id, cf.name.in_(cf_names)), - and_(cfv.document_id == None, cf.name.in_(cf_names)), - ) - ) - .subquery() - ) - return subq - - subq = get_subq() - doc = aliased(Document) - dt = aliased(DocumentType) - dtcf = aliased(DocumentTypeCustomField) - stmt = ( - select( - doc.id.label("doc_id"), + stmt = """ + SELECT + doc.basetreenode_ptr_id AS doc_id, doc.document_type_id, - subq.c.cf_name.label("cf_name"), - subq.c.cf_data_type.label("cf_type"), - subq.c.cf_extra_data.label("cf_extra_data"), - subq.c.cf_id.label("cf_id"), - subq.c.cfv_id.label("cfv_id"), - case( - (subq.c.cf_data_type == "monetary", subq.c.value_monetary), - (subq.c.cf_data_type == "string", subq.c.value_text), - (subq.c.cf_data_type == "date", subq.c.value_date), - "--", - ).label("cf_value"), - ) - .select_from(doc) - .join(dt, dt.id == doc.document_type_id) - .join(dtcf, dt.id == dtcf.document_type_id) - .join(subq, subq.c.cf_id == dtcf.custom_field_id) - .where(subq.c.cf_name.in_(cf_names), doc.id == document_id) - ) + cf.cf_id AS cf_id, + cf.cf_name, + cf.cf_type AS cf_type, + cf.cf_extra_data, + cfv.id AS cfv_id, + CASE + WHEN(cf.cf_type = 'monetary') THEN cfv.value_monetary + WHEN(cf.cf_type = 'string') THEN cfv.value_text + WHEN(cf.cf_type = 'date') THEN cfv.value_date + WHEN(cf.cf_type = 'boolean') THEN cfv.value_bool + END AS cf_value + FROM core_document AS doc + JOIN core_documenttypecustomfield AS dtcf ON dtcf.document_type_id = doc.document_type_id + JOIN( + SELECT + sub_cf1.id AS cf_id, + sub_cf1.name AS cf_name, + sub_cf1.data_type AS cf_type, + sub_cf1.extra_data AS cf_extra_data + FROM core_document AS sub_doc1 + JOIN core_documenttypecustomfield AS sub_dtcf1 + ON sub_dtcf1.document_type_id = sub_doc1.document_type_id + JOIN core_customfield AS sub_cf1 + ON sub_cf1.id = sub_dtcf1.custom_field_id + WHERE sub_doc1.basetreenode_ptr_id = :document_id + ) AS cf ON cf.cf_id = dtcf.custom_field_id + LEFT OUTER JOIN core_customfieldvalue AS cfv + ON cfv.field_id = cf.cf_id AND cfv.document_id = :document_id + WHERE + doc.basetreenode_ptr_id = :document_id + """ result = [] - for row in session.execute(stmt): + str_doc_id = str(document_id).replace("-", "") + for row in session.execute(text(stmt), {"document_id": str_doc_id}): result.append( schemas.CFV( document_id=row.doc_id, @@ -230,12 +179,13 @@ def update_document_custom_fields( custom_fields: dict, # if of the document ) -> list[schemas.CFV]: """ """ - items = get_doc_cfv( - session, document_id=document_id, cf_names=list(custom_fields.keys()) - ) + items = get_doc_cfv(session, document_id=document_id) insert_values = [] update_values = [] for item in items: + if item.name not in custom_fields.keys(): + continue + mapped_type = CUSTOM_FIELD_DATA_TYPE_MAP.get(item.type) if item.custom_field_value_id is None: # prepare insert values @@ -245,7 +195,7 @@ def update_document_custom_fields( field_id=item.custom_field_id, ) if item.type == "date": - v[f"value_{mapped_type}"] = str2date(item.value) + v[f"value_{mapped_type}"] = str2date(custom_fields[item.name]) else: v[f"value_{mapped_type}"] = item.value insert_values.append(v) diff --git a/papermerge/core/db/models.py b/papermerge/core/db/models.py index 172c8254b..f91010cbf 100644 --- a/papermerge/core/db/models.py +++ b/papermerge/core/db/models.py @@ -266,9 +266,6 @@ class CustomFieldValue(Base): ForeignKey("core_document.basetreenode_ptr_id") ) field_id: Mapped[UUID] = mapped_column(ForeignKey("core_customfield.id")) - field = relationship( - "CustomField", primaryjoin="CustomField.id == CustomFieldValue.field_id" - ) value_text: Mapped[str] value_bool: Mapped[bool] value_url: Mapped[str]