24481 Tombstone pipeline - Updates to support dissolution (bcgov#3109)

* 24481 - tombstone pipeline - update to support dissolution Signed-off-by: Hongjing Chen <[email protected]> * comment Signed-off-by: Hongjing Chen <[email protected]> --------- Signed-off-by: Hongjing Chen <[email protected]>
leodube-aot · Dec 3, 2024 · 969d921 · 969d921
1 parent d96b756
commit 969d921
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 15 deletions.
diff --git a/data-tool/flows/tombstone/tombstone_mappings.py b/data-tool/flows/tombstone/tombstone_mappings.py
@@ -87,11 +87,13 @@ class EventFilings(str, Enum):
     # Dissolution
     ## voluntary
     FILE_ADVD2 = 'FILE_ADVD2'
-    FILE_DISLV = 'FILE_DISLV'
     FILE_ADVDS = 'FILE_ADVDS'
+    DISLV_NULL = 'DISLV_NULL'
     ## admin
     DISLC_NULL = 'DISLC_NULL'
     FILE_ADCOL = 'FILE_ADCOL'
+    SYSDA_NULL = 'SYSDA_NULL'
+    SYSDS_NULL = 'SYSDS_NULL'
     # involuntary
     SYSDF_NULL = 'SYSDF_NULL'
     SYSDT_NULL = 'SYSDT_NULL'
@@ -100,7 +102,7 @@ class EventFilings(str, Enum):
     FILE_ICORP = 'FILE_ICORP'
     FILE_ICORU = 'FILE_ICORU'
     FILE_ICORC = 'FILE_ICORC'
-    CONVICORP_NULL = 'CONVICORP_NULL'
+    CONVICORP_NULL = 'CONVICORP_NULL'  # TODO: may need to be removed
 
     # TODO: Ledger - unsupported
     # TODO: Liquidation - unsupported
@@ -130,7 +132,9 @@ class EventFilings(str, Enum):
     FILE_TRANP = 'FILE_TRANP'
 
     # TODO:
-    # Other COLIN events: CONV*, Adim Corp (ADCORP, BNUPD, ADMIN), XPRO filing
+    # Other COLIN events:
+    # CONV*, Adim Corp (ADCORP, BNUPD, ADMIN), XPRO filing
+    # SYSDL, SYST
 
 
     @classmethod
@@ -208,10 +212,12 @@ def has_value(cls, value):
     EventFilings.DISD2_DISDE: 'delayOfDissolution',
 
     EventFilings.FILE_ADVD2: ['dissolution', 'voluntary'],
-    EventFilings.FILE_DISLV: ['dissolution', 'voluntary'],
     EventFilings.FILE_ADVDS: ['dissolution', 'voluntary'],
+    EventFilings.DISLV_NULL: ['dissolution', 'voluntary'],
     EventFilings.DISLC_NULL: ['dissolution', 'administrative'],
     EventFilings.FILE_ADCOL: ['dissolution', 'administrative'],
+    EventFilings.SYSDA_NULL: ['dissolution', 'administrative'],
+    EventFilings.SYSDS_NULL: ['dissolution', 'administrative'],
     EventFilings.SYSDF_NULL: ['dissolution', 'involuntary'],
     EventFilings.SYSDT_NULL: ['dissolution', 'involuntary'],
 

diff --git a/data-tool/flows/tombstone/tombstone_queries.py b/data-tool/flows/tombstone/tombstone_queries.py
@@ -24,6 +24,7 @@ def get_unprocessed_corps_query(flow_name, environment, batch_size):
 --    and c.corp_num = 'BC0002567' -- lots of filings - IA, ARs, transition, COD
 --    and c.corp_num in ('BC0068889', 'BC0441359') -- test users mapping
 --    and c.corp_num in ('BC0326163', 'BC0046540', 'BC0883637', 'BC0043406', 'BC0068889', 'BC0441359')
+--    and c.corp_num in ('BC0472301', 'BC0649417', 'BC0808085', 'BC0803411', 'BC0756111', 'BC0511226', 'BC0833000', 'BC0343855', 'BC0149266') -- dissolution
     and c.corp_type_cd in ('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE') -- TODO: update transfer script
     and cs.end_event_id is null
     and ((cp.processed_status is null or cp.processed_status != 'COMPLETED'))
@@ -452,7 +453,7 @@ def get_filings_query(corp_num):
 --          and e.corp_num = 'BC0068889'
 --          and e.corp_num = 'BC0449924'  -- AR, ADCORP
 --        and e.trigger_dts is not null
-        order by e.event_id
+        order by e.event_timerstamp
         ;
     """
     return query

diff --git a/data-tool/flows/tombstone/tombstone_utils.py b/data-tool/flows/tombstone/tombstone_utils.py
@@ -5,14 +5,13 @@
 import pandas as pd
 import pytz
 from sqlalchemy import Connection, text
-from tombstone.tombstone_base_data import (ALIAS, FILING, OFFICE, PARTY,
-                                           PARTY_ROLE, RESOLUTION,
+from tombstone.tombstone_base_data import (ALIAS, FILING, FILING_JSON, OFFICE,
+                                           PARTY, PARTY_ROLE, RESOLUTION,
                                            SHARE_CLASSES, USER)
 from tombstone.tombstone_mappings import (EVENT_FILING_LEAR_TARGET_MAPPING,
                                           LEAR_FILING_BUSINESS_UPDATE_MAPPING,
                                           LEAR_STATE_FILINGS)
 
-
 unsupported_event_file_types = set()
 
 
@@ -246,13 +245,8 @@ def format_filings_data(data: dict) -> list[dict]:
         if not effective_date:
             effective_date = x['e_event_dt_str']
 
-        meta_data = {
-            'colinFilingInfo': {
-                'eventType': x['e_event_type_cd'],
-                'filingType': x['f_filing_type_cd']
-            },
-            'isLedgerPlaceholder': True,
-        }
+        filing_json, meta_data = build_filing_json_meta_data(filing_type, filing_subtype,
+                                                             effective_date, x)
 
         filing = copy.deepcopy(FILING)
 
@@ -267,6 +261,7 @@ def format_filings_data(data: dict) -> list[dict]:
             'filing_sub_type': filing_subtype,
             'completion_date': effective_date,
             'effective_date': effective_date,
+            'filing_json': filing_json,
             'meta_data': meta_data,
             'submitter_id': user_id  # will be updated to real user_id when loading data into db
         }
@@ -361,6 +356,35 @@ def get_target_filing_type(event_file_type: str) -> tuple[str, str]:
     return filing_type, filing_subtype
 
 
+def build_filing_json_meta_data(filing_type: str, filing_subtype: str, effective_date: str, data: dict) -> tuple[dict, dict]:
+    filing_json = copy.deepcopy(FILING_JSON)
+    filing_json['filing'][filing_type] = {}
+
+    meta_data = {
+        'colinFilingInfo': {
+            'eventType': data['e_event_type_cd'],
+            'filingType': data['f_filing_type_cd']
+        },
+        'isLedgerPlaceholder': True,
+    }
+
+    if filing_type == 'annualReport':
+        meta_data['annualReport'] = {
+            'annualReportFilingYear': int(effective_date[:4]),
+        }
+    elif filing_type == 'dissolution':
+        meta_data['dissolution'] = {
+            'dissolutionType': filing_subtype,
+        }
+        filing_json['filing']['dissolution'] = {
+            'dissolutionType': filing_subtype,
+            'dissolutionDate': effective_date[:10],
+        }
+    # TODO: populate meta_data for correction to display correct filing name
+
+    return filing_json, meta_data
+
+
 def build_epoch_filing(business_id: int) -> dict:
     now = datetime.utcnow().replace(tzinfo=pytz.UTC)
     filing = copy.deepcopy(FILING)