24454 - EFT TDI17 Processing updates (#1830)

jobs/payment-jobs/pyproject.toml

@@ -7,7 +7,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.12"
-pay-api = {git = "https://github.com/seeker25/sbc-pay.git", branch = "20767_p3", subdirectory = "pay-api"}
+pay-api = {git = "https://github.com/ochiu/sbc-pay.git", branch = "24454-EFT-TDI17-Processing-Updates", subdirectory = "pay-api"}
 flask = "^3.0.2"
 flask-sqlalchemy = "^3.1.1"
 sqlalchemy = "^2.0.28"

pay-api/migrations/versions/2024_11_19_8bd139bbb602_.py

@@ -0,0 +1,26 @@
+"""24454 - EFT File column clean up
+
+Revision ID: 8bd139bbb602
+Revises: 474917a13bd4
+Create Date: 2024-11-19 21:20:17.080264
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '8bd139bbb602'
+down_revision = '474917a13bd4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('eft_files', schema=None) as batch_op:
+        batch_op.drop_column('total_deposit_cents')
+        batch_op.drop_column('number_of_details')
+
+
+def downgrade():
+    with op.batch_alter_table('eft_files', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('number_of_details', sa.INTEGER(), autoincrement=False, nullable=True))
+        batch_op.add_column(sa.Column('total_deposit_cents', sa.BIGINT(), autoincrement=False, nullable=True))

pay-api/src/pay_api/models/eft_file.py

@@ -46,9 +46,7 @@ class EFTFile(BaseModel):  # pylint: disable=too-many-instance-attributes
             "deposit_to_date",
             "file_creation_date",
             "file_ref",
-            "number_of_details",
-            "status_code",
-            "total_deposit_cents",
+            "status_code"
         ]
     }
 
@@ -63,8 +61,6 @@ class EFTFile(BaseModel):  # pylint: disable=too-many-instance-attributes
     deposit_from_date = db.Column("deposit_from_date", db.DateTime, nullable=True)
     deposit_to_date = db.Column("deposit_to_date", db.DateTime, nullable=True)
     file_creation_date = db.Column("file_creation_date", db.DateTime, nullable=True)
-    number_of_details = db.Column("number_of_details", db.Integer, nullable=True)
-    total_deposit_cents = db.Column("total_deposit_cents", db.BigInteger, nullable=True)
     file_ref = db.Column("file_ref", db.String, nullable=False, index=True)
     status_code = db.Column(
         db.String,

pay-api/src/pay_api/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = "1.22.8"  # pylint: disable=invalid-name
+__version__ = "1.22.9"  # pylint: disable=invalid-name

pay-queue/devops/vaults.gcp.env

@@ -31,3 +31,6 @@ NOTIFY_API_URL="op://API/$APP_ENV/notify-api/NOTIFY_API_URL"
 NOTIFY_API_VERSION="op://API/$APP_ENV/notify-api/NOTIFY_API_VERSION"
 JWT_OIDC_ISSUER="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_ISSUER"
 PAY_CONNECTOR_AUTH="op://relationship/$APP_ENV/pay-api/PAY_CONNECTOR_AUTH"
+EFT_TDI17_LOCATION_ID="op://relationship/$APP_ENV/pay-queue/EFT_TDI17_LOCATION_ID"
+EFT_WIRE_PATTERNS="op://relationship/$APP_ENV/pay-queue/EFT_WIRE_PATTERNS"
+EFT_PATTERNS="op://relationship/$APP_ENV/pay-queue/EFT_PATTERNS"

pay-queue/pyproject.toml

@@ -19,7 +19,7 @@ itsdangerous = "^2.1.2"
 protobuf = "4.25.3"
 launchdarkly-server-sdk = "^8.2.1"
 cachecontrol = "^0.14.0"
-pay-api = {git = "https://github.com/seeker25/sbc-pay.git", branch = "20767_p3", subdirectory = "pay-api"}
+pay-api = {git = "https://github.com/ochiu/sbc-pay.git", branch = "24454-EFT-TDI17-Processing-Updates", subdirectory = "pay-api"}
 pg8000 = "^1.30.5"
 
 

pay-queue/src/pay_queue/config.py

@@ -50,6 +50,11 @@ def get_named_config(config_name: str = "production"):
     return app_config
 
 
+def get_comma_delimited_string_as_tuple(value: str) -> tuple:
+    """Get comma delimited string as a tuple."""
+    return tuple(val.strip() for val in value.split(",") if val.strip())
+
+
 class _Config:  # pylint: disable=too-few-public-methods,protected-access
     """Base class configuration that should set reasonable defaults.
 
@@ -93,6 +98,8 @@ class _Config:  # pylint: disable=too-few-public-methods,protected-access
 
     # EFT Config
     EFT_TDI17_LOCATION_ID = os.getenv("EFT_TDI17_LOCATION_ID")
+    EFT_WIRE_PATTERNS = get_comma_delimited_string_as_tuple(os.getenv("EFT_WIRE_PATTERNS", ""))
+    EFT_PATTERNS = get_comma_delimited_string_as_tuple(os.getenv("EFT_PATTERNS", ""))
 
     # Secret key for encrypting bank account
     ACCOUNT_SECRET_KEY = os.getenv("ACCOUNT_SECRET_KEY")

pay-queue/src/pay_queue/services/eft/eft_base.py

@@ -58,7 +58,7 @@
 """This manages the EFT base class."""
 import decimal
 from datetime import datetime
-from typing import List
+from typing import List, Tuple
 
 from pay_queue.services.eft.eft_enums import EFTConstants
 from pay_queue.services.eft.eft_errors import EFTError
@@ -140,6 +140,13 @@ def parse_datetime(self, datetime_str: str, error: EFTError) -> decimal:
 
         return result
 
+    def find_matching_pattern(self, patterns: Tuple, value: str) -> str:
+        """Find matching pattern for a value."""
+        for pattern in patterns:
+            if value.startswith(pattern):
+                return pattern
+        return None
+
     def add_error(self, error: EFTParseError):
         """Add parse error to error array."""
         error.index = self.index

pay-queue/src/pay_queue/services/eft/eft_reconciliation.py

@@ -61,6 +61,22 @@ def eft_error_handling(self, row, error_msg: str, capture_error: bool = True, ta
             send_error_email(email_service_params)
 
 
+def _validate_configuration(context: EFTReconciliation) -> bool:
+    """Validate required configuration is available."""
+    is_valid = True
+    if current_app.config.get("EFT_TDI17_LOCATION_ID") is None:
+        is_valid = False
+        context.eft_error_handling("N/A", "Missing EFT_TDI17_LOCATION_ID configuration")
+    if not current_app.config.get("EFT_WIRE_PATTERNS"):
+        is_valid = False
+        context.eft_error_handling("N/A", "Missing EFT_WIRE_PATTERNS configuration")
+    if not current_app.config.get("EFT_PATTERNS"):
+        is_valid = False
+        context.eft_error_handling("N/A", "Missing EFT_PATTERNS configuration")
+
+    return is_valid
+
+
 def reconcile_eft_payments(ce):  # pylint: disable=too-many-locals
     """Read the TDI17 file, create processing records and update payment details.
 
@@ -76,12 +92,11 @@ def reconcile_eft_payments(ce):  # pylint: disable=too-many-locals
     8: Finalize and complete
     """
     context = EFTReconciliation(ce)
+    if not _validate_configuration(context):
+        return
 
     # Used to filter transactions by location id to isolate EFT specific transactions from the TDI17
     eft_location_id = current_app.config.get("EFT_TDI17_LOCATION_ID")
-    if eft_location_id is None:
-        context.eft_error_handling("N/A", "Missing EFT_TDI17_LOCATION_ID configuration")
-        return
 
     # Fetch EFT File
     file = get_object(context.minio_location, context.file_name)
@@ -268,9 +283,6 @@ def _process_eft_trailer(eft_trailer: EFTTrailer, eft_file_model: EFTFileModel)
         )
         return False
 
-    # Populate header and trailer data on EFT File record - values will return None if parsing failed
-    _set_eft_trailer_on_file(eft_trailer, eft_file_model)
-
     # Errors on parsing trailer - create EFT error records
     if eft_trailer is not None and eft_trailer.has_errors():
         _save_eft_trailer_error(eft_trailer, eft_file_model)
@@ -335,12 +347,6 @@ def _set_eft_header_on_file(eft_header: EFTHeader, eft_file_model: EFTFileModel)
     eft_file_model.deposit_to_date = getattr(eft_header, "ending_deposit_date", None)
 
 
-def _set_eft_trailer_on_file(eft_trailer: EFTTrailer, eft_file_model: EFTFileModel):
-    """Set EFT Trailer information on EFTFile model."""
-    eft_file_model.number_of_details = getattr(eft_trailer, "number_of_details", None)
-    eft_file_model.total_deposit_cents = getattr(eft_trailer, "total_deposit_amount", None)
-
-
 def _set_eft_base_error(line_type: str, index: int, eft_file_id: int, error_messages: [str]) -> EFTTransactionModel:
     """Instantiate EFT Transaction model error record."""
     eft_transaction_model = (

pay-queue/src/pay_queue/services/eft/eft_record.py

@@ -16,6 +16,7 @@
 from datetime import datetime
 from typing import Tuple
 
+from flask import current_app
 from pay_api.utils.enums import EFTShortnameType
 
 from pay_queue.services.eft.eft_base import EFTBase
@@ -27,11 +28,8 @@
 class EFTRecord(EFTBase):
     """Defines the structure of the transaction record of a received EFT file."""
 
-    PAD_DESCRIPTION_PATTERN = "MISC PAYMENT BCONLINE"
-    EFT_DESCRIPTION_PATTERN = "MISC PAYMENT"
-    WIRE_DESCRIPTION_PATTERN = "FUNDS TRANSFER CR TT"
-    FEDERAL_PAYMENT_DESCRIPTION_PATTERN = "FEDERAL PAYMENT CANADA"
-    GENERATE_SHORT_NAME_PATTERNS: Tuple = (FEDERAL_PAYMENT_DESCRIPTION_PATTERN,)
+    eft_patterns: Tuple
+    eft_wire_patterns: Tuple
 
     ministry_code: str
     program_code: str
@@ -54,6 +52,8 @@ class EFTRecord(EFTBase):
     def __init__(self, content: str, index: int):
         """Return an EFT Transaction record."""
         super().__init__(content, index)
+        self.eft_patterns = current_app.config.get("EFT_PATTERNS")
+        self.eft_wire_patterns: Tuple = current_app.config.get("EFT_WIRE_PATTERNS")
         self._process()
 
     @staticmethod
@@ -113,20 +113,17 @@ def parse_transaction_description(self):
         if not self.transaction_description:
             return
 
-        if self.transaction_description.startswith(self.GENERATE_SHORT_NAME_PATTERNS):
-            self.short_name_type = EFTShortnameType.EFT.value
-            self.transaction_description = self.FEDERAL_PAYMENT_DESCRIPTION_PATTERN.strip()
-            self.generate_short_name = True
-            return
-
-        if self.transaction_description.startswith(self.WIRE_DESCRIPTION_PATTERN):
+        if matching_pattern := self.find_matching_pattern(self.eft_wire_patterns, self.transaction_description):
             self.short_name_type = EFTShortnameType.WIRE.value
-            self.transaction_description = self.transaction_description[len(self.WIRE_DESCRIPTION_PATTERN) :].strip()
+            self.transaction_description = self.transaction_description[len(matching_pattern) :].strip()
             return
 
-        # Check if this a PAD or EFT Transaction
-        if self.transaction_description.startswith(
-            self.EFT_DESCRIPTION_PATTERN
-        ) and not self.transaction_description.startswith(self.PAD_DESCRIPTION_PATTERN):
+        if matching_pattern := self.find_matching_pattern(self.eft_patterns, self.transaction_description):
             self.short_name_type = EFTShortnameType.EFT.value
-            self.transaction_description = self.transaction_description[len(self.EFT_DESCRIPTION_PATTERN) :].strip()
+            self.transaction_description = self.transaction_description[len(matching_pattern) :].strip()
+            return
+
+        # Undefined patterns will generate a short name and default to type EFT
+        self.short_name_type = EFTShortnameType.EFT.value
+        self.transaction_description = self.transaction_description.strip()
+        self.generate_short_name = True

pay-queue/tests/conftest.py

@@ -25,6 +25,7 @@
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
 from pay_queue import create_app
+from pay_queue.config import get_comma_delimited_string_as_tuple
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -181,6 +182,10 @@ def publish(self, *args, **kwargs):
 
 
 @pytest.fixture(scope="session", autouse=True)
-def set_eft_tdi17_location_id(app):
-    """Set TDI17 Location ID for tests."""
+def set_eft_configuration(app):
+    """Set EFT TDI17 processing configuration for tests."""
     app.config["EFT_TDI17_LOCATION_ID"] = "85004"
+    app.config["EFT_WIRE_PATTERNS"] = get_comma_delimited_string_as_tuple("FUNDS TRANSFER CR TT")
+    app.config["EFT_PATTERNS"] = get_comma_delimited_string_as_tuple(
+        "ACCOUNT PAYABLE PMT,BILL PAYMENT,COMM BILL PAYMENT,MISC PAYMENT,PAYROLL DEPOSIT"
+    )