MHR API verify new CI script. (#2081)

* MHR API verify new CI script. Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script. Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script - remove legacy test. Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script - update db upgrade. Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script - ut report url. Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script update tests Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script update tests Signed-off-by: Doug Lovett <[email protected]> * MHR API verify new CI script update tests Signed-off-by: Doug Lovett <[email protected]> --------- Signed-off-by: Doug Lovett <[email protected]>
bcgov · Dec 12, 2024 · b84120a · b84120a
1 parent dde71c3
commit b84120a
Show file tree

Hide file tree

Showing 15 changed files with 59 additions and 581 deletions.
diff --git a/mhr-api/devops/vaults.gcp.env b/mhr-api/devops/vaults.gcp.env
@@ -46,4 +46,3 @@ DOC_SERVICE_CONFIG='op://ppr/$APP_ENV/mhr-api/DOC_SERVICE_CONFIG'
 GUNICORN_PROCESSES="op://ppr/$APP_ENV/mhr-api/GUNICORN_PROCESSES"
 GUNICORN_THREADS="op://ppr/$APP_ENV/mhr-api/GUNICORN_THREADS"
 REPORT_VERSION="op://ppr/$APP_ENV/mhr-api/REPORT_VERSION"
-REPORT_API_AUDIENCE=op://API/$APP_ENV/report-api-gotenberg/REPORT_API_GOTENBERG_URL"
diff --git a/mhr-api/gunicorn_config.py b/mhr-api/gunicorn_config.py
@@ -17,8 +17,8 @@
 
 import os
 
-workers = int(os.environ.get('GUNICORN_PROCESSES', '1'))  # pylint: disable=invalid-name
-threads = int(os.environ.get('GUNICORN_THREADS', '1'))  # pylint: disable=invalid-name
+workers = int(os.environ.get("GUNICORN_PROCESSES", "1"))  # pylint: disable=invalid-name
+threads = int(os.environ.get("GUNICORN_THREADS", "1"))  # pylint: disable=invalid-name
 
-forwarded_allow_ips = '*'  # pylint: disable=invalid-name
-secure_scheme_headers = {'X-Forwarded-Proto': 'https'}  # pylint: disable=invalid-name
+forwarded_allow_ips = "*"  # pylint: disable=invalid-name
+secure_scheme_headers = {"X-Forwarded-Proto": "https"}  # pylint: disable=invalid-name
diff --git a/mhr-api/migrations/versions/0001_rebase_2024-10.py b/mhr-api/migrations/versions/0001_rebase_2024-10.py
@@ -23,26 +23,9 @@
 
 def upgrade():
     # ### Manually install extensions. ###
-    # op.execute("CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;")
-    # op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-    # op.execute("CREATE EXTENSION IF NOT EXISTS btree_gist;")
-    public_fuzzystrmatch = PGExtension(
-        schema="public",
-        signature="fuzzystrmatch"
-    )
-    op.create_entity(public_fuzzystrmatch)
-
-    public_btree_gist = PGExtension(
-        schema="public",
-        signature="btree_gist"
-    )
-    op.create_entity(public_btree_gist)
-
-    public_pg_trgm = PGExtension(
-        schema="public",
-        signature="pg_trgm"
-    )
-    op.create_entity(public_pg_trgm)
+    op.execute("CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS btree_gist;")
 
     # ### Manually create sequences and add them to pk columns. ###
     # PPR sequences
@@ -3626,25 +3609,6 @@ def downgrade():
     op.execute(DropSequence(Sequence('word_id_seq')))
     op.execute(DropSequence(Sequence('name_id_seq')))
 
-    # Manually added drop extensions
-    public_pg_trgm = PGExtension(
-        schema="public",
-        signature="pg_trgm"
-    )
-    op.drop_entity(public_pg_trgm)
-
-    public_btree_gist = PGExtension(
-        schema="public",
-        signature="btree_gist"
-    )
-    op.drop_entity(public_btree_gist)
-
-    public_fuzzystrmatch = PGExtension(
-        schema="public",
-        signature="fuzzystrmatch"
-    )
-    op.drop_entity(public_fuzzystrmatch)
-
     public_mhr_account_reg_vw = PGView(
         schema="public",
         signature="mhr_account_reg_vw",

diff --git a/mhr-api/src/mhr_api/__init__.py b/mhr-api/src/mhr_api/__init__.py
@@ -34,9 +34,7 @@
 from mhr_api.utils.auth import jwt
 from mhr_api.utils.logging import logger, setup_logging
 
-setup_logging(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml")
-)  # important to do this first
+setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml"))  # important to do this first
 
 
 def create_app(service_environment=APP_RUNNING_ENVIRONMENT, **kwargs):
@@ -49,16 +47,12 @@ def create_app(service_environment=APP_RUNNING_ENVIRONMENT, **kwargs):
 
     db.init_app(app)
     Migrate(app, db)
-    if (
-        app.config.get("DEPLOYMENT_ENV", "") == "unitTesting"
-    ):  # CI only run upgrade for unit testing.
+    if app.config.get("DEPLOYMENT_ENV", "") == "testing":  # CI only run upgrade for unit testing.
         logger.info("Running db upgrade.")
         with app.app_context():
             upgrade(directory="migrations", revision="head", sql=False, tag=None)
         # Alembic has it's own logging config, we'll need to restore our logging here.
-        setup_logging(
-            os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml")
-        )
+        setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml"))
         logger.info("Finished db upgrade.")
     else:
         logger.info("Logging, migrate set up.")
@@ -112,9 +106,7 @@ def setup_test_data():
     """Load unit test data in the dev/local environment. Delete all existing test data as a first step."""
     try:
         test_path = os.getcwd()
-        logger.info(
-            f"Executing DB scripts to create test data from test data dir {test_path}..."
-        )
+        logger.info(f"Executing DB scripts to create test data from test data dir {test_path}...")
         # execute_script(db.session, os.path.join(test_path, "test_data/postgres_test_reset.sql"))
         execute_script(db.session, "test_data/postgres_create_first.sql")
         filenames = os.listdir(os.path.join(test_path, "test_data/postgres_data_files"))
@@ -126,16 +118,12 @@ def setup_test_data():
             )
         # execute_script(db.session, "test_data/postgres_test_reset_ppr.sql")
         execute_script(db.session, "test_data/postgres_create_first_ppr.sql")
-        filenames = os.listdir(
-            os.path.join(os.getcwd(), "test_data/postgres_data_files_ppr")
-        )
+        filenames = os.listdir(os.path.join(os.getcwd(), "test_data/postgres_data_files_ppr"))
         sorted_names = sorted(filenames)
         for filename in sorted_names:
             execute_script(
                 db.session,
-                os.path.join(
-                    os.getcwd(), ("test_data/postgres_data_files_ppr/" + filename)
-                ),
+                os.path.join(os.getcwd(), ("test_data/postgres_data_files_ppr/" + filename)),
             )
     except Exception as err:  # pylint: disable=broad-except # noqa F841;
         logger.error(f"setup_test_data failed: {str(err)}")

diff --git a/mhr-api/src/mhr_api/config.py b/mhr-api/src/mhr_api/config.py
@@ -82,7 +82,7 @@ class Config:  # pylint: disable=too-few-public-methods
     AUTH_API_VERSION = os.getenv("AUTH_API_VERSION", "")
     PAY_API_URL = os.getenv("PAY_API_URL", "")
     PAY_API_VERSION = os.getenv("PAY_API_VERSION", "")
-    REPORT_API_URL = os.getenv("REPORT_API_URL", "")
+    REPORT_API_URL = os.getenv("REPORT_API_URL", "https://gotenberg-p56lvhvsqa-nn.a.run.app")
 
     AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}"
     PAYMENT_SVC_URL = f"{PAY_API_URL + PAY_API_VERSION}"
@@ -101,9 +101,7 @@ class Config:  # pylint: disable=too-few-public-methods
     if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None):
         SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}"
     else:
-        SQLALCHEMY_DATABASE_URI = (
-            f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-        )
+        SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
 
     # Connection pool settings
     DB_MIN_POOL_SIZE = os.getenv("DATABASE_MIN_POOL_SIZE", "2")
@@ -142,9 +140,7 @@ class Config:  # pylint: disable=too-few-public-methods
     # ACCOUNT_SVC_TIMEOUT = os.g
 
     # DB Query limits on result set sizes
-    ACCOUNT_REGISTRATIONS_MAX_RESULTS = os.getenv(
-        "ACCOUNT_REGISTRATIONS_MAX_RESULTS", "100"
-    )
+    ACCOUNT_REGISTRATIONS_MAX_RESULTS = os.getenv("ACCOUNT_REGISTRATIONS_MAX_RESULTS", "100")
     ACCOUNT_DRAFTS_MAX_RESULTS = os.getenv("ACCOUNT_DRAFTS_MAX_RESULTS", "1000")
     ACCOUNT_SEARCH_MAX_RESULTS = os.getenv("ACCOUNT_SEARCH_MAX_RESULTS", "1000")
 
@@ -159,45 +155,31 @@ class Config:  # pylint: disable=too-few-public-methods
 
     # Google APIs and cloud storage
     GOOGLE_DEFAULT_SA = os.getenv("GOOGLE_DEFAULT_SA")
-    GCP_CS_SA_SCOPES = os.getenv(
-        "GCP_CS_SA_SCOPES", "https://www.googleapis.com/auth/cloud-platform"
-    )
+    GCP_CS_SA_SCOPES = os.getenv("GCP_CS_SA_SCOPES", "https://www.googleapis.com/auth/cloud-platform")
     # Storage of search reports
     GCP_CS_BUCKET_ID = os.getenv("GCP_CS_BUCKET_ID", "mhr_search_result_report_dev")
     # Storage of registration verification reports
-    GCP_CS_BUCKET_ID_REGISTRATION = os.getenv(
-        "GCP_CS_BUCKET_ID_REGISTRATION", "mhr_registration_report_dev"
-    )
+    GCP_CS_BUCKET_ID_REGISTRATION = os.getenv("GCP_CS_BUCKET_ID_REGISTRATION", "mhr_registration_report_dev")
     # Storage of batch reports
     GCP_CS_BUCKET_ID_BATCH = os.getenv("GCP_CS_BUCKET_ID_BATCH", "mhr_batch_report_dev")
     # Storage of service agreement terms reports
-    GCP_CS_BUCKET_ID_TERMS = os.getenv(
-        "GCP_CS_BUCKET_ID_TERMS", "mhr_service_agreement_dev"
-    )
+    GCP_CS_BUCKET_ID_TERMS = os.getenv("GCP_CS_BUCKET_ID_TERMS", "mhr_service_agreement_dev")
 
     # Pub/Sub
     GCP_PS_PROJECT_ID = os.getenv("DEPLOYMENT_PROJECT", "eogruh-dev")
-    GCP_PS_SEARCH_REPORT_TOPIC = os.getenv(
-        "GCP_PS_SEARCH_REPORT_TOPIC", "mhr-search-report"
-    )
-    GCP_PS_REGISTRATION_REPORT_TOPIC = os.getenv(
-        "GCP_PS_REGISTRATION_REPORT_TOPIC", "mhr-registration-report"
-    )
+    GCP_PS_SEARCH_REPORT_TOPIC = os.getenv("GCP_PS_SEARCH_REPORT_TOPIC", "mhr-search-report")
+    GCP_PS_REGISTRATION_REPORT_TOPIC = os.getenv("GCP_PS_REGISTRATION_REPORT_TOPIC", "mhr-registration-report")
 
     GATEWAY_URL = os.getenv("GATEWAY_URL", "https://bcregistry-dev.apigee.net")
-    GATEWAY_LTSA_URL = os.getenv(
-        "GATEWAY_LTSA_URL", "https://bcregistry-test.apigee.net/ltsa/api/v1"
-    )
+    GATEWAY_LTSA_URL = os.getenv("GATEWAY_LTSA_URL", "https://bcregistry-test.apigee.net/ltsa/api/v1")
     SUBSCRIPTION_API_KEY = os.getenv("SUBSCRIPTION_API_KEY")
     GATEWAY_API_KEY = os.getenv("GATEWAY_API_KEY")
 
     # Search results data size threshold for real time reports.
     MAX_SIZE_SEARCH_RT: int = int(os.getenv("MAX_SIZE_SEARCH_RT", "200000"))
     # Default 2, set to 1 to revert to original report api client
     REPORT_VERSION = os.getenv("REPORT_VERSION", "2")
-    REPORT_API_AUDIENCE = os.getenv(
-        "REPORT_API_AUDIENCE", "https://gotenberg-p56lvhvsqa-nn.a.run.app"
-    )
+    REPORT_API_AUDIENCE = os.getenv("REPORT_API_AUDIENCE", "https://gotenberg-p56lvhvsqa-nn.a.run.app")
 
     NOTIFY_MAN_REG_CONFIG = os.getenv("NOTIFY_MAN_REG_CONFIG")
     NOTIFY_LOCATION_CONFIG = os.getenv("NOTIFY_LOCATION_CONFIG")
@@ -250,9 +232,7 @@ class UnitTestingConfig(Config):  # pylint: disable=too-few-public-methods
     DB_HOST = os.getenv("DATABASE_TEST_HOST", "")
     DB_PORT = os.getenv("DATABASE_TEST_PORT", "5432")
     # SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-    SQLALCHEMY_DATABASE_URI = (
-        f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-    )
+    SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
 
     # JWT OIDC settings
     # JWT_OIDC_TEST_MODE will set jwt_manager to use

diff --git a/mhr-api/src/mhr_api/utils/admin_validator.py b/mhr-api/src/mhr_api/utils/admin_validator.py
@@ -18,7 +18,7 @@
 from mhr_api.models import MhrRegistration
 from mhr_api.models import registration_utils as reg_utils
 from mhr_api.models.type_tables import MhrDocumentTypes, MhrNoteStatusTypes, MhrRegistrationTypes
-from mhr_api.utils import validator_utils, validator_owner_utils
+from mhr_api.utils import validator_owner_utils, validator_utils
 from mhr_api.utils.logging import logger
 
 NCAN_DOC_TYPES = " CAU CAUC CAUE NCON NPUB REGC REST "  # Set of doc types NCAN can cancel.