generated from bcgov/quickstart-openshift
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Co-authored-by: Derek Roberts <[email protected]>
- Loading branch information
1 parent
0e3d4f1
commit 69f57d2
Showing
14 changed files
with
367 additions
and
657 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
sync/config/SQL/SPAR/POSTGRES_SEEDLOT_PARENT_TREE_GEN_QLTY.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| SELECT | ||
| soqgq.seedlot_number | ||
| , soqgq.parent_tree_id | ||
| , soqgq.genetic_type_code | ||
| , soqgq.genetic_worth_code | ||
| , soqgq.genetic_quality_value | ||
| , soqgq.estimated_ind | ||
| , soqgq.untested_ind | ||
| , aos.seed_plan_unit_id | ||
| , soqgq.revision_count | ||
| FROM spar.seedlot_parent_tree_gen_qlty soqgq | ||
| LEFT | ||
| OUTER | ||
| JOIN spar.seedlot_orchard so | ||
| ON so.seedlot_number = soqgq.seedlot_number | ||
| AND so.primary_ind = true | ||
| LEFT | ||
| OUTER | ||
| JOIN spar.active_orchard_spu aos | ||
| ON aos.orchard_id = so.orchard_id | ||
| WHERE soqgq.seedlot_number = %(p_seedlot_number)s | ||
| ORDER BY soqgq.seedlot_number | ||
| , soqgq.parent_tree_id |
11 changes: 11 additions & 0 deletions
11
sync/config/SQL/SPAR/POSTGRES_SEEDLOT_PARENT_TREE_SMP_MIX.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| SELECT | ||
| seedlot_number | ||
| , parent_tree_id | ||
| , genetic_type_code | ||
| , genetic_worth_code | ||
| , genetic_quality_value smp_mix_value | ||
| , revision_count | ||
| FROM spar.seedlot_parent_tree_smp_mix | ||
| WHERE seedlot_number = %(p_seedlot_number)s | ||
| ORDER BY seedlot_number | ||
| , parent_tree_id |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| import json | ||
| import module.data_synchronization as data_sync | ||
| from logging import config as logging_config, basicConfig as loggingBasicConfig, DEBUG as loggingDEBUG, INFO as loggingINFO | ||
| import os | ||
| import sys | ||
| import yaml | ||
|
|
||
| def env_var_is_filled(variable): | ||
| if os.environ.get(variable) is None: | ||
| print("Error: "+variable+" environment variable is None") | ||
| return False | ||
| return True | ||
|
|
||
| def generate_db_config(type_,schema_,settings): | ||
| dbconfig = {} | ||
| ssl_required = settings["ssl_required"] | ||
| version_column = settings["version_column"] | ||
| max_rows_upsert = settings["max_rows_upsert"] | ||
| if type_ == "ORACLE": | ||
| dbconfig = { | ||
| "type": "ORACLE", | ||
| "username": os.environ.get("ORACLE_USER"), | ||
| "password": os.environ.get("ORACLE_PASSWORD"), | ||
| "host": os.environ.get("ORACLE_HOST"), | ||
| "port": os.environ.get("ORACLE_PORT"), | ||
| "service_name": os.environ.get("ORACLE_SERVICE"), | ||
| "schema": schema_, | ||
| "test_query": "SELECT 'SUCCESS' a FROM DUAL", | ||
| "ssl_required": ssl_required, | ||
| "version_column": version_column, | ||
| "max_rows_upsert": max_rows_upsert | ||
| } | ||
| if type_ == "POSTGRES": | ||
| dbconfig = { | ||
| "type": "POSTGRES", | ||
| "username": os.environ.get("POSTGRES_USER"), | ||
| "password": os.environ.get("POSTGRES_PASSWORD"), | ||
| "host": os.environ.get("POSTGRES_HOST"), | ||
| "database": os.environ.get("POSTGRES_DATABASE"), | ||
| "port": os.environ.get("POSTGRES_PORT"), | ||
| "schema": schema_, | ||
| "test_query": "SELECT 'SUCCESS' a", | ||
| "ssl_required": ssl_required, | ||
| "version_column": version_column, | ||
| "max_rows_upsert": max_rows_upsert | ||
| } | ||
| return dbconfig | ||
|
|
||
| def required_variables_exists(): | ||
| ret = True | ||
|
|
||
| print("-------------------------------------") | ||
| print("----- ETL Tool: Unit test Execution ") | ||
| print("----- 1. Checking if required variables are defined") | ||
| print("-------------------------------------") | ||
|
|
||
| if not env_var_is_filled("TEST_MODE") or \ | ||
| not env_var_is_filled("EXECUTION_ID") or \ | ||
| not env_var_is_filled("POSTGRES_HOST") or \ | ||
| not env_var_is_filled("POSTGRES_PORT") or \ | ||
| not env_var_is_filled("POSTGRES_USER") or \ | ||
| not env_var_is_filled("POSTGRES_PASSWORD") or \ | ||
| not env_var_is_filled("POSTGRES_DATABASE") or \ | ||
| not env_var_is_filled("ORACLE_PORT") or \ | ||
| not env_var_is_filled("ORACLE_HOST") or \ | ||
| not env_var_is_filled("ORACLE_SERVICE") or \ | ||
| not env_var_is_filled("ORACLE_USER") or \ | ||
| not env_var_is_filled("ORACLE_PASSWORD"): | ||
| ret = False | ||
|
|
||
| if ret: | ||
| print("Required variable tests passed!") | ||
| else: | ||
| raise Exception("Not all required variables to execute a instance of Data Sync Engine exists.") | ||
|
|
||
| def testOracleConnection(settings): | ||
| print("-------------------------------------") | ||
| print("-- 3. Checking if Oracle connection is available and reachable") | ||
| print("-------------------------------------") | ||
| from module.test_db_connection import test_db_connection | ||
| dbConfig = generate_db_config("ORACLE","THE",settings) | ||
| d = test_db_connection.do_test(dbConfig) | ||
| print(d) | ||
|
|
||
| def testPostgresConnection(settings): | ||
| print("-------------------------------------") | ||
| print("-- 2. Checking if Postgres connection is available and reachable") | ||
| print("-------------------------------------") | ||
| from module.test_db_connection import test_db_connection | ||
| dbConfig = generate_db_config("POSTGRES","spar",settings) | ||
| d = test_db_connection.do_test(dbConfig) | ||
| print(d) | ||
|
|
||
| def read_settings(): | ||
| file = os.path.join(os.path.abspath(os.path.dirname(__file__)) , "settings.yml") | ||
| try: | ||
| with open(file, 'r') as stream: | ||
| data_loaded = yaml.safe_load(stream) | ||
| if data_loaded["postgres"]["max_rows_upsert"] or data_loaded["postgres"]["version_column"] or \ | ||
| data_loaded["oracle"]["max_rows_upsert"] or data_loaded["oracle"]["version_column"] or \ | ||
| data_loaded["postgres"]["ssl_version"] or data_loaded["oracle"]["ssl_version"]: | ||
| return data_loaded | ||
| except FileNotFoundError: | ||
| print("Error: settings.yml not found") | ||
| except KeyError: | ||
| print("Error: settings.yml is not well formated or does not have required settings") | ||
| except Exception as err: | ||
| print(f"A fatal error has occurred when trying to load settings.yml ({type(err)}): {err}") | ||
|
|
||
|
|
||
| def main() -> None: | ||
| definition_of_yes = ["Y","YES","1","T","TRUE"] | ||
| # print(os.environ.get("TEST_MODE")) | ||
| if os.environ.get("TEST_MODE") is None: | ||
| print("Error: test mode variable is None") | ||
| elif os.environ.get("EXECUTION_ID") is None: | ||
| print("Error: EXECUTION_ID is None, no execution defined to be executed in this run.") | ||
| else: | ||
| this_is_a_test = os.environ.get("TEST_MODE") | ||
| settings = read_settings() | ||
| print("<------------------ settings ----------------->") | ||
| print(settings) | ||
| print("<------------------ settings ----------------->") | ||
| if this_is_a_test in definition_of_yes: | ||
| print("Executing in Test mode") | ||
| required_variables_exists() | ||
| testPostgresConnection(settings["postgres"]) | ||
| testOracleConnection(settings["oracle"]) | ||
| # Vault disabled | ||
| # testVault() | ||
| else: | ||
| print("-------------------------------------") | ||
| print("Starting ETL main process ") | ||
| print("-------------------------------------") | ||
|
|
||
| dbOracle = generate_db_config("ORACLE","THE",settings["oracle"]) | ||
| dbPostgres = generate_db_config("POSTGRES","spar",settings["postgres"]) | ||
|
|
||
| execute_etl(dbPostgres, dbOracle) | ||
|
|
||
| print("-------------------------------------") | ||
| print("ETL Main process finished ") | ||
| print("-------------------------------------") | ||
|
|
||
| # MAIN Execution | ||
| def execute_etl(dbPostgres, dbOracle) -> None: | ||
| loggingBasicConfig(level=loggingDEBUG, stream=sys.stdout) | ||
| data_sync.execute_instance( oracle_config = dbOracle, postgres_config = dbPostgres ,track_config = dbPostgres) | ||
|
|
||
| if __name__ == '__main__': | ||
| main() | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.