Remove unused function

bin/consortium_to_public.py

@@ -4,12 +4,13 @@
 import os
 import synapseclient
 import subprocess
-import time
 
 from genie import dashboard_table_updater
 from genie import process_functions
 from genie import consortium_to_public
 from genie import database_to_staging
+from genie import extract
+from genie import load
 
 logger = logging.getLogger(__name__)
 
@@ -127,12 +128,15 @@ def main(args):
     releaseSynId = databaseSynIdMappingDf["Id"][
         databaseSynIdMappingDf["Database"] == "releaseFolder"
     ].values[0]
+    processTrackerSynId = databaseSynIdMappingDf["Id"][
+        databaseSynIdMappingDf["Database"] == "processTracker"
+    ].values[0]
     # TEST run of the infrastructure will always
     # Map to a specific folder
     if args.test:
         officialPublic = {"TESTpublic": "syn12299959"}
     else:
-        officialPublic = consortium_to_public.get_public_to_consortium_synid_mapping(
+        officialPublic = extract.get_public_to_consortium_synid_mapping(
             syn, releaseSynId
         )
     if args.genieVersion not in officialPublic.keys():
@@ -143,16 +147,13 @@ def main(args):
 
     args.releaseId = officialPublic[args.genieVersion]
     if not args.test and not args.staging:
-        processTrackerSynId = databaseSynIdMappingDf["Id"][
-            databaseSynIdMappingDf["Database"] == "processTracker"
-        ].values[0]
-        processTracker = syn.tableQuery(
-            "SELECT timeStartProcessing FROM %s where center = 'SAGE' "
-            "and processingType = 'public'" % processTrackerSynId
+        load.update_process_trackingdf(
+            syn=syn,
+            process_trackerdb_synid=processTrackerSynId,
+            center="SAGE",
+            process_type="public",
+            start=True,
         )
-        processTrackerDf = processTracker.asDataFrame()
-        processTrackerDf["timeStartProcessing"].iloc[0] = str(int(time.time() * 1000))
-        syn.store(synapseclient.Table(processTrackerSynId, processTrackerDf))
 
     caseListEntities, genePanelEntities = consortium_to_public.consortiumToPublic(
         syn,
@@ -207,13 +208,13 @@ def main(args):
     )
     # Don't update process tracker is testing or staging
     if not args.test and not args.staging:
-        processTracker = syn.tableQuery(
-            "SELECT timeEndProcessing FROM %s where center = 'SAGE' and "
-            "processingType = 'public'" % processTrackerSynId
+        load.update_process_trackingdf(
+            syn=syn,
+            process_trackerdb_synid=processTrackerSynId,
+            center="SAGE",
+            process_type="public",
+            start=False,
         )
-        processTrackerDf = processTracker.asDataFrame()
-        processTrackerDf["timeEndProcessing"].iloc[0] = str(int(time.time() * 1000))
-        syn.store(synapseclient.Table(processTrackerSynId, processTrackerDf))
 
     if not args.test:
         logger.info("DASHBOARD UPDATE")

bin/database_to_staging.py

@@ -11,6 +11,7 @@
     create_case_lists,
     dashboard_table_updater,
     database_to_staging,
+    load,
     process_functions,
 )
 
@@ -172,13 +173,17 @@ def main(
     ].values[0]
     # get syn id of case list folder in consortium release
     # caseListSynId = findCaseListId(syn, consortiumSynId)
-    caseListSynId, _ = database_to_staging.search_and_create_folder(
+    caseListSynId = database_to_staging.search_or_create_folder(
         syn, consortiumSynId, "case_lists"
     )
 
     if not staging:
-        database_to_staging.update_process_trackingdf(
-            syn, processTrackerSynId, "SAGE", "dbToStage", start=True
+        load.update_process_trackingdf(
+            syn=syn,
+            process_trackerdb_synid=processTrackerSynId,
+            center="SAGE",
+            process_type="dbToStage",
+            start=True,
         )
 
     centerMappingSynId = databaseSynIdMappingDf["Id"][
@@ -234,8 +239,11 @@ def main(
     for casePath in caseListFiles:
         casePath = os.path.join(database_to_staging.CASE_LIST_PATH, casePath)
         caseListEntities.append(
-            database_to_staging.store_file(
-                syn, casePath, parent=caseListSynId, genieVersion=genie_version
+            load.store_file(
+                syn=syn,
+                filepath=casePath,
+                parentid=caseListSynId,
+                version_comment=genie_version,
             )
         )
 
@@ -294,8 +302,12 @@ def main(
     )
 
     if not staging:
-        database_to_staging.update_process_trackingdf(
-            syn, processTrackerSynId, "SAGE", "dbToStage", start=False
+        load.update_process_trackingdf(
+            syn=syn,
+            process_trackerdb_synid=processTrackerSynId,
+            center="SAGE",
+            process_type="dbToStage",
+            start=False,
         )
 
     if not test:
@@ -317,11 +329,11 @@ def main(
         genie_user=genie_user,
         genie_pass=genie_pass,
     )
-    database_to_staging.store_file(
-        syn,
-        data_guide_pdf,
-        genieVersion=genie_version,
-        parent=folders["release_folder"],
+    load.store_file(
+        syn=syn,
+        filepath=data_guide_pdf,
+        version_comment=genie_version,
+        parentid=folders["release_folder"],
     )
     logger.info("COMPLETED DATABASE TO STAGING")
 

bin/input_to_database.py

@@ -8,6 +8,7 @@
 
 from genie import (
     config,
+    extract,
     input_to_database,
     process_functions,
     write_invalid_reasons,
@@ -56,7 +57,7 @@ def main(
     syn = process_functions.synLogin(pemfile, debug=debug)
 
     # Get project GENIE configurations
-    genie_config = process_functions.get_genie_config(syn=syn, project_id=project_id)
+    genie_config = extract.get_genie_config(syn=syn, project_id=project_id)
 
     # Filter for specific center
     if center is not None:
@@ -115,6 +116,9 @@ def main(
 
     # Start GENIE processing
     for process_center in centers:
+        # Check if the genie genome nexus is up, if not then don't run
+        # processing
+        process_functions.checkUrl("http://genie.genomenexus.org/")
         input_to_database.center_input_to_database(
             syn=syn,
             project_id=project_id,

genie/__init__.py

@@ -1,12 +1,13 @@
 # Import logging last to not take in synapseclient logging
 import logging
+import os
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 logging.getLogger("keyring").setLevel(logging.WARNING)
 
 # create version in __init__.py
 # https://packaging.python.org/en/latest/guides/single-sourcing-package-version/
-__version__ = "14.4.0"
+__version__ = "15.0.0"
 
 __all__ = ["__version__"]