Switched duckdb sort to be pandas based

cumulus_library/databases/duckdb.py

@@ -191,12 +191,17 @@ def export_table_as_parquet(
         if table_size[0] == 0:
             return None
         query = f"""COPY
-            (SELECT * FROM {table_name} ORDER BY ALL desc)
+            (SELECT * FROM {table_name})
             TO '{parquet_path}'
             (FORMAT parquet)
             """  # noqa: S608
         self.connection.execute(query)
 
+        df = pandas.read_parquet(parquet_path)
+        df = df.sort_values(
+            by=list(df.columns), ascending=False, ignore_index=True, na_position="first"
+        )
+        df.to_parquet(parquet_path)
         return parquet_path
 
     def create_schema(self, schema_name):

tests/test_data/duckdb_data/expected_export/core/core__count_condition_month.cube.csv

@@ -1,3 +1,3 @@
 cnt,category_code,recordedDate_month,code_display
-15,encounter-diagnosis,,
 15,,,
+15,encounter-diagnosis,,

tests/test_data/duckdb_data/expected_export/core/core__count_diagnosticreport_month.cube.csv

@@ -1,20 +1,20 @@
 cnt,category_display,code_display,issued_month
-13,Laboratory,CBC panel - Blood by Automated count,
+13,,,
+13,,History and physical note,
+13,,Evaluation + Plan note,
+13,,CBC panel - Blood by Automated count,
 13,Laboratory,,
+13,Laboratory,CBC panel - Blood by Automated count,
+13,History and physical note,,
 13,History and physical note,History and physical note,
 13,History and physical note,Evaluation + Plan note,
-13,History and physical note,,
+13,Evaluation + Plan note,,
 13,Evaluation + Plan note,History and physical note,
 13,Evaluation + Plan note,Evaluation + Plan note,
-13,Evaluation + Plan note,,
-13,,History and physical note,
-13,,Evaluation + Plan note,
-13,,CBC panel - Blood by Automated count,
-13,,,
 12,cumulus__none,,
-11,cumulus__none,Generalized anxiety disorder 7 item (GAD-7),
 11,,Generalized anxiety disorder 7 item (GAD-7),
-10,cumulus__none,Patient Health Questionnaire 2 item (PHQ-2) [Reported],
-10,cumulus__none,Alcohol Use Disorder Identification Test - Consumption [AUDIT-C],
+11,cumulus__none,Generalized anxiety disorder 7 item (GAD-7),
 10,,Patient Health Questionnaire 2 item (PHQ-2) [Reported],
 10,,Alcohol Use Disorder Identification Test - Consumption [AUDIT-C],
+10,cumulus__none,Patient Health Questionnaire 2 item (PHQ-2) [Reported],
+10,cumulus__none,Alcohol Use Disorder Identification Test - Consumption [AUDIT-C],

tests/test_data/duckdb_data/expected_export/core/core__count_documentreference_month.cube.csv

@@ -1,19 +1,19 @@
 cnt,type_display,author_month,class_display
+50,,,
 50,Evaluation + Plan note,,
 50,Emergency department note,,
-50,,,
+46,,,ambulatory
 46,Evaluation + Plan note,,ambulatory
 46,Emergency department note,,ambulatory
-46,,,ambulatory
+26,,2018-07-01,
 26,Evaluation + Plan note,2018-07-01,
 26,Emergency department note,2018-07-01,
-26,,2018-07-01,
+24,,2018-07-01,ambulatory
+24,,2018-06-01,
 24,Evaluation + Plan note,2018-07-01,ambulatory
 24,Evaluation + Plan note,2018-06-01,
 24,Emergency department note,2018-07-01,ambulatory
 24,Emergency department note,2018-06-01,
-24,,2018-07-01,ambulatory
-24,,2018-06-01,
+22,,2018-06-01,ambulatory
 22,Evaluation + Plan note,2018-06-01,ambulatory
 22,Emergency department note,2018-06-01,ambulatory
-22,,2018-06-01,ambulatory

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_all_types.cube.csv

@@ -1,9 +1,9 @@
 cnt,class_display,type_display,serviceType_display,priority_display
-50,,,cumulus__none,cumulus__none
-50,,,cumulus__none,
-50,,,,cumulus__none
 50,,,,
-46,ambulatory,,cumulus__none,cumulus__none
-46,ambulatory,,cumulus__none,
-46,ambulatory,,,cumulus__none
+50,,,,cumulus__none
+50,,,cumulus__none,
+50,,,cumulus__none,cumulus__none
 46,ambulatory,,,
+46,ambulatory,,,cumulus__none
+46,ambulatory,,cumulus__none,
+46,ambulatory,,cumulus__none,cumulus__none

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_all_types_month.cube.csv

@@ -1,25 +1,25 @@
 cnt,class_display,type_display,serviceType_display,priority_display,period_start_month
-50,,,cumulus__none,cumulus__none,
-50,,,cumulus__none,,
-50,,,,cumulus__none,
 50,,,,,
-46,ambulatory,,cumulus__none,cumulus__none,
-46,ambulatory,,cumulus__none,,
-46,ambulatory,,,cumulus__none,
+50,,,,cumulus__none,
+50,,,cumulus__none,,
+50,,,cumulus__none,cumulus__none,
 46,ambulatory,,,,
-26,,,cumulus__none,cumulus__none,2018-07-01
-26,,,cumulus__none,,2018-07-01
-26,,,,cumulus__none,2018-07-01
+46,ambulatory,,,cumulus__none,
+46,ambulatory,,cumulus__none,,
+46,ambulatory,,cumulus__none,cumulus__none,
 26,,,,,2018-07-01
-24,ambulatory,,cumulus__none,cumulus__none,2018-07-01
-24,ambulatory,,cumulus__none,,2018-07-01
-24,ambulatory,,,cumulus__none,2018-07-01
-24,ambulatory,,,,2018-07-01
-24,,,cumulus__none,cumulus__none,2018-06-01
-24,,,cumulus__none,,2018-06-01
-24,,,,cumulus__none,2018-06-01
+26,,,,cumulus__none,2018-07-01
+26,,,cumulus__none,,2018-07-01
+26,,,cumulus__none,cumulus__none,2018-07-01
 24,,,,,2018-06-01
-22,ambulatory,,cumulus__none,cumulus__none,2018-06-01
-22,ambulatory,,cumulus__none,,2018-06-01
-22,ambulatory,,,cumulus__none,2018-06-01
+24,,,,cumulus__none,2018-06-01
+24,,,cumulus__none,,2018-06-01
+24,,,cumulus__none,cumulus__none,2018-06-01
+24,ambulatory,,,,2018-07-01
+24,ambulatory,,,cumulus__none,2018-07-01
+24,ambulatory,,cumulus__none,,2018-07-01
+24,ambulatory,,cumulus__none,cumulus__none,2018-07-01
 22,ambulatory,,,,2018-06-01
+22,ambulatory,,,cumulus__none,2018-06-01
+22,ambulatory,,cumulus__none,,2018-06-01
+22,ambulatory,,cumulus__none,cumulus__none,2018-06-01

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_month.cube.csv

@@ -3,63 +3,63 @@ cnt,period_start_month,class_display,age_at_visit,gender,race_display,ethnicity_
 47,,,,,white,
 46,,ambulatory,,,,
 45,,,,,,not hispanic or latino
-43,,ambulatory,,,white,
 43,,,,,white,not hispanic or latino
+43,,ambulatory,,,white,
 42,,ambulatory,,,,not hispanic or latino
 40,,ambulatory,,,white,not hispanic or latino
 29,,,,female,,
 28,,,,female,white,
-27,,ambulatory,,female,,
 27,,,,female,,not hispanic or latino
-26,2018-07-01,,,,white,
-26,2018-07-01,,,,,
-26,,ambulatory,,female,white,
+27,,ambulatory,,female,,
 26,,,,female,white,not hispanic or latino
+26,,ambulatory,,female,white,
+26,2018-07-01,,,,,
+26,2018-07-01,,,,white,
 25,,ambulatory,,female,,not hispanic or latino
-24,2018-07-01,ambulatory,,,white,
-24,2018-07-01,ambulatory,,,,
-24,2018-07-01,,,,white,not hispanic or latino
+24,,ambulatory,,female,white,not hispanic or latino
 24,2018-07-01,,,,,not hispanic or latino
+24,2018-07-01,,,,white,not hispanic or latino
+24,2018-07-01,ambulatory,,,,
+24,2018-07-01,ambulatory,,,white,
 24,2018-06-01,,,,,
-24,,ambulatory,,female,white,not hispanic or latino
-23,2018-07-01,ambulatory,,,white,not hispanic or latino
 23,2018-07-01,ambulatory,,,,not hispanic or latino
+23,2018-07-01,ambulatory,,,white,not hispanic or latino
 22,2018-06-01,ambulatory,,,,
-21,2018-06-01,,,,white,
-21,2018-06-01,,,,,not hispanic or latino
 21,,,,male,,
-19,2018-06-01,ambulatory,,,white,
-19,2018-06-01,ambulatory,,,,not hispanic or latino
-19,2018-06-01,,,,white,not hispanic or latino
-19,,ambulatory,,male,,
+21,2018-06-01,,,,,not hispanic or latino
+21,2018-06-01,,,,white,
 19,,,,male,white,
+19,,ambulatory,,male,,
+19,2018-06-01,,,,white,not hispanic or latino
+19,2018-06-01,ambulatory,,,,not hispanic or latino
+19,2018-06-01,ambulatory,,,white,
 18,,,,male,,not hispanic or latino
-17,2018-06-01,ambulatory,,,white,not hispanic or latino
-17,,ambulatory,,male,white,
-17,,ambulatory,,male,,not hispanic or latino
 17,,,,male,white,not hispanic or latino
+17,,ambulatory,,male,,not hispanic or latino
+17,,ambulatory,,male,white,
+17,2018-06-01,ambulatory,,,white,not hispanic or latino
 16,,ambulatory,,male,white,not hispanic or latino
 15,2018-06-01,,,female,,
-14,2018-07-01,,,female,white,not hispanic or latino
-14,2018-07-01,,,female,white,
-14,2018-07-01,,,female,,not hispanic or latino
 14,2018-07-01,,,female,,
-14,2018-06-01,ambulatory,,female,,
+14,2018-07-01,,,female,,not hispanic or latino
+14,2018-07-01,,,female,white,
+14,2018-07-01,,,female,white,not hispanic or latino
 14,2018-06-01,,,female,white,
-13,2018-07-01,ambulatory,,female,white,not hispanic or latino
-13,2018-07-01,ambulatory,,female,white,
-13,2018-07-01,ambulatory,,female,,not hispanic or latino
+14,2018-06-01,ambulatory,,female,,
 13,2018-07-01,ambulatory,,female,,
-13,2018-06-01,ambulatory,,female,white,
+13,2018-07-01,ambulatory,,female,,not hispanic or latino
+13,2018-07-01,ambulatory,,female,white,
+13,2018-07-01,ambulatory,,female,white,not hispanic or latino
 13,2018-06-01,,,female,,not hispanic or latino
-12,2018-07-01,,,male,white,
+13,2018-06-01,ambulatory,,female,white,
 12,2018-07-01,,,male,,
-12,2018-06-01,ambulatory,,female,,not hispanic or latino
+12,2018-07-01,,,male,white,
 12,2018-06-01,,,female,white,not hispanic or latino
-11,2018-07-01,ambulatory,,male,white,
+12,2018-06-01,ambulatory,,female,,not hispanic or latino
 11,2018-07-01,ambulatory,,male,,
+11,2018-07-01,ambulatory,,male,white,
 11,2018-06-01,ambulatory,,female,white,not hispanic or latino
-10,2018-07-01,ambulatory,,male,white,not hispanic or latino
-10,2018-07-01,ambulatory,,male,,not hispanic or latino
-10,2018-07-01,,,male,white,not hispanic or latino
 10,2018-07-01,,,male,,not hispanic or latino
+10,2018-07-01,,,male,white,not hispanic or latino
+10,2018-07-01,ambulatory,,male,,not hispanic or latino
+10,2018-07-01,ambulatory,,male,white,not hispanic or latino

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_priority_month.cube.csv

@@ -1,13 +1,13 @@
 cnt,class_display,priority_display,period_start_month
-50,,cumulus__none,
 50,,,
-46,ambulatory,cumulus__none,
+50,,cumulus__none,
 46,ambulatory,,
-26,,cumulus__none,2018-07-01
+46,ambulatory,cumulus__none,
 26,,,2018-07-01
-24,ambulatory,cumulus__none,2018-07-01
-24,ambulatory,,2018-07-01
-24,,cumulus__none,2018-06-01
+26,,cumulus__none,2018-07-01
 24,,,2018-06-01
-22,ambulatory,cumulus__none,2018-06-01
+24,,cumulus__none,2018-06-01
+24,ambulatory,,2018-07-01
+24,ambulatory,cumulus__none,2018-07-01
 22,ambulatory,,2018-06-01
+22,ambulatory,cumulus__none,2018-06-01

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_service_month.cube.csv

@@ -1,13 +1,13 @@
 cnt,class_display,serviceType_display,period_start_month
-50,,cumulus__none,
 50,,,
-46,ambulatory,cumulus__none,
+50,,cumulus__none,
 46,ambulatory,,
-26,,cumulus__none,2018-07-01
+46,ambulatory,cumulus__none,
 26,,,2018-07-01
-24,ambulatory,cumulus__none,2018-07-01
-24,ambulatory,,2018-07-01
-24,,cumulus__none,2018-06-01
+26,,cumulus__none,2018-07-01
 24,,,2018-06-01
-22,ambulatory,cumulus__none,2018-06-01
+24,,cumulus__none,2018-06-01
+24,ambulatory,,2018-07-01
+24,ambulatory,cumulus__none,2018-07-01
 22,ambulatory,,2018-06-01
+22,ambulatory,cumulus__none,2018-06-01

tests/test_data/duckdb_data/expected_export/core/core__count_encounter_type_month.cube.csv

@@ -2,6 +2,6 @@ cnt,class_display,type_display,period_start_month
 50,,,
 46,ambulatory,,
 26,,,2018-07-01
-24,ambulatory,,2018-07-01
 24,,,2018-06-01
+24,ambulatory,,2018-07-01
 22,ambulatory,,2018-06-01

tests/test_data/duckdb_data/expected_export/core/core__count_medicationrequest_month.cube.csv

@@ -1,13 +1,13 @@
 cnt,status,intent,authoredon_month,medication_display
-27,,order,,
 27,,,,
-26,stopped,order,,
+27,,order,,
 26,stopped,,,
-15,stopped,order,2018-07-01,
-15,stopped,,2018-07-01,
-15,,order,2018-07-01,
+26,stopped,order,,
 15,,,2018-07-01,
-12,,order,2018-06-01,
+15,,order,2018-07-01,
+15,stopped,,2018-07-01,
+15,stopped,order,2018-07-01,
 12,,,2018-06-01,
-11,stopped,order,2018-06-01,
+12,,order,2018-06-01,
 11,stopped,,2018-06-01,
+11,stopped,order,2018-06-01,

tests/test_data/duckdb_data/expected_export/core/core__count_observation_lab_month.cube.csv

@@ -1,15 +1,15 @@
 cnt,effectiveDateTime_month,observation_code,valueCodeableConcept_display,class_display
-20,,,,ambulatory
 20,,,,
-10,,5778-6,Brown color (qualifier value),ambulatory
-10,,5778-6,Brown color (qualifier value),
-10,,5778-6,,ambulatory
-10,,5778-6,,
-10,,34533-0,Urine smell ammoniacal (finding),ambulatory
-10,,34533-0,Urine smell ammoniacal (finding),
-10,,34533-0,,ambulatory
-10,,34533-0,,
-10,,,Urine smell ammoniacal (finding),ambulatory
+20,,,,ambulatory
 10,,,Urine smell ammoniacal (finding),
-10,,,Brown color (qualifier value),ambulatory
+10,,,Urine smell ammoniacal (finding),ambulatory
 10,,,Brown color (qualifier value),
+10,,,Brown color (qualifier value),ambulatory
+10,,5778-6,,
+10,,5778-6,,ambulatory
+10,,5778-6,Brown color (qualifier value),
+10,,5778-6,Brown color (qualifier value),ambulatory
+10,,34533-0,,
+10,,34533-0,,ambulatory
+10,,34533-0,Urine smell ammoniacal (finding),
+10,,34533-0,Urine smell ammoniacal (finding),ambulatory

tests/test_data/duckdb_data/expected_export/core/core__count_procedure_month.cube.csv

@@ -1,33 +1,33 @@
 cnt,category_display,code_display,performedDateTime_month
-13,cumulus__none,Medication Reconciliation (procedure),cumulus__none
-13,cumulus__none,Medication Reconciliation (procedure),
-13,cumulus__none,,cumulus__none
-13,cumulus__none,,
-13,,Medication Reconciliation (procedure),cumulus__none
-13,,Medication Reconciliation (procedure),
-13,,,cumulus__none
 13,,,
-11,cumulus__none,Depression screening (procedure),cumulus__none
-11,cumulus__none,Depression screening (procedure),
-11,cumulus__none,Assessment of substance use (procedure),cumulus__none
-11,cumulus__none,Assessment of substance use (procedure),
-11,cumulus__none,Assessment of anxiety (procedure),cumulus__none
-11,cumulus__none,Assessment of anxiety (procedure),
-11,,Depression screening (procedure),cumulus__none
+13,,,cumulus__none
+13,,Medication Reconciliation (procedure),
+13,,Medication Reconciliation (procedure),cumulus__none
+13,cumulus__none,,
+13,cumulus__none,,cumulus__none
+13,cumulus__none,Medication Reconciliation (procedure),
+13,cumulus__none,Medication Reconciliation (procedure),cumulus__none
 11,,Depression screening (procedure),
-11,,Assessment of substance use (procedure),cumulus__none
+11,,Depression screening (procedure),cumulus__none
 11,,Assessment of substance use (procedure),
-11,,Assessment of anxiety (procedure),cumulus__none
+11,,Assessment of substance use (procedure),cumulus__none
 11,,Assessment of anxiety (procedure),
-10,cumulus__none,Depression screening using Patient Health Questionnaire Two-Item score (procedure),cumulus__none
-10,cumulus__none,Depression screening using Patient Health Questionnaire Two-Item score (procedure),
-10,cumulus__none,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),cumulus__none
-10,cumulus__none,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),
-10,cumulus__none,Assessment of health and social care needs (procedure),cumulus__none
-10,cumulus__none,Assessment of health and social care needs (procedure),
-10,,Depression screening using Patient Health Questionnaire Two-Item score (procedure),cumulus__none
+11,,Assessment of anxiety (procedure),cumulus__none
+11,cumulus__none,Depression screening (procedure),
+11,cumulus__none,Depression screening (procedure),cumulus__none
+11,cumulus__none,Assessment of substance use (procedure),
+11,cumulus__none,Assessment of substance use (procedure),cumulus__none
+11,cumulus__none,Assessment of anxiety (procedure),
+11,cumulus__none,Assessment of anxiety (procedure),cumulus__none
 10,,Depression screening using Patient Health Questionnaire Two-Item score (procedure),
-10,,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),cumulus__none
+10,,Depression screening using Patient Health Questionnaire Two-Item score (procedure),cumulus__none
 10,,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),
-10,,Assessment of health and social care needs (procedure),cumulus__none
+10,,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),cumulus__none
 10,,Assessment of health and social care needs (procedure),
+10,,Assessment of health and social care needs (procedure),cumulus__none
+10,cumulus__none,Depression screening using Patient Health Questionnaire Two-Item score (procedure),
+10,cumulus__none,Depression screening using Patient Health Questionnaire Two-Item score (procedure),cumulus__none
+10,cumulus__none,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),
+10,cumulus__none,Assessment using Alcohol Use Disorders Identification Test - Consumption (procedure),cumulus__none
+10,cumulus__none,Assessment of health and social care needs (procedure),
+10,cumulus__none,Assessment of health and social care needs (procedure),cumulus__none