Merge pull request #417 from clemente-lab/Enhancement-DumpReload

Enhancement dump reload

.github/workflows/tests.yml

@@ -26,7 +26,7 @@ jobs:
         run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.;
 
       - name: Python install
-        run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta@Enhancement-StudySequencingSeparation;
+        run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta;
 
       - name: install jupyter
         run: conda env create -f ./conda_env_files/jupyter_env.yaml -p /home/runner/conda_envs/jupyter
@@ -83,7 +83,7 @@ jobs:
         run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.;
 
       - name: Python install
-        run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta@Enhancement-StudySequencingSeparation;
+        run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta;
 
       - name: install pheniqs
         run: conda env create -f ./conda_env_files/pheniqs_env.yaml -p /home/runner/conda_envs/pheniqs

mmeds/config.py

@@ -361,6 +361,7 @@
 TEST_USER = 'testuser'
 SERVER_USER = 'serveruser'
 TEST_USER_0 = 'testuser0'
+REUPLOAD_USER = 'reupload'
 TEST_CODE = 'singlereads'
 TEST_CODE_SHORT = 'singlereadsshort'
 TEST_CODE_PAIRED = 'pairedreads'
@@ -401,13 +402,15 @@
 TEST_SUBJECT_SIMPLIFIED = str(TEST_PATH / 'test_subject_simplified.tsv')
 TEST_ADD_SUBJECT = str(TEST_PATH / 'test_add_subject.tsv')
 TEST_ANIMAL_SUBJECT = str(TEST_PATH / 'test_animal_subject.tsv')
+TEST_MIXED_SUBJECT = str(TEST_PATH / 'test_mixed_subject.tsv')
 TEST_SUBJECT_ERROR = str(TEST_PATH / 'validation_files/test_subject_error.tsv')
 TEST_SUBJECT_WARN = str(TEST_PATH / 'validation_files/test_subject_warn.tsv')
 TEST_SUBJECT_ALT = str(TEST_PATH / 'test_subject_alt.tsv')
 TEST_SUBJECT_SHORT_DUAL = str(TEST_PATH / 'test_subject_short.tsv')
 TEST_SUBJECT_SHORT = str(TEST_PATH / 'test_subject_short.tsv')
 TEST_METADATA = str(TEST_PATH / 'test_metadata.tsv')
 TEST_ANIMAL_METADATA = str(TEST_PATH / 'test_animal_metadata.tsv')
+TEST_MIXED_METADATA = str(TEST_PATH / 'test_mixed_metadata.tsv')
 TEST_METADATA_ALT = str(TEST_PATH / 'test_metadata_alt.tsv')
 TEST_METADATA_WARN = str(TEST_PATH / 'validation_files/test_metadata_warn.tsv')
 TEST_METADATA_SHORT = str(TEST_PATH / 'short_metadata.tsv')
@@ -542,6 +545,8 @@
     'AdditionalMetaData'
 }
 
+MIXED_SUBJECT_TABLES = SUBJECT_TABLES.union(ANIMAL_SUBJECT_TABLES)
+
 # Tables that should exist in the specimen metadata
 SPECIMEN_TABLES = ((set(TABLE_ORDER) - SUBJECT_TABLES) - ANIMAL_SUBJECT_TABLES) | {'AdditionalMetaData'}
 
@@ -592,7 +597,6 @@
 
 ICD_TABLES = {'IllnessBroadCategory', 'IllnessCategory', 'IllnessDetails'}
 
-
 # These are the tables that users are given direct access to
 PUBLIC_TABLES = set(set(TABLE_ORDER) - set(PROTECTED_TABLES) - set(['AdditionalMetaData', 'ICDCode']))
 
@@ -605,13 +609,14 @@
 COLUMN_TYPES_SPECIMEN = defaultdict(dict)
 COLUMN_TYPES_SUBJECT = defaultdict(dict)
 COLUMN_TYPES_ANIMAL_SUBJECT = defaultdict(dict)
+COLUMN_TYPES_MIXED_SUBJECT = defaultdict(dict)
 COL_TO_TABLE = {}
 
 # Summary Gradient Colors for use in continuous variable plots
 CONTINUOUS_GRADIENTS = [
-    ("0000FF", "FF0000"), # Blue - Red
-    ("FF9900", "9900CC"), # Orange - Purple
-    ("00FFFF", "FF33CC")  # Cyan - Pink
+    ("0000FF", "FF0000"),  # Blue - Red
+    ("FF9900", "9900CC"),  # Orange - Purple
+    ("00FFFF", "FF33CC")   # Cyan - Pink
 ]
 
 # Try connecting via the testing setup
@@ -685,7 +690,8 @@
 
     for test_file, col_types, tables in [(TEST_SPECIMEN, COLUMN_TYPES_SPECIMEN, SPECIMEN_TABLES),
                                          (TEST_SUBJECT, COLUMN_TYPES_SUBJECT, SUBJECT_TABLES),
-                                         (TEST_ANIMAL_SUBJECT, COLUMN_TYPES_ANIMAL_SUBJECT, ANIMAL_SUBJECT_TABLES)]:
+                                         (TEST_ANIMAL_SUBJECT, COLUMN_TYPES_ANIMAL_SUBJECT, ANIMAL_SUBJECT_TABLES),
+                                         (TEST_MIXED_SUBJECT, COLUMN_TYPES_MIXED_SUBJECT, MIXED_SUBJECT_TABLES)]:
         tdf = read_csv(test_file,
                        sep='\t',
                        header=[0, 1],

mmeds/database/metadata_uploader.py

@@ -166,7 +166,7 @@ def run(self):
 
         if not self.temporary:
             # Read in the metadata file to import
-            if self.subject_type == 'human':
+            if self.subject_type == 'human' or self.subject_type == 'mixed':
                 df = parse_ICD_codes(load_metadata(metadata_copy))
             elif self.subject_type == 'animal':
                 df = load_metadata(metadata_copy)

mmeds/html/upload_select_page.html

@@ -60,6 +60,7 @@ <h5><b> <i class="fa fa-cloud-upload "></i> Upload Study</b></h5>
             <select class="w3-select w3-border" name="subjectType" form="upload_form" placeholder="Subject Type" required>
                 <option value="human" selected="selected">Human</option>
                 <option value="animal">Animal</option>
+                <option value="mixed">Mixed</option>
             </select>
             </p>
             <p>

mmeds/secrets.py

@@ -23,5 +23,6 @@
 TEST_PASS = 'testpassL33TG@m3r'
 TEST_USER_PASS = 'password'
 TEST_ROOT_PASS = 'root'
+REUPLOAD_PASS = 'r3Up10@D'
 AUTH_KEY = b'F1ndTh3W@tch3r'
 WATCHER_PORT = 52953

mmeds/server.py

@@ -472,7 +472,19 @@ def run_validate(self, myMetaData, simplified):
                 # Perform additional checks that the Validator has insufficient information
                 # to perform during it's run
                 if cp.session['metadata_type'] == 'subject':
-                    warnings += db.check_repeated_subjects(subjects, cp.session['subject_type'])
+                    # Give separate subject tables for animal and human
+                    if cp.session['subject_type'] == 'mixed':
+                        human_df = subjects.iloc[:, 0:4]
+                        animal_df = subjects.iloc[:, 4:8]
+                        # Drop rows of other DF
+                        human_df.dropna(how='all')
+                        animal_df.dropna(how='all')
+
+                        warnings += db.check_repeated_subjects(human_df, 'human')
+                        warnings += db.check_repeated_subjects(animal_df, 'animal')
+
+                    else:
+                        warnings += db.check_repeated_subjects(subjects, cp.session['subject_type'])
                     cp.session['subject_ids'] = subjects
                 elif cp.session['metadata_type'] == 'specimen':
                     errors += db.check_user_study_name(cp.session['study_name'])

mmeds/tests/server/test_server.py

@@ -599,7 +599,7 @@ def upload_metadata_fail(self):
         headers, body = self.upload_files(['myMetaData'], [fig.TEST_SPECIMEN], ['text/tab-seperated-values'])
         self.getPage('/upload/validate_metadata?barcodes_type=single', headers + self.cookies, 'POST', body)
         self.assertStatus('200 OK')
-        standard_error = ('-1\t-1\tIllegal Table Error: Table {} should not be the metadata')
+        standard_error = ('-1\t-1\tIllegal Table Error: Table {} should not be in the metadata')
         error_message = [('-1\t-1\tMissing Table Error: Missing tables Ethnicity, Genotypes, Heights, ICDCode,' +
                           ' Illness, Intervention, Interventions, SubjectType, Subjects, Weights')]
         error_categories = ['Aliquot',

mmeds/tests/unit/test.py

@@ -105,7 +105,9 @@ def setup_tests(tests):
                            'Test_Single_0',
                            testing,
                            fig.TEST_CODE + '0'))
+
     for setup in test_setup:
+        print(setup)
         assert 0 == upload_metadata(setup)
 
 

mmeds/tests/unit/test_database.py

@@ -73,7 +73,7 @@ def test_a_tables(self):
         tables.sort(key=lambda x: fig.TABLE_ORDER.index(x))
         del tables[tables.index('AdditionalMetaData')]
         del tables[tables.index('ICDCode')]
-        Logger.info("tables: ", tables)
+        Logger.info(f"tables: {tables}")
         for row in range(len(self.df)):
             for table in tables:
                 Logger.info('Query table {}'.format(table))

mmeds/tests/unit/test_validate.py

@@ -68,6 +68,7 @@ def test_b_error_files(self):
                     subject_ids = subjects
 
                 # Check the correct error is raised
+                Logger.debug(f"name: {name}\nexpected error: {error}\nactual errors: {errors}")
                 assert error in errors[0].lower()
 
                 # Check the messages are foramtted correctly