Merge pull request #530 from isb-cgc/isb-cgc-prod-sp

Sprint 25 release

README.md

@@ -1 +1 @@
-# ISB-CGC-Common
+# ISB-CGC-Common  

accounts/models.py

@@ -19,7 +19,7 @@
 from django.db import models
 from django.contrib.auth.models import User
 import logging
-from datetime import datetime, timedelta
+import datetime
 import pytz
 
 logger = logging.getLogger('main_logger')
@@ -149,7 +149,7 @@ def get_auth_datasets(self):
         return result
 
     def is_expired(self):
-        expired_time = pytz.utc.localize(datetime.utcnow() + timedelta(days=-7, minutes=10))
+        expired_time = pytz.utc.localize(datetime.datetime.utcnow() + datetime.timedelta(days=-7, minutes=10))
         return self.authorized_date < expired_time
 
 

accounts/tests/test_unlink_accounts.py

@@ -23,7 +23,7 @@
 from django.contrib.auth.models import User
 from django.core.exceptions import ObjectDoesNotExist
 from accounts.models import NIH_User
-from accounts.views import unlink_accounts_and_get_acl_tasks, ACLDeleteAction
+from accounts.sa_utils import unlink_accounts_and_get_acl_tasks, ACLDeleteAction
 
 logging.basicConfig(
     level=logging.INFO

accounts/urls.py

@@ -44,5 +44,6 @@
     url(r'^users/(?P<user_id>\d+)/delete_bucket/(?P<bucket_id>\d+)/$', views.delete_bucket, name='delete_bucket'),
     url(r'^users/(?P<user_id>\d+)/register_bqdataset/(?P<gcp_id>\d+)/$', views.register_bqdataset, name='register_bqdataset'),
     url(r'^users/(?P<user_id>\d+)/delete_bqdataset/(?P<bqdataset_id>\d+)/$', views.delete_bqdataset, name='delete_bqdataset'),
+    url(r'^users/(?P<user_id>\d+)/datasets/$', views.get_user_datasets, name='dataset_info'),
 ]
 

cohorts/metadata_counting.py

@@ -25,7 +25,7 @@
 import re
 from metadata_helpers import *
 from projects.models import Program, Project, User_Data_Tables, Public_Metadata_Tables
-from google_helpers.bigquery_service import authorize_credentials_with_Google
+from google_helpers.bigquery.service import authorize_credentials_with_Google
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 
 BQ_ATTEMPT_MAX = 10
@@ -36,7 +36,6 @@
 
 MAX_FILE_LIST_ENTRIES = settings.MAX_FILE_LIST_REQUEST
 MAX_SEL_FILES = settings.MAX_FILES_IGV
-WHITELIST_RE = settings.WHITELIST_RE
 BQ_SERVICE = None
 
 logger = logging.getLogger('main_logger')
@@ -418,7 +417,7 @@ def count_public_metadata(user, cohort_id=None, inc_filters=None, program_id=Non
 
             # for-each result, add to list
 
-            if results.__len__() > 0:
+            if len(results) > 0:
                 for barcode in results:
                     barcodes.append(str(barcode['f'][0]['v']))
 
@@ -427,7 +426,7 @@ def count_public_metadata(user, cohort_id=None, inc_filters=None, program_id=Non
                 # Put in one 'not found' entry to zero out the rest of the queries
                 barcodes = ['NONE_FOUND', ]
 
-            tmp_mut_table = 'bq_res_table_' + user.id.__str__() + "_" + make_id(6)
+            tmp_mut_table = 'bq_res_table_' + str(user.id) + "_" + make_id(6)
 
             make_tmp_mut_table_str = """
                 CREATE TEMPORARY TABLE %s (
@@ -709,7 +708,6 @@ def count_public_metadata(user, cohort_id=None, inc_filters=None, program_id=Non
 
         stop = time.time()
         logger.debug('[BENCHMARKING] Time to query filter count set in metadata_counts:'+(stop - start).__str__())
-        logger.debug('[BENCHMARKING] Filters requested: '+str(inc_filters))
 
         # query sample and case counts
         count_query = 'SELECT COUNT(DISTINCT %s) FROM %s'

cohorts/metadata_helpers.py

@@ -22,7 +22,6 @@
 import random
 import string
 import logging
-import traceback
 import warnings
 import copy
 import MySQLdb
@@ -271,7 +270,7 @@ def fetch_build_data_attr(build):
                             if val not in METADATA_DATA_ATTR[build][attr]['values']:
                                 METADATA_DATA_ATTR[build][attr]['values'][val] = {
                                     'displ_value': val,
-                                    'value': re.sub(r"[^A-Za-z0-9.:_\-]","",re.sub(r"\s+","-", val)),
+                                    'value': re.sub(r"[^A-Za-z0-9_\-]","",re.sub(r"\s+","-", val)),
                                     'name': val
                                 }
 

cohorts/models.py

@@ -22,6 +22,7 @@
 from django.db.models import Count
 from django.contrib.auth.models import User
 from django.db.models import Q
+from django.utils.html import escape
 from projects.models import Project, Program, User_Feature_Definitions
 from sharing.models import Shared_Resource
 from metadata_helpers import fetch_metadata_value_set, fetch_program_data_types, MOLECULAR_DISPLAY_STRINGS
@@ -266,24 +267,24 @@ def get_revision_history(self):
                         prog_filters[cohort_filter['program']].append(cohort_filter)
                     revision_list.append({'type': 'filter', 'vals': prog_filters})
                 elif source.type == Source.CLONE:
-                    revision_list.append('Cloned from %s.' % source.parent.name)
+                    revision_list.append('Cloned from %s.' % escape(source.parent.name))
                 elif source.type == Source.PLOT_SEL:
-                    revision_list.append('Selected from plot of %s.' % source.parent.name)
+                    revision_list.append('Selected from plot of %s.' % escape(source.parent.name))
                 sources = Source.objects.filter(cohort=source.parent)
 
             # multiple parents
             if sources.count() > 1:
                 if sources[0].type == Source.SET_OPS:
-                    revision_list.append(sources[0].notes)
+                    revision_list.append(escape(sources[0].notes))
                 if sources[0].type == Source.PLOT_SEL:
                     ret_str = 'Selected from plot of '
                     first = True
                     for source in sources:
                         if first:
-                            ret_str += source.parent.name
+                            ret_str += escape(source.parent.name)
                             first = False
                         else:
-                            ret_str += ', ' + source.parent.name
+                            ret_str += ', ' + escape(source.parent.name)
                     revision_list.append(ret_str)
                 # TODO: Actually traverse the tree, but this will require a most sophisticated way of displaying
                 # Currently only getting parents history, and not grandparents history.

cohorts/urls.py

@@ -32,9 +32,9 @@
     url(r'^filelist_ajax/(?P<cohort_id>\d+)/$',     views.cohort_filelist_ajax, name='cohort_filelist_ajax'),
     url(r'^filelist_ajax/(?P<cohort_id>\d+)/panel/(?P<panel_type>[A-Za-z]+)/$',
                                                     views.cohort_filelist_ajax, name='cohort_filelist_ajax_panel'),
+    url(r'^filelist/(?P<cohort_id>\d+)/export',     views.export_file_list_to_bq, name='export_cohort_filelist_to_bq'),
     url(r'^save_cohort/',                           views.save_cohort, name='save_cohort'),
-    url(r'^export_cohort/$',                        views.export_cohort, name='export_cohort'),
-    url(r'^export_cohort/(?P<cohort_id>\d+)/',      views.export_cohort_to_bq, name='export_cohort_to_bq'),
+    url(r'^export_cohort/(?P<cohort_id>\d+)/',      views.export_cohort_to_bq, name='export_cohort'),
     url(r'^save_cohort_from_plot/',                 views.save_cohort_from_plot, name='save_cohort_from_plot'),
     url(r'^delete_cohort/',                         views.delete_cohort, name='delete_cohort'),
     url(r'^clone_cohort/(?P<cohort_id>\d+)/',       views.clone_cohort, name='clone_cohort'),