test case improvements and general fixes

- Separated User and Profile functionality for better modularity.
- Fixed backend tests.
- Resolved migration issues.
- Set default language during wizard creation.
- Shifted PGP handling from UserProfile to User.

backend/globaleaks/db/__init__.py

@@ -119,8 +119,6 @@ def create_default_user_profiles(session, roles):
             "name": role.capitalize(),
             "role": role,
             "language": "en",
-            "pgp_key_public": "",
-            "pgp_key_remove": False
         }
         db_create_user_profile(session, user_desc)
 
@@ -278,7 +276,7 @@ def db_refresh_tenant_cache(session, to_refresh=None):
                     update_cache(profile[var_name], tid)
                 elif tid:
                     update_cache(default_cfg, tid)
-    query = (session.query(models.User.tid,models.User.mail_address,models.UserProfile.pgp_key_public)
+    query = (session.query(models.User.tid,models.User.mail_address,models.User.pgp_key_public)
             .join(models.UserProfile, models.User.profile_id == models.UserProfile.id)
             .filter(
                 models.User.role == 'admin',

backend/globaleaks/db/migration.py

@@ -35,7 +35,7 @@
 from globaleaks.db.migrations.update_66 import SubmissionSubStatus_v_65
 from globaleaks.db.migrations.update_67 import \
         InternalTip_v_66, ReceiverFile_v_66, Redaction_v_66, User_v_66, WhistleblowerFile_v_66
-from globaleaks.db.migrations.update_68 import Subscriber_v_67
+from globaleaks.db.migrations.update_68 import Subscriber_v_67, User_v_67
 from globaleaks.db.migrations.update_69 import Tenant_v_68
 
 
@@ -82,7 +82,7 @@
     ('Step', [models._Step, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
     ('Subscriber', [Subscriber_v_52, Subscriber_v_62, 0, 0, 0, 0, 0, 0, 0, 0, 0, Subscriber_v_67, 0, 0, 0, 0, models._Subscriber, 0]),
     ('Tenant', [Tenant_v_52, models._Tenant, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Tenant_v_68]),
-    ('User', [User_v_52, User_v_54, 0, User_v_56, 0, User_v_61, 0, 0, 0, 0, User_v_64, 0, 0, User_v_66, 0, models._User, 0, 0]),
+    ('User', [User_v_52, User_v_54, 0, User_v_56, 0, User_v_61, 0, 0, 0, 0, User_v_64, 0, 0, User_v_66, 0, User_v_67, 0, models._User]),
     ('WhistleblowerFile', [WhistleblowerFile_v_57, 0, 0, 0, 0, 0, WhistleblowerFile_v_64, 0, 0, 0, 0, 0, 0, WhistleblowerFile_v_66, 0, models._WhistleblowerFile, 0, 0]),
 
     ('WhistleblowerTip', [WhistleblowerTip_v_59, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1])

backend/globaleaks/db/migrations/update_68/__init__.py

@@ -1,8 +1,9 @@
 # -*- coding: UTF-8
+from globaleaks.models.enums import EnumUserRole
 from globaleaks.db.migrations.update import MigrationBase
 from globaleaks.models import Model
 from globaleaks.models.properties import *
-from globaleaks.utils.utility import datetime_now
+from globaleaks.utils.utility import datetime_now, datetime_null
 
 class Subscriber_v_67(Model):
     __tablename__ = 'subscriber'
@@ -25,6 +26,48 @@ class Subscriber_v_67(Model):
     tos1 = Column(UnicodeText, default='', nullable=False)
     tos2 = Column(UnicodeText, default='', nullable=False)
 
+class User_v_67(Model):
+    __tablename__ = 'user'
+
+    id = Column(UnicodeText(36), primary_key=True, default=uuid4)
+    tid = Column(Integer, default=1, nullable=False)
+    creation_date = Column(DateTime, default=datetime_now, nullable=False)
+    username = Column(UnicodeText, default='', nullable=False)
+    salt = Column(UnicodeText(24), default='', nullable=False)
+    hash = Column(UnicodeText(44), default='', nullable=False)
+    name = Column(UnicodeText, default='', nullable=False)
+    description = Column(JSON, default=dict, nullable=False)
+    public_name = Column(UnicodeText, default='', nullable=False)
+    role = Column(Enum(EnumUserRole), default='receiver', nullable=False)
+    enabled = Column(Boolean, default=True, nullable=False)
+    last_login = Column(DateTime, default=datetime_null, nullable=False)
+    mail_address = Column(UnicodeText, default='', nullable=False)
+    language = Column(UnicodeText(12), nullable=False)
+    password_change_needed = Column(Boolean, default=True, nullable=False)
+    password_change_date = Column(DateTime, default=datetime_null, nullable=False)
+    crypto_prv_key = Column(UnicodeText(84), default='', nullable=False)
+    crypto_pub_key = Column(UnicodeText(56), default='', nullable=False)
+    crypto_rec_key = Column(UnicodeText(80), default='', nullable=False)
+    crypto_bkp_key = Column(UnicodeText(84), default='', nullable=False)
+    crypto_escrow_prv_key = Column(UnicodeText(84), default='', nullable=False)
+    crypto_escrow_bkp1_key = Column(UnicodeText(84), default='', nullable=False)
+    crypto_escrow_bkp2_key = Column(UnicodeText(84), default='', nullable=False)
+    change_email_address = Column(UnicodeText, default='', nullable=False)
+    change_email_token = Column(UnicodeText, unique=True)
+    change_email_date = Column(DateTime, default=datetime_null, nullable=False)
+    notification = Column(Boolean, default=True, nullable=False)
+    forcefully_selected = Column(Boolean, default=False, nullable=False)
+    can_delete_submission = Column(Boolean, default=False, nullable=False)
+    can_postpone_expiration = Column(Boolean, default=True, nullable=False)
+    can_grant_access_to_reports = Column(Boolean, default=False, nullable=False)
+    can_transfer_access_to_reports = Column(Boolean, default=False, nullable=False)
+    can_redact_information = Column(Boolean, default=False, nullable=False)
+    can_mask_information = Column(Boolean, default=True, nullable=False)
+    can_reopen_reports = Column(Boolean, default=True, nullable=False)
+    can_edit_general_settings = Column(Boolean, default=False, nullable=False)
+    readonly = Column(Boolean, default=False, nullable=False)
+    two_factor_secret = Column(UnicodeText(32), default='', nullable=False)
+    reminder_date = Column(DateTime, default=datetime_null, nullable=False)
 
 class MigrationScript(MigrationBase):
     def migrate_Subscriber(self):

backend/globaleaks/handlers/admin/node.py

@@ -42,7 +42,18 @@ def db_update_enabled_languages(session, tid, languages, default_language):
 
     to_remove = list(set(cur_enabled_langs) - set(languages))
     if to_remove:
-        session.query(models.User).filter(models.User.tid == tid, models.User.language.in_(to_remove)).update({'language': default_language}, synchronize_session=False)
+        profile_ids = session.query(models.UserProfile.id) \
+            .join(models.User, models.User.profile_id == models.UserProfile.id) \
+            .filter(models.User.tid == tid, models.UserProfile.language.in_(to_remove)) \
+            .all()
+
+        profile_ids = [pid[0] for pid in profile_ids]
+
+        if profile_ids:
+            session.query(models.UserProfile) \
+                .filter(models.UserProfile.id.in_(profile_ids)) \
+                .update({'language': default_language}, synchronize_session=False)
+
         db_del(session, models.EnabledLanguage, (models.EnabledLanguage.tid == tid, models.EnabledLanguage.name.in_(to_remove)))
 
 

backend/globaleaks/handlers/admin/user.py

@@ -27,9 +27,6 @@ def serialize_user_profile(user):
         'role': user.role,
         'enabled': user.enabled,
         'language': user.language,
-        'pgp_key_remove': False,
-        'pgp_key_fingerprint': user.pgp_key_fingerprint,
-        'pgp_key_public': user.pgp_key_public,
         'notification': user.notification,
         'can_edit_general_settings': user.can_edit_general_settings,
         'can_delete_submission': user.can_delete_submission,
@@ -58,8 +55,6 @@ def db_create_user_profile(session, request):
     for key, value in request.items():
         setattr(user, key, value)
 
-    parse_pgp_options(user, request)
-
     existing_user = session.query(models.UserProfile).filter(models.UserProfile.name == user.name).first()
     if existing_user:
         raise errors.DuplicateUserError
@@ -111,8 +106,6 @@ def db_admin_update_user_profile(session, user_id, request):
     """
     user = session.query(models.UserProfile).filter(models.UserProfile.id == user_id).first()
 
-    parse_pgp_options(user, request)
-
     for key, value in request.items():
         if hasattr(user, key):
             setattr(user, key, value) 
@@ -192,6 +185,9 @@ def db_create_user(session, tid, user_session, request, language):
 
     user.salt = GCE.generate_salt()
 
+    # The various options related in manage PGP keys are used here.
+    parse_pgp_options(user, request)
+
     if user_session and user_session.ek:
         db_set_user_password(session, tid, user, generateRandomPassword(16))
 
@@ -260,6 +256,9 @@ def db_admin_update_user(session, tid, user_session, user_id, request, language)
     if user.password_change_needed:
         request['password_change_needed'] = True
 
+    # The various options related in manage PGP keys are used here.
+    parse_pgp_options(user, request)
+
     user.update(request)
 
     return user_serialize_user(session, user, language)

backend/globaleaks/handlers/recipient/export.py

@@ -118,7 +118,7 @@ def get_tip_export(session, tid, user_id, itip_id, language):
     if itip.status == 'new':
         db_update_submission_status(session, tid, user_id, itip, 'opened', None, profile)
 
-    return profile.pgp_key_public, serialize_rtip_export(session, user, itip, rtip, context, language)
+    return user.pgp_key_public, serialize_rtip_export(session, user, itip, rtip, context, language)
 
 
 def create_pdf_report(input_text, data):

backend/globaleaks/handlers/recipient/rtip.py

@@ -1242,8 +1242,9 @@ class WhistleblowerFileDownload(BaseHandler):
 
     @transact
     def download_wbfile(self, session, tid, user_id, file_id):
-        ifile, wbfile, rtip, profile = db_get(session,
-                                           (models.InternalFile,
+        user, ifile, wbfile, rtip, profile = db_get(session,
+                                           (models.User,
+                                            models.InternalFile,
                                             models.WhistleblowerFile,
                                             models.ReceiverTip,
                                             models.UserProfile),
@@ -1268,7 +1269,7 @@ def download_wbfile(self, session, tid, user_id, file_id):
         log.debug("Download of file %s by receiver %s" %
                   (wbfile.internalfile_id, rtip.receiver_id))
 
-        return ifile.name, ifile.id, wbfile.id, rtip.crypto_tip_prv_key, rtip.deprecated_crypto_files_prv_key, profile.pgp_key_public
+        return ifile.name, ifile.id, wbfile.id, rtip.crypto_tip_prv_key, rtip.deprecated_crypto_files_prv_key, user.pgp_key_public
 
     @inlineCallbacks
     def get(self, wbfile_id):
@@ -1326,7 +1327,7 @@ def download_rfile(self, session, tid, user_id, file_id):
             rfile, rtip, pgp_key = db_get(session,
                                           (models.ReceiverFile,
                                            models.ReceiverTip,
-                                           models.UserProfile.pgp_key_public),
+                                           models.User.pgp_key_public),
                                           (models.User.id == user_id,
                                            models.User.profile_id == models.UserProfile.id,
                                            models.User.id == models.ReceiverTip.receiver_id,

backend/globaleaks/handlers/user/__init__.py

@@ -69,9 +69,9 @@ def user_serialize_user(session, user, language):
         'language': profile.language,
         'password_change_needed': user.password_change_needed,
         'password_change_date': user.password_change_date,
-        'pgp_key_fingerprint': profile.pgp_key_fingerprint,
-        'pgp_key_public': profile.pgp_key_public,
-        'pgp_key_expiration': profile.pgp_key_expiration,
+        'pgp_key_fingerprint': user.pgp_key_fingerprint,
+        'pgp_key_public': user.pgp_key_public,
+        'pgp_key_expiration': user.pgp_key_expiration,
         'pgp_key_remove': False,
         'picture': picture,
         'tid': user.tid,

backend/globaleaks/handlers/wizard.py

@@ -73,6 +73,9 @@ def db_wizard(session, tid, hostname, request):
 
         if admin_profile:
            admin_desc['profile_id'] = admin_profile.id
+           admin_profile.language = language
+           session.add(admin_profile)
+           session.commit()
 
         admin_user = db_create_user(session, tid, None, admin_desc, language)
         db_set_user_password(session, tid, admin_user, request['admin_password'])
@@ -94,6 +97,9 @@ def db_wizard(session, tid, hostname, request):
 
         if receiver_profile:
            receiver_desc['profile_id'] = receiver_profile.id
+           receiver_profile.language = language
+           session.add(receiver_profile)
+           session.commit()
 
         receiver_user = db_create_user(session, tid, None, receiver_desc, language)
         db_set_user_password(session, tid, receiver_user, request['receiver_password'])

backend/globaleaks/jobs/delivery.py

@@ -34,9 +34,8 @@ def file_delivery(session):
         ifile.new = False
         src = ifile.id
 
-        for rtip, user, profle in session.query(models.ReceiverTip, models.User, models.UserProfile) \
+        for rtip, user in session.query(models.ReceiverTip, models.User) \
                                  .filter(models.ReceiverTip.internaltip_id == ifile.internaltip_id,
-                                         models.UserProfile.id == models.User.profile_id,
                                          models.User.id == models.ReceiverTip.receiver_id):
             receiverfile = models.WhistleblowerFile()
             receiverfile.internalfile_id = ifile.id
@@ -58,7 +57,7 @@ def file_delivery(session):
 
             receiverfiles_maps[ifile.id]['wbfiles'].append({
                 'dst': os.path.abspath(os.path.join(Settings.attachments_path, receiverfile.internalfile_id)),
-                'pgp_key_public': profle.pgp_key_public
+                'pgp_key_public': user.pgp_key_public
             })
 
     for rfile, itip in session.query(models.ReceiverFile, models.InternalTip)\

backend/globaleaks/jobs/pgp_check.py

@@ -22,9 +22,9 @@ def db_get_expired_or_expiring_pgp_users(session, tids_list):
     threshold = datetime_now() + timedelta(15)
 
     query = (session.query(models.User, models.UserProfile).join(models.UserProfile, models.User.profile_id == models.UserProfile.id)
-             .filter(models.UserProfile.pgp_key_public != '',
-                     models.UserProfile.pgp_key_expiration != datetime_null(),
-                     models.UserProfile.pgp_key_expiration < threshold,
+             .filter(models.User.pgp_key_public != '',
+                     models.User.pgp_key_expiration != datetime_null(),
+                     models.User.pgp_key_expiration < threshold,
                      models.User.tid.in_(tids_list)))
 
     return query.all()
@@ -72,10 +72,10 @@ def perform_pgp_validation_checks(self, session):
             tenant_expiry_map.setdefault(user.tid, []).append(user_desc)
 
             log.info('Removing expired PGP key of: %s', user.username, tid=user.tid)
-            if profile.pgp_key_expiration < datetime_now():
-                profile.pgp_key_public = ''
-                profile.pgp_key_fingerprint = ''
-                profile.pgp_key_expiration = datetime_null()
+            if user.pgp_key_expiration < datetime_now():
+                user.pgp_key_public = ''
+                user.pgp_key_fingerprint = ''
+                user.pgp_key_expiration = datetime_null()
 
         for tid, expired_or_expiring in tenant_expiry_map.items():
             for user_desc in expired_or_expiring:

backend/globaleaks/models/__init__.py

@@ -951,6 +951,11 @@ class _User(Model):
 
     accepted_privacy_policy = Column(DateTime, default=datetime_null, nullable=False)
     clicked_recovery_key = Column(Boolean, default=False, nullable=False)
+    # BEGIN of PGP key fields
+    pgp_key_fingerprint = Column(UnicodeText, default='', nullable=False)
+    pgp_key_public = Column(UnicodeText, default='', nullable=False)
+    pgp_key_expiration = Column(DateTime, default=datetime_null, nullable=False)
+    # END of PGP key fields
 
     unicode_keys = ['username', 'role',
                     'language', 'mail_address',
@@ -1010,12 +1015,6 @@ class _UserProfile(Model):
     can_reopen_reports = Column(Boolean, default=True, nullable=False)
     can_edit_general_settings = Column(Boolean, default=False, nullable=False)
     language = Column(UnicodeText(12), nullable=False)
-    # BEGIN of PGP key fields
-    pgp_key_fingerprint = Column(UnicodeText, default='', nullable=False)
-    pgp_key_public = Column(UnicodeText, default='', nullable=False)
-    pgp_key_expiration = Column(DateTime, default=datetime_null, nullable=False)
-    # END of PGP key fields
-
 
 class _ReceiverFile(Model):
     """

backend/globaleaks/rest/requests.py

@@ -133,7 +133,9 @@ def get_multilang_request_format(request_format, localized_strings):
     'role': user_role_regexp,
     'password_change_needed': bool,
     'mail_address': email_regexp,
+    'pgp_key_public': str,
     'pgp_key_remove': bool,
+    'pgp_key_fingerprint': str,
     'pgp_key_expiration': str,
     'profile_id': str
 }
@@ -142,9 +144,6 @@ def get_multilang_request_format(request_format, localized_strings):
     'name': str,
     'role': user_role_regexp,
     'enabled': bool,
-    'pgp_key_fingerprint': str,
-    'pgp_key_public': str,
-    'pgp_key_remove': bool,
     'notification': bool,
     'can_edit_general_settings': bool,
     'can_delete_submission': bool,

backend/globaleaks/tests/handlers/admin/test_user.py

@@ -63,7 +63,7 @@ class TestReceiverInstance(TestAdminInstance):
             'name': 'Mario Rossi',
             'mail_address': '[email protected]',
             'language': 'en',
-            'forcefully_selected': True
+            'forcefully_selected': False
         }
     }