resolving conflict

.travis.yml

@@ -20,6 +20,7 @@ before_install:
   - sudo service postgresql restart 11
 install:
   - psql -p 5433 -c 'create database travis_ci_test;' -U postgres
+  - pip install --upgrade setuptools==70.0.0
   - pip install --upgrade pip
   - pip install -r requirements-dev.txt
   - source travis_env.sh

Makefile

@@ -308,8 +308,6 @@ celery-start:
 	--pidfile "$(CELERY_WORKER_RUNNER_PID_FILE)" \
 	--logfile "$(CELERY_WORKER_RUNNER_LOGFILE)" &
 
-flower:
-	flower -A beagle_etl --port=5555 -logLevel=debug --broker=$(CELERY_BROKER_URL)
 # check that the Celery processes are running
 celery-check:
 	-ps auxww | grep 'celery' | grep -v 'grep' | grep -v 'make' | grep '$(CURDIR)'

beagle/__init__.py

@@ -1 +1 @@
-__version__ = "1.82.0"
+__version__ = "1.87.0"

beagle/settings.py

@@ -35,6 +35,7 @@
 
 CORS_ORIGIN_ALLOW_ALL = True
 
+SESSION_COOKIE_NAME = os.environ.get("BEAGLE_COOKIE_SESSION_NAME", "beagle_prod_session")
 
 # Application definition
 
@@ -306,8 +307,6 @@
 LIMS_PASSWORD = os.environ.get("BEAGLE_LIMS_PASSWORD")
 ETL_USER = os.environ.get("BEAGLE_ETL_USER")
 
-# NATS
-
 # SSL
 NATS_SSL_CERTFILE = os.environ.get("BEAGLE_NATS_SSL_CERTFILE")
 NATS_SSL_KEYFILE = os.environ.get("BEAGLE_NATS_SSL_KEYFILE")
@@ -369,7 +368,7 @@
         "django_auth_ldap": {"level": "DEBUG", "handlers": ["console"]},
         "django": {
             "handlers": ["file", "console"],
-            "level": "INFO",
+            "level": "DEBUG",
             "propagate": True,
         },
     },
@@ -382,7 +381,9 @@
 NOTIFIER_CC = os.environ.get("BEAGLE_NOTIFIER_CC", "")  # Put "CC [~webbera] and [~socci]" for production
 NOTIFIER_STORAGE_DIR = os.environ.get("BEAGLE_NOTIFIER_STORAGE_DIR", "/tmp")
 NOTIFIER_FILE_GROUP = os.environ.get("BEAGLE_NOTIFIER_FILE_GROUP")
+NOTIFIER_LOCAL_ATTACHMENTS_DIR = os.environ.get("BEAGLE_NOTIFIER_LOCAL_ATTACHMENTS_DIR", "/tmp")
 
+JIRA_CLOUD = os.environ.get("JIRA_CLOUD", "True") == "True"
 JIRA_PREFIX = os.environ.get("JIRA_PREFIX", "VADEV-")
 JIRA_URL = os.environ.get("JIRA_URL", "")
 JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "")
@@ -400,6 +401,7 @@
 BEAGLE_NATS_UPDATE_SAMPLE_QUEUE = os.environ.get("BEAGLE_NATS_UPDATE_SAMPLE_QUEUE", "beagle_nats_update_sample_queue")
 BEAGLE_RUNNER_QUEUE = os.environ.get("BEAGLE_RUNNER_QUEUE", "beagle_runner_queue")
 BEAGLE_DEFAULT_QUEUE = os.environ.get("BEAGLE_DEFAULT_QUEUE", "beagle_default_queue")
+BEAGLE_CHECK_FILES_QUEUE = os.environ.get("BEAGLE_CHECK_FILES_QUEUE", "beagle_check_files_queue")
 BEAGLE_JOB_SCHEDULER_QUEUE = os.environ.get("BEAGLE_JOB_SCHEDULER_QUEUE", "beagle_job_scheduler_queue")
 BEAGLE_SHARED_TMPDIR = os.environ.get("BEAGLE_SHARED_TMPDIR", "/juno/work/ci/temp")
 
@@ -444,6 +446,9 @@
 SEX_METADATA_KEY = "sex"
 RUN_DATE_METADATA_KEY = "runDate"
 
+VALIDATE_PRIMARY_ID_FOR_GENE_PANELS = os.environ.get("BEAGLE_VALIDATE_PRIMARY_ID_FOR_GENE_PANELS", "").split(",")
+PRIMARY_ID_REGEX = os.environ.get("BEAGLE_PRIMARY_ID_REGEX", "[a-z]*_[0-9]*")
+
 BEAGLE_NOTIFIER_EMAIL_GROUP = os.environ.get("BEAGLE_NOTIFIER_EMAIL_GROUP", "946a922c-8c6b-4cba-8754-16df02f05d2a")
 BEAGLE_NOTIFIER_EMAIL_ABOUT_NEW_USERS = os.environ.get("BEAGLE_NOTIFIER_EMAIL_ABOUT_NEW_USERS")
 BEAGLE_NOTIFIER_EMAIL_FROM = os.environ.get("BEAGLE_NOTIFIER_EMAIL_FROM")
@@ -453,6 +458,8 @@
 BEAGLE_NOTIFIER_VOYAGER_STATUS_PIPELINES = os.environ.get("BEAGLE_NOTIFIER_VOYAGER_STATUS_PIPELINES", "").split(",")
 BEAGLE_NOTIFIER_VOYAGER_STATUS_NOTIFY_EXTERNAL = os.environ.get("BEAGLE_NOTIFIER_NOTIFY_EXTERNAL", "").split(",")
 
+DELIVERY_FILE_SERVER = os.environ.get("BEAGLE_DELIVERY_FILE_SERVER", "http://test-server.com")
+
 ASSAYS_ADMIN_HOLD_ONLY_NORMALS = os.environ.get(
     "BEAGLE_ASSAYS_ADMIN_HOLD_ONLY_NORMALS",
     "IMPACT341,IMPACT+ (341 genes plus custom content),IMPACT468,HemePACT_v4,HemePACT_v3,IMPACT505,IMPACT410",
@@ -468,6 +475,8 @@
 NOTIFIER_WES_CC = os.environ.get("BEAGLE_NOTIFIER_WHOLE_EXOME_SEQUENCING_CC", "")
 
 DEFAULT_MAPPING = json.loads(os.environ.get("BEAGLE_COPY_MAPPING", "{}"))
+
+MAPPING = json.loads(os.environ.get("BEAGLE_FILE_MAPPING", "{}"))
 COPY_FILE_PERMISSION = 0o644
 COPY_DIR_PERMISSION = 0o750
 COPY_GROUP_OWNERSHIP = os.environ.get("BEAGLE_GROUP_OWNERSHIP", "cmoigo")
@@ -481,5 +490,17 @@
 DATADOG_RUN_ERROR_URL = os.environ.get("DATADOG_RUN_ERROR_URL", "")
 DATADOG_JOB_ERROR_URL = os.environ.get("DATADOG_JOB_ERROR_URL", "")
 
+# Pipeline Default Permissions
+
+DEFAULT_OUTPUTS_PERMISSIONS = os.environ.get("BEAGLE_DEFAULT_PERMISSIONS")
+DEFAULT_OUTPUTS_UID = int(os.environ.get("BEAGLE_DEFAULT_OUTPUTS_UID", 0))
+DEFAULT_OUTPUTS_GID = int(os.environ.get("BEAGLE_DEFAULT_OUTPUTS_GID", 0))
+
 DEFAULT_RESTART_COUNT = os.environ.get("DEFAULT_RESTART_COUNT", 0)
 DEFAULT_RESUME_COUNT = os.environ.get("DEFAULT_RESUME_COUNT", 3)
+
+MANUAL_RESTART_REPORT_THRESHOLD = os.environ.get("MANUAL_RESTART_REPORT_THRESHOLD", 4)
+MANUAL_RESTART_REPORT_PATH = os.environ.get("MANUAL_RESTART_REPORT_PATH", "/tmp/report.txt")
+
+MISSING_FILES_REPORT_PATH = os.environ.get("BEAGLE_MISSING_FILES_REPORT_PATH")
+MISSING_FILES_REPORT_COUNT = int(os.environ.get("BEAGLE_MISSING_FILES_REPORT_COUNT", 10))

beagle_etl/celery.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 import os
 from celery import Celery
+from celery.schedules import crontab
 from django.conf import settings
 from celery.app.log import TaskFormatter
 from celery.signals import after_setup_task_logger, worker_ready
@@ -42,13 +43,15 @@ def at_start(sender, **k):
     "runner.tasks.submit_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.create_jobs_from_request": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.create_jobs_from_chaining": {"queue": settings.BEAGLE_RUNNER_QUEUE},
+    "runner.tasks.create_jobs_from_pairs": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.add_pipeline_to_cache": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.running_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.terminate_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.complete_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.fail_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "notifier.tasks.send_notification": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "file_system.tasks.populate_job_group_notifier_metadata": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
+    "file_system.tasks.check_fastq_files": {"queue": settings.BEAGLE_CHECK_FILES_QUEUE},
     "beagle_etl.tasks.job_processor": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.tasks.process_smile_events": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.tasks.fetch_request_nats": {"queue": settings.BEAGLE_NATS_NEW_REQUEST_QUEUE},
@@ -76,6 +79,11 @@ def at_start(sender, **k):
         "schedule": settings.CHECK_JOB_STATUS_PERIOD,
         "options": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     },
+    "check_operator_run_alerts": {
+        "task": "runner.tasks.check_operator_run_alerts",
+        "schedule": crontab(minute=0, hour=0),
+        "options": {"queue": settings.BEAGLE_RUNNER_QUEUE},
+    },
     "process_triggers": {
         "task": "runner.tasks.process_triggers",
         "schedule": settings.PROCESS_TRIGGERS_PERIOD,
@@ -91,4 +99,9 @@ def at_start(sender, **k):
         "schedule": settings.CHECK_JOB_TIMEOUTS,
         "options": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     },
+    "check_missing_files": {
+        "task": "file_system.tasks.check_fastq_files",
+        "schedule": crontab(day_of_week=1, hour=0, minute=0),
+        "options": {"queue": settings.BEAGLE_CHECK_FILES_QUEUE},
+    },
 }

beagle_etl/exceptions.py

@@ -84,3 +84,19 @@ class DuplicatedFilesException(ETLExceptions):
     """
 
     code = 110
+
+
+class FailedToLocateTheFileException(ETLExceptions):
+    """
+    File not located in the file_system
+    """
+
+    code = 111
+
+
+class IncorrectlyFormattedPrimaryId(ETLExceptions):
+    """
+    Incorrectly formatted primaryId
+    """
+
+    code = 112

beagle_etl/fixtures/beagle_etl.operator.json

@@ -203,7 +203,7 @@
     "fields": {
       "active": true,
       "recipes": "[\"None\"]",
-      "class_name": "runner.operator.access.v2_1_0.manifest.AccessManifestOperator",
+      "class_name": "runner.operator.manifest.AccessManifestOperator",
       "version": "v2.1.0",
       "slug": "AccessManifestOperator_2_1_0"
     }
@@ -229,6 +229,27 @@
       "version": "v1.2.0",
       "slug": "ArgosOperator_v1_2_0"
     }
+  },
+  {
+    "model": "beagle_etl.operator",
+    "pk": 23,
+    "fields": {
+      "active": true,
+      "recipes": "[\"None\"]",
+      "class_name": "runner.operator.cmo_ch.v2_1_0.chip_var.CMOCHChipVarOperator",
+      "version": "v1.2.0",
+      "slug": "CMOCHChipVarOperator_v1_2_0"
+    }
+  },
+  {
+    "model": "beagle_etl.operator",
+    "pk": 24,
+    "fields": {
+      "active": true,
+      "recipes": "[\"None\"]",
+      "class_name": "runner.operator.argos_bam_operator.v1_0_0.argos_bam_operator.ArgosBamOperator",
+      "version": "v1.2.0",
+      "slug": "ArgosBamOperator_v1_0_0"
+    }
   }
-
-]
+]

beagle_etl/jobs/helper_jobs.py

@@ -9,6 +9,7 @@
     FailedToCopyFilePermissionDeniedException,
     DuplicatedFilesException,
     FailedToRegisterFileException,
+    FailedToLocateTheFileException,
 )
 
 
@@ -75,3 +76,15 @@ def check_file_exist(path):
         raise FailedToRegisterFileException("File %s not registered", path)
     elif file_cnt > 1:
         raise DuplicatedFilesException("Duplicated file %s", path)
+
+
+def locate_file(path):
+    if os.path.exists(path):
+        return path
+    for k, v in settings.MAPPING.items():
+        if path.startswith(k):
+            for r in v:
+                check_path = path.replace(k, r)
+                if os.path.exists(check_path):
+                    return check_path
+    raise FailedToLocateTheFileException(f"Unable to locate file: {path} on file system")