XFEL GUI updates (#992)

Series of updates from most recent beamtime:

* Switch extant usage of mpi4py to libtbx.mpi4py
* Refresh cached database connection after 5 minutes
* add pydrive2 to psana_environment
* XFEL GUI: only move on to next task if the previous one is in the DONE state
* XFEL GUI: more tweaks for energy tab: add max events box, fix for parameter loading at GUI start, hide energy tab if not LCLS, fix for ListCtrl if clicked and empty, remove resizing code, fix invisible boxes in ListCtrl
* Prevent duplicate gdrive uploads
* Improve mpi safety
* only disable mpi if the gui actually runs
* XFEL GUI: energy tab, label eBeam using average
* XFEL GUI: open image viewer when clicking on a dot (LCLS only for now)
* XFEL GUI: better checks for closed image viewer

Co-authored-by: Daniel Paley <[email protected]>

libtbx/__init__.py

@@ -68,6 +68,9 @@ def __new__(cls):
 
 Auto = AutoType()
 
+class mpi_import_guard:
+  disable_mpi = False
+
 class slots_getstate_setstate(object):
   """
   Implements getstate and setstate for classes with __slots__ defined. Allows an

libtbx/mpi4py.py

@@ -67,13 +67,23 @@ def size(self):
 
 mpiEmulator.COMM_WORLD = mpiCommEmulator()
 
+class MpiDisabledError(Exception):
+  pass
+
 try:
+  import libtbx
+  if libtbx.mpi_import_guard.disable_mpi:
+    raise MpiDisabledError
   from mpi4py import MPI
   using_mpi = True
 except ImportError:
   print ("\nWarning: could not import mpi4py. Running as a single process.\n")
   MPI = mpiEmulator()
   using_mpi = False
+except MpiDisabledError:
+  MPI = mpiEmulator()
+  using_mpi = False
+
 
 def mpi_abort_on_exception(func):
   """

prime/command_line/mpi_run.py

@@ -3,7 +3,7 @@
 Find initial scaling factors for all integration results
 """
 from __future__ import absolute_import, division, print_function
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 import sys, os
 from prime.postrefine.mod_input import process_input, read_pickles
 from prime.postrefine.mod_util import intensities_scaler

prime/command_line/mpi_scale.py

@@ -3,7 +3,7 @@
 Find initial scaling factors for all integration results
 """
 from __future__ import absolute_import, division, print_function
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 import sys, os
 from prime.postrefine.mod_input import process_input, read_pickles
 from prime.postrefine.mod_util import intensities_scaler

simtbx/command_line/errors.py

@@ -10,7 +10,7 @@
 parser.add_argument("--ndev", type=int, default=1, help="number of gpu devices")
 args = parser.parse_args()
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 COMM = MPI.COMM_WORLD
 
 import os

simtbx/command_line/estimate_Ncells_Eta.py

@@ -17,7 +17,7 @@
 #parser.add_argument("--njobs", type=int, default=5, help="number of jobs (only runs on single node, no MPI)")
 parser.add_argument("--plot", action="store_true", help="show a histogram at the end")
 args = parser.parse_args()
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 COMM = MPI.COMM_WORLD
 #from joblib import Parallel, delayed
 import json

simtbx/command_line/integrate.py

@@ -21,7 +21,7 @@
 
 args = parser.parse_args()
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 COMM = MPI.COMM_WORLD
 
 import logging

xfel/amo/pnccd_ana/mpi_fxs_bg.py

@@ -17,7 +17,7 @@
 #times. Here ignoring those errors.
 np.seterr(divide='ignore', invalid='ignore')
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -141,7 +141,7 @@ def compute_bg(argv=None) :
     argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/mpi_fxs_c2.py

@@ -18,7 +18,7 @@
 #times. Here ignoring those errors.
 np.seterr(divide='ignore', invalid='ignore')
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -148,7 +148,7 @@ def compute_c2(argv=None) :
      argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/mpi_fxs_calib.py

@@ -16,7 +16,7 @@
 #times. Here ignoring those errors.
 np.seterr(divide='ignore', invalid='ignore')
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -135,7 +135,7 @@ def compute_calib(argv=None) :
     argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/mpi_fxs_index.py

@@ -17,7 +17,7 @@
 #times. Here ignoring those errors.
 np.seterr(divide='ignore', invalid='ignore')
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -136,7 +136,7 @@ def compute_index(argv=None) :
     argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/mpi_fxs_launch.py

@@ -77,7 +77,7 @@ def launch(argv=None) :
      argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/mpi_fxs_mask.py

@@ -18,7 +18,7 @@
 #times. Here ignoring those errors.
 np.seterr(divide='ignore', invalid='ignore')
 
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -140,7 +140,7 @@ def compute_mask(argv=None) :
     argv = sys.argv[1:]
 
   try:
-     from mpi4py import MPI
+     from libtbx.mpi4py import MPI
   except ImportError:
      raise Sorry("MPI not found")
 

xfel/amo/pnccd_ana/pnccd_hit.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, division, print_function
 import numpy as np
-from mpi4py import MPI
+from libtbx.mpi4py import MPI
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()

xfel/command_line/FEE_average_plot.py

@@ -92,7 +92,7 @@ def run(args):
   ds = DataSource(dataset_name)
   src = Source('DetInfo(%s)'%params.input.address)
   # set up multiprocessing with MPI
-  from mpi4py import MPI
+  from libtbx.mpi4py import MPI
   comm = MPI.COMM_WORLD
   rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
   size = comm.Get_size() # size: number of processes running in this job

xfel/command_line/cxi_xtc_process.py

@@ -137,7 +137,7 @@ def run(self):
     print("Processing run %d of experiment %s using config file %s"%(params.input.run_num, params.input.experiment, params.input.cfg))
 
     if params.mp.method == "mpi":
-      from mpi4py import MPI
+      from libtbx.mpi4py import MPI
       comm = MPI.COMM_WORLD
       rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
       size = comm.Get_size() # size: number of processes running in this job

xfel/command_line/mpi_average.py

@@ -22,7 +22,7 @@ def average(argv=None):
     argv = sys.argv[1:]
 
   try:
-    from mpi4py import MPI
+    from libtbx.mpi4py import MPI
   except ImportError:
     raise Sorry("MPI not found")
 

xfel/command_line/upload_mtz.py

@@ -5,6 +5,7 @@
 from dials.util import Sorry
 import os, sys
 import re
+import fcntl
 
 
 help_message = """
@@ -71,6 +72,21 @@ def _get_log_fname(mtz_fname):
   assert len(hit.groups()) == 1
   return hit.groups()[0] + '_main.log'
 
+class Locker:
+  """ See https://stackoverflow.com/a/60214222
+  """
+  def __enter__(self):
+    try:
+      self.fp = open(os.path.expanduser('~/.upload_mtz.lock'), 'wb')
+    except FileNotFoundError:
+      self.fp = None
+    if self.fp is not None:
+      fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX)
+  def __exit__(self, *args, **kwargs):
+    if self.fp is not None:
+      fcntl.flock(self.fp.fileno(), fcntl.LOCK_UN)
+      self.fp.close()
+
 class pydrive2_interface:
   """
   Wrapper for uploading versioned mtzs and logs using Pydrive2. Constructed from
@@ -92,26 +108,29 @@ def __init__(self, cred_file, folder_id):
     self.drive = GoogleDrive(gauth)
     self.top_folder_id = folder_id
 
+
+
   def _fetch_or_create_folder(self, fname, parent_id):
-    query = {
-        "q": "'{}' in parents and title='{}'".format(parent_id, fname),
-        "supportsTeamDrives": "true",
-        "includeItemsFromAllDrives": "true",
-        "corpora": "allDrives"
-    }
-    hits = self.drive.ListFile(query).GetList()
-    if hits:
-      assert len(hits)==1
-      return hits[0]['id']
-    else:
+    with Locker():
       query = {
-        "title": fname,
-        "mimeType": "application/vnd.google-apps.folder",
-        "parents": [{"kind": "drive#fileLink", "id": parent_id}]
+          "q": "'{}' in parents and title='{}'".format(parent_id, fname),
+          "supportsTeamDrives": "true",
+          "includeItemsFromAllDrives": "true",
+          "corpora": "allDrives"
       }
-      f = self.drive.CreateFile(query)
-      f.Upload()
-      return f['id']
+      hits = self.drive.ListFile(query).GetList()
+      if hits:
+        assert len(hits)==1
+        return hits[0]['id']
+      else:
+        query = {
+          "title": fname,
+          "mimeType": "application/vnd.google-apps.folder",
+          "parents": [{"kind": "drive#fileLink", "id": parent_id}]
+        }
+        f = self.drive.CreateFile(query)
+        f.Upload()
+        return f['id']
 
   def _upload_detail(self, file_path, parent_id):
     title = os.path.split(file_path)[1]

xfel/command_line/xfel_process.py

@@ -3,6 +3,7 @@
 # LIBTBX_SET_DISPATCHER_NAME cctbx.xfel.process
 
 from __future__ import absolute_import, division, print_function
+from libtbx.mpi4py import mpi_abort_on_exception
 
 help_message = '''
 
@@ -85,6 +86,12 @@ def __init__(self):
       epilog=help_message
       )
 
+  @mpi_abort_on_exception
+  def run(self):
+    super().run()
+
+
+
 if __name__ == '__main__':
   import dials.command_line.stills_process
   dials.command_line.stills_process.Processor = DialsProcessorWithLogging

xfel/command_line/xtc_dump.py

@@ -156,7 +156,7 @@ def run(self):
     self.params = params
     self.options = options
 
-    from mpi4py import MPI
+    from libtbx.mpi4py import MPI
     comm = MPI.COMM_WORLD
     rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
     size = comm.Get_size() # size: number of processes running in this job

xfel/command_line/xtc_process.py

@@ -615,7 +615,7 @@ def run(self):
     self.options = options
 
     if params.mp.method == "mpi":
-      from mpi4py import MPI
+      from libtbx.mpi4py import MPI
       comm = MPI.COMM_WORLD
       rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
       size = comm.Get_size() # size: number of processes running in this job

xfel/conda_envs/psana_environment.yml

@@ -76,6 +76,7 @@ dependencies:
  # xfel gui
  - mysql
  - mysqlclient
+ - pydrive2
 
  # Avoid numpy 1.21.[01234]
  # See https://github.com/cctbx/cctbx_project/issues/627

xfel/cxi/cspad_ana/mod_event_code.py

@@ -50,7 +50,7 @@ def __init__(
         self.size = int(os.environ['SGE_TASK_LAST']) - int(os.environ['SGE_TASK_FIRST']) + 1
     else:
       try:
-        from mpi4py import MPI
+        from libtbx.mpi4py import MPI
       except ImportError:
         self.rank = 0
         self.size = 1

xfel/merging/application/phil/phil.py

@@ -687,7 +687,7 @@
 
 diffbragg_phil = """
 diffBragg {
-  include scope simtbx.command_line.hopper.phil_scope
+  include scope simtbx.diffBragg.phil.phil_scope
 }
 """
 

xfel/small_cell/command_line/small_cell_index.py

@@ -118,7 +118,7 @@ def run(argv=None):
       files = os.listdir(path)
 
       try:
-        from mpi4py import MPI
+        from libtbx.mpi4py import MPI
         comm = MPI.COMM_WORLD
         rank = comm.Get_rank()
         size = comm.Get_size()