Add energy (FEE/Ebeam) calibration to the XFEL UI

- EnergyTab is a new tab in the GUI where the user can use the FEE
  calibration tools, generate and view associated plots, and export
  a phil file with the results. The phil can be imported into
  rungroups directly, avoiding manual transcription errors. This
  includes the FEE spectrometer calibration from the notch scan as
  well as visualizing and calculating the Ebeam offset -- that is,
  spectrum_eV_offset, spectrum_eV_per_pixel, and wavelength_offset.
  (Note, we really should consider renaming these and doing all
  adjustments in eV.)

- CalibWorker is a new thread running in the background, executing
  any requested calibration tasks (so far just for EnergyTab).

- ebeam_plotter.py is the plotting tool to replace the command line
  script thing.py for use in the GUI.

- Small necessary changes were made to existing calibration and
  visualization scripts to make them able to accept an existing
  matplotlib figure and add plots to it, and to write out results.

- Note of caution, window resizing seems to be crash-prone. Not
  entirely sure why, but given this fact, I haven't put work into
  making things resize nicely for different screen sizes. This is
  all optimized for the control room at MFX.

serialtbx/util/energy_scan_notch_finder.py

@@ -48,7 +48,7 @@ def find_notch(data_x, data_y, kernel_size, fit_half_range, baseline_cutoff, ref
   fitted_min_y = notch_shape.__call__(fitted_min_x)
   return ((fitted_min_x, fitted_min_y), notch_shape, smoothed_y, flattened_y)
 
-def plot_notches(runs, rundata, notches, per_run_plots=False):
+def plot_notches(runs, rundata, notches, per_run_plots=False, use_figure=None):
   """Plot the energy scan, optionally one plot per spectrum for troubleshooting misses when automatically identifying notch positions, and always as an overlay of spectra in the scan with notch positions marked."""
   if per_run_plots:
     for run, data_y, notch in zip(runs, rundata, notches):
@@ -64,36 +64,43 @@ def plot_notches(runs, rundata, notches, per_run_plots=False):
       plt.legend()
       plt.figure()
 
+  fig = use_figure or plt.figure()
+  ax = fig.subplots()
   for run, data, notch in zip(runs, rundata, notches):
     (notch_x, notch_y), notch_shape, smoothed_y, flattened_y = notch
-    plt.plot(range(len(data)), data, '-', label=f"run {run}: notch at {int(notch_x)} pixels".format())
-    plt.plot([notch_x], [notch_y], 'k+', label="_nolegend_")
+    ax.plot(range(len(data)), data, '-', label=f"run {run}: notch at {int(notch_x)} pixels".format())
+    ax.plot([notch_x], [notch_y], 'k+', label="_nolegend_")
   # repeat last one to add legend
-  plt.plot([notch_x], [notch_y], 'k+', label="identified notches")
+  ax.plot([notch_x], [notch_y], 'k+', label="identified notches")
 
-  plt.legend()
-  plt.title("Energy scan")
-  plt.xlabel("FEE spectrometer pixels")
-  plt.ylabel("Mean counts")
-  plt.figure()
+  ax.legend()
+  ax.set_title("Energy scan")
+  ax.set_xlabel("FEE spectrometer pixels")
+  ax.set_ylabel("Mean counts")
 
-def calibrate_energy(notches, energies):
+def calibrate_energy(notches, energies, return_trendline=False, use_figure=None):
   """Having identified the pixel positions on the FEE spectrometer corresponding to known energy values, get a linear fit of these ordered pairs and report the eV offset and eV per pixel matching the fit."""
   pixels = [n[0][0] for n in notches]
   linear_fit = Poly.fit(pixels, energies, 1).convert()
   eV_offset, eV_per_pixel = linear_fit.coef
   print(f"Calibrated eV offset of {eV_offset} and eV per pixel of {eV_per_pixel}".format())
-  plt.scatter(pixels, energies, color='k', label="known energy positions")
+  fig = use_figure or plt.figure()
+  ax = fig.subplots()
+  ax.scatter(pixels, energies, color='k', label="known energy positions")
   px_min = int(min(pixels))
   px_max = int(max(pixels))
   px_range = px_max - px_min
   trendline_x = np.arange(px_min-0.1*px_range, px_max+0.1*px_range, int(px_range/10))
   trendline_y = linear_fit.__call__(trendline_x)
-  plt.plot(trendline_x, trendline_y, 'b-', label=f"linear fit: y = {eV_per_pixel:.4f}*x + {eV_offset:.4f}".format())
-  plt.xlabel("FEE spectrometer pixels")
-  plt.ylabel("Energy (eV)")
-  plt.title("Energy calibration")
-  plt.legend()
-  plt.show()
-  return (eV_offset, eV_per_pixel)
+  ax.plot(trendline_x, trendline_y, 'b-', label=f"linear fit: y = {eV_per_pixel:.4f}*x + {eV_offset:.4f}".format())
+  ax.set_xlabel("FEE spectrometer pixels")
+  ax.set_ylabel("Energy (eV)")
+  ax.set_title("Energy calibration")
+  ax.legend()
+  if use_figure is None:
+    plt.show()
+  if return_trendline:
+    return ((eV_offset, eV_per_pixel), (linear_fit, trendline_x, trendline_y))
+  else:
+    return (eV_offset, eV_per_pixel)
 

xfel/command_line/fee_calibration.py

@@ -15,6 +15,9 @@
 verbose = False
   .type = bool
   .help = print all possible output
+output_phil = None
+  .type = path
+  .help = path where calibrated values should be written as a phil file
 """
 
 phil_scope = parse(fee_phil_string + notch_phil_string)
@@ -97,6 +100,15 @@ def run(args):
              for data in rundata]
   plot_notches(runs, rundata, notches, params.per_run_plots)
   eV_offset, eV_per_pixel = calibrate_energy(notches, energies)
+  args_str = ' '.join(args)
+  with open('fee_calib.out', 'a') as outfile:
+    outfile.write(f'using {args_str}, eV_offset={eV_offset} eV_per_pixel={eV_per_pixel}\n')
+  print('wrote calibrated values to fee_calib.out')
+  if params.output_phil:
+    with open(params.output_phil, 'w') as outfile:
+      outfile.write(f'spectrum_eV_offset={eV_offset}\n')
+      outfile.write(f'spectrum_eV_per_pixel={eV_per_pixel}\n')
+    print(f'wrote calibrated values to {params.output_phil}')
 
 if __name__ == "__main__":
   import sys

xfel/ui/components/ebeam_plotter.py

@@ -0,0 +1,62 @@
+from __future__ import division
+
+import dxtbx
+from xfel.cxi.cspad_ana import cspad_tbx
+from matplotlib import pyplot as plt
+import numpy as np
+from simtbx.nanoBragg.utils import ENERGY_CONV
+
+def compare_ebeams_with_fees(locfiles, runs=None, plot=True, use_figure=None):
+  if plot:
+    fig = use_figure or plt.figure()
+    ax = fig.subplots()
+
+  ebeam_eV_offsets = []
+  ebeam_wav_offsets = []
+
+  for i in range(len(locfiles)):
+    if locfiles[i] is None:
+      if plot:
+        ax.plot([],[], label='No data for run {runs[i]}')
+      continue
+
+    img = dxtbx.load(locfiles[i])
+    n_img = img.get_num_images()
+
+    ebeams_eV = []
+    ebeams_wav = []
+    fee_coms_eV = []
+    fee_coms_wav = []
+
+    for j in range(n_img):
+      if not img.get_spectrum(j):
+        continue # no FEE
+      ewav = cspad_tbx.evt_wavelength(img._get_event(j))
+      if not ewav:
+        continue # no ebeam
+      fee_coms_wav.append(fwav:=img.get_beam(j).get_wavelength())
+      fee_coms_eV.append(feV:=ENERGY_CONV/fwav)
+      ebeams_wav.append(ewav)
+      ebeams_eV.append(eeV:=ENERGY_CONV/ewav)
+      print(f'{i}: {int(feV)} eV FEE / {int(eeV)} eV Ebeam')
+
+    if plot:
+      ax.hist(ebeams_eV, alpha=0.5, bins=40, label=f'run {runs[i]} ebeams ({int(eeV)} eV)')
+      ax.hist(fee_coms_eV, alpha=0.5, bins=40, label=f'run {runs[i]} FEE COMs ({int(feV)} eV)')
+
+    diffs_eV = np.array(fee_coms_eV) - np.array(ebeams_eV)
+    ebeam_eV_offsets.append(sum(diffs_eV)/len(diffs_eV))
+    diffs_wav = np.array(fee_coms_wav) - np.array(ebeams_wav)
+    ebeam_wav_offsets.append(sum(diffs_wav)/len(diffs_wav))
+
+  if plot:
+    ax.legend()
+    ax.set_xlabel('Energy (eV)')
+    ax.set_ylabel('Counts')
+    ax.set_title('Ebeam vs Calibrated FEE')
+    if not use_figure:
+      plt.show()
+
+  return (sum(ebeam_eV_offsets)/len(ebeam_eV_offsets),
+          sum(ebeam_wav_offsets)/len(ebeam_wav_offsets))
+