Skip to content

Commit

Permalink
Merge pull request #18 from NSLS-II-HXN/deploy-2020-2
Browse files Browse the repository at this point in the history
Fixes to work with collection-2021* conda envs
  • Loading branch information
danielballan authored Apr 13, 2022
2 parents a22f7f3 + 27d2f9f commit d724c46
Show file tree
Hide file tree
Showing 19 changed files with 1,533 additions and 337 deletions.
8 changes: 6 additions & 2 deletions .ci/bl-specific.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
#!/bin/bash

# cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/...
# TODO: watch for https://github.com/NSLS-II-HXN/hxnfly/pull/8 to be
# merged/released, then this install can be removed from here.
python3 -m pip install --no-deps -vv git+https://github.com/NSLS-II-HXN/hxnfly@master

conda install -y -c ${CONDA_CHANNEL_NAME} 03-id-hxn-collection
sudo mkdir -v -p /home/xf03id/
sudo chown -Rv $USER: /home/xf03id/
touch /home/xf03id/benchmark.out
35 changes: 20 additions & 15 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -64,23 +64,28 @@ security
profile_default

*.sw*
profile_collection/startup/20-detectors.py.bak.write-on-gpfs-on-merlin_matt_20200219
profile_collection/startup/20-detectors.py.write-local-on-merlin-test_matt_20200219
startup/20-detectors.py.bak.write-on-gpfs-on-merlin_matt_20200219
startup/20-detectors.py.write-local-on-merlin-test_matt_20200219
*~
profile_collection/startup/#*.py#
profile_collection/startup/eLog.pdf
profile_collection/startup/eLog_info.obj
startup/#*.py#
startup/eLog.pdf
startup/eLog_info.obj
old/
# Quick test:
profile_collection/startup/20-detectors-eiger.py
startup/20-detectors-eiger.py

# Ignore temporary files
startup/tmp_fig.pdf
startup/tmp_img.png
startup/eLog_info.obj

# PostScript files to ignore (no .ps extension):
profile_collection/startup/functools
profile_collection/startup/gridspec
profile_collection/startup/h5py
profile_collection/startup/np
profile_collection/startup/os
profile_collection/startup/plt
profile_collection/startup/shutil
profile_collection/startup/sys
profile_collection/startup/css-workspace-xf03id-ws1/
startup/functools
startup/gridspec
startup/h5py
startup/np
startup/os
startup/plt
startup/shutil
startup/sys
startup/css-workspace-xf03id-ws1/
9 changes: 6 additions & 3 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@ resources:
endpoint: github

jobs:
- template: collection-2021-1.2.yml@templates
parameters:
beamline_acronym: HXN
- template: nsls2-collection-2021-2.2.yml@templates
parameters:
beamline_acronym: HXN
- template: nsls2-collection-2021-2.2-py39.yml@templates
parameters:
beamline_acronym: HXN
- template: nsls2-collection-2021-3.0-py37.yml@templates
parameters:
beamline_acronym: HXN
- template: nsls2-collection-2021-3.0-py39.yml@templates
parameters:
beamline_acronym: HXN
105 changes: 78 additions & 27 deletions startup/00-startup.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,85 @@
import certifi
import functools
import os
import time
import uuid
import warnings
from collections import deque
from datetime import datetime, timedelta, tzinfo

# The following code allows to call Matplotlib API from threads (experimental)
# Requires https://github.com/tacaswell/mpl-qtthread (not packaged yet)
import matplotlib
import matplotlib.backends.backend_qt5
import mpl_qtthread
# set up the teleporter
mpl_qtthread.backend.initialize_qt_teleporter()
# tell Matplotlib to use this backend
matplotlib.use("module://mpl_qtthread.backend_agg")
# suppress (now) spurious warnings for mpl3.3+
mpl_qtthread.monkeypatch_pyplot()

import certifi
import ophyd
import pandas as pd
import pymongo
import six
from ophyd.signal import EpicsSignalBase

import certifi
import ophyd
import pandas as pd
import pymongo
import six
from ophyd.signal import EpicsSignalBase

EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10)

# Set up a Broker.
# TODO clean this up
from bluesky_kafka import Publisher
from databroker import Broker
from databroker.headersource.mongo import MDS
from databroker.v0 import Broker
from databroker.assets.mongo import Registry

from databroker.headersource.core import doc_or_uid_to_uid

from datetime import timedelta, datetime, tzinfo

import pymongo
from pymongo import MongoClient

import uuid
from databroker.headersource.mongo import MDS
from databroker.v0 import Broker
from jsonschema import validate as js_validate
import six
from collections import deque
from pymongo import MongoClient

import os
os.environ["PPMAC_HOST"] = "xf03idc-ppmac1"

bootstrap_servers = os.getenv("BLUESKY_KAFKA_BOOTSTRAP_SERVERS", None)
if bootstrap_servers is None:
# https://github.com/NSLS-II/nslsii/blob/b332c34813adf798c38184292d21537ef4f653bb/nslsii/__init__.py#L710-L712
msg = ("The 'BLUESKY_KAFKA_BOOTSTRAP_SERVERS' environment variable must "
"be defined as a comma-delimited list of Kafka server addresses "
"or hostnames and ports as a string such as "
"``'kafka1:9092,kafka2:9092``")
raise RuntimeError(msg)

kafka_password = os.getenv("BLUESKY_KAFKA_PASSWORD", None)
if kafka_password is None:
msg = "The 'BLUESKY_KAFKA_PASSWORD' environment variable must be set."
raise RuntimeError(msg)

kafka_publisher = Publisher(
topic="hxn.bluesky.datum.documents",
bootstrap_servers=os.environ['BLUESKY_KAFKA_BOOTSTRAP_SERVERS'],
bootstrap_servers=bootstrap_servers,
key=str(uuid.uuid4()),
producer_config={
"acks": 1,
"message.timeout.ms": 3000,
"queue.buffering.max.kbytes": 10 * 1048576,
"compression.codec": "snappy",
"security.protocol": "SSL",
"ssl.ca.location": certifi.where()
},
"ssl.ca.location": certifi.where(),
"security.protocol": "SASL_SSL",
"sasl.mechanisms": "PLAIN",
"sasl.username": "beamline",
"sasl.password": kafka_password,
},
flush_on_stop_doc=True,
)


# DB1
db1_name = 'rs'
db1_addr = 'mongodb://xf03id1-mdb01:27017,xf03id1-mdb02:27017,xf03id1-mdb03:27017'
Expand Down Expand Up @@ -257,6 +294,8 @@ def bulk_register_datum_table(self, resource_uid, dkwargs_table, validate=False)
mds_db1 = MDS(_mds_config_db1, auth=False)
db1 = Broker(mds_db1, CompositeRegistry(_fs_config_db1))


# wrapper for two databases
class CompositeBroker(Broker):
"""wrapper for two databases"""

Expand Down Expand Up @@ -326,16 +365,18 @@ def insert(self, name, doc):
db = CompositeBroker(mds_db1, CompositeRegistry(_fs_config_db1))

from hxntools.handlers import register as _hxn_register_handlers

_hxn_register_handlers(db)
del _hxn_register_handlers
# do the rest of the standard configuration
from IPython import get_ipython
from nslsii import configure_base, configure_olog

configure_base(get_ipython().user_ns, db, bec=False)
configure_base(get_ipython().user_ns, db, bec=False, ipython_logging=False)
# configure_olog(get_ipython().user_ns)

from bluesky.callbacks.best_effort import BestEffortCallback

bec = BestEffortCallback()

# un import *
Expand All @@ -355,11 +396,11 @@ def insert(self, name, doc):
RE.md['beamline_id'] = 'HXN'
RE.verbose = True

# set up some HXN specific callbacks
from ophyd.callbacks import UidPublish
from hxntools.scan_number import HxnScanNumberPrinter
from hxntools.scan_status import HxnScanStatus
from ophyd import EpicsSignal
# set up some HXN specific callbacks
from ophyd.callbacks import UidPublish

uid_signal = EpicsSignal('XF:03IDC-ES{BS-Scan}UID-I', name='uid_signal')
uid_broadcaster = UidPublish(uid_signal)
Expand All @@ -372,7 +413,7 @@ def flush_on_stop_doc(name, doc):
kafka_publisher.flush()

# This is needed to prevent the local buffer from filling.
RE.subscribe('stop', flush_on_stop_doc)
RE.subscribe(flush_on_stop_doc, 'stop')

# Pass on only start/stop documents to a few subscriptions
for _event in ('start', 'stop'):
Expand All @@ -389,6 +430,7 @@ def ensure_proposal_id(md):

# be nice on segfaults
import faulthandler

faulthandler.enable()

# set up logging framework
Expand Down Expand Up @@ -417,8 +459,8 @@ def ensure_proposal_id(md):
pd.options.display.max_rows = None
pd.options.display.max_columns = 10

from bluesky.plan_stubs import mov

from bluesky.plan_stubs import mov
# from bluesky.utils import register_transform

def register_transform(RE, *, prefix='<'):
Expand All @@ -435,6 +477,7 @@ def register_transform(RE, *, prefix='<'):
valid python syntax or an existing transform you are on your own.
'''
import IPython

# from IPython.core.inputtransformer2 import StatelessInputTransformer

# @StatelessInputTransformer.wrap
Expand Down Expand Up @@ -522,7 +565,17 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs):
if as_string is None:
as_string = self._string

with self._lock:
###########################################
# Usedf only for old ophyd 1.3.3 and older.
from distutils.version import LooseVersion

import ophyd

if ophyd.__version__ < LooseVersion('1.4'):
self._metadata_lock = self._lock
###########################################

with self._metadata_lock:
if not self._read_pv.connected:
if not self._read_pv.wait_for_connection(connection_timeout):
raise TimeoutError('Failed to connect to %s' %
Expand Down Expand Up @@ -557,11 +610,9 @@ def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs):
return ret


from ophyd import EpicsSignal
from ophyd import EpicsSignalRO
from ophyd import EpicsSignal, EpicsSignalRO
from ophyd.areadetector import EpicsSignalWithRBV

EpicsSignal.get = _epicssignal_get
EpicsSignalRO.get = _epicssignal_get
EpicsSignalWithRBV.get = _epicssignal_get

12 changes: 6 additions & 6 deletions startup/13-mll.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ class HxnMLLSample(NamedDevice):
sz = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:2}Mtr', doc='coarse z')

kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillAll-Cmd.PROC',
doc='kill all piezos')
doc='kill all piezos', kind='omitted')
zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1}KillZero-Cmd.PROC',
doc='zero all piezos')
doc='zero all piezos', kind='omitted')
# sz1 = Cpt(EpicsMotor, 'XF:03IDC-ES{ANC350:3-Ax:3}Mtr', doc='coarse z1')
# sz1 was replaced with vz when controller 2 died

Expand Down Expand Up @@ -69,12 +69,12 @@ class HxnMLLDiffractionSample(NamedDevice):
#dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y')
#dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_z')
# swap back dssx and dssz (08/07/19, X. Huang)
dssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_x')
dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y')
dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssz}Mtr', doc='fine_z')
dssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssx}Mtr', doc='fine_x', timeout = 120)
dssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssy}Mtr', doc='fine_y', timeout = 120)
dssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-dssz}Mtr', doc='fine_z', timeout = 120)

kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-Diff}Kill-Cmd.PROC',
doc='kill all piezos')
doc='kill all piezos', kind='omitted')


smlld = HxnMLLDiffractionSample('', name='smlld')
Expand Down
10 changes: 5 additions & 5 deletions startup/15-zp.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ class SmarpodRotationAxis(SmarpodBase):
class HxnZPSample(NamedDevice):
# Zoneplate module fine sample stage axes (closed on cap
# sensors/interferometer)
zpssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssx}Mtr', doc='fine x')
zpssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssy}Mtr', doc='fine y')
zpssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssz}Mtr', doc='fine z')
zpssx = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssx}Mtr', doc='fine x', timeout = 120)
zpssy = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssy}Mtr', doc='fine y', timeout = 120)
zpssz = Cpt(EpicsMotor, 'XF:03IDC-ES{Ppmac:1-zpssz}Mtr', doc='fine z', timeout = 120)

# rotary underneath sample
zpsth = Cpt(EpicsMotor, 'XF:03IDC-ES{SC210:1-Ax:1}Mtr', doc='theta')
Expand All @@ -58,8 +58,8 @@ class HxnZPSample(NamedDevice):
smarthy = Cpt(SmarpodRotationAxis, axis=3, doc='smarpod theta around y')
smarthz = Cpt(SmarpodRotationAxis, axis=1, doc='smarpod theta around z')

kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC')
zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC')
kill = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Kill-Cmd.PROC', kind='omitted')
zero = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Zero-Cmd.PROC', kind='omitted')
mode = Cpt(EpicsSignal, 'XF:03IDC-ES{Ppmac:1-ZP}Mode-I')


Expand Down
18 changes: 13 additions & 5 deletions startup/21-xspress3.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from ophyd.device import (Component as Cpt)

from hxntools.detectors.xspress3 import (Xspress3FileStore,
Xspress3Channel)
from nslsii.detectors.xspress3 import (Xspress3FileStore,
Xspress3Channel)
from hxntools.detectors.hxn_xspress3 import HxnXspress3DetectorBase
import threading
from ophyd import DeviceStatus
Expand Down Expand Up @@ -139,7 +139,12 @@ def unstage(self, *args, **kwargs):


# Create directories on the xspress3 server, otherwise scans can fail:
xspress3.make_directories.put(True)
#====================================================================
# Note from DAMA: commented it out during the 2020-2 deployment visit.
# The corresponding code is in
# https://github.com/NSLS-II/nslsii/blob/master/nslsii/detectors/xspress3.py
#====================================================================
# xspress3.make_directories.put(True)


elem_K_list = np.array(['Na','Mg','Al','Si','P','S','Cl','Ar','K','Ca','Sc','Ti','V','Cr','Mn','Fe','Co','Ni','Cu','Zn','Ga','Ge','As','Se','Br','Kr','Rb','Sr','Y','Zr','Nb','Mo','Tc','Ru','Rh','Pd','Ag','Cd','In','Sn','Sb','Te','I','Xe','Cs','Ba','La','Hf','Ta','W','Re','Os','Ir','Pt','Au','Hg','Tl','Pb','Bi','Po','At','Rn','Fr','Ra','Ac','Ce','Pr','Nd','Pm','Sm','Eu','Gd','Tb','Dy','Ho','Er','Tm','Yb','Lu','Th','Pa','U','Np','Pu','Am','Cm','Bk','Cf'])
Expand All @@ -156,7 +161,7 @@ def unstage(self, *args, **kwargs):


def xspress3_roi_setup():
elem_list = np.array(['Mg','Si','Ge','Cl','Ga','Ti','Fe','Zn','Mn','Cu','Ni','Cr','Co','Pt_L','Hf_L','W_L'])
elem_list = np.array(['V','Si','S','P','Ca','K','Fe','Zn','Al','Ti','Ni','Cr','Co','Pt_M','Au_L','Mn'])
num_elem = np.size(elem_list)
if num_elem > 16:
num_elem = 16
Expand All @@ -173,6 +178,10 @@ def xspress3_roi_setup():
break
channel.set_roi(i+1, energy-150, energy+150, name=elem_list[i])

# Uncomment the following lines for debugging liveplots without beam.
# if elem_list[i] == "Au_L":
# channel.set_roi(i+1, 0, 40000, name=elem_list[i])


'''
def xspress3_roi_setup():
Expand Down Expand Up @@ -267,4 +276,3 @@ def configure_xspress3(sclr):
attr.kind = 'config'

configure_xspress3(xspress3)

Loading

0 comments on commit d724c46

Please sign in to comment.