Adapt the webap for cutouts files. Adapt the webap to use provenance …

…tags. WEbap tests.

improc/tools.py

@@ -239,10 +239,10 @@ def strip_wcs_keywords( hdr ):
 
     """
 
-    basematch = re.compile( "^C(RVAL|RPIX|UNIT|DELT|TYPE)[12]$" )
-    cdmatch = re.compile( "^CD[12]_[12]$" )
-    sipmatch = re.compile( "^[AB]P?_(ORDER|(\d+)_(\d+))$" )
-    tpvmatch = re.compile( "^P[CV]\d+_\d+$" )
+    basematch = re.compile( r"^C(RVAL|RPIX|UNIT|DELT|TYPE)[12]$" )
+    cdmatch = re.compile( r"^CD[12]_[12]$" )
+    sipmatch = re.compile( r"^[AB]P?_(ORDER|(\d+)_(\d+))$" )
+    tpvmatch = re.compile( r"^P[CV]\d+_\d+$" )
 
     tonuke = set()
     for kw in hdr.keys():

models/provenance.py

@@ -384,6 +384,14 @@ class ProvenanceTagExistsError(Exception):
     pass
 
 class ProvenanceTag(Base, AutoIDMixin):
+    """A human-readable tag to associate with provenances.
+
+    A well-defined provenane tag will have a provenance defined for every step, but there will
+    only be a *single* provenance for each step (except for refrenceing, where there could be
+    multiple provenances defined).  The class method validate can check this for duplicates.
+
+    """
+
     __tablename__ = "provenance_tags"
 
     __table_args__ = ( UniqueConstraint( 'tag', 'provenance_id', name='_provenancetag_prov_tag_uc' ), )
@@ -437,15 +445,15 @@ def newtag( cls, tag, provs, session=None ):
 
         with SmartSession( session ) as sess:
             # Get all the provenance IDs we're going to insert
-            provids = []
+            provids = set()
             for prov in provs:
                 if isinstance( prov, Provenance ):
-                    provids.append( prov.id )
+                    provids.add( prov.id )
                 elif isinstance( prov, str ):
                     provobj = sess.get( Provenance, prov )
                     if provobj is None:
                         raise ValueError( f"Unknown Provenance ID {prov}" )
-                    provids.append( provobj.id )
+                    provids.add( provobj.id )
                 else:
                     raise TypeError( f"Everything in the provs list must be Provenance or str, not {type(prov)}" )
 

tests/conftest.py

@@ -33,8 +33,8 @@
 #   at the end of tests.  In general, we want this to be True, so we can make sure
 #   that our tests are properly cleaning up after themselves.  However, the errors
 #   from this can hide other errors and failures, so when debugging, set it to False.
-# verify_archive_database_empty = True
-verify_archive_database_empty = False
+verify_archive_database_empty = True
+# verify_archive_database_empty = False
 
 
 pytest_plugins = [

tests/webap/test_webap.py

@@ -1,20 +1,157 @@
+import re
+import time
 import pytest
 
+import sqlalchemy as sa
+
 import selenium
 import selenium.webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions
 from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support.ui import Select
 from selenium.webdriver.remote.webelement import WebElement
 
+from models.base import SmartSession
+from models.provenance import CodeVersion, Provenance, ProvenanceTag
+
+from util.logger import SCLogger
+
 def test_webap( browser, webap_url, decam_datastore ):
-    import pdb; pdb.set_trace()
+    ds = decam_datastore
+    junkprov = None
+
+    try:
+        # Create a new provenance tag, tagging the provenances that are in decam_datastore
+        ProvenanceTag.newtag( 'test_webap',
+                              [ ds.exposure.provenance,
+                                ds.image.provenance,
+                                ds.sources.provenance,
+                                ds.reference.provenance,
+                                ds.sub_image.provenance,
+                                ds.detections.provenance,
+                                ds.cutouts.provenance,
+                                ds.measurements[0].provenance ] )
+
+        # Create a throwaway provenance and provenance tag so we can test
+        #  things *not* being found
+        with SmartSession() as session:
+            cv = session.query( CodeVersion ).first()
+            junkprov = Provenance( process='no_process', code_version=cv, is_testing=True )
+            session.add( junkprov )
+            session.commit()
+            ProvenanceTag.newtag( 'no_such_tag', [ junkprov ] )
+
+        browser.get( webap_url )
+        WebDriverWait( browser, timeout=10 ).until(
+            lambda d: d.find_element(By.ID, 'seechange_context_render_page_complete' ) )
+
+        # The "test_webap" option in the provtag_wid select widget won't necessarily
+        # be there immediately, because it's filled in with a callback from a web request
+        tries = 5
+        while ( tries > 0 ):
+            provtag_wid = browser.find_element( By.ID, "provtag_wid" )
+            options = provtag_wid.find_elements( By.TAG_NAME, "option" )
+            if any( [ o.text == 'test_webap' for o in options ] ):
+                break
+            tries -= 1
+            if tries < 0:
+                assert False, "Failed to find the test_webap option in the provenances select widget"
+            else:
+                SCLogger.debug( "Didn't find test_webap in the provtag_wid select, sleeping 1s and retrying" )
+                time.sleep( 1 )
+
+        buttons = browser.find_elements( By.XPATH, "//input[@type='button']" )
+        buttons = { b.get_attribute("value") : b for b in buttons }
+
+        # Make sure we get no exposures if we ask for the junk tag
+
+        select = Select( provtag_wid )
+        select.select_by_visible_text( 'no_such_tag' )
+        buttons['Show Exposures'].click()
+
+        WebDriverWait( browser, timeout=10 ).until(
+            lambda d: d.find_element( By.XPATH, "//h2[contains(.,'Exposures from')]" ) )
+
+        # Make sure that the "Exposure List" div is what's shown
+        # WARNING -- this absolute xpath might change if the page layout is changed!
+        tabcontentdiv = browser.find_element( By.XPATH, "html/body/div/div/div/div/div/div/div[2]" )
+        assert tabcontentdiv.text[:15] == 'Exposures from '
+        explist = browser.find_element( By.ID, "exposure_list_table" )
+        rows = explist.find_elements( By.TAG_NAME, "tr" )
+        assert len(rows) == 1   # Just the header row
+
+        # Now ask for the test_webap tag, see if we get the one exposure we expect
+
+        select.select_by_visible_text( 'test_webap' )
+        buttons['Show Exposures'].click()
+        # Give it half a second to go at least get to the "loading" screen; that's
+        #  all javascript with no server communcation, so should be fast.
+        time.sleep( 0.5 )
+        WebDriverWait( browser, timeout=10 ).until(
+            lambda d: d.find_element( By.XPATH, "//h2[contains(.,'Exposures from')]" ) )
+
+        tabcontentdiv = browser.find_element( By.XPATH, "html/body/div/div/div/div/div/div/div[2]" )
+        assert tabcontentdiv.text[:15] == 'Exposures from '
+        explist = browser.find_element( By.ID, "exposure_list_table" )
+        rows = explist.find_elements( By.TAG_NAME, "tr" )
+        assert len(rows) == 2   # Just the header row
+
+        cols = rows[1].find_elements( By.XPATH, "./*" )
+        assert cols[0].text == 'c4d_230702_080904_ori.fits.fz'
+        assert cols[2].text == 'ELAIS-E1'
+        assert cols[5].text == '1'    # n_images
+        assert cols[6].text == '172'  # detections
+        assert cols[7].text == '6'    # sources
+
+        # Try to click on the exposure name, make sure we get the exposure details
+        expnamelink = cols[0].find_element( By.TAG_NAME, 'a' )
+        expnamelink.click()
+        WebDriverWait( browser, timeout=10 ).until(
+            lambda d: d.find_element( By.XPATH, "//h2[contains(.,'Exposure c4d_230702_080904_ori.fits.fz')]" ) )
+
+        # OMG I nest a lot of divs
+        tabcontentdiv = browser.find_element( By.XPATH, "html/body/div/div/div/div/div/div/div[2]" )
+        imagesdiv = tabcontentdiv.find_element( By.XPATH, "./div/div/div/div[2]/div" )
+        assert re.search( r"^Exposure has 1 images and 1 completed subtractions.*"
+                          r"6 out of 172 detections pass preliminary cuts",
+                          imagesdiv.text, re.DOTALL ) is not None
+
+
+        imagestab = imagesdiv.find_element( By.TAG_NAME, 'table' )
+        rows = imagestab.find_elements( By.TAG_NAME, 'tr' )
+        assert len(rows) == 2
+        cols = rows[1].find_elements( By.XPATH, "./*" )
+        assert re.search( r'^c4d_20230702_080904_S3_r_Sci', cols[1].text ) is not None
+
+        # Find the sources tab and click on that
+        tabbuttonsdiv = tabcontentdiv.find_element( By.XPATH, "./div/div/div/div[1]" )
+        sourcestab = tabbuttonsdiv.find_element( By.XPATH, "//.[.='Sources']" )
+        sourcestab.click()
+        # Give it half a second to go at least get to the "loading" screen; that's
+        #  all javascript with no server communcation, so should be fast.
+        time.sleep( 0.5 )
+        WebDriverWait( browser, timeout=10 ).until(
+            lambda d: d.find_element( By.XPATH, "//p[contains(.,'Sources for all successfully completed chips')]" ) )
+
+        # Now imagesdiv should have information about the sources
+        tabcontentdiv = browser.find_element( By.XPATH, "html/body/div/div/div/div/div/div/div[2]" )
+        imagesdiv = tabcontentdiv.find_element( By.XPATH, "./div/div/div/div[2]/div" )
+
+        sourcestab = imagesdiv.find_element( By.TAG_NAME, 'table' )
+        rows = sourcestab.find_elements( By.TAG_NAME, 'tr' )
+        assert len(rows) == 7
+        # check stuff about the rows?
+
+        # There is probably more we should be testing here.  Definitely.
+
+    finally:
+        # Clean up the junk Provenance, and the ProvenanceTags we created
+        with SmartSession() as session:
+            session.execute( sa.text( "DELETE FROM provenance_tags "
+                                      "WHERE tag IN ('test_webap', 'no_such_tag')" ) )
+            if junkprov is not None:
+                session.delete( junkprov )
+            session.commit()
 
-    browser.get( webap_url )
-    WebDriverWait( browser, timeout=10 ).until(
-        lambda d: d.find_element(By.ID, 'seechange_context_render_page_complete' ) )
-
-
-    pass
 
-

util/radec.py

@@ -5,8 +5,8 @@
 from astropy.coordinates import SkyCoord, BarycentricTrueEcliptic
 import astropy.units as u
 
-_radecparse = re.compile( '^ *(?P<sign>[\-\+])? *(?P<d>[0-9]{1,2}): *(?P<m>[0-9]{1,2}):'
-                          ' *(?P<s>[0-9]{1,2}(\.[0-9]*)?) *$' )
+_radecparse = re.compile( r'^ *(?P<sign>[\-\+])? *(?P<d>[0-9]{1,2}): *(?P<m>[0-9]{1,2}):'
+                          r' *(?P<s>[0-9]{1,2}(\.[0-9]*)?) *$' )
 
 
 def parse_sexigesimal_degrees( strval, hours=False, positive=None ):

util/util.py

@@ -453,7 +453,7 @@ def as_datetime( string ):
     ---------
       string : str or datetime.datetime
          The string to convert.  If a datetime.datetime, the return
-         value is just this.  If none or an empty string ("^\s*$"), will
+         value is just this.  If none or an empty string ("^\\s*$"), will
          return None.  Otherwise, must be a string that
          dateutil.parser.parse can handle.