From 09c34b02019decf7464af583a42179f9f5369a0f Mon Sep 17 00:00:00 2001
From: Rasmus Handberg <rasmush@phys.au.dk>
Date: Tue, 8 Feb 2022 11:37:30 +0100
Subject: [PATCH 1/5] Disable tests of TNS query on GitHub Actions
Until we set up a way to avoid "too many requests" errors during testing
---
tests/test_tns.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/tests/test_tns.py b/tests/test_tns.py
index e90ab33..c16d2fd 100644
--- a/tests/test_tns.py
+++ b/tests/test_tns.py
@@ -7,12 +7,14 @@
"""
import pytest
+import os
import datetime
from astropy.coordinates import SkyCoord
from conftest import capture_cli
from flows import tns
#--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
def test_tns_search(SETUP_CONFIG):
coo_centre = SkyCoord(
@@ -28,6 +30,7 @@ def test_tns_search(SETUP_CONFIG):
assert res[0]['prefix'] == 'SN'
#--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
def test_tns_get_obj(SETUP_CONFIG):
res = tns.tns_get_obj('2019yvr')
@@ -37,6 +40,7 @@ def test_tns_get_obj(SETUP_CONFIG):
assert res['name_prefix'] == 'SN'
#--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
@pytest.mark.parametrize('date_begin,date_end', [
('2019-01-01', '2019-02-01'),
(datetime.date(2019, 1, 1), datetime.date(2019, 2, 1)),
@@ -69,6 +73,7 @@ def test_tns_getnames_wronginput(SETUP_CONFIG):
)
#--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
def test_run_querytns(SETUP_CONFIG):
# Run the command line interface:
From 3d6e499fb9a675b3fab17795e734a4ce8038afa9 Mon Sep 17 00:00:00 2001
From: Rasmus Handberg <rasmush@phys.au.dk>
Date: Thu, 10 Feb 2022 10:50:27 +0100
Subject: [PATCH 2/5] Bugfix ALFOSC B-filter
---
flows/load_image.py | 1 +
tests/test_load_image.py | 5 +++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/flows/load_image.py b/flows/load_image.py
index 35073eb..3693582 100644
--- a/flows/load_image.py
+++ b/flows/load_image.py
@@ -183,6 +183,7 @@ def load_image(FILENAME, target_coord=None):
filters_used.append(hdr.get(check_headers).strip())
if len(filters_used) == 1:
image.photfilter = {
+ 'B_Bes 440_100': 'B',
'V_Bes 530_80': 'V',
'R_Bes 650_130': 'R',
"g'_SDSS 480_145": 'gp',
diff --git a/tests/test_load_image.py b/tests/test_load_image.py
index bd64679..3dd3927 100644
--- a/tests/test_load_image.py
+++ b/tests/test_load_image.py
@@ -26,6 +26,7 @@
#['SN2021rhu_59465.86130221_B.fits.gz', 22],
#['20200613_SN2020lao_u_stacked_meandiff.fits.gz', 1],
#['2021aess_20220104_K.fits.gz', 5],
+ #['2021aess_B01_20220207v1.fits.gz', 5],
])
def test_load_image(fpath, siteid):
# Get list of all available filters:
@@ -54,11 +55,11 @@ def test_load_image(fpath, siteid):
assert isinstance(img.obstime, Time)
assert isinstance(img.exptime, float)
assert img.exptime > 0
- assert isinstance(img.photfilter, str)
- assert img.photfilter in all_filters
assert isinstance(img.wcs, WCS)
assert isinstance(img.site, dict)
assert img.site['siteid'] == siteid
+ assert isinstance(img.photfilter, str)
+ assert img.photfilter in all_filters
#--------------------------------------------------------------------------------------------------
if __name__ == '__main__':
From c3f14eaa18a1b1ba66a61a827930c48210831799 Mon Sep 17 00:00:00 2001
From: Rasmus Handberg <rasmush@phys.au.dk>
Date: Thu, 10 Feb 2022 11:19:18 +0100
Subject: [PATCH 3/5] Better logging of ingest errors
---
run_ingest.py | 32 +++++++++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/run_ingest.py b/run_ingest.py
index f510b0d..406f9ad 100644
--- a/run_ingest.py
+++ b/run_ingest.py
@@ -119,6 +119,10 @@ def ingest_from_inbox():
db.cursor.execute("SELECT archive,path FROM aadc.files_archives;")
archives_list = db.cursor.fetchall()
+ # Get list of all available filters:
+ db.cursor.execute("SELECT photfilter FROM flows.photfilters;")
+ all_filters = set([row['photfilter'] for row in db.cursor.fetchall()])
+
for inputtype in ('science', 'templates', 'subtracted', 'replace'): #
for fpath in glob.iglob(os.path.join(rootdir_inbox, '*', inputtype, '*')):
logger.info("="*72)
@@ -131,6 +135,7 @@ def ingest_from_inbox():
uploadlogid = row['logid']
else:
uploadlogid = None
+ logger.info("Uploadlog ID: %s", uploadlogid)
# Only accept FITS file, or already compressed FITS files:
if not fpath.endswith('.fits') and not fpath.endswith('.fits.gz'):
@@ -272,8 +277,12 @@ def ingest_from_inbox():
# Try to load the image using the same function as the pipeline would:
try:
img = load_image(fpath, target_coord=target_coord)
- except: # noqa: E722, pragma: no cover
+ except Exception as e: # pragma: no cover
logger.exception("Could not load FITS image")
+ if uploadlogid:
+ errmsg = str(e) if hasattr(e, 'message') else str(e.message)
+ db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", ['Load Image Error: ' + errmsg, uploadlogid])
+ db.conn.commit()
continue
# Use the WCS in the file to calculate the pixel-positon of the target:
@@ -281,6 +290,10 @@ def ingest_from_inbox():
target_pixels = img.wcs.all_world2pix(target_radec, 0).flatten()
except: # noqa: E722, pragma: no cover
logger.exception("Could not find target position using the WCS.")
+ if uploadlogid:
+ errmsg = "Could not find target position using the WCS."
+ db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+ db.conn.commit()
continue
# Check that the position of the target actually falls within
@@ -288,11 +301,28 @@ def ingest_from_inbox():
if target_pixels[0] < -0.5 or target_pixels[1] < -0.5 \
or target_pixels[0] > img.shape[1]-0.5 or target_pixels[1] > img.shape[0]-0.5:
logger.error("Target position does not fall within image. Check the WCS.")
+ if uploadlogid:
+ errmsg = "Target position does not fall within image. Check the WCS."
+ db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+ db.conn.commit()
continue
# Check that the site was found:
if img.site is None or img.site['siteid'] is None:
logger.error("Unknown SITE")
+ if uploadlogid:
+ errmsg = "Unknown site"
+ db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+ db.conn.commit()
+ continue
+
+ # Check that the extracted photometric filter is valid:
+ if img.photfilter not in all_filters:
+ logger.error("Unknown PHOTFILTER: %s", img.photfilter)
+ if uploadlogid:
+ errmsg = "Unknown PHOTFILTER: '" + str(img.photfilter) + "'"
+ db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+ db.conn.commit()
continue
# Do a deep check to ensure that there is not already another file with the same
From 3757322b9a40d597fc603d3f1c3ddfd817692029 Mon Sep 17 00:00:00 2001
From: Rasmus Handberg <rasmush@phys.au.dk>
Date: Thu, 10 Feb 2022 11:19:57 +0100
Subject: [PATCH 4/5] Bumped version
---
VERSION | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/VERSION b/VERSION
index 870328a..76279d7 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-master-v0.9.1
\ No newline at end of file
+master-v0.9.2
\ No newline at end of file
From e3e464fd8a16d652444abbfe865a405336b291a8 Mon Sep 17 00:00:00 2001
From: Rasmus Handberg <rasmush@phys.au.dk>
Date: Thu, 10 Feb 2022 11:21:45 +0100
Subject: [PATCH 5/5] Update notes
---
notes/update_all_catalogs.py | 24 ++++++++++++++----------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/notes/update_all_catalogs.py b/notes/update_all_catalogs.py
index e5b1618..7356441 100644
--- a/notes/update_all_catalogs.py
+++ b/notes/update_all_catalogs.py
@@ -4,7 +4,7 @@
import logging
import sys
import os.path
-from tqdm import tqdm
+import tqdm
from astropy.coordinates import SkyCoord
if os.path.abspath('..') not in sys.path:
sys.path.insert(0, os.path.abspath('..'))
@@ -37,12 +37,16 @@ def emit(self, record):
logger.addHandler(console)
logger.setLevel(logging.INFO)
- for target in tqdm(flows.api.get_targets()):
- #if target['targetid'] != 1942:
- # continue
-
- donefile = f"{target['targetid']:05d}.done"
- if not os.path.exists(donefile):
- flows.catalogs.download_catalog(target['targetid'], update_existing=True)
-
- open(donefile, 'w').close()
+ # Do it by status, just to prioritize things a bit:
+ for tgtstatus in ('target', 'candidate', 'rejected'):
+ targetids = sorted([tgt['targetid'] for tgt in flows.api.get_targets() if tgt['target_status'] == tgtstatus])[::-1]
+
+ for targetid in tqdm.tqdm(targetids, desc=tgtstatus):
+ donefile = f"catalog_updates/{targetid:05d}.done"
+ if not os.path.exists(donefile):
+ try:
+ flows.catalogs.download_catalog(targetid, update_existing=True)
+ except:
+ logger.exception("targetid=%d", targetid)
+ else:
+ open(donefile, 'w').close()