From 38e0c4fe35f9cc30cfb35d46f097d4c1e25acf05 Mon Sep 17 00:00:00 2001 From: Eric Bellm Date: Tue, 26 Sep 2023 14:11:24 -0700 Subject: [PATCH 1/2] update to v4.01 --- schema/alert.avsc | 15 ++-- schema/candidate.avsc | 164 +++++++++++++++++++------------------- schema/cutout.avsc | 2 +- schema/fp_hist.avsc | 38 +++++++++ schema/prv_candidate.avsc | 106 ++++++++++++------------ 5 files changed, 183 insertions(+), 142 deletions(-) create mode 100644 schema/fp_hist.avsc diff --git a/schema/alert.avsc b/schema/alert.avsc index dcb7958..95f4bc7 100644 --- a/schema/alert.avsc +++ b/schema/alert.avsc @@ -3,18 +3,21 @@ "type": "record", "name": "alert", "doc": "avro alert schema for ZTF (www.ztf.caltech.edu)", - "version": "3.3", + "version": "4.01", "fields": [ {"name": "schemavsn", "type": "string", "doc": "schema version used"}, {"name": "publisher", "type": "string", "doc": "origin of alert packet"}, {"name": "objectId", "type": "string", "doc": "object identifier or name"}, {"name": "candid", "type": "long"}, {"name": "candidate", "type": "ztf.alert.candidate"}, - {"name": "prv_candidates", "type": [{ + {"name": "prv_candidates", "type": ["null", { "type": "array", - "items": "ztf.alert.prv_candidate"}, "null" ], "default": null}, - {"name": "cutoutScience", "type": ["ztf.alert.cutout", "null"], "default": null}, - {"name": "cutoutTemplate", "type": ["ztf.alert.cutout", "null"], "default": null}, - {"name": "cutoutDifference", "type": ["ztf.alert.cutout", "null"], "default": null} + "items": "ztf.alert.prv_candidate"}], "default": null}, + {"name": "fp_hists", "type": ["null", { + "type": "array", + "items": "ztf.alert.fp_hist"}], "default": null}, + {"name": "cutoutScience", "type": ["null", "ztf.alert.cutout"], "default": null}, + {"name": "cutoutTemplate", "type": ["null", "ztf.alert.cutout"], "default": null}, + {"name": "cutoutDifference", "type": ["null", "ztf.alert.cutout"], "default": null} ] } diff --git a/schema/candidate.avsc b/schema/candidate.avsc index 40c11cf..1773767 100644 --- a/schema/candidate.avsc +++ b/schema/candidate.avsc @@ -2,111 +2,111 @@ "namespace": "ztf.alert", "name": "candidate", "doc": "avro alert schema", - "version": "3.3", + "version": "4.01", "type": "record", "fields": [ {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, {"name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)"}, {"name": "pid", "type": "long", "doc": "Processing ID for science image to facilitate archive retrieval"}, - {"name": "diffmaglim", "type": ["float", "null"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, - {"name": "pdiffimfilename", "type": ["string", "null"], "default": null, "doc": "filename of positive (sci minus ref) difference image"}, - {"name": "programpi", "type": ["string", "null"], "default": null, "doc": "Principal investigator attached to program ID"}, + {"name": "diffmaglim", "type": ["null", "float"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, + {"name": "pdiffimfilename", "type": ["null", "string"], "default": null, "doc": "filename of positive (sci minus ref) difference image"}, + {"name": "programpi", "type": ["null", "string"], "default": null, "doc": "Principal investigator attached to program ID"}, {"name": "programid", "type": "int", "doc": "Program ID: encodes either public, collab, or caltech mode"}, {"name": "candid", "type": "long", "doc": "Candidate ID from operations DB"}, {"name": "isdiffpos", "type": "string", "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, - {"name": "tblid", "type": ["long", "null"], "default": null, "doc": "Internal pipeline table extraction ID"}, - {"name": "nid", "type": ["int", "null"], "default": null, "doc": "Night ID"}, - {"name": "rcid", "type": ["int", "null"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, - {"name": "field", "type": ["int", "null"], "default": null, "doc": "ZTF field ID"}, - {"name": "xpos", "type": ["float", "null"], "default": null, "doc": "x-image position of candidate [pixels]"}, - {"name": "ypos", "type": ["float", "null"], "default": null, "doc": "y-image position of candidate [pixels]"}, + {"name": "tblid", "type": ["null", "long"], "default": null, "doc": "Internal pipeline table extraction ID"}, + {"name": "nid", "type": ["null", "int"], "default": null, "doc": "Night ID"}, + {"name": "rcid", "type": ["null", "int"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, + {"name": "field", "type": ["null", "int"], "default": null, "doc": "ZTF field ID"}, + {"name": "xpos", "type": ["null", "float"], "default": null, "doc": "x-image position of candidate [pixels]"}, + {"name": "ypos", "type": ["null", "float"], "default": null, "doc": "y-image position of candidate [pixels]"}, {"name": "ra", "type": "double", "doc": "Right Ascension of candidate; J2000 [deg]"}, {"name": "dec", "type": "double", "doc": "Declination of candidate; J2000 [deg]"}, {"name": "magpsf", "type": "float", "doc": "Magnitude from PSF-fit photometry [mag]"}, {"name": "sigmapsf", "type": "float", "doc": "1-sigma uncertainty in magpsf [mag]"}, - {"name": "chipsf", "type": ["float", "null"], "default": null, "doc": "Reduced chi-square for PSF-fit"}, - {"name": "magap", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, - {"name": "sigmagap", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, - {"name": "distnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, - {"name": "magnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, - {"name": "sigmagnr", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, - {"name": "chinr", "type": ["float", "null"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, - {"name": "sharpnr", "type": ["float", "null"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, - {"name": "sky", "type": ["float", "null"], "default": null, "doc": "Local sky background estimate [DN]"}, - {"name": "magdiff", "type": ["float", "null"], "default": null, "doc": "Difference: magap - magpsf [mag]"}, - {"name": "fwhm", "type": ["float", "null"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, - {"name": "classtar", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, - {"name": "mindtoedge", "type": ["float", "null"], "default": null, "doc": "Distance to nearest edge in image [pixels]"}, - {"name": "magfromlim", "type": ["float", "null"], "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, - {"name": "seeratio", "type": ["float", "null"], "default": null, "doc": "Ratio: difffwhm / fwhm"}, - {"name": "aimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, - {"name": "bimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, - {"name": "aimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / fwhm"}, - {"name": "bimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: bimage / fwhm"}, - {"name": "elong", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / bimage"}, - {"name": "nneg", "type": ["int", "null"], "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, - {"name": "nbad", "type": ["int", "null"], "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, - {"name": "rb", "type": ["float", "null"], "default": null, "doc": "RealBogus quality score from Random Forest classifier; range is 0 to 1 where closer to 1 is more reliable"}, - {"name": "ssdistnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, - {"name": "ssmagnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, - {"name": "ssnamenr", "type": ["string", "null"], "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, - {"name": "sumrat", "type": ["float", "null"], "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, - {"name": "magapbig", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, - {"name": "sigmagapbig", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, + {"name": "chipsf", "type": ["null", "float"], "default": null, "doc": "Reduced chi-square for PSF-fit"}, + {"name": "magap", "type": ["null", "float"], "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, + {"name": "sigmagap", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, + {"name": "distnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, + {"name": "magnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, + {"name": "sigmagnr", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, + {"name": "chinr", "type": ["null", "float"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, + {"name": "sharpnr", "type": ["null", "float"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, + {"name": "sky", "type": ["null", "float"], "default": null, "doc": "Local sky background estimate [DN]"}, + {"name": "magdiff", "type": ["null", "float"], "default": null, "doc": "Difference: magap - magpsf [mag]"}, + {"name": "fwhm", "type": ["null", "float"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, + {"name": "classtar", "type": ["null", "float"], "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, + {"name": "mindtoedge", "type": ["null", "float"], "default": null, "doc": "Distance to nearest edge in image [pixels]"}, + {"name": "magfromlim", "type": ["null", "float"], "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, + {"name": "seeratio", "type": ["null", "float"], "default": null, "doc": "Ratio: difffwhm / fwhm"}, + {"name": "aimage", "type": ["null", "float"], "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, + {"name": "bimage", "type": ["null", "float"], "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, + {"name": "aimagerat", "type": ["null", "float"], "default": null, "doc": "Ratio: aimage / fwhm"}, + {"name": "bimagerat", "type": ["null", "float"], "default": null, "doc": "Ratio: bimage / fwhm"}, + {"name": "elong", "type": ["null", "float"], "default": null, "doc": "Ratio: aimage / bimage"}, + {"name": "nneg", "type": ["null", "int"], "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, + {"name": "nbad", "type": ["null", "int"], "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, + {"name": "rb", "type": ["null", "float"], "default": null, "doc": "RealBogus quality score from Random Forest classifier; range is 0 to 1 where closer to 1 is more reliable"}, + {"name": "ssdistnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, + {"name": "ssmagnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, + {"name": "ssnamenr", "type": ["null", "string"], "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, + {"name": "sumrat", "type": ["null", "float"], "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, + {"name": "magapbig", "type": ["null", "float"], "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, + {"name": "sigmagapbig", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, {"name": "ranr", "type": "double", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, {"name": "decnr", "type": "double", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "sgmag1", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag1", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag1", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag1", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "sgscore1", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr1", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, + {"name": "sgmag1", "type": ["null", "float"], "default": null, "doc": "g-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "srmag1", "type": ["null", "float"], "default": null, "doc": "r-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "simag1", "type": ["null", "float"], "default": null, "doc": "i-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "szmag1", "type": ["null", "float"], "default": null, "doc": "z-band PSF-fit magnitude of closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "sgscore1", "type": ["null", "float"], "default": null, "doc": "Star/Galaxy score of closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, + {"name": "distpsnr1", "type": ["null", "float"], "default": null, "doc": "Distance to closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, {"name": "ndethist", "type": "int", "doc": "Number of spatially-coincident detections falling within 1.5 arcsec going back to beginning of survey; only detections that fell on the same field and readout-channel ID where the input candidate was observed are counted. All raw detections down to a photometric S/N of ~ 3 are included."}, {"name": "ncovhist", "type": "int", "doc": "Number of times input candidate position fell on any field and readout-channel going back to beginning of survey"}, - {"name": "jdstarthist", "type": ["double", "null"], "default": null, "doc": "Earliest Julian date of epoch corresponding to ndethist [days]"}, - {"name": "jdendhist", "type": ["double", "null"], "default": null, "doc": "Latest Julian date of epoch corresponding to ndethist [days]"}, - {"name": "scorr", "type": ["double", "null"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, - {"name": "tooflag", "type": ["int", "null"], "default": 0, "doc": "1 => candidate is from a Target-of-Opportunity (ToO) exposure; 0 => candidate is from a non-ToO exposure"}, - {"name": "objectidps1", "type": ["long", "null"], "default": null, "doc": "Object ID of closest source from PS1 catalog; if exists within 30 arcsec"}, - {"name": "objectidps2", "type": ["long", "null"], "default": null, "doc": "Object ID of second closest source from PS1 catalog; if exists within 30 arcsec"}, - {"name": "sgmag2", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag2", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag2", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag2", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "sgscore2", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of second closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr2", "type": ["float", "null"], "default": null, "doc": "Distance to second closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, - {"name": "objectidps3", "type": ["long", "null"], "default": null, "doc": "Object ID of third closest source from PS1 catalog; if exists within 30 arcsec"}, - {"name": "sgmag3", "type": ["float", "null"], "default": null, "doc": "g-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "srmag3", "type": ["float", "null"], "default": null, "doc": "r-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "simag3", "type": ["float", "null"], "default": null, "doc": "i-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "szmag3", "type": ["float", "null"], "default": null, "doc": "z-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, - {"name": "sgscore3", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy score of third closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, - {"name": "distpsnr3", "type": ["float", "null"], "default": null, "doc": "Distance to third closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, + {"name": "jdstarthist", "type": ["null", "double"], "default": null, "doc": "Earliest Julian date of epoch corresponding to ndethist [days]"}, + {"name": "jdendhist", "type": ["null", "double"], "default": null, "doc": "Latest Julian date of epoch corresponding to ndethist [days]"}, + {"name": "scorr", "type": ["null", "double"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, + {"name": "tooflag", "type": ["null", "int"], "default": 0, "doc": "1 => candidate is from a Target-of-Opportunity (ToO) exposure; 0 => candidate is from a non-ToO exposure"}, + {"name": "objectidps1", "type": ["null", "long"], "default": null, "doc": "Object ID of closest source from PS1 catalog; if exists within 30 arcsec"}, + {"name": "objectidps2", "type": ["null", "long"], "default": null, "doc": "Object ID of second closest source from PS1 catalog; if exists within 30 arcsec"}, + {"name": "sgmag2", "type": ["null", "float"], "default": null, "doc": "g-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "srmag2", "type": ["null", "float"], "default": null, "doc": "r-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "simag2", "type": ["null", "float"], "default": null, "doc": "i-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "szmag2", "type": ["null", "float"], "default": null, "doc": "z-band PSF-fit magnitude of second closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "sgscore2", "type": ["null", "float"], "default": null, "doc": "Star/Galaxy score of second closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, + {"name": "distpsnr2", "type": ["null", "float"], "default": null, "doc": "Distance to second closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, + {"name": "objectidps3", "type": ["null", "long"], "default": null, "doc": "Object ID of third closest source from PS1 catalog; if exists within 30 arcsec"}, + {"name": "sgmag3", "type": ["null", "float"], "default": null, "doc": "g-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "srmag3", "type": ["null", "float"], "default": null, "doc": "r-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "simag3", "type": ["null", "float"], "default": null, "doc": "i-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "szmag3", "type": ["null", "float"], "default": null, "doc": "z-band PSF-fit magnitude of third closest source from PS1 catalog; if exists within 30 arcsec [mag]"}, + {"name": "sgscore3", "type": ["null", "float"], "default": null, "doc": "Star/Galaxy score of third closest source from PS1 catalog; if exists within 30 arcsec: 0 <= sgscore <= 1 where closer to 1 implies higher likelihood of being a star"}, + {"name": "distpsnr3", "type": ["null", "float"], "default": null, "doc": "Distance to third closest source from PS1 catalog; if exists within 30 arcsec [arcsec]"}, {"name": "nmtchps", "type": "int", "doc": "Number of source matches from PS1 catalog falling within 30 arcsec"}, {"name": "rfid", "type": "long", "doc": "Processing ID for reference image to facilitate archive retrieval"}, {"name": "jdstartref", "type": "double", "doc": "Observation Julian date of earliest exposure used to generate reference image [days]"}, {"name": "jdendref", "type": "double", "doc": "Observation Julian date of latest exposure used to generate reference image [days]"}, {"name": "nframesref", "type": "int", "doc": "Number of frames (epochal images) used to generate reference image"}, {"name": "rbversion", "type": "string", "doc": "version of Random Forest classifier model used to assign RealBogus (rb) quality score"}, - {"name": "dsnrms", "type": ["float", "null"], "default": null, "doc": "Ratio: D/stddev(D) on event position where D = difference image"}, - {"name": "ssnrms", "type": ["float", "null"], "default": null, "doc": "Ratio: S/stddev(S) on event position where S = image of convolution: D (x) PSF(D)"}, - {"name": "dsdiff", "type": ["float", "null"], "default": null, "doc": "Difference of statistics: dsnrms - ssnrms"}, - {"name": "magzpsci", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, - {"name": "magzpsciunc", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, - {"name": "magzpscirms", "type": ["float", "null"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, + {"name": "dsnrms", "type": ["null", "float"], "default": null, "doc": "Ratio: D/stddev(D) on event position where D = difference image"}, + {"name": "ssnrms", "type": ["null", "float"], "default": null, "doc": "Ratio: S/stddev(S) on event position where S = image of convolution: D (x) PSF(D)"}, + {"name": "dsdiff", "type": ["null", "float"], "default": null, "doc": "Difference of statistics: dsnrms - ssnrms"}, + {"name": "magzpsci", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, + {"name": "magzpsciunc", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, + {"name": "magzpscirms", "type": ["null", "float"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, {"name": "nmatches", "type": "int", "doc": "Number of PS1 photometric calibrators used to calibrate science image from science image processing"}, - {"name": "clrcoeff", "type": ["float", "null"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, - {"name": "clrcounc", "type": ["float", "null"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, - {"name": "zpclrcov", "type": ["float", "null"], "default": null, "doc": "Covariance in magzpsci and clrcoeff from science image processing [mag^2]"}, - {"name": "zpmed", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point from median of all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, - {"name": "clrmed", "type": ["float", "null"], "default": null, "doc": "Median color of all PS1 photometric calibrators used from science image processing [mag]: for filter (fid) = 1, 2, 3, PS1 color used = g-r, g-r, r-i respectively"}, - {"name": "clrrms", "type": ["float", "null"], "default": null, "doc": "RMS color (deviation from average) of all PS1 photometric calibrators used from science image processing [mag]"}, - {"name": "neargaia", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [arcsec]"}, - {"name": "neargaiabright", "type": ["float", "null"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [arcsec]"}, - {"name": "maggaia", "type": ["float", "null"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [mag]"}, - {"name": "maggaiabright", "type": ["float", "null"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [mag]"}, - {"name": "exptime", "type": ["float", "null"], "default": null, "doc": "Integration time of camera exposure [sec]"}, - {"name": "drb", "type": ["float", "null"], "default": null, "doc": "RealBogus quality score from Deep-Learning-based classifier; range is 0 to 1 where closer to 1 is more reliable"}, + {"name": "clrcoeff", "type": ["null", "float"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, + {"name": "clrcounc", "type": ["null", "float"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, + {"name": "zpclrcov", "type": ["null", "float"], "default": null, "doc": "Covariance in magzpsci and clrcoeff from science image processing [mag^2]"}, + {"name": "zpmed", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point from median of all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, + {"name": "clrmed", "type": ["null", "float"], "default": null, "doc": "Median color of all PS1 photometric calibrators used from science image processing [mag]: for filter (fid) = 1, 2, 3, PS1 color used = g-r, g-r, r-i respectively"}, + {"name": "clrrms", "type": ["null", "float"], "default": null, "doc": "RMS color (deviation from average) of all PS1 photometric calibrators used from science image processing [mag]"}, + {"name": "neargaia", "type": ["null", "float"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [arcsec]"}, + {"name": "neargaiabright", "type": ["null", "float"], "default": null, "doc": "Distance to closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [arcsec]"}, + {"name": "maggaia", "type": ["null", "float"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog irrespective of magnitude; if exists within 90 arcsec [mag]"}, + {"name": "maggaiabright", "type": ["null", "float"], "default": null, "doc": "Gaia (G-band) magnitude of closest source from Gaia DR1 catalog brighter than magnitude 14; if exists within 90 arcsec [mag]"}, + {"name": "exptime", "type": ["null", "float"], "default": null, "doc": "Integration time of camera exposure [sec]"}, + {"name": "drb", "type": ["null", "float"], "default": null, "doc": "RealBogus quality score from Deep-Learning-based classifier; range is 0 to 1 where closer to 1 is more reliable"}, {"name": "drbversion", "type": "string", "doc": "version of Deep-Learning-based classifier model used to assign RealBogus (drb) quality score"} ] } diff --git a/schema/cutout.avsc b/schema/cutout.avsc index 43fe382..9843e23 100644 --- a/schema/cutout.avsc +++ b/schema/cutout.avsc @@ -3,7 +3,7 @@ "type": "record", "name": "cutout", "doc": "avro alert schema", - "version": "3.3", + "version": "4.01", "fields": [ {"name": "fileName", "type": "string"}, {"name": "stampData", "type": "bytes", "doc": "fits.gz"} diff --git a/schema/fp_hist.avsc b/schema/fp_hist.avsc new file mode 100644 index 0000000..a7486ce --- /dev/null +++ b/schema/fp_hist.avsc @@ -0,0 +1,38 @@ +{ + "namespace": "ztf.alert", + "name": "fp_hist", + "doc": "avro alert schema", + "version": "4.01", + "type": "record", + "fields": [ + {"name": "field", "type": ["null", "int"], "default": null, "doc": "ZTF field ID"}, + {"name": "rcid", "type": ["null", "int"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, + {"name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)"}, + {"name": "pid", "type": "long", "doc": "Processing ID for image"}, + {"name": "rfid", "type": "long", "doc": "Processing ID for reference image to facilitate archive retrieval"}, + {"name": "sciinpseeing", "type": ["null", "float"], "default": null, "doc": "Effective FWHM of sci image [pixels]"}, + {"name": "scibckgnd", "type": ["null", "float"], "default": null, "doc": "Background level in sci image [DN]"}, + {"name": "scisigpix", "type": ["null", "float"], "default": null, "doc": "Robust sigma per pixel in sci image [DN]"}, + {"name": "magzpsci", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, + {"name": "magzpsciunc", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, + {"name": "magzpscirms", "type": ["null", "float"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, + {"name": "clrcoeff", "type": ["null", "float"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, + {"name": "clrcounc", "type": ["null", "float"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, + {"name": "exptime", "type": ["null", "float"], "default": null, "doc": "Integration time of camera exposure [sec]"}, + {"name": "adpctdif1", "type": ["null", "float"], "default": null, "doc": "Full sci image astrometric RMS along R.A. with respect to Gaia1 [arcsec]"}, + {"name": "adpctdif2", "type": ["null", "float"], "default": null, "doc": "Full sci image astrometric RMS along Dec. with respect to Gaia1 [arcsec]"}, + {"name": "diffmaglim", "type": ["null", "float"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, + {"name": "programid", "type": "int", "doc": "Program ID: encodes either public, collab, or caltech mode"}, + {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, + {"name": "forcediffimflux", "type": ["null", "float"], "default": null, "doc": "Forced difference image PSF-fit flux [DN]"}, + {"name": "forcediffimfluxunc", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in forcediffimflux [DN]"}, + {"name": "procstatus", "type": ["null", "string"], "default": null, "doc": "Forced photometry processing status codes (0 => no warnings); see documentation"}, + {"name": "distnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [arcsec]"}, + {"name": "ranr", "type": "double", "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, + {"name": "decnr", "type": "double", "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, + {"name": "magnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, + {"name": "sigmagnr", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, + {"name": "chinr", "type": ["null", "float"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, + {"name": "sharpnr", "type": ["null", "float"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"} + ] +} diff --git a/schema/prv_candidate.avsc b/schema/prv_candidate.avsc index fc56ab7..ce30da7 100644 --- a/schema/prv_candidate.avsc +++ b/schema/prv_candidate.avsc @@ -2,65 +2,65 @@ "namespace": "ztf.alert", "name": "prv_candidate", "doc": "avro alert schema", - "version": "3.3", + "version": "4.01", "type": "record", "fields": [ {"name": "jd", "type": "double", "doc": "Observation Julian date at start of exposure [days]"}, {"name": "fid", "type": "int", "doc": "Filter ID (1=g; 2=R; 3=i)"}, {"name": "pid", "type": "long", "doc": "Processing ID for image"}, - {"name": "diffmaglim", "type": ["float", "null"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, - {"name": "pdiffimfilename", "type": ["string", "null"], "default": null, "doc": "filename of positive (sci minus ref) difference image"}, - {"name": "programpi", "type": ["string", "null"], "default": null, "doc": "Principal investigator attached to program ID"}, + {"name": "diffmaglim", "type": ["null", "float"], "default": null, "doc": "Expected 5-sigma mag limit in difference image based on global noise estimate [mag]"}, + {"name": "pdiffimfilename", "type": ["null", "string"], "default": null, "doc": "filename of positive (sci minus ref) difference image"}, + {"name": "programpi", "type": ["null", "string"], "default": null, "doc": "Principal investigator attached to program ID"}, {"name": "programid", "type": "int", "doc": "Program ID: encodes either public, collab, or caltech mode"}, - {"name": "candid", "type": ["long", "null"], "doc": "Candidate ID from operations DB"}, - {"name": "isdiffpos", "type": ["string", "null"], "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, - {"name": "tblid", "type": ["long", "null"], "default": null, "doc": "Internal pipeline table extraction ID"}, - {"name": "nid", "type": ["int", "null"], "default": null, "doc": "Night ID"}, - {"name": "rcid", "type": ["int", "null"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, - {"name": "field", "type": ["int", "null"], "default": null, "doc": "ZTF field ID"}, - {"name": "xpos", "type": ["float", "null"], "default": null, "doc": "x-image position of candidate [pixels]"}, - {"name": "ypos", "type": ["float", "null"], "default": null, "doc": "y-image position of candidate [pixels]"}, - {"name": "ra", "type": ["double", "null"], "doc": "Right Ascension of candidate; J2000 [deg]"}, - {"name": "dec", "type": ["double", "null"], "doc": "Declination of candidate; J2000 [deg]"}, - {"name": "magpsf", "type": ["float", "null"], "doc": "Magnitude from PSF-fit photometry [mag]"}, - {"name": "sigmapsf", "type": ["float", "null"], "doc": "1-sigma uncertainty in magpsf [mag]"}, - {"name": "chipsf", "type": ["float", "null"], "default": null, "doc": "Reduced chi-square for PSF-fit"}, - {"name": "magap", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, - {"name": "sigmagap", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, - {"name": "distnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, - {"name": "magnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, - {"name": "sigmagnr", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, - {"name": "chinr", "type": ["float", "null"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, - {"name": "sharpnr", "type": ["float", "null"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, - {"name": "sky", "type": ["float", "null"], "default": null, "doc": "Local sky background estimate [DN]"}, - {"name": "magdiff", "type": ["float", "null"], "default": null, "doc": "Difference: magap - magpsf [mag]"}, - {"name": "fwhm", "type": ["float", "null"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, - {"name": "classtar", "type": ["float", "null"], "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, - {"name": "mindtoedge", "type": ["float", "null"], "default": null, "doc": "Distance to nearest edge in image [pixels]"}, - {"name": "magfromlim", "type": ["float", "null"], "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, - {"name": "seeratio", "type": ["float", "null"], "default": null, "doc": "Ratio: difffwhm / fwhm"}, - {"name": "aimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, - {"name": "bimage", "type": ["float", "null"], "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, - {"name": "aimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / fwhm"}, - {"name": "bimagerat", "type": ["float", "null"], "default": null, "doc": "Ratio: bimage / fwhm"}, - {"name": "elong", "type": ["float", "null"], "default": null, "doc": "Ratio: aimage / bimage"}, - {"name": "nneg", "type": ["int", "null"], "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, - {"name": "nbad", "type": ["int", "null"], "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, - {"name": "rb", "type": ["float", "null"], "default": null, "doc": "RealBogus quality score; range is 0 to 1 where closer to 1 is more reliable"}, - {"name": "ssdistnr", "type": ["float", "null"], "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, - {"name": "ssmagnr", "type": ["float", "null"], "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, - {"name": "ssnamenr", "type": ["string", "null"], "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, - {"name": "sumrat", "type": ["float", "null"], "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, - {"name": "magapbig", "type": ["float", "null"], "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, - {"name": "sigmagapbig", "type": ["float", "null"], "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, - {"name": "ranr", "type": ["double", "null"], "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "decnr", "type": ["double", "null"], "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, - {"name": "scorr", "type": ["double", "null"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, - {"name": "magzpsci", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, - {"name": "magzpsciunc", "type": ["float", "null"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, - {"name": "magzpscirms", "type": ["float", "null"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, - {"name": "clrcoeff", "type": ["float", "null"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, - {"name": "clrcounc", "type": ["float", "null"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, + {"name": "candid", "type": ["null", "long"], "doc": "Candidate ID from operations DB"}, + {"name": "isdiffpos", "type": ["null", "string"], "doc": "t or 1 => candidate is from positive (sci minus ref) subtraction; f or 0 => candidate is from negative (ref minus sci) subtraction"}, + {"name": "tblid", "type": ["null", "long"], "default": null, "doc": "Internal pipeline table extraction ID"}, + {"name": "nid", "type": ["null", "int"], "default": null, "doc": "Night ID"}, + {"name": "rcid", "type": ["null", "int"], "default": null, "doc": "Readout channel ID [00 .. 63]"}, + {"name": "field", "type": ["null", "int"], "default": null, "doc": "ZTF field ID"}, + {"name": "xpos", "type": ["null", "float"], "default": null, "doc": "x-image position of candidate [pixels]"}, + {"name": "ypos", "type": ["null", "float"], "default": null, "doc": "y-image position of candidate [pixels]"}, + {"name": "ra", "type": ["null", "double"], "doc": "Right Ascension of candidate; J2000 [deg]"}, + {"name": "dec", "type": ["null", "double"], "doc": "Declination of candidate; J2000 [deg]"}, + {"name": "magpsf", "type": ["null", "float"], "doc": "Magnitude from PSF-fit photometry [mag]"}, + {"name": "sigmapsf", "type": ["null", "float"], "doc": "1-sigma uncertainty in magpsf [mag]"}, + {"name": "chipsf", "type": ["null", "float"], "default": null, "doc": "Reduced chi-square for PSF-fit"}, + {"name": "magap", "type": ["null", "float"], "default": null, "doc": "Aperture mag using 14 pixel diameter aperture [mag]"}, + {"name": "sigmagap", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magap [mag]"}, + {"name": "distnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest source in reference image PSF-catalog [pixels]"}, + {"name": "magnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest source in reference image PSF-catalog [mag]"}, + {"name": "sigmagnr", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magnr [mag]"}, + {"name": "chinr", "type": ["null", "float"], "default": null, "doc": "DAOPhot chi parameter of nearest source in reference image PSF-catalog"}, + {"name": "sharpnr", "type": ["null", "float"], "default": null, "doc": "DAOPhot sharp parameter of nearest source in reference image PSF-catalog"}, + {"name": "sky", "type": ["null", "float"], "default": null, "doc": "Local sky background estimate [DN]"}, + {"name": "magdiff", "type": ["null", "float"], "default": null, "doc": "Difference: magap - magpsf [mag]"}, + {"name": "fwhm", "type": ["null", "float"], "default": null, "doc": "Full Width Half Max assuming a Gaussian core, from SExtractor [pixels]"}, + {"name": "classtar", "type": ["null", "float"], "default": null, "doc": "Star/Galaxy classification score from SExtractor"}, + {"name": "mindtoedge", "type": ["null", "float"], "default": null, "doc": "Distance to nearest edge in image [pixels]"}, + {"name": "magfromlim", "type": ["null", "float"], "default": null, "doc": "Difference: diffmaglim - magap [mag]"}, + {"name": "seeratio", "type": ["null", "float"], "default": null, "doc": "Ratio: difffwhm / fwhm"}, + {"name": "aimage", "type": ["null", "float"], "default": null, "doc": "Windowed profile RMS afloat major axis from SExtractor [pixels]"}, + {"name": "bimage", "type": ["null", "float"], "default": null, "doc": "Windowed profile RMS afloat minor axis from SExtractor [pixels]"}, + {"name": "aimagerat", "type": ["null", "float"], "default": null, "doc": "Ratio: aimage / fwhm"}, + {"name": "bimagerat", "type": ["null", "float"], "default": null, "doc": "Ratio: bimage / fwhm"}, + {"name": "elong", "type": ["null", "float"], "default": null, "doc": "Ratio: aimage / bimage"}, + {"name": "nneg", "type": ["null", "int"], "default": null, "doc": "number of negative pixels in a 5 x 5 pixel stamp"}, + {"name": "nbad", "type": ["null", "int"], "default": null, "doc": "number of prior-tagged bad pixels in a 5 x 5 pixel stamp"}, + {"name": "rb", "type": ["null", "float"], "default": null, "doc": "RealBogus quality score; range is 0 to 1 where closer to 1 is more reliable"}, + {"name": "ssdistnr", "type": ["null", "float"], "default": null, "doc": "distance to nearest known solar system object if exists within 30 arcsec [arcsec]"}, + {"name": "ssmagnr", "type": ["null", "float"], "default": null, "doc": "magnitude of nearest known solar system object if exists within 30 arcsec (usually V-band from MPC archive) [mag]"}, + {"name": "ssnamenr", "type": ["null", "string"], "default": null, "doc": "name of nearest known solar system object if exists within 30 arcsec (from MPC archive)"}, + {"name": "sumrat", "type": ["null", "float"], "default": null, "doc": "Ratio: sum(pixels) / sum(|pixels|) in a 5 x 5 pixel stamp where stamp is first median-filtered to mitigate outliers"}, + {"name": "magapbig", "type": ["null", "float"], "default": null, "doc": "Aperture mag using 18 pixel diameter aperture [mag]"}, + {"name": "sigmagapbig", "type": ["null", "float"], "default": null, "doc": "1-sigma uncertainty in magapbig [mag]"}, + {"name": "ranr", "type": ["null", "double"], "doc": "Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg]"}, + {"name": "decnr", "type": ["null", "double"], "doc": "Declination of nearest source in reference image PSF-catalog; J2000 [deg]"}, + {"name": "scorr", "type": ["null", "double"], "default": null, "doc": "Peak-pixel signal-to-noise ratio in point source matched-filtered detection image"}, + {"name": "magzpsci", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point for photometry estimates [mag]"}, + {"name": "magzpsciunc", "type": ["null", "float"], "default": null, "doc": "Magnitude zero point uncertainty (in magzpsci) [mag]"}, + {"name": "magzpscirms", "type": ["null", "float"], "default": null, "doc": "RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag]"}, + {"name": "clrcoeff", "type": ["null", "float"], "default": null, "doc": "Color coefficient from linear fit from photometric calibration of science image"}, + {"name": "clrcounc", "type": ["null", "float"], "default": null, "doc": "Color coefficient uncertainty from linear fit (corresponding to clrcoeff)"}, {"name": "rbversion", "type": "string", "doc": "version of RealBogus model/classifier used to assign rb quality score"} ] } From a9ed6a9028f29ca952723d060f75e17e04612bb3 Mon Sep 17 00:00:00 2001 From: Eric Bellm Date: Tue, 26 Sep 2023 14:19:44 -0700 Subject: [PATCH 2/2] update schema docs for v4.01 --- docs/schema.md | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/schema.md b/docs/schema.md index d50fc45..3170bad 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -1,13 +1,13 @@ ZTF Avro Schemas ================ -These documents are for schema v3.3. +These documents are for schema v4.01. ## Schema Heirarchy ZTF uses nested schemas to organize the data in the alert packet. -`ztf.alert` (defined in [`alert.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/alert.avsc)) is the top-level namespace. `ztf.alert` in turn relies on [`candidate.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/candidate.avsc), [`prv_candidate.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/prv_candidate.avsc), and [`cutout.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/cutout.avsc). +`ztf.alert` (defined in [`alert.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/alert.avsc)) is the top-level namespace. `ztf.alert` in turn relies on [`candidate.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/candidate.avsc), [`prv_candidate.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/prv_candidate.avsc), [`fp_hist.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/fp_hist.avsc) and [`cutout.avsc`](https://github.com/ZwickyTransientFacility/ztf-avro-alert/blob/master/schema/cutout.avsc). ### ztf.alert @@ -22,6 +22,7 @@ The top-level alert contains the following fields: | `candid` | long | unique identifier for the subtraction candidate | | `candidate` | `ztf.alert.candidate` | candidate record | | `prv_candidates` | array of `ztf.alert.prv_candidate` or null | candidate records for 30 days' past history | +| `fp_hists` | array of `ztf.alert.fp_hist` or null | forced photometry measurements at the candidate position on past difference images | | `cutoutScience` | `ztf.alert.cutout` or null | cutout of the science image | | `cutoutTemplate` | `ztf.alert.cutout` or null | cutout of the coadded reference image | | `cutoutDifference` | `ztf.alert.cutout` or null | cutout of the resulting difference image | @@ -144,6 +145,42 @@ The fields for an individual `prv_candidate` are identical to `candidate` except Additionally, if the previous image has a nondetection at position of the new candidate, `candid`, `isdiffpos`, `ra`, `dec`, `magpsf`, `sigmapsf`, `ranr`, and `decr` will be null. In this case `diffmaglim` provides an estimate of the limiting magnitude over the entire image. +### ztf.alert.fp_hist + +Introduced in the v4 schemas in 2023, these records provide forced photometry measurements at the candidate position on earlier differnece images. + +| Field | Type | Contents | +|:--------|:-------|:--------| +| `field` | [null, int], default: null | ZTF field ID | +| `rcid` | [null, int], default: null | Readout channel ID [00 .. 63] | +| `fid` | int | Filter ID (1=g; 2=R; 3=i) | +| `pid` | long | Processing ID for image | +| `rfid` | long | Processing ID for reference image to facilitate archive retrieval | +| `sciinpseeing` | [null, float], default: null | Effective FWHM of sci image [pixels] | +| `scibckgnd` | [null, float], default: null | Background level in sci image [DN] | +| `scisigpix` | [null, float], default: null | Robust sigma per pixel in sci image [DN] | +| `magzpsci` | [null, float], default: null | Magnitude zero point for photometry estimates [mag] | +| `magzpsciunc` | [null, float], default: null | Magnitude zero point uncertainty (in magzpsci) [mag] | +| `magzpscirms` | [null, float], default: null | RMS (deviation from average) in all differences between instrumental photometry and matched photometric calibrators from science image processing [mag] | +| `clrcoeff` | [null, float], default: null | Color coefficient from linear fit from photometric calibration of science image | +| `clrcounc` | [null, float], default: null | Color coefficient uncertainty from linear fit (corresponding to clrcoeff) | +| `exptime` | [null, float], default: null | Integration time of camera exposure [sec] | +| `adpctdif1` | [null, float], default: null | Full sci image astrometric RMS along R.A. with respect to Gaia1 [arcsec] | +| `adpctdif2` | [null, float], default: null | Full sci image astrometric RMS along Dec. with respect to Gaia1 [arcsec] | +| `diffmaglim` | [null, float], default: null | Expected 5-sigma mag limit in difference image based on global noise estimate [mag] | +| `programid` | int | Program ID: encodes either public, collab, or caltech mode | +| `jd` | double | Observation Julian date at start of exposure [days] | +| `forcediffimflux` | [null, float], default: null | Forced difference image PSF-fit flux [DN] | +| `forcediffimfluxunc` | [null, float], default: null | 1-sigma uncertainty in forcediffimflux [DN] | +| `procstatus` | [null, string], default: null | Forced photometry processing status codes (0 => no warnings); see documentation | +| `distnr` | [null, float], default: null | distance to nearest source in reference image PSF-catalog [arcsec] | +| `ranr` | double | Right Ascension of nearest source in reference image PSF-catalog; J2000 [deg] | +| `decnr` | double | Declination of nearest source in reference image PSF-catalog; J2000 [deg] | +| `magnr` | [null, float], default: null | magnitude of nearest source in reference image PSF-catalog [mag] | +| `sigmagnr` | [null, float], default: null | 1-sigma uncertainty in magnr [mag] | +| `chinr` | [null, float], default: null | DAOPhot chi parameter of nearest source in reference image PSF-catalog | +| `sharpnr` | [null, float], default: null | DAOPhot sharp parameter of nearest source in reference image PSF-catalog | + ### ztf.alert.cutout Each cutout contains two fields: