default_config.yaml

overrides:
  - local_overrides.yaml
augments:
  - local_augments.yaml

path:
  data_root: null
  data_temp: null

db:
  engine: postgresql
  user: postgres
  password: fragile
  host: localhost
  port: 5432
  database: seechange

storage:
  images:
    # can choose hdf5 as well, but this is not yet implemented
    format: fits
    # should Image object save the weights/flags/etc in a single file with the image data?
    single_file: false
    # The convention for building filenames for images
    # Use any of the following: short_name, date, time, section_id, filter, ra, dec, prov_id
    # Can also use section_id_int if the section_id is always an integer
    # Can also use ra_int and ra_frac to get the integer number before/after the decimal point
    # (the same can be done for dec). Also use ra_int_h to get the number in hours.
    # to get the declination with "p" or "m" replacing the sign, use dec_int_pm.
    # The string given here is fed into the python format() function
    # so you can use e.g., {ra_int:03d} to get a 3 digit zero padded right ascension.
    # The name convention can also include subfolders (e.g., using {ra_int}/...).
    # The minimal set of fields to make the filenames unique include:
    # inst_name (short instrument name), date, time, section_id, prov_hash
    # (in this example, the first six characters of the provenance unique hash)
    name_convention: "{ra_int:03d}/{inst_name}_{date}_{time}_{section_id}_{filter}_{im_type}_{prov_hash:.6s}"


# ======================================================================
# Archive:
#
# Set to null if there is no archive; otherwise, a dict
# Subfields:
#   url: the URL of the archive server, or null if archive is on the filesystem
#   verify_cert: boolean, should we verify the SSL cert of the archive server
#   path_base: the base of the collection on the archive server (a string unique to this dataset)
#   read_dir: the directory to read from if the archive is on the local filesystem, or null
#   write_dir: the directory to write to if the archive is on the local filesystem, or null

archive: null

# ======================================================================
# Gaia DR3 server
#
# There are two ways we can get it : through the server defined in the
#  submodule extern/nersc-desi-gaia-dr3-server, and via NOIRLab
#  using their queryClient.
# Set use_server to True to use the custom one
# Set use_datalab to True to use NOIRLab datalab.  This will
#   be ignored if use_server is True
# Set fallback_datalab to True to try to use NOIRLab datalab
#   if the custom server doesn't return after five tries.
# server_url is the server where the custom server runs

catalog_gaiadr3:
  use_server: True
  use_datalab: False
  fallback_datalab: True
  server_url: https://ls4-gaia-dr3.lbl.gov
  server_timeout_sec: 5.


# ======================================================================
# Pipeline components
#
# For documentation on the parameters, see the Parameters subclass
# in the file that defines each part of the pipeline

pipeline:
  # save images and their products before the stage where we look for a reference and make a subtraction
  save_before_subtraction: true
  # automatically save all the products at the end of the pipeline run
  save_at_finish: true

preprocessing:
  # these steps need to be done on the images: either they came like that or we do it in the pipeline
  steps_required: [ 'overscan', 'linearity', 'flat', 'fringe' ]

extraction:
  sources:  # this part of the extraction parameters is for finding sources and calculating the PSF
    method: sextractor
    measure_psf: true
    apertures: [1.0, 2.0, 3.0, 5.0]
    inf_aper_num: -1
    best_aper_num: 0
    aperunit: fwhm
    separation_fwhm: 1.0
    threshold: 3.0
    subtraction: false
  bg:  # this part of the extraction parameters is for estimating the background
    format: map
    method: sep
    poly_order: 1
    sep_box_size: 128
    sep_filt_size: 3
  wcs:  # this part of the extraction parameters is for finding the world coordinates system (astrometric calibration)
    cross_match_catalog: gaia_dr3
    solution_method: scamp
    max_catalog_mag: [22.0]
    mag_range_catalog: 6.0
    min_catalog_stars: 50
    max_sources_to_use: [2000, 1000, 500, 200]
  zp:  # this part of the extraction parameters is for finding the zero point (photometric calibration)
    cross_match_catalog: gaia_dr3
    max_catalog_mag: [22.0]
    mag_range_catalog: 6.0
    min_catalog_stars: 50

# how to do the subtractions
subtraction:
  method: zogy
  # set refset to null to only make references (no subtraction will happen).
  # to start running subtractions, first make a ref set and use the name of that
  # in this field.
  refset: null
  alignment:
    method: swarp
    to_index: new

# how to extract sources (detections) from the subtration image
detection:
  subtraction: true  # this sets up the Detector object to run on subtraction images
  method: filter  # when using ZOGY subtraction, detection method must be "filter"!
  threshold: 5.0

# how to make the cutouts
cutting:
  cutout_size: 25

# how to measure things like fluxes and centroids, and make analytical cuts
measuring:
  annulus_radii: [10, 15]
  annulus_units: pixels
  use_annulus_for_centroids: true
   # TODO: remove these in favor of the thresholds dict (and put None to not threshold any one of them)? Issue #319
  analytical_cuts: ['negatives', 'bad pixels', 'offsets', 'filter bank']
  outlier_sigma: 3.0  # how many times the noise rms counts as a positive/negative outlier
  bad_pixel_radius: 3.0  # how far from the centroid counts as having a bad pixel
  bad_pixel_exclude: []  # which types of bad pixels are ok to have near the source
  streak_filter_angle_step: 5.0  # how many degrees to step through the angles for the streak filter
  width_filter_multipliers: [0.25, 2.0, 5.0, 10.0]  # try different width, if they trigger a high S/N, disqualify it
  association_radius: 2.0  # when matching sources, how close counts as being from the same underlying object
  thresholds:  # any of the analytical cuts that score above these thresholds will be disqualified
    negatives: 0.3
    bad pixels: 1
    offsets: 5.0
    filter bank: 1
  deletion_thresholds:  # any of the analytical cuts that score above these thresholds will be deleted
    negatives: 0.3
    bad pixels: 1
    offsets: 5.0
    filter bank: 1


# use these parameters when running the coaddition pipeline, e.g., for making weekly coadds
# the coaddition pipeline will load the regular configuration first, then the coaddition config:
coaddition:
  # the pipeline handles any top level configuration (e.g., how to choose images)
  pipeline:
    # how many days back you'd like to collect the images for
    date_range: 7.0
  coaddition:
    method: zogy
    noise_estimator: sep
    flag_fwhm_factor: 1.0
    alignment:
      method: swarp
      to_index: last
    inpainting:
      multi_image_method: median
      feather_width: 2
      rescale_method: median
      single_image_method: biharmonic
      ignore_flags: 0
  # The following are used to override the regular "extraction" parameters
  extraction:
    sources:  # override the regular source and psf extraction parameters
      measure_psf: true
      threshold: 3.0
      method: sextractor
    bg:  # override the regular background estimation parameters
      format: map
      method: sep
    wcs:  # override the regular astrometric calibration parameters
      cross_match_catalog: gaia_dr3
      solution_method: scamp
      max_catalog_mag: [22.0]
      mag_range_catalog: 6.0
      min_catalog_stars: 50
    zp:  # override the regular photometric calibration parameters
      cross_match_catalog: gaia_dr3
      max_catalog_mag: [22.0]
      mag_range_catalog: 6.0
      min_catalog_stars: 50

# use these parameters to make references by coadding images
# the reference pipeline will load the regular configuration first,
# then the coaddition config, and only in the end, override with this:
referencing:
  maker:  # these handles the top level configuration (e.g., how to choose images that go into the reference maker)
    name: best_references  # the name of the reference set that you want to make references for
    allow_append: true  # can we add different image load criteria to an existing reference set with this name?
    start_time: null  # only grab images after this time
    end_time: null  # only grab images before this time
    instrument: null  # only grab images from this instrument (if list, will make cross-instrument references)
    filter: null  # only grab images with this filter
    project: null  # only grab images with this project
    min_airmass: null  # only grab images with airmass above this
    max_airmass: null  # only grab images with airmass below this
    min_background: null  # only grab images with background rms above this
    max_background: null  # only grab images with background rms below this
    min_seeing: null  # only grab images with seeing above this
    max_seeing: null  # only grab images with seeing below this
    min_lim_mag: null  # only grab images with limiting magnitude above this
    max_lim_mag: null  # only grab images with limiting magnitude below this
    min_exp_time: null  # only grab images with exposure time above this
    max_exp_time: null  # only grab images with exposure time below this
    min_number: 7  # only create a reference if this many images can be found
    max_number: 30  # do not add more than this number of images to the reference
    seeing_quality_factor: 3.0  # linear coefficient for adding lim_mag and seeing to get the "image quality"
    save_new_refs: true  # should the new references be saved to disk and committed to the database?
  pipeline:  # The following are used to override the regular "extraction" parameters
    extraction:
      sources:  # override the regular source and psf extraction parameters
        measure_psf: true
#        threshold: 3.0
#        method: sextractor
#      bg:
#        format: map
#        method: sep
#        poly_order: 1
#        sep_box_size: 128
#        sep_filt_size: 3
#      wcs:  # override the regular astrometric calibration parameters
#        cross_match_catalog: gaia_dr3
#        solution_method: scamp
#        max_catalog_mag: [22.0]
#        mag_range_catalog: 6.0
#        min_catalog_stars: 50
#      zp:  # override the regular photometric calibration parameters
#        cross_match_catalog: gaia_dr3
#        max_catalog_mag: [22.0]
#        mag_range_catalog: 6.0
#        min_catalog_stars: 50

  coaddition:  # override the coaddition parameters in the general "coaddition" config
    coaddition:  # override the way coadds are made, from the general "coaddition" config
      method: zogy
    extraction:  # override the coaddition/regular pipeline config, when extracting for the coadd images
      sources: # override the regular source and psf extraction parameters and the coadd extraction parameters
        measure_psf: true
#        threshold: 3.0
#        method: sextractor
#      bg: # override the regular background estimation parameters and the coadd extraction parameters
#        format: map
#        method: sep
#        poly_order: 1
#        sep_box_size: 128
#        sep_filt_size: 3
#      wcs:  # override the regular astrometric calibration parameters and the coadd extraction parameters
#        cross_match_catalog: gaia_dr3
#        solution_method: scamp
#        max_catalog_mag: [22.0]
#        mag_range_catalog: 6.0
#        min_catalog_stars: 50
#      zp:  # override the regular photometric calibration parameters and the coadd extraction parameters
#        cross_match_catalog: gaia_dr3
#        max_catalog_mag: [22.0]
#        mag_range_catalog: 6.0
#        min_catalog_stars: 50


# Specific configuration for specific instruments.
# Instruments should override the two defaults from
# instrument_default; they may add additional
# configuration that their code needs.

instrument_default:
  calibratorset: nightly
  flattype: sky


# DECam

# For the observatory-supplied calibrations, NOIRLab distributes them on
# a Google Drive, which is a nightmare to try to download
# programmatically.  What's more, they come in ginormous tar files,
# which are slow to process; especially for tests, where we just want to
# do a couple of chips, it's not worth the time.  So, I've untarred the
# files on the NERSC web portal to allow them to be grabbed
# individually.  These can be found on Perlmutter at
# /global/cfs/dirs/m2218/www/decam_calibration_files

DECam:
  calibratorset: externally_supplied
  flattype: externally_supplied
  calibfiles:
    mjd: 56876
    urlbase: https://portal.nersc.gov/cfs/m4616/decam_calibration_files/
    linearity: DECamMasterCal_56475/linearity/linearity_table_v0.4.fits
    fringebase: DECamMasterCal_56876/fringecor/DECam_Master_20131115v1
    flatbase: DECamMasterCal_56876/starflat/DECam_Master_20130829v3
    bpmbase: DECamMasterCal_56876/bpm/DECam_Master_20140209v2_cd_


# Config for astromatic utilities (sextractor, scamp, swarp, psfex)
astromatic:
  # An absolute path to where astromatic config files are
  config_dir: null
  # A path relative to models/base/CODE_ROOT where the astromatic
  # config files are. Ignored if config_dir is not null
  config_subdir: data/astromatic_config