fix conflicts in change log

bin/run_mtl_loop

@@ -12,10 +12,10 @@ ap = ArgumentParser(description='Make an initial HEALPixel-split ledger for a Me
 ap.add_argument("obscon",
                 help="String matching ONE obscondition in the bitmask yaml file \
                 (e.g. 'BRIGHT'). Controls priorities when merging targets,      \
-                which tiles to process, etc.")
+                which tiles to process, etc.", choices=["DARK", "BRIGHT", "BACKUP"])
 ap.add_argument("-s", "--survey",
                 help="Flavor of survey to run. Defaults to [{}]".format(survey),
-                default=survey)
+                default=survey, choices=["main", "sv3", "sv2"])
 ap.add_argument('--zcatdir',
                 help="Full path to the directory that hosts the redshift        \
                 catalogs. Default is to use the $ZCAT_DIR environment variable",
@@ -33,6 +33,11 @@ ap.add_argument("--reprocess", action='store_true',
 
 ns = ap.parse_args()
 
+if ns.obscon == "BACKUP" and ns.survey == "main":
+    msg = "Updating BACKUP ledgers is not yet supported for the Main Survey!!!"
+    raise RuntimeError(msg)
+    log.error(msg)
+
 scndstates = [False]
 
 # ADM if nosecondary wasn't passed, also process the secondary ledger

doc/changes.rst

@@ -5,20 +5,29 @@ desitarget Change Log
 2.3.1 (unreleased)
 ------------------
 
-* Restore ``select_mock_targets`` to a working state [`PR #788`].
-* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and ``BGS`` 
-  color-cut selection functions to support the `desisim` template-generating
-  code (see `desisim/PR #556`) [`PR #786`].
+* A couple of MTL maintenance updates [`PR #789`_]. Includes:
+    * Turn off BACKUP MTL processing for the Main Survey (for now).
+    * Allow the file format for override ledgers to be forced.
+        * Addresses `issue #784`_.
+* Restore ``select_mock_targets`` to a working state [`PR #788`_].
+* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and
+  ``BGS`` color-cut selection functions to support the `desisim`
+  template-generating code (see `desisim/PR #556`_) [`PR #786`_].
+* Extra functionality for splitting randoms by HEALPixel [`PR #785`_]:
+    * Correctly process catalogs with an existing ``HPXPIXEL`` column.
 * Functionality to split random catalogs by HEALPixel [`PR #783`_].
     * Allows the io/reading utilities to be used on the resulting files.
 * More accurately round pixel coordinates for randoms [`PR #782`_].
     * Alters brick-based values for randoms (e.g. NOBS/MASKBITS/PSFSIZE).
 
 .. _`PR #782`: https://github.com/desihub/desitarget/pull/782
 .. _`PR #783`: https://github.com/desihub/desitarget/pull/783
+.. _`PR #785`: https://github.com/desihub/desitarget/pull/785
 .. _`desisim/PR #556`: https://github.com/desihub/desisim/pull/556
+.. _`issue #784`: https://github.com/desihub/desitarget/issues/784
 .. _`PR #786`: https://github.com/desihub/desitarget/pull/786
 .. _`PR #788`: https://github.com/desihub/desitarget/pull/788
+.. _`PR #789`: https://github.com/desihub/desitarget/pull/789
 
 2.3.0 (2021-12-14)
 ------------------

py/desitarget/io.py

@@ -5,6 +5,8 @@
 =============
 
 Functions for reading, writing and manipulating files related to targeting.
+
+.. _`desitarget issue 784`: https://github.com/desihub/desitarget/issues/784#issuecomment-1011452303
 """
 from __future__ import (absolute_import, division)
 #
@@ -1485,9 +1487,10 @@ def write_randoms_in_hp(randoms, outdirname, nside, pixlist, hpx=None,
 
     Notes
     -------
-        - A new column HPXPIXEL is added to `randoms`. HPXNSIDE=`nside`
-          and FILENSID=`nside` and "HPXNEST"=``True`` are added to the
-          output header (or overwritten, if they already exist).
+        - A new column HPXPIXEL is added to `randoms` (or overwritten if
+          it already exists). HPXNSIDE=`nside` and FILENSID=`nside` and
+          "HPXNEST"=``True`` are added to the output header (or
+          overwritten, if they already exist).
     """
     t0 = time()
 
@@ -1516,18 +1519,22 @@ def write_randoms_in_hp(randoms, outdirname, nside, pixlist, hpx=None,
             # ADM set up a new array restricted to just the pixel of
             # ADM interest and add the HPXPIXEL column.
             nrows = np.sum(inpix)
-            dt = randoms.dtype.descr + [('HPXPIXEL', '>i8')]
+            if "HPXPIXEL" in randoms.dtype.names:
+                dt = randoms.dtype.descr
+            else:
+                dt = randoms.dtype.descr + [('HPXPIXEL', '>i8')]
             outran = np.zeros(nrows, dtype=dt)
-            outran["HPXPIXEL"] = pix
-            for col in randoms.dtype.names:
+            for col in outran.dtype.names:
                 outran[col] = randoms[col][inpix]
+            outran["HPXPIXEL"] = pix
             # ADM add the pixel to the output file hdr. dict is in case
             # ADM hdr was passed as a FITSHDR, which can't be copied.
             outhdr = dict(hdr).copy()
             outhdr["FILEHPX"] = pix
             # ADM write out the randoms in this pixel.
-            nt, fn = write_randoms(outdirname, outran, indir=infile,
-                                   nsidefile=nside, hpxlist=pix, extra=outhdr)
+            nt, fn = write_randoms(
+                outdirname, outran, indir=infile, nsidefile=nside, hpxlist=pix,
+                extra=outhdr, hpxsplit=True)
             if verbose:
                 log.info('{} targets written to {}...t={:.1f}s'.format(
                     nt, fn, time()-t0))
@@ -1537,7 +1544,7 @@ def write_randoms_in_hp(randoms, outdirname, nside, pixlist, hpx=None,
 
 def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False,
                   nsidefile=None, hpxlist=None, resolve=True, north=None,
-                  extra=None):
+                  extra=None, hpxsplit=False):
     """Write a catalogue of randoms and associated pixel-level info.
 
     Parameters
@@ -1577,6 +1584,9 @@ def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False,
     extra : :class:`dict`, optional
         If passed (and not ``None``), write these extra dictionary keys
         and values to the output header.
+    hpxsplit : :class:`bool`, optional, defaults to ``False``
+        If ``True`` use the "INFILE" key in `hdr` to determine an extra,
+        final, directory to add to the output path.
 
     Returns
     -------
@@ -1684,6 +1694,18 @@ def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False,
     # ADM add whether or not the randoms were resolved to the header.
     hdr["RESOLVE"] = resolve
 
+    # ADM if performing a split-by-HEALPixel, add an extra directory.
+    if hpxsplit:
+        if "INFILE" in hdr:
+            # ADM determine an extra directory to add to the output path.
+            xdirname = os.path.splitext(os.path.basename(hdr["INFILE"]))[0]
+            dirname, basename = os.path.split(filename)
+            filename = os.path.join(dirname, xdirname, basename)
+        else:
+            msg = "hpxsplit is True: hdr must include the INFILE key!!!"
+            log.error(msg)
+            raise RuntimeError(msg)
+
     # ADM create necessary directories, if they don't exist.
     os.makedirs(os.path.dirname(filename), exist_ok=True)
 
@@ -3027,24 +3049,27 @@ def read_ecsv_header(filename, cleanup=True):
     return hdr
 
 
-def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False):
-    """Construct an MTL filename just from the header in the file
+def find_mtl_file_format_from_header(hpdirname, returnoc=False,
+                                     override=False, forceoverride=False):
+    """Construct MTL filename from a directory containing an MTL ledger.
 
     Parameters
     ----------
     hpdirname : :class:`str`
-        Full path to either a directory containing MTL ledgers that have
-        been partitioned by HEALPixel. Or the name of a single ledger.
+        Full path to a directory containing MTL ledgers that have been
+        partitioned by HEALPixel.
     returnoc : :class:`bool`, optional, defaults to ``False``
         If ``True`` then also return the OBSCON header keyword
         for files in this directory.
     override : :class:`bool`, optional, defaults to ``False``
         If ``True``, return the file form for an override ledger instead
-        of a standard MTL ledger IF the location of a standard MTL ledger
-        or ledgers has been passed as `hpdirname`. If the location of an
-        override ledger or ledgers has been passed as `hpdirname`, then
-        the fact that we're working with override ledgers is detected
-        automatically and `override`=``True`` does not need to be passed.
+        of a standard MTL ledger BUT if an OVERRIDE value is detected in
+        the header of the first file found in `hpdirname` let it take
+        precedence over `override`. See `desitarget issue 784`_.
+    forceoverride : :class:`bool`, optional, defaults to ``False``
+        If ``True``, return the file form for an override ledger instead
+        of a standard MTL ledger REGARDLESS of any OVERRIDE value found
+        in the header of the first file found in `hpdirname`.
 
     Returns
     -------
@@ -3058,15 +3083,20 @@ def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False):
     Notes
     -----
         - Should work for both .ecsv and .fits files.
+        - If both `override` and `forceoverride` are ``True``,
+          `forceoverride` takes precedence.
     """
     # ADM grab information from the target directory.
     surv = read_keyword_from_mtl_header(hpdirname, "SURVEY")
     oc = read_keyword_from_mtl_header(hpdirname, "OBSCON")
     # ADM detect whether we're working with the override ledgers.
-    try:
-        override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE")
-    except KeyError:
-        pass
+    if forceoverride:
+        override = forceoverride
+    else:
+        try:
+            override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE")
+        except KeyError:
+            pass
 
     from desitarget.mtl import get_mtl_ledger_format
     ender = get_mtl_ledger_format()

py/desitarget/mtl.py

@@ -1449,7 +1449,8 @@ def force_overrides(hpdirname, pixlist):
     # ADM find the general format for the ledger files in `hpdirname`.
     fileform = io.find_mtl_file_format_from_header(hpdirname)
     # ADM this is the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM before making updates, check all suggested ledgers exist.
     for pix in pixlist:
@@ -1550,7 +1551,8 @@ def reprocess_ledger(hpdirname, zcat, obscon="DARK"):
     # ADM also returning the obsconditions.
     fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True)
     # ADM also find the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM check the obscondition is as expected.
     if obscon != oc:
@@ -1809,7 +1811,8 @@ def update_ledger(hpdirname, zcat, targets=None, obscon="DARK",
     # ADM also returning the obsconditions.
     fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True)
     # ADM this is the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM check the obscondition is as expected.
     if obscon != oc: