Merge pull request #9 from lsst/tickets/OPSIM-1099

lsst · Nov 15, 2023 · 50f50e2 · 50f50e2
2 parents a0bd203 + 897d7ff
commit 50f50e2
Show file tree

Hide file tree

Showing 26 changed files with 532 additions and 241 deletions.
diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml
@@ -18,17 +18,18 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: "3.11"
-          channels: conda-forge
+          python-version: ${{ matrix.python-version }}
+          miniforge-variant: Mambaforge
+          channels: conda-forge,defaults
+          use-mamba: true
           channel-priority: strict
           show-channel-urls: true
 
-      - name: configure and install requirements and documenteer
+      - name: configure conda and install requirements
         shell: bash -l {0}
         run: |
-          conda config --set always_yes yes
-          conda install --quiet --file=requirements.txt
-          conda install --quiet pip
+          mamba install --quiet --file=requirements.txt          
+          mamba install --quiet pip
           pip install "documenteer[guide]"
 
       - name: install rubin_scheduler
@@ -38,11 +39,23 @@ jobs:
           ls ${{ github.workspace }}
           python -m pip install . --no-deps
 
-      - name: download data needed for scheduler docs
+      - name: Access rubin-sim-data cache
+        id: cache-rs
+        uses: actions/cache@v3
+        env:
+          cache-name: cached-rubin-sim-data
+        with:
+          path: ~/rubin_sim_data
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
+
+      - name: Update rubin-sim-data if needed
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}
-          scheduler_download_data --tdqm_disable -d 'scheduler'
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          # Download anything that is not current
+          scheduler_download_data --tdqm_disable --update
 
       - name: check conda and documenteer
         shell: bash -l {0}
@@ -52,7 +65,7 @@ jobs:
       - name: build docs
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
           cd docs
           package-docs build
 

diff --git a/.github/workflows/test_and_build.yaml b/.github/workflows/test_and_build.yaml
@@ -1,4 +1,4 @@
-name: Run Tests and Build Documentation
+name: Run Tests and Publish PyPI
 
 on:
   # Trigger the workflow on push (to main) or pull request
@@ -25,6 +25,7 @@ jobs:
         with:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
+          miniforge-variant: Mambaforge
           channels: conda-forge,defaults
           use-mamba: true
           channel-priority: strict
@@ -33,7 +34,6 @@ jobs:
       - name: configure conda and install requirements
         shell: bash -l {0}
         run: |
-          mamba config --set always_yes yes
           mamba install --quiet --file=requirements.txt
           mamba install --quiet --file=test-requirements.txt
 
@@ -42,13 +42,25 @@ jobs:
         run: |
           echo `pwd`
           ls ${{ github.workspace }}
-          python -m pip install .
+          python -m pip install . --no-deps
 
-      - name: download rubin_sim_data components needed for unit tests
+      - name: Access rubin-sim-data cache
+        id: cache-rs
+        uses: actions/cache@v3
+        env:
+          cache-name: cached-rubin-sim-data
+        with:
+          path: ~/rubin_sim_data
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
+
+      - name: Update rubin-sim-data if needed
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir
-          scheduler_download_data --tdqm_disable -d 'site_models,scheduler,skybrightness_pre,utils'
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          # Download anything that is not current
+          scheduler_download_data --tdqm_disable --update
 
       - name: conda list
         shell: bash -l {0}
@@ -57,7 +69,8 @@ jobs:
       - name: run tests
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          #pytest -r a -v 
           pytest -r a -v --cov=rubin_scheduler --cov=tests --cov-report=xml --cov-report=term --cov-branch
 
   pypi:

diff --git a/ci/Jenkinsfile_rubin_scheduler.conda b/ci/Jenkinsfile_rubin_scheduler.conda
@@ -0,0 +1,2 @@
+@Library('JenkinsShared')_
+ExternalCondaPipeline("rubin-scheduler")
diff --git a/docs/api.rst b/docs/api.rst
@@ -1,16 +1,19 @@
+.. py:currentmodule:: rubin_scheduler
+
 .. _api:
 
 API
 ===
 
 .. toctree::
-    :maxdepth: 4
+    :maxdepth: 2
 
-.. automodule:: rubin_scheduler.scheduler
+    FBS Scheduler <fbs-api>
 
-.. automodule:: rubin_scheduler.site_models
+    Site Models <site-models-api>
 
-.. automodule:: rubin_scheduler.utils
+    Skybrightness Pre <skybrightness-pre-api>
 
-.. automodule:: rubin_scheduler.data
+    Utils <utils-api>
 
+    Data <data-api>
diff --git a/docs/data-api.rst b/docs/data-api.rst
@@ -0,0 +1,12 @@
+.. py:currentmodule:: rubin_scheduler.data
+
+.. _data-api:
+
+========
+Data API
+========
+
+.. automodule:: rubin_scheduler.data
+    :imported-members:
+    :members:
+    :show-inheritance:
diff --git a/docs/data_download.rst → docs/data-download.rst b/docs/data_download.rst → docs/data-download.rst
diff --git a/docs/fbs-api.rst b/docs/fbs-api.rst
@@ -0,0 +1,79 @@
+.. py:currentmodule:: rubin_scheduler.scheduler
+
+.. _fbs-api:
+
+=================
+FBS Scheduler API
+=================
+
+Sim Runner
+^^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.sim_runner
+    :members:
+    :show-inheritance:
+
+Example Scheduler
+^^^^^^^^^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.example.example_scheduler
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Model Observatory
+^^^^^^^^^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.model_observatory
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Schedulers
+^^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.schedulers
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Surveys
+^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.surveys
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Basis Functions
+^^^^^^^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.basis_functions
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Features
+^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.features
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Detailers
+^^^^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.detailers
+    :imported-members:
+    :members:
+    :show-inheritance:
+
+Utils
+^^^^^^
+
+.. automodule:: rubin_scheduler.scheduler.utils
+    :imported-members:
+    :members:
+    :show-inheritance:
+
diff --git a/docs/fbs.rst b/docs/fbs.rst
@@ -10,14 +10,20 @@ The `rubin_scheduler.scheduler` module implements the scheduling algorithms for
 
 Scripts to use the scheduler code to create a simulated survey can be
 found in the github repo at
-`lsst-sims/sims_featureScheduler_runs2.0
-<https://github.com/lsst-sims/sims_featureScheduler_runs2.0>`_.
-To be able to simulate a full 10 years of observations, additional skybrightness
-data files must be downloaded (about 250GB), which can be done using the
-script `rubin_scheduler/bin/rs_download_sky <https://github.com/lsst/rubin_scheduler/bin/rs_download_sky>`_.
+`lsst-sims/sims_featureScheduler_runs3.3
+<https://github.com/lsst-sims/sims_featureScheduler_runs3.3>`_.
+To simulate a full 10 years of observations, additional skybrightness
+data files must be downloaded, as described in :ref:`Precalculated Skybrightness <skybrightness-pre>`, using
+
+`rs_download_sky`
+
 Running a typical simulation will take on the order of 6 hours to complete.
 
 The scheduler outputs a sqlite database containing the pointing history of
 the telescope, along with information about the conditions of each
 observation (visit).
-Description of the :doc:`schema for the output database <output_schema>`.
+Description of the :ref:`schema for the output database <output-schema>`.
+
+.. toctree::
+
+    Simulation output schema <output-schema>
diff --git a/docs/index.rst b/docs/index.rst
@@ -9,9 +9,10 @@ rubin_scheduler
 Scheduling and simulating millions of observations for the Rubin C. Observatory's Legacy Survey of Space and Time (LSST).
 
 .. toctree::
+    :maxdepth: 2
 
     Installation <installation>
-    Usage <user-guide>
+    User Guide <user-guide>
     API <api>
 
 
diff --git a/docs/output_schema.rst → docs/output-schema.rst b/docs/output_schema.rst → docs/output-schema.rst
@@ -1,3 +1,7 @@
+.. py:currentmodule:: rubin_scheduler
+
+.. _output-schema:
+
 =======================
 Scheduler Output Schema
 =======================

diff --git a/docs/site-models-api.rst b/docs/site-models-api.rst
@@ -0,0 +1,12 @@
+.. py:currentmodule:: rubin_scheduler.site_models
+
+.. _site-models-api:
+
+===============
+Site Models API
+===============
+
+.. automodule:: rubin_scheduler.site_models
+    :imported-members:
+    :members:
+    :show-inheritance:
diff --git a/docs/site_models.rst → docs/site-models.rst b/docs/site_models.rst → docs/site-models.rst
diff --git a/docs/skybrightness-pre-api.rst b/docs/skybrightness-pre-api.rst
@@ -0,0 +1,12 @@
+.. py:currentmodule:: rubin_scheduler.skybrightness_pre
+
+.. _skybrightness-pre-api:
+
+=====================
+Skybrightness Pre API
+=====================
+
+.. automodule:: rubin_scheduler.skybrightness_pre
+    :imported-members:
+    :members:
+    :show-inheritance:
diff --git a/docs/skybrightness-pre.rst b/docs/skybrightness-pre.rst
@@ -0,0 +1,29 @@
+.. py:currentmodule:: rubin_scheduler.skybrightness_pre
+
+.. _skybrightness-pre:
+
+=============================
+Skybrightness (Precalculated)
+=============================
+
+The `rubin_scheduler.skybrightness_pre` module generates and accesses pre-calculated skybrightness values
+over the sky in each bandpass during the expected on-sky period for LSST. The values calculated
+by the rubin_sim.skybrightness module are packaged into data files which are then read and passed to
+the scheduler by rubin_sim.skybrightness_pre.
+
+
+Downloading Skybrightness Data
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The standard rubin_sim_data download for skybrightness_pre contains a small quantity of skybrightness data,
+limited to a period around the survey start date. For full simulations, additional skybrightness data will
+be needed. This can be downloaded using
+
+`rs_download_sky`
+
+which will download a series of files from
+`https://s3df.slac.stanford.edu/data/rubin/sim-data/sims_skybrightness_pre/h5_2023_09_12/
+<https://s3df.slac.stanford.edu/data/rubin/sim-data/sims_skybrightness_pre/h5_2023_09_12/>`_.
+These files are identified by the MJD range of data contained within each file (i.e. `60841_61054.h5` contains
+pre-calculated skybrightness data covering MJD = 60841 to 61054).
+
diff --git a/docs/skybrightness_pre.rst b/docs/skybrightness_pre.rst
diff --git a/docs/user-guide.rst b/docs/user-guide.rst
@@ -6,12 +6,12 @@
 User Guide
 ##########
 
-Software modules within rubin_scheduler:
 
 .. toctree::
 
-    Scheduler <fbs>
-    Site Models <site_models>
-    Skybrightness(Pre) <skybrightness_pre>
-    Data Download <data_download>
-
+    FBS Scheduler <fbs>
+    Site Models <site-models>
+    Precalculated Skybrightness <skybrightness-pre>
+    Utils <utils>
+    Data Download Utilities <data-download>
+