Merge pull request #318 from Subaru-PFS/u/wanqqq/exetime-upperlimit

force the pointing simulation to stop exactly at exetime

README.md

@@ -78,9 +78,6 @@ OUTPUT_DIR="data"
 # maximum execution time (s) to terminate the calculation (default: 900s = 15min, 0 = no limit)
 # MAX_EXETIME=0
 
-# maximum number of pointings to be considered (default: 200, 0 = no limit)
-# MAX_NPPC=100
-
 # email setting (email will be sent at each submission)
 # EMAIL_FROM=
 # EMAIL_TO=
@@ -101,6 +98,10 @@ ANN_FILE="user_announcement.md"
 # SQLite database file to be used for the duplication check of upload_id
 # The file will be created under $OUTPUT_DIR
 UPLOADID_DB="upload_id.sqlite"
+
+# loggging level
+# DEBUG, INFO (default), WARNING, ERROR, or CRITICAL
+LOG_LEVEL="INFO"
 ```
 
 ## Preparing database

docs/docs/PPP.md

@@ -66,19 +66,20 @@ The online PPP will give a status report of the pointing simulation.
 
     - (Usually under **Classical** mode) No fibers can be assigned since the input pointings can not complete any targets. For example, if a target requests 1800 sec, but only one pointing with an individual exposure time of 900 sec is given, no fiber can be assigned to the target since it can not be completed. Adding pointings or modifying individual exposure time can solve the problem.
     - No fibers can be assigned due to no available fibers. Slightly shifting the pointing by ~0.2-0.5 degree can solve the problem in most cases.
+    - The running time exceeds 15 minutes.
+
 
 !!! warning "Warnings are raised in the following cases:"
 
     - The total requested time exceeds the 5-night upper limit for the normal program (35 hours).
-    - The running time exceeds 15 minutes.
 
 #### Examples of status
 
 <figure markdown>
   ![Status indicators](images/ppp_warning_35h.png){ width="1000" }
-  ![Status indicators](images/ppp_warning_exetime.png){ width="995" }
+  ![Status indicators](images/ppp_error_exetime.png){ width="995" }
   <figcaption>(Top) A warning to indicate that the total time to complete all targets in the list is estimated to exceed 5 nights</figcaption>
-  <figcaption>(Bottom) A warning to indicate that the running time exceeds 15 minutes.</figcaption>
+  <figcaption>(Bottom) An error to indicate that the running time exceeds 15 minutes.</figcaption>
 </figure>
 
 <figure markdown>

docs/docs/cli.md

@@ -78,8 +78,7 @@ $ pfs-uploader-cli simulate [OPTIONS] INPUT_LIST
 * `--date-begin TEXT`: Begin date (e.g., 2023-02-01). The default is the first date of the next Subaru semester.
 * `--date-end TEXT`: End date (e.g., 2023-07-31). The default is the last date of the next Subaru semester.
 * `--single-exptime INTEGER`: Single exposure time (s).  [default: 900]
-* `--max-exec-time INTEGER`: Max execution time (s). Default is 0 (no limit).
-* `--max-nppc INTEGER`: Max number of pointings to consider. Default is 0 (no limit).
+* `--max-exec-time INTEGER`: Max execution time (s). 0 means no limit.  [default: 0]
 * `--obs-type [queue|classical|filler]`: Observation type.  [default: queue]
 * `--log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]`: Set the log level.  [default: INFO]
 * `--help`: Show this message and exit.

docs/docs/developer.md

@@ -85,9 +85,6 @@ OUTPUT_DIR="data"
 # maximum execution time (s) to terminate the calculation (default: 900s = 15min, 0 = no limit)
 # MAX_EXETIME=0
 
-# maximum number of pointings to be considered (default: 200, 0 = no limit)
-# MAX_NPPC=100
-
 # email setting (email will be sent at each submission)
 # EMAIL_FROM=
 # EMAIL_TO=
@@ -108,6 +105,10 @@ ANN_FILE="user_announcement.md"
 # SQLite database file to be used for the duplication check of upload_id
 # The file will be created under $OUTPUT_DIR
 UPLOADID_DB="upload_id.sqlite"
+
+# loggging level
+# DEBUG, INFO (default), WARNING, ERROR, or CRITICAL
+LOG_LEVEL="INFO"
 ```
 
 ## Preparing database

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
     "ets-fiber-assigner @ git+https://github.com/Subaru-PFS/ets_fiberalloc.git",
     "ics-cobraOps @ git+https://github.com/Subaru-PFS/ics_cobraOps.git",
     "pfs-utils @ git+https://github.com/Subaru-PFS/pfs_utils.git",
+    "psutil>=6.0.0",
 ]
 requires-python = ">=3.10,<3.13"
 readme = "README.md"

src/pfs_target_uploader/cli/cli_main.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import glob
+import multiprocessing as mp
 import os
 import sys
 from datetime import date
@@ -154,23 +155,16 @@ def simulate(
     max_exec_time: Annotated[
         int,
         typer.Option(
-            "--max-exec-time", help="Max execution time (s). Default is 0 (no limit)."
+            "--max-exec-time", help="Max execution time (s). 0 means no limit."
         ),
-    ] = None,
-    max_nppc: Annotated[
-        int,
-        typer.Option(
-            "--max-nppc",
-            help="Max number of pointings to consider. Default is 0 (no limit).",
-        ),
-    ] = None,
+    ] = 0,
     obs_type: Annotated[ObsType, typer.Option(help="Observation type.")] = "queue",
     log_level: Annotated[
         LogLevel, typer.Option(case_sensitive=False, help="Set the log level.")
     ] = LogLevel.INFO,
 ):
     logger.remove(0)
-    logger.add(sys.stderr, level=log_level.value)
+    logger.add(sys.stderr, level=log_level.value, enqueue=True)
 
     if obs_type != "classical":
         logger.warning(
@@ -189,11 +183,6 @@ def simulate(
         date_begin = None if date_begin is None else date.fromisoformat(date_begin)
         date_end = None if date_end is None else date.fromisoformat(date_end)
 
-        if max_exec_time is None:
-            max_exec_time = 0
-        if max_nppc is None:
-            max_nppc = 0
-
         validation_status, df_validated = validate_input(
             df_input, date_begin=date_begin, date_end=date_end
         )
@@ -206,6 +195,45 @@ def simulate(
     tb_visible = tb_input[validation_status["visibility"]["success"]]
 
     logger.info("Running the online PPP to simulate pointings")
+
+    ppp_run_results = mp.Manager().Queue()
+
+    ppp_run = mp.Process(
+        target=PPPrunStart,
+        name="PPP",
+        args=(
+            tb_visible,
+            None,  # uPPC
+            None,  # weight_para
+            single_exptime,  # single_exptime
+            1.38,  # d_pfi
+            True,  # quiet
+            "HDBSCAN",  # clustering_algorithm
+            max_exec_time,  # max_exetime
+            ppp_run_results,  # queue
+            logger,  # logger
+        ),
+    )
+
+    # start run PPP
+    ppp_run.start()
+
+    # Wait max_exetime for PPP
+    ppp_run.join(max_exec_time if max_exec_time > 0 else None)
+
+    if ppp_run.is_alive():
+        # if ppp is still running after max_exetime, kill it
+        logger.error("Pointing simulation failed (runout time)")
+
+        # Terminate PPP
+        ppp_run.terminate()
+
+        # Cleanup
+        ppp_run.join()
+
+        # exit
+        sys.exit(1)
+
     (
         uS_L2,
         _,
@@ -218,16 +246,10 @@ def simulate(
         sub_m,
         obj_allo_M_fin,
         _,  # ppp_status
-    ) = PPPrunStart(
-        tb_visible,
-        None,  # uPPC
-        None,  # weight_para
-        exetime=max_exec_time,
-        max_nppc=max_nppc,
-        single_exptime=single_exptime,
-    )
+    ) = ppp_run_results.get()
 
     logger.info("Summarizing the results")
+
     _, p_result_fig, p_result_ppc, p_result_tab = ppp_result(
         cR_L_,
         sub_l,

src/pfs_target_uploader/pn_app.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 from datetime import datetime, timezone
 
 import gurobipy
@@ -38,21 +39,22 @@ def _toggle_widgets(widgets: list, disabled: bool = True):
 def target_uploader_app(use_panel_cli=False):
     pn.state.notifications.position = "bottom-left"
 
+    config = dotenv_values(".env.shared")
+
+    # configure logger to be multiprocessing-safe
+    log_level = config["LOG_LEVEL"] if "LOG_LEVEL" in config.keys() else "INFO"
+    logger.info(f"Log level is set to {log_level}")
+    logger.remove()
+    logger.add(sys.stderr, level=log_level, enqueue=True)
+
     logger.info(f"{pn.state.headers=}")
     logger.info(f"{pn.state.location.href=}")
 
-    config = dotenv_values(".env.shared")
-
     if "MAX_EXETIME" not in config.keys():
         max_exetime: int = 900
     else:
         max_exetime = int(config["MAX_EXETIME"])
 
-    if "MAX_NPPC" not in config.keys():
-        max_nppc: int = 200
-    else:
-        max_nppc = int(config["MAX_NPPC"])
-
     if "PPP_QUIET" not in config.keys():
         ppp_quiet: bool = True
     else:
@@ -88,7 +90,6 @@ def target_uploader_app(use_panel_cli=False):
         logger.info("No upload ID database is used. Scan output directories directly.")
 
     logger.info(f"Maximum execution time for the PPP is set to {max_exetime} sec.")
-    logger.info(f"Maximum number of PPCs is set to {max_nppc}.")
 
     logger.info(f"config params from dotenv: {config}")
 
@@ -126,7 +127,7 @@ def target_uploader_app(use_panel_cli=False):
 
     panel_results = ValidationResultWidgets()
     panel_targets = TargetWidgets()
-    panel_ppp = PppResultWidgets(exetime=max_exetime, max_nppc=max_nppc)
+    panel_ppp = PppResultWidgets()
 
     panel_input.reset()
     panel_input.db_path = db_path
@@ -439,7 +440,10 @@ def cb_PPP(event):
                 single_exptime=panel_obs_type.single_exptime.value,
                 clustering_algorithm=clustering_algorithm,
                 quiet=ppp_quiet,
+                max_exetime=max_exetime,
+                logger=logger,
             )
+
             panel_ppp.show_results()
 
             tab_panels.active = 2