Merge pull request #296 from Subaru-PFS/u/wanqqq/user_ppc_code_fix

add pointing simulation status & show target list on admin

src/pfs_target_uploader/pn_app.py

@@ -409,7 +409,9 @@ def cb_PPP(event):
 
         try:
             panel_ppp.origname = panel_input.file_input.filename
+            panel_ppp.origname_ppc = panel_ppcinput.file_input.filename
             panel_ppp.origdata = panel_input.file_input.value
+            panel_ppp.origdata_ppc = panel_input.file_input.value
             panel_ppp.df_summary = panel_status.df_summary
 
             if not validation_status["status"]:
@@ -514,14 +516,24 @@ def cb_submit(event):
         panel_ppp.df_input = df_validated
         panel_ppp.df_summary = panel_status.df_summary
         panel_ppp.origname = panel_input.file_input.filename
+        panel_ppp.origname_ppc = panel_ppcinput.file_input.filename
         panel_ppp.origdata = panel_input.file_input.value
+        panel_ppp.origdata_ppc = panel_input.file_input.value
         panel_ppp.upload_time = datetime.now(timezone.utc)
         panel_ppp.secret_token = panel_input.secret_token
 
+        if panel_ppp.status_ == 2:
+            ppc_status_ = "user"
+        elif panel_ppp.status_ == 0:
+            ppc_status_ = "skip"
+        else:
+            ppc_status_ = "auto"
+
         outdir, outfile_zip, _ = panel_ppp.upload(
             outdir_prefix=config["OUTPUT_DIR"],
             single_exptime=panel_obs_type.single_exptime.value,
             observation_type=panel_obs_type.obs_type.value,
+            ppc_status=ppc_status_,
         )
 
         try:
@@ -596,6 +608,8 @@ def list_files_app(use_panel_cli=False):
 
     logger.info(f"config params from dotenv: {config}")
 
+    panel_targets = TargetWidgets()
+
     if not os.path.exists(config["OUTPUT_DIR"]):
         logger.error(f"{config['OUTPUT_DIR']} not found")
         raise ValueError
@@ -660,24 +674,7 @@ def list_files_app(use_panel_cli=False):
         step=1,
     )
 
-    # setup panel components
-
     # Target & psl summary table
-
-    """def execute_javascript(script):
-        script = f'<script type="text/javascript">{script}</script>'
-        js_panel.object = script
-        js_panel.object = ""
-
-    def open_panel_download(event):
-        if event.column == "download":
-            p_href = df_files_tgt["fullpath"][event.row].replace(
-                config["OUTPUT_DIR"], "data", 1
-            )
-            # c.f. https://www.w3schools.com/jsref/met_win_open.asp
-            script = f"window.open('{p_href}', '_blank')"
-            execute_javascript(script)#"""
-
     def Table_files_tgt_psl(column_checkbox_):
         if psl_info_input.value is not None:
             df_psl_info = load_input(
@@ -743,8 +740,12 @@ def execute_javascript(script):
         def open_panel_download(event):
             if event.column == "download":
                 href = df_files_tgt_psl["fullpath_tgt"][event.row]
+                href_mod = href.replace(config["OUTPUT_DIR"], "data", 1)
+                logger.info(f"{href=}")
+                logger.info(f"{href_mod=}")
+
                 # c.f. https://www.w3schools.com/jsref/met_win_open.asp
-                script = f"window.open('{href}', '_blank')"
+                script = f"window.open('{href_mod}', '_blank')"
                 execute_javascript(script)
 
         def open_panel_magnify(event):
@@ -753,7 +754,7 @@ def open_panel_magnify(event):
                 table_ppc.clear()
 
                 # move to "PPC details" tab
-                tab_panels.active = 1
+                tab_panels.active = 2
 
                 u_id = _df_files_tgt_psl["Upload ID"][row_target]
                 p_ppc = os.path.split(_df_files_tgt_psl["fullpath_psl"][row_target])[0]
@@ -772,6 +773,8 @@ def open_panel_magnify(event):
                 except FileNotFoundError:
                     table_tac_t = Table()
 
+                panel_targets.show_results(Table.to_pandas(table_tgt_t))
+
                 (
                     nppc_fin,
                     p_result_fig_fin,
@@ -793,7 +796,6 @@ def open_panel_magnify(event):
                     raise ValueError
 
                 path_t_server = path_t_all[0]
-                tac_ppc_list_file_server = f"{path_t_server}/TAC_ppc_{u_id}.ecsv"
 
                 path_t = path_t_server.replace(config["OUTPUT_DIR"], "data", 1)
                 tac_ppc_list_file = f"{path_t}/TAC_ppc_{u_id}.ecsv"
@@ -962,8 +964,8 @@ def tab_ppc_save(event):
             "timestamp",
             "TAC_FH_L",
             "TAC_FH_M",
-            "TAC_nppc_L",
-            "TAC_nppc_M",
+            "observation_type",
+            "pointing_status",
         ],
         options=list(df_files_tgt_psl.columns)
         + ["proposal ID", "PI name", "rank", "grade"],
@@ -1013,6 +1015,7 @@ def tab_ppc_save(event):
                 js_panel,
             ),
         ),
+        ("Target list", panel_targets.pane),
         ("PPC details", table_ppc),
     )
 

src/pfs_target_uploader/utils/io.py

@@ -129,14 +129,17 @@ def upload_file(
     ppp_fig,
     outdir_prefix=".",
     origname="example.csv",
+    origname_ppc=None,
     origdata=None,
+    origdata_ppc=None,
     secret_token=None,
     upload_time=None,
     ppp_status=True,
     export=False,
     skip_subdirectories=False,
     single_exptime=900,
     observation_type="queue",
+    ppc_status="auto",
 ):
     # use the current UTC time and random hash string to construct an output filename
     if upload_time is None:
@@ -221,41 +224,61 @@ def upload_file(
     }
 
     for file_prefix, obj, type in zip(
-        ["target", "target_summary", "psl", "ppc", "ppp_figure", "", ""],
+        ["target", "target_summary", "psl", "ppc", "ppp_figure", "", "", ""],
         [
             tb_target,
             tb_target_summary,
             tb_psl,
             tb_ppc,
             ppp_fig,
             origdata,
+            origdata_ppc,
             generate_readme_text(),
         ],
-        ["table", "table", "table", "table", "figure", "original", "readme"],
+        [
+            "table",
+            "table",
+            "table",
+            "table",
+            "figure",
+            "original",
+            "original_ppc",
+            "readme",
+        ],
     ):
         logger.info(f"Adding metadata to {file_prefix} file")
         if type == "table":
             # add metadata
             obj.meta["original_filename"] = origname
+            obj.meta["original_filename_ppc"] = origname_ppc
+            obj.meta["ppp_status"] = ppp_status
+            obj.meta["single_exptime"] = single_exptime
+            obj.meta["observation_type"] = observation_type
+            obj.meta["ppc_status"] = ppc_status
             if not export:
+                # add upload-related metadata
                 obj.meta["upload_id"] = secret_token
                 obj.meta["upload_at"] = upload_time
-                obj.meta["ppp_status"] = ppp_status
-                obj.meta["single_exptime"] = single_exptime
-                obj.meta["observation_type"] = observation_type
             filename = f"{file_prefix}_{secret_token}.ecsv"
         elif type == "figure":
             filename = f"{file_prefix}_{secret_token}.html"
         elif type == "original":
             filename = origname
+        elif type == "original_ppc":
+            filename = origname_ppc
         elif type == "readme":
             filename = "README.txt"
 
         outfiles_dict["filename"].append(filename)
         outfiles_dict["object"].append(obj)
         outfiles_dict["type"].append(type)
-        outfiles_dict["absname"].append(os.path.join(outdir, filename))
-        outfiles_dict["arcname"].append(os.path.join(outfile_zip_prefix, filename))
+
+        outfiles_dict["absname"].append(
+            os.path.join(outdir, filename) if filename is not None else None
+        )
+        outfiles_dict["arcname"].append(
+            os.path.join(outfile_zip_prefix, filename) if filename is not None else None
+        )
 
     outdir, outfile_zip, sio = upload_write(
         outfiles_dict, outfile_zip_prefix, outdir, export=export
@@ -275,6 +298,8 @@ def upload_write(outfiles_dict, outfile_zip_prefix, outdir, export=False):
             if export:
                 dest = StringIO()
             else:
+                if outfiles_dict["filename"][i] is None:
+                    continue
                 dest = os.path.join(outdir, outfiles_dict["filename"][i])
 
             if outfiles_dict["type"][i] == "table":
@@ -304,14 +329,16 @@ def upload_write(outfiles_dict, outfile_zip_prefix, outdir, export=False):
             arcname = outfiles_dict["arcname"][i]
 
             if export:
-                if outfiles_dict["type"][i] == "original":
-                    zipfile.writestr(arcname, outfiles_dict["object"][i])
+                if outfiles_dict["type"][i] in ["original", "original_ppc"]:
+                    if outfiles_dict["filename"][i] is not None:
+                        zipfile.writestr(arcname, outfiles_dict["object"][i])
                 else:
                     zipfile.writestr(arcname, dest.getvalue())
             else:
-                if outfiles_dict["type"][i] == "original":
-                    with open(dest, "wb") as f:
-                        f.write(outfiles_dict["object"][i])
+                if outfiles_dict["type"][i] in ["original", "original_ppc"]:
+                    if outfiles_dict["filename"][i] is not None:
+                        with open(dest, "wb") as f:
+                            f.write(outfiles_dict["object"][i])
                 zipfile.write(absname, arcname=outfiles_dict["arcname"][i])
 
             logger.info(f"File {outfiles_dict['filename'][i]} is saved under {outdir}.")
@@ -329,6 +356,7 @@ def load_file_properties(datadir, ext="ecsv", n_uid=16):
     n_files = len(dirs)
 
     orignames = np.full(n_files, None, dtype=object)
+    orignames_ppc = np.full(n_files, None, dtype=object)
     upload_ids = np.full(n_files, None, dtype=object)
     timestamps = np.full(n_files, None, dtype="datetime64[s]")
     filesizes = np.zeros(n_files, dtype=float)
@@ -353,6 +381,7 @@ def load_file_properties(datadir, ext="ecsv", n_uid=16):
     tac_rot_m = np.zeros(n_files, dtype=float)
     single_exptime = np.full(n_files, 900, dtype=int)
     observation_type = np.full(n_files, None, dtype=object)
+    ppc_status = np.full(n_files, None, dtype=object)
 
     for i, d in enumerate(dirs):
         uid = d[-n_uid:]
@@ -388,14 +417,14 @@ def load_file_properties(datadir, ext="ecsv", n_uid=16):
                 orignames[i] = None
 
             try:
-                upload_ids[i] = tb_target.meta["upload_id"]
+                orignames_ppc[i] = tb_target.meta["original_filename_ppc"]
             except KeyError:
-                upload_ids[i] = None
+                orignames_ppc[i] = None
 
             try:
-                observation_type[i] = tb_target.meta["observation_type"]
+                upload_ids[i] = tb_target.meta["upload_id"]
             except KeyError:
-                observation_type[i] = None
+                upload_ids[i] = None
 
             try:
                 if isinstance(tb_target.meta["upload_at"], str):
@@ -410,6 +439,16 @@ def load_file_properties(datadir, ext="ecsv", n_uid=16):
             except KeyError:
                 pass
 
+            try:
+                observation_type[i] = tb_target.meta["observation_type"]
+            except KeyError:
+                observation_type[i] = None
+
+            try:
+                ppc_status[i] = tb_target.meta["ppc_status"]
+            except KeyError:
+                ppc_status[i] = None
+
             n_obj[i] = tb_target["ob_code"].size
             t_exp[i] = np.sum(tb_target["exptime"]) / 3600.0
 
@@ -474,6 +513,8 @@ def load_file_properties(datadir, ext="ecsv", n_uid=16):
             "fullpath_psl": fullpath_psl,
             "single_exptime": single_exptime,
             "observation_type": observation_type,
+            "pointing_status": ppc_status,
+            "Filename_pointing": orignames_ppc,
         }
     )