From 320f8f48369621adca8f2e68de6293e347165838 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 22 Jan 2025 11:59:48 +0000 Subject: [PATCH] cat-log: list log files available via tailer/viewer command * The `--mode=list-dir` option lists log files which are available to view. * This commit includes remote `job.out` and `job.err` files which are available via platform tailer/viewer commands. * Note, these log files are only listed if accessible via *both* a tailer and a viewer in order to prevent ambiguity (these are typically configured together anyway). --- changes.d/6480.fix.md | 1 + cylc/flow/scripts/cat_log.py | 49 ++++++++++---- .../cylc-cat-log/12-remote-out-err-tailer.t | 64 +++++++++++++++++++ .../12-remote-out-err-tailer/flow.cylc | 25 ++++++++ 4 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 changes.d/6480.fix.md create mode 100644 tests/functional/cylc-cat-log/12-remote-out-err-tailer.t create mode 100644 tests/functional/cylc-cat-log/12-remote-out-err-tailer/flow.cylc diff --git a/changes.d/6480.fix.md b/changes.d/6480.fix.md new file mode 100644 index 00000000000..064c379b17e --- /dev/null +++ b/changes.d/6480.fix.md @@ -0,0 +1 @@ +cat-log: List log files which are availably via a configured tailer/viewer command. diff --git a/cylc/flow/scripts/cat_log.py b/cylc/flow/scripts/cat_log.py index ec652287044..f7a12740416 100755 --- a/cylc/flow/scripts/cat_log.py +++ b/cylc/flow/scripts/cat_log.py @@ -596,28 +596,49 @@ def _main( cmd.append('--prepend-path') cmd.append(workflow_id) # TODO: Add Intelligent Host selection to this + proc = None with suppress(KeyboardInterrupt): # (Ctrl-C while tailing) # NOTE: This will raise NoHostsError if the platform is not # contactable - remote_cylc_cmd( + proc = remote_cylc_cmd( cmd, platform, - capture_process=False, + capture_process=(mode == 'list-dir'), manage=(mode == 'tail'), - text=False + text=(mode == 'list-dir'), ) - if ( - mode == 'list-dir' - and os.path.exists( - os.path.join( - local_log_dir, - 'job-activity.log' - ) - ) - ): - # add the local-only job-activity.log file to the remote-list - print('job-activity.log') + + # add and missing items to file listing results + if isinstance(proc, Popen): + # i.e: if mode=='list-dir' and ctrl+c not pressed + out, err = proc.communicate() + files = out.splitlines() + + # add files which can be accessed via a tailer + if live_job_id is not None: + if ( + # NOTE: only list the file if it can be viewed in + # both modes + (platform['out tailer'] and platform['out viewer']) + and 'job.out' not in files + ): + files.append('job.out') + if ( + (platform['err tailer'] and platform['err viewer']) + and 'job.err' not in files + ): + files.append('job.err') + + # add the job-activity.log file which is always local + if os.path.exists( + os.path.join(local_log_dir, 'job-activity.log') + ): + files.append('job-activity.log') + + files.sort() + print('\n'.join(files)) + else: # Local task job or local job log. logpath = os.path.join(local_log_dir, options.filename) diff --git a/tests/functional/cylc-cat-log/12-remote-out-err-tailer.t b/tests/functional/cylc-cat-log/12-remote-out-err-tailer.t new file mode 100644 index 00000000000..001bdd1c6de --- /dev/null +++ b/tests/functional/cylc-cat-log/12-remote-out-err-tailer.t @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test "cylc cat-log" with custom out/err tailers +export REQUIRE_PLATFORM='loc:remote runner:background fs:indep comms:tcp' +. "$(dirname "$0")/test_header" +#------------------------------------------------------------------------------- +set_test_number 9 +create_test_global_config "" " +[platforms] + [[$CYLC_TEST_PLATFORM]] + out tailer = echo OUT + err tailer = echo ERR + out viewer = echo OUT + err viewer = echo ERR + " +#------------------------------------------------------------------------------- +# run the workflow +TEST_NAME="${TEST_NAME_BASE}-validate" +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" +run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" +workflow_run_ok "${TEST_NAME_BASE}-run" cylc play -N "${WORKFLOW_NAME}" +#------------------------------------------------------------------------------- +# change the platform the task ran on to the remote platform +# sqlite3 \ +# "${HOME}/cylc-run/${WORKFLOW_NAME}/.service/db" \ +# 'SELECT * FROM task_jobs;' >&2 +sqlite3 \ + "${HOME}/cylc-run/${WORKFLOW_NAME}/log/db" \ + "UPDATE task_jobs SET platform_name = '${CYLC_TEST_PLATFORM}' WHERE name = 'foo' and cycle = '1';" +# sqlite3 \ +# "${HOME}/cylc-run/${WORKFLOW_NAME}/.service/db" \ +# 'SELECT * FROM task_jobs;' >&2 +#------------------------------------------------------------------------------- +# test cylc cat-log --mode=list-dir lists the tailed files +TEST_NAME="${TEST_NAME_BASE}-list-dir" +run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -m 'list-dir' +# the job.out and job.err filees +grep_ok "job.out" "${TEST_NAME}.stdout" +grep_ok "job.err" "${TEST_NAME}.stdout" +#------------------------------------------------------------------------------- +# test cylc cat-log runs the custom tailers +TEST_NAME="${TEST_NAME_BASE}-cat-out" +run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m t +grep_ok "OUT" "${TEST_NAME}.stdout" +run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f e -m t +grep_ok "ERR" "${TEST_NAME}.stdout" +#------------------------------------------------------------------------------- +purge +exit diff --git a/tests/functional/cylc-cat-log/12-remote-out-err-tailer/flow.cylc b/tests/functional/cylc-cat-log/12-remote-out-err-tailer/flow.cylc new file mode 100644 index 00000000000..21113656994 --- /dev/null +++ b/tests/functional/cylc-cat-log/12-remote-out-err-tailer/flow.cylc @@ -0,0 +1,25 @@ +[scheduler] + [[events]] + abort on stall timeout = True + stall timeout = PT2M + +[scheduling] + [[graph]] + R1 = foo + +[runtime] + [[foo]] + script = """ + # wait for the started message to be received + cylc__job__poll_grep_workflow_log -E 'foo.*running' + + # remove the out/err files + rm "${CYLC_TASK_LOG_DIR}/job.out" + rm "${CYLC_TASK_LOG_DIR}/job.err" + + # stop the workflow, orphaning this job + cylc stop --now --now "${CYLC_WORKFLOW_ID}" 2>/dev/null >/dev/null + + # suppress any subsequent messages + rm "${CYLC_WORKFLOW_RUN_DIR}/.service/contact" + """