diff --git a/doc/source/user/projectconf.rst b/doc/source/user/projectconf.rst index ab14722c..f9017fbe 100644 --- a/doc/source/user/projectconf.rst +++ b/doc/source/user/projectconf.rst @@ -239,28 +239,29 @@ Project specs .. _projectconf general: -General Settings -================ +General Settings - Environment variables +======================================== Aside from the project specific configuration, a few options can also be defined in general. There are two ways to set these options: * set the value in the ``~/.jfremote.yaml`` configuration file. -* export the variable name prepended by the ``jfremote`` prefix:: +* set an environment variable composed by the name of the variable and + prepended by the ``JFREMOTE_`` prefix:: - export jfremote_project=project_name + export JFREMOTE_PROJECT=project_name .. note:: - The name of the exported variables is case-insensitive (i.e. JFREMOTE_PROJECT + The name of the exported variables is case-insensitive (i.e. jfremote_project is equally valid). The most useful variable to set is the ``project`` one, allowing to select the default project to be used in a multi-project environment. Other generic options are the location of the projects folder, instead of -``~/.jfremote`` (``projects_folder``) and the path to the ``~/.jfremote.yaml`` -file itself (``config_file``). +``~/.jfremote`` (``JFREMOTE_PROJECT_FOLDER``) and the path to the ``~/.jfremote.yaml`` +file itself (``JFREMOTE_CONFIG_FILE``). Some customization options are also available for the behaviour of the CLI. For more details see the API documentation :py:class:`jobflow_remote.config.settings.JobflowRemoteSettings`. diff --git a/src/jobflow_remote/cli/admin.py b/src/jobflow_remote/cli/admin.py index 08073572..c6d6837d 100644 --- a/src/jobflow_remote/cli/admin.py +++ b/src/jobflow_remote/cli/admin.py @@ -39,6 +39,16 @@ @app_admin.command() def reset( + validation: Annotated[ + Optional[str], + typer.Argument( + help=( + "If the number of flows in the DB exceed 25 it will be required to pass " + "today's date in the YYYY-MM-DD to proceed with the reset" + ), + metavar="DATE", + ), + ] = None, reset_output: Annotated[ bool, typer.Option( @@ -47,16 +57,6 @@ def reset( help="Also delete all the documents in the current store", ), ] = False, - max_limit: Annotated[ - int, - typer.Option( - "--max", - "-m", - help=( - "The database will be reset only if the number of Flows is lower than the specified limit. 0 means no limit" - ), - ), - ] = 25, force: force_opt = False, ) -> None: """ @@ -81,45 +81,16 @@ def reset( with loading_spinner(processing=False) as progress: progress.add_task(description="Resetting the DB...", total=None) jc = get_job_controller() - done = jc.reset(reset_output=reset_output, max_limit=max_limit) + done = jc.reset(reset_output=reset_output, max_limit=25, validation=validation) not_text = "" if done else "[bold]NOT [/bold]" out_console.print(f"The database was {not_text}reset") if not done and SETTINGS.cli_suggestions: out_console.print( - "Check the amount of Flows and change --max-limit if this is the correct project to reset", + "Check the amount of Flows and set the DATE argument if this is the correct project to reset", style="yellow", ) -@app_admin.command(hidden=True) -def remove_lock( - job_id: job_ids_indexes_opt = None, - db_id: db_ids_opt = None, - state: job_state_opt = None, - start_date: start_date_opt = None, - end_date: end_date_opt = None, - force: force_opt = False, -) -> None: - """ - DEPRECATED: use unlock instead - Forcibly removes the lock from the documents of the selected jobs. - WARNING: can lead to inconsistencies if the processes is actually running. - """ - out_console.print( - "remove-lock command has been DEPRECATED. Use unlock instead.", - style="bold yellow", - ) - - unlock( - job_id=job_id, - db_id=db_id, - state=state, - start_date=start_date, - end_date=end_date, - force=force, - ) - - @app_admin.command() def unlock( job_id: job_ids_indexes_opt = None, diff --git a/src/jobflow_remote/jobs/jobcontroller.py b/src/jobflow_remote/jobs/jobcontroller.py index 22ab0062..1fad2ea1 100644 --- a/src/jobflow_remote/jobs/jobcontroller.py +++ b/src/jobflow_remote/jobs/jobcontroller.py @@ -2516,7 +2516,12 @@ def unlock_flows( ) return result.modified_count - def reset(self, reset_output: bool = False, max_limit: int = 25) -> bool: + def reset( + self, + reset_output: bool = False, + max_limit: int = 25, + validation: str | None = None, + ) -> bool: """ Reset the content of the queue database and builds the indexes. Optionally deletes the content of the JobStore with the outputs. @@ -2529,8 +2534,13 @@ def reset(self, reset_output: bool = False, max_limit: int = 25) -> bool: If True also reset the JobStore containing the outputs. max_limit Maximum number of Flows present in the DB. If number is larger - the database will not be reset. Set 0 for not limit. - + the database a validation should be passed. Set 0 for not limit. + Setting max_limit to a large number or 0 will always lead to a + reset of the DB without validation. Prefer setting the password + to avoid unwanted deletions. + validation + A string representing today's date in the format YYYY-MM-DD. + Required if the number of Flows to delete exceed max_limit. Returns ------- bool @@ -2540,10 +2550,12 @@ def reset(self, reset_output: bool = False, max_limit: int = 25) -> bool: # what if the outputs are in other stores? Should take those as well if max_limit: n_flows = self.flows.count_documents({}) - if n_flows >= max_limit: + today = datetime.now().strftime("%Y-%m-%d") + if n_flows >= max_limit and today != validation: logger.warning( f"The database contains {n_flows} flows and will not be reset. " - "Increase the max_limit value or set it to 0" + "Pass today's date in the YYYY-MM-DD format to validate the reset " + "or change the max_limit value." ) return False diff --git a/tests/db/cli/test_admin.py b/tests/db/cli/test_admin.py index 0149fd45..766c068b 100644 --- a/tests/db/cli/test_admin.py +++ b/tests/db/cli/test_admin.py @@ -1,21 +1,41 @@ import pytest -def test_reset(job_controller, two_flows_four_jobs) -> None: +def test_reset(job_controller, one_job) -> None: + from datetime import datetime + + from jobflow import Flow + + from jobflow_remote import submit_flow + from jobflow_remote.testing import add from jobflow_remote.testing.cli import run_check_cli run_check_cli( - ["admin", "reset", "-m", "1"], + ["admin", "reset"], + required_out="The database was reset", + cli_input="y", + ) + assert job_controller.count_jobs() == 0 + + for _ in range(26): + f = Flow(add(1, 2)) + submit_flow(f, worker="test_local_worker") + + run_check_cli( + ["admin", "reset"], required_out="The database was NOT reset", cli_input="y", ) - assert job_controller.count_jobs() == 4 + assert job_controller.count_jobs() == 26 + + run_check_cli(["admin", "reset"], cli_input="n") + assert job_controller.count_jobs() == 26 - run_check_cli(["admin", "reset", "-m", "10"], cli_input="n") - assert job_controller.count_jobs() == 4 + run_check_cli(["admin", "reset", "1220-01-01"], cli_input="y") + assert job_controller.count_jobs() == 26 run_check_cli( - ["admin", "reset", "-m", "10"], + ["admin", "reset", datetime.now().strftime("%Y-%m-%d")], required_out="The database was reset", cli_input="y", ) diff --git a/tests/db/conftest.py b/tests/db/conftest.py index b91d2219..d745a96e 100644 --- a/tests/db/conftest.py +++ b/tests/db/conftest.py @@ -164,7 +164,7 @@ def job_controller(random_project_name): from jobflow_remote.jobs.jobcontroller import JobController jc = JobController.from_project_name(random_project_name) - assert jc.reset() + assert jc.reset(max_limit=0) return jc diff --git a/tests/db/jobs/test_jobcontroller.py b/tests/db/jobs/test_jobcontroller.py index baf919e3..d0b2c860 100644 --- a/tests/db/jobs/test_jobcontroller.py +++ b/tests/db/jobs/test_jobcontroller.py @@ -558,13 +558,33 @@ def test_set_job_doc_properties(job_controller, one_job) -> None: def test_reset(job_controller, two_flows_four_jobs): + from datetime import datetime + + from jobflow import Flow + + from jobflow_remote import submit_flow + from jobflow_remote.testing import add + assert job_controller.count_jobs() == 4 assert not job_controller.reset(max_limit=1) + assert job_controller.count_jobs() == 4 assert job_controller.reset(max_limit=10, reset_output=True) assert job_controller.count_jobs() == 0 + for _ in range(4): + f = Flow(add(1, 2)) + submit_flow(f, worker="test_local_worker") + + assert job_controller.count_jobs() == 4 + assert not job_controller.reset(max_limit=1, validation="1327-01-01") + assert job_controller.count_jobs() == 4 + assert job_controller.reset( + max_limit=1, validation=datetime.now().strftime("%Y-%m-%d") + ) + assert job_controller.count_jobs() == 0 + def test_delete_job(job_controller, two_flows_four_jobs, runner): from jobflow import Flow diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index da7f9351..d305315d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -286,5 +286,5 @@ def job_controller(random_project_name): from jobflow_remote.jobs.jobcontroller import JobController jc = JobController.from_project_name(random_project_name) - assert jc.reset() + assert jc.reset(max_limit=0) return jc