From 0efd0a065cb19a686b6f237d50cb5ad577913fc7 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Tue, 13 Jun 2023 10:58:09 -0400 Subject: [PATCH 1/3] ec2 architect uninterruptable shutdown --- mephisto/abstractions/architects/ec2/ec2_architect.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mephisto/abstractions/architects/ec2/ec2_architect.py b/mephisto/abstractions/architects/ec2/ec2_architect.py index 545102fa1..94b7ce9dd 100644 --- a/mephisto/abstractions/architects/ec2/ec2_architect.py +++ b/mephisto/abstractions/architects/ec2/ec2_architect.py @@ -360,4 +360,12 @@ def shutdown(self) -> None: in the db. """ if self.created: # only delete the server if it's created by us + def cant_cancel_shutdown(sig, frame): + logger.warn( + "Ignoring ^C during ec2 cleanup. ^| if you NEED to exit and you will " + "have to clean up this server with cleanup_ec2_server_by_name.py after." + ) + + old_handler = signal.signal(signal.SIGINT, cant_cancel_shutdown) self.__delete_ec2_server() + signal.signal(signal.SIGINT, old_handler) From 4f72d05926ba421d4a33f8d82e6c3c004bd10710 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Mon, 26 Jun 2023 14:36:32 -0400 Subject: [PATCH 2/3] adding handler --- .../abstractions/architects/ec2/ec2_architect.py | 3 ++- mephisto/abstractions/architects/ec2/ec2_helpers.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mephisto/abstractions/architects/ec2/ec2_architect.py b/mephisto/abstractions/architects/ec2/ec2_architect.py index 94b7ce9dd..a3bf8453c 100644 --- a/mephisto/abstractions/architects/ec2/ec2_architect.py +++ b/mephisto/abstractions/architects/ec2/ec2_architect.py @@ -7,7 +7,7 @@ import os import sh # type: ignore import shutil -import time +import signal import requests import re import json @@ -360,6 +360,7 @@ def shutdown(self) -> None: in the db. """ if self.created: # only delete the server if it's created by us + def cant_cancel_shutdown(sig, frame): logger.warn( "Ignoring ^C during ec2 cleanup. ^| if you NEED to exit and you will " diff --git a/mephisto/abstractions/architects/ec2/ec2_helpers.py b/mephisto/abstractions/architects/ec2/ec2_helpers.py index 24e056e8f..8b6caa603 100644 --- a/mephisto/abstractions/architects/ec2/ec2_helpers.py +++ b/mephisto/abstractions/architects/ec2/ec2_helpers.py @@ -13,6 +13,7 @@ import json import getpass import hashlib +import signal from mephisto.abstractions.providers.mturk.mturk_utils import setup_aws_credentials from mephisto.abstractions.architects.router import build_router @@ -45,6 +46,13 @@ MAX_RETRIES = 10 +def cant_cancel_shutdown(sig, frame): + logger.warn( + "Ignoring ^C during ec2 cleanup. ^| if you NEED to exit and you will " + "have to clean up resources yourself from AWS." + ) + + def get_owner_tag() -> Dict[str, str]: """ Creates a tag with the user's username @@ -962,7 +970,9 @@ def deploy_fallback_server( ) detete_instance_address(session, allocation_id, association_id) except Exception as e: + old_handler = signal.signal(signal.SIGINT, cant_cancel_shutdown) detete_instance_address(session, allocation_id, association_id) + signal.signal(signal.SIGINT, old_handler) raise e return True @@ -1011,7 +1021,9 @@ def deploy_to_routing_server( detete_instance_address(session, allocation_id, association_id) print("Server setup complete!") except Exception as e: + old_handler = signal.signal(signal.SIGINT, cant_cancel_shutdown) detete_instance_address(session, allocation_id, association_id) + signal.signal(signal.SIGINT, old_handler) raise e return True From 13ad2affb4e98a2baec08bed8226659e36904bab Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Tue, 27 Jun 2023 11:57:54 -0400 Subject: [PATCH 3/3] Include catching ctrl-C --- mephisto/abstractions/architects/ec2/ec2_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mephisto/abstractions/architects/ec2/ec2_helpers.py b/mephisto/abstractions/architects/ec2/ec2_helpers.py index 8b6caa603..bbbbffef5 100644 --- a/mephisto/abstractions/architects/ec2/ec2_helpers.py +++ b/mephisto/abstractions/architects/ec2/ec2_helpers.py @@ -969,7 +969,7 @@ def deploy_fallback_server( env=dict(os.environ, SSH_AUTH_SOCK=""), ) detete_instance_address(session, allocation_id, association_id) - except Exception as e: + except (Exception, KeyboardInterrupt) as e: old_handler = signal.signal(signal.SIGINT, cant_cancel_shutdown) detete_instance_address(session, allocation_id, association_id) signal.signal(signal.SIGINT, old_handler) @@ -1020,7 +1020,7 @@ def deploy_to_routing_server( ) detete_instance_address(session, allocation_id, association_id) print("Server setup complete!") - except Exception as e: + except (Exception, KeyboardInterrupt) as e: old_handler = signal.signal(signal.SIGINT, cant_cancel_shutdown) detete_instance_address(session, allocation_id, association_id) signal.signal(signal.SIGINT, old_handler)