diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e6a4991b6e..29ebb59719 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,21 @@ in development Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8. +Several st2.conf database options have been renamed or deprecated. Most of the options will continue to work using their old name. +However, if you use `[database].ssl_keyfile` and/or `[database].ssl_certfile`, you MUST migrate to `[database].tls_certificate_key_file`. +This new option expects the key and certificate in the same file. Use something like the following to create that file from your old files: + +``` +cat path/to/ssl_keyfile path/to/ssl_certfile > path/to/tls_certificate_key_file +``` + +Other options that were renamed under `[database]` are (more details available in `st2.conf.sample`): + +* `ssl` -> `tls` +* `ssl_cert_reqs` -> `tls_allow_invalid_certificates` (opt type change: string -> boolean) +* `ssl_ca_certs` -> `tls_ca_file` +* `ssl_match_hostnames` -> `tls_allow_invalid_hostnames` (meaning is inverted: the new option is the opposite of the old) + Fixed ~~~~~ * Fixed #6021 and #5327 by adding max_page_size to api_opts and added limit and offset to list_values() methods of @@ -31,6 +46,11 @@ Changed * Updated unit tests to use redis for coordination instead of the NoOp driver. This will hopefully make CI more stable. #6245 Contributed by @FileMagic, @guzzijones, and @cognifloyd +* Renamed `[database].ssl*` options to support pymongo 4, which we have to update to support newer MongoDB servers. + Please see the note above about migrating to the newer options, especially if you use `[database].ssl_keyfile` + and/or `[database].ssl_certfile`, as those options are ignored in StackStorm 3.9.0. #6250 + Contributed by @cognifloyd + Added ~~~~~ * Continue introducing `pants `_ to improve DX (Developer Experience) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 82da2ae2e1..5eed1ffd4c 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -19,7 +19,7 @@ logging = /etc/st2/logging.actionrunner.conf # List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment. pip_opts = # comma separated list allowed here. # Python binary which will be used by Python actions. -python_binary = /usr/bin/python +python_binary = /usr/bin/python3 # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter. python_runner_log_level = DEBUG # Time interval between subsequent queries to check running executions. @@ -106,11 +106,13 @@ index_url = https://index.stackstorm.org/v1/index.json # comma separated list al pack_group = st2packs # Paths which will be searched for integration packs. packs_base_paths = None -# Paths which will be searched for runners. NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0 +# Paths which will be searched for runners. +# DEPRECATED FOR REMOVAL since 3.0.0: Option unused since StackStorm v3.0.0 runners_base_paths = None # Path to the directory which contains system packs. system_packs_base_path = /opt/stackstorm/packs -# Path to the directory which contains system runners. NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0 +# Path to the directory which contains system runners. +# DEPRECATED FOR REMOVAL since 3.0.0: Option unused since StackStorm v3.0.0 system_runners_base_path = /opt/stackstorm/runners [coordination] @@ -142,19 +144,33 @@ host = 127.0.0.1 password = None # port of db server port = 27017 -# Create the connection to mongodb using SSL -ssl = False -# ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from MongoDB. -ssl_ca_certs = None # Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided +# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_allow_invalid_certificates with the following: The 'optional' and 'required' values are equivalent to tls_allow_invalid_certificates=False. The 'none' value is equivalent to tls_allow_invalid_certificates=True. This option is a needlessly more complex version of tls_allow_invalid_certificates. # Valid values: none, optional, required ssl_cert_reqs = None # Certificate file used to identify the localconnection +# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_certificate_key_file with a path to a file containing the concatenation of the files from ssl_keyfile and ssl_certfile. This option is ignored by pymongo. ssl_certfile = None # Private keyfile used to identify the local connection against MongoDB. +# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_certificate_key_file with a path to a file containing the concatenation of the files from ssl_keyfile and ssl_certfile. This option is ignored by pymongo. ssl_keyfile = None # If True and `ssl_cert_reqs` is not None, enables hostname verification +# DEPRECATED FOR REMOVAL since 3.9.0: Use tls_allow_invalid_hostnames with the opposite value from this option. ssl_match_hostname = True +# Create the connection to mongodb using TLS. +# This option has a deprecated alias: ssl +tls = False +# Specifies whether MongoDB is allowed to pass an invalid certificate. This defaults to False to have security by default. Only temporarily set to True if you need to debug the connection. +tls_allow_invalid_certificates = False +# If True and `tlsAllowInvalidCertificates` is True, disables hostname verification. This defaults to False to have security by default. Only temporarily set to True if you need to debug the connection. +tls_allow_invalid_hostnames = False +# ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from MongoDB. +# This option has a deprecated alias: ssl_ca_certs +tls_ca_file = None +# Client certificate used to identify the local connection against MongoDB. The certificate file must contain one or both of private key and certificate. Supplying separate files for private key (ssl_keyfile) and certificate (ssl_certfile) is no longer supported. If encrypted, pass the password or passphrase in tls_certificate_key_file_password. +tls_certificate_key_file = None +# The password or passphrase to decrypt the file in tls_certificate_key_file. Only set this if tls_certificate_key_file is encrypted. +tls_certificate_key_file_password = None # username for db login username = None # Compression level when compressors is set to zlib. Valid values are -1 to 9. Defaults to 6. diff --git a/contrib/packs/actions/pack_mgmt/unload.py b/contrib/packs/actions/pack_mgmt/unload.py index b26182329d..2d4aaf989d 100644 --- a/contrib/packs/actions/pack_mgmt/unload.py +++ b/contrib/packs/actions/pack_mgmt/unload.py @@ -59,11 +59,12 @@ def initialize(self): cfg.CONF.database.port, username=username, password=password, - ssl=cfg.CONF.database.ssl, - ssl_keyfile=cfg.CONF.database.ssl_keyfile, - ssl_certfile=cfg.CONF.database.ssl_certfile, - ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs, - ssl_ca_certs=cfg.CONF.database.ssl_ca_certs, + tls=cfg.CONF.database.tls, + tls_certificate_key_file=cfg.CONF.database.tls_certificate_key_file, + tls_certificate_key_file_password=cfg.CONF.database.tls_certificate_key_file_password, + tls_allow_invalid_certificates=cfg.CONF.database.tls_allow_invalid_certificates, + tls_ca_file=cfg.CONF.database.tls_ca_file, + ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs, # deprecated authentication_mechanism=cfg.CONF.database.authentication_mechanism, ssl_match_hostname=cfg.CONF.database.ssl_match_hostname, ) diff --git a/pants-plugins/sample_conf/rules.py b/pants-plugins/sample_conf/rules.py index 97195943d5..675b7cf6d8 100644 --- a/pants-plugins/sample_conf/rules.py +++ b/pants-plugins/sample_conf/rules.py @@ -27,7 +27,7 @@ FileContent, Snapshot, ) -from pants.engine.process import FallibleProcessResult +from pants.engine.process import ProcessResult from pants.engine.rules import Get, collect_rules, rule from pants.engine.target import FieldSet from pants.util.logging import LogLevel @@ -64,7 +64,7 @@ async def generate_sample_conf_via_fmt( pex = await Get(VenvPex, PexFromTargetsRequest, subsystem.pex_request()) result = await Get( - FallibleProcessResult, + ProcessResult, VenvPexProcess( pex, description="Regenerating st2.conf.sample", diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index f7e9cdc302..bbf73752b3 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -143,8 +143,10 @@ def register_opts(ignore_errors=False): cfg.StrOpt( "system_runners_base_path", default=system_runners_base_path, - help="Path to the directory which contains system runners. " - "NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0", + help="Path to the directory which contains system runners.", + deprecated_for_removal=True, + deprecated_reason="Option unused since StackStorm v3.0.0", + deprecated_since="3.0.0", ), cfg.StrOpt( "packs_base_paths", @@ -154,8 +156,10 @@ def register_opts(ignore_errors=False): cfg.StrOpt( "runners_base_paths", default=None, - help="Paths which will be searched for runners. " - "NOTE: This option has been deprecated and it's unused since StackStorm v3.0.0", + help="Paths which will be searched for runners.", + deprecated_for_removal=True, + deprecated_reason="Option unused since StackStorm v3.0.0", + deprecated_since="3.0.0", ), cfg.ListOpt( "index_url", @@ -172,6 +176,7 @@ def register_opts(ignore_errors=False): cfg.StrOpt( "webui_base_url", default="https://%s" % socket.getfqdn(), + sample_default="https://localhost", help="Base https URL to access st2 Web UI. This is used to construct history URLs " "that are sent out when chatops is used to kick off executions.", ) @@ -184,7 +189,7 @@ def register_opts(ignore_errors=False): cfg.IntOpt("port", default=27017, help="port of db server"), cfg.StrOpt("db_name", default="st2", help="name of database"), cfg.StrOpt("username", help="username for db login"), - cfg.StrOpt("password", help="password for db login"), + cfg.StrOpt("password", help="password for db login", secret=True), cfg.IntOpt( "connection_timeout", default=3 * 1000, @@ -206,35 +211,108 @@ def register_opts(ignore_errors=False): help="Backoff multiplier (seconds).", ), cfg.BoolOpt( - "ssl", default=False, help="Create the connection to mongodb using SSL" + "tls", + deprecated_name="ssl", + default=False, + help="Create the connection to mongodb using TLS.", + ), + cfg.StrOpt( + "tls_certificate_key_file", + default=None, + help=( + "Client certificate used to identify the local connection against MongoDB. " + "The certificate file must contain one or both of private key and certificate. " + "Supplying separate files for private key (ssl_keyfile) and certificate (ssl_certfile) " + "is no longer supported. " + "If encrypted, pass the password or passphrase in tls_certificate_key_file_password." + ), + ), + cfg.StrOpt( + "tls_certificate_key_file_password", + default=None, + help=( + "The password or passphrase to decrypt the file in tls_certificate_key_file. " + "Only set this if tls_certificate_key_file is encrypted." + ), + secret=True, ), cfg.StrOpt( "ssl_keyfile", default=None, help="Private keyfile used to identify the local connection against MongoDB.", + deprecated_for_removal=True, + deprecated_reason=( + "Use tls_certificate_key_file with a path to a file containing " + "the concatenation of the files from ssl_keyfile and ssl_certfile. " + "This option is ignored by pymongo." + ), + deprecated_since="3.9.0", ), cfg.StrOpt( "ssl_certfile", default=None, help="Certificate file used to identify the localconnection", + deprecated_for_removal=True, + deprecated_reason=( + "Use tls_certificate_key_file with a path to a file containing " + "the concatenation of the files from ssl_keyfile and ssl_certfile. " + "This option is ignored by pymongo. " + ), + deprecated_since="3.9.0", + ), + cfg.BoolOpt( + "tls_allow_invalid_certificates", + default=None, + sample_default=False, + help=( + "Specifies whether MongoDB is allowed to pass an invalid certificate. " + "This defaults to False to have security by default. " + "Only temporarily set to True if you need to debug the connection." + ), ), cfg.StrOpt( "ssl_cert_reqs", default=None, choices=["none", "optional", "required"], - help="Specifies whether a certificate is required from the other side of the " - "connection, and whether it will be validated if provided", + help=( + "Specifies whether a certificate is required from the other side of the " + "connection, and whether it will be validated if provided" + ), + deprecated_for_removal=True, + deprecated_reason=( + "Use tls_allow_invalid_certificates with the following: " + "The 'optional' and 'required' values are equivalent to tls_allow_invalid_certificates=False. " + "The 'none' value is equivalent to tls_allow_invalid_certificates=True. " + "This option is a needlessly more complex version of tls_allow_invalid_certificates." + ), + deprecated_since="3.9.0", ), cfg.StrOpt( - "ssl_ca_certs", + "tls_ca_file", + deprecated_name="ssl_ca_certs", default=None, - help="ca_certs file contains a set of concatenated CA certificates, which are " - "used to validate certificates passed from MongoDB.", + help=( + "ca_certs file contains a set of concatenated CA certificates, which are " + "used to validate certificates passed from MongoDB." + ), + ), + cfg.BoolOpt( + "tls_allow_invalid_hostnames", + default=None, + sample_default=False, + help=( + "If True and `tlsAllowInvalidCertificates` is True, disables hostname verification. " + "This defaults to False to have security by default. " + "Only temporarily set to True if you need to debug the connection." + ), ), cfg.BoolOpt( "ssl_match_hostname", default=True, help="If True and `ssl_cert_reqs` is not None, enables hostname verification", + deprecated_for_removal=True, + deprecated_reason="Use tls_allow_invalid_hostnames with the opposite value from this option.", + deprecated_since="3.9.0", ), cfg.StrOpt( "authentication_mechanism", @@ -460,11 +538,13 @@ def register_opts(ignore_errors=False): cfg.StrOpt( "python_binary", default=default_python_bin_path, + sample_default="/usr/bin/python3", help="Python binary which will be used by Python actions.", ), cfg.StrOpt( "virtualenv_binary", default=default_virtualenv_bin_path, + sample_default="/usr/bin/virtualenv", help="Virtualenv binary which should be used to create pack virtualenvs.", ), cfg.StrOpt( diff --git a/st2common/st2common/database_setup.py b/st2common/st2common/database_setup.py index 2e2e7d2a17..68b7583942 100644 --- a/st2common/st2common/database_setup.py +++ b/st2common/st2common/database_setup.py @@ -36,13 +36,15 @@ def db_config(): "db_port": cfg.CONF.database.port, "username": username, "password": password, - "ssl": cfg.CONF.database.ssl, - "ssl_keyfile": cfg.CONF.database.ssl_keyfile, - "ssl_certfile": cfg.CONF.database.ssl_certfile, - "ssl_cert_reqs": cfg.CONF.database.ssl_cert_reqs, - "ssl_ca_certs": cfg.CONF.database.ssl_ca_certs, + "tls": cfg.CONF.database.tls, + "tls_certificate_key_file": cfg.CONF.database.tls_certificate_key_file, + "tls_certificate_key_file_password": cfg.CONF.database.tls_certificate_key_file_password, + "tls_allow_invalid_certificates": cfg.CONF.database.tls_allow_invalid_certificates, + "tls_ca_file": cfg.CONF.database.tls_ca_file, + "tls_allow_invalid_hostnames": cfg.CONF.database.tls_allow_invalid_hostnames, + "ssl_cert_reqs": cfg.CONF.database.ssl_cert_reqs, # deprecated "authentication_mechanism": cfg.CONF.database.authentication_mechanism, - "ssl_match_hostname": cfg.CONF.database.ssl_match_hostname, + "ssl_match_hostname": cfg.CONF.database.ssl_match_hostname, # deprecated } diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index 1cecf3a247..018a419c17 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -48,7 +48,6 @@ import copy import importlib import traceback -import ssl as ssl_lib import six from oslo_config import cfg @@ -127,13 +126,15 @@ def _db_connect( db_port, username=None, password=None, - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): if "://" in db_host: @@ -161,14 +162,16 @@ def _db_connect( % (db_name, host_string, str(username_string)) ) - ssl_kwargs = _get_ssl_kwargs( - ssl=ssl, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, + tls_kwargs = _get_tls_kwargs( + tls=tls, + tls_certificate_key_file=tls_certificate_key_file, + tls_certificate_key_file_password=tls_certificate_key_file_password, + tls_allow_invalid_certificates=tls_allow_invalid_certificates, + tls_ca_file=tls_ca_file, + tls_allow_invalid_hostnames=tls_allow_invalid_hostnames, + ssl_cert_reqs=ssl_cert_reqs, # deprecated authentication_mechanism=authentication_mechanism, - ssl_match_hostname=ssl_match_hostname, + ssl_match_hostname=ssl_match_hostname, # deprecated ) compressor_kwargs = {} @@ -194,7 +197,7 @@ def _db_connect( password=password, connectTimeoutMS=connection_timeout, serverSelectionTimeoutMS=connection_timeout, - **ssl_kwargs, + **tls_kwargs, **compressor_kwargs, ) @@ -231,13 +234,15 @@ def db_setup( username=None, password=None, ensure_indexes=True, - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): connection = _db_connect( @@ -246,13 +251,15 @@ def db_setup( db_port, username=username, password=password, - ssl=ssl, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, + tls=tls, + tls_certificate_key_file=tls_certificate_key_file, + tls_certificate_key_file_password=tls_certificate_key_file_password, + tls_allow_invalid_certificates=tls_allow_invalid_certificates, + tls_ca_file=tls_ca_file, + tls_allow_invalid_hostnames=tls_allow_invalid_hostnames, + ssl_cert_reqs=ssl_cert_reqs, # deprecated authentication_mechanism=authentication_mechanism, - ssl_match_hostname=ssl_match_hostname, + ssl_match_hostname=ssl_match_hostname, # deprecated ) # Create all the indexes upfront to prevent race-conditions caused by @@ -397,13 +404,15 @@ def db_cleanup( db_port, username=None, password=None, - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): connection = _db_connect( @@ -412,13 +421,15 @@ def db_cleanup( db_port, username=username, password=password, - ssl=ssl, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, + tls=tls, + tls_certificate_key_file=tls_certificate_key_file, + tls_certificate_key_file_password=tls_certificate_key_file_password, + tls_allow_invalid_certificates=tls_allow_invalid_certificates, + tls_ca_file=tls_ca_file, + tls_allow_invalid_hostnames=tls_allow_invalid_hostnames, + ssl_cert_reqs=ssl_cert_reqs, # deprecated authentication_mechanism=authentication_mechanism, - ssl_match_hostname=ssl_match_hostname, + ssl_match_hostname=ssl_match_hostname, # deprecated ) LOG.info( @@ -433,46 +444,54 @@ def db_cleanup( return connection -def _get_ssl_kwargs( - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, +def _get_tls_kwargs( + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): # NOTE: In pymongo 3.9.0 some of the ssl related arguments have been renamed - # https://api.mongodb.com/python/current/changelog.html#changes-in-version-3-9-0 - # Old names still work, but we should eventually update to new argument names. - ssl_kwargs = { - "ssl": ssl, + # Old names stopped working in pymongo 4, so we migrated to the new names in st2 3.9.0. + # https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#renamed-uri-options + tls_kwargs = { + "tls": tls, } - if ssl_keyfile: - ssl_kwargs["ssl"] = True - ssl_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - ssl_kwargs["ssl"] = True - ssl_kwargs["ssl_certfile"] = ssl_certfile - if ssl_cert_reqs: - if ssl_cert_reqs == "none": - ssl_cert_reqs = ssl_lib.CERT_NONE - elif ssl_cert_reqs == "optional": - ssl_cert_reqs = ssl_lib.CERT_OPTIONAL - elif ssl_cert_reqs == "required": - ssl_cert_reqs = ssl_lib.CERT_REQUIRED - ssl_kwargs["ssl_cert_reqs"] = ssl_cert_reqs - if ssl_ca_certs: - ssl_kwargs["ssl"] = True - ssl_kwargs["ssl_ca_certs"] = ssl_ca_certs + # pymongo 4 ignores ssl_keyfile and ssl_certfile, so we do not need to pass them on. + if tls_certificate_key_file: + tls_kwargs["tls"] = True + tls_kwargs["tlsCertificateKeyFile"] = tls_certificate_key_file + if tls_certificate_key_file_password: + tls_kwargs[ + "tlsCertificateKeyFilePassword" + ] = tls_certificate_key_file_password + if tls_allow_invalid_certificates is not None: + tls_kwargs["tlsAllowInvalidCertificates"] = tls_allow_invalid_certificates + elif ssl_cert_reqs: # fall back to old option + # possible values: none, optional, required + # ssl lib docs say 'optional' is the same as 'required' for clients: + # https://docs.python.org/3/library/ssl.html#ssl.CERT_OPTIONAL + tls_kwargs["tlsAllowInvalidCertificates"] = ssl_cert_reqs == "none" + if tls_ca_file: + tls_kwargs["tls"] = True + tls_kwargs["tlsCAFile"] = tls_ca_file if authentication_mechanism: - ssl_kwargs["ssl"] = True - ssl_kwargs["authentication_mechanism"] = authentication_mechanism - if ssl_kwargs.get("ssl", False): - # pass in ssl_match_hostname only if ssl is True. The right default value - # for ssl_match_hostname in almost all cases is True. - ssl_kwargs["ssl_match_hostname"] = ssl_match_hostname - return ssl_kwargs + tls_kwargs["tls"] = True + tls_kwargs["authentication_mechanism"] = authentication_mechanism + if tls_kwargs.get("tls", False): + # pass in tlsAllowInvalidHostname only if tls is True. The right default value + # for tlsAllowInvalidHostname in almost all cases is False. + tls_kwargs["tlsAllowInvalidHostnames"] = ( + tls_allow_invalid_hostnames + if tls_allow_invalid_hostnames is not None + else not ssl_match_hostname + ) + return tls_kwargs class MongoDBAccess(object): diff --git a/st2common/st2common/persistence/cleanup.py b/st2common/st2common/persistence/cleanup.py index 06c48dec86..f633e576c7 100644 --- a/st2common/st2common/persistence/cleanup.py +++ b/st2common/st2common/persistence/cleanup.py @@ -44,13 +44,15 @@ def db_cleanup_with_retry( db_port, username=None, password=None, - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): """ This method is a retry version of db_cleanup. @@ -62,13 +64,15 @@ def db_cleanup_with_retry( db_port, username=username, password=password, - ssl=ssl, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, + tls=tls, + tls_certificate_key_file=tls_certificate_key_file, + tls_certificate_key_file_password=tls_certificate_key_file_password, + tls_allow_invalid_certificates=tls_allow_invalid_certificates, + tls_ca_file=tls_ca_file, + tls_allow_invalid_hostnames=tls_allow_invalid_hostnames, + ssl_cert_reqs=ssl_cert_reqs, # deprecated authentication_mechanism=authentication_mechanism, - ssl_match_hostname=ssl_match_hostname, + ssl_match_hostname=ssl_match_hostname, # deprecated ) diff --git a/st2common/st2common/persistence/db_init.py b/st2common/st2common/persistence/db_init.py index c7c0bd9f83..a8fa2711fe 100644 --- a/st2common/st2common/persistence/db_init.py +++ b/st2common/st2common/persistence/db_init.py @@ -65,13 +65,15 @@ def db_setup_with_retry( username=None, password=None, ensure_indexes=True, - ssl=False, - ssl_keyfile=None, - ssl_certfile=None, - ssl_cert_reqs=None, - ssl_ca_certs=None, + tls=False, + tls_certificate_key_file=None, + tls_certificate_key_file_password=None, + tls_allow_invalid_certificates=None, + tls_ca_file=None, + tls_allow_invalid_hostnames=None, + ssl_cert_reqs=None, # deprecated authentication_mechanism=None, - ssl_match_hostname=True, + ssl_match_hostname=True, # deprecated ): """ This method is a retry version of db_setup. @@ -84,11 +86,13 @@ def db_setup_with_retry( username=username, password=password, ensure_indexes=ensure_indexes, - ssl=ssl, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, + tls=tls, + tls_certificate_key_file=tls_certificate_key_file, + tls_certificate_key_file_password=tls_certificate_key_file_password, + tls_allow_invalid_certificates=tls_allow_invalid_certificates, + tls_ca_file=tls_ca_file, + tls_allow_invalid_hostnames=tls_allow_invalid_hostnames, + ssl_cert_reqs=ssl_cert_reqs, # deprecated authentication_mechanism=authentication_mechanism, - ssl_match_hostname=ssl_match_hostname, + ssl_match_hostname=ssl_match_hostname, # deprecated ) diff --git a/st2common/tests/unit/test_db.py b/st2common/tests/unit/test_db.py index ed6b88649d..89814fdf87 100644 --- a/st2common/tests/unit/test_db.py +++ b/st2common/tests/unit/test_db.py @@ -21,7 +21,6 @@ monkey_patch() -import ssl import time import jsonschema @@ -38,7 +37,7 @@ from st2common.util import schema as util_schema from st2common.util import reference from st2common.models.db import db_setup -from st2common.models.db import _get_ssl_kwargs +from st2common.models.db import _get_tls_kwargs from st2common.util import date as date_utils from st2common.exceptions.db import StackStormDBObjectNotFoundError from st2common.models.db.trigger import TriggerTypeDB, TriggerDB, TriggerInstanceDB @@ -226,85 +225,130 @@ def test_network_level_compression(self): self.assertTrue("compressors=['zlib']" in str(connection)) self.assertTrue("zlibcompressionlevel=9" in str(connection)) - def test_get_ssl_kwargs(self): + def test_get_tls_kwargs(self): # 1. No SSL kwargs provided - ssl_kwargs = _get_ssl_kwargs() - self.assertEqual(ssl_kwargs, {"ssl": False}) + tls_kwargs = _get_tls_kwargs() + self.assertEqual(tls_kwargs, {"tls": False}) - # 2. ssl kwarg provided - ssl_kwargs = _get_ssl_kwargs(ssl=True) - self.assertEqual(ssl_kwargs, {"ssl": True, "ssl_match_hostname": True}) + # 2. tls kwarg provided + tls_kwargs = _get_tls_kwargs(tls=True) + self.assertEqual(tls_kwargs, {"tls": True, "tlsAllowInvalidHostnames": False}) - # 2. authentication_mechanism kwarg provided - ssl_kwargs = _get_ssl_kwargs(authentication_mechanism="MONGODB-X509") + # 3. authentication_mechanism kwarg provided + tls_kwargs = _get_tls_kwargs(authentication_mechanism="MONGODB-X509") self.assertEqual( - ssl_kwargs, + tls_kwargs, { - "ssl": True, - "ssl_match_hostname": True, + "tls": True, + "tlsAllowInvalidHostnames": False, "authentication_mechanism": "MONGODB-X509", }, ) - # 3. ssl_keyfile provided - ssl_kwargs = _get_ssl_kwargs(ssl_keyfile="/tmp/keyfile") + # 4a. tls_certificate_key_file provided + tls_kwargs = _get_tls_kwargs(tls_certificate_key_file="/tmp/keyfile") self.assertEqual( - ssl_kwargs, - {"ssl": True, "ssl_keyfile": "/tmp/keyfile", "ssl_match_hostname": True}, + tls_kwargs, + { + "tls": True, + "tlsCertificateKeyFile": "/tmp/keyfile", + "tlsAllowInvalidHostnames": False, + }, ) - # 4. ssl_certfile provided - ssl_kwargs = _get_ssl_kwargs(ssl_certfile="/tmp/certfile") + # 4b. tls_certificate_key_file_password provided with tls_certificate_key_file + tls_kwargs = _get_tls_kwargs( + tls_certificate_key_file="/tmp/keyfile", + tls_certificate_key_file_password="pass", + ) self.assertEqual( - ssl_kwargs, - {"ssl": True, "ssl_certfile": "/tmp/certfile", "ssl_match_hostname": True}, + tls_kwargs, + { + "tls": True, + "tlsCertificateKeyFile": "/tmp/keyfile", + "tlsCertificateKeyFilePassword": "pass", + "tlsAllowInvalidHostnames": False, + }, ) - # 5. ssl_ca_certs provided - ssl_kwargs = _get_ssl_kwargs(ssl_ca_certs="/tmp/ca_certs") + # 4c. tls_certificate_key_file_password provided without tls_certificate_key_file + tls_kwargs = _get_tls_kwargs(tls_certificate_key_file_password="pass") + self.assertEqual(tls_kwargs, {"tls": False}) + + # 5. tls_ca_file provided + tls_kwargs = _get_tls_kwargs(tls_ca_file="/tmp/ca_certs") self.assertEqual( - ssl_kwargs, - {"ssl": True, "ssl_ca_certs": "/tmp/ca_certs", "ssl_match_hostname": True}, + tls_kwargs, + { + "tls": True, + "tlsCAFile": "/tmp/ca_certs", + "tlsAllowInvalidHostnames": False, + }, ) - # 6. ssl_ca_certs and ssl_cert_reqs combinations - ssl_kwargs = _get_ssl_kwargs(ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="none") + # 6. tls_ca_file and ssl_cert_reqs combinations + tls_kwargs = _get_tls_kwargs(tls_ca_file="/tmp/ca_certs", ssl_cert_reqs="none") self.assertEqual( - ssl_kwargs, + tls_kwargs, { - "ssl": True, - "ssl_ca_certs": "/tmp/ca_certs", - "ssl_cert_reqs": ssl.CERT_NONE, - "ssl_match_hostname": True, + "tls": True, + "tlsCAFile": "/tmp/ca_certs", + "tlsAllowInvalidCertificates": True, + "tlsAllowInvalidHostnames": False, }, ) - ssl_kwargs = _get_ssl_kwargs( - ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="optional" + tls_kwargs = _get_tls_kwargs( + tls_ca_file="/tmp/ca_certs", ssl_cert_reqs="optional" ) self.assertEqual( - ssl_kwargs, + tls_kwargs, { - "ssl": True, - "ssl_ca_certs": "/tmp/ca_certs", - "ssl_cert_reqs": ssl.CERT_OPTIONAL, - "ssl_match_hostname": True, + "tls": True, + "tlsCAFile": "/tmp/ca_certs", + "tlsAllowInvalidCertificates": False, + "tlsAllowInvalidHostnames": False, }, ) - ssl_kwargs = _get_ssl_kwargs( - ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="required" + tls_kwargs = _get_tls_kwargs( + tls_ca_file="/tmp/ca_certs", ssl_cert_reqs="required" ) self.assertEqual( - ssl_kwargs, + tls_kwargs, { - "ssl": True, - "ssl_ca_certs": "/tmp/ca_certs", - "ssl_cert_reqs": ssl.CERT_REQUIRED, - "ssl_match_hostname": True, + "tls": True, + "tlsCAFile": "/tmp/ca_certs", + "tlsAllowInvalidCertificates": False, + "tlsAllowInvalidHostnames": False, }, ) + # 7. tls_allow_invalid_certificates provided (does not implicitly enable tls) + for allow_invalid in (True, False): + tls_kwargs = _get_tls_kwargs(tls_allow_invalid_certificates=allow_invalid) + self.assertEqual( + tls_kwargs, + { + "tls": False, + "tlsAllowInvalidCertificates": allow_invalid, + }, + ) + + # make sure ssl_cert_reqs is ignored if tls_allow_invalid_certificates is set + for ssl_cert_reqs in ("none", "optional", "required"): + tls_kwargs = _get_tls_kwargs( + ssl_cert_reqs=ssl_cert_reqs, + tls_allow_invalid_certificates=allow_invalid, + ) + self.assertEqual( + tls_kwargs, + { + "tls": False, + "tlsAllowInvalidCertificates": allow_invalid, + }, + ) + @mock.patch("st2common.models.db.mongoengine") def test_db_setup(self, mock_mongoengine): db_setup( @@ -330,8 +374,8 @@ def test_db_setup(self, mock_mongoengine): "password": "password", "tz_aware": True, "authentication_mechanism": "MONGODB-X509", - "ssl": True, - "ssl_match_hostname": True, + "tls": True, + "tlsAllowInvalidHostnames": False, "connectTimeoutMS": 3000, "serverSelectionTimeoutMS": 3000, }, @@ -536,7 +580,7 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_name=db_name, db_host=db_host, db_port=db_port, - ssl=True, + tls=True, ensure_indexes=False, ) end = time.time() @@ -555,7 +599,7 @@ def test_db_connect_server_selection_timeout_ssl_on_non_ssl_listener(self): db_name=db_name, db_host=db_host, db_port=db_port, - ssl=True, + tls=True, ensure_indexes=False, ) end = time.time() diff --git a/st2reactor/st2reactor/container/sensor_wrapper.py b/st2reactor/st2reactor/container/sensor_wrapper.py index 6b1975bc53..cc4723043a 100644 --- a/st2reactor/st2reactor/container/sensor_wrapper.py +++ b/st2reactor/st2reactor/container/sensor_wrapper.py @@ -231,11 +231,12 @@ def __init__( username=username, password=password, ensure_indexes=db_ensure_indexes, - ssl=cfg.CONF.database.ssl, - ssl_keyfile=cfg.CONF.database.ssl_keyfile, - ssl_certfile=cfg.CONF.database.ssl_certfile, - ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs, - ssl_ca_certs=cfg.CONF.database.ssl_ca_certs, + tls=cfg.CONF.database.tls, + tls_certificate_key_file=cfg.CONF.database.tls_certificate_key_file, + tls_certificate_key_file_password=cfg.CONF.database.tls_certificate_key_file_password, + tls_allow_invalid_certificates=cfg.CONF.database.tls_allow_invalid_certificates, + tls_ca_file=cfg.CONF.database.tls_ca_file, + ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs, # deprecated authentication_mechanism=cfg.CONF.database.authentication_mechanism, ssl_match_hostname=cfg.CONF.database.ssl_match_hostname, ) diff --git a/tools/config_gen.py b/tools/config_gen.py index f139474c13..972171af71 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -76,7 +76,7 @@ STATIC_OPTION_VALUES = { "actionrunner": { "virtualenv_binary": "/usr/bin/virtualenv", - "python_binary": "/usr/bin/python", + "python_binary": "/usr/bin/python3", }, "webui": {"webui_base_url": "https://localhost"}, } @@ -164,30 +164,48 @@ def _print_options(opt_group, options): if opt.name in SKIP_OPTIONS: continue + opt_default = opt.default if opt.sample_default is None else opt.sample_default + # Special case for options which could change during this script run static_option_value = STATIC_OPTION_VALUES.get(opt_group.name, {}).get( opt.name, None ) if static_option_value: - opt.default = static_option_value + assert ( + opt_default == static_option_value + ), f"opt_default={opt_default} != static_option_value={static_option_value}" # Special handling for list options if isinstance(opt, cfg.ListOpt): - if opt.default: - value = ",".join(opt.default) + if opt_default: + value = ",".join(opt_default) else: value = "" value += " # comma separated list allowed here." - elif isinstance(opt.default, dict): + elif isinstance(opt_default, dict): # this is for [sensorcontainer].partition_provider which # is a generic cfg.Opt(type=types.Dict(value_type=types.String()) - value = " ".join([f"{k}:{v}" for k, v in opt.default.items()]) + value = " ".join([f"{k}:{v}" for k, v in opt_default.items()]) else: - value = opt.default + value = opt_default print(("# %s" % opt.help).strip()) + for deprecated_opt in opt.deprecated_opts: + deprecated_opt: cfg.DeprecatedOpt + alias = ( + deprecated_opt.name + if deprecated_opt.group is None + else f"{deprecated_opt.group}.{deprecated_opt.name}" + ) + print(f"# This option has a deprecated alias: {alias}") + + if opt.deprecated_for_removal: + print( + f"# DEPRECATED FOR REMOVAL since {opt.deprecated_since}: {opt.deprecated_reason}".strip() + ) + if isinstance(opt, cfg.StrOpt) and opt.type.choices: if isinstance(opt.type.choices, OrderedDict): valid_values = ", ".join([str(x) for x in opt.type.choices])