diff --git a/Dockerfile b/Dockerfile index 699e279344..d3f61fb32c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,10 @@ RUN sudo python3 setup.py install RUN sudo python3 prerequisites.py --yes --cmsuser=cmsuser install -RUN sudo sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@testdb:5432/cmsdbfortesting"|' ./config/cms.conf.sample \ - | sudo tee /usr/local/etc/cms-testdb.conf -RUN sudo sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@devdb:5432/cmsdb"|' ./config/cms.conf.sample \ - | sudo tee /usr/local/etc/cms-devdb.conf +RUN sudo sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@testdb:5432/cmsdbfortesting"|' ./config/cms.toml.sample \ + | sudo tee /usr/local/etc/cms-testdb.toml +RUN sudo sed 's|/cmsuser:your_password_here@localhost:5432/cmsdb"|/postgres@devdb:5432/cmsdb"|' ./config/cms.toml.sample \ + | sudo tee /usr/local/etc/cms-devdb.toml ENV LANG C.UTF-8 diff --git a/cms/cms_conf_legacy_mapping.toml.jinja b/cms/cms_conf_legacy_mapping.toml.jinja new file mode 100644 index 0000000000..8ba27afd93 --- /dev/null +++ b/cms/cms_conf_legacy_mapping.toml.jinja @@ -0,0 +1,157 @@ +### System-wide ### + +temp_dir = {{(temp_dir or "/tmp")|tojson}} + +# Whether to have a backdoor (see doc for the risks). +backdoor = {{(backdoor or false)|tojson}} + +# The user/group that CMS will be run as. +cmsuser = {{(cmsuser or "cmsuser")|tojson}} + + +### Database ### + +# Connection string for the database. +database = {{(database or "postgresql+psycopg2://cmsuser:your_password_here@localhost:5432/cmsdb")|tojson}} + +# Whether SQLAlchemy prints DB queries on stdout. +database_debug = {{(database_debug or false)|tojson}} + +# Whether to use two-phase commit. +twophase_commit = {{(twophase_commit or false)|tojson}} + + +### Worker ### + +# Don't delete the sandbox directory under /tmp/ when they +# are not needed anymore. Warning: this can easily eat GB +# of space very soon. +keep_sandbox = {{(keep_sandbox or false)|tojson}} + + +### Sandbox ### + +# Do not allow contestants' solutions to write files bigger +# than this size (expressed in KB; defaults to 1 GB). +max_file_size = {{(max_file_size or 1048576)|tojson}} + + +### WebServers ### + +# This key is used to encode information that can be seen +# by the user, namely cookies and auto-incremented +# numbers. It should be changed for each +# contest. Particularly, you should not use this example +# for other than testing. It must be a 16 bytes long +# hexadecimal number. You can easily create a key calling +# python -c 'from cmscommon import crypto; print(crypto.get_hex_random_key())' +secret_key = {{(secret_key or "8e045a51e4b102ea803c06f92841a1fb")|tojson}} + +# Whether Tornado prints debug information on stdout. +tornado_debug = {{(tornado_debug or false)|tojson}} + + +### ContestWebServer ### + +# Listening HTTP addresses and ports for the CWSs listed above +# in core_services. If you access them through a proxy (acting +# as a load balancer) running on the same host you could put +# 127.0.0.1 here for additional security. +contest_listen_address = {{(contest_listen_address or "")|tojson}} +contest_listen_port = {{(contest_listen_port or [8888])|tojson}} + +# Login cookie duration in seconds. The duration is refreshed +# on every manual request. +contest_cookie_duration = {{(cookie_duration or 1800)|tojson}} + +# If CWSs write submissions to disk before storing them in +# the DB, and where to save them. %s = DATA_DIR. +submit_local_copy = {{(submit_local_copy or true)|tojson}} +submit_local_copy_path = {{(submit_local_copy_path or "%s/submissions/")|tojson}} + +# The number of proxies that will be crossed before CWSs get +# the request. This is used to decide whether to assume that +# the real source IP address is the one listed in the request +# headers or not. For example, if you're using nginx as a load +# balancer, you will likely want to set this value to 1. +contest_num_proxies_used = {{(num_proxies_used or 0)|tojson}} + +# Maximum size of a submission in bytes. If you use a proxy +# and set these sizes to large values remember to change +# client_max_body_size in nginx.conf too. +max_submission_length = {{(max_submission_length or 100000)|tojson}} +max_input_length = {{(max_input_length or 5000000)|tojson}} + +# STL documentation path in the system (exposed in CWS). +stl_path = {{(stl_path or "/usr/share/cppreference/doc/html/")|tojson}} + + +### AdminWebserver ### + +# Listening HTTP address and port for the AWS. If you access +# it through a proxy running on the same host you could put +# 127.0.0.1 here for additional security. +admin_listen_address = {{(admin_listen_address or "")|tojson}} +admin_listen_port = {{(admin_listen_port or 8889)|tojson}} + +# Login cookie duration for admins in seconds. +# The duration is refreshed on every manual request. +admin_cookie_duration = {{(admin_cookie_duration or 36000)|tojson}} + +# The number of proxies that will be crossed before AWS gets +# the request. This is used to determine the request's real +# source IP address. For example, if you're using nginx as +# a proxy, you will likely want to set this value to 1. +admin_num_proxies_used = {{(admin_num_proxies_used or 0)|tojson}} + + +### ProxyService ### + +# List of URLs (with embedded username and password) of the +# RWSs where the scores are to be sent. Don't include the +# load balancing proxy (if any), just the backends. If any +# of them uses HTTPS specify a file with the certificates +# you trust. +rankings = {{(rankings or ["http://usern4me:passw0rd@localhost:8890/"])|tojson}} +https_certfile = {{(https_certfile or "")|tojson}} + + +### PrintingService ### + +# Maximum size of a print job in bytes. +max_print_length = {{(max_print_length or 10000000)|tojson}} + +# Printer name (can be found out using 'lpstat -p'; +# if "", printing is disabled) +printer = {{(printer or "")|tojson}} + +# Output paper size (probably A4 or Letter) +paper_size = {{(paper_size or "A4")|tojson}} + +# Maximum number of pages a user can print per print job +# (excluding the title page). Text files are cropped to this +# length. Too long pdf files are rejected. +max_pages_per_job = {{(max_pages_per_job or 10)|tojson}} +max_jobs_per_user = {{(max_jobs_per_user or 10)|tojson}} +pdf_printing_allowed = {{(pdf_printing_allowed or false)|tojson}} + +{{stray}} + +### Services ### + +[core_services] + +LogService = {{(core_services.LogService or [])|tojson}} +ResourceService = {{(core_services.ResourceService or [])|tojson}} +ScoringService = {{(core_services.ScoringService or [])|tojson}} +Checker = {{(core_services.Checker or [])|tojson}} +EvaluationService = {{(core_services.EvaluationService or [])|tojson}} +Worker = {{(core_services.Worker or [])|tojson}} +ContestWebServer = {{(core_services.ContestWebServer or [])|tojson}} +AdminWebServer = {{(core_services.AdminWebServer or [])|tojson}} +ProxyService = {{(core_services.ProxyService or [])|tojson}} +PrintingService = {{(core_services.PrintingService or [])|tojson}} + +[other_services] + +TestFileCacher = {{(other_services.TestFileCacher or [])|tojson}} diff --git a/cms/conf.py b/cms/conf.py index 9ccc2b8e74..e704738808 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -7,6 +7,7 @@ # Copyright © 2013 Luca Wehrstedt # Copyright © 2014 Fabian Gundlach <320pointsguy@gmail.com> # Copyright © 2016 Myungwoo Chun +# Copyright © 2022-2023 Manuel Gundlach # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,12 +22,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from __future__ import annotations + import errno +import jinja2 +import jinja2.meta import json import logging import os import sys +import tomli from collections import namedtuple +from typing import Optional from .log import set_detailed_logs @@ -44,6 +51,7 @@ class ServiceCoord(namedtuple("ServiceCoord", "name shard")): service (thus identifying it). """ + def __repr__(self): return "%s,%d" % (self.name, self.shard) @@ -77,10 +85,11 @@ class AsyncConfig: class Config: """This class will contain the configuration for CMS. This needs to be populated at the initilization stage. This is loaded by - default with some sane data. See cms.conf.sample in the config + default with some sane data. See cms.toml.sample in the config directory for information on the meaning of the fields. """ + def __init__(self): """Default values for configuration, plus decide if this instance is running from the system path or from the source @@ -89,77 +98,77 @@ def __init__(self): """ self.async_config = async_config - # System-wide - self.cmsuser = "cmsuser" - self.temp_dir = "/tmp" - self.backdoor = False - self.file_log_debug = False - self.stream_log_detailed = False + # System-wide. + self.cmsuser: str = "cmsuser" + self.temp_dir: str = "/tmp" + self.backdoor: bool = False + self.file_log_debug: bool = False + self.stream_log_detailed: bool = False # Database. - self.database = "postgresql+psycopg2://cmsuser@localhost/cms" - self.database_debug = False - self.twophase_commit = False + self.database: str = "postgresql+psycopg2://cmsuser@localhost/cms" + self.database_debug: bool = False + self.twophase_commit: bool = False # Worker. - self.keep_sandbox = True - self.use_cgroups = True - self.sandbox_implementation = 'isolate' + self.keep_sandbox: bool = False + self.use_cgroups: bool = True + self.sandbox_implementation: str = 'isolate' # Sandbox. # Max size of each writable file during an evaluation step, in KiB. - self.max_file_size = 1024 * 1024 # 1 GiB + self.max_file_size: int = 1024 * 1024 # 1 GiB # Max processes, CPU time (s), memory (KiB) for compilation runs. - self.compilation_sandbox_max_processes = 1000 - self.compilation_sandbox_max_time_s = 10.0 - self.compilation_sandbox_max_memory_kib = 512 * 1024 # 512 MiB + self.compilation_sandbox_max_processes: int = 1000 + self.compilation_sandbox_max_time_s: float = 10.0 + self.compilation_sandbox_max_memory_kib: int = 512 * 1024 # 512 MiB # Max processes, CPU time (s), memory (KiB) for trusted runs. - self.trusted_sandbox_max_processes = 1000 - self.trusted_sandbox_max_time_s = 10.0 - self.trusted_sandbox_max_memory_kib = 4 * 1024 * 1024 # 4 GiB + self.trusted_sandbox_max_processes: int = 1000 + self.trusted_sandbox_max_time_s: float = 10.0 + self.trusted_sandbox_max_memory_kib: int = 4 * 1024 * 1024 # 4 GiB # WebServers. - self.secret_key_default = "8e045a51e4b102ea803c06f92841a1fb" - self.secret_key = self.secret_key_default - self.tornado_debug = False + self.secret_key_default: str = "8e045a51e4b102ea803c06f92841a1fb" + self.secret_key: str = self.secret_key_default + self.tornado_debug: bool = False # ContestWebServer. - self.contest_listen_address = [""] - self.contest_listen_port = [8888] - self.cookie_duration = 30 * 60 # 30 minutes - self.submit_local_copy = True - self.submit_local_copy_path = "%s/submissions/" - self.tests_local_copy = True - self.tests_local_copy_path = "%s/tests/" - self.is_proxy_used = None # (deprecated in favor of num_proxies_used) - self.num_proxies_used = None - self.max_submission_length = 100_000 # 100 KB - self.max_input_length = 5_000_000 # 5 MB - self.stl_path = "/usr/share/cppreference/doc/html/" + self.contest_listen_address: list[str] = [""] + self.contest_listen_port: list[int] = [8888] + self.contest_cookie_duration: int = 30 * 60 # 30 minutes + self.submit_local_copy: bool = True + self.submit_local_copy_path: str = "%s/submissions/" + self.tests_local_copy: bool = True + self.tests_local_copy_path: str = "%s/tests/" + self.is_proxy_used: Optional[bool] = None # (deprecated) + self.contest_num_proxies_used: Optional[int] = None + self.max_submission_length: int = 100_000 # 100 KB + self.max_input_length: int = 5_000_000 # 5 MB + self.stl_path: str = "/usr/share/cppreference/doc/html/" # Prefix of 'shared-mime-info'[1] installation. It can be found # out using `pkg-config --variable=prefix shared-mime-info`, but # it's almost universally the same (i.e. '/usr') so it's hardly # necessary to change it. # [1] http://freedesktop.org/wiki/Software/shared-mime-info - self.shared_mime_info_prefix = "/usr" + self.shared_mime_info_prefix: str = "/usr" # AdminWebServer. - self.admin_listen_address = "" - self.admin_listen_port = 8889 - self.admin_cookie_duration = 10 * 60 * 60 # 10 hours - self.admin_num_proxies_used = None + self.admin_listen_address: str = "" + self.admin_listen_port: int = 8889 + self.admin_cookie_duration: int = 10 * 60 * 60 # 10 hours + self.admin_num_proxies_used: Optional[int] = None # ProxyService. - self.rankings = ["http://usern4me:passw0rd@localhost:8890/"] - self.https_certfile = None + self.rankings: list[str] = ["http://usern4me:passw0rd@localhost:8890/"] + self.https_certfile: Optional[str] = None - # PrintingService - self.max_print_length = 10_000_000 # 10 MB - self.printer = None - self.paper_size = "A4" - self.max_pages_per_job = 10 - self.max_jobs_per_user = 10 - self.pdf_printing_allowed = False + # PrintingService. + self.max_print_length: int = 10_000_000 # 10 MB + self.printer: Optional[str] = None + self.paper_size: str = "A4" + self.max_pages_per_job: int = 10 + self.max_jobs_per_user: int = 10 + self.pdf_printing_allowed: bool = False # Installed or from source? # We declare we are running from installed if the program was @@ -178,20 +187,23 @@ def __init__(self): self.cache_dir = os.path.join("/", "var", "local", "cache", "cms") self.data_dir = os.path.join("/", "var", "local", "lib", "cms") self.run_dir = os.path.join("/", "var", "local", "run", "cms") - paths = [os.path.join("/", "usr", "local", "etc", "cms.conf"), - os.path.join("/", "etc", "cms.conf")] + etc_paths = [os.path.join("/", "usr", "local", "etc"), + os.path.join("/", "etc")] else: self.log_dir = "log" self.cache_dir = "cache" self.data_dir = "lib" self.run_dir = "run" - paths = [os.path.join(".", "config", "cms.conf")] + etc_paths = [os.path.join(".", "config")] if '__file__' in globals(): - paths += [os.path.abspath(os.path.join( - os.path.dirname(__file__), - '..', 'config', 'cms.conf'))] - paths += [os.path.join("/", "usr", "local", "etc", "cms.conf"), - os.path.join("/", "etc", "cms.conf")] + etc_paths += [os.path.abspath(os.path.join( + os.path.dirname(__file__), + '..', 'config'))] + etc_paths += [os.path.join("/", "usr", "local", "etc"), + os.path.join("/", "etc")] + + paths = [os.path.join(p, "cms.conf") for p in etc_paths] + \ + [os.path.join(p, "cms.toml") for p in etc_paths] # Allow user to override config file path using environment # variable 'CMS_CONFIG'. @@ -215,12 +227,12 @@ def _load(self, paths): if self._load_unique(conf_file): break else: - logging.warning("No configuration file found: " + logging.warning("No valid configuration file found: " "falling back to default values.") def _load_unique(self, path): """Populate the Config class with everything that sits inside - the JSON file path (usually something like /etc/cms.conf). The + the TOML file path (usually something like /etc/cms.toml). The only pieces of data treated differently are the elements of core_services and other_services that are sent to async config. @@ -228,56 +240,145 @@ def _load_unique(self, path): Services whose name begins with an underscore are ignored, so they can be commented out in the configuration file. - path (string): the path of the JSON config file. + path (string): the path of the TOML (or JSON) config file. """ + # Load config file. - try: - with open(path, 'rt', encoding='utf-8') as f: - data = json.load(f) - except FileNotFoundError: - logger.debug("Couldn't find config file %s.", path) - return False - except OSError as error: - logger.warning("I/O error while opening file %s: [%s] %s", - path, errno.errorcode[error.errno], - os.strerror(error.errno)) - return False - except ValueError as error: - logger.warning("Invalid syntax in file %s: %s", path, error) + for loader, loader_name, success_handler in \ + ((tomli, "TOML", lambda p, d: None), + (json, "JSON", self._suggest_updated_legacy_config)): + try: + with open(path, 'rb') as f: + data = loader.load(f) + except FileNotFoundError: + logger.debug("Couldn't find config file %s.", path) + return False + except OSError as error: + logger.warning("I/O error while opening file %s: [%s] %s", + path, errno.errorcode[error.errno], + os.strerror(error.errno)) + return False + except ValueError as error: + logger.warning("Invalid syntax (assuming %s) in file %s: %s", + loader_name, path, error) + else: + success_handler(path, data) + break + else: return False logger.info("Using configuration file %s.", path) if "is_proxy_used" in data: logger.warning("The 'is_proxy_used' setting is deprecated, please " - "use 'num_proxies_used' instead.") + "use 'contest_num_proxies_used' instead.") # Put core and test services in async_config, ignoring those # whose name begins with "_". - for service in data["core_services"]: - if service.startswith("_"): - continue - for shard_number, shard in \ - enumerate(data["core_services"][service]): - coord = ServiceCoord(service, shard_number) - self.async_config.core_services[coord] = Address(*shard) - del data["core_services"] - - for service in data["other_services"]: - if service.startswith("_"): - continue - for shard_number, shard in \ - enumerate(data["other_services"][service]): - coord = ServiceCoord(service, shard_number) - self.async_config.other_services[coord] = Address(*shard) - del data["other_services"] + for part in ("core_services", "other_services"): + for service in data[part]: + if service.startswith("_"): + continue + for shard_number, shard in \ + enumerate(data[part][service]): + coord = ServiceCoord(service, shard_number) + getattr(self.async_config, part)[coord] = Address(*shard) + del data[part] + + # These keys have been renamed. If the old key name is still used, it + # is still regarded. + for key in ("cookie_duration", "num_proxies_used"): + if key in data: + new_key = "contest_" + key + if new_key in data: + logger.error("Conflicting keys %s and %s.", key, new_key) + continue + else: + data[new_key] = data[key] + del data[key] + + # A value of "" means None for these attributes + for key in ("https_certfile", "printer"): + if key in data and data[key] == "": + data[key] = None # Put everything else in self. for key, value in data.items(): + # Ignore keys whose name begins with "_". + if key.startswith("_"): + continue + # Warn about unknown keys. + if not hasattr(self, key): + logger.warning("Key %s unknown.", key) setattr(self, key, value) return True + def _suggest_updated_legacy_config(self, path, legacy_data): + logger.error("Legacy json config file found at %s. " + "The format for configuration files has changed to TOML. " + "You should rewrite your configuration file for the new " + "format. A suggested translation will follow.", path) + + # Load data into the TOML config template + + # NOTE Values are rendered using json.dumps. This is only + # heuristically correct and might in some cases not adhere to the + # TOML specification. However, for the usual configuration values of + # CMS this should be fine. + + jinja_env = jinja2.Environment( + loader=jinja2.PackageLoader("cms", "")) + + ast = jinja_env.parse( + jinja_env.loader.get_source(jinja_env, + "cms_conf_legacy_mapping.toml.jinja") + ) + attr_in_template = jinja2.meta.find_undeclared_variables(ast) + + legacy_attr_not_in_template = { + k: v for k, v in legacy_data.items() + if k not in attr_in_template and not k.startswith('_') + } + template_attr_not_in_legacy = [ + k for k in attr_in_template + if k not in legacy_data and k != "stray" + ] + + for k in legacy_attr_not_in_template: + logger.warning("Key %s in legacy config unknown (but will be " + "exported).", k) + + # Attributes in the legacy config that don't appear in the TOML + # template will be put under a 'stray' "header". + if len(legacy_attr_not_in_template) == 0: + stray = "" + else: + stray = "\n### stray ###\n\n" + \ + '\n'.join("{} = {}".format(k, json.dumps(v)) + for k, v in legacy_attr_not_in_template.items()) + \ + "\n" + assert "stray" not in legacy_data + template_data = legacy_data.copy() + template_data["stray"] = stray + + for k in template_attr_not_in_legacy: + logger.warning("Key %s is missing in legacy config; " + "the value will be set to the default.", k) + + template = jinja_env.get_template("cms_conf_legacy_mapping.toml.jinja") + updated_config = template.render(template_data) + + logger.info("==== Config heuristically translated to new format below ====\n" + "%s\n" + "==== Config heuristically translated to new format above ====\n" + "You can find your legacy config updated to the " + "current config format above. " + "Please check the output, save it as %s and remove %s. ", + updated_config, + os.path.join(os.path.dirname(path), "cms.toml"), + path) + config = Config() diff --git a/cms/db/util.py b/cms/db/util.py index 28fd97ae69..e5d432532d 100644 --- a/cms/db/util.py +++ b/cms/db/util.py @@ -54,7 +54,7 @@ def test_db_connection(): except OperationalError as e: logger.error(e) raise ConfigError("Operational error while talking to the DB. " - "Is the connection string in cms.conf correct?") + "Is the connection string in cms.toml correct?") def get_contest_list(session=None): diff --git a/cms/io/service.py b/cms/io/service.py index c93d10339d..ca94b13a02 100644 --- a/cms/io/service.py +++ b/cms/io/service.py @@ -95,7 +95,7 @@ def __init__(self, shard=0): address = get_service_address(self._my_coord) except KeyError: raise ConfigError("Unable to find address for service %r. " - "Is it specified in core_services in cms.conf?" % + "Is it specified in core_services in cms.toml?" % (self._my_coord,)) self.rpc_server = StreamServer(address, self._connection_handler) @@ -192,7 +192,7 @@ def connect_to(self, coord, on_connect=None, on_disconnect=None, # the service was optional. if must_be_present: raise ConfigError("Missing address and port for %s " - "in cms.conf." % (coord, )) + "in cms.toml." % (coord, )) else: service = FakeRemoteServiceClient(coord, None) service.connect() diff --git a/cms/io/web_service.py b/cms/io/web_service.py index 41d2e2829c..8357a22fb6 100644 --- a/cms/io/web_service.py +++ b/cms/io/web_service.py @@ -56,7 +56,7 @@ def __init__(self, listen_port, handlers, parameters, shard=0, rpc_auth = parameters.pop('rpc_auth', None) auth_middleware = parameters.pop('auth_middleware', None) is_proxy_used = parameters.pop('is_proxy_used', None) - num_proxies_used = parameters.pop('num_proxies_used', None) + contest_num_proxies_used = parameters.pop('contest_num_proxies_used', None) self.wsgi_app = tornado_wsgi.WSGIApplication(handlers, **parameters) self.wsgi_app.service = self @@ -90,14 +90,14 @@ def __init__(self, listen_port, handlers, parameters, shard=0, # only if all requests come from a trusted source (if clients # were allowed to directlty communicate with the server they # could fake their IP and compromise the security of IP lock). - if num_proxies_used is None: + if contest_num_proxies_used is None: if is_proxy_used: - num_proxies_used = 1 + contest_num_proxies_used = 1 else: - num_proxies_used = 0 + contest_num_proxies_used = 0 - if num_proxies_used > 0: - self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies_used) + if contest_num_proxies_used > 0: + self.wsgi_app = ProxyFix(self.wsgi_app, contest_num_proxies_used) self.web_server = WSGIServer((listen_address, listen_port), self) diff --git a/cms/server/admin/templates/base.html b/cms/server/admin/templates/base.html index 32f84b5cfb..822ecab742 100644 --- a/cms/server/admin/templates/base.html +++ b/cms/server/admin/templates/base.html @@ -85,7 +85,7 @@

{{ contest.name }}

{% if config.secret_key == config.secret_key_default %}
- Change secret_key in cms.conf!
+ Change secret_key in cms.toml!
For example,
{{ get_hex_random_key() }}
diff --git a/cms/server/contest/authentication.py b/cms/server/contest/authentication.py index 579a5933e8..e6557cda33 100644 --- a/cms/server/contest/authentication.py +++ b/cms/server/contest/authentication.py @@ -321,9 +321,9 @@ def log_failed_attempt(msg, *args): *args) # Check if the cookie is expired. - if timestamp - last_update > timedelta(seconds=config.cookie_duration): + if timestamp - last_update > timedelta(seconds=config.contest_cookie_duration): log_failed_attempt("cookie expired (lasts %d seconds)", - config.cookie_duration) + config.contest_cookie_duration) return None, None # Load participation from DB and make sure it exists. diff --git a/cms/server/contest/server.py b/cms/server/contest/server.py index a299af4aa6..ca16ed98d2 100644 --- a/cms/server/contest/server.py +++ b/cms/server/contest/server.py @@ -68,7 +68,7 @@ def __init__(self, shard, contest_id=None): "cookie_secret": hex_to_bin(config.secret_key), "debug": config.tornado_debug, "is_proxy_used": config.is_proxy_used, - "num_proxies_used": config.num_proxies_used, + "contest_num_proxies_used": config.contest_num_proxies_used, "xsrf_cookies": True, } @@ -79,7 +79,7 @@ def __init__(self, shard, contest_id=None): raise ConfigError("Wrong shard number for %s, or missing " "address/port configuration. Please check " "contest_listen_address and contest_listen_port " - "in cms.conf." % __name__) + "in cms.toml." % __name__) self.contest_id = contest_id diff --git a/cmsranking/Config.py b/cmsranking/Config.py index e7d624f328..dc6c46e93a 100644 --- a/cmsranking/Config.py +++ b/cmsranking/Config.py @@ -2,6 +2,7 @@ # Contest Management System - http://cms-dev.github.io/ # Copyright © 2011-2016 Luca Wehrstedt +# Copyright © 2023 Manuel Gundlach # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -17,7 +18,7 @@ # along with this program. If not, see . import errno -import json +import tomli import logging import os import sys @@ -37,6 +38,7 @@ class Config: """An object holding the current configuration. """ + def __init__(self): """Fill this object with the default values for each key. @@ -74,15 +76,15 @@ def __init__(self): self.lib_dir = os.path.join("/", "var", "local", "lib", "cms", "ranking") self.conf_paths = [os.path.join("/", "usr", "local", "etc", - "cms.ranking.conf"), - os.path.join("/", "etc", "cms.ranking.conf")] + "cms.ranking.toml"), + os.path.join("/", "etc", "cms.ranking.toml")] else: self.log_dir = os.path.join("log", "ranking") self.lib_dir = os.path.join("lib", "ranking") - self.conf_paths = [os.path.join(".", "config", "cms.ranking.conf"), + self.conf_paths = [os.path.join(".", "config", "cms.ranking.toml"), os.path.join("/", "usr", "local", "etc", - "cms.ranking.conf"), - os.path.join("/", "etc", "cms.ranking.conf")] + "cms.ranking.toml"), + os.path.join("/", "etc", "cms.ranking.toml")] # Allow users to override config file path using environment # variable 'CMS_RANKING_CONFIG'. @@ -138,7 +140,7 @@ def _load_many(self, conf_paths): """ for conf_path in conf_paths: try: - with open(conf_path, "rt", encoding="utf-8") as conf_fobj: + with open(conf_path, "rb") as conf_fobj: logger.info("Using config file %s.", conf_path) return self._load_one(conf_fobj) except FileNotFoundError: @@ -164,10 +166,11 @@ def _load_one(self, conf_fobj): """ # Parse config file. + # TODO Advice if legacy JSON file detected try: - data = json.load(conf_fobj) + data = tomli.load(conf_fobj) except ValueError: - logger.critical("Config file is invalid JSON.") + logger.critical("Config file is invalid TOML.") return False # Store every config property. diff --git a/cmstestsuite/functionaltestframework.py b/cmstestsuite/functionaltestframework.py index 9b47827c10..95674c1beb 100644 --- a/cmstestsuite/functionaltestframework.py +++ b/cmstestsuite/functionaltestframework.py @@ -22,11 +22,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import logging import re import sys import time +import tomli from cmstestsuite import CONFIG, TestException, sh from cmstestsuite.web import Browser @@ -73,7 +73,7 @@ def __new__(cls): return FunctionalTestFramework.__instance def __init__(self): - # This holds the decoded-JSON of the cms.conf configuration file. + # This holds the decoded-TOML of the cms.toml configuration file. # Lazily loaded, to be accessed through the getter method. self._cms_config = None @@ -141,8 +141,8 @@ def initialize_aws(self): def get_cms_config(self): if self._cms_config is None: - with open("%(CONFIG_PATH)s" % CONFIG, "rt", encoding="utf-8") as f: - self._cms_config = json.load(f) + with open("%(CONFIG_PATH)s" % CONFIG, "rb") as f: + self._cms_config = tomli.load(f) return self._cms_config def admin_req(self, path, args=None, files=None): diff --git a/cmstestsuite/testrunner.py b/cmstestsuite/testrunner.py index 33e128cfef..cc7fee6ae3 100644 --- a/cmstestsuite/testrunner.py +++ b/cmstestsuite/testrunner.py @@ -93,9 +93,9 @@ def load_cms_conf(self): except subprocess.CalledProcessError: git_root = None CONFIG["TEST_DIR"] = git_root - CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] + CONFIG["CONFIG_PATH"] = "%s/config/cms.toml" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: - CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" + CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.toml" # Override CMS config path when environment variable is present CMS_CONFIG_ENV_VAR = "CMS_CONFIG" diff --git a/cmstestsuite/unit_tests/server/contest/authentication_test.py b/cmstestsuite/unit_tests/server/contest/authentication_test.py index b099be0363..a40a01d9d3 100755 --- a/cmstestsuite/unit_tests/server/contest/authentication_test.py +++ b/cmstestsuite/unit_tests/server/contest/authentication_test.py @@ -227,7 +227,7 @@ def assertFailure(self, **kwargs): self.assertIsNone(authenticated_participation) self.assertIsNone(cookie) - @patch.object(config, "cookie_duration", 10) + @patch.object(config, "contest_cookie_duration", 10) def test_cookie_contains_timestamp(self): self.contest.ip_autologin = False self.contest.allow_password_authentication = True diff --git a/config/.gitignore b/config/.gitignore index 5dcae6fd36..c8d7a8ef00 100644 --- a/config/.gitignore +++ b/config/.gitignore @@ -1,2 +1,2 @@ -cms.conf -cms.ranking.conf +cms.toml +cms.ranking.toml diff --git a/config/cms.conf.sample b/config/cms.conf.sample deleted file mode 100644 index 8bb0d730b8..0000000000 --- a/config/cms.conf.sample +++ /dev/null @@ -1,192 +0,0 @@ -{ - "_help": "There is no way to put comments in a JSON file; the", - "_help": "fields starting with '_' are meant to be comments.", - - - - "_section": "System-wide configuration", - - "temp_dir": "/tmp", - - "_help": "Whether to have a backdoor (see doc for the risks).", - "backdoor": false, - - "_help": "The user/group that CMS will be run as.", - "cmsuser": "cmsuser", - - - "_section": "AsyncLibrary", - - "core_services": - { - "LogService": [["localhost", 29000]], - "ResourceService": [["localhost", 28000]], - "ScoringService": [["localhost", 28500]], - "Checker": [["localhost", 22000]], - "EvaluationService": [["localhost", 25000]], - "Worker": [["localhost", 26000], - ["localhost", 26001], - ["localhost", 26002], - ["localhost", 26003], - ["localhost", 26004], - ["localhost", 26005], - ["localhost", 26006], - ["localhost", 26007], - ["localhost", 26008], - ["localhost", 26009], - ["localhost", 26010], - ["localhost", 26011], - ["localhost", 26012], - ["localhost", 26013], - ["localhost", 26014], - ["localhost", 26015]], - "ContestWebServer": [["localhost", 21000]], - "AdminWebServer": [["localhost", 21100]], - "ProxyService": [["localhost", 28600]], - "PrintingService": [["localhost", 25123]] - }, - - "other_services": - { - "TestFileCacher": [["localhost", 27501]] - }, - - - - "_section": "Database", - - "_help": "Connection string for the database.", - "database": "postgresql+psycopg2://cmsuser:your_password_here@localhost:5432/cmsdb", - - "_help": "Whether SQLAlchemy prints DB queries on stdout.", - "database_debug": false, - - "_help": "Whether to use two-phase commit.", - "twophase_commit": false, - - - - "_section": "Worker", - - "_help": "Don't delete the sandbox directory under /tmp/ when they", - "_help": "are not needed anymore. Warning: this can easily eat GB", - "_help": "of space very soon.", - "keep_sandbox": false, - - - - "_section": "Sandbox", - - "_help": "Do not allow contestants' solutions to write files bigger", - "_help": "than this size (expressed in KB; defaults to 1 GB).", - "max_file_size": 1048576, - - - - "_section": "WebServers", - - "_help": "This key is used to encode information that can be seen", - "_help": "by the user, namely cookies and auto-incremented", - "_help": "numbers. It should be changed for each", - "_help": "contest. Particularly, you should not use this example", - "_help": "for other than testing. It must be a 16 bytes long", - "_help": "hexadecimal number. You can easily create a key calling:", - "_help": "python -c 'from cmscommon import crypto; print(crypto.get_hex_random_key())'", - "secret_key": "8e045a51e4b102ea803c06f92841a1fb", - - "_help": "Whether Tornado prints debug information on stdout.", - "tornado_debug": false, - - - - "_section": "ContestWebServer", - - "_help": "Listening HTTP addresses and ports for the CWSs listed above", - "_help": "in core_services. If you access them through a proxy (acting", - "_help": "as a load balancer) running on the same host you could put", - "_help": "127.0.0.1 here for additional security.", - "contest_listen_address": [""], - "contest_listen_port": [8888], - - "_help": "Login cookie duration in seconds. The duration is refreshed", - "_help": "on every manual request.", - "cookie_duration": 10800, - - "_help": "If CWSs write submissions to disk before storing them in", - "_help": "the DB, and where to save them. %s = DATA_DIR.", - "submit_local_copy": true, - "submit_local_copy_path": "%s/submissions/", - - "_help": "The number of proxies that will be crossed before CWSs get", - "_help": "the request. This is used to decide whether to assume that", - "_help": "the real source IP address is the one listed in the request", - "_help": "headers or not. For example, if you're using nginx as a load", - "_help": "balancer, you will likely want to set this value to 1.", - "num_proxies_used": 0, - - "_help": "Maximum size of a submission in bytes. If you use a proxy", - "_help": "and set these sizes to large values remember to change", - "_help": "client_max_body_size in nginx.conf too.", - "max_submission_length": 100000, - "max_input_length": 5000000, - - "_help": "STL documentation path in the system (exposed in CWS).", - "stl_path": "/usr/share/cppreference/doc/html/", - - - - "_section": "AdminWebServer", - - "_help": "Listening HTTP address and port for the AWS. If you access", - "_help": "it through a proxy running on the same host you could put", - "_help": "127.0.0.1 here for additional security.", - "admin_listen_address": "", - "admin_listen_port": 8889, - - "_help": "Login cookie duration for admins in seconds.", - "_help": "The duration is refreshed on every manual request.", - "admin_cookie_duration": 36000, - - "_help": "The number of proxies that will be crossed before AWS gets", - "_help": "the request. This is used to determine the request's real", - "_help": "source IP address. For example, if you're using nginx as", - "_help": "a proxy, you will likely want to set this value to 1.", - "admin_num_proxies_used": 0, - - - - "_section": "ScoringService", - - "_help": "List of URLs (with embedded username and password) of the", - "_help": "RWSs where the scores are to be sent. Don't include the", - "_help": "load balancing proxy (if any), just the backends. If any", - "_help": "of them uses HTTPS specify a file with the certificates", - "_help": "you trust.", - "rankings": ["http://usern4me:passw0rd@localhost:8890/"], - "https_certfile": null, - - - - "_section": "PrintingService", - - "_help": "Maximum size of a print job in bytes.", - "max_print_length": 10000000, - - "_help": "Printer name (can be found out using 'lpstat -p';", - "_help": "if null, printing is disabled)", - "printer": null, - - "_help": "Output paper size (probably A4 or Letter)", - "paper_size": "A4", - - "_help": "Maximum number of pages a user can print per print job", - "_help": "(excluding the title page). Text files are cropped to this", - "_help": "length. Too long pdf files are rejected.", - "max_pages_per_job": 10, - "max_jobs_per_user": 10, - "pdf_printing_allowed": false, - - - - "_help": "This is the end of this file." -} diff --git a/config/cms.ranking.conf.sample b/config/cms.ranking.conf.sample deleted file mode 100644 index 3cdf8f383f..0000000000 --- a/config/cms.ranking.conf.sample +++ /dev/null @@ -1,16 +0,0 @@ -{ - "_help": "There is no way to put comments in a JSON file; the", - "_help": "fields starting with '_' are meant to be comments.", - - "_help": "Listening address for RankingWebServer.", - "bind_address": "", - - "_help": "Listening port for RankingWebServer.", - "http_port": 8890, - - "_help": "Login information for adding and editing data.", - "username": "usern4me", - "password": "passw0rd", - - "_help": "This is the end of this file." -} diff --git a/config/cms.ranking.toml.sample b/config/cms.ranking.toml.sample new file mode 100644 index 0000000000..6ddd4d18c4 --- /dev/null +++ b/config/cms.ranking.toml.sample @@ -0,0 +1,9 @@ +# Listening address for RankingWebServer. +bind_address = "" + +# Listening port for RankingWebServer. +http_port = 8890 + +# Login information for adding and editing data. +username = "usern4me" +password = "passw0rd" diff --git a/config/cms.toml.sample b/config/cms.toml.sample new file mode 100644 index 0000000000..439c50ca93 --- /dev/null +++ b/config/cms.toml.sample @@ -0,0 +1,159 @@ +### System-wide ### + +temp_dir = "/tmp" + +# Whether to have a backdoor (see doc for the risks). +backdoor = false + +# The user/group that CMS will be run as. +cmsuser = "cmsuser" + + +### Database ### + +# Connection string for the database. +database = "postgresql+psycopg2://cmsuser:your_password_here@localhost:5432/cmsdb" + +# Whether SQLAlchemy prints DB queries on stdout. +database_debug = false + +# Whether to use two-phase commit. +twophase_commit = false + + +### Worker ### + +# Don't delete the sandbox directory under /tmp/ when they +# are not needed anymore. Warning: this can easily eat GB +# of space very soon. +keep_sandbox = false + + +### Sandbox ### + +# Do not allow contestants' solutions to write files bigger +# than this size (expressed in KB; defaults to 1 GB). +max_file_size = 1048576 + + +### WebServers ### + +# This key is used to encode information that can be seen +# by the user, namely cookies and auto-incremented +# numbers. It should be changed for each +# contest. Particularly, you should not use this example +# for other than testing. It must be a 16 bytes long +# hexadecimal number. You can easily create a key calling +# python -c 'from cmscommon import crypto; print(crypto.get_hex_random_key())' +secret_key = "8e045a51e4b102ea803c06f92841a1fb" + +# Whether Tornado prints debug information on stdout. +tornado_debug = false + + +### ContestWebServer ### + +# Listening HTTP addresses and ports for the CWSs listed above +# in core_services. If you access them through a proxy (acting +# as a load balancer) running on the same host you could put +# 127.0.0.1 here for additional security. +contest_listen_address = [""] +contest_listen_port = [8888] + +# Login cookie duration in seconds. The duration is refreshed +# on every manual request. +contest_cookie_duration = 1800 + +# If CWSs write submissions to disk before storing them in +# the DB, and where to save them. %s = DATA_DIR. +submit_local_copy = true +submit_local_copy_path = "%s/submissions/" + +# The number of proxies that will be crossed before CWSs get +# the request. This is used to decide whether to assume that +# the real source IP address is the one listed in the request +# headers or not. For example, if you're using nginx as a load +# balancer, you will likely want to set this value to 1. +contest_num_proxies_used = 0 + +# Maximum size of a submission in bytes. If you use a proxy +# and set these sizes to large values remember to change +# client_max_body_size in nginx.conf too. +max_submission_length = 100000 +max_input_length = 5000000 + +# STL documentation path in the system (exposed in CWS). +stl_path = "/usr/share/cppreference/doc/html/" + + +### AdminWebserver ### + +# Listening HTTP address and port for the AWS. If you access +# it through a proxy running on the same host you could put +# 127.0.0.1 here for additional security. +admin_listen_address = "" +admin_listen_port = 8889 + +# Login cookie duration for admins in seconds. +# The duration is refreshed on every manual request. +admin_cookie_duration = 36000 + +# The number of proxies that will be crossed before AWS gets +# the request. This is used to determine the request's real +# source IP address. For example, if you're using nginx as +# a proxy, you will likely want to set this value to 1. +admin_num_proxies_used = 0 + + +### ProxyService ### + +# List of URLs (with embedded username and password) of the +# RWSs where the scores are to be sent. Don't include the +# load balancing proxy (if any), just the backends. If any +# of them uses HTTPS specify a file with the certificates +# you trust. +rankings = ["http://usern4me:passw0rd@localhost:8890/"] +https_certfile = "" + + +### PrintingService ### + +# Maximum size of a print job in bytes. +max_print_length = 10000000 + +# Printer name (can be found out using 'lpstat -p'; +# if "", printing is disabled) +printer = "" + +# Output paper size (probably A4 or Letter) +paper_size = "A4" + +# Maximum number of pages a user can print per print job +# (excluding the title page). Text files are cropped to this +# length. Too long pdf files are rejected. +max_pages_per_job = 10 +max_jobs_per_user = 10 +pdf_printing_allowed = false + + +### Services ### + +[core_services] + +LogService = [["localhost", 29000]] +ResourceService = [["localhost", 28000]] +ScoringService = [["localhost", 28500]] +Checker = [["localhost", 22000]] +EvaluationService = [["localhost", 25000]] +Worker = [["localhost", 26000], + ["localhost", 26001], + ["localhost", 26002], + ["localhost", 26003]] +ContestWebServer = [["localhost", 21000]] +AdminWebServer = [["localhost", 21100]] +ProxyService = [["localhost", 28600]] +PrintingService = [["localhost", 25123]] + +[other_services] + +TestFileCacher = [["localhost", 27501]] diff --git a/config/nginx.conf.sample b/config/nginx.conf.sample index 5960e29cc6..e2d24eb6dc 100644 --- a/config/nginx.conf.sample +++ b/config/nginx.conf.sample @@ -166,7 +166,7 @@ http { proxy_set_header Connection ""; # Needs to be as large as the maximum allowed submission - # and input lengths set in cms.conf. + # and input lengths set in cms.toml. client_max_body_size 50M; } } diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 64a35faf71..2e1a838eac 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,7 +9,7 @@ services: depends_on: - "testdb" environment: - CMS_CONFIG: /usr/local/etc/cms-testdb.conf + CMS_CONFIG: /usr/local/etc/cms-testdb.toml # Could be removed in the future, see: # - https://github.com/pytest-dev/pytest/issues/7443 # - https://github.com/actions/runner/issues/241 diff --git a/docs/Configuring a contest.rst b/docs/Configuring a contest.rst index 820b922afd..84c6b24923 100644 --- a/docs/Configuring a contest.rst +++ b/docs/Configuring a contest.rst @@ -135,16 +135,16 @@ In general, each user can have multiple ranges of IP addresses associated to it. .. warning:: - If a reverse-proxy (like nginx) is in use then it is necessary to set ``num_proxies_used`` (in :file:`cms.conf`) to ``1`` and configure the proxy in order to properly pass the ``X-Forwarded-For``-style headers (see :ref:`running-cms_recommended-setup`). That configuration option can be set to a higher number if there are more proxies between the origin and the server. + If a reverse-proxy (like nginx) is in use then it is necessary to set ``contest_num_proxies_used`` (in :file:`cms.toml`) to ``1`` and configure the proxy in order to properly pass the ``X-Forwarded-For``-style headers (see :ref:`running-cms_recommended-setup`). That configuration option can be set to a higher number if there are more proxies between the origin and the server. Logging in with credentials --------------------------- If the autologin is not enabled, users can log in with username and password, which have to be specified in the user configuration (in cleartext, for the moment). The password can also be overridden for a specific contest in the participation configuration. These credentials need to be inserted by the admins (i.e. there's no way to sign up, of log in as a "guest", etc.). -A successfully logged in user needs to reauthenticate after ``cookie_duration`` seconds (specified in the :file:`cms.conf` file) from when they last visited a page. +A successfully logged in user needs to reauthenticate after ``contest_cookie_duration`` seconds (specified in the :file:`cms.toml` file) from when they last visited a page. -Even without autologin, it is possible to restrict the IP address or subnet that the user is using for accessing CWS, using the "IP-based login restriction" option in the contest configuration (in which case, admins need to set ``num_proxies_used`` as before). If this is set, then the login will fail if the IP address that attempted it does not match at least one of the addresses or subnets specified in the participation settings. If the participation IP address is not set, then no restriction applies. +Even without autologin, it is possible to restrict the IP address or subnet that the user is using for accessing CWS, using the "IP-based login restriction" option in the contest configuration (in which case, admins need to set ``contest_num_proxies_used`` as before). If this is set, then the login will fail if the IP address that attempted it does not match at least one of the addresses or subnets specified in the participation settings. If the participation IP address is not set, then no restriction applies. Failure to login ---------------- diff --git a/docs/Creating a contest.rst b/docs/Creating a contest.rst index e1063546f2..0f6d24ac9a 100644 --- a/docs/Creating a contest.rst +++ b/docs/Creating a contest.rst @@ -6,7 +6,7 @@ Creating a contest from scratch The most immediate (but often less practical) way to create a contest in CMS is using the admin interface. You can start the AdminWebServer using the command ``cmsAdminWebServer`` (or using the ResourceService). -After that, you can connect to the server using the address and port specified in :file:`cms.conf`; by default, http://localhost:8889/. Here, you can create a contest clicking on the link in the left column. After this, you must similarly add tasks and users. +After that, you can connect to the server using the address and port specified in :file:`cms.toml`; by default, http://localhost:8889/. Here, you can create a contest clicking on the link in the left column. After this, you must similarly add tasks and users. Since the details of contests, tasks and users usually live somewhere in the filesystem, it is more practical to use an importer to create them in CMS automatically. diff --git a/docs/Installation.rst b/docs/Installation.rst index 9ad179c2db..07fcd221b1 100644 --- a/docs/Installation.rst +++ b/docs/Installation.rst @@ -271,8 +271,8 @@ There are still a few steps to complete manually in this case. First, add CMS an export PATH=$PATH:./isolate/ export PYTHONPATH=./ - cp config/cms.conf.sample config/cms.conf - cp config/cms.ranking.conf.sample config/cms.ranking.conf + cp config/cms.toml.sample config/cms.toml + cp config/cms.ranking.toml.sample config/cms.ranking.toml Second, perform these tasks (that require root permissions): diff --git a/docs/RankingWebServer.rst b/docs/RankingWebServer.rst index 69c3b9db71..efa9c107cb 100644 --- a/docs/RankingWebServer.rst +++ b/docs/RankingWebServer.rst @@ -13,9 +13,9 @@ To start RWS you have to execute ``cmsRankingWebServer``. Configuring it -------------- -The configuration file is named :file:`cms.ranking.conf` and RWS will search for it in :file:`/usr/local/etc` and in :file:`/etc` (in this order!). In case it's not found in any of these, RWS will use a hard-coded default configuration that can be found in :gh_blob:`cmsranking/Config.py`. If RWS is not installed then the :gh_tree:`config` directory will also be checked for configuration files (note that for this to work your working directory needs to be root of the repository). In any case, as soon as you start it, RWS will tell you which configuration file it's using. +The configuration file is named :file:`cms.ranking.toml` and RWS will search for it in :file:`/usr/local/etc` and in :file:`/etc` (in this order!). In case it's not found in any of these, RWS will use a hard-coded default configuration that can be found in :gh_blob:`cmsranking/Config.py`. If RWS is not installed then the :gh_tree:`config` directory will also be checked for configuration files (note that for this to work your working directory needs to be root of the repository). In any case, as soon as you start it, RWS will tell you which configuration file it's using. -The configuration file is a JSON object. The most important parameters are: +The configuration file is a TOML document. The most important parameters are: * ``bind_address`` @@ -37,7 +37,7 @@ The configuration file is a JSON object. The most important parameters are: Remember to change the ``username`` and ``password`` every time you set up a RWS. Keeping the default ones will leave your scoreboard open to illegitimate access. -To connect the rest of CMS to your new RWS you need to add its connection parameters to the configuration file of CMS (i.e. :file:`cms.conf`). Note that you can connect CMS to multiple RWSs, each on a different server and/or port. The parameter you need to change is ``rankings``, a list of URLs in the form:: +To connect the rest of CMS to your new RWS you need to add its connection parameters to the configuration file of CMS (i.e. :file:`cms.toml`). Note that you can connect CMS to multiple RWSs, each on a different server and/or port. The parameter you need to change is ``rankings``, a list of URLs in the form:: ://:@:/ @@ -70,9 +70,9 @@ We support the following extensions: .png, .jpg, .gif and .bmp. Removing data ------------- -PS is only able to create or update data on RWS, but not to delete it. This means that, for example, when a user or a task is removed from CMS it will continue to be shown on RWS. To fix this you will have to intervene manually. The ``cmsRWSHelper`` script is designed to make this operation straightforward. For example, calling :samp:`cmsRWSHelper delete user {username}` will cause the user *username* to be removed from all the RWSs that are specified in :file:`cms.conf`. See ``cmsRWSHelper --help`` and :samp:`cmsRWSHelper {action} --help` for more usage details. +PS is only able to create or update data on RWS, but not to delete it. This means that, for example, when a user or a task is removed from CMS it will continue to be shown on RWS. To fix this you will have to intervene manually. The ``cmsRWSHelper`` script is designed to make this operation straightforward. For example, calling :samp:`cmsRWSHelper delete user {username}` will cause the user *username* to be removed from all the RWSs that are specified in :file:`cms.toml`. See ``cmsRWSHelper --help`` and :samp:`cmsRWSHelper {action} --help` for more usage details. -In case using ``cmsRWSHelper`` is impossible (for example because no :file:`cms.conf` is available) there are alternative ways to achieve the same result, presented in decreasing order of difficulty and increasing order of downtime needed. +In case using ``cmsRWSHelper`` is impossible (for example because no :file:`cms.toml` is available) there are alternative ways to achieve the same result, presented in decreasing order of difficulty and increasing order of downtime needed. * You can send a hand-crafted HTTP request to RWS (a ``DELETE`` method on the :samp:`/{entity_type}/{entity_id}` resource, giving credentials by Basic Auth) and it will, all by itself, delete that object and all the ones that depend on it, recursively (that is, when deleting a task or a user it will delete its submissions and, for each of them, its subchanges). diff --git a/docs/Running CMS.rst b/docs/Running CMS.rst index d33b85b331..6d3ad3b6a0 100644 --- a/docs/Running CMS.rst +++ b/docs/Running CMS.rst @@ -51,11 +51,11 @@ Finally you have to create the database schema for CMS, by running: Configuring CMS =============== -There are two configuration files, one for CMS itself and one for the rankings. Samples for both files are in the directory :gh_tree:`config/`. You want to copy them to the same file names but without the ``.sample`` suffix (that is, to :file:`config/cms.conf` and :file:`config/cms.ranking.conf`) before modifying them. +There are two configuration files, one for CMS itself and one for the rankings. Samples for both files are in the directory :gh_tree:`config/`. You want to copy them to the same file names but without the ``.sample`` suffix (that is, to :file:`config/cms.toml` and :file:`config/cms.ranking.toml`) before modifying them. -* :file:`cms.conf` is intended to be the same on all machines; all configurations options are explained in the file; of particular importance is the definition of ``core_services``, that specifies where and how many services are going to be run, and the connecting line for the database, in which you need to specify the name of the user created above and its password. +* :file:`cms.toml` is intended to be the same on all machines; all configurations options are explained in the file; of particular importance is the definition of ``core_services``, that specifies where and how many services are going to be run, and the connecting line for the database, in which you need to specify the name of the user created above and its password. -* :file:`cms.ranking.conf` is not necessarily meant to be the same on each server that will host a ranking, since it just controls settings relevant for one single server. The addresses and log-in information of each ranking must be the same as the ones found in :file:`cms.conf`. +* :file:`cms.ranking.toml` is not necessarily meant to be the same on each server that will host a ranking, since it just controls settings relevant for one single server. The addresses and log-in information of each ranking must be the same as the ones found in :file:`cms.toml`. These files are a pretty good starting point if you want to try CMS. There are some mandatory changes to do though: @@ -63,7 +63,7 @@ These files are a pretty good starting point if you want to try CMS. There are s * if you are running low on disk space, you may want to make sure ``keep_sandbox`` is set to ``false``; -If you are organizing a real contest, you must also change ``secret_key`` to a random key (the admin interface will suggest one if you visit it when ``secret_key`` is the default). You will also need to think about how to distribute your services and change ``core_services`` accordingly. Finally, you should change the ranking section of :file:`cms.conf`, and :file:`cms.ranking.conf`, using non-trivial username and password. +If you are organizing a real contest, you must also change ``secret_key`` to a random key (the admin interface will suggest one if you visit it when ``secret_key`` is the default). You will also need to think about how to distribute your services and change ``core_services`` accordingly. Finally, you should change the ranking section of :file:`cms.toml`, and :file:`cms.ranking.toml`, using non-trivial username and password. .. warning:: @@ -78,7 +78,7 @@ Here we will assume you installed CMS. If not, you should replace all commands p At this point, you should have CMS installed on all the machines you want run services on, with the same configuration file, and a running PostgreSQL instance. To run CMS, you need a contest in the database. To create a contest, follow :doc:`these instructions `. -CMS is composed of a number of services, potentially replicated several times, and running on several machines. You can start all the services by hand, but this is a tedious task. Luckily, there is a service (ResourceService) that takes care of starting all the services on the machine it is running, limiting thus the number of binaries you have to run. Services started by ResourceService do not show their logs to the standard output; so it is expected that you run LogService to inspect the logs as they arrive (logs are also saved to disk). To start LogService, you need to issue, in the machine specified in cms.conf for LogService, this command: +CMS is composed of a number of services, potentially replicated several times, and running on several machines. You can start all the services by hand, but this is a tedious task. Luckily, there is a service (ResourceService) that takes care of starting all the services on the machine it is running, limiting thus the number of binaries you have to run. Services started by ResourceService do not show their logs to the standard output; so it is expected that you run LogService to inspect the logs as they arrive (logs are also saved to disk). To start LogService, you need to issue, in the machine specified in cms.toml for LogService, this command: .. sourcecode:: bash diff --git a/docs/Troubleshooting.rst b/docs/Troubleshooting.rst index 47bd996af9..712a2cdaf5 100644 --- a/docs/Troubleshooting.rst +++ b/docs/Troubleshooting.rst @@ -38,7 +38,7 @@ Servers - *Symptom.* Message from ContestWebServer such as: :samp:`WARNING:root:Invalid cookie signature KFZzdW5kdWRlCnAwCkkxMzI5MzQzNzIwCnRw...` - *Possible cause.* The contest secret key (defined in cms.conf) may have been changed and users' browsers are still attempting to use cookies signed with the old key. If this is the case, the problem should correct itself and won't be seen by users. + *Possible cause.* The contest secret key (defined in cms.toml) may have been changed and users' browsers are still attempting to use cookies signed with the old key. If this is the case, the problem should correct itself and won't be seen by users. - *Symptom.* Ranking Web Server displays wrong data, or too much data. @@ -61,7 +61,7 @@ Sandbox - *Symptom.* Contestants' solutions fail when trying to write large outputs. - *Possible cause.* CMS limits the maximum output size from programs being evaluated for security reasons. Currently the limit is 1 GB and can be configured by changing the parameter ``max_file_size`` in :file:`cms.conf`. + *Possible cause.* CMS limits the maximum output size from programs being evaluated for security reasons. Currently the limit is 1 GB and can be configured by changing the parameter ``max_file_size`` in :file:`cms.toml`. Evaluations =========== diff --git a/prerequisites.py b/prerequisites.py index 46e6ed9556..6da8f484a0 100755 --- a/prerequisites.py +++ b/prerequisites.py @@ -247,7 +247,7 @@ def install_conf(): root = pwd.getpwnam("root") cmsuser = pwd.getpwnam(CMSUSER) makedir(os.path.join(USR_ROOT, "etc"), root, 0o755) - for conf_file_name in ["cms.conf", "cms.ranking.conf"]: + for conf_file_name in ["cms.toml", "cms.ranking.toml"]: conf_file = os.path.join(USR_ROOT, "etc", conf_file_name) # Skip if destination is a symlink if os.path.islink(conf_file): @@ -377,7 +377,7 @@ def uninstall(): print("===== Deleting configuration to /usr/local/etc/") if ask("Type Y if you really want to remove configuration files: "): - for conf_file_name in ["cms.conf", "cms.ranking.conf"]: + for conf_file_name in ["cms.toml", "cms.ranking.toml"]: try_delete(os.path.join(USR_ROOT, "etc", conf_file_name)) print("===== Deleting empty directories") diff --git a/requirements.txt b/requirements.txt index d6c04249d2..c6cd81b71a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ chardet>=3.0,<3.1 # https://pypi.python.org/pypi/chardet babel>=2.6,<2.7 # http://babel.pocoo.org/en/latest/changelog.html pyxdg>=0.26,<0.27 # https://freedesktop.org/wiki/Software/pyxdg/ Jinja2>=2.10,<2.11 # http://jinja.pocoo.org/docs/latest/changelog/ +tomli # See https://github.com/pallets/markupsafe/issues/286 but breaking change in # MarkupSafe causes jinja to break diff --git a/setup.py b/setup.py index 15c0a7d5a0..a90c4f55a3 100755 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ # Copyright © 2016 Myungwoo Chun # Copyright © 2016 Masaki Hara # Copyright © 2016 Peyman Jabbarzade Ganje +# Copyright © 2021-2023 Manuel Gundlach # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -36,6 +37,9 @@ PACKAGE_DATA = { + "cms": [ + "cms_conf_legacy_mapping.toml.jinja", + ], "cms.server": [ "static/*.*", "static/jq/*.*",