Clean up API server config and cli args (apache#47083)

This included renaming the following args to better match
fastapi/uvicorn:
  - `--hostname` -> `--host`
  - `--debug` -> `--dev`

The following config options were removed (there isn't an equivalent
currently):
  - `[webserver] error_logfile`
  - `[webserver] access_logformat`

The following config options were moved/renamed to the `api` section.
  - `[webserver] web_server_host` has been moved to `[api] host`
  - `[webserver] web_server_port` has been moved to `[api] port`
  - `[webserver] workers` has been moved to `[api] workers`
  - `[webserver] web_server_worker_timeout` has been moved to `[api] worker_timeout`
  - `[webserver] web_server_ssl_cert` has been moved to `[api] ssl_cert`
  - `[webserver] web_server_ssl_key` has been moved to `[api] ssl_key`
  - `[webserver] access_logfile` has been moved to `[api] access_logfile`

airflow/cli/cli_config.py

@@ -638,75 +638,16 @@ def string_lower_type(val):
     default=False,
 )
 
-# webserver
-ARG_PORT = Arg(
-    ("-p", "--port"),
-    default=conf.get("webserver", "WEB_SERVER_PORT"),
-    type=int,
-    help="The port on which to run the server",
-)
-ARG_SSL_CERT = Arg(
-    ("--ssl-cert",),
-    default=conf.get("webserver", "WEB_SERVER_SSL_CERT"),
-    help="Path to the SSL certificate for the webserver",
-)
-ARG_SSL_KEY = Arg(
-    ("--ssl-key",),
-    default=conf.get("webserver", "WEB_SERVER_SSL_KEY"),
-    help="Path to the key to use with the SSL certificate",
-)
-ARG_WORKERS = Arg(
-    ("-w", "--workers"),
-    default=conf.get("webserver", "WORKERS"),
-    type=int,
-    help="Number of workers to run the webserver on",
-)
-ARG_WORKERCLASS = Arg(
-    ("-k", "--workerclass"),
-    default=conf.get("webserver", "WORKER_CLASS"),
-    choices=["sync", "eventlet", "gevent", "tornado"],
-    help="The worker class to use for Gunicorn",
-)
-ARG_WORKER_TIMEOUT = Arg(
-    ("-t", "--worker-timeout"),
-    default=conf.get("webserver", "WEB_SERVER_WORKER_TIMEOUT"),
-    type=int,
-    help="The timeout for waiting on webserver workers",
-)
-ARG_HOSTNAME = Arg(
-    ("-H", "--hostname"),
-    default=conf.get("webserver", "WEB_SERVER_HOST"),
-    help="Set the hostname on which to run the web server",
-)
-ARG_DEBUG = Arg(
-    ("-d", "--debug"), help="Use the server that ships with Flask in debug mode", action="store_true"
-)
-ARG_ACCESS_LOGFILE = Arg(
-    ("-A", "--access-logfile"),
-    default=conf.get("webserver", "ACCESS_LOGFILE"),
-    help="The logfile to store the webserver access log. Use '-' to print to stdout",
-)
-ARG_ERROR_LOGFILE = Arg(
-    ("-E", "--error-logfile"),
-    default=conf.get("webserver", "ERROR_LOGFILE"),
-    help="The logfile to store the webserver error log. Use '-' to print to stderr",
-)
-ARG_ACCESS_LOGFORMAT = Arg(
-    ("-L", "--access-logformat"),
-    default=conf.get("webserver", "ACCESS_LOGFORMAT"),
-    help="The access log format for gunicorn logs",
-)
-
 # api-server
 ARG_API_SERVER_PORT = Arg(
     ("-p", "--port"),
-    default=9091,
+    default=conf.get("api", "port"),
     type=int,
     help="The port on which to run the API server",
 )
 ARG_API_SERVER_WORKERS = Arg(
     ("-w", "--workers"),
-    default=4,
+    default=conf.get("api", "workers"),
     type=int,
     help="Number of workers to run on the API server",
 )
@@ -717,22 +658,15 @@ def string_lower_type(val):
     help="The timeout for waiting on API server workers",
 )
 ARG_API_SERVER_HOSTNAME = Arg(
-    ("-H", "--hostname"),
-    default="0.0.0.0",  # nosec
-    help="Set the hostname on which to run the API server",
+    ("-H", "--host"),
+    default=conf.get("api", "host"),
+    help="Set the host on which to run the API server",
 )
 ARG_API_SERVER_ACCESS_LOGFILE = Arg(
     ("-A", "--access-logfile"),
+    default=conf.get("api", "access_logfile"),
     help="The logfile to store the access log. Use '-' to print to stdout",
 )
-ARG_API_SERVER_ERROR_LOGFILE = Arg(
-    ("-E", "--error-logfile"),
-    help="The logfile to store the error log. Use '-' to print to stderr",
-)
-ARG_API_SERVER_ACCESS_LOGFORMAT = Arg(
-    ("-L", "--access-logformat"),
-    help="The access log format for gunicorn logs",
-)
 ARG_API_SERVER_APPS = Arg(
     ("--apps",),
     help="Applications to run (comma-separated). Default is all. Options: core, execution, all",
@@ -743,7 +677,17 @@ def string_lower_type(val):
     help="Enable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info.",
     action="store_true",
 )
-
+ARG_SSL_CERT = Arg(
+    ("--ssl-cert",),
+    default=conf.get("api", "ssl_cert"),
+    help="Path to the SSL certificate for the webserver",
+)
+ARG_SSL_KEY = Arg(
+    ("--ssl-key",),
+    default=conf.get("api", "ssl_key"),
+    help="Path to the key to use with the SSL certificate",
+)
+ARG_DEV = Arg(("-d", "--dev"), help="Start FastAPI in development mode", action="store_true")
 
 # scheduler
 ARG_NUM_RUNS = Arg(
@@ -1871,13 +1815,11 @@ class GroupCommand(NamedTuple):
             ARG_STDOUT,
             ARG_STDERR,
             ARG_API_SERVER_ACCESS_LOGFILE,
-            ARG_API_SERVER_ERROR_LOGFILE,
-            ARG_API_SERVER_ACCESS_LOGFORMAT,
             ARG_API_SERVER_APPS,
             ARG_LOG_FILE,
             ARG_SSL_CERT,
             ARG_SSL_KEY,
-            ARG_DEBUG,
+            ARG_DEV,
             ARG_API_SERVER_ALLOW_PROXY_FORWARDING,
         ),
     ),

airflow/cli/commands/local_commands/api_server_command.py

@@ -48,13 +48,12 @@ def api_server(args):
 
     apps = args.apps
     access_logfile = args.access_logfile or "-"
-    access_logformat = args.access_logformat
     num_workers = args.workers
     worker_timeout = args.worker_timeout
     proxy_headers = args.proxy_headers
 
-    if args.debug:
-        print(f"Starting the API server on port {args.port} and host {args.hostname} debug.")
+    if args.dev:
+        print(f"Starting the API server on port {args.port} and host {args.host} in development mode.")
         log.warning("Running in dev mode, ignoring uvicorn args")
 
         run_args = [
@@ -64,13 +63,13 @@ def api_server(args):
             "--port",
             str(args.port),
             "--host",
-            str(args.hostname),
+            str(args.host),
         ]
 
         if args.proxy_headers:
             run_args.append("--proxy-headers")
 
-        # There is no way to pass the apps to airflow/api_fastapi/main.py in the debug mode
+        # There is no way to pass the apps to airflow/api_fastapi/main.py in the development mode
         # because fastapi dev command does not accept any additional arguments
         # so environment variable is being used to pass it
         os.environ["AIRFLOW_API_APPS"] = apps
@@ -91,18 +90,17 @@ def api_server(args):
                 Running the uvicorn with:
                 Apps: {apps}
                 Workers: {num_workers}
-                Host: {args.hostname}:{args.port}
+                Host: {args.host}:{args.port}
                 Timeout: {worker_timeout}
                 Logfiles: {access_logfile}
-                Access Logformat: {access_logformat}
                 ================================================================="""
             )
         )
         ssl_cert, ssl_key = _get_ssl_cert_and_key_filepaths(args)
-        setproctitle(f"airflow api_server -- host:{args.hostname} port:{args.port}")
+        setproctitle(f"airflow api_server -- host:{args.host} port:{args.port}")
         uvicorn.run(
             "airflow.api_fastapi.main:app",
-            host=args.hostname,
+            host=args.host,
             port=args.port,
             workers=num_workers,
             timeout_keep_alive=worker_timeout,

airflow/cli/commands/remote_commands/config_command.py

@@ -292,6 +292,42 @@ def message(self) -> str:
         config=ConfigParameter("webserver", "force_log_out_after"),
         renamed_to=ConfigParameter("webserver", "session_lifetime_minutes"),
     ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "web_server_host"),
+        renamed_to=ConfigParameter("api", "host"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "web_server_port"),
+        renamed_to=ConfigParameter("api", "port"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "workers"),
+        renamed_to=ConfigParameter("api", "workers"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "web_server_worker_timeout"),
+        renamed_to=ConfigParameter("api", "worker_timeout"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "web_server_ssl_cert"),
+        renamed_to=ConfigParameter("api", "ssl_cert"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "web_server_ssl_key"),
+        renamed_to=ConfigParameter("api", "ssl_key"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "access_logfile"),
+        renamed_to=ConfigParameter("api", "access_logfile"),
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "error_logfile"),
+        was_deprecated=False,
+    ),
+    ConfigChange(
+        config=ConfigParameter("webserver", "access_logformat"),
+        was_deprecated=False,
+    ),
     # policy
     ConfigChange(
         config=ConfigParameter("policy", "airflow_local_settings"),

airflow/config_templates/config.yml

@@ -1322,6 +1322,57 @@ api:
       type: string
       example: ~
       default: "http://localhost:9091"
+    host:
+      description: |
+        The ip specified when starting the api server
+      version_added: ~
+      type: string
+      example: ~
+      default: "0.0.0.0"
+    port:
+      description: |
+        The port on which to run the api server
+      version_added: ~
+      type: string
+      example: ~
+      default: "9091"
+    workers:
+      description: |
+        Number of workers to run on the API server
+      version_added: ~
+      type: string
+      example: ~
+      default: "4"
+    worker_timeout:
+      description: |
+        Number of seconds the API server waits before timing out on a worker
+      version_added: ~
+      type: string
+      example: ~
+      default: "120"
+    access_logfile:
+      description: |
+        Log files for the api server. '-' means log to stderr.
+      version_added: ~
+      type: string
+      example: ~
+      default: "-"
+    ssl_cert:
+      description: |
+        Paths to the SSL certificate and key for the api server. When both are
+        provided SSL will be enabled. This does not change the api server port.
+      version_added: ~
+      type: string
+      example: ~
+      default: ""
+    ssl_key:
+      description: |
+        Paths to the SSL certificate and key for the api server. When both are
+        provided SSL will be enabled. This does not change the api server port.
+      version_added: ~
+      type: string
+      example: ~
+      default: ""
     auth_backends:
       description: |
         Comma separated list of auth backends to authenticate users of the API. See
@@ -1534,36 +1585,6 @@ webserver:
       example: "America/New_York"
       # Default is left as UTC for now so the date's don't "suddenly" change on upgrade
       default: "UTC"
-    web_server_host:
-      description: |
-        The ip specified when starting the web server
-      version_added: ~
-      type: string
-      example: ~
-      default: "0.0.0.0"
-    web_server_port:
-      description: |
-        The port on which to run the web server
-      version_added: ~
-      type: string
-      example: ~
-      default: "8080"
-    web_server_ssl_cert:
-      description: |
-        Paths to the SSL certificate and key for the web server. When both are
-        provided SSL will be enabled. This does not change the web server port.
-      version_added: ~
-      type: string
-      example: ~
-      default: ""
-    web_server_ssl_key:
-      description: |
-        Paths to the SSL certificate and key for the web server. When both are
-        provided SSL will be enabled. This does not change the web server port.
-      version_added: ~
-      type: string
-      example: ~
-      default: ""
     session_backend:
       description: |
         The type of backend used to store web session data, can be ``database`` or ``securecookie``. For the
@@ -1590,13 +1611,6 @@ webserver:
       type: string
       example: ~
       default: "120"
-    web_server_worker_timeout:
-      description: |
-        Number of seconds the gunicorn webserver waits before timing out on a worker
-      version_added: ~
-      type: string
-      example: ~
-      default: "120"
     worker_refresh_batch_size:
       description: |
         Number of workers to refresh at a time through Gunicorn's built-in worker management.
@@ -1636,13 +1650,6 @@ webserver:
       sensitive: true
       example: ~
       default: "{SECRET_KEY}"
-    workers:
-      description: |
-        Number of workers to run the Gunicorn web server
-      version_added: ~
-      type: string
-      example: ~
-      default: "4"
     worker_class:
       description: |
         The worker class gunicorn should use. Choices include
@@ -1666,30 +1673,6 @@ webserver:
       type: string
       example: ~
       default: "sync"
-    access_logfile:
-      description: |
-        Log files for the gunicorn webserver. '-' means log to stderr.
-      version_added: ~
-      type: string
-      example: ~
-      default: "-"
-    error_logfile:
-      description: |
-        Log files for the gunicorn webserver. '-' means log to stderr.
-      version_added: ~
-      type: string
-      example: ~
-      default: "-"
-    access_logformat:
-      description: |
-        Access log format for gunicorn webserver.
-        default format is ``%%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s"``
-        See `Gunicorn Settings: 'access_log_format' Reference
-        <https://docs.gunicorn.org/en/stable/settings.html#access-log-format>`__ for more details
-      version_added: 2.0.0
-      type: string
-      example: ~
-      default: ""
     expose_config:
       description: |
         Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values