Kp 8688 logrotate rsyslog

roles/korp-backend/defaults/main.yml

@@ -45,8 +45,8 @@ mk_additional_directories:
   - {path: /v/appl/, owner: root, group: root, mode: "u+rwx,g+rwxs,o+rx"}
   - {path: /v/corpora/data, owner: root, group: clarin, mode: "u+rwx,g+rwxs,o+rx"}
   - {path: "{{ cwb_registry }}", owner: root, group: clarin, mode: "u+rwx,g+rwxs,o+rx"}
-  - {path: /data1/korp/log/, owner: root, group: root, mode: "u+rwx,g+rwxs,o+rwx"}
-  - {path: /data1/korp/log/korp-py, owner: gunicorn, group: root, mode: "u+rwx,g+rwxs,o+rwx"}
+  - {path: /var/log/korp/, owner: root, group: root, mode: "u+rwx,g+rwxs,o+rwx"}
+  - {path: /data1/korp/log/, owner: root, group: root, mode: "u+rwx,g+rwxs,o+rwx"} # just because of old cgi stuff
   - {path: "{{ backend_cache_dir }}", owner: gunicorn, group: apache, mode: "u+rwx,g+rwxs,o-rwx"}
   - {path: "{{ korp_git_root }}", owner: root, group: clarin, mode: "u+rwx,g+rwxs,o+rx"}
   - {path: /v/appl/utils/, owner: root, group: clarin, mode: "u+rwx,g+rwxs,o-rwx"}

roles/korp-backend/files/11-korp-py.conf

@@ -0,0 +1,9 @@
+$MaxMessageSize 64k
+
+:msg, startswith, "[korp.py"
+local0.* /var/log/korp/korp.log
+&stop
+
+:msg, startswith, "[auth.py"
+local0.* /var/log/korp/korp-authserver.log
+&stop

roles/korp-backend/files/LOG_README.md

@@ -0,0 +1,23 @@
+# Logging
+
+There are three server applications that in theory can write logs on Korp:
+
+1. `korp.py`, ie. the Korp backend
+2. `korp-auth.py`, an authoritization checking server
+3. `korp_download.cgi` and `korp.cgi`, legacy cgi programs that handle
+   downloading the results of queries
+
+`korp.py` and `korp-auth.py` are configured to write logs to syslog,
+`/dev/log`. A syslog service called `rsyslog` is configured on the
+Korp machine to write their logs to `/var/log/korp/` as `korp.log` and
+`korp-auth.log`, respectively. rsyslog is also capable of forwarding
+the logs to an external log consumer. See
+`/etc/rsyslog.d/11-korp-py.conf`.
+
+`korp_download.cgi` and `korp.cgi` write logs in the old-fashioned way
+(without any protection against simultaneous writes) to files in
+`/v/korp/log`. If this situation persists for a long time, we could
+consider handling their logs with syslog too.
+
+All these logs are currently being rotated monthly and deleted after
+15 months.

roles/korp-backend/handlers/main.yml

@@ -25,3 +25,9 @@
     name: korp-authserver
     state: restarted
   become: yes
+
+- name: restart rsyslog
+  systemd:
+    name: rsyslog
+    state: restarted
+  become: yes

roles/korp-backend/tasks/main.yml

@@ -332,6 +332,20 @@
     mode: "0644"
   notify: restart korp-backend
 
+- name: Install rsyslog configuration and logging README
+  ansible.builtin.copy:
+    src: "{{ item.src }}" 11-korp-py.conf
+    dest: "{{ item.dest }}" /etc/rsyslog.d/11-korp-py.conf
+    mode: "0644"
+  loop:
+    - src: 11-korp-py.conf
+      dest: /etc/rsyslog.d/11-korp-py.conf
+    - src: LOG_README.md
+      dest: /var/log/korp/README.md
+    - src: LOG_README.md
+      dest: /v/korp/log/README.md
+  notify: restart rsyslog
+
 - name: configure logrotate for backend
   ansible.builtin.template:
     src: "korp_logrotate"

roles/korp-backend/templates/authserver_config.py

@@ -5,13 +5,16 @@
 # For log levels
 import logging
 
+LOG_USING_NATIVE_PYTHON = False
+LOG_USING_SYSLOG = True
+
 # Host and port for the WSGI server
 WSGI_HOST = "0.0.0.0"
 WSGI_PORT = 1235
 
 # Database host and port
 DBHOST = "{{ korp_db_server }}"
-DBPORT = {{ korp_db_port }}
+DBPORT = {{korp_db_port}}
 # Database name
 DBNAME = "korp_auth"
 # Username and password for database access
@@ -21,3 +24,4 @@
 # Log file and level
 LOG_FILE = "/data1/korp/log/korp-auth-py.log"
 LOG_LEVEL = logging.INFO  # in non-logging version, WARNING
+LOG_FORMAT = "[auth.py %(levelname)s %(process)d @ %(asctime)s] %(message)s"

roles/korp-backend/templates/korp_logrotate

@@ -1,13 +1,27 @@
-/v/korp/log/*log {
+/var/log/korp/*.log {
     monthly
-    rotate 6
+    rotate 15
+    # to delete old logfiles even if there aren't new ones coming in, due
+    # to changes in naming or whatever. 455 days ~= 15 months.
+    maxage 455
     compress
+    delaycompress
+    postrotate
+        systemctl restart rsyslog
+    endscript
+    # no missingok, we want to know if log files don't exist
+    # no notifempty, then leave an empty rotated file, we want to know
+    # no sharedscripts, wait as little as possible before restarting rsyslogd
+}
 
+# This is now just for legacy cgi-based download stuff
+/v/korp/log/*log {
+    monthly
+    rotate 15
+    maxage 455
+    compress
     missingok
     notifempty
     sharedscripts
     delaycompress
-    postrotate
-        /bin/systemctl restart korp-backend.service
-    endscript
 }

roles/korp-backend/templates/logger_plugin_config.py

@@ -7,18 +7,8 @@
 
 import logging
 
-
-# Base directory for log files
-LOG_BASEDIR = "/v/korp/log"
-
-# Log filename format string (for str.format()), which may include a
-# directory part, to be appended to LOG_BASEDIR to get the full log
-# file path. The supported format keys are: year, mon, mday, hour,
-# min, sec (the corresponding tm_X values returned by
-# time.localtime()), and pid (process id), which should be included to
-# avoid having multiple processes writing to the same log file when
-# using a WSGI server with multiple worker processes.
-LOG_FILENAME_FORMAT = "korp-py.log"
+LOG_USING_NATIVE_PYTHON = False
+LOG_USING_SYSLOG = True
 
 # Default log level
 LOG_LEVEL = logging.INFO
@@ -41,7 +31,7 @@
 #   since the epoch
 # %(message)s contains the actual message of the form "Item: Value".
 LOG_FORMAT = (
-    "[korp.py %(levelname)s %(process)d:%(request)d @ %(asctime)s]" " %(message)s"
+    "[korp.py %(levelname)s %(request)d @ %(asctime)s]" " %(message)s"
 )
 
 # The maximum length of a log message, including the fixed part; 0 for