Enabled TLS encrypted communication for etcd

automation/add_pgnode.yml

@@ -236,6 +236,8 @@
       when: pg_probackup_install|bool
 
     - role: tls_certificate/copy
+      vars:
+        copy_for: "pg"
       when: tls_cert_generate|bool
 
     - role: pgbouncer

automation/deploy_pgcluster.yml

@@ -94,6 +94,11 @@
         timescale_minimal_pg_version: 12 # if enable_timescale is defined
       tags: always
 
+    - role: hostname
+
+    - role: tls_certificate/generate
+      when: tls_cert_generate|bool
+
   tasks:
     - name: Clean dnf cache
       ansible.builtin.command: dnf clean all
@@ -356,8 +361,10 @@
 
     - role: cron
 
-    - role: tls_certificate
+    - role: tls_certificate/copy
       when: tls_cert_generate|bool
+      vars:
+        copy_for: "pg"
 
     - role: pgbouncer
       when: pgbouncer_install|bool

automation/molecule/tests/etcd/etcd.yml

@@ -1,8 +1,13 @@
 ---
 - name: Check etcd health
   ansible.builtin.uri:
-    url: "http://{{ inventory_hostname }}:2379/health"
+    url: "{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379/health"
+    method: GET
     return_content: true
+    validate_certs: "{{ tls_cert_generate | default(false) | bool }}"
+    ca_path: "{{ tls_etcd_ca_cert_path if tls_cert_generate | default(false) | bool else omit }}"
+    client_cert: "{{ tls_etcd_cert_path if tls_cert_generate | default(false) | bool else omit }}"
+    client_key: "{{ tls_etcd_privatekey_path if tls_cert_generate | default(false) | bool else omit }}"
   register: etcd_health_status
   failed_when: "(etcd_health_status.content | from_json).health != 'true'"
   when: dcs_type == "etcd"

automation/roles/confd/templates/confd.toml.j2

@@ -4,7 +4,7 @@ watch = true
 nodes = [
 {% if not dcs_exists|bool and dcs_type == 'etcd' %}
   {% for host in groups['etcd_cluster'] %}
-  "http://{{ hostvars[host]['inventory_hostname'] }}:2379",
+  "{{ patroni_etcd_protocol | default('http', true) }}://{{ hostvars[host]['inventory_hostname'] }}:2379",
   {% endfor %}
 {% endif %}
 {% if dcs_exists|bool and dcs_type == 'etcd' %}

automation/roles/etcd/tasks/main.yml

@@ -86,6 +86,14 @@
     state: directory
   tags: etcd, etcd_conf
 
+- name: Fetch etcd TLS certificate, key and CA from the master node
+  ansible.builtin.include_role:
+    name: ../roles/tls_certificate/copy
+  vars:
+    copy_for: "etcd"
+  when: tls_cert_generate|bool
+  tags: etcd, etcd_conf
+
 - name: Create etcd data directory
   ansible.builtin.file:
     path: "{{ etcd_data_dir }}"
@@ -128,7 +136,12 @@
     - name: Wait until the etcd cluster is healthy
       ansible.builtin.command: >
         /usr/local/bin/etcdctl endpoint health
-        --endpoints=http://{{ inventory_hostname }}:2379
+        --endpoints={{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379
+        {% if tls_cert_generate | default(false) | bool %}
+        --cacert={{ tls_etcd_ca_cert_path | default('/etc/etcd/ca.crt') }}
+        --cert={{ tls_etcd_cert_path | default('/etc/etcd/server.crt') }}
+        --key={{ tls_etcd_privatekey_path | default('/etc/etcd/server.key') }}
+        {% endif %}
       environment:
         ETCDCTL_API: "3"
       register: etcd_health_result

automation/roles/etcd/templates/etcd.conf.j2

@@ -1,13 +1,24 @@
 ETCD_NAME="{{ ansible_hostname }}"
-ETCD_LISTEN_CLIENT_URLS="http://{{ inventory_hostname }}:2379,http://127.0.0.1:2379"
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ inventory_hostname }}:2379"
-ETCD_LISTEN_PEER_URLS="http://{{ inventory_hostname }}:2380"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ inventory_hostname }}:2380"
+ETCD_LISTEN_CLIENT_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379,{{ patroni_etcd_protocol | default('http', true) }}://127.0.0.1:2379"
+ETCD_ADVERTISE_CLIENT_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379"
+ETCD_LISTEN_PEER_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2380"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2380"
 ETCD_INITIAL_CLUSTER_TOKEN="{{ etcd_cluster_name }}"
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd_cluster'] %}{{ hostvars[host]['ansible_hostname'] }}=http://{{ hostvars[host]['inventory_hostname'] }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd_cluster'] %}{{ hostvars[host]['ansible_hostname'] }}={{ patroni_etcd_protocol | default('http', true) }}://{{ hostvars[host]['inventory_hostname'] }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_DATA_DIR="{{ etcd_data_dir }}"
 ETCD_ELECTION_TIMEOUT="5000"
 ETCD_HEARTBEAT_INTERVAL="1000"
 ETCD_INITIAL_ELECTION_TICK_ADVANCE="false"
 ETCD_AUTO_COMPACTION_RETENTION="1"
+{% if tls_cert_generate | default(false) | bool %}
+ETCD_CERT_FILE="{{ tls_etcd_cert_path | default('/etc/etcd/server.crt') }}"
+ETCD_KEY_FILE="{{ tls_etcd_privatekey_path | default('/etc/etcd/server.key') }}"
+ETCD_TRUSTED_CA_FILE="{{ tls_etcd_ca_cert_path | default('/etc/etcd/ca.crt') }}"
+ETCD_PEER_CERT_FILE="{{ tls_etcd_cert_path | default('/etc/etcd/server.crt') }}"
+ETCD_PEER_KEY_FILE="{{ tls_etcd_privatekey_path | default('/etc/etcd/server.key') }}"
+ETCD_PEER_TRUSTED_CA_FILE="{{ tls_etcd_ca_cert_path | default('/etc/etcd/ca.crt') }}"
+ETCD_PEER_CLIENT_CERT_AUTH="true"
+ETCD_CLIENT_CERT_AUTH="true"
+ETCD_TLS_MIN_VERSION="TLS1.2"
+{% endif %}

automation/roles/patroni/templates/patroni.yml.j2

@@ -38,20 +38,32 @@ restapi:
 {% if not dcs_exists|bool and dcs_type == 'etcd' %}
 etcd3:
   hosts: {% for host in groups['etcd_cluster'] %}{{ hostvars[host]['inventory_hostname'] }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+  {% if tls_cert_generate | default(false) | bool %}
+  protocol: https
+  cacert: {{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}
+  cert: {{ tls_cert_path | default('/etc/tls/server.crt') }}
+  key: {{ tls_privatekey_path | default('/etc/tls/server.key') }}
+  {% endif %}
 {% endif %}
+
 {% if dcs_exists|bool and dcs_type == 'etcd' %}
 etcd3:
   hosts: {% for etcd_hosts in patroni_etcd_hosts %}{{etcd_hosts.host}}:{{etcd_hosts.port}}{% if not loop.last %},{% endif %}{% endfor %}
 
+  {% if tls_cert_generate | default(false) | bool %}
+  protocol: https
+  cacert: {{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}
+  cert: {{ tls_cert_path | default('/etc/tls/server.crt') }}
+  key: {{ tls_privatekey_path | default('/etc/tls/server.key') }}
+  {% endif %}
+
   {% if patroni_etcd_username | default('') | length > 0 %}
   username: {{ patroni_etcd_username | default('') }}
   {% endif %}
   {% if patroni_etcd_password | default('') | length > 0 %}
   password: {{ patroni_etcd_password }}
   {% endif %}
-  {% if patroni_etcd_protocol | default('') | length > 0 %}
-  protocol: {{ patroni_etcd_protocol }}
-  {% endif %}
 {% endif %}
 
 {% if dcs_type == 'consul' %}

automation/roles/patroni/templates/pg_hba.conf.j2

@@ -85,14 +85,14 @@
 
 # TYPE      DATABASE                 USER                     ADDRESS                  METHOD
 {% for client in postgresql_pg_hba %}
-  {{ client.type.ljust(10) |default('host') }}{{ client.database.ljust(25) |default('all') }}{{ client.user.ljust(25) |default('all') }}{{ client.address.ljust(25) |default('') }}{{ client.method |default('md5') }} {{ client.options |default(None) }}
+  {{ client.type.ljust(10) |default('{% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}') }}{{ client.database.ljust(25) |default('all') }}{{ client.user.ljust(25) |default('all') }}{{ client.address.ljust(25) |default('') }}{{ client.method |default('md5') }} {{ client.options |default(None) }}
 {% endfor %}
 {% for patroni in groups['postgres_cluster'] %}
-  host      all                      all                      {{ hostvars[patroni]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      all                      all                      {{ hostvars[patroni]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
 {% endfor %}
 # Allow replication connections from localhost, by a user with the
 # replication privilege.
-  host      replication              {{ patroni_replication_username }}               localhost                trust
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      replication              {{ patroni_replication_username }}               localhost                trust
 {% for host in groups['postgres_cluster'] %}
-  host      replication              {{ patroni_replication_username }}               {{ hostvars[host]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      replication              {{ patroni_replication_username }}               {{ hostvars[host]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
 {% endfor %}

automation/roles/tls_certificate/copy/tasks/main.yml

@@ -1,42 +1,46 @@
 ---
-# for add_pgnode.yml
-
-- name: Ensure TLS directories exist
-  ansible.builtin.file:
-    path: "{{ item | dirname }}"
-    state: directory
-    owner: "{{ tls_owner | default('postgres') }}"
-    group: "{{ tls_owner | default('postgres') }}"
-    mode: "0750"
-  loop:
-    - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
-    - "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
-
-- name: Fetch TLS certificate and key from master
+- name: Fetch TLS certificate, key and CA from the master node into memory
   run_once: true
-  ansible.builtin.fetch:
+  ansible.builtin.slurp:
     src: "{{ item }}"
-    dest: "files/tls/"
-    validate_checksum: true
-    flat: true
   delegate_to: "{{ groups.master[0] }}"
+  register: tls_files
   loop:
-    - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
-    - "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
+    - "/etc/tls/server.key"
+    - "/etc/tls/server.crt"
+    - "/etc/tls/ca.crt"
 
-- name: Copy TLS certificate and key to replica
+- name: Copy etcd TLS certificate, key and CA to all nodes from memory
   ansible.builtin.copy:
-    src: "files/tls/{{ item.path | basename }}"
+    content: "{{ tls_files.results[item.index].content | b64decode }}"
     dest: "{{ item.path }}"
-    owner: "{{ tls_owner | default('postgres') }}"
-    group: "{{ tls_owner | default('postgres') }}"
+    owner: "etcd"
+    group: "etcd"
     mode: "{{ item.mode }}"
   loop:
-    - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "{{ tls_privatekey_mode | default('0400') }}" }
-    - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "{{ tls_cert_mode | default('0644') }}" }
+    - { index: 0, path: "{{ tls_etcd_privatekey_path | default('/etc/etcd/server.key') }}", mode: "0400" }
+    - { index: 1, path: "{{ tls_etcd_cert_path | default('/etc/etcd/server.crt') }}", mode: "0644" }
+    - { index: 2, path: "{{ tls_etcd_ca_cert_path | default('/etc/etcd/ca.crt') }}", mode: "0644" }
+  when: copy_for == 'etcd'
+
+- block:
+    - name: Create directory {{ tls_privatekey_path | dirname }}
+      ansible.builtin.file:
+        dest: "{{ tls_privatekey_path | dirname }}"
+        state: directory
+        owner: "{{ tls_owner }}"
+        group: "{{ tls_owner }}"
+        mode: "0755"
 
-- name: Delete TLS certificate and key from the ansible controller
-  ansible.builtin.file:
-    path: "files/tls/"
-    state: absent
-  delegate_to: localhost
+    - name: Copy PostgreSQL TLS certificate, key and CA to all nodes
+      ansible.builtin.copy:
+        content: "{{ tls_files.results[item.index].content | b64decode }}"
+        dest: "{{ item.path }}"
+        owner: "{{ tls_owner }}"
+        group: "{{ tls_owner }}"
+        mode: "{{ item.mode }}"
+      loop:
+        - { index: 0, path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "0400" }
+        - { index: 1, path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "0644" }
+        - { index: 2, path: "{{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}", mode: "0644" }
+  when: copy_for == 'pg'

automation/roles/tls_certificate/generate/tasks/main.yml

@@ -0,0 +1,122 @@
+---
+- name: Make sure that the python3-cryptography package is present
+  ansible.builtin.package:
+    name: python3-cryptography
+    state: present
+  register: pack_status
+  until: pack_status is success
+  delay: 5
+  retries: 3
+
+- name: "Clean up existing certificates"
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
+    - "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
+    - "{{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}"
+    - "{{ tls_ca_privatekey_path | default('/etc/tls/ca.key') }}"
+    - "{{ tls_etcd_cert_path | default('/etc/etcd/server.crt') }}"
+    - "{{ tls_etcd_ca_cert_path | default('/etc/etcd/ca.crt') }}"
+    - "{{ tls_etcd_privatekey_path | default('/etc/etcd/server.key') }}"
+    - "/etc/tls"
+
+- ansible.builtin.set_fact:
+    all_san_entries: []
+
+- name: "Gather host-specific network information"
+  ansible.builtin.set_fact:
+    san_entry: >-
+      DNS:{{ ansible_hostname }},DNS:{{ ansible_fqdn }},IP:{{ inventory_hostname }}
+
+- block:
+    - name: "Aggregate all subjectAltName entries"
+      ansible.builtin.set_fact:
+        all_san_entries: "{{ all_san_entries + [hostvars[item].san_entry] }}"
+      with_items: "{{ ansible_play_hosts }}"
+
+    - name: "Join subjectAltName entries into a single string"
+      ansible.builtin.set_fact:
+        subject_alt_name: "{{ all_san_entries | join(',') + ',DNS:localhost,IP:127.0.0.1' }}"
+      when: ansible_play_hosts | length > 1
+
+    - name: "Display Certificate subjectAltName future value"
+      ansible.builtin.debug:
+        var: subject_alt_name
+
+######## Generate CA ########
+    - name: "Ensure TLS directory exist"
+      ansible.builtin.file:
+        path: "/etc/tls"
+        state: directory
+        mode: "0700"
+
+    - name: "Generate CA private key"
+      community.crypto.openssl_privatekey:
+        path: "/etc/tls/ca.key"
+        size: "{{ tls_privatekey_size | default(4096) }}"
+        type: "{{ tls_privatekey_type | default('RSA') }}"
+
+    - name: "Create CSR for CA certificate"
+      community.crypto.openssl_csr_pipe:
+        privatekey_path: "/etc/tls/ca.key"
+        common_name: PostgreSQL CA
+        use_common_name_for_san: false
+        basic_constraints:
+          - 'CA:TRUE'
+        basic_constraints_critical: true
+        key_usage:
+          - keyCertSign
+        key_usage_critical: true
+      register: ca_csr
+
+    - name: "Create self-signed CA certificate from CSR"
+      community.crypto.x509_certificate:
+        path: "/etc/tls/ca.crt"
+        csr_content: "{{ ca_csr.csr }}"
+        privatekey_path: "/etc/tls/ca.key"
+        provider: "{{ tls_cert_provider | default('selfsigned') }}"
+        entrust_not_after: "+{{ tls_cert_valid_days | default(3650) }}d"
+
+######## Generate Server cert/key ########
+    - name: "Create server private key"
+      community.crypto.openssl_privatekey:
+        path: "/etc/tls/server.key"
+        size: "{{ tls_privatekey_size | default(4096) }}"
+        type: "{{ tls_privatekey_type | default('RSA') }}"
+
+    - name: "Create server CSR"
+      community.crypto.openssl_csr_pipe:
+        privatekey_path: "/etc/tls/server.key"
+        common_name: "{{ patroni_cluster_name }}"
+        key_usage:
+          - digitalSignature
+          - keyEncipherment
+          - dataEncipherment
+        extended_key_usage:
+          - clientAuth
+          - serverAuth
+        subject:
+          C: "AL"
+          O: "autobase"
+          CN: "{{ patroni_cluster_name }}"
+        subject_alt_name: "{{ subject_alt_name }}"
+      register: csr
+
+    - name: "Sign server certificate with the CA"
+      community.crypto.x509_certificate_pipe:
+        csr_content: "{{ csr.csr }}"
+        provider: ownca
+        ownca_path: "/etc/tls/ca.crt"
+        ownca_privatekey_path: "/etc/tls/ca.key"
+        ownca_not_after: +{{ tls_cert_valid_days | default(3650) }}d
+        ownca_not_before: "-1d"
+      register: certificate
+
+    - name: "Write server certificate"
+      ansible.builtin.copy:
+        dest: "/etc/tls/server.crt"
+        content: "{{ certificate.certificate }}"
+  delegate_to: "{{ groups.master[0] }}"
+  run_once: true