Merge "Add support for encrypting backend Keystone HAProxy traffic"

ansible/certificates.yml

@@ -1,4 +1,7 @@
 ---
+- import_playbook: gather-facts.yml
+  when: kolla_enable_tls_backend | default(false) | bool
+
 - name: Apply role certificates
   hosts: localhost
   roles:

ansible/group_vars/all.yml

@@ -747,11 +747,18 @@ haproxy_user: "openstack"
 haproxy_enable_external_vip: "{{ 'no' if kolla_same_external_internal_vip | bool else 'yes' }}"
 kolla_enable_tls_internal: "no"
 kolla_enable_tls_external: "{{ kolla_enable_tls_internal if kolla_same_external_internal_vip | bool else 'no' }}"
-kolla_external_fqdn_cert: "{{ node_config }}/certificates/haproxy.pem"
+kolla_certificates_dir: "{{ node_config }}/certificates"
-kolla_internal_fqdn_cert: "{{ node_config }}/certificates/haproxy-internal.pem"
+kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy.pem"
-kolla_external_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy.crt"
+kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy-internal.pem"
-kolla_internal_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy-internal.crt"
+kolla_external_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy.crt"
+kolla_internal_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy-internal.crt"
 kolla_copy_ca_into_containers: "no"
+kolla_verify_tls_backend: "yes"
+haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
+haproxy_backend_cacert_dir: "/etc/ssl/certs"
+kolla_enable_tls_backend: "no"
+kolla_tls_backend_cert: "{{ kolla_certificates_dir }}/backend-cert.pem"
+kolla_tls_backend_key: "{{ kolla_certificates_dir }}/backend-key.pem"
 
 ####################
 # Kibana options

ansible/inventory/all-in-one

@@ -35,6 +35,9 @@ compute
 [baremetal:children]
 control
 
+[tls-backend:children]
+control
+
 [grafana:children]
 monitoring
 

ansible/inventory/multinode

@@ -39,6 +39,9 @@ compute
 storage
 monitoring
 
+[tls-backend:children]
+control
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]

ansible/roles/certificates/defaults/main.yml

@@ -1,3 +0,0 @@
----
-# Directory on deploy node (localhost) in which certificates are generated.
-certificates_dir: "{{ node_config }}/certificates"

ansible/roles/certificates/tasks/generate.yml

@@ -1,21 +1,33 @@
 ---
 - name: Ensuring private internal directory exist
   file:
-    path: "{{ certificates_dir }}/private/internal"
+    path: "{{ kolla_certificates_dir }}/private/internal"
     state: "directory"
     recurse: yes
     mode: "0770"
 
 - name: Ensuring private external directory exist
   file:
-    path: "{{ certificates_dir }}/private/external"
+    path: "{{ kolla_certificates_dir }}/private/external"
     state: "directory"
     recurse: yes
     mode: "0770"
 
+- name: Ensuring backend certificate and key directories exist
+  file:
+    path: "{{ item | dirname }}"
+    state: "directory"
+    recurse: yes
+    mode: "0770"
+  when:
+    - kolla_enable_tls_backend | bool
+  with_items:
+    - "{{ kolla_tls_backend_cert }}"
+    - "{{ kolla_tls_backend_key }}"
+
 - name: Ensuring ca directory exist
   file:
-    path: "{{ certificates_dir }}/ca"
+    path: "{{ kolla_certificates_dir }}/ca"
     state: "directory"
     recurse: yes
     mode: "0770"
@@ -24,36 +36,36 @@
     - name: Creating external SSL configuration file
       template:
         src: "{{ item }}.j2"
-        dest: "{{ certificates_dir }}/{{ item }}"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
         mode: "0660"
       with_items:
         - "openssl-kolla.cnf"
     - name: Creating external Key
       command: creates="{{ item }}" openssl genrsa -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/external/external.key"
+        - "{{ kolla_certificates_dir }}/private/external/external.key"
     - name: Setting permissions on external key
       file:
-        path: "{{ certificates_dir }}/private/external/external.key"
+        path: "{{ kolla_certificates_dir }}/private/external/external.key"
         mode: "0660"
         state: file
     - name: Creating external Server Certificate
       command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
-        -config {{ certificates_dir }}/openssl-kolla.cnf \
+        -config {{ kolla_certificates_dir }}/openssl-kolla.cnf \
         -days 3650 \
         -extensions v3_req \
-        -key {{ certificates_dir }}/private/external/external.key \
+        -key {{ kolla_certificates_dir }}/private/external/external.key \
         -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/external/external.crt"
+        - "{{ kolla_certificates_dir }}/private/external/external.crt"
     - name: Creating external CA Certificate File
       copy:
-        src: "{{ certificates_dir }}/private/external/external.crt"
+        src: "{{ kolla_certificates_dir }}/private/external/external.crt"
         dest: "{{ kolla_external_fqdn_cacert }}"
         mode: "0660"
     - name: Creating external Server PEM File
       assemble:
-        src: "{{ certificates_dir }}/private/external"
+        src: "{{ kolla_certificates_dir }}/private/external"
         dest: "{{ kolla_external_fqdn_cert }}"
         mode: "0660"
   when:
@@ -62,14 +74,14 @@
 - block:
     - name: Copy the external certificate crt to be the internal when internal + external are same network
       copy:
-        src: "{{ certificates_dir }}/private/external/external.crt"
+        src: "{{ kolla_certificates_dir }}/private/external/external.crt"
-        dest: "{{ certificates_dir }}/private/internal/internal.crt"
+        dest: "{{ kolla_certificates_dir }}/private/internal/internal.crt"
         remote_src: yes
         mode: "0660"
     - name: Copy the external certificate key to be the internal when internal + external are same network
       copy:
-        src: "{{ certificates_dir }}/private/external/external.key"
+        src: "{{ kolla_certificates_dir }}/private/external/external.key"
-        dest: "{{ certificates_dir }}/private/internal/internal.key"
+        dest: "{{ kolla_certificates_dir }}/private/internal/internal.key"
         remote_src: yes
         mode: "0660"
     - name: Copy the external PEM file to be the internal when internal + external are same network
@@ -93,38 +105,72 @@
     - name: Creating internal SSL configuration file
       template:
         src: "{{ item }}.j2"
-        dest: "{{ certificates_dir }}/{{ item }}"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
         mode: "0660"
       with_items:
         - "openssl-kolla-internal.cnf"
     - name: Creating internal Key
       command: creates="{{ item }}" openssl genrsa -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/internal/internal.key"
+        - "{{ kolla_certificates_dir }}/private/internal/internal.key"
     - name: Setting permissions on internal key
       file:
-        path: "{{ certificates_dir }}/private/internal/internal.key"
+        path: "{{ kolla_certificates_dir }}/private/internal/internal.key"
         mode: "0660"
         state: file
     - name: Creating internal Server Certificate
       command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
-        -config {{ certificates_dir }}/openssl-kolla-internal.cnf \
+        -config {{ kolla_certificates_dir }}/openssl-kolla-internal.cnf \
         -days 3650 \
         -extensions v3_req \
-        -key {{ certificates_dir }}/private/internal/internal.key \
+        -key {{ kolla_certificates_dir }}/private/internal/internal.key \
         -out {{ item }}
       with_items:
-        - "{{ certificates_dir }}/private/internal/internal.crt"
+        - "{{ kolla_certificates_dir }}/private/internal/internal.crt"
     - name: Creating internal CA Certificate File
       copy:
-        src: "{{ certificates_dir }}/private/internal/internal.crt"
+        src: "{{ kolla_certificates_dir }}/private/internal/internal.crt"
         dest: "{{ kolla_internal_fqdn_cacert }}"
         mode: "0660"
     - name: Creating internal Server PEM File
       assemble:
-        src: "{{ certificates_dir }}/private/internal"
+        src: "{{ kolla_certificates_dir }}/private/internal"
         dest: "{{ kolla_internal_fqdn_cert }}"
         mode: "0660"
   when:
     - kolla_enable_tls_internal | bool
     - not kolla_same_external_internal_vip | bool
+
+- block:
+    - name: Creating backend SSL configuration file
+      template:
+        src: "{{ item }}.j2"
+        dest: "{{ kolla_certificates_dir }}/{{ item }}"
+        mode: "0660"
+      with_items:
+        - "openssl-kolla-backend.cnf"
+    - name: Creating backend Key
+      command: creates="{{ item }}" openssl genrsa -out {{ item }}
+      with_items:
+        - "{{ kolla_tls_backend_key }}"
+    - name: Setting permissions on backend key
+      file:
+        path: "{{ kolla_tls_backend_key }}"
+        mode: "0660"
+        state: file
+    - name: Creating backend Server Certificate
+      command: creates="{{ item }}" openssl req -new -nodes -sha256 -x509 \
+        -config {{ kolla_certificates_dir }}/openssl-kolla-backend.cnf \
+        -days 3650 \
+        -extensions v3_req \
+        -key {{ kolla_tls_backend_key }} \
+        -out {{ item }}
+      with_items:
+        - "{{ kolla_tls_backend_cert }}"
+    - name: Creating backend Certificate file to be included in container trusted ca-certificates
+      copy:
+        src: "{{ kolla_tls_backend_cert }}"
+        dest: "{{ kolla_certificates_dir }}/ca/backend-cert.crt"
+        mode: "0660"
+  when:
+    - kolla_enable_tls_backend | bool

ansible/roles/certificates/templates/openssl-kolla-backend.cnf.j2

@@ -0,0 +1,18 @@
+[req]
+prompt = no
+distinguished_name = req_distinguished_name
+req_extensions = v3_req
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = NC
+localityName = RTP
+organizationalUnitName = kolla
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+{% for host in groups['tls-backend']%}
+IP.{{ loop.index }} = {{ 'api' | kolla_address(host) }}
+{% endfor %}

ansible/roles/haproxy-config/defaults/main.yml

@@ -13,3 +13,4 @@ haproxy_backend_http_extra: []
 haproxy_backend_tcp_extra: []
 
 haproxy_health_check: "check inter 2000 rise 2 fall 5"
+haproxy_health_check_ssl: "check check-ssl inter 2000 rise 2 fall 5"

ansible/roles/haproxy-config/templates/haproxy_single_service_listen.cfg.j2

@@ -10,7 +10,7 @@ userlist {{ service_name }}-user
 {%- macro listen_macro(service_name, service_port, listen_port,
                        service_mode, external,
                        haproxy_http_extra, haproxy_tcp_extra, host_group,
-                       custom_member_list, auth_user, auth_pass) %}
+                       custom_member_list, auth_user, auth_pass, tls_backend) %}
 listen {{ service_name }}
     {% if service_mode == 'redirect' %}
     mode http
@@ -59,10 +59,21 @@ listen {{ service_name }}
     {{ custom_member }}
             {% endfor %}
         {% else %}
+            {% set backend_tls_info = '' %}
+            {% if tls_backend|bool %}
+                {% set haproxy_health_check_final = haproxy_health_check_ssl %}
+                {% if kolla_verify_tls_backend|bool %}
+                    {% set backend_tls_info = 'ssl verify required ca-file %s'|format(haproxy_backend_cacert) %}
+                {% else %}
+                    {% set backend_tls_info = 'ssl verify none' %}
+                {% endif %}
+            {% else %}
+                {% set haproxy_health_check_final = haproxy_health_check %}
+            {% endif %}
             {% for host in groups[host_group] %}
                 {% set host_name = hostvars[host]['ansible_hostname'] %}
                 {% set host_ip = 'api' | kolla_address(host) %}
-    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check }}
+    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check_final }} {{ backend_tls_info }}
             {% endfor %}
         {% endif %}
     {% endif %}
@@ -86,6 +97,7 @@ listen {{ service_name }}
         {# Additional options can be defined in config, and are additive to the global extras #}
         {% set haproxy_tcp_extra = haproxy_service.frontend_tcp_extra|default([]) + haproxy_service.backend_tcp_extra|default([]) + haproxy_frontend_tcp_extra + haproxy_backend_tcp_extra %}
         {% set haproxy_http_extra = haproxy_service.frontend_http_extra|default([]) + haproxy_service.backend_http_extra|default([]) + haproxy_frontend_http_extra + haproxy_backend_http_extra %}
+        {% set tls_backend = haproxy_service.tls_backend|default(false) %}
         {# Allow for basic auth #}
         {% set auth_user = haproxy_service.auth_user|default() %}
         {% set auth_pass = haproxy_service.auth_pass|default() %}
@@ -94,6 +106,6 @@ listen {{ service_name }}
         {% endif %}
 {{ listen_macro(haproxy_name, haproxy_service.port, listen_port,
                 mode, external, haproxy_http_extra, haproxy_tcp_extra,
-                host_group, custom_member_list, auth_user, auth_pass) }}
+                host_group, custom_member_list, auth_user, auth_pass, tls_backend) }}
     {% endif %}
 {%- endfor -%}

ansible/roles/haproxy-config/templates/haproxy_single_service_split.cfg.j2

@@ -53,7 +53,7 @@ frontend {{ service_name }}_front
 
 {%- macro backend_macro(service_name, listen_port, service_mode, host_group,
                         custom_member_list, backend_http_extra,
-                        backend_tcp_extra, auth_user, auth_pass) %}
+                        backend_tcp_extra, auth_user, auth_pass, tls_backend) %}
 backend {{ service_name }}_back
     {% if service_mode == 'redirect' %}
     mode http
@@ -79,10 +79,21 @@ backend {{ service_name }}_back
     {{ custom_member }}
         {% endfor %}
     {% else %}
+        {% set backend_tls_info = '' %}
+        {% if tls_backend|bool %}
+            {% set haproxy_health_check_final = haproxy_health_check_ssl %}
+            {% if kolla_verify_tls_backend|bool %}
+                {% set backend_tls_info = 'ssl verify required ca-file %s'|format(haproxy_backend_cacert) %}
+            {% else %}
+                {% set backend_tls_info = 'ssl verify none' %}
+            {% endif %}
+        {% else %}
+            {% set haproxy_health_check_final = haproxy_health_check %}
+        {% endif %}
         {% for host in groups[host_group] %}
             {% set host_name = hostvars[host]['ansible_hostname'] %}
             {% set host_ip = 'api' | kolla_address(host) %}
-    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check }}
+    server {{ host_name }} {{ host_ip }}:{{ listen_port }} {{ haproxy_health_check_final }} {{ backend_tls_info }}
         {% endfor %}
     {% endif %}
 {% endmacro %}
@@ -107,6 +118,7 @@ backend {{ service_name }}_back
         {% set backend_tcp_extra = haproxy_service.backend_tcp_extra|default([]) %}
         {% set frontend_http_extra = haproxy_service.frontend_http_extra|default([]) + haproxy_frontend_http_extra %}
         {% set backend_http_extra = haproxy_service.backend_http_extra|default([]) %}
+        {% set tls_backend = haproxy_service.tls_backend|default(false) %}
         {# Allow for basic auth #}
         {% set auth_user = haproxy_service.auth_user|default() %}
         {% set auth_pass = haproxy_service.auth_pass|default() %}
@@ -119,7 +131,7 @@ backend {{ service_name }}_back
         {% if haproxy_service.mode != 'redirect' %}
 {{ backend_macro(haproxy_name, listen_port, mode, host_group,
                  custom_member_list, backend_http_extra, backend_tcp_extra,
-                 auth_user, auth_pass) }}
+                 auth_user, auth_pass, tls_backend) }}
         {% endif %}
     {% endif %}
 {%- endfor -%}

ansible/roles/haproxy/tasks/config.yml

@@ -125,6 +125,20 @@
   notify:
     - Restart haproxy container
 
+- name: Copying over extra CA certificates
+  vars:
+    service: "{{ haproxy_services['haproxy'] }}"
+  become: true
+  copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/haproxy/ca-certificates"
+    mode: "0644"
+  when:
+    - inventory_hostname in groups[service.group]
+    - kolla_copy_ca_into_containers | bool
+  notify:
+    - Restart haproxy container
+
 - name: Copying over haproxy start script
   vars:
     service: "{{ haproxy_services['haproxy'] }}"

ansible/roles/haproxy/templates/haproxy_main.cfg.j2

@@ -18,6 +18,9 @@ global
     ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
     tune.ssl.default-dh-param 4096
     {% endif %}
+    {% if kolla_enable_tls_internal | bool or kolla_enable_tls_external | bool %}
+    ca-base {{ haproxy_backend_cacert_dir }}
+    {% endif %}
 
 defaults
     log global

ansible/roles/keystone/defaults/main.yml

@@ -14,18 +14,21 @@ keystone_services:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: false
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_public_port }}"
         listen_port: "{{ keystone_public_listen_port }}"
       keystone_external:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: true
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_public_port }}"
         listen_port: "{{ keystone_public_listen_port }}"
       keystone_admin:
         enabled: "{{ enable_keystone }}"
         mode: "http"
         external: false
+        tls_backend: "{{ keystone_enable_tls_backend }}"
         port: "{{ keystone_admin_port }}"
         listen_port: "{{ keystone_admin_listen_port }}"
   keystone-ssh:
@@ -141,3 +144,8 @@ keystone_ks_services:
       - {'interface': 'admin', 'url': '{{ keystone_admin_url }}'}
       - {'interface': 'internal', 'url': '{{ keystone_internal_url }}'}
       - {'interface': 'public', 'url': '{{ keystone_public_url }}'}
+
+####################
+# TLS
+####################
+keystone_enable_tls_backend: "{{ kolla_enable_tls_backend }}"

ansible/roles/keystone/tasks/config.yml

@@ -38,19 +38,9 @@
   run_once: True
   register: keystone_domain_directory
 
-- name: Copying over extra CA certificates
+- include_tasks: copy-certs.yml
-  become: true
-  copy:
-    src: "{{ node_config }}/certificates/ca/"
-    dest: "{{ node_config_directory }}/{{ item.key }}/ca-certificates"
-    mode: "0644"
   when:
-    - item.value.enabled | bool
+    - kolla_copy_ca_into_containers | bool or keystone_enable_tls_backend | bool
-    - inventory_hostname in groups[item.value.group]
-    - kolla_copy_ca_into_containers | bool
-  with_dict: "{{ keystone_services }}"
-  notify:
-    - "Restart {{ item.key }} container"
 
 - name: Copying over config.json files for services
   template:

ansible/roles/keystone/tasks/copy-certs.yml

@@ -0,0 +1,6 @@
+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ keystone_services }}"

ansible/roles/keystone/templates/keystone.json.j2

@@ -34,7 +34,19 @@
             "dest": "/etc/{{ keystone_dir }}/wsgi-keystone.conf",
             "owner": "keystone",
             "perm": "0600"
-        }
+        }{% if keystone_enable_tls_backend | bool %},
+        {
+            "source": "{{ container_config_directory }}/keystone-cert.pem",
+            "dest": "/etc/keystone/certs/keystone-cert.pem",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/keystone-key.pem",
+            "dest": "/etc/keystone/certs/keystone-key.pem",
+            "owner": "keystone",
+            "perm": "0600"
+        }{% endif %}
     ],
     "permissions": [
         {

ansible/roles/keystone/templates/wsgi-keystone.conf.j2

@@ -5,6 +5,9 @@
 {% set python_path = '/var/lib/kolla/venv/lib/python' + distro_python_version + '/site-packages' %}
 {% endif %}
 {% set binary_path = '/usr/bin' if keystone_install_type == 'binary' else '/var/lib/kolla/venv/bin' %}
+{% if keystone_enable_tls_backend | bool %}
+LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so
+{% endif %}
 Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_public_listen_port }}
 Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_admin_listen_port }}
 
@@ -42,6 +45,12 @@ LogLevel info
     ErrorLog "{{ keystone_log_dir }}/keystone-apache-public-error.log"
     LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
     CustomLog "{{ keystone_log_dir }}/keystone-apache-public-access.log" logformat
+
+{% if keystone_enable_tls_backend | bool %}
+    SSLEngine on
+    SSLCertificateFile /etc/keystone/certs/keystone-cert.pem
+    SSLCertificateKeyFile /etc/keystone/certs/keystone-key.pem
+{% endif %}
 </VirtualHost>
 
 <VirtualHost *:{{ keystone_admin_listen_port }}>
@@ -56,4 +65,10 @@ LogLevel info
     ErrorLog "{{ keystone_log_dir }}/keystone-apache-admin-error.log"
     LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
     CustomLog "{{ keystone_log_dir }}/keystone-apache-admin-access.log" logformat
+
+{% if keystone_enable_tls_backend | bool %}
+    SSLEngine on
+    SSLCertificateFile /etc/keystone/certs/keystone-cert.pem
+    SSLCertificateKeyFile /etc/keystone/certs/keystone-key.pem
+{% endif %}
 </VirtualHost>

ansible/roles/service-cert-copy/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+- name: "{{ project_name }} | Copying over extra CA certificates"
+  become: true
+  copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ item.key }}/ca-certificates"
+    mode: "0644"
+  when:
+    - kolla_copy_ca_into_containers | bool
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: "{{ project_name }} | Copying over backend internal TLS certificate"
+  vars:
+    certs:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
+      - "{{ kolla_certificates_dir }}/{{ project_name }}-cert.pem"
+      - "{{ kolla_tls_backend_cert }}"
+    backend_tls_cert: "{{ lookup('first_found', certs) }}"
+  copy:
+    src: "{{ backend_tls_cert }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ project_name }}-cert.pem"
+    mode: "0644"
+  become: true
+  when:
+    - item.value.haproxy is defined
+    - item.value.haproxy.values() | selectattr('enabled', 'defined') | map(attribute='enabled') | map('bool') | select | list | length > 0
+    - item.value.haproxy.values() | selectattr('tls_backend', 'defined') | map(attribute='tls_backend') | map('bool') | select | list | length > 0
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: "{{ project_name }} | Copying over backend internal TLS key"
+  vars:
+    keys:
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/{{ project_name }}-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
+      - "{{ kolla_certificates_dir }}/{{ project_name }}-key.pem"
+      - "{{ kolla_tls_backend_key }}"
+    backend_tls_key: "{{ lookup('first_found', keys) }}"
+  copy:
+    src: "{{ backend_tls_key }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ project_name }}-key.pem"
+    mode: "0600"
+  become: true
+  when:
+    - item.value.haproxy is defined
+    - item.value.haproxy.values() | selectattr('enabled', 'defined') | map(attribute='enabled') | map('bool') | select | list | length > 0
+    - item.value.haproxy.values() | selectattr('tls_backend', 'defined') | map(attribute='tls_backend') | map('bool') | select | list | length > 0
+  with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - "Restart {{ item.key }} container"

doc/source/admin/advanced-configuration.rst

@@ -99,12 +99,12 @@ The default for TLS is disabled, to enable TLS networking:
 .. code-block:: yaml
 
    kolla_enable_tls_external: "yes"
-   kolla_external_fqdn_cert: "{{ node_config }}/certificates/mycert.pem"
+   kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/mycert.pem"
 
    and/or
 
    kolla_enable_tls_internal: "yes"
-   kolla_internal_fqdn_cert: "{{ node_config }}/certificates/mycert-internal.pem"
+   kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/mycert-internal.pem"
 
 
 .. note::
@@ -181,7 +181,7 @@ service containers to enable trust for those CA certificates. This is required
 for any certificates that are either self-signed or signed by a private CA,
 and are not already present in the service image trust store.
 
-All certificate file names will have the "kolla-customca-" prefix appended to
+All certificate file names will have the "kolla-customca-" prefix prepended to
 it when it is copied into the containers. For example, if a certificate file is
 named "internal.crt", it will be named "kolla-customca-internal.crt" in the
 containers.
@@ -192,6 +192,11 @@ the ``/usr/local/share/ca-certificates/`` directory.
 For Centos and Red Hat Linux containers, the certificate files will be copied
 to the ``/etc/pki/ca-trust/source/anchors/`` directory.
 
+In addition, the ``openstack_cacert`` should be configured with the path to
+the cacert in the container. For example, if the self-signed certificate task
+was used and the deployment is on ubuntu, the path would be:
+"/etc/pki/ca-trust/source/anchors/kolla-customca-haproxy-internal.crt"
+
 .. _service-config:
 
 OpenStack Service Configuration in Kolla

etc/kolla/globals.yml

@@ -184,11 +184,18 @@
 # allow clients to perform authentication.
 #kolla_enable_tls_internal: "no"
 #kolla_enable_tls_external: "{{ kolla_enable_tls_internal if kolla_same_external_internal_vip | bool else 'no' }}"
-#kolla_external_fqdn_cert: "{{ node_config }}/certificates/haproxy.pem"
+#kolla_certificates_dir: "{{ node_config }}/certificates"
-#kolla_internal_fqdn_cert: "{{ node_config }}/certificates/haproxy-internal.pem"
+#kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy.pem"
-#kolla_external_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy.crt"
+#kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/haproxy-internal.pem"
-#kolla_internal_fqdn_cacert: "{{ node_config }}/certificates/ca/haproxy-internal.crt"
+#kolla_external_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy.crt"
+#kolla_internal_fqdn_cacert: "{{ kolla_certificates_dir }}/ca/haproxy-internal.crt"
 #kolla_copy_ca_into_containers: "no"
+#kolla_verify_tls_backend: "yes"
+#haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
+#haproxy_backend_cacert_dir: "/etc/ssl/certs"
+#kolla_enable_tls_backend: "no"
+#kolla_tls_backend_cert: "{{ kolla_certificates_dir }}/backend-cert.pem"
+#kolla_tls_backend_key: "{{ kolla_certificates_dir }}/backend-key.pem"
 
 ################
 # Region options

releasenotes/notes/copy-certificate-authority-into-containers-860cbda3384dd731.yaml

@@ -12,10 +12,8 @@ features:
 
 issues:
   - |
-    Python <= 2.7.9 will not trust self-signed or privately signed CAs even
+    Python Requests library will not trust self-signed or privately signed CAs
-    if they are added into the OS trusted CA folder and update-ca-trust is
+    even if they are added into the OS trusted CA folder and update-ca-trust is
-    executed. This is also true for the Python Requests library, regardless of
+    executed. For services that rely on the Python Requests library, either CA
-    Python version. For services that run Python <= 2.7.9 or rely on the
+    verification must be explicitly disabled in the service or the path to the
-    Python Requests library, either CA verification must be explicitly disabled
+    CA certificate must be configured using the ``openstack_cacert`` parameter.
-    in the service or the path to the CA certificate must be configured using
-    the ``openstack_cacert`` parameter.

releasenotes/notes/encrypt-backend-haproxy-keystone-fb96285d74fb464c.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Added configuration options to enable backend TLS encryption from HAProxy
+    to the Keystone service. When used in conjunction with enabling TLS for
+    service API endpoints, network communcation will be encrypted end to end,
+    from client through HAProxy to the Keystone service.

tests/check-config.sh

@@ -16,6 +16,8 @@ function check_config {
     for f in $(sudo find /etc/kolla \
                 -not -regex /etc/kolla/config.* \
                 -not -regex /etc/kolla/certificates.* \
+                -not -regex .*pem \
+                -not -regex .*key \
                 -not -regex ".*ca-certificates.*" \
                 -not -path /etc/kolla \
                 -not -name admin-openrc.sh \

tests/templates/globals-default.j2

@@ -117,8 +117,8 @@ ceph_nova_user: "cinder"
 {% if tls_enabled %}
 kolla_enable_tls_external: "yes"
 kolla_enable_tls_internal: "yes"
-kolla_verify_internal_ca_certs: "no"
 kolla_copy_ca_into_containers: "yes"
+kolla_enable_tls_backend: "yes"
 {% if base_distro == "ubuntu" or base_distro == "debian" %}
 openstack_cacert: "/usr/local/share/ca-certificates/kolla-customca-haproxy-internal.crt"
 {% endif %}

tests/templates/inventory.j2

@@ -53,6 +53,9 @@ compute
 storage
 monitoring
 
+[tls-backend:children]
+control
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]