diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index 491593793c..7952b5f7ee 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -72,6 +72,7 @@ fluentd_image_full: "{{ fluentd_image }}:{{ fluentd_tag }}" syslog_swift_facility: "local0" syslog_haproxy_facility: "local1" +syslog_glance_tls_proxy_facility: "local2" kolla_toolbox_default_volumes: - "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro" @@ -125,6 +126,8 @@ fluentd_input_openstack_services: enabled: "{{ enable_freezer | bool }}" - name: glance enabled: "{{ enable_glance | bool }}" + - name: glance-tls-proxy + enabled: "{{ enable_glance | bool }}" - name: gnocchi enabled: "{{ enable_gnocchi | bool }}" - name: heat diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index ecbfee9ad9..31bb143b48 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -294,6 +294,7 @@ - { name: "etcd", enabled: "{{ enable_etcd }}" } - { name: "freezer", enabled: "{{ enable_freezer }}" } - { name: "glance", enabled: "{{ enable_glance }}" } + - { name: "glance-tls-proxy", enabled: "{{ enable_glance }}" } - { name: "global", enabled: "yes" } - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" } - { name: "grafana", enabled: "{{ enable_grafana }}" } diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 index 911320798f..238efee3de 100644 --- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 @@ -36,6 +36,13 @@ </record> </filter> +<filter syslog.{{ syslog_glance_tls_proxy_facility }}.**> + @type record_transformer + <record> + programname glance-tls-proxy + </record> +</filter> + # Rename internal Fluent message field to match other logs. This removes # all other fields by default, including the original message field. This is # intented to avoid duplication of the log message and to prevent passing diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 71b5da2f3b..9b42b30850 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -105,3 +105,58 @@ </store> {% endif %} </match> + +<match syslog.{{ syslog_glance_tls_proxy_facility }}.**> + @type copy + <store> + @type file + path /var/log/kolla/glance-tls-proxy/glance-tls-proxy.*.log + output_tag false + output_time false + utc + append true + compress gzip + </store> +{% if log_direct_to_elasticsearch %} + <store> + type elasticsearch + host {{ elasticsearch_address }} + port {{ elasticsearch_port }} + scheme {{ fluentd_elasticsearch_scheme }} +{% if fluentd_elasticsearch_path != '' %} + path {{ fluentd_elasticsearch_path }} +{% endif %} +{% if fluentd_elasticsearch_scheme == 'https' %} + ssl_version {{ fluentd_elasticsearch_ssl_version }} + ssl_verify {{ fluentd_elasticsearch_ssl_verify }} +{% endif %} +{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%} + user {{ fluentd_elasticsearch_user }} + password {{ fluentd_elasticsearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ kibana_log_prefix }} + flush_interval 15s + reconnect_on_error true + buffer_type file + buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.* + </store> +{% elif enable_monasca | bool %} + type copy + <store> + @type monasca + keystone_url {{ keystone_internal_url }} + monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} + monasca_log_api_version v3.0 + username {{ monasca_agent_user }} + password {{ monasca_agent_password }} + domain_id default + project_name {{ monasca_control_plane_project }} + message_field_name Payload + buffer_type file + buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.* + max_retry_wait 1800s + disable_retry_limit true + </store> +{% endif %} +</match> diff --git a/ansible/roles/common/templates/cron-logrotate-glance-tls-proxy.conf.j2 b/ansible/roles/common/templates/cron-logrotate-glance-tls-proxy.conf.j2 new file mode 100644 index 0000000000..5f53099180 --- /dev/null +++ b/ansible/roles/common/templates/cron-logrotate-glance-tls-proxy.conf.j2 @@ -0,0 +1,3 @@ +"/var/log/kolla/glance-tls-proxy/*.log" +{ +} diff --git a/ansible/roles/common/templates/cron.json.j2 b/ansible/roles/common/templates/cron.json.j2 index f207e3db3c..b050437e32 100644 --- a/ansible/roles/common/templates/cron.json.j2 +++ b/ansible/roles/common/templates/cron.json.j2 @@ -16,6 +16,7 @@ ( 'etcd', enable_etcd ), ( 'freezer', enable_freezer ), ( 'glance', enable_glance ), + ( 'glance-tls-proxy', enable_glance ), ( 'gnocchi', enable_gnocchi ), ( 'grafana', enable_grafana ), ( 'haproxy', enable_haproxy ), diff --git a/ansible/roles/glance/defaults/main.yml b/ansible/roles/glance/defaults/main.yml index 9cbb3c74a1..145fd53683 100644 --- a/ansible/roles/glance/defaults/main.yml +++ b/ansible/roles/glance/defaults/main.yml @@ -14,7 +14,7 @@ glance_services: dimensions: "{{ glance_api_dimensions }}" haproxy: glance_api: - enabled: "{{ enable_glance }}" + enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}" mode: "http" external: false port: "{{ glance_api_port }}" @@ -24,7 +24,7 @@ glance_services: - "timeout server {{ haproxy_glance_api_server_timeout }}" custom_member_list: "{{ haproxy_members.split(';') }}" glance_api_external: - enabled: "{{ enable_glance }}" + enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}" mode: "http" external: true port: "{{ glance_api_port }}" @@ -33,12 +33,43 @@ glance_services: backend_http_extra: - "timeout server {{ haproxy_glance_api_server_timeout }}" custom_member_list: "{{ haproxy_members.split(';') }}" - + glance-tls-proxy: + container_name: glance_tls_proxy + group: glance-api + host_in_groups: "{{ inventory_hostname in glance_api_hosts }}" + enabled: "{{ glance_enable_tls_backend }}" + image: "{{ glance_tls_proxy_image_full }}" + volumes: "{{ glance_tls_proxy_default_volumes + glance_tls_proxy_extra_volumes }}" + dimensions: "{{ glance_tls_proxy_dimensions }}" + haproxy: + glance_tls_proxy: + enabled: "{{ enable_glance | bool and glance_enable_tls_backend | bool }}" + mode: "http" + external: false + port: "{{ glance_api_port }}" + frontend_http_extra: + - "timeout client {{ haproxy_glance_api_client_timeout }}" + backend_http_extra: + - "timeout server {{ haproxy_glance_api_server_timeout }}" + custom_member_list: "{{ haproxy_tls_members.split(';') }}" + tls_backend: "yes" + glance_tls_proxy_external: + enabled: "{{ enable_glance | bool and glance_enable_tls_backend | bool }}" + mode: "http" + external: true + port: "{{ glance_api_port }}" + frontend_http_extra: + - "timeout client {{ haproxy_glance_api_client_timeout }}" + backend_http_extra: + - "timeout server {{ haproxy_glance_api_server_timeout }}" + custom_member_list: "{{ haproxy_tls_members.split(';') }}" + tls_backend: "yes" #################### # HAProxy #################### haproxy_members: "{% for host in glance_api_hosts %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ glance_api_listen_port }} check inter 2000 rise 2 fall 5;{% endfor %}" +haproxy_tls_members: "{% for host in glance_api_hosts %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ glance_api_listen_port }} check inter 2000 rise 2 fall 5 ssl verify required ca-file {{ haproxy_backend_cacert }};{% endfor %}" #################### # Keystone @@ -92,7 +123,12 @@ glance_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ dock glance_api_tag: "{{ glance_tag }}" glance_api_image_full: "{{ glance_api_image }}:{{ glance_api_tag }}" +glance_tls_proxy_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ glance_install_type }}-haproxy" +glance_tls_proxy_tag: "{{ glance_tag }}" +glance_tls_proxy_image_full: "{{ glance_tls_proxy_image }}:{{ glance_tls_proxy_tag }}" + glance_api_dimensions: "{{ default_container_dimensions }}" +glance_tls_proxy_dimensions: "{{ default_container_dimensions }}" glance_api_default_volumes: - "{{ node_config_directory }}/glance-api/:{{ container_config_directory }}/:ro" @@ -104,9 +140,15 @@ glance_api_default_volumes: # NOTE(yoctozepto): below to support Cinder iSCSI backends - "{% if enable_cinder | bool and enable_cinder_backend_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}" - "{% if enable_cinder | bool and enable_cinder_backend_iscsi | bool %}/dev:/dev{% endif %}" +glance_tls_proxy_default_volumes: + - "{{ node_config_directory }}/glance-tls-proxy/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" + - "kolla_logs:/var/log/kolla/" glance_extra_volumes: "{{ default_extra_volumes }}" glance_api_extra_volumes: "{{ glance_extra_volumes }}" +glance_tls_proxy_extra_volumes: "{{ glance_extra_volumes }}" #################### # Glance @@ -166,3 +208,27 @@ vmware_datastore_name: ################### # Default maximum size of 10Gb glance_cache_max_size: "10737418240" + +#################### +# TLS +#################### +glance_enable_tls_backend: "{{ kolla_enable_tls_backend }}" + +#################### +# Backend TLS proxy +#################### +syslog_server: "{{ api_interface_address }}" +syslog_glance_tls_proxy_facility: "local2" + +glance_tls_proxy_max_connections: 40000 +glance_tls_proxy_processes: 1 +glance_tls_proxy_process_cpu_map: "no" +glance_tls_proxy_defaults_max_connections: 10000 +glance_tls_proxy_http_request_timeout: "10s" +glance_tls_proxy_queue_timeout: "1m" +glance_tls_proxy_connect_timeout: "10s" +glance_tls_proxy_client_timeout: "{{ haproxy_glance_api_client_timeout}}" +glance_tls_proxy_server_timeout: "{{ haproxy_glance_api_server_timeout }}" +glance_tls_proxy_check_timeout: "10s" +# Check http://www.haproxy.org/download/1.5/doc/configuration.txt for available options +glance_tls_proxy_defaults_balance: "roundrobin" diff --git a/ansible/roles/glance/handlers/main.yml b/ansible/roles/glance/handlers/main.yml index e3370f8d9c..5abf923756 100644 --- a/ansible/roles/glance/handlers/main.yml +++ b/ansible/roles/glance/handlers/main.yml @@ -15,3 +15,18 @@ dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" + +- name: Restart glance-tls-proxy container + vars: + service_name: "glance-tls-proxy" + service: "{{ glance_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/glance/tasks/check-containers.yml b/ansible/roles/glance/tasks/check-containers.yml index 7739559377..548fff7cf1 100644 --- a/ansible/roles/glance/tasks/check-containers.yml +++ b/ansible/roles/glance/tasks/check-containers.yml @@ -6,7 +6,7 @@ common_options: "{{ docker_common_options }}" name: "{{ item.value.container_name }}" image: "{{ item.value.image }}" - privileged: "{{ item.value.privileged }}" + privileged: "{{ item.value.privileged | default(omit) }}" environment: "{{ item.value.environment | default(omit) }}" volumes: "{{ item.value.volumes|reject('equalto', '')|list }}" dimensions: "{{ item.value.dimensions }}" diff --git a/ansible/roles/glance/tasks/config.yml b/ansible/roles/glance/tasks/config.yml index 0874a2576c..4d4240ecd5 100644 --- a/ansible/roles/glance/tasks/config.yml +++ b/ansible/roles/glance/tasks/config.yml @@ -37,7 +37,18 @@ - include_tasks: copy-certs.yml when: - - kolla_copy_ca_into_containers | bool + - kolla_copy_ca_into_containers | bool or glance_enable_tls_backend | bool + +- name: Creating TLS backend PEM File + assemble: + src: "{{ node_config_directory }}/glance-tls-proxy/" + dest: "{{ node_config_directory }}/glance-tls-proxy/glance-cert-and-key.pem" + mode: "0660" + regexp: "^glance-(cert|key)\\.pem$" + remote_src: true + become: true + when: + - glance_enable_tls_backend | bool - name: Copying over config.json files for services template: @@ -52,25 +63,24 @@ notify: - Restart {{ item.key }} container -- name: Copying over glance-*.conf +- name: Copying over glance-api.conf vars: - service_name: "{{ item.key }}" + glance_api: "{{ glance_services['glance-api'] }}" merge_configs: sources: - - "{{ role_path }}/templates/{{ item.key }}.conf.j2" + - "{{ role_path }}/templates/glance-api.conf.j2" - "{{ node_custom_config }}/global.conf" - "{{ node_custom_config }}/glance.conf" - - "{{ node_custom_config }}/glance/{{ item.key }}.conf" - - "{{ node_custom_config }}/glance/{{ inventory_hostname }}/{{ item.key }}.conf" - dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.conf" + - "{{ node_custom_config }}/glance/glance-api.conf" + - "{{ node_custom_config }}/glance/{{ inventory_hostname }}/glance-api.conf" + dest: "{{ node_config_directory }}/glance-api/glance-api.conf" mode: "0660" become: true when: - - item.value.enabled | bool - - item.value.host_in_groups | bool - with_dict: "{{ glance_services }}" + - glance_api.enabled | bool + - glance_api.host_in_groups | bool notify: - - Restart {{ item.key }} container + - Restart glance-api container - name: Copying over glance-cache.conf for glance_api vars: @@ -111,18 +121,37 @@ - Restart glance-api container - name: Copying over existing policy file + vars: + glance_api: "{{ glance_services['glance-api'] }}" template: src: "{{ glance_policy_file_path }}" - dest: "{{ node_config_directory }}/{{ item.key }}/{{ glance_policy_file }}" + dest: "{{ node_config_directory }}/glance-api/{{ glance_policy_file }}" mode: "0660" become: true when: - glance_policy_file is defined - - item.value.host_in_groups | bool - - item.value.enabled | bool - with_dict: "{{ glance_services }}" + - glance_api.host_in_groups | bool + - glance_api.enabled | bool notify: - - Restart {{ item.key }} container + - Restart glance-api container + +- name: Copying over glance-haproxy-tls.cfg + vars: + glance_tls_proxy: "{{ glance_services['glance-tls-proxy'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/glance-tls-proxy/glance-tls-proxy.cfg" + mode: "0660" + become: true + with_first_found: + - "{{ node_custom_config }}/glance/{{ inventory_hostname }}/glance-tls-proxy.cfg" + - "{{ node_custom_config }}/glance/glance-tls-proxy.cfg" + - "glance-tls-proxy.cfg.j2" + when: + - glance_tls_proxy.enabled | bool + - glance_tls_proxy.host_in_groups | bool + notify: + - Restart glance-tls-proxy container - include_tasks: check-containers.yml when: kolla_action != "config" diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2 index 296ba757d1..2964503029 100644 --- a/ansible/roles/glance/templates/glance-api.conf.j2 +++ b/ansible/roles/glance/templates/glance-api.conf.j2 @@ -5,7 +5,11 @@ debug = {{ glance_logging_debug }} log_file = /var/log/kolla/glance/glance-api.log use_forwarded_for = true +{% if glance_enable_tls_backend | bool %} +bind_host = 127.0.0.1 +{% else %} bind_host = {{ api_interface_address }} +{% endif %} bind_port = {{ glance_api_listen_port }} workers = {{ openstack_service_workers }} diff --git a/ansible/roles/glance/templates/glance-tls-proxy.cfg.j2 b/ansible/roles/glance/templates/glance-tls-proxy.cfg.j2 new file mode 100644 index 0000000000..d58b00420d --- /dev/null +++ b/ansible/roles/glance/templates/glance-tls-proxy.cfg.j2 @@ -0,0 +1,37 @@ +#jinja2: lstrip_blocks: True +global + chroot /var/lib/haproxy + user glance + group glance + daemon + log {{ syslog_server }}:{{ syslog_udp_port }} {{ syslog_glance_tls_proxy_facility }} + maxconn {{ glance_tls_proxy_max_connections }} + nbproc {{ glance_tls_proxy_processes }} + {% if (glance_tls_proxy_processes | int > 1) and (glance_tls_proxy_process_cpu_map | bool) %} + {% for cpu_idx in range(0, glance_tls_proxy_processes) %} + cpu-map {{ cpu_idx + 1 }} {{ cpu_idx }} + {% endfor %} + {% endif %} + ssl-default-bind-ciphers DEFAULT:!MEDIUM:!3DES + ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 + tune.ssl.default-dh-param 4096 + +defaults + log global + option redispatch + retries 3 + timeout http-request {{ glance_tls_proxy_http_request_timeout }} + timeout queue {{ glance_tls_proxy_queue_timeout }} + timeout connect {{ glance_tls_proxy_connect_timeout }} + timeout client {{ glance_tls_proxy_client_timeout }} + timeout server {{ glance_tls_proxy_server_timeout }} + timeout check {{ glance_tls_proxy_check_timeout }} + balance {{ glance_tls_proxy_defaults_balance }} + maxconn {{ glance_tls_proxy_defaults_max_connections }} + +frontend glance_backend_tls + bind {{ api_interface_address }}:{{ glance_api_listen_port }} ssl crt /etc/glance/certs/glance-cert-and-key.pem + default_backend glance_api + +backend glance_api + server glance-api 127.0.0.1:{{ glance_api_listen_port }} check diff --git a/ansible/roles/glance/templates/glance-tls-proxy.json.j2 b/ansible/roles/glance/templates/glance-tls-proxy.json.j2 new file mode 100644 index 0000000000..27546f2d17 --- /dev/null +++ b/ansible/roles/glance/templates/glance-tls-proxy.json.j2 @@ -0,0 +1,17 @@ +{ + "command": "/usr/sbin/haproxy -W -db -p /run/haproxy.pid -f /etc/glance/glance-tls-proxy.cfg", + "config_files": [ + { + "source": "{{ container_config_directory }}/glance-tls-proxy.cfg", + "dest": "/etc/glance/glance-tls-proxy.cfg", + "owner": "glance", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/glance-cert-and-key.pem", + "dest": "/etc/glance/certs/glance-cert-and-key.pem", + "owner": "glance", + "perm": "0600" + } + ] +} diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst index 34c21a06ed..bed050a5c0 100644 --- a/doc/source/admin/advanced-configuration.rst +++ b/doc/source/admin/advanced-configuration.rst @@ -382,6 +382,10 @@ By default, Swift and HAProxy use ``local0`` and ``local1``, respectively. syslog_swift_facility: "local0" syslog_haproxy_facility: "local1" +If Glance TLS backend is enabled (``glance_enable_tls_backend``), the syslog +facility for the ``glance_tls_proxy`` service uses ``local2`` by default. This +can be set via ``syslog_glance_tls_proxy_facility``. + Mount additional Docker volumes in containers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/releasenotes/notes/encrypt-backend-haproxy-fb96285d74fb464c.yaml b/releasenotes/notes/encrypt-backend-haproxy-fb96285d74fb464c.yaml index 61baa77537..7385b29302 100644 --- a/releasenotes/notes/encrypt-backend-haproxy-fb96285d74fb464c.yaml +++ b/releasenotes/notes/encrypt-backend-haproxy-fb96285d74fb464c.yaml @@ -2,6 +2,7 @@ features: - | Added configuration options to enable backend TLS encryption from HAProxy - to the Keystone, Heat, and cinder service. When used in conjunction with - enabling TLS for service API endpoints, network communcation will be - encrypted end to end, from client through HAProxy to the backend service. + to the Keystone, Glance, Heat, and Cinder services. When used in + conjunction with enabling TLS for service API endpoints, network + communcation will be encrypted end to end, from client through HAProxy to + the backend service.