
Keepalived and haproxy cooperate to provide control plane HA in kolla-ansible deployments. Certain care should be exerted to avoid prolonged availability loss during reconfigurations and upgrades. This patch aims to provide this care. There is nothing special about keepalived upgrade compared to reconfig, hence it is simplified to run the same code as for deploy. The broken logic of safe upgrade is replaced by common handler code which's goal is to ensure we down current master only after we have backups ready. This change introduces a switch to kolla_docker module that allows to ignore missing containers (as they are logically stopped). ignore_missing is the switch's name. All tests are included. Change-Id: I22ddec5f7ee4a7d3d502649a158a7e005fe29c48
180 lines
5.6 KiB
YAML
180 lines
5.6 KiB
YAML
---
|
|
# NOTE(yoctozepto): this handler dance is to ensure we delay restarting master
|
|
# keepalived and haproxy which control VIP address until we have working backups.
|
|
# This could be improved by checking if backup keepalived do not report FAULT state.
|
|
# Master node is handled specially to let it close down connections and only then
|
|
# drop the VIP address by stopping keepalived service.
|
|
|
|
# NOTE(yoctozepto): we need fresh VIP address placement info (facts may be old)
|
|
- name: Check IP addresses on the API interface
|
|
vars:
|
|
version: "{{ '6' if api_address_family == 'ipv6' else '4' }}"
|
|
become: true
|
|
command: ip -{{ version }} -o addr show dev {{ api_interface }}
|
|
register: ip_addr_output
|
|
changed_when: false
|
|
when:
|
|
- kolla_action != "config"
|
|
listen:
|
|
- Restart haproxy container
|
|
- Restart keepalived container
|
|
|
|
- name: Group HA nodes by status
|
|
vars:
|
|
re_safe_address: "{{ kolla_internal_vip_address | regex_escape }}"
|
|
group_by:
|
|
key: kolla_ha_is_master_{{ ip_addr_output.stdout is regex('\b' + re_safe_address + '\b') }}
|
|
when:
|
|
- kolla_action != "config"
|
|
listen:
|
|
- Restart haproxy container
|
|
- Restart keepalived container
|
|
|
|
- name: Stop backup keepalived container
|
|
become: true
|
|
kolla_docker:
|
|
action: "stop_container"
|
|
# NOTE(yoctozepto): backup node might not have keepalived yet - ignore
|
|
ignore_missing: true
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "keepalived"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_False is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_False
|
|
listen:
|
|
- Restart keepalived container
|
|
|
|
- name: Restart backup haproxy container
|
|
vars:
|
|
service_name: "haproxy"
|
|
service: "{{ haproxy_services[service_name] }}"
|
|
become: true
|
|
kolla_docker:
|
|
action: "recreate_or_restart_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "{{ service.container_name }}"
|
|
image: "{{ service.image }}"
|
|
privileged: "{{ service.privileged | default(False) }}"
|
|
volumes: "{{ service.volumes }}"
|
|
dimensions: "{{ service.dimensions }}"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_False is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_False
|
|
listen:
|
|
- Restart haproxy container
|
|
- Restart keepalived container
|
|
notify:
|
|
- Wait for backup haproxy to start
|
|
|
|
- name: Wait for backup haproxy to start
|
|
wait_for:
|
|
host: "{{ api_interface_address }}"
|
|
port: "{{ haproxy_monitor_port }}"
|
|
|
|
- name: Start backup keepalived container
|
|
vars:
|
|
service_name: "keepalived"
|
|
service: "{{ haproxy_services[service_name] }}"
|
|
become: true
|
|
kolla_docker:
|
|
action: "recreate_or_restart_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "{{ service.container_name }}"
|
|
image: "{{ service.image }}"
|
|
privileged: "{{ service.privileged | default(False) }}"
|
|
volumes: "{{ service.volumes }}"
|
|
dimensions: "{{ service.dimensions }}"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_False is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_False
|
|
listen:
|
|
- Restart keepalived container
|
|
notify:
|
|
- Wait for virtual IP to appear
|
|
|
|
# NOTE(yoctozepto): This is to ensure haproxy can close any open connections
|
|
# to the VIP address.
|
|
- name: Stop master haproxy container
|
|
become: true
|
|
kolla_docker:
|
|
action: "stop_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "haproxy"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_True is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_True
|
|
listen:
|
|
- Restart keepalived container
|
|
|
|
- name: Stop master keepalived container
|
|
become: true
|
|
kolla_docker:
|
|
action: "stop_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "keepalived"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_True is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_True
|
|
listen:
|
|
- Restart keepalived container
|
|
|
|
- name: Start master haproxy container
|
|
vars:
|
|
service_name: "haproxy"
|
|
service: "{{ haproxy_services[service_name] }}"
|
|
become: true
|
|
kolla_docker:
|
|
action: "recreate_or_restart_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "{{ service.container_name }}"
|
|
image: "{{ service.image }}"
|
|
privileged: "{{ service.privileged | default(False) }}"
|
|
volumes: "{{ service.volumes }}"
|
|
dimensions: "{{ service.dimensions }}"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_True is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_True
|
|
listen:
|
|
- Restart haproxy container
|
|
- Restart keepalived container
|
|
notify:
|
|
- Wait for master haproxy to start
|
|
|
|
- name: Wait for master haproxy to start
|
|
wait_for:
|
|
host: "{{ api_interface_address }}"
|
|
port: "{{ haproxy_monitor_port }}"
|
|
|
|
- name: Start master keepalived container
|
|
vars:
|
|
service_name: "keepalived"
|
|
service: "{{ haproxy_services[service_name] }}"
|
|
become: true
|
|
kolla_docker:
|
|
action: "recreate_or_restart_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "{{ service.container_name }}"
|
|
image: "{{ service.image }}"
|
|
privileged: "{{ service.privileged | default(False) }}"
|
|
volumes: "{{ service.volumes }}"
|
|
dimensions: "{{ service.dimensions }}"
|
|
when:
|
|
- kolla_action != "config"
|
|
- groups.kolla_ha_is_master_True is defined
|
|
- inventory_hostname in groups.kolla_ha_is_master_True
|
|
listen:
|
|
- Restart keepalived container
|
|
notify:
|
|
- Wait for virtual IP to appear
|
|
|
|
- name: Wait for virtual IP to appear
|
|
wait_for:
|
|
host: "{{ kolla_internal_vip_address }}"
|
|
port: "{{ haproxy_monitor_port }}"
|