Radosław Piliszek c2d0bf30ea Coordinate haproxy and keepalived restarts
Keepalived and haproxy cooperate to provide control plane HA in
kolla-ansible deployments.
Certain care should be exerted to avoid prolonged availability
loss during reconfigurations and upgrades.
This patch aims to provide this care.
There is nothing special about keepalived upgrade compared to
reconfig, hence it is simplified to run the same code as for
deploy.
The broken logic of safe upgrade is replaced by common handler
code which's goal is to ensure we down current master only after
we have backups ready.

This change introduces a switch to kolla_docker module that allows
to ignore missing containers (as they are logically stopped).
ignore_missing is the switch's name.
All tests are included.

Change-Id: I22ddec5f7ee4a7d3d502649a158a7e005fe29c48
2020-10-04 16:58:24 +02:00

180 lines
5.6 KiB
YAML

---
# NOTE(yoctozepto): this handler dance is to ensure we delay restarting master
# keepalived and haproxy which control VIP address until we have working backups.
# This could be improved by checking if backup keepalived do not report FAULT state.
# Master node is handled specially to let it close down connections and only then
# drop the VIP address by stopping keepalived service.
# NOTE(yoctozepto): we need fresh VIP address placement info (facts may be old)
- name: Check IP addresses on the API interface
vars:
version: "{{ '6' if api_address_family == 'ipv6' else '4' }}"
become: true
command: ip -{{ version }} -o addr show dev {{ api_interface }}
register: ip_addr_output
changed_when: false
when:
- kolla_action != "config"
listen:
- Restart haproxy container
- Restart keepalived container
- name: Group HA nodes by status
vars:
re_safe_address: "{{ kolla_internal_vip_address | regex_escape }}"
group_by:
key: kolla_ha_is_master_{{ ip_addr_output.stdout is regex('\b' + re_safe_address + '\b') }}
when:
- kolla_action != "config"
listen:
- Restart haproxy container
- Restart keepalived container
- name: Stop backup keepalived container
become: true
kolla_docker:
action: "stop_container"
# NOTE(yoctozepto): backup node might not have keepalived yet - ignore
ignore_missing: true
common_options: "{{ docker_common_options }}"
name: "keepalived"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_False is defined
- inventory_hostname in groups.kolla_ha_is_master_False
listen:
- Restart keepalived container
- name: Restart backup haproxy container
vars:
service_name: "haproxy"
service: "{{ haproxy_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_False is defined
- inventory_hostname in groups.kolla_ha_is_master_False
listen:
- Restart haproxy container
- Restart keepalived container
notify:
- Wait for backup haproxy to start
- name: Wait for backup haproxy to start
wait_for:
host: "{{ api_interface_address }}"
port: "{{ haproxy_monitor_port }}"
- name: Start backup keepalived container
vars:
service_name: "keepalived"
service: "{{ haproxy_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_False is defined
- inventory_hostname in groups.kolla_ha_is_master_False
listen:
- Restart keepalived container
notify:
- Wait for virtual IP to appear
# NOTE(yoctozepto): This is to ensure haproxy can close any open connections
# to the VIP address.
- name: Stop master haproxy container
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "haproxy"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_True is defined
- inventory_hostname in groups.kolla_ha_is_master_True
listen:
- Restart keepalived container
- name: Stop master keepalived container
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "keepalived"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_True is defined
- inventory_hostname in groups.kolla_ha_is_master_True
listen:
- Restart keepalived container
- name: Start master haproxy container
vars:
service_name: "haproxy"
service: "{{ haproxy_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_True is defined
- inventory_hostname in groups.kolla_ha_is_master_True
listen:
- Restart haproxy container
- Restart keepalived container
notify:
- Wait for master haproxy to start
- name: Wait for master haproxy to start
wait_for:
host: "{{ api_interface_address }}"
port: "{{ haproxy_monitor_port }}"
- name: Start master keepalived container
vars:
service_name: "keepalived"
service: "{{ haproxy_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- groups.kolla_ha_is_master_True is defined
- inventory_hostname in groups.kolla_ha_is_master_True
listen:
- Restart keepalived container
notify:
- Wait for virtual IP to appear
- name: Wait for virtual IP to appear
wait_for:
host: "{{ kolla_internal_vip_address }}"
port: "{{ haproxy_monitor_port }}"