e6d2b92200
During an upgrade, nova pins the version of RPC calls to the minimum seen across all services. This ensures that old services do not receive data they cannot handle. After the upgrade is complete, all nova services are supposed to be reloaded via SIGHUP to cause them to check again the RPC versions of services and use the new latest version which should now be supported by all running services. Due to a bug [1] in oslo.service, sending services SIGHUP is currently broken. We replaced the HUP with a restart for the nova_compute container for bug 1821362, but not other nova services. It seems we need to restart all nova services to allow the RPC version pin to be removed. Testing in a Queens to Rocky upgrade, we find the following in the logs: Automatically selected compute RPC version 5.0 from minimum service version 30 However, the service version in Rocky is 35. There is a second issue in that it takes some time for the upgraded services to update the nova services database table with their new version. We need to wait until all nova-compute services have done this before the restart is performed, otherwise the RPC version cap will remain in place. There is currently no interface in nova available for checking these versions [2], so as a workaround we use a configurable delay with a default duration of 30 seconds. Testing showed it takes about 10 seconds for the version to be updated, so this gives us some headroom. This change restarts all nova services after an upgrade, after a 30 second delay. [1] https://bugs.launchpad.net/oslo.service/+bug/1715374 [2] https://bugs.launchpad.net/nova/+bug/1833542 Change-Id: Ia6fc9011ee6f5461f40a1307b72709d769814a79 Closes-Bug: #1833069 Related-Bug: #1833542
47 lines
1.3 KiB
YAML
47 lines
1.3 KiB
YAML
---
|
|
# Create new set of configs on nodes
|
|
- include_tasks: config.yml
|
|
|
|
- include_tasks: bootstrap_service.yml
|
|
|
|
- name: Stopping all nova services except nova-compute
|
|
become: true
|
|
kolla_docker:
|
|
action: "stop_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
name: "{{ item.value.container_name }}"
|
|
with_dict: "{{ nova_services }}"
|
|
when:
|
|
- "'nova-compute' not in item.key"
|
|
- inventory_hostname in groups[item.value.group]
|
|
- item.value.enabled | bool
|
|
- nova_safety_upgrade | bool
|
|
|
|
# TODO(donghm): Flush_handlers to restart nova services
|
|
# should be run in serial nodes to decrease downtime if
|
|
# the previous task did not run. Update when the
|
|
# Ansible strategy module for rolling upgrade is finished.
|
|
|
|
- name: Flush handlers
|
|
meta: flush_handlers
|
|
|
|
- name: Migrate Nova database
|
|
vars:
|
|
nova_api: "{{ nova_services['nova-api'] }}"
|
|
become: true
|
|
kolla_docker:
|
|
action: "start_container"
|
|
common_options: "{{ docker_common_options }}"
|
|
detach: False
|
|
environment:
|
|
KOLLA_OSM:
|
|
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
|
|
image: "{{ nova_api.image }}"
|
|
labels:
|
|
BOOTSTRAP:
|
|
name: "bootstrap_nova"
|
|
restart_policy: "never"
|
|
volumes: "{{ nova_api.volumes }}"
|
|
run_once: True
|
|
delegate_to: "{{ groups[nova_api.group][0] }}"
|