From 93b61841746e9d781524dd733df895370ad86dfc Mon Sep 17 00:00:00 2001 From: Guillaume Boutry Date: Fri, 7 Feb 2025 12:07:58 +0100 Subject: [PATCH] [ops-sunbeam] Run DB Sync before starting services All services needing to run DB Sync actually need the database to be populated before they start. Starting the services before the database lead to a lot of error logs misleading on root case. Modify configure_unit lifecycle to make db sync run first. Change-Id: I6da0483aba0b0547c84946d540b362c3e5a46d82 Signed-off-by: Guillaume Boutry --- charms/cinder-ceph-k8s/src/charm.py | 8 ++-- charms/designate-bind-k8s/src/charm.py | 3 +- charms/designate-k8s/src/charm.py | 3 +- charms/horizon-k8s/src/charm.py | 3 +- charms/keystone-k8s/src/charm.py | 3 +- charms/nova-k8s/src/charm.py | 17 --------- charms/octavia-k8s/src/charm.py | 4 +- ops-sunbeam/ops_sunbeam/charm.py | 16 +++++++- ops-sunbeam/ops_sunbeam/container_handlers.py | 38 ++++++++++++------- .../ops_sunbeam/ovn/container_handlers.py | 4 +- 10 files changed, 54 insertions(+), 45 deletions(-) diff --git a/charms/cinder-ceph-k8s/src/charm.py b/charms/cinder-ceph-k8s/src/charm.py index 29ab3877..88907c8b 100755 --- a/charms/cinder-ceph-k8s/src/charm.py +++ b/charms/cinder-ceph-k8s/src/charm.py @@ -325,8 +325,8 @@ class CinderCephOperatorCharm(charm.OSBaseOperatorCharmK8S): """Provide database name for cinder services.""" return {"database": "cinder"} - def init_container_services(self): - """Setp ceph keyring and init pebble handlers that are ready.""" + def configure_containers(self): + """Setp ceph keyring and configure container that are ready.""" for ph in self.pebble_handlers: if ph.pebble_ready: # The code for managing ceph client config should move to @@ -359,7 +359,7 @@ class CinderCephOperatorCharm(charm.OSBaseOperatorCharmK8S): ], exception_on_error=True, ) - ph.init_service(self.contexts()) + ph.configure_container(self.contexts()) else: logging.debug( f"Not running init for {ph.service_name}," @@ -368,7 +368,7 @@ class CinderCephOperatorCharm(charm.OSBaseOperatorCharmK8S): raise sunbeam_guard.WaitingExceptionError( "Payload container not ready" ) - super().init_container_services() + super().configure_containers() def _set_or_update_rbd_secret( self, diff --git a/charms/designate-bind-k8s/src/charm.py b/charms/designate-bind-k8s/src/charm.py index eb1cba5a..867b1212 100755 --- a/charms/designate-bind-k8s/src/charm.py +++ b/charms/designate-bind-k8s/src/charm.py @@ -258,9 +258,10 @@ class BindOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): self.check_relation_handlers_ready(event) self.update_owned_relation_data() self.open_ports() + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() - self.run_db_sync() self._state.unit_bootstrapped = True def configure_app_leader(self, event: ops.EventBase) -> None: diff --git a/charms/designate-k8s/src/charm.py b/charms/designate-k8s/src/charm.py index 13dd6708..e028d630 100755 --- a/charms/designate-k8s/src/charm.py +++ b/charms/designate-k8s/src/charm.py @@ -378,9 +378,10 @@ class DesignateOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm): self.check_leader_ready() self.check_relation_handlers_ready(event) self.open_ports() + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() - self.run_db_sync() self.run_pool_update() self._state.unit_bootstrapped = True diff --git a/charms/horizon-k8s/src/charm.py b/charms/horizon-k8s/src/charm.py index 6eaa8985..cd95cdc4 100755 --- a/charms/horizon-k8s/src/charm.py +++ b/charms/horizon-k8s/src/charm.py @@ -236,9 +236,10 @@ class HorizonOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm): """Run configuration on this unit.""" self.check_leader_ready() self.check_relation_handlers_ready(event) + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() - self.run_db_sync() self.configure_plugins(event) self._state.unit_bootstrapped = True diff --git a/charms/keystone-k8s/src/charm.py b/charms/keystone-k8s/src/charm.py index 68e0ab35..3a4fe134 100755 --- a/charms/keystone-k8s/src/charm.py +++ b/charms/keystone-k8s/src/charm.py @@ -1643,9 +1643,10 @@ export OS_AUTH_VERSION=3 self.check_leader_ready() self.check_relation_handlers_ready(event) self.open_ports() + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() - self.run_db_sync() pre_update_fernet_ready = self.unit_fernet_bootstrapped() self.update_fernet_keys_from_peer() # If the wsgi service was running with no tokens it will be in a diff --git a/charms/nova-k8s/src/charm.py b/charms/nova-k8s/src/charm.py index 20efd171..b913c5c6 100755 --- a/charms/nova-k8s/src/charm.py +++ b/charms/nova-k8s/src/charm.py @@ -816,24 +816,7 @@ class NovaOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm): else: logger.debug("Metadata secret not ready") return - # Do not run service check for nova scheduler as it is broken until db - # migrations have run. - scheduler_handler = self.get_named_pebble_handler( - NOVA_SCHEDULER_CONTAINER - ) - scheduler_handler.enable_service_check = False - super().configure_charm(event) - if scheduler_handler.pebble_ready: - logging.debug("Starting nova scheduler service, pebble ready") - # Restart nova-scheduler service after cell1 is created - # Creation of cell1 is part of bootstrap process - scheduler_handler.start_all() - scheduler_handler.enable_service_check = True - else: - logging.debug( - "Not starting nova scheduler service, pebble not ready" - ) def set_config_from_event(self, event: ops.framework.EventBase) -> None: """Set config in relation data.""" diff --git a/charms/octavia-k8s/src/charm.py b/charms/octavia-k8s/src/charm.py index 0d68c496..59383bca 100755 --- a/charms/octavia-k8s/src/charm.py +++ b/charms/octavia-k8s/src/charm.py @@ -374,6 +374,8 @@ class OctaviaOVNOperatorCharm(OctaviaOperatorCharm): self.check_leader_ready() self.check_relation_handlers_ready(event) self.open_ports() + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() for container in [ @@ -388,8 +390,6 @@ class OctaviaOVNOperatorCharm(OctaviaOperatorCharm): OCTAVIA_AGENT_SOCKET_DIR, ] ) - - self.run_db_sync() self._state.unit_bootstrapped = True diff --git a/ops-sunbeam/ops_sunbeam/charm.py b/ops-sunbeam/ops_sunbeam/charm.py index f8536278..1b899bc1 100644 --- a/ops-sunbeam/ops_sunbeam/charm.py +++ b/ops-sunbeam/ops_sunbeam/charm.py @@ -649,6 +649,19 @@ class OSBaseOperatorCharmK8S(OSBaseOperatorCharm): if h.container_name in container_names ] + def configure_containers(self): + """Configure containers.""" + for ph in self.pebble_handlers: + if ph.pebble_ready: + ph.configure_container(self.contexts()) + else: + logging.debug( + f"Not configuring {ph.service_name}, container not ready" + ) + raise sunbeam_guard.WaitingExceptionError( + "Payload container not ready" + ) + def init_container_services(self): """Run init on pebble handlers that are ready.""" for ph in self.pebble_handlers: @@ -689,9 +702,10 @@ class OSBaseOperatorCharmK8S(OSBaseOperatorCharm): self.check_leader_ready() self.check_relation_handlers_ready(event) self.open_ports() + self.configure_containers() + self.run_db_sync() self.init_container_services() self.check_pebble_handlers_ready() - self.run_db_sync() self._state.unit_bootstrapped = True def get_relation_handlers( diff --git a/ops-sunbeam/ops_sunbeam/container_handlers.py b/ops-sunbeam/ops_sunbeam/container_handlers.py index 1f770a8f..85a67799 100644 --- a/ops-sunbeam/ops_sunbeam/container_handlers.py +++ b/ops-sunbeam/ops_sunbeam/container_handlers.py @@ -81,6 +81,7 @@ class PebbleHandler(ops.framework.Object): self.framework.observe( self.charm.on.update_status, self._on_update_status ) + self._files_changed: list[str] = [] def setup_pebble_handler(self) -> None: """Configure handler for pebble ready event.""" @@ -179,15 +180,24 @@ class PebbleHandler(ops.framework.Object): d.path, user=d.user, group=d.group, make_parents=True ) + def _reset_files_changed(self) -> None: + """Reset list of files changed.""" + self._files_changed = [] + + def configure_container( + self, context: sunbeam_core.OPSCharmContexts + ) -> None: + """Write configuration files to the container.""" + self.setup_dirs() + self._files_changed.extend(self.write_config(context)) + self.files_changed(self._files_changed) + def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None: """Initialise service ready for use. - Write configuration files to the container and record - that service is ready for us. + Record that service is ready for us. """ - self.setup_dirs() - changes = self.write_config(context) - self.files_changed(changes) + self.configure_container(context) self.status.set(ActiveStatus("")) def default_container_configs( @@ -213,7 +223,7 @@ class PebbleHandler(ops.framework.Object): return False container = self.charm.unit.get_container(self.container_name) services = container.get_services() - return all([s.is_running() for s in services.values()]) + return all(s.is_running() for s in services.values()) def execute( self, cmd: list[str], exception_on_error: bool = False, **kwargs @@ -324,6 +334,7 @@ class PebbleHandler(ops.framework.Object): f"Starting {service_name} in {self.container_name}" ) container.start(service_name) + self._reset_files_changed() continue if restart: @@ -331,6 +342,7 @@ class PebbleHandler(ops.framework.Object): f"Restarting {service_name} in {self.container_name}" ) container.restart(service_name) + self._reset_files_changed() def stop_all(self) -> None: """Stop services in container.""" @@ -360,10 +372,8 @@ class ServicePebbleHandler(PebbleHandler): Write configuration files to the container and record that service is ready for us. """ - self.setup_dirs() - files_changed = self.write_config(context) - self.files_changed(files_changed) - if files_changed: + self.configure_container(context) + if self._files_changed: self.start_service(restart=True) else: self.start_service(restart=False) @@ -483,7 +493,7 @@ class WSGIPebbleHandler(PebbleHandler): def init_service(self, context: sunbeam_core.OPSCharmContexts) -> None: """Enable and start WSGI service.""" container = self.charm.unit.get_container(self.container_name) - files_changed = self.write_config(context) + self.configure_container(context) try: process = container.exec( ["a2ensite", self.wsgi_service_name], timeout=5 * 60 @@ -499,9 +509,9 @@ class WSGIPebbleHandler(PebbleHandler): ) # ignore for now - pebble is raising an exited too quickly, but it # appears to work properly. - files_changed.extend(self.write_config(context)) - self.files_changed(files_changed) - if files_changed: + self._files_changed.extend(self.write_config(context)) + self.files_changed(self._files_changed) + if self._files_changed: self.start_wsgi(restart=True) else: self.start_wsgi(restart=False) diff --git a/ops-sunbeam/ops_sunbeam/ovn/container_handlers.py b/ops-sunbeam/ops_sunbeam/ovn/container_handlers.py index 810e4ef8..b6793ff7 100644 --- a/ops-sunbeam/ops_sunbeam/ovn/container_handlers.py +++ b/ops-sunbeam/ops_sunbeam/ovn/container_handlers.py @@ -42,9 +42,7 @@ class OVNPebbleHandler(sunbeam_chandlers.ServicePebbleHandler): NOTE: Override default to services being automatically started """ - self.setup_dirs() - changes = self.write_config(context) - self.files_changed(changes) + self.configure_container(context) self.status.set(ops.ActiveStatus("")) @property