openstack/sunbeam-charms
Code Issues Proposed changes

[keystone-k8s] configure charm when rotate id-svc fails

Browse Source 
secret-rotate can be the first hook executed on certain failure modes,
such as the re-creation of a pod during a secret-rotate execution. This
can lead the keystone to be unconfigured + unstarted. Rotating
identity-service credentials necessitates keystone to be up and running.

Try to configure the charm if can't connect to keystone while trying to
create the new service account.

If after the new configure, the charm status is not active (happens when
a mandatory relation is missing for example), fail the event to be
retried later.

Closes-Bug: #2051257
Change-Id: I782232ca61c58f1040d1834c54f4c07c18a2e5d0
This commit is contained in:
Guillaume Boutry 2024-06-11 13:57:45 +02:00
parent 5b902a232a
commit 914456f91e
No known key found for this signature in database
GPG Key ID: E95E3326872E55DE
2 changed files with 160 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
charms/keystone-k8s/src/charm.py
View File

@ -46,6 +46,7 @@ import charms.keystone_k8s.v0.identity_credentials as sunbeam_cc_svc
import charms.keystone_k8s.v0.identity_resource as sunbeam_ops_svc
import charms.keystone_k8s.v1.identity_service as sunbeam_id_svc
import jinja2
import keystoneauth1.exceptions
import ops.charm
import ops.pebble
import ops_sunbeam.charm as sunbeam_charm
@ -781,10 +782,10 @@ export OS_AUTH_VERSION=3
and len(CREDENTIALS_SECRET_PREFIX) : # noqa: E203
]
username = f"{username}-{suffix}"
password = pwgen.pwgen(12)
password = str(pwgen.pwgen(12))
logger.info(f"Creating service account with username {username}")
self.keystone_manager.create_service_account(username, password)
self._create_service_account_with_retry(event, username, password)
olduser = event.secret.get_content(refresh=True).get("username")
event.secret.set_content(
{"username": username, "password": password}
@ -799,6 +800,29 @@ export OS_AUTH_VERSION=3
{"old_service_users": json.dumps(service_users_to_delete)}
)
def _create_service_account_with_retry(
self, event: ops.EventBase, username: str, password: str
) -> dict:
"""Create a service account, tries to configure the charm if connection fails."""
try:
return self.keystone_manager.create_service_account(
username, password
)
except keystoneauth1.exceptions.ConnectFailure:
logger.debug("Failed to connect to keystone", exc_info=True)
self.configure_charm(event)
if self.status.status.name != "active":
logger.debug("Configure charm did not configure to completion")
# note(gboutry): This is not caught by a sunbeam guard,
# this will fail the hook.
raise sunbeam_guard.BlockedExceptionError(
"Failed to create service account"
)
# note(gboutry): If successfully configured, retry the service account creation
# let bubble up this exception, if it fails again,
# this hook should be retried again in the future.
return self.keystone_manager.create_service_account(username, password)
def _on_secret_remove(self, event: ops.charm.SecretRemoveEvent):
logger.info(f"secret-remove triggered for label {event.secret.label}")
if (

135
charms/keystone-k8s/tests/unit/test_keystone_charm.py
View File

@ -26,6 +26,8 @@ from unittest.mock import (
)
import charm
import keystoneauth1.exceptions
import ops_sunbeam.guard as sunbeam_guard
import ops_sunbeam.test_utils as test_utils
@ -56,7 +58,7 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
"pwgen",
]
def add_id_relation(self) -> str:
def add_id_relation(self) -> int:
"""Add amqp relation."""
rel_id = self.harness.add_relation("identity-service", "cinder")
self.harness.add_relation_unit(rel_id, "cinder/0")
@ -415,6 +417,137 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
"""Test secret-rotate event for label credential_keys on non leader unit."""
self._test_non_leader_on_secret_rotate(label="credential-keys")
def test_leader_on_secret_rotate_identity_service_secret(self):
"""Test secret-rotate event for label identity_service_secret on leader unit."""
create_service_account = MagicMock(side_effect=[{"name": "cinder"}])
configure_mock = MagicMock(
side_effect=self.harness.charm.configure_charm
)
self._test_secret_rotate_identity_credentials(
create_service_mock=create_service_account,
configure_mock=configure_mock,
)
self.assertEqual(create_service_account.call_count, 1)
self.assertEqual(configure_mock.call_count, 0)
def test_leader_on_secret_rotate_identity_service_secret_when_failing_to_connect_once(
self,
):
"""When the secret rotate hook fails to connect once, it should retry.
The hook will try to configure the charm, and then retry to create the service account.
"""
create_service_account = MagicMock(
side_effect=[
keystoneauth1.exceptions.ConnectFailure(
"Failed to connect..."
),
{"name": "cinder"},
]
)
configure_mock = MagicMock(
side_effect=self.harness.charm.configure_charm
)
self._test_secret_rotate_identity_credentials(
create_service_mock=create_service_account,
configure_mock=configure_mock,
)
self.assertEqual(create_service_account.call_count, 2)
self.assertEqual(configure_mock.call_count, 1)
def test_leader_on_secret_rotate_identity_service_secret_when_failing_to_connect_twice(
self,
):
"""This will fail to connect twice, and then raise an error."""
create_service_account = MagicMock(
side_effect=[
keystoneauth1.exceptions.ConnectFailure(
"Failed to connect..."
),
keystoneauth1.exceptions.ConnectFailure(
"Failed to connect..."
),
]
)
configure_mock = MagicMock(
side_effect=self.harness.charm.configure_charm
)
with self.assertRaises(keystoneauth1.exceptions.ConnectFailure):
self._test_secret_rotate_identity_credentials(
create_service_mock=create_service_account,
configure_mock=configure_mock,
)
self.assertEqual(create_service_account.call_count, 2)
self.assertEqual(configure_mock.call_count, 1)
def test_leader_on_secret_rotate_identity_service_secret_when_unexpected_error(
self,
):
"""This is an unhandled exception, it should have been bubbled up."""
create_service_account = MagicMock(
side_effect=Exception("I am unexpected..."),
)
configure_mock = MagicMock(
side_effect=self.harness.charm.configure_charm
)
with self.assertRaises(Exception):
self._test_secret_rotate_identity_credentials(
create_service_mock=create_service_account,
configure_mock=configure_mock,
)
self.assertEqual(create_service_account.call_count, 1)
self.assertEqual(configure_mock.call_count, 0)
def test_leader_on_secret_rotate_identity_service_secret_when_configured_not_active(
self,
):
"""Keystone is not ready, it should raise a blocked exception."""
create_service_account = MagicMock(
side_effect=keystoneauth1.exceptions.ConnectFailure(
"Failed to connect..."
),
)
configure_mock = MagicMock(
side_effect=self.harness.charm.configure_charm
)
with self.assertRaises(sunbeam_guard.BlockedExceptionError):
self._test_secret_rotate_identity_credentials(
create_service_mock=create_service_account,
configure_mock=configure_mock,
remove_ingress=True,
)
self.assertEqual(create_service_account.call_count, 1)
self.assertEqual(configure_mock.call_count, 1)
def _test_secret_rotate_identity_credentials(
self,
create_service_mock: MagicMock,
configure_mock: MagicMock,
remove_ingress=False,
):
test_utils.add_complete_ingress_relation(self.harness)
test_utils.add_complete_db_relation(self.harness)
test_utils.add_complete_peer_relation(self.harness)
self.harness.set_leader()
self.harness.container_pebble_ready("keystone")
rel_id = self.add_id_relation()
rel_data = self.harness.get_relation_data(
rel_id, self.harness.model.app.name
)
label = charm.CREDENTIALS_SECRET_PREFIX + "svc_" + "cinder"
secret_id = rel_data["service-credentials"]
if remove_ingress:
rel = self.harness.charm.model.get_relation("ingress-public")
rel_id = rel.id
self.harness.remove_relation(rel_id)
self.km_mock.create_service_account = create_service_mock
self.harness.charm.configure_charm = configure_mock
self.harness.trigger_secret_rotation(secret_id, label=label)
def test_on_secret_changed_with_fernet_keys_and_fernet_secret_same(self):
"""Test secret change event when fernet keys and secret have same content."""
test_utils.add_complete_ingress_relation(self.harness)