From 95424ebb641b1a60e43aeacf73939416222bb24e Mon Sep 17 00:00:00 2001 From: Chi Wai Chan Date: Thu, 4 Jul 2024 18:48:41 +0800 Subject: [PATCH] Add TLS support for tempest-k8s Updated charmcraft.yaml to include receive-ca-cert relation, and inject the cacert to tempest container, and expose the path to the cacert to pebble. Change-Id: I3d7f3775d700643f688fb3b6140058dfb257f2bc --- charms/tempest-k8s/.sunbeam-build.yaml | 3 + charms/tempest-k8s/charmcraft.yaml | 2 + charms/tempest-k8s/src/charm.py | 27 +++++++-- charms/tempest-k8s/src/handlers.py | 58 ++++++++++--------- .../src/templates/tempest-run-wrapper.j2 | 2 +- charms/tempest-k8s/src/utils/cleanup.py | 29 +++++++++- charms/tempest-k8s/src/utils/constants.py | 7 +++ charms/tempest-k8s/tests/unit/test_cleanup.py | 2 + tests/tempest/smoke.yaml.j2 | 2 + 9 files changed, 98 insertions(+), 34 deletions(-) diff --git a/charms/tempest-k8s/.sunbeam-build.yaml b/charms/tempest-k8s/.sunbeam-build.yaml index 03f366df..9f4a96b0 100644 --- a/charms/tempest-k8s/.sunbeam-build.yaml +++ b/charms/tempest-k8s/.sunbeam-build.yaml @@ -2,5 +2,8 @@ external-libraries: - charms.observability_libs.v1.kubernetes_service_patch - charms.grafana_k8s.v0.grafana_dashboard - charms.loki_k8s.v1.loki_push_api + - charms.certificate_transfer_interface.v0.certificate_transfer internal-libraries: - charms.keystone_k8s.v0.identity_resource +templates: + - ca-bundle.pem.j2 diff --git a/charms/tempest-k8s/charmcraft.yaml b/charms/tempest-k8s/charmcraft.yaml index 705d5f69..2f9781e5 100644 --- a/charms/tempest-k8s/charmcraft.yaml +++ b/charms/tempest-k8s/charmcraft.yaml @@ -62,6 +62,8 @@ requires: interface: keystone-resources logging: interface: loki_push_api + receive-ca-cert: + interface: certificate_transfer optional: true provides: diff --git a/charms/tempest-k8s/src/charm.py b/charms/tempest-k8s/src/charm.py index bb9f1986..0e54a053 100755 --- a/charms/tempest-k8s/src/charm.py +++ b/charms/tempest-k8s/src/charm.py @@ -55,12 +55,10 @@ from utils.alert_rules import ( ensure_alert_rules_disabled, update_alert_rules_files, ) -from utils.cleanup import ( - CleanUpError, - run_extensive_cleanup, -) from utils.constants import ( CONTAINER, + OS_CACERT_PATH, + RECEIVE_CA_CERT_RELATION_NAME, TEMPEST_ACCOUNTS_COUNT, TEMPEST_ADHOC_OUTPUT, TEMPEST_CONCURRENCY, @@ -141,6 +139,12 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): "tempest", 0o750, ), + sunbeam_core.ContainerConfigFile( + OS_CACERT_PATH, + "root", + "tempest", + 0o640, + ), ] def get_schedule(self) -> Schedule: @@ -224,6 +228,12 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): if (value := os.environ.get(proxy_var)) } + def _get_os_cacert_environment(self) -> Dict[str, str]: + """Return the path to the OS cacert file if receive-ca-cert relation exist.""" + if not list(self.model.relations[RECEIVE_CA_CERT_RELATION_NAME]): + return {} + return {"OS_CACERT": OS_CACERT_PATH} + def _get_environment_for_tempest( self, variant: TempestEnvVariant ) -> Dict[str, str]: @@ -259,6 +269,7 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): "TEMPEST_OUTPUT": variant.output_path(), } tempest_env.update(self._get_proxy_environment()) + tempest_env.update(self._get_os_cacert_environment()) return tempest_env def _get_cleanup_env(self) -> Dict[str, str]: @@ -278,6 +289,7 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"), } cleanup_env.update(self._get_proxy_environment()) + cleanup_env.update(self._get_os_cacert_environment()) return cleanup_env def get_unit_data(self, key: str) -> Optional[str]: @@ -309,10 +321,13 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): env = self._get_environment_for_tempest(TempestEnvVariant.PERIODIC) pebble = self.pebble_handler() + # Push auxiliary files first before doing anything + pebble.push_auxiliary_files() + try: # do an extensive clean-up before tempest init to remove stalled resources - run_extensive_cleanup(self._get_cleanup_env()) - except CleanUpError: + pebble.run_extensive_cleanup(self._get_cleanup_env()) + except ops.pebble.ExecError: logger.debug("Clean-up failed and tempest init not run.") self.set_tempest_ready(False) return diff --git a/charms/tempest-k8s/src/handlers.py b/charms/tempest-k8s/src/handlers.py index 56625c5f..f2c35f4b 100644 --- a/charms/tempest-k8s/src/handlers.py +++ b/charms/tempest-k8s/src/handlers.py @@ -40,10 +40,6 @@ import ops_sunbeam.relation_handlers as sunbeam_rhandlers from utils.alert_rules import ( ALERT_RULES_PATH, ) -from utils.cleanup import ( - CleanUpError, - run_extensive_cleanup, -) from utils.constants import ( OPENSTACK_DOMAIN, OPENSTACK_PROJECT, @@ -160,14 +156,13 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler): return [x.name for x in files] @assert_ready - def init_tempest(self, env: Dict[str, str]): - """Init the openstack environment for tempest. + def push_auxiliary_files(self) -> None: + """Push auxiliary files to the container. - Raise a RuntimeError if something goes wrong. + The auxiliary files are: + * the cleanup script + * the exclude list for tempest """ - # push auxiliary files to the container - # * the cleanup script - # * the exclude list for tempest aux_files = [ "src/utils/cleanup.py", "src/utils/tempest_exclude_list.txt", @@ -182,6 +177,12 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler): make_dirs=True, ) + @assert_ready + def init_tempest(self, env: Dict[str, str]): + """Init the openstack environment for tempest. + + Raise a RuntimeError if something goes wrong. + """ # Pebble runs cron, which runs tempest periodically # when periodic checks are enabled. # This ensures that tempest gets the env, inherited from cron. @@ -284,6 +285,21 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler): return summary + @assert_ready + def run_extensive_cleanup(self, env: Dict[str, str]) -> None: + """Wrapper for running extensive cleanup.""" + try: + self.execute( + ["python3", "cleanup.py", "extensive"], + user="tempest", + group="tempest", + working_dir=TEMPEST_HOME, + exception_on_error=True, + environment=env, + ) + except ops.pebble.ExecError: + logger.warning("Clean-up failed") + class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): """Relation handler for identity ops.""" @@ -619,22 +635,12 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): # and the environment should be inited again if rejoined. self.charm.set_tempest_ready(False) - credential = self.get_user_credential() - if credential and credential.get("auth-url"): - env = { - "OS_AUTH_URL": credential.get("auth-url"), - "OS_USERNAME": credential.get("username"), - "OS_PASSWORD": credential.get("password"), - "OS_PROJECT_NAME": credential.get("project-name"), - "OS_DOMAIN_ID": credential.get("domain-id"), - "OS_USER_DOMAIN_ID": credential.get("domain-id"), - "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"), - } - try: - # do an extensive clean-up upon identity relation removal - run_extensive_cleanup(env) - except CleanUpError as e: - logger.warning("Clean-up failed: %s", str(e)) + # Do an extensive clean-up upon identity relation removal if credential + # exists. + env = self.charm._get_cleanup_env() + if env and env.get("OS_AUTH_URL"): + pebble = self.charm.pebble_handler() + pebble.run_extensive_cleanup(env) # Delete the stored keystone credentials, # because they are no longer valid. diff --git a/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 b/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 index 1aef0d6d..2a8e0847 100644 --- a/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 +++ b/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 @@ -23,7 +23,7 @@ echo ":: discover-tempest-config" >> "$TMP_FILE" if discover-tempest-config --test-accounts "$TEMPEST_TEST_ACCOUNTS" --out "$TEMPEST_CONF" >> "$TMP_FILE" 2>&1; then echo ":: tempest run" >> "$TMP_FILE" tempest run --exclude-list "$TEMPEST_EXCLUDE_LIST" --workspace "$TEMPEST_WORKSPACE" -w "$TEMPEST_CONCURRENCY" "$@" >> "$TMP_FILE" 2>&1 - python3 "$TEMPEST_HOME/cleanup.py" + python3 "$TEMPEST_HOME/cleanup.py" quick else echo ":: skipping tempest run because discover-tempest-config had errors" >> "$TMP_FILE" fi diff --git a/charms/tempest-k8s/src/utils/cleanup.py b/charms/tempest-k8s/src/utils/cleanup.py index 6528bd7c..edf5b1b0 100644 --- a/charms/tempest-k8s/src/utils/cleanup.py +++ b/charms/tempest-k8s/src/utils/cleanup.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Utils for cleaning up tempest-related resources.""" +import argparse import os from collections.abc import ( Callable, @@ -47,6 +48,7 @@ def _connect_to_os(env: dict) -> Connection: password=env["OS_PASSWORD"], user_domain_id=env["OS_USER_DOMAIN_ID"], project_domain_id=env["OS_USER_DOMAIN_ID"], + cacert=env["OS_CACERT"], ) @@ -298,6 +300,25 @@ def run_extensive_cleanup(env: dict) -> None: raise CleanUpError("\n".join(failure_message)) +def parse_command_line() -> argparse.Namespace: + """Parse command line interface.""" + parser = argparse.ArgumentParser( + description="Clean up OpenStack resources created by tempest or discover-tempest-conf." + ) + subparsers = parser.add_subparsers(dest="command") + subparsers.add_parser( + "quick", + description="Run a quick cleanup, suitable in between tempest test runs.", + help="run a quick cleanup, suitable in between tempest test runs", + ) + subparsers.add_parser( + "extensive", + description="Run an extensive cleanup, suitable for a clean environment before creating initial tempest resources.", + help="run an extensive cleanup, suitable for a clean environment before creating initial tempest resources", + ) + return parser.parse_args() + + def main() -> None: """Entrypoint for executing the script directly. @@ -305,6 +326,7 @@ def main() -> None: Quick cleanup will be performed. """ env = { + "OS_CACERT": os.getenv("OS_CACERT", ""), "OS_AUTH_URL": os.getenv("OS_AUTH_URL", ""), "OS_USERNAME": os.getenv("OS_USERNAME", ""), "OS_PASSWORD": os.getenv("OS_PASSWORD", ""), @@ -315,7 +337,12 @@ def main() -> None: "TEMPEST_TEST_ACCOUNTS": os.getenv("TEMPEST_TEST_ACCOUNTS", ""), } - run_quick_cleanup(env) + args = parse_command_line() + + if args.command == "quick": + run_quick_cleanup(env) + else: + run_extensive_cleanup(env) if __name__ == "__main__": diff --git a/charms/tempest-k8s/src/utils/constants.py b/charms/tempest-k8s/src/utils/constants.py index 29cb895c..f1220254 100644 --- a/charms/tempest-k8s/src/utils/constants.py +++ b/charms/tempest-k8s/src/utils/constants.py @@ -32,6 +32,13 @@ def get_tempest_concurrency() -> str: return str(min(4, cpu_count())) +# It's the target location for the OS cacert template file. See +# https://opendev.org/openstack/sunbeam-charms/src/branch/main/templates/ca-bundle.pem.j2 +OS_CACERT_PATH = "/usr/local/share/ca-certificates/ca-bundle.pem" +# The relation name of 'receive-ca-cert'. See +# https://opendev.org/openstack/sunbeam-charms/src/branch/main/ops-sunbeam/ops_sunbeam/charm.py#L194 +RECEIVE_CA_CERT_RELATION_NAME = "receive-ca-cert" + TEMPEST_CONCURRENCY = get_tempest_concurrency() # It's desirable to have more accounts than the concurrency, diff --git a/charms/tempest-k8s/tests/unit/test_cleanup.py b/charms/tempest-k8s/tests/unit/test_cleanup.py index dd0d2985..8319f764 100644 --- a/charms/tempest-k8s/tests/unit/test_cleanup.py +++ b/charms/tempest-k8s/tests/unit/test_cleanup.py @@ -63,6 +63,7 @@ class TestCleanup(unittest.TestCase): def test_connect_to_os(self): """Test establishing OS connection.""" env = { + "OS_CACERT": "/usr/local/share/ca-certificates/ca-bundle.pem", "OS_AUTH_URL": "http://10.6.0.20/openstack-keystone", "OS_USERNAME": "test_user", "OS_PASSWORD": "userpass", @@ -80,6 +81,7 @@ class TestCleanup(unittest.TestCase): password=env["OS_PASSWORD"], user_domain_id=env["OS_USER_DOMAIN_ID"], project_domain_id=env["OS_USER_DOMAIN_ID"], + cacert=env["OS_CACERT"], ) def test_get_exclude_resources(self): diff --git a/tests/tempest/smoke.yaml.j2 b/tests/tempest/smoke.yaml.j2 index f36406c4..882dc570 100644 --- a/tests/tempest/smoke.yaml.j2 +++ b/tests/tempest/smoke.yaml.j2 @@ -210,3 +210,5 @@ relations: - - tempest:identity-ops - keystone:identity-ops +- - tempest:receive-ca-cert + - keystone:send-ca-cert