From b0151aa6ee36fca920f79b363e0e49315708e819 Mon Sep 17 00:00:00 2001 From: Tianqi Xiao Date: Thu, 8 Feb 2024 04:25:21 +0000 Subject: [PATCH] Handle tempest resources clean-up Three levels of clean-up is implemented to facilitate different scenarios: 1. quick clean-up that removes resources created by tempest tests. This can be used in-between test runs. 2. extensive clean-up that, in addition to 1, also removes test accounts, project, and the associated network resources. This is used after charm refresh and identity-ops relation departed where the previous test accounts becomes obsoletes. 3. full cleanup that, in addition to 1 and 2, deletes the testing domain, user, and project created through keystone relation. This is used at the initial stage of keystone relation joins to get a fresh env. Change-Id: If172e9dde0e04c53849d7a0b30fc27f62c5bdbd9 --- charms/tempest-k8s/requirements.txt | 3 + charms/tempest-k8s/src/charm.py | 38 +- charms/tempest-k8s/src/handlers.py | 66 ++- .../src/templates/tempest-run-wrapper.j2 | 1 + charms/tempest-k8s/src/utils/cleanup.py | 317 ++++++++++ charms/tempest-k8s/src/utils/constants.py | 3 +- charms/tempest-k8s/tests/unit/test_cleanup.py | 552 ++++++++++++++++++ .../tests/unit/test_tempest_charm.py | 23 +- test-requirements.txt | 1 + 9 files changed, 995 insertions(+), 9 deletions(-) create mode 100644 charms/tempest-k8s/src/utils/cleanup.py create mode 100644 charms/tempest-k8s/tests/unit/test_cleanup.py diff --git a/charms/tempest-k8s/requirements.txt b/charms/tempest-k8s/requirements.txt index f29e573a..87a019a4 100644 --- a/charms/tempest-k8s/requirements.txt +++ b/charms/tempest-k8s/requirements.txt @@ -10,3 +10,6 @@ tenacity # for validating cron expressions croniter + +# for handling cleanup +openstacksdk \ No newline at end of file diff --git a/charms/tempest-k8s/src/charm.py b/charms/tempest-k8s/src/charm.py index 49920e23..592f90d3 100755 --- a/charms/tempest-k8s/src/charm.py +++ b/charms/tempest-k8s/src/charm.py @@ -50,6 +50,10 @@ from ops.model import ( from ops_sunbeam.config_contexts import ( ConfigContext, ) +from utils.cleanup import ( + CleanUpError, + run_extensive_cleanup, +) from utils.constants import ( CONTAINER, TEMPEST_ADHOC_OUTPUT, @@ -210,6 +214,9 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): "OS_PROJECT_NAME": credential.get("project-name"), "OS_PROJECT_DOMAIN_NAME": credential.get("domain-name"), "OS_DOMAIN_NAME": credential.get("domain-name"), + "OS_DOMAIN_ID": credential.get("domain-id"), + "OS_USER_DOMAIN_ID": credential.get("domain-id"), + "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"), "TEMPEST_CONCURRENCY": TEMPEST_CONCURRENCY, "TEMPEST_CONF": TEMPEST_CONF, "TEMPEST_HOME": TEMPEST_HOME, @@ -220,6 +227,23 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): "TEMPEST_OUTPUT": variant.output_path(), } + def _get_cleanup_env(self) -> Dict[str, str]: + """Return a dictionary of environment variables. + + To be used with tempest resource cleanup functions. + """ + logger.debug("Retrieving OpenStack credentials") + credential = self.user_id_ops.get_user_credential() + return { + "OS_AUTH_URL": credential.get("auth-url"), + "OS_USERNAME": credential.get("username"), + "OS_PASSWORD": credential.get("password"), + "OS_PROJECT_NAME": credential.get("project-name"), + "OS_DOMAIN_ID": credential.get("domain-id"), + "OS_USER_DOMAIN_ID": credential.get("domain-id"), + "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"), + } + def get_unit_data(self, key: str) -> Optional[str]: """Retrieve a value set for this unit on the peer relation.""" return self.peers.interface.peers_rel.data[self.unit].get(key) @@ -248,6 +272,15 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): # for periodic checks. env = self._get_environment_for_tempest(TempestEnvVariant.PERIODIC) pebble = self.pebble_handler() + + try: + # do an extensive clean-up before tempest init to remove stalled resources + run_extensive_cleanup(self._get_cleanup_env()) + except CleanUpError: + logger.debug("Clean-up failed and tempest init not run.") + self.set_tempest_ready(False) + return + try: pebble.init_tempest(env) except RuntimeError: @@ -296,17 +329,16 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S): test_list: str = event.params["test-list"].strip() env = self._get_environment_for_tempest(TempestEnvVariant.ADHOC) + try: summary = self.pebble_handler().run_tempest_tests( regexes, exclude_regex, test_list, serial, env ) except RuntimeError as e: - # put the message in set_results instead of event.fail, - # because event.fail message is not always displayed to the user: - # https://bugs.launchpad.net/juju/+bug/2052765 event.set_results({"error": str(e)}) event.fail() return + event.set_results( { "summary": summary, diff --git a/charms/tempest-k8s/src/handlers.py b/charms/tempest-k8s/src/handlers.py index ae1b35f3..1bb3864d 100644 --- a/charms/tempest-k8s/src/handlers.py +++ b/charms/tempest-k8s/src/handlers.py @@ -36,6 +36,10 @@ import ops.model import ops.pebble import ops_sunbeam.container_handlers as sunbeam_chandlers import ops_sunbeam.relation_handlers as sunbeam_rhandlers +from utils.cleanup import ( + CleanUpError, + run_extensive_cleanup, +) from utils.constants import ( OPENSTACK_DOMAIN, OPENSTACK_PROJECT, @@ -157,6 +161,16 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler): Raise a RuntimeError if something goes wrong. """ + # push the cleanup script to container + with open("src/utils/cleanup.py") as f: + self.container.push( + f"{TEMPEST_HOME}/cleanup.py", + f, + user="tempest", + group="tempest", + make_dirs=True, + ) + # Pebble runs cron, which runs tempest periodically # when periodic checks are enabled. # This ensures that tempest gets the env, inherited from cron. @@ -456,6 +470,10 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): def _teardown_tempest_resource_ops(self) -> List[dict]: """Tear down openstack resource ops.""" + credential_id = self._ensure_credential() + credential_secret = self.model.get_secret(id=credential_id) + content = credential_secret.get_content() + username = content.get("username") teardown_ops = [ { "name": "show_domain", @@ -463,6 +481,20 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): "name": OPENSTACK_DOMAIN, }, }, + { + "name": "delete_project", + "params": { + "name": OPENSTACK_PROJECT, + "domain": "{{ show_domain[0].id }}", + }, + }, + { + "name": "delete_user", + "params": { + "name": username, + "domain": "{{ show_domain[0].id }}", + }, + }, { "name": "update_domain", "params": { @@ -517,6 +549,19 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): self._set_secret({"auth-url": auth_url}) return + def _process_setup_tempest_resource_response(self, response: dict) -> None: + """Process extra ops request: "_setup_tempest_resource_request".""" + for op in response.get("ops", []): + if op.get("name") != "create_domain": + continue + if op.get("return-code") != 0: + logger.warning("Create domain ops failed.") + return + domain_id = op.get("value", {}).get("id") + if domain_id is not None: + self._set_secret({"domain-id": domain_id}) + return + def _on_provider_ready(self, event) -> None: """Handles response available events.""" if not self.model.unit.is_leader(): @@ -535,15 +580,30 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler): response = self.interface.response logger.info("%s", json.dumps(response, indent=4)) self._process_list_endpoint_response(response) + self._process_setup_tempest_resource_response(response) self.callback_f(event) def _on_provider_goneaway(self, event) -> None: """Handle gone_away event.""" if not self.model.unit.is_leader(): return - logger.info( - "Identity ops provider gone away: teardown tempest resources" - ) + logger.info("Identity ops provider gone away") + credential = self.get_user_credential() + env = { + "OS_AUTH_URL": credential.get("auth-url"), + "OS_USERNAME": credential.get("username"), + "OS_PASSWORD": credential.get("password"), + "OS_PROJECT_NAME": credential.get("project-name"), + "OS_DOMAIN_ID": credential.get("domain-id"), + "OS_USER_DOMAIN_ID": credential.get("domain-id"), + "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"), + } + try: + # do an extensive clean-up upon identity relation removal + run_extensive_cleanup(env) + + except CleanUpError as e: + logger.warning("Clean-up failed: %s", str(e)) self.callback_f(event) diff --git a/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 b/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 index 50c7d8d7..c16cc539 100644 --- a/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 +++ b/charms/tempest-k8s/src/templates/tempest-run-wrapper.j2 @@ -15,6 +15,7 @@ echo ":: discover-tempest-config" >> "$TMP_FILE" if discover-tempest-config --test-accounts "$TEMPEST_TEST_ACCOUNTS" --out "$TEMPEST_CONF" >> "$TMP_FILE" 2>&1; then echo ":: tempest run" >> "$TMP_FILE" tempest run --workspace "$TEMPEST_WORKSPACE" "$@" >> "$TMP_FILE" 2>&1 + python3 "$TEMPEST_HOME/cleanup.py" else echo ":: skipping tempest run because discover-tempest-config had errors" >> "$TMP_FILE" fi diff --git a/charms/tempest-k8s/src/utils/cleanup.py b/charms/tempest-k8s/src/utils/cleanup.py new file mode 100644 index 00000000..597b9c39 --- /dev/null +++ b/charms/tempest-k8s/src/utils/cleanup.py @@ -0,0 +1,317 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utils for cleaning up tempest-related resources.""" +import os +from collections.abc import ( + Callable, +) + +import yaml +from keystoneauth1.exceptions.catalog import ( + EndpointNotFound, +) +from keystoneauth1.exceptions.http import ( + Unauthorized, +) +from openstack.connection import ( + Connection, +) +from openstack.exceptions import ( + ForbiddenException, +) + +RESOURCE_PREFIX = "tempest-" + + +class CleanUpError(Exception): + """Exception raised when clean-up process terminated unsuccessfully.""" + + +def _connect_to_os(env: dict) -> Connection: + """Establish connection to the OpenStack cloud.""" + return Connection( + auth_url=env["OS_AUTH_URL"], + project_name=env["OS_PROJECT_NAME"], + username=env["OS_USERNAME"], + password=env["OS_PASSWORD"], + user_domain_id=env["OS_USER_DOMAIN_ID"], + project_domain_id=env["OS_USER_DOMAIN_ID"], + ) + + +def _get_exclusion_resources(account_file: str) -> dict[str, set[str]]: + """Get users and projects to be excluded from clean-up. + + The resources are looked up in an account_file which is generated by + `tempest account-generator`. + + Note that this lookup will only work in workload container as it needs + access to the generated test accounts file. + """ + try: + with open(account_file, "r") as file: + parsed_file = yaml.safe_load(file) + + return { + "projects": {account["project_name"] for account in parsed_file}, + "users": {account["username"] for account in parsed_file}, + } + except FileNotFoundError as e: + raise CleanUpError("Test account file doesn't exist.") from e + except KeyError as e: + raise CleanUpError("Malformated test account file.") from e + + +def _get_test_projects_in_domain( + conn: Connection, domain_id: str, exclude_projects: set[str] = set() +) -> list[str]: + """Get test projects in domain to do cleanup for. + + Projects in exclude_projects will not be included in the returned list. + """ + return [ + project.id + for project in conn.identity.projects(domain_id=domain_id) + if project.name.startswith(RESOURCE_PREFIX) + and project.name not in exclude_projects + ] + + +def _cleanup_compute_resources(conn: Connection, project_id: str) -> None: + """Delete compute resources with names starting with prefix in the specified project. + + The compute resources to be removed are instances and keypairs. + """ + # Delete instances + for server in conn.compute.servers(project_id=project_id): + if server.name.startswith(RESOURCE_PREFIX): + conn.compute.delete_server(server.id) + + # Delete keypairs + for keypair in conn.compute.keypairs(): + if keypair.name.startswith(RESOURCE_PREFIX): + conn.compute.delete_keypair(keypair) + + +def _cleanup_block_resources(conn: Connection, project_id: str) -> None: + """Delete block storage resources with names starting with prefix in the specified project. + + The block storage resources to be removed are snapshots and instances. + """ + # Delete snapshots + for snapshot in conn.block_store.snapshots( + details=True, project_id=project_id + ): + if snapshot.name.startswith(RESOURCE_PREFIX): + conn.block_store.delete_snapshot(snapshot.id) + + # Delete volumes + for volume in conn.block_store.volumes( + details=True, project_id=project_id + ): + if volume.name.startswith(RESOURCE_PREFIX): + conn.block_store.delete_volume(volume.id) + + +def _cleanup_images(conn: Connection, project_id: str) -> None: + """Delete images with names starting with prefix and owned by the specified project.""" + for image in conn.image.images(): + # TODO: to be extra careful, we should also check the prefix of the image + # However, some tempest tests are not creating images with the prefix, so + # we should wait until https://review.opendev.org/c/openstack/tempest/+/908358 + # is released. + if image.owner == project_id: + conn.image.delete_image(image.id) + + +def _cleanup_networks_resources(conn: Connection, project_id: str) -> None: + """Delete network resources with names starting with prefix in the specified project. + + The network resources to be removed are ports, routers, and networks. + """ + # Delete ports and routers + for router in conn.network.routers(project_id=project_id): + if router.name.startswith(RESOURCE_PREFIX): + for port in conn.network.ports(device_id=router.id): + conn.network.remove_interface_from_router( + router, port_id=port.id + ) + conn.network.delete_router(router.id) + + # Delete networks + for network in conn.network.networks(project_id=project_id): + if network.name.startswith(RESOURCE_PREFIX): + conn.network.delete_network(network.id) + + +def _cleanup_stacks(conn: Connection, project_id: str) -> None: + """Delete stacks with names starting with prefix and owned by the specified project. + + If Heat service is not found in the cloud, this clean-up will be skipped. + """ + try: + for stack in conn.orchestration.stacks(project_id=project_id): + if stack.name.startswith(RESOURCE_PREFIX): + conn.orchestration.delete_stack(stack.id) + except EndpointNotFound: + # do nothing if the heat endpoint is not found + pass + + +def _cleanup_users( + conn: Connection, domain_id: str, exclude_users: set[str] = set() +) -> None: + """Delete users with names starting with prefix in the specified domain. + + If exclude_users is specified, users in exclude_users will not be removed. + """ + for user in conn.identity.users(domain_id=domain_id): + if ( + user.name.startswith(RESOURCE_PREFIX) + and user.name not in exclude_users + ): + conn.identity.delete_user(user.id) + + +def _cleanup_project(conn: Connection, project_id: str) -> None: + """Delete a project given its id.""" + conn.identity.delete_project(project_id) + + +def _run_cleanup_functions( + conn: Connection, projects: list[str], functions: list[Callable] +) -> list[str]: + """Run clean-up function on a list of projects.""" + failure_message = [] + for project_id in projects: + for func in functions: + try: + func(conn, project_id) + except Exception as e: + failure_message.append(f"Error calling {func.__name__}: {e}") + return failure_message + + +def run_quick_cleanup(env: dict) -> None: + """Perform the quick cleanup of tempest resources under a specific domain. + + This clean-up removes compute instances, keypairs, volumes, snapshots, + images, networks, projects, users, and stacks that are not associated with + the pre-generated test accounts and projects defined in account_file. + + Note that this clean up will only work in workload container as it needs + access to the generated test accounts file. + """ + conn = _connect_to_os(env) + cleanup_funcs = [ + _cleanup_compute_resources, + _cleanup_block_resources, + _cleanup_images, + _cleanup_stacks, + ] + filtered_cleanup_funcs = [ + _cleanup_networks_resources, + _cleanup_project, + ] + exclude_resources = _get_exclusion_resources(env["TEMPEST_TEST_ACCOUNTS"]) + failure_message = [] + + try: + # get all projects with prefix in domain + test_projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"]) + + # get projects that are not found in account file + filtered_test_projects = _get_test_projects_in_domain( + conn, env["OS_DOMAIN_ID"], exclude_resources["projects"] + ) + except (ForbiddenException, Unauthorized) as e: + raise CleanUpError("Operation not authorized.") from e + + failure_message.extend( + _run_cleanup_functions(conn, test_projects, cleanup_funcs) + ) + failure_message.extend( + _run_cleanup_functions( + conn, filtered_test_projects, filtered_cleanup_funcs + ) + ) + + try: + _cleanup_users(conn, env["OS_DOMAIN_ID"], exclude_resources["users"]) + except Exception as e: + failure_message.append(f"\nError cleaning up users: {e}") + + if failure_message: + raise CleanUpError("\n".join(failure_message)) + + +def run_extensive_cleanup(env: dict) -> None: + """Perform the extensive cleanup of tempest resources under a specific domain. + + This clean-up removes compute instances, keypairs, volumes, snapshots, + images, and stacks, as well as generated test accounts, projects, and the + network resources associated with them. + """ + conn = _connect_to_os(env) + cleanup_funcs = [ + _cleanup_compute_resources, + _cleanup_block_resources, + _cleanup_images, + _cleanup_stacks, + _cleanup_networks_resources, + _cleanup_project, + ] + failure_message = [] + + try: + # get all projects with prefix in domain + projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"]) + except (ForbiddenException, Unauthorized) as e: + raise CleanUpError("Operation not authorized.") from e + + failure_message.extend( + _run_cleanup_functions(conn, projects, cleanup_funcs) + ) + + try: + _cleanup_users(conn, env["OS_DOMAIN_ID"]) + except Exception as e: + failure_message.append(f"\nError cleaning up users: {e}") + + if failure_message: + raise CleanUpError("\n".join(failure_message)) + + +def main() -> None: + """Entrypoint for executing the script directly. + + This will be used in periodic test runs. + Quick cleanup will be performed. + """ + env = { + "OS_AUTH_URL": os.getenv("OS_AUTH_URL", ""), + "OS_USERNAME": os.getenv("OS_USERNAME", ""), + "OS_PASSWORD": os.getenv("OS_PASSWORD", ""), + "OS_PROJECT_NAME": os.getenv("OS_PROJECT_NAME", ""), + "OS_DOMAIN_ID": os.getenv("OS_DOMAIN_ID", ""), + "OS_USER_DOMAIN_ID": os.getenv("OS_USER_DOMAIN_ID", ""), + "OS_PROJECT_DOMAIN_ID": os.getenv("OS_PROJECT_DOMAIN_ID", ""), + "TEMPEST_TEST_ACCOUNTS": os.getenv("TEMPEST_TEST_ACCOUNTS", ""), + } + + run_quick_cleanup(env) + + +if __name__ == "__main__": + main() diff --git a/charms/tempest-k8s/src/utils/constants.py b/charms/tempest-k8s/src/utils/constants.py index c794502e..52b5dc6d 100644 --- a/charms/tempest-k8s/src/utils/constants.py +++ b/charms/tempest-k8s/src/utils/constants.py @@ -29,7 +29,8 @@ TEMPEST_WORKSPACE = "tempest" OPENSTACK_USER = "tempest" OPENSTACK_DOMAIN = "tempest" -OPENSTACK_PROJECT = "tempest-CloudValidation" +# not use tempest as prefix to exclude this project from utils/cleanup.py scope +OPENSTACK_PROJECT = "CloudValidation-tempest" OPENSTACK_ROLE = "admin" # keys for application data diff --git a/charms/tempest-k8s/tests/unit/test_cleanup.py b/charms/tempest-k8s/tests/unit/test_cleanup.py new file mode 100644 index 00000000..dd0d2985 --- /dev/null +++ b/charms/tempest-k8s/tests/unit/test_cleanup.py @@ -0,0 +1,552 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for tempest-related resources cleanup.""" +import textwrap +import unittest +from unittest.mock import ( + MagicMock, + call, + mock_open, + patch, +) + +from keystoneauth1.exceptions.catalog import ( + EndpointNotFound, +) +from keystoneauth1.exceptions.http import ( + Unauthorized, +) +from openstack.exceptions import ( + ForbiddenException, +) +from utils.cleanup import ( + CleanUpError, + _cleanup_block_resources, + _cleanup_compute_resources, + _cleanup_images, + _cleanup_networks_resources, + _cleanup_project, + _cleanup_stacks, + _cleanup_users, + _connect_to_os, + _get_exclusion_resources, + _get_test_projects_in_domain, + _run_cleanup_functions, + run_extensive_cleanup, + run_quick_cleanup, +) + + +class TestCleanup(unittest.TestCase): + """Test tempest resources clean-up.""" + + def setUp(self): + """Set up test construction.""" + self.patcher = patch("utils.cleanup.Connection") + self.mock_connection = self.patcher.start() + + def tearDown(self): + """Tear down test construction.""" + self.patcher.stop() + + def test_connect_to_os(self): + """Test establishing OS connection.""" + env = { + "OS_AUTH_URL": "http://10.6.0.20/openstack-keystone", + "OS_USERNAME": "test_user", + "OS_PASSWORD": "userpass", + "OS_PROJECT_NAME": "test_project", + "OS_DOMAIN_ID": "domain_id", + "OS_USER_DOMAIN_ID": "domain_id", + "OS_PROJECT_DOMAIN_ID": "domain_id", + } + _connect_to_os(env) + + self.mock_connection.assert_called_once_with( + auth_url=env["OS_AUTH_URL"], + project_name=env["OS_PROJECT_NAME"], + username=env["OS_USERNAME"], + password=env["OS_PASSWORD"], + user_domain_id=env["OS_USER_DOMAIN_ID"], + project_domain_id=env["OS_USER_DOMAIN_ID"], + ) + + def test_get_exclude_resources(self): + """Test get exclude resources from test accounts file.""" + account_file_content = textwrap.dedent( + """ + - domain_name: mydomain + password: password1 + project_name: tempest-test_creds-11949114 + resources: + network: tempest-test_creds-9369097-network + username: tempest-test_creds-11949114-project + - domain_name: mydomain + password: password2 + project_name: tempest-test_creds-18083146 + resources: + network: tempest-test_creds-20041716-network + username: tempest-test_creds-18083146-project + """ + ) + + expected_result = { + "projects": { + "tempest-test_creds-11949114", + "tempest-test_creds-18083146", + }, + "users": { + "tempest-test_creds-11949114-project", + "tempest-test_creds-18083146-project", + }, + } + + with patch( + "utils.cleanup.open", + new_callable=mock_open, + read_data=account_file_content, + ): + result = _get_exclusion_resources("test_account_file.yaml") + + self.assertEqual(result, expected_result) + + def test_get_test_projects_in_domain(self): + """Test get tempest projects of a specified domain.""" + tempest_project1 = MagicMock() + tempest_project1.configure_mock(id="1", name="tempest-project-1") + tempest_project2 = MagicMock() + tempest_project2.configure_mock(id="2", name="tempest-project-2") + non_tempest_project = MagicMock() + non_tempest_project.configure_mock(id="3", name="non-tempest-project") + self.mock_connection.identity.projects.return_value = [ + tempest_project1, + tempest_project2, + non_tempest_project, + ] + + projects = _get_test_projects_in_domain( + self.mock_connection, "tempest_domain" + ) + + self.assertEqual(projects, ["1", "2"]) + + def test_get_test_projects_in_domain_with_exclusion(self): + """Test get tempest projects of a specified domain with exclusion.""" + tempest_project1 = MagicMock() + tempest_project1.configure_mock(id="1", name="tempest-project-1") + tempest_project2 = MagicMock() + tempest_project2.configure_mock(id="2", name="tempest-project-2") + non_tempest_project = MagicMock() + non_tempest_project.configure_mock(id="3", name="non-tempest-project") + self.mock_connection.identity.projects.return_value = [ + tempest_project1, + tempest_project2, + non_tempest_project, + ] + + projects = _get_test_projects_in_domain( + self.mock_connection, + domain_id="tempest_domain", + exclude_projects={"tempest-project-1"}, + ) + + self.assertEqual(projects, ["2"]) + + def test_cleanup_block_resources(self): + """Test cleanup volumes and snapshots.""" + tempest_snapshots = MagicMock() + tempest_snapshots.configure_mock(id="1", name="tempest-snapshots-1") + non_tempest_snapshots = MagicMock() + non_tempest_snapshots.configure_mock( + id="2", name="non-tempest-snapshots" + ) + self.mock_connection.block_store.snapshots.return_value = [ + tempest_snapshots, + non_tempest_snapshots, + ] + + tempest_volume = MagicMock() + tempest_volume.configure_mock(id="1", name="tempest-volume-1") + non_tempest_volume = MagicMock() + non_tempest_volume.configure_mock(id="2", name="non-tempest-volume") + self.mock_connection.block_store.volumes.return_value = [ + tempest_volume, + non_tempest_volume, + ] + + _cleanup_block_resources(self.mock_connection, "test_project") + + self.mock_connection.block_store.delete_snapshot.assert_called_once_with( + "1" + ) + self.mock_connection.block_store.delete_volume.assert_called_once_with( + "1" + ) + + def test_cleanup_networks_resources(self): + """Test cleanup network resources.""" + tempest_router = MagicMock() + tempest_router.configure_mock(id="1", name="tempest-router") + non_tempest_router = MagicMock() + non_tempest_router.configure_mock(id="2", name="non-tempest-router") + self.mock_connection.network.routers.return_value = [ + tempest_router, + non_tempest_router, + ] + + tempest_port1 = MagicMock() + tempest_port1.configure_mock(id="1") + tempest_port2 = MagicMock() + tempest_port2.configure_mock(id="2") + self.mock_connection.network.ports.return_value = [ + tempest_port1, + tempest_port2, + ] + + tempest_network = MagicMock() + tempest_network.configure_mock(id="1", name="tempest-network") + non_tempest_network = MagicMock() + non_tempest_network.configure_mock(id="2", name="non-tempest-network") + self.mock_connection.network.networks.return_value = [ + tempest_network, + non_tempest_network, + ] + + _cleanup_networks_resources(self.mock_connection, "project_id") + + self.mock_connection.network.remove_interface_from_router.call_args_list == [ + call(tempest_router, port_id="1"), + call(tempest_router, port_id="2"), + ] + self.mock_connection.network.delete_router.assert_called_once_with("1") + self.mock_connection.network.delete_network.assert_called_once_with( + "1" + ) + + def test_cleanup_compute_resources(self): + """Test cleanup instances and keypairs.""" + tempest_instance = MagicMock() + tempest_instance.configure_mock(id="1", name="tempest-server-1") + non_tempest_instance = MagicMock() + non_tempest_instance.configure_mock(id="2", name="non-tempest-server") + self.mock_connection.compute.servers.return_value = [ + tempest_instance, + non_tempest_instance, + ] + + tempest_keypair = MagicMock() + tempest_keypair.configure_mock(id="1", name="tempest-keypair-1") + non_tempest_keypair = MagicMock() + non_tempest_keypair.configure_mock( + id="2", name="non-tempest-keypair-2" + ) + + self.mock_connection.compute.keypairs.return_value = [ + tempest_keypair, + non_tempest_keypair, + ] + + _cleanup_compute_resources(self.mock_connection, "test_project") + + self.mock_connection.compute.delete_server.assert_called_once_with("1") + self.mock_connection.compute.delete_keypair.assert_called_once_with( + tempest_keypair + ) + + def test_cleanup_images(self): + """Test cleanup images.""" + image_owned_by_project = MagicMock() + image_owned_by_project.configure_mock( + id="1", name="image-1", owner="test_project" + ) + image_not_owned_by_project = MagicMock() + image_not_owned_by_project.configure_mock( + id="3", name="image-3", owner="not_test_project" + ) + self.mock_connection.image.images.return_value = [ + image_owned_by_project, + image_not_owned_by_project, + ] + + _cleanup_images(self.mock_connection, "test_project") + + self.mock_connection.image.delete_image.assert_called_once_with("1") + + def test_cleanup_project(self): + """Test cleanup projects.""" + project_id = "tempest_project_id" + + _cleanup_project(self.mock_connection, project_id) + + self.mock_connection.identity.delete_project.assert_called_with( + project_id + ) + + def test_cleanup_users(self): + """Test cleanup users.""" + tempest_user = MagicMock() + tempest_user.configure_mock( + domain_id="tempest", id="1", name="tempest-user-1" + ) + non_tempest_user = MagicMock() + non_tempest_user.configure_mock( + domain_id="tempest", id="2", name="non-tempest-user-2" + ) + + self.mock_connection.identity.users.return_value = [ + tempest_user, + non_tempest_user, + ] + + _cleanup_users(self.mock_connection, domain_id="tempest") + + self.mock_connection.identity.delete_user.assert_called_once_with("1") + + def test_cleanup_users_with_exclusion(self): + """Test cleanup users with exclusion.""" + tempest_user1 = MagicMock() + tempest_user1.configure_mock( + domain_id="tempest", id="1", name="tempest-user-1" + ) + tempest_user2 = MagicMock() + tempest_user2.configure_mock( + domain_id="tempest", id="2", name="tempest-user-2" + ) + non_tempest_user = MagicMock() + non_tempest_user.configure_mock( + domain_id="tempest", id="3", name="non-tempest-user" + ) + + self.mock_connection.identity.users.return_value = [ + tempest_user1, + tempest_user2, + non_tempest_user, + ] + + _cleanup_users( + self.mock_connection, + domain_id="tempest", + exclude_users={"tempest-user-1"}, + ) + + self.mock_connection.identity.delete_user.assert_called_once_with("2") + + def test_cleanup_stacks_success(self): + """Test cleanup heat stacks.""" + tempest_stack = MagicMock() + tempest_stack.configure_mock(id="1", name="tempest-stack-1") + non_tempest_stack = MagicMock() + non_tempest_stack.configure_mock(id="2", name="non-tempest-stack-2") + self.mock_connection.orchestration.stacks.return_value = [ + tempest_stack, + non_tempest_stack, + ] + + _cleanup_stacks(self.mock_connection, "test_project") + + self.mock_connection.orchestration.delete_stack.assert_called_once_with( + "1" + ) + + def test_cleanup_stacks_endpoint_not_found(self): + """Test cleanup stacks when heat endpoint is not found.""" + self.mock_connection.orchestration.stacks.side_effect = ( + EndpointNotFound + ) + + _cleanup_stacks(self.mock_connection, "test_project") + + self.mock_connection.orchestration.delete_stack.assert_not_called() + + @patch("utils.cleanup._cleanup_compute_resources") + @patch("utils.cleanup._cleanup_block_resources") + @patch("utils.cleanup._cleanup_images") + @patch("utils.cleanup._cleanup_stacks") + @patch("utils.cleanup._cleanup_networks_resources") + @patch("utils.cleanup._cleanup_project") + def test_run_cleanup_functions( + self, + mock_cleanup_project, + mock_cleanup_networks_resources, + mock_cleanup_stacks, + mock_cleanup_images, + mock_cleanup_block_resources, + mock_cleanup_compute_resources, + ): + """Test run cleanup functions.""" + cleanup_funcs = [ + mock_cleanup_compute_resources, + mock_cleanup_block_resources, + mock_cleanup_images, + mock_cleanup_stacks, + mock_cleanup_networks_resources, + mock_cleanup_project, + ] + projects = ["tempest_project_id"] + + _run_cleanup_functions(self.mock_connection, projects, cleanup_funcs) + + mock_cleanup_stacks.assert_called_once() + mock_cleanup_images.assert_called_once() + mock_cleanup_block_resources.assert_called_once() + mock_cleanup_compute_resources.assert_called_once() + mock_cleanup_networks_resources.assert_called_once() + mock_cleanup_project.assert_called_once() + + @patch("utils.cleanup._cleanup_compute_resources") + @patch("utils.cleanup._cleanup_block_resources") + @patch("utils.cleanup._cleanup_images") + @patch("utils.cleanup._cleanup_stacks") + @patch("utils.cleanup._cleanup_networks_resources") + @patch("utils.cleanup._cleanup_project") + def test_run_cleanup_functions_unsuccessful( + self, + mock_cleanup_project, + mock_cleanup_networks_resources, + mock_cleanup_stacks, + mock_cleanup_images, + mock_cleanup_block_resources, + mock_cleanup_compute_resources, + ): + """Test run cleanup functions.""" + cleanup_funcs = [ + mock_cleanup_compute_resources, + mock_cleanup_block_resources, + mock_cleanup_images, + mock_cleanup_stacks, + mock_cleanup_networks_resources, + mock_cleanup_project, + ] + projects = ["tempest_project_id"] + mock_cleanup_images.__name__ = "_cleanup_images" + mock_cleanup_images.side_effect = Exception + + _run_cleanup_functions(self.mock_connection, projects, cleanup_funcs) + + mock_cleanup_stacks.assert_called_once() + mock_cleanup_images.assert_called_once() + mock_cleanup_block_resources.assert_called_once() + mock_cleanup_compute_resources.assert_called_once() + mock_cleanup_networks_resources.assert_called_once() + mock_cleanup_project.assert_called_once() + + @patch("utils.cleanup._run_cleanup_functions") + @patch("utils.cleanup._cleanup_users") + @patch("utils.cleanup._connect_to_os") + @patch("utils.cleanup._get_test_projects_in_domain") + @patch("utils.cleanup._get_exclusion_resources") + def test_run_quick_cleanup( + self, + mock_get_exclusion_resources, + mock_get_test_projects_in_domain, + mock_connect_to_os, + mock_cleanup_users, + mock_run_cleanup_functions, + ): + """Test run quick cleanup.""" + env = MagicMock() + mock_get_test_projects_in_domain.side_effect = [ + ["tempest_project_id"], + ["tempest_project_id"], + ] + + run_quick_cleanup(env) + + mock_connect_to_os.assert_called_once_with(env) + mock_get_exclusion_resources.assert_called_once() + mock_get_test_projects_in_domain.assert_called() + mock_run_cleanup_functions.assert_called() + mock_cleanup_users.assert_called_once() + + @patch("utils.cleanup._run_cleanup_functions") + @patch("utils.cleanup._cleanup_users") + @patch("utils.cleanup._connect_to_os") + @patch("utils.cleanup._get_test_projects_in_domain") + @patch("utils.cleanup._get_exclusion_resources") + def test_run_quick_cleanup_error( + self, + mock_get_exclusion_resources, + mock_get_test_projects_in_domain, + mock_connect_to_os, + mock_cleanup_users, + mock_run_cleanup_functions, + ): + """Test run quick cleanup with failure.""" + env = MagicMock() + mock_get_test_projects_in_domain.side_effect = ForbiddenException + + with self.assertRaises(CleanUpError) as error: + run_quick_cleanup(env) + + mock_connect_to_os.assert_called_once_with(env) + mock_get_exclusion_resources.assert_called_once() + mock_get_test_projects_in_domain.assert_called_once() + self.assertEqual(str(error.exception), "Operation not authorized.") + + mock_run_cleanup_functions.assert_not_called() + mock_cleanup_users.assert_not_called() + + @patch("utils.cleanup._run_cleanup_functions") + @patch("utils.cleanup._cleanup_users") + @patch("utils.cleanup._connect_to_os") + @patch("utils.cleanup._get_test_projects_in_domain") + @patch("utils.cleanup._get_exclusion_resources") + def test_run_extensive_cleanup( + self, + mock_get_exclusion_resources, + mock_get_test_projects_in_domain, + mock_connect_to_os, + mock_cleanup_users, + mock_run_cleanup_functions, + ): + """Test run extensive cleanup.""" + env = MagicMock() + mock_get_test_projects_in_domain.side_effect = [ + ["tempest_project_id"], + ["tempest_project_id"], + ] + + run_extensive_cleanup(env) + + mock_connect_to_os.assert_called_once_with(env) + mock_get_test_projects_in_domain.assert_called() + mock_run_cleanup_functions.assert_called_once() + mock_cleanup_users.assert_called_once() + + @patch("utils.cleanup._run_cleanup_functions") + @patch("utils.cleanup._cleanup_users") + @patch("utils.cleanup._connect_to_os") + @patch("utils.cleanup._get_test_projects_in_domain") + @patch("utils.cleanup._get_exclusion_resources") + def test_run_extensive_cleanup_permission_error( + self, + mock_get_exclusion_resources, + mock_get_test_projects_in_domain, + mock_connect_to_os, + mock_cleanup_users, + mock_run_cleanup_functions, + ): + """Test run extensive cleanup with failure.""" + env = MagicMock() + mock_get_test_projects_in_domain.side_effect = Unauthorized + + with self.assertRaises(CleanUpError) as error: + run_extensive_cleanup(env) + + mock_connect_to_os.assert_called_once_with(env) + mock_get_test_projects_in_domain.assert_called_once() + self.assertEqual(str(error.exception), "Operation not authorized.") + + mock_run_cleanup_functions.assert_not_called() + mock_cleanup_users.assert_not_called() diff --git a/charms/tempest-k8s/tests/unit/test_tempest_charm.py b/charms/tempest-k8s/tests/unit/test_tempest_charm.py index 2c36a0a2..0c9010f3 100644 --- a/charms/tempest-k8s/tests/unit/test_tempest_charm.py +++ b/charms/tempest-k8s/tests/unit/test_tempest_charm.py @@ -18,6 +18,9 @@ import json import pathlib +from unittest.mock import ( + patch, +) import charm import mock @@ -42,9 +45,12 @@ TEST_TEMPEST_ENV = { "OS_USERNAME": "tempest", "OS_PASSWORD": "password", "OS_USER_DOMAIN_NAME": "tempest", - "OS_PROJECT_NAME": "tempest-CloudValidation", + "OS_PROJECT_NAME": "CloudValidation-tempest", "OS_PROJECT_DOMAIN_NAME": "tempest", "OS_DOMAIN_NAME": "tempest", + "OS_PROJECT_DOMAIN_ID": "tempest-domain-id", + "OS_USER_DOMAIN_ID": "tempest-domain-id", + "OS_DOMAIN_ID": "tempest-domain-id", "TEMPEST_CONCURRENCY": "4", "TEMPEST_CONF": "/var/lib/tempest/workspace/etc/tempest.conf", "TEMPEST_HOME": "/var/lib/tempest", @@ -97,6 +103,18 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase): self.addCleanup(self.harness.cleanup) self.harness.begin() self.harness.set_leader() + self.patcher1 = patch("utils.cleanup.Connection") + self.patcher2 = patch( + "utils.cleanup._get_exclusion_resources", + return_value={"projects": set(), "users": set()}, + ) + self.patcher1.start() + self.patcher2.start() + + def tearDown(self): + """Tear down test construction.""" + self.patcher1.stop() + self.patcher2.stop() def add_identity_ops_relation(self, harness): """Add identity resource relation.""" @@ -108,7 +126,8 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase): "username": "tempest", "password": "password", "domain-name": "tempest", - "project-name": "tempest-CloudValidation", + "domain-id": "tempest-domain-id", + "project-name": "CloudValidation-tempest", "auth-url": "http://10.6.0.23/openstack-keystone/v3", }, ) diff --git a/test-requirements.txt b/test-requirements.txt index e5577480..e7b8e147 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,3 +18,4 @@ croniter # tempest-k8s git+https://github.com/juju/charm-helpers.git#egg=charmhelpers # cinder-ceph-k8s,glance-k8s,gnocchi-k8s git+https://opendev.org/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client # cinder-ceph-k8s requests-unixsocket # sunbeam-clusterd +openstacksdk # tempest-k8s