Handle tempest resources clean-up

Three levels of clean-up is implemented to facilitate different
scenarios:
1. quick clean-up that removes resources created by tempest tests.
This can be used in-between test runs.
2. extensive clean-up that, in addition to 1, also removes test
accounts, project, and the associated network resources. This is
used after charm refresh and identity-ops relation departed where
the previous test accounts becomes obsoletes.
3. full cleanup that, in addition to 1 and 2, deletes the testing
domain, user, and project created through keystone relation. This
is used at the initial stage of keystone relation joins to get
a fresh env.

Change-Id: If172e9dde0e04c53849d7a0b30fc27f62c5bdbd9

charms/tempest-k8s/requirements.txt

@@ -10,3 +10,6 @@ tenacity
 
 # for validating cron expressions
 croniter
+
+# for handling cleanup
+openstacksdk

charms/tempest-k8s/src/charm.py

@@ -50,6 +50,10 @@ from ops.model import (
 from ops_sunbeam.config_contexts import (
     ConfigContext,
 )
+from utils.cleanup import (
+    CleanUpError,
+    run_extensive_cleanup,
+)
 from utils.constants import (
     CONTAINER,
     TEMPEST_ADHOC_OUTPUT,
@@ -210,6 +214,9 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
             "OS_PROJECT_NAME": credential.get("project-name"),
             "OS_PROJECT_DOMAIN_NAME": credential.get("domain-name"),
             "OS_DOMAIN_NAME": credential.get("domain-name"),
+            "OS_DOMAIN_ID": credential.get("domain-id"),
+            "OS_USER_DOMAIN_ID": credential.get("domain-id"),
+            "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
             "TEMPEST_CONCURRENCY": TEMPEST_CONCURRENCY,
             "TEMPEST_CONF": TEMPEST_CONF,
             "TEMPEST_HOME": TEMPEST_HOME,
@@ -220,6 +227,23 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
             "TEMPEST_OUTPUT": variant.output_path(),
         }
 
+    def _get_cleanup_env(self) -> Dict[str, str]:
+        """Return a dictionary of environment variables.
+
+        To be used with tempest resource cleanup functions.
+        """
+        logger.debug("Retrieving OpenStack credentials")
+        credential = self.user_id_ops.get_user_credential()
+        return {
+            "OS_AUTH_URL": credential.get("auth-url"),
+            "OS_USERNAME": credential.get("username"),
+            "OS_PASSWORD": credential.get("password"),
+            "OS_PROJECT_NAME": credential.get("project-name"),
+            "OS_DOMAIN_ID": credential.get("domain-id"),
+            "OS_USER_DOMAIN_ID": credential.get("domain-id"),
+            "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
+        }
+
     def get_unit_data(self, key: str) -> Optional[str]:
         """Retrieve a value set for this unit on the peer relation."""
         return self.peers.interface.peers_rel.data[self.unit].get(key)
@@ -248,6 +272,15 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
         # for periodic checks.
         env = self._get_environment_for_tempest(TempestEnvVariant.PERIODIC)
         pebble = self.pebble_handler()
+
+        try:
+            # do an extensive clean-up before tempest init to remove stalled resources
+            run_extensive_cleanup(self._get_cleanup_env())
+        except CleanUpError:
+            logger.debug("Clean-up failed and tempest init not run.")
+            self.set_tempest_ready(False)
+            return
+
         try:
             pebble.init_tempest(env)
         except RuntimeError:
@@ -296,17 +329,16 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
         test_list: str = event.params["test-list"].strip()
 
         env = self._get_environment_for_tempest(TempestEnvVariant.ADHOC)
+
         try:
             summary = self.pebble_handler().run_tempest_tests(
                 regexes, exclude_regex, test_list, serial, env
             )
         except RuntimeError as e:
-            # put the message in set_results instead of event.fail,
-            # because event.fail message is not always displayed to the user:
-            # https://bugs.launchpad.net/juju/+bug/2052765
             event.set_results({"error": str(e)})
             event.fail()
             return
+
         event.set_results(
             {
                 "summary": summary,

charms/tempest-k8s/src/handlers.py

@@ -36,6 +36,10 @@ import ops.model
 import ops.pebble
 import ops_sunbeam.container_handlers as sunbeam_chandlers
 import ops_sunbeam.relation_handlers as sunbeam_rhandlers
+from utils.cleanup import (
+    CleanUpError,
+    run_extensive_cleanup,
+)
 from utils.constants import (
     OPENSTACK_DOMAIN,
     OPENSTACK_PROJECT,
@@ -157,6 +161,16 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
 
         Raise a RuntimeError if something goes wrong.
         """
+        # push the cleanup script to container
+        with open("src/utils/cleanup.py") as f:
+            self.container.push(
+                f"{TEMPEST_HOME}/cleanup.py",
+                f,
+                user="tempest",
+                group="tempest",
+                make_dirs=True,
+            )
+
         # Pebble runs cron, which runs tempest periodically
         # when periodic checks are enabled.
         # This ensures that tempest gets the env, inherited from cron.
@@ -456,6 +470,10 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
 
     def _teardown_tempest_resource_ops(self) -> List[dict]:
         """Tear down openstack resource ops."""
+        credential_id = self._ensure_credential()
+        credential_secret = self.model.get_secret(id=credential_id)
+        content = credential_secret.get_content()
+        username = content.get("username")
         teardown_ops = [
             {
                 "name": "show_domain",
@@ -463,6 +481,20 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
                     "name": OPENSTACK_DOMAIN,
                 },
             },
+            {
+                "name": "delete_project",
+                "params": {
+                    "name": OPENSTACK_PROJECT,
+                    "domain": "{{ show_domain[0].id }}",
+                },
+            },
+            {
+                "name": "delete_user",
+                "params": {
+                    "name": username,
+                    "domain": "{{ show_domain[0].id }}",
+                },
+            },
             {
                 "name": "update_domain",
                 "params": {
@@ -517,6 +549,19 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
                     self._set_secret({"auth-url": auth_url})
                     return
 
+    def _process_setup_tempest_resource_response(self, response: dict) -> None:
+        """Process extra ops request: "_setup_tempest_resource_request"."""
+        for op in response.get("ops", []):
+            if op.get("name") != "create_domain":
+                continue
+            if op.get("return-code") != 0:
+                logger.warning("Create domain ops failed.")
+                return
+            domain_id = op.get("value", {}).get("id")
+            if domain_id is not None:
+                self._set_secret({"domain-id": domain_id})
+                return
+
     def _on_provider_ready(self, event) -> None:
         """Handles response available events."""
         if not self.model.unit.is_leader():
@@ -535,15 +580,30 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
         response = self.interface.response
         logger.info("%s", json.dumps(response, indent=4))
         self._process_list_endpoint_response(response)
+        self._process_setup_tempest_resource_response(response)
         self.callback_f(event)
 
     def _on_provider_goneaway(self, event) -> None:
         """Handle gone_away event."""
         if not self.model.unit.is_leader():
             return
-        logger.info(
+        logger.info("Identity ops provider gone away")
-            "Identity ops provider gone away: teardown tempest resources"
+        credential = self.get_user_credential()
-        )
+        env = {
+            "OS_AUTH_URL": credential.get("auth-url"),
+            "OS_USERNAME": credential.get("username"),
+            "OS_PASSWORD": credential.get("password"),
+            "OS_PROJECT_NAME": credential.get("project-name"),
+            "OS_DOMAIN_ID": credential.get("domain-id"),
+            "OS_USER_DOMAIN_ID": credential.get("domain-id"),
+            "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
+        }
+        try:
+            # do an extensive clean-up upon identity relation removal
+            run_extensive_cleanup(env)
+
+        except CleanUpError as e:
+            logger.warning("Clean-up failed: %s", str(e))
         self.callback_f(event)
 
 

charms/tempest-k8s/src/templates/tempest-run-wrapper.j2

@@ -15,6 +15,7 @@ echo ":: discover-tempest-config" >> "$TMP_FILE"
 if discover-tempest-config --test-accounts "$TEMPEST_TEST_ACCOUNTS" --out "$TEMPEST_CONF" >> "$TMP_FILE" 2>&1; then
     echo ":: tempest run" >> "$TMP_FILE"
     tempest run --workspace "$TEMPEST_WORKSPACE" "$@" >> "$TMP_FILE" 2>&1
+    python3 "$TEMPEST_HOME/cleanup.py"
 else
     echo ":: skipping tempest run because discover-tempest-config had errors" >> "$TMP_FILE"
 fi

charms/tempest-k8s/src/utils/cleanup.py

@@ -0,0 +1,317 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utils for cleaning up tempest-related resources."""
+import os
+from collections.abc import (
+    Callable,
+)
+
+import yaml
+from keystoneauth1.exceptions.catalog import (
+    EndpointNotFound,
+)
+from keystoneauth1.exceptions.http import (
+    Unauthorized,
+)
+from openstack.connection import (
+    Connection,
+)
+from openstack.exceptions import (
+    ForbiddenException,
+)
+
+RESOURCE_PREFIX = "tempest-"
+
+
+class CleanUpError(Exception):
+    """Exception raised when clean-up process terminated unsuccessfully."""
+
+
+def _connect_to_os(env: dict) -> Connection:
+    """Establish connection to the OpenStack cloud."""
+    return Connection(
+        auth_url=env["OS_AUTH_URL"],
+        project_name=env["OS_PROJECT_NAME"],
+        username=env["OS_USERNAME"],
+        password=env["OS_PASSWORD"],
+        user_domain_id=env["OS_USER_DOMAIN_ID"],
+        project_domain_id=env["OS_USER_DOMAIN_ID"],
+    )
+
+
+def _get_exclusion_resources(account_file: str) -> dict[str, set[str]]:
+    """Get users and projects to be excluded from clean-up.
+
+    The resources are looked up in an account_file which is generated by
+    `tempest account-generator`.
+
+    Note that this lookup will only work in workload container as it needs
+    access to the generated test accounts file.
+    """
+    try:
+        with open(account_file, "r") as file:
+            parsed_file = yaml.safe_load(file)
+
+        return {
+            "projects": {account["project_name"] for account in parsed_file},
+            "users": {account["username"] for account in parsed_file},
+        }
+    except FileNotFoundError as e:
+        raise CleanUpError("Test account file doesn't exist.") from e
+    except KeyError as e:
+        raise CleanUpError("Malformated test account file.") from e
+
+
+def _get_test_projects_in_domain(
+    conn: Connection, domain_id: str, exclude_projects: set[str] = set()
+) -> list[str]:
+    """Get test projects in domain to do cleanup for.
+
+    Projects in exclude_projects will not be included in the returned list.
+    """
+    return [
+        project.id
+        for project in conn.identity.projects(domain_id=domain_id)
+        if project.name.startswith(RESOURCE_PREFIX)
+        and project.name not in exclude_projects
+    ]
+
+
+def _cleanup_compute_resources(conn: Connection, project_id: str) -> None:
+    """Delete compute resources with names starting with prefix in the specified project.
+
+    The compute resources to be removed are instances and keypairs.
+    """
+    # Delete instances
+    for server in conn.compute.servers(project_id=project_id):
+        if server.name.startswith(RESOURCE_PREFIX):
+            conn.compute.delete_server(server.id)
+
+    # Delete keypairs
+    for keypair in conn.compute.keypairs():
+        if keypair.name.startswith(RESOURCE_PREFIX):
+            conn.compute.delete_keypair(keypair)
+
+
+def _cleanup_block_resources(conn: Connection, project_id: str) -> None:
+    """Delete block storage resources with names starting with prefix in the specified project.
+
+    The block storage resources to be removed are snapshots and instances.
+    """
+    # Delete snapshots
+    for snapshot in conn.block_store.snapshots(
+        details=True, project_id=project_id
+    ):
+        if snapshot.name.startswith(RESOURCE_PREFIX):
+            conn.block_store.delete_snapshot(snapshot.id)
+
+    # Delete volumes
+    for volume in conn.block_store.volumes(
+        details=True, project_id=project_id
+    ):
+        if volume.name.startswith(RESOURCE_PREFIX):
+            conn.block_store.delete_volume(volume.id)
+
+
+def _cleanup_images(conn: Connection, project_id: str) -> None:
+    """Delete images with names starting with prefix and owned by the specified project."""
+    for image in conn.image.images():
+        # TODO: to be extra careful, we should also check the prefix of the image
+        # However, some tempest tests are not creating images with the prefix, so
+        # we should wait until https://review.opendev.org/c/openstack/tempest/+/908358
+        # is released.
+        if image.owner == project_id:
+            conn.image.delete_image(image.id)
+
+
+def _cleanup_networks_resources(conn: Connection, project_id: str) -> None:
+    """Delete network resources with names starting with prefix in the specified project.
+
+    The network resources to be removed are ports, routers, and networks.
+    """
+    # Delete ports and routers
+    for router in conn.network.routers(project_id=project_id):
+        if router.name.startswith(RESOURCE_PREFIX):
+            for port in conn.network.ports(device_id=router.id):
+                conn.network.remove_interface_from_router(
+                    router, port_id=port.id
+                )
+            conn.network.delete_router(router.id)
+
+    # Delete networks
+    for network in conn.network.networks(project_id=project_id):
+        if network.name.startswith(RESOURCE_PREFIX):
+            conn.network.delete_network(network.id)
+
+
+def _cleanup_stacks(conn: Connection, project_id: str) -> None:
+    """Delete stacks with names starting with prefix and owned by the specified project.
+
+    If Heat service is not found in the cloud, this clean-up will be skipped.
+    """
+    try:
+        for stack in conn.orchestration.stacks(project_id=project_id):
+            if stack.name.startswith(RESOURCE_PREFIX):
+                conn.orchestration.delete_stack(stack.id)
+    except EndpointNotFound:
+        # do nothing if the heat endpoint is not found
+        pass
+
+
+def _cleanup_users(
+    conn: Connection, domain_id: str, exclude_users: set[str] = set()
+) -> None:
+    """Delete users with names starting with prefix in the specified domain.
+
+    If exclude_users is specified, users in exclude_users will not be removed.
+    """
+    for user in conn.identity.users(domain_id=domain_id):
+        if (
+            user.name.startswith(RESOURCE_PREFIX)
+            and user.name not in exclude_users
+        ):
+            conn.identity.delete_user(user.id)
+
+
+def _cleanup_project(conn: Connection, project_id: str) -> None:
+    """Delete a project given its id."""
+    conn.identity.delete_project(project_id)
+
+
+def _run_cleanup_functions(
+    conn: Connection, projects: list[str], functions: list[Callable]
+) -> list[str]:
+    """Run clean-up function on a list of projects."""
+    failure_message = []
+    for project_id in projects:
+        for func in functions:
+            try:
+                func(conn, project_id)
+            except Exception as e:
+                failure_message.append(f"Error calling {func.__name__}: {e}")
+    return failure_message
+
+
+def run_quick_cleanup(env: dict) -> None:
+    """Perform the quick cleanup of tempest resources under a specific domain.
+
+    This clean-up removes compute instances, keypairs, volumes, snapshots,
+    images, networks, projects, users, and stacks that are not associated with
+    the pre-generated test accounts and projects defined in account_file.
+
+    Note that this clean up will only work in workload container as it needs
+    access to the generated test accounts file.
+    """
+    conn = _connect_to_os(env)
+    cleanup_funcs = [
+        _cleanup_compute_resources,
+        _cleanup_block_resources,
+        _cleanup_images,
+        _cleanup_stacks,
+    ]
+    filtered_cleanup_funcs = [
+        _cleanup_networks_resources,
+        _cleanup_project,
+    ]
+    exclude_resources = _get_exclusion_resources(env["TEMPEST_TEST_ACCOUNTS"])
+    failure_message = []
+
+    try:
+        # get all projects with prefix in domain
+        test_projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"])
+
+        # get projects that are not found in account file
+        filtered_test_projects = _get_test_projects_in_domain(
+            conn, env["OS_DOMAIN_ID"], exclude_resources["projects"]
+        )
+    except (ForbiddenException, Unauthorized) as e:
+        raise CleanUpError("Operation not authorized.") from e
+
+    failure_message.extend(
+        _run_cleanup_functions(conn, test_projects, cleanup_funcs)
+    )
+    failure_message.extend(
+        _run_cleanup_functions(
+            conn, filtered_test_projects, filtered_cleanup_funcs
+        )
+    )
+
+    try:
+        _cleanup_users(conn, env["OS_DOMAIN_ID"], exclude_resources["users"])
+    except Exception as e:
+        failure_message.append(f"\nError cleaning up users: {e}")
+
+    if failure_message:
+        raise CleanUpError("\n".join(failure_message))
+
+
+def run_extensive_cleanup(env: dict) -> None:
+    """Perform the extensive cleanup of tempest resources under a specific domain.
+
+    This clean-up removes compute instances, keypairs, volumes, snapshots,
+    images, and stacks, as well as generated test accounts, projects, and the
+    network resources associated with them.
+    """
+    conn = _connect_to_os(env)
+    cleanup_funcs = [
+        _cleanup_compute_resources,
+        _cleanup_block_resources,
+        _cleanup_images,
+        _cleanup_stacks,
+        _cleanup_networks_resources,
+        _cleanup_project,
+    ]
+    failure_message = []
+
+    try:
+        # get all projects with prefix in domain
+        projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"])
+    except (ForbiddenException, Unauthorized) as e:
+        raise CleanUpError("Operation not authorized.") from e
+
+    failure_message.extend(
+        _run_cleanup_functions(conn, projects, cleanup_funcs)
+    )
+
+    try:
+        _cleanup_users(conn, env["OS_DOMAIN_ID"])
+    except Exception as e:
+        failure_message.append(f"\nError cleaning up users: {e}")
+
+    if failure_message:
+        raise CleanUpError("\n".join(failure_message))
+
+
+def main() -> None:
+    """Entrypoint for executing the script directly.
+
+    This will be used in periodic test runs.
+    Quick cleanup will be performed.
+    """
+    env = {
+        "OS_AUTH_URL": os.getenv("OS_AUTH_URL", ""),
+        "OS_USERNAME": os.getenv("OS_USERNAME", ""),
+        "OS_PASSWORD": os.getenv("OS_PASSWORD", ""),
+        "OS_PROJECT_NAME": os.getenv("OS_PROJECT_NAME", ""),
+        "OS_DOMAIN_ID": os.getenv("OS_DOMAIN_ID", ""),
+        "OS_USER_DOMAIN_ID": os.getenv("OS_USER_DOMAIN_ID", ""),
+        "OS_PROJECT_DOMAIN_ID": os.getenv("OS_PROJECT_DOMAIN_ID", ""),
+        "TEMPEST_TEST_ACCOUNTS": os.getenv("TEMPEST_TEST_ACCOUNTS", ""),
+    }
+
+    run_quick_cleanup(env)
+
+
+if __name__ == "__main__":
+    main()

charms/tempest-k8s/src/utils/constants.py

@@ -29,7 +29,8 @@ TEMPEST_WORKSPACE = "tempest"
 
 OPENSTACK_USER = "tempest"
 OPENSTACK_DOMAIN = "tempest"
-OPENSTACK_PROJECT = "tempest-CloudValidation"
+# not use tempest as prefix to exclude this project from utils/cleanup.py scope
+OPENSTACK_PROJECT = "CloudValidation-tempest"
 OPENSTACK_ROLE = "admin"
 
 # keys for application data

charms/tempest-k8s/tests/unit/test_cleanup.py

@@ -0,0 +1,552 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for tempest-related resources cleanup."""
+import textwrap
+import unittest
+from unittest.mock import (
+    MagicMock,
+    call,
+    mock_open,
+    patch,
+)
+
+from keystoneauth1.exceptions.catalog import (
+    EndpointNotFound,
+)
+from keystoneauth1.exceptions.http import (
+    Unauthorized,
+)
+from openstack.exceptions import (
+    ForbiddenException,
+)
+from utils.cleanup import (
+    CleanUpError,
+    _cleanup_block_resources,
+    _cleanup_compute_resources,
+    _cleanup_images,
+    _cleanup_networks_resources,
+    _cleanup_project,
+    _cleanup_stacks,
+    _cleanup_users,
+    _connect_to_os,
+    _get_exclusion_resources,
+    _get_test_projects_in_domain,
+    _run_cleanup_functions,
+    run_extensive_cleanup,
+    run_quick_cleanup,
+)
+
+
+class TestCleanup(unittest.TestCase):
+    """Test tempest resources clean-up."""
+
+    def setUp(self):
+        """Set up test construction."""
+        self.patcher = patch("utils.cleanup.Connection")
+        self.mock_connection = self.patcher.start()
+
+    def tearDown(self):
+        """Tear down test construction."""
+        self.patcher.stop()
+
+    def test_connect_to_os(self):
+        """Test establishing OS connection."""
+        env = {
+            "OS_AUTH_URL": "http://10.6.0.20/openstack-keystone",
+            "OS_USERNAME": "test_user",
+            "OS_PASSWORD": "userpass",
+            "OS_PROJECT_NAME": "test_project",
+            "OS_DOMAIN_ID": "domain_id",
+            "OS_USER_DOMAIN_ID": "domain_id",
+            "OS_PROJECT_DOMAIN_ID": "domain_id",
+        }
+        _connect_to_os(env)
+
+        self.mock_connection.assert_called_once_with(
+            auth_url=env["OS_AUTH_URL"],
+            project_name=env["OS_PROJECT_NAME"],
+            username=env["OS_USERNAME"],
+            password=env["OS_PASSWORD"],
+            user_domain_id=env["OS_USER_DOMAIN_ID"],
+            project_domain_id=env["OS_USER_DOMAIN_ID"],
+        )
+
+    def test_get_exclude_resources(self):
+        """Test get exclude resources from test accounts file."""
+        account_file_content = textwrap.dedent(
+            """
+        - domain_name: mydomain
+          password: password1
+          project_name: tempest-test_creds-11949114
+          resources:
+            network: tempest-test_creds-9369097-network
+          username: tempest-test_creds-11949114-project
+        - domain_name: mydomain
+          password: password2
+          project_name: tempest-test_creds-18083146
+          resources:
+            network: tempest-test_creds-20041716-network
+          username: tempest-test_creds-18083146-project
+        """
+        )
+
+        expected_result = {
+            "projects": {
+                "tempest-test_creds-11949114",
+                "tempest-test_creds-18083146",
+            },
+            "users": {
+                "tempest-test_creds-11949114-project",
+                "tempest-test_creds-18083146-project",
+            },
+        }
+
+        with patch(
+            "utils.cleanup.open",
+            new_callable=mock_open,
+            read_data=account_file_content,
+        ):
+            result = _get_exclusion_resources("test_account_file.yaml")
+
+        self.assertEqual(result, expected_result)
+
+    def test_get_test_projects_in_domain(self):
+        """Test get tempest projects of a specified domain."""
+        tempest_project1 = MagicMock()
+        tempest_project1.configure_mock(id="1", name="tempest-project-1")
+        tempest_project2 = MagicMock()
+        tempest_project2.configure_mock(id="2", name="tempest-project-2")
+        non_tempest_project = MagicMock()
+        non_tempest_project.configure_mock(id="3", name="non-tempest-project")
+        self.mock_connection.identity.projects.return_value = [
+            tempest_project1,
+            tempest_project2,
+            non_tempest_project,
+        ]
+
+        projects = _get_test_projects_in_domain(
+            self.mock_connection, "tempest_domain"
+        )
+
+        self.assertEqual(projects, ["1", "2"])
+
+    def test_get_test_projects_in_domain_with_exclusion(self):
+        """Test get tempest projects of a specified domain with exclusion."""
+        tempest_project1 = MagicMock()
+        tempest_project1.configure_mock(id="1", name="tempest-project-1")
+        tempest_project2 = MagicMock()
+        tempest_project2.configure_mock(id="2", name="tempest-project-2")
+        non_tempest_project = MagicMock()
+        non_tempest_project.configure_mock(id="3", name="non-tempest-project")
+        self.mock_connection.identity.projects.return_value = [
+            tempest_project1,
+            tempest_project2,
+            non_tempest_project,
+        ]
+
+        projects = _get_test_projects_in_domain(
+            self.mock_connection,
+            domain_id="tempest_domain",
+            exclude_projects={"tempest-project-1"},
+        )
+
+        self.assertEqual(projects, ["2"])
+
+    def test_cleanup_block_resources(self):
+        """Test cleanup volumes and snapshots."""
+        tempest_snapshots = MagicMock()
+        tempest_snapshots.configure_mock(id="1", name="tempest-snapshots-1")
+        non_tempest_snapshots = MagicMock()
+        non_tempest_snapshots.configure_mock(
+            id="2", name="non-tempest-snapshots"
+        )
+        self.mock_connection.block_store.snapshots.return_value = [
+            tempest_snapshots,
+            non_tempest_snapshots,
+        ]
+
+        tempest_volume = MagicMock()
+        tempest_volume.configure_mock(id="1", name="tempest-volume-1")
+        non_tempest_volume = MagicMock()
+        non_tempest_volume.configure_mock(id="2", name="non-tempest-volume")
+        self.mock_connection.block_store.volumes.return_value = [
+            tempest_volume,
+            non_tempest_volume,
+        ]
+
+        _cleanup_block_resources(self.mock_connection, "test_project")
+
+        self.mock_connection.block_store.delete_snapshot.assert_called_once_with(
+            "1"
+        )
+        self.mock_connection.block_store.delete_volume.assert_called_once_with(
+            "1"
+        )
+
+    def test_cleanup_networks_resources(self):
+        """Test cleanup network resources."""
+        tempest_router = MagicMock()
+        tempest_router.configure_mock(id="1", name="tempest-router")
+        non_tempest_router = MagicMock()
+        non_tempest_router.configure_mock(id="2", name="non-tempest-router")
+        self.mock_connection.network.routers.return_value = [
+            tempest_router,
+            non_tempest_router,
+        ]
+
+        tempest_port1 = MagicMock()
+        tempest_port1.configure_mock(id="1")
+        tempest_port2 = MagicMock()
+        tempest_port2.configure_mock(id="2")
+        self.mock_connection.network.ports.return_value = [
+            tempest_port1,
+            tempest_port2,
+        ]
+
+        tempest_network = MagicMock()
+        tempest_network.configure_mock(id="1", name="tempest-network")
+        non_tempest_network = MagicMock()
+        non_tempest_network.configure_mock(id="2", name="non-tempest-network")
+        self.mock_connection.network.networks.return_value = [
+            tempest_network,
+            non_tempest_network,
+        ]
+
+        _cleanup_networks_resources(self.mock_connection, "project_id")
+
+        self.mock_connection.network.remove_interface_from_router.call_args_list == [
+            call(tempest_router, port_id="1"),
+            call(tempest_router, port_id="2"),
+        ]
+        self.mock_connection.network.delete_router.assert_called_once_with("1")
+        self.mock_connection.network.delete_network.assert_called_once_with(
+            "1"
+        )
+
+    def test_cleanup_compute_resources(self):
+        """Test cleanup instances and keypairs."""
+        tempest_instance = MagicMock()
+        tempest_instance.configure_mock(id="1", name="tempest-server-1")
+        non_tempest_instance = MagicMock()
+        non_tempest_instance.configure_mock(id="2", name="non-tempest-server")
+        self.mock_connection.compute.servers.return_value = [
+            tempest_instance,
+            non_tempest_instance,
+        ]
+
+        tempest_keypair = MagicMock()
+        tempest_keypair.configure_mock(id="1", name="tempest-keypair-1")
+        non_tempest_keypair = MagicMock()
+        non_tempest_keypair.configure_mock(
+            id="2", name="non-tempest-keypair-2"
+        )
+
+        self.mock_connection.compute.keypairs.return_value = [
+            tempest_keypair,
+            non_tempest_keypair,
+        ]
+
+        _cleanup_compute_resources(self.mock_connection, "test_project")
+
+        self.mock_connection.compute.delete_server.assert_called_once_with("1")
+        self.mock_connection.compute.delete_keypair.assert_called_once_with(
+            tempest_keypair
+        )
+
+    def test_cleanup_images(self):
+        """Test cleanup images."""
+        image_owned_by_project = MagicMock()
+        image_owned_by_project.configure_mock(
+            id="1", name="image-1", owner="test_project"
+        )
+        image_not_owned_by_project = MagicMock()
+        image_not_owned_by_project.configure_mock(
+            id="3", name="image-3", owner="not_test_project"
+        )
+        self.mock_connection.image.images.return_value = [
+            image_owned_by_project,
+            image_not_owned_by_project,
+        ]
+
+        _cleanup_images(self.mock_connection, "test_project")
+
+        self.mock_connection.image.delete_image.assert_called_once_with("1")
+
+    def test_cleanup_project(self):
+        """Test cleanup projects."""
+        project_id = "tempest_project_id"
+
+        _cleanup_project(self.mock_connection, project_id)
+
+        self.mock_connection.identity.delete_project.assert_called_with(
+            project_id
+        )
+
+    def test_cleanup_users(self):
+        """Test cleanup users."""
+        tempest_user = MagicMock()
+        tempest_user.configure_mock(
+            domain_id="tempest", id="1", name="tempest-user-1"
+        )
+        non_tempest_user = MagicMock()
+        non_tempest_user.configure_mock(
+            domain_id="tempest", id="2", name="non-tempest-user-2"
+        )
+
+        self.mock_connection.identity.users.return_value = [
+            tempest_user,
+            non_tempest_user,
+        ]
+
+        _cleanup_users(self.mock_connection, domain_id="tempest")
+
+        self.mock_connection.identity.delete_user.assert_called_once_with("1")
+
+    def test_cleanup_users_with_exclusion(self):
+        """Test cleanup users with exclusion."""
+        tempest_user1 = MagicMock()
+        tempest_user1.configure_mock(
+            domain_id="tempest", id="1", name="tempest-user-1"
+        )
+        tempest_user2 = MagicMock()
+        tempest_user2.configure_mock(
+            domain_id="tempest", id="2", name="tempest-user-2"
+        )
+        non_tempest_user = MagicMock()
+        non_tempest_user.configure_mock(
+            domain_id="tempest", id="3", name="non-tempest-user"
+        )
+
+        self.mock_connection.identity.users.return_value = [
+            tempest_user1,
+            tempest_user2,
+            non_tempest_user,
+        ]
+
+        _cleanup_users(
+            self.mock_connection,
+            domain_id="tempest",
+            exclude_users={"tempest-user-1"},
+        )
+
+        self.mock_connection.identity.delete_user.assert_called_once_with("2")
+
+    def test_cleanup_stacks_success(self):
+        """Test cleanup heat stacks."""
+        tempest_stack = MagicMock()
+        tempest_stack.configure_mock(id="1", name="tempest-stack-1")
+        non_tempest_stack = MagicMock()
+        non_tempest_stack.configure_mock(id="2", name="non-tempest-stack-2")
+        self.mock_connection.orchestration.stacks.return_value = [
+            tempest_stack,
+            non_tempest_stack,
+        ]
+
+        _cleanup_stacks(self.mock_connection, "test_project")
+
+        self.mock_connection.orchestration.delete_stack.assert_called_once_with(
+            "1"
+        )
+
+    def test_cleanup_stacks_endpoint_not_found(self):
+        """Test cleanup stacks when heat endpoint is not found."""
+        self.mock_connection.orchestration.stacks.side_effect = (
+            EndpointNotFound
+        )
+
+        _cleanup_stacks(self.mock_connection, "test_project")
+
+        self.mock_connection.orchestration.delete_stack.assert_not_called()
+
+    @patch("utils.cleanup._cleanup_compute_resources")
+    @patch("utils.cleanup._cleanup_block_resources")
+    @patch("utils.cleanup._cleanup_images")
+    @patch("utils.cleanup._cleanup_stacks")
+    @patch("utils.cleanup._cleanup_networks_resources")
+    @patch("utils.cleanup._cleanup_project")
+    def test_run_cleanup_functions(
+        self,
+        mock_cleanup_project,
+        mock_cleanup_networks_resources,
+        mock_cleanup_stacks,
+        mock_cleanup_images,
+        mock_cleanup_block_resources,
+        mock_cleanup_compute_resources,
+    ):
+        """Test run cleanup functions."""
+        cleanup_funcs = [
+            mock_cleanup_compute_resources,
+            mock_cleanup_block_resources,
+            mock_cleanup_images,
+            mock_cleanup_stacks,
+            mock_cleanup_networks_resources,
+            mock_cleanup_project,
+        ]
+        projects = ["tempest_project_id"]
+
+        _run_cleanup_functions(self.mock_connection, projects, cleanup_funcs)
+
+        mock_cleanup_stacks.assert_called_once()
+        mock_cleanup_images.assert_called_once()
+        mock_cleanup_block_resources.assert_called_once()
+        mock_cleanup_compute_resources.assert_called_once()
+        mock_cleanup_networks_resources.assert_called_once()
+        mock_cleanup_project.assert_called_once()
+
+    @patch("utils.cleanup._cleanup_compute_resources")
+    @patch("utils.cleanup._cleanup_block_resources")
+    @patch("utils.cleanup._cleanup_images")
+    @patch("utils.cleanup._cleanup_stacks")
+    @patch("utils.cleanup._cleanup_networks_resources")
+    @patch("utils.cleanup._cleanup_project")
+    def test_run_cleanup_functions_unsuccessful(
+        self,
+        mock_cleanup_project,
+        mock_cleanup_networks_resources,
+        mock_cleanup_stacks,
+        mock_cleanup_images,
+        mock_cleanup_block_resources,
+        mock_cleanup_compute_resources,
+    ):
+        """Test run cleanup functions."""
+        cleanup_funcs = [
+            mock_cleanup_compute_resources,
+            mock_cleanup_block_resources,
+            mock_cleanup_images,
+            mock_cleanup_stacks,
+            mock_cleanup_networks_resources,
+            mock_cleanup_project,
+        ]
+        projects = ["tempest_project_id"]
+        mock_cleanup_images.__name__ = "_cleanup_images"
+        mock_cleanup_images.side_effect = Exception
+
+        _run_cleanup_functions(self.mock_connection, projects, cleanup_funcs)
+
+        mock_cleanup_stacks.assert_called_once()
+        mock_cleanup_images.assert_called_once()
+        mock_cleanup_block_resources.assert_called_once()
+        mock_cleanup_compute_resources.assert_called_once()
+        mock_cleanup_networks_resources.assert_called_once()
+        mock_cleanup_project.assert_called_once()
+
+    @patch("utils.cleanup._run_cleanup_functions")
+    @patch("utils.cleanup._cleanup_users")
+    @patch("utils.cleanup._connect_to_os")
+    @patch("utils.cleanup._get_test_projects_in_domain")
+    @patch("utils.cleanup._get_exclusion_resources")
+    def test_run_quick_cleanup(
+        self,
+        mock_get_exclusion_resources,
+        mock_get_test_projects_in_domain,
+        mock_connect_to_os,
+        mock_cleanup_users,
+        mock_run_cleanup_functions,
+    ):
+        """Test run quick cleanup."""
+        env = MagicMock()
+        mock_get_test_projects_in_domain.side_effect = [
+            ["tempest_project_id"],
+            ["tempest_project_id"],
+        ]
+
+        run_quick_cleanup(env)
+
+        mock_connect_to_os.assert_called_once_with(env)
+        mock_get_exclusion_resources.assert_called_once()
+        mock_get_test_projects_in_domain.assert_called()
+        mock_run_cleanup_functions.assert_called()
+        mock_cleanup_users.assert_called_once()
+
+    @patch("utils.cleanup._run_cleanup_functions")
+    @patch("utils.cleanup._cleanup_users")
+    @patch("utils.cleanup._connect_to_os")
+    @patch("utils.cleanup._get_test_projects_in_domain")
+    @patch("utils.cleanup._get_exclusion_resources")
+    def test_run_quick_cleanup_error(
+        self,
+        mock_get_exclusion_resources,
+        mock_get_test_projects_in_domain,
+        mock_connect_to_os,
+        mock_cleanup_users,
+        mock_run_cleanup_functions,
+    ):
+        """Test run quick cleanup with failure."""
+        env = MagicMock()
+        mock_get_test_projects_in_domain.side_effect = ForbiddenException
+
+        with self.assertRaises(CleanUpError) as error:
+            run_quick_cleanup(env)
+
+        mock_connect_to_os.assert_called_once_with(env)
+        mock_get_exclusion_resources.assert_called_once()
+        mock_get_test_projects_in_domain.assert_called_once()
+        self.assertEqual(str(error.exception), "Operation not authorized.")
+
+        mock_run_cleanup_functions.assert_not_called()
+        mock_cleanup_users.assert_not_called()
+
+    @patch("utils.cleanup._run_cleanup_functions")
+    @patch("utils.cleanup._cleanup_users")
+    @patch("utils.cleanup._connect_to_os")
+    @patch("utils.cleanup._get_test_projects_in_domain")
+    @patch("utils.cleanup._get_exclusion_resources")
+    def test_run_extensive_cleanup(
+        self,
+        mock_get_exclusion_resources,
+        mock_get_test_projects_in_domain,
+        mock_connect_to_os,
+        mock_cleanup_users,
+        mock_run_cleanup_functions,
+    ):
+        """Test run extensive cleanup."""
+        env = MagicMock()
+        mock_get_test_projects_in_domain.side_effect = [
+            ["tempest_project_id"],
+            ["tempest_project_id"],
+        ]
+
+        run_extensive_cleanup(env)
+
+        mock_connect_to_os.assert_called_once_with(env)
+        mock_get_test_projects_in_domain.assert_called()
+        mock_run_cleanup_functions.assert_called_once()
+        mock_cleanup_users.assert_called_once()
+
+    @patch("utils.cleanup._run_cleanup_functions")
+    @patch("utils.cleanup._cleanup_users")
+    @patch("utils.cleanup._connect_to_os")
+    @patch("utils.cleanup._get_test_projects_in_domain")
+    @patch("utils.cleanup._get_exclusion_resources")
+    def test_run_extensive_cleanup_permission_error(
+        self,
+        mock_get_exclusion_resources,
+        mock_get_test_projects_in_domain,
+        mock_connect_to_os,
+        mock_cleanup_users,
+        mock_run_cleanup_functions,
+    ):
+        """Test run extensive cleanup with failure."""
+        env = MagicMock()
+        mock_get_test_projects_in_domain.side_effect = Unauthorized
+
+        with self.assertRaises(CleanUpError) as error:
+            run_extensive_cleanup(env)
+
+        mock_connect_to_os.assert_called_once_with(env)
+        mock_get_test_projects_in_domain.assert_called_once()
+        self.assertEqual(str(error.exception), "Operation not authorized.")
+
+        mock_run_cleanup_functions.assert_not_called()
+        mock_cleanup_users.assert_not_called()

charms/tempest-k8s/tests/unit/test_tempest_charm.py

@@ -18,6 +18,9 @@
 
 import json
 import pathlib
+from unittest.mock import (
+    patch,
+)
 
 import charm
 import mock
@@ -42,9 +45,12 @@ TEST_TEMPEST_ENV = {
     "OS_USERNAME": "tempest",
     "OS_PASSWORD": "password",
     "OS_USER_DOMAIN_NAME": "tempest",
-    "OS_PROJECT_NAME": "tempest-CloudValidation",
+    "OS_PROJECT_NAME": "CloudValidation-tempest",
     "OS_PROJECT_DOMAIN_NAME": "tempest",
     "OS_DOMAIN_NAME": "tempest",
+    "OS_PROJECT_DOMAIN_ID": "tempest-domain-id",
+    "OS_USER_DOMAIN_ID": "tempest-domain-id",
+    "OS_DOMAIN_ID": "tempest-domain-id",
     "TEMPEST_CONCURRENCY": "4",
     "TEMPEST_CONF": "/var/lib/tempest/workspace/etc/tempest.conf",
     "TEMPEST_HOME": "/var/lib/tempest",
@@ -97,6 +103,18 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase):
         self.addCleanup(self.harness.cleanup)
         self.harness.begin()
         self.harness.set_leader()
+        self.patcher1 = patch("utils.cleanup.Connection")
+        self.patcher2 = patch(
+            "utils.cleanup._get_exclusion_resources",
+            return_value={"projects": set(), "users": set()},
+        )
+        self.patcher1.start()
+        self.patcher2.start()
+
+    def tearDown(self):
+        """Tear down test construction."""
+        self.patcher1.stop()
+        self.patcher2.stop()
 
     def add_identity_ops_relation(self, harness):
         """Add identity resource relation."""
@@ -108,7 +126,8 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase):
                 "username": "tempest",
                 "password": "password",
                 "domain-name": "tempest",
-                "project-name": "tempest-CloudValidation",
+                "domain-id": "tempest-domain-id",
+                "project-name": "CloudValidation-tempest",
                 "auth-url": "http://10.6.0.23/openstack-keystone/v3",
             },
         )

test-requirements.txt

@@ -18,3 +18,4 @@ croniter                   # tempest-k8s
 git+https://github.com/juju/charm-helpers.git#egg=charmhelpers                               # cinder-ceph-k8s,glance-k8s,gnocchi-k8s
 git+https://opendev.org/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client  # cinder-ceph-k8s
 requests-unixsocket # sunbeam-clusterd
+openstacksdk               # tempest-k8s