Handle tempest resources clean-up

Three levels of clean-up is implemented to facilitate different
scenarios:
1. quick clean-up that removes resources created by tempest tests.
This can be used in-between test runs.
2. extensive clean-up that, in addition to 1, also removes test
accounts, project, and the associated network resources. This is
used after charm refresh and identity-ops relation departed where
the previous test accounts becomes obsoletes.
3. full cleanup that, in addition to 1 and 2, deletes the testing
domain, user, and project created through keystone relation. This
is used at the initial stage of keystone relation joins to get
a fresh env.

Change-Id: If172e9dde0e04c53849d7a0b30fc27f62c5bdbd9
This commit is contained in:
Tianqi Xiao 2024-02-08 04:25:21 +00:00 committed by Tianqi
parent 2e2dd81e7e
commit b0151aa6ee
9 changed files with 995 additions and 9 deletions

View File

@ -10,3 +10,6 @@ tenacity
# for validating cron expressions
croniter
# for handling cleanup
openstacksdk

View File

@ -50,6 +50,10 @@ from ops.model import (
from ops_sunbeam.config_contexts import (
ConfigContext,
)
from utils.cleanup import (
CleanUpError,
run_extensive_cleanup,
)
from utils.constants import (
CONTAINER,
TEMPEST_ADHOC_OUTPUT,
@ -210,6 +214,9 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
"OS_PROJECT_NAME": credential.get("project-name"),
"OS_PROJECT_DOMAIN_NAME": credential.get("domain-name"),
"OS_DOMAIN_NAME": credential.get("domain-name"),
"OS_DOMAIN_ID": credential.get("domain-id"),
"OS_USER_DOMAIN_ID": credential.get("domain-id"),
"OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
"TEMPEST_CONCURRENCY": TEMPEST_CONCURRENCY,
"TEMPEST_CONF": TEMPEST_CONF,
"TEMPEST_HOME": TEMPEST_HOME,
@ -220,6 +227,23 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
"TEMPEST_OUTPUT": variant.output_path(),
}
def _get_cleanup_env(self) -> Dict[str, str]:
"""Return a dictionary of environment variables.
To be used with tempest resource cleanup functions.
"""
logger.debug("Retrieving OpenStack credentials")
credential = self.user_id_ops.get_user_credential()
return {
"OS_AUTH_URL": credential.get("auth-url"),
"OS_USERNAME": credential.get("username"),
"OS_PASSWORD": credential.get("password"),
"OS_PROJECT_NAME": credential.get("project-name"),
"OS_DOMAIN_ID": credential.get("domain-id"),
"OS_USER_DOMAIN_ID": credential.get("domain-id"),
"OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
}
def get_unit_data(self, key: str) -> Optional[str]:
"""Retrieve a value set for this unit on the peer relation."""
return self.peers.interface.peers_rel.data[self.unit].get(key)
@ -248,6 +272,15 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
# for periodic checks.
env = self._get_environment_for_tempest(TempestEnvVariant.PERIODIC)
pebble = self.pebble_handler()
try:
# do an extensive clean-up before tempest init to remove stalled resources
run_extensive_cleanup(self._get_cleanup_env())
except CleanUpError:
logger.debug("Clean-up failed and tempest init not run.")
self.set_tempest_ready(False)
return
try:
pebble.init_tempest(env)
except RuntimeError:
@ -296,17 +329,16 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
test_list: str = event.params["test-list"].strip()
env = self._get_environment_for_tempest(TempestEnvVariant.ADHOC)
try:
summary = self.pebble_handler().run_tempest_tests(
regexes, exclude_regex, test_list, serial, env
)
except RuntimeError as e:
# put the message in set_results instead of event.fail,
# because event.fail message is not always displayed to the user:
# https://bugs.launchpad.net/juju/+bug/2052765
event.set_results({"error": str(e)})
event.fail()
return
event.set_results(
{
"summary": summary,

View File

@ -36,6 +36,10 @@ import ops.model
import ops.pebble
import ops_sunbeam.container_handlers as sunbeam_chandlers
import ops_sunbeam.relation_handlers as sunbeam_rhandlers
from utils.cleanup import (
CleanUpError,
run_extensive_cleanup,
)
from utils.constants import (
OPENSTACK_DOMAIN,
OPENSTACK_PROJECT,
@ -157,6 +161,16 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
Raise a RuntimeError if something goes wrong.
"""
# push the cleanup script to container
with open("src/utils/cleanup.py") as f:
self.container.push(
f"{TEMPEST_HOME}/cleanup.py",
f,
user="tempest",
group="tempest",
make_dirs=True,
)
# Pebble runs cron, which runs tempest periodically
# when periodic checks are enabled.
# This ensures that tempest gets the env, inherited from cron.
@ -456,6 +470,10 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
def _teardown_tempest_resource_ops(self) -> List[dict]:
"""Tear down openstack resource ops."""
credential_id = self._ensure_credential()
credential_secret = self.model.get_secret(id=credential_id)
content = credential_secret.get_content()
username = content.get("username")
teardown_ops = [
{
"name": "show_domain",
@ -463,6 +481,20 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
"name": OPENSTACK_DOMAIN,
},
},
{
"name": "delete_project",
"params": {
"name": OPENSTACK_PROJECT,
"domain": "{{ show_domain[0].id }}",
},
},
{
"name": "delete_user",
"params": {
"name": username,
"domain": "{{ show_domain[0].id }}",
},
},
{
"name": "update_domain",
"params": {
@ -517,6 +549,19 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
self._set_secret({"auth-url": auth_url})
return
def _process_setup_tempest_resource_response(self, response: dict) -> None:
"""Process extra ops request: "_setup_tempest_resource_request"."""
for op in response.get("ops", []):
if op.get("name") != "create_domain":
continue
if op.get("return-code") != 0:
logger.warning("Create domain ops failed.")
return
domain_id = op.get("value", {}).get("id")
if domain_id is not None:
self._set_secret({"domain-id": domain_id})
return
def _on_provider_ready(self, event) -> None:
"""Handles response available events."""
if not self.model.unit.is_leader():
@ -535,15 +580,30 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
response = self.interface.response
logger.info("%s", json.dumps(response, indent=4))
self._process_list_endpoint_response(response)
self._process_setup_tempest_resource_response(response)
self.callback_f(event)
def _on_provider_goneaway(self, event) -> None:
"""Handle gone_away event."""
if not self.model.unit.is_leader():
return
logger.info(
"Identity ops provider gone away: teardown tempest resources"
)
logger.info("Identity ops provider gone away")
credential = self.get_user_credential()
env = {
"OS_AUTH_URL": credential.get("auth-url"),
"OS_USERNAME": credential.get("username"),
"OS_PASSWORD": credential.get("password"),
"OS_PROJECT_NAME": credential.get("project-name"),
"OS_DOMAIN_ID": credential.get("domain-id"),
"OS_USER_DOMAIN_ID": credential.get("domain-id"),
"OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
}
try:
# do an extensive clean-up upon identity relation removal
run_extensive_cleanup(env)
except CleanUpError as e:
logger.warning("Clean-up failed: %s", str(e))
self.callback_f(event)

View File

@ -15,6 +15,7 @@ echo ":: discover-tempest-config" >> "$TMP_FILE"
if discover-tempest-config --test-accounts "$TEMPEST_TEST_ACCOUNTS" --out "$TEMPEST_CONF" >> "$TMP_FILE" 2>&1; then
echo ":: tempest run" >> "$TMP_FILE"
tempest run --workspace "$TEMPEST_WORKSPACE" "$@" >> "$TMP_FILE" 2>&1
python3 "$TEMPEST_HOME/cleanup.py"
else
echo ":: skipping tempest run because discover-tempest-config had errors" >> "$TMP_FILE"
fi

View File

@ -0,0 +1,317 @@
# Copyright 2024 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utils for cleaning up tempest-related resources."""
import os
from collections.abc import (
Callable,
)
import yaml
from keystoneauth1.exceptions.catalog import (
EndpointNotFound,
)
from keystoneauth1.exceptions.http import (
Unauthorized,
)
from openstack.connection import (
Connection,
)
from openstack.exceptions import (
ForbiddenException,
)
RESOURCE_PREFIX = "tempest-"
class CleanUpError(Exception):
"""Exception raised when clean-up process terminated unsuccessfully."""
def _connect_to_os(env: dict) -> Connection:
"""Establish connection to the OpenStack cloud."""
return Connection(
auth_url=env["OS_AUTH_URL"],
project_name=env["OS_PROJECT_NAME"],
username=env["OS_USERNAME"],
password=env["OS_PASSWORD"],
user_domain_id=env["OS_USER_DOMAIN_ID"],
project_domain_id=env["OS_USER_DOMAIN_ID"],
)
def _get_exclusion_resources(account_file: str) -> dict[str, set[str]]:
"""Get users and projects to be excluded from clean-up.
The resources are looked up in an account_file which is generated by
`tempest account-generator`.
Note that this lookup will only work in workload container as it needs
access to the generated test accounts file.
"""
try:
with open(account_file, "r") as file:
parsed_file = yaml.safe_load(file)
return {
"projects": {account["project_name"] for account in parsed_file},
"users": {account["username"] for account in parsed_file},
}
except FileNotFoundError as e:
raise CleanUpError("Test account file doesn't exist.") from e
except KeyError as e:
raise CleanUpError("Malformated test account file.") from e
def _get_test_projects_in_domain(
conn: Connection, domain_id: str, exclude_projects: set[str] = set()
) -> list[str]:
"""Get test projects in domain to do cleanup for.
Projects in exclude_projects will not be included in the returned list.
"""
return [
project.id
for project in conn.identity.projects(domain_id=domain_id)
if project.name.startswith(RESOURCE_PREFIX)
and project.name not in exclude_projects
]
def _cleanup_compute_resources(conn: Connection, project_id: str) -> None:
"""Delete compute resources with names starting with prefix in the specified project.
The compute resources to be removed are instances and keypairs.
"""
# Delete instances
for server in conn.compute.servers(project_id=project_id):
if server.name.startswith(RESOURCE_PREFIX):
conn.compute.delete_server(server.id)
# Delete keypairs
for keypair in conn.compute.keypairs():
if keypair.name.startswith(RESOURCE_PREFIX):
conn.compute.delete_keypair(keypair)
def _cleanup_block_resources(conn: Connection, project_id: str) -> None:
"""Delete block storage resources with names starting with prefix in the specified project.
The block storage resources to be removed are snapshots and instances.
"""
# Delete snapshots
for snapshot in conn.block_store.snapshots(
details=True, project_id=project_id
):
if snapshot.name.startswith(RESOURCE_PREFIX):
conn.block_store.delete_snapshot(snapshot.id)
# Delete volumes
for volume in conn.block_store.volumes(
details=True, project_id=project_id
):
if volume.name.startswith(RESOURCE_PREFIX):
conn.block_store.delete_volume(volume.id)
def _cleanup_images(conn: Connection, project_id: str) -> None:
"""Delete images with names starting with prefix and owned by the specified project."""
for image in conn.image.images():
# TODO: to be extra careful, we should also check the prefix of the image
# However, some tempest tests are not creating images with the prefix, so
# we should wait until https://review.opendev.org/c/openstack/tempest/+/908358
# is released.
if image.owner == project_id:
conn.image.delete_image(image.id)
def _cleanup_networks_resources(conn: Connection, project_id: str) -> None:
"""Delete network resources with names starting with prefix in the specified project.
The network resources to be removed are ports, routers, and networks.
"""
# Delete ports and routers
for router in conn.network.routers(project_id=project_id):
if router.name.startswith(RESOURCE_PREFIX):
for port in conn.network.ports(device_id=router.id):
conn.network.remove_interface_from_router(
router, port_id=port.id
)
conn.network.delete_router(router.id)
# Delete networks
for network in conn.network.networks(project_id=project_id):
if network.name.startswith(RESOURCE_PREFIX):
conn.network.delete_network(network.id)
def _cleanup_stacks(conn: Connection, project_id: str) -> None:
"""Delete stacks with names starting with prefix and owned by the specified project.
If Heat service is not found in the cloud, this clean-up will be skipped.
"""
try:
for stack in conn.orchestration.stacks(project_id=project_id):
if stack.name.startswith(RESOURCE_PREFIX):
conn.orchestration.delete_stack(stack.id)
except EndpointNotFound:
# do nothing if the heat endpoint is not found
pass
def _cleanup_users(
conn: Connection, domain_id: str, exclude_users: set[str] = set()
) -> None:
"""Delete users with names starting with prefix in the specified domain.
If exclude_users is specified, users in exclude_users will not be removed.
"""
for user in conn.identity.users(domain_id=domain_id):
if (
user.name.startswith(RESOURCE_PREFIX)
and user.name not in exclude_users
):
conn.identity.delete_user(user.id)
def _cleanup_project(conn: Connection, project_id: str) -> None:
"""Delete a project given its id."""
conn.identity.delete_project(project_id)
def _run_cleanup_functions(
conn: Connection, projects: list[str], functions: list[Callable]
) -> list[str]:
"""Run clean-up function on a list of projects."""
failure_message = []
for project_id in projects:
for func in functions:
try:
func(conn, project_id)
except Exception as e:
failure_message.append(f"Error calling {func.__name__}: {e}")
return failure_message
def run_quick_cleanup(env: dict) -> None:
"""Perform the quick cleanup of tempest resources under a specific domain.
This clean-up removes compute instances, keypairs, volumes, snapshots,
images, networks, projects, users, and stacks that are not associated with
the pre-generated test accounts and projects defined in account_file.
Note that this clean up will only work in workload container as it needs
access to the generated test accounts file.
"""
conn = _connect_to_os(env)
cleanup_funcs = [
_cleanup_compute_resources,
_cleanup_block_resources,
_cleanup_images,
_cleanup_stacks,
]
filtered_cleanup_funcs = [
_cleanup_networks_resources,
_cleanup_project,
]
exclude_resources = _get_exclusion_resources(env["TEMPEST_TEST_ACCOUNTS"])
failure_message = []
try:
# get all projects with prefix in domain
test_projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"])
# get projects that are not found in account file
filtered_test_projects = _get_test_projects_in_domain(
conn, env["OS_DOMAIN_ID"], exclude_resources["projects"]
)
except (ForbiddenException, Unauthorized) as e:
raise CleanUpError("Operation not authorized.") from e
failure_message.extend(
_run_cleanup_functions(conn, test_projects, cleanup_funcs)
)
failure_message.extend(
_run_cleanup_functions(
conn, filtered_test_projects, filtered_cleanup_funcs
)
)
try:
_cleanup_users(conn, env["OS_DOMAIN_ID"], exclude_resources["users"])
except Exception as e:
failure_message.append(f"\nError cleaning up users: {e}")
if failure_message:
raise CleanUpError("\n".join(failure_message))
def run_extensive_cleanup(env: dict) -> None:
"""Perform the extensive cleanup of tempest resources under a specific domain.
This clean-up removes compute instances, keypairs, volumes, snapshots,
images, and stacks, as well as generated test accounts, projects, and the
network resources associated with them.
"""
conn = _connect_to_os(env)
cleanup_funcs = [
_cleanup_compute_resources,
_cleanup_block_resources,
_cleanup_images,
_cleanup_stacks,
_cleanup_networks_resources,
_cleanup_project,
]
failure_message = []
try:
# get all projects with prefix in domain
projects = _get_test_projects_in_domain(conn, env["OS_DOMAIN_ID"])
except (ForbiddenException, Unauthorized) as e:
raise CleanUpError("Operation not authorized.") from e
failure_message.extend(
_run_cleanup_functions(conn, projects, cleanup_funcs)
)
try:
_cleanup_users(conn, env["OS_DOMAIN_ID"])
except Exception as e:
failure_message.append(f"\nError cleaning up users: {e}")
if failure_message:
raise CleanUpError("\n".join(failure_message))
def main() -> None:
"""Entrypoint for executing the script directly.
This will be used in periodic test runs.
Quick cleanup will be performed.
"""
env = {
"OS_AUTH_URL": os.getenv("OS_AUTH_URL", ""),
"OS_USERNAME": os.getenv("OS_USERNAME", ""),
"OS_PASSWORD": os.getenv("OS_PASSWORD", ""),
"OS_PROJECT_NAME": os.getenv("OS_PROJECT_NAME", ""),
"OS_DOMAIN_ID": os.getenv("OS_DOMAIN_ID", ""),
"OS_USER_DOMAIN_ID": os.getenv("OS_USER_DOMAIN_ID", ""),
"OS_PROJECT_DOMAIN_ID": os.getenv("OS_PROJECT_DOMAIN_ID", ""),
"TEMPEST_TEST_ACCOUNTS": os.getenv("TEMPEST_TEST_ACCOUNTS", ""),
}
run_quick_cleanup(env)
if __name__ == "__main__":
main()

View File

@ -29,7 +29,8 @@ TEMPEST_WORKSPACE = "tempest"
OPENSTACK_USER = "tempest"
OPENSTACK_DOMAIN = "tempest"
OPENSTACK_PROJECT = "tempest-CloudValidation"
# not use tempest as prefix to exclude this project from utils/cleanup.py scope
OPENSTACK_PROJECT = "CloudValidation-tempest"
OPENSTACK_ROLE = "admin"
# keys for application data

View File

@ -0,0 +1,552 @@
# Copyright 2024 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for tempest-related resources cleanup."""
import textwrap
import unittest
from unittest.mock import (
MagicMock,
call,
mock_open,
patch,
)
from keystoneauth1.exceptions.catalog import (
EndpointNotFound,
)
from keystoneauth1.exceptions.http import (
Unauthorized,
)
from openstack.exceptions import (
ForbiddenException,
)
from utils.cleanup import (
CleanUpError,
_cleanup_block_resources,
_cleanup_compute_resources,
_cleanup_images,
_cleanup_networks_resources,
_cleanup_project,
_cleanup_stacks,
_cleanup_users,
_connect_to_os,
_get_exclusion_resources,
_get_test_projects_in_domain,
_run_cleanup_functions,
run_extensive_cleanup,
run_quick_cleanup,
)
class TestCleanup(unittest.TestCase):
"""Test tempest resources clean-up."""
def setUp(self):
"""Set up test construction."""
self.patcher = patch("utils.cleanup.Connection")
self.mock_connection = self.patcher.start()
def tearDown(self):
"""Tear down test construction."""
self.patcher.stop()
def test_connect_to_os(self):
"""Test establishing OS connection."""
env = {
"OS_AUTH_URL": "http://10.6.0.20/openstack-keystone",
"OS_USERNAME": "test_user",
"OS_PASSWORD": "userpass",
"OS_PROJECT_NAME": "test_project",
"OS_DOMAIN_ID": "domain_id",
"OS_USER_DOMAIN_ID": "domain_id",
"OS_PROJECT_DOMAIN_ID": "domain_id",
}
_connect_to_os(env)
self.mock_connection.assert_called_once_with(
auth_url=env["OS_AUTH_URL"],
project_name=env["OS_PROJECT_NAME"],
username=env["OS_USERNAME"],
password=env["OS_PASSWORD"],
user_domain_id=env["OS_USER_DOMAIN_ID"],
project_domain_id=env["OS_USER_DOMAIN_ID"],
)
def test_get_exclude_resources(self):
"""Test get exclude resources from test accounts file."""
account_file_content = textwrap.dedent(
"""
- domain_name: mydomain
password: password1
project_name: tempest-test_creds-11949114
resources:
network: tempest-test_creds-9369097-network
username: tempest-test_creds-11949114-project
- domain_name: mydomain
password: password2
project_name: tempest-test_creds-18083146
resources:
network: tempest-test_creds-20041716-network
username: tempest-test_creds-18083146-project
"""
)
expected_result = {
"projects": {
"tempest-test_creds-11949114",
"tempest-test_creds-18083146",
},
"users": {
"tempest-test_creds-11949114-project",
"tempest-test_creds-18083146-project",
},
}
with patch(
"utils.cleanup.open",
new_callable=mock_open,
read_data=account_file_content,
):
result = _get_exclusion_resources("test_account_file.yaml")
self.assertEqual(result, expected_result)
def test_get_test_projects_in_domain(self):
"""Test get tempest projects of a specified domain."""
tempest_project1 = MagicMock()
tempest_project1.configure_mock(id="1", name="tempest-project-1")
tempest_project2 = MagicMock()
tempest_project2.configure_mock(id="2", name="tempest-project-2")
non_tempest_project = MagicMock()
non_tempest_project.configure_mock(id="3", name="non-tempest-project")
self.mock_connection.identity.projects.return_value = [
tempest_project1,
tempest_project2,
non_tempest_project,
]
projects = _get_test_projects_in_domain(
self.mock_connection, "tempest_domain"
)
self.assertEqual(projects, ["1", "2"])
def test_get_test_projects_in_domain_with_exclusion(self):
"""Test get tempest projects of a specified domain with exclusion."""
tempest_project1 = MagicMock()
tempest_project1.configure_mock(id="1", name="tempest-project-1")
tempest_project2 = MagicMock()
tempest_project2.configure_mock(id="2", name="tempest-project-2")
non_tempest_project = MagicMock()
non_tempest_project.configure_mock(id="3", name="non-tempest-project")
self.mock_connection.identity.projects.return_value = [
tempest_project1,
tempest_project2,
non_tempest_project,
]
projects = _get_test_projects_in_domain(
self.mock_connection,
domain_id="tempest_domain",
exclude_projects={"tempest-project-1"},
)
self.assertEqual(projects, ["2"])
def test_cleanup_block_resources(self):
"""Test cleanup volumes and snapshots."""
tempest_snapshots = MagicMock()
tempest_snapshots.configure_mock(id="1", name="tempest-snapshots-1")
non_tempest_snapshots = MagicMock()
non_tempest_snapshots.configure_mock(
id="2", name="non-tempest-snapshots"
)
self.mock_connection.block_store.snapshots.return_value = [
tempest_snapshots,
non_tempest_snapshots,
]
tempest_volume = MagicMock()
tempest_volume.configure_mock(id="1", name="tempest-volume-1")
non_tempest_volume = MagicMock()
non_tempest_volume.configure_mock(id="2", name="non-tempest-volume")
self.mock_connection.block_store.volumes.return_value = [
tempest_volume,
non_tempest_volume,
]
_cleanup_block_resources(self.mock_connection, "test_project")
self.mock_connection.block_store.delete_snapshot.assert_called_once_with(
"1"
)
self.mock_connection.block_store.delete_volume.assert_called_once_with(
"1"
)
def test_cleanup_networks_resources(self):
"""Test cleanup network resources."""
tempest_router = MagicMock()
tempest_router.configure_mock(id="1", name="tempest-router")
non_tempest_router = MagicMock()
non_tempest_router.configure_mock(id="2", name="non-tempest-router")
self.mock_connection.network.routers.return_value = [
tempest_router,
non_tempest_router,
]
tempest_port1 = MagicMock()
tempest_port1.configure_mock(id="1")
tempest_port2 = MagicMock()
tempest_port2.configure_mock(id="2")
self.mock_connection.network.ports.return_value = [
tempest_port1,
tempest_port2,
]
tempest_network = MagicMock()
tempest_network.configure_mock(id="1", name="tempest-network")
non_tempest_network = MagicMock()
non_tempest_network.configure_mock(id="2", name="non-tempest-network")
self.mock_connection.network.networks.return_value = [
tempest_network,
non_tempest_network,
]
_cleanup_networks_resources(self.mock_connection, "project_id")
self.mock_connection.network.remove_interface_from_router.call_args_list == [
call(tempest_router, port_id="1"),
call(tempest_router, port_id="2"),
]
self.mock_connection.network.delete_router.assert_called_once_with("1")
self.mock_connection.network.delete_network.assert_called_once_with(
"1"
)
def test_cleanup_compute_resources(self):
"""Test cleanup instances and keypairs."""
tempest_instance = MagicMock()
tempest_instance.configure_mock(id="1", name="tempest-server-1")
non_tempest_instance = MagicMock()
non_tempest_instance.configure_mock(id="2", name="non-tempest-server")
self.mock_connection.compute.servers.return_value = [
tempest_instance,
non_tempest_instance,
]
tempest_keypair = MagicMock()
tempest_keypair.configure_mock(id="1", name="tempest-keypair-1")
non_tempest_keypair = MagicMock()
non_tempest_keypair.configure_mock(
id="2", name="non-tempest-keypair-2"
)
self.mock_connection.compute.keypairs.return_value = [
tempest_keypair,
non_tempest_keypair,
]
_cleanup_compute_resources(self.mock_connection, "test_project")
self.mock_connection.compute.delete_server.assert_called_once_with("1")
self.mock_connection.compute.delete_keypair.assert_called_once_with(
tempest_keypair
)
def test_cleanup_images(self):
"""Test cleanup images."""
image_owned_by_project = MagicMock()
image_owned_by_project.configure_mock(
id="1", name="image-1", owner="test_project"
)
image_not_owned_by_project = MagicMock()
image_not_owned_by_project.configure_mock(
id="3", name="image-3", owner="not_test_project"
)
self.mock_connection.image.images.return_value = [
image_owned_by_project,
image_not_owned_by_project,
]
_cleanup_images(self.mock_connection, "test_project")
self.mock_connection.image.delete_image.assert_called_once_with("1")
def test_cleanup_project(self):
"""Test cleanup projects."""
project_id = "tempest_project_id"
_cleanup_project(self.mock_connection, project_id)
self.mock_connection.identity.delete_project.assert_called_with(
project_id
)
def test_cleanup_users(self):
"""Test cleanup users."""
tempest_user = MagicMock()
tempest_user.configure_mock(
domain_id="tempest", id="1", name="tempest-user-1"
)
non_tempest_user = MagicMock()
non_tempest_user.configure_mock(
domain_id="tempest", id="2", name="non-tempest-user-2"
)
self.mock_connection.identity.users.return_value = [
tempest_user,
non_tempest_user,
]
_cleanup_users(self.mock_connection, domain_id="tempest")
self.mock_connection.identity.delete_user.assert_called_once_with("1")
def test_cleanup_users_with_exclusion(self):
"""Test cleanup users with exclusion."""
tempest_user1 = MagicMock()
tempest_user1.configure_mock(
domain_id="tempest", id="1", name="tempest-user-1"
)
tempest_user2 = MagicMock()
tempest_user2.configure_mock(
domain_id="tempest", id="2", name="tempest-user-2"
)
non_tempest_user = MagicMock()
non_tempest_user.configure_mock(
domain_id="tempest", id="3", name="non-tempest-user"
)
self.mock_connection.identity.users.return_value = [
tempest_user1,
tempest_user2,
non_tempest_user,
]
_cleanup_users(
self.mock_connection,
domain_id="tempest",
exclude_users={"tempest-user-1"},
)
self.mock_connection.identity.delete_user.assert_called_once_with("2")
def test_cleanup_stacks_success(self):
"""Test cleanup heat stacks."""
tempest_stack = MagicMock()
tempest_stack.configure_mock(id="1", name="tempest-stack-1")
non_tempest_stack = MagicMock()
non_tempest_stack.configure_mock(id="2", name="non-tempest-stack-2")
self.mock_connection.orchestration.stacks.return_value = [
tempest_stack,
non_tempest_stack,
]
_cleanup_stacks(self.mock_connection, "test_project")
self.mock_connection.orchestration.delete_stack.assert_called_once_with(
"1"
)
def test_cleanup_stacks_endpoint_not_found(self):
"""Test cleanup stacks when heat endpoint is not found."""
self.mock_connection.orchestration.stacks.side_effect = (
EndpointNotFound
)
_cleanup_stacks(self.mock_connection, "test_project")
self.mock_connection.orchestration.delete_stack.assert_not_called()
@patch("utils.cleanup._cleanup_compute_resources")
@patch("utils.cleanup._cleanup_block_resources")
@patch("utils.cleanup._cleanup_images")
@patch("utils.cleanup._cleanup_stacks")
@patch("utils.cleanup._cleanup_networks_resources")
@patch("utils.cleanup._cleanup_project")
def test_run_cleanup_functions(
self,
mock_cleanup_project,
mock_cleanup_networks_resources,
mock_cleanup_stacks,
mock_cleanup_images,
mock_cleanup_block_resources,
mock_cleanup_compute_resources,
):
"""Test run cleanup functions."""
cleanup_funcs = [
mock_cleanup_compute_resources,
mock_cleanup_block_resources,
mock_cleanup_images,
mock_cleanup_stacks,
mock_cleanup_networks_resources,
mock_cleanup_project,
]
projects = ["tempest_project_id"]
_run_cleanup_functions(self.mock_connection, projects, cleanup_funcs)
mock_cleanup_stacks.assert_called_once()
mock_cleanup_images.assert_called_once()
mock_cleanup_block_resources.assert_called_once()
mock_cleanup_compute_resources.assert_called_once()
mock_cleanup_networks_resources.assert_called_once()
mock_cleanup_project.assert_called_once()
@patch("utils.cleanup._cleanup_compute_resources")
@patch("utils.cleanup._cleanup_block_resources")
@patch("utils.cleanup._cleanup_images")
@patch("utils.cleanup._cleanup_stacks")
@patch("utils.cleanup._cleanup_networks_resources")
@patch("utils.cleanup._cleanup_project")
def test_run_cleanup_functions_unsuccessful(
self,
mock_cleanup_project,
mock_cleanup_networks_resources,
mock_cleanup_stacks,
mock_cleanup_images,
mock_cleanup_block_resources,
mock_cleanup_compute_resources,
):
"""Test run cleanup functions."""
cleanup_funcs = [
mock_cleanup_compute_resources,
mock_cleanup_block_resources,
mock_cleanup_images,
mock_cleanup_stacks,
mock_cleanup_networks_resources,
mock_cleanup_project,
]
projects = ["tempest_project_id"]
mock_cleanup_images.__name__ = "_cleanup_images"
mock_cleanup_images.side_effect = Exception
_run_cleanup_functions(self.mock_connection, projects, cleanup_funcs)
mock_cleanup_stacks.assert_called_once()
mock_cleanup_images.assert_called_once()
mock_cleanup_block_resources.assert_called_once()
mock_cleanup_compute_resources.assert_called_once()
mock_cleanup_networks_resources.assert_called_once()
mock_cleanup_project.assert_called_once()
@patch("utils.cleanup._run_cleanup_functions")
@patch("utils.cleanup._cleanup_users")
@patch("utils.cleanup._connect_to_os")
@patch("utils.cleanup._get_test_projects_in_domain")
@patch("utils.cleanup._get_exclusion_resources")
def test_run_quick_cleanup(
self,
mock_get_exclusion_resources,
mock_get_test_projects_in_domain,
mock_connect_to_os,
mock_cleanup_users,
mock_run_cleanup_functions,
):
"""Test run quick cleanup."""
env = MagicMock()
mock_get_test_projects_in_domain.side_effect = [
["tempest_project_id"],
["tempest_project_id"],
]
run_quick_cleanup(env)
mock_connect_to_os.assert_called_once_with(env)
mock_get_exclusion_resources.assert_called_once()
mock_get_test_projects_in_domain.assert_called()
mock_run_cleanup_functions.assert_called()
mock_cleanup_users.assert_called_once()
@patch("utils.cleanup._run_cleanup_functions")
@patch("utils.cleanup._cleanup_users")
@patch("utils.cleanup._connect_to_os")
@patch("utils.cleanup._get_test_projects_in_domain")
@patch("utils.cleanup._get_exclusion_resources")
def test_run_quick_cleanup_error(
self,
mock_get_exclusion_resources,
mock_get_test_projects_in_domain,
mock_connect_to_os,
mock_cleanup_users,
mock_run_cleanup_functions,
):
"""Test run quick cleanup with failure."""
env = MagicMock()
mock_get_test_projects_in_domain.side_effect = ForbiddenException
with self.assertRaises(CleanUpError) as error:
run_quick_cleanup(env)
mock_connect_to_os.assert_called_once_with(env)
mock_get_exclusion_resources.assert_called_once()
mock_get_test_projects_in_domain.assert_called_once()
self.assertEqual(str(error.exception), "Operation not authorized.")
mock_run_cleanup_functions.assert_not_called()
mock_cleanup_users.assert_not_called()
@patch("utils.cleanup._run_cleanup_functions")
@patch("utils.cleanup._cleanup_users")
@patch("utils.cleanup._connect_to_os")
@patch("utils.cleanup._get_test_projects_in_domain")
@patch("utils.cleanup._get_exclusion_resources")
def test_run_extensive_cleanup(
self,
mock_get_exclusion_resources,
mock_get_test_projects_in_domain,
mock_connect_to_os,
mock_cleanup_users,
mock_run_cleanup_functions,
):
"""Test run extensive cleanup."""
env = MagicMock()
mock_get_test_projects_in_domain.side_effect = [
["tempest_project_id"],
["tempest_project_id"],
]
run_extensive_cleanup(env)
mock_connect_to_os.assert_called_once_with(env)
mock_get_test_projects_in_domain.assert_called()
mock_run_cleanup_functions.assert_called_once()
mock_cleanup_users.assert_called_once()
@patch("utils.cleanup._run_cleanup_functions")
@patch("utils.cleanup._cleanup_users")
@patch("utils.cleanup._connect_to_os")
@patch("utils.cleanup._get_test_projects_in_domain")
@patch("utils.cleanup._get_exclusion_resources")
def test_run_extensive_cleanup_permission_error(
self,
mock_get_exclusion_resources,
mock_get_test_projects_in_domain,
mock_connect_to_os,
mock_cleanup_users,
mock_run_cleanup_functions,
):
"""Test run extensive cleanup with failure."""
env = MagicMock()
mock_get_test_projects_in_domain.side_effect = Unauthorized
with self.assertRaises(CleanUpError) as error:
run_extensive_cleanup(env)
mock_connect_to_os.assert_called_once_with(env)
mock_get_test_projects_in_domain.assert_called_once()
self.assertEqual(str(error.exception), "Operation not authorized.")
mock_run_cleanup_functions.assert_not_called()
mock_cleanup_users.assert_not_called()

View File

@ -18,6 +18,9 @@
import json
import pathlib
from unittest.mock import (
patch,
)
import charm
import mock
@ -42,9 +45,12 @@ TEST_TEMPEST_ENV = {
"OS_USERNAME": "tempest",
"OS_PASSWORD": "password",
"OS_USER_DOMAIN_NAME": "tempest",
"OS_PROJECT_NAME": "tempest-CloudValidation",
"OS_PROJECT_NAME": "CloudValidation-tempest",
"OS_PROJECT_DOMAIN_NAME": "tempest",
"OS_DOMAIN_NAME": "tempest",
"OS_PROJECT_DOMAIN_ID": "tempest-domain-id",
"OS_USER_DOMAIN_ID": "tempest-domain-id",
"OS_DOMAIN_ID": "tempest-domain-id",
"TEMPEST_CONCURRENCY": "4",
"TEMPEST_CONF": "/var/lib/tempest/workspace/etc/tempest.conf",
"TEMPEST_HOME": "/var/lib/tempest",
@ -97,6 +103,18 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase):
self.addCleanup(self.harness.cleanup)
self.harness.begin()
self.harness.set_leader()
self.patcher1 = patch("utils.cleanup.Connection")
self.patcher2 = patch(
"utils.cleanup._get_exclusion_resources",
return_value={"projects": set(), "users": set()},
)
self.patcher1.start()
self.patcher2.start()
def tearDown(self):
"""Tear down test construction."""
self.patcher1.stop()
self.patcher2.stop()
def add_identity_ops_relation(self, harness):
"""Add identity resource relation."""
@ -108,7 +126,8 @@ class TestTempestOperatorCharm(test_utils.CharmTestCase):
"username": "tempest",
"password": "password",
"domain-name": "tempest",
"project-name": "tempest-CloudValidation",
"domain-id": "tempest-domain-id",
"project-name": "CloudValidation-tempest",
"auth-url": "http://10.6.0.23/openstack-keystone/v3",
},
)

View File

@ -18,3 +18,4 @@ croniter # tempest-k8s
git+https://github.com/juju/charm-helpers.git#egg=charmhelpers # cinder-ceph-k8s,glance-k8s,gnocchi-k8s
git+https://opendev.org/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client # cinder-ceph-k8s
requests-unixsocket # sunbeam-clusterd
openstacksdk # tempest-k8s