[chore] General tidy, unit tests

This commit is contained in:
James Page 2021-10-05 09:39:32 +01:00
parent f4fe94d0d8
commit 83370af83c
13 changed files with 635 additions and 99 deletions

View File

@ -5,3 +5,5 @@ build/
.coverage
__pycache__/
*.py[cod]
.tox
.stestr

View File

@ -0,0 +1,3 @@
[DEFAULT]
test_path=./unit_tests
top_dir=./

View File

@ -0,0 +1,2 @@
mysql lib does not seem to be published
so as a tectical solution keep it in tree

View File

@ -0,0 +1,105 @@
"""
## Overview
This document explains how to integrate with the MySQL charm for the purposes of consuming a mysql database. It also explains how alternative implementations of the MySQL charm may maintain the same interface and be backward compatible with all currently integrated charms. Finally this document is the authoritative reference on the structure of relation data that is shared between MySQL charms and any other charm that intends to use the database.
## Consumer Library Usage
The MySQL charm library uses the [Provider and Consumer](https://ops.readthedocs.io/en/latest/#module-ops.relation) objects from the Operator Framework. Charms that would like to use a MySQL database must use the `MySQLConsumer` object from the charm library. Using the `MySQLConsumer` object requires instantiating it, typically in the constructor of your charm. The `MySQLConsumer` constructor requires the name of the relation over which a database will be used. This relation must use the `mysql_datastore` interface. In addition the constructor also requires a `consumes` specification, which is a dictionary with key `mysql` (also see Provider Library Usage below) and a value that represents the minimum acceptable version of MySQL. This version string can be in any format that is compatible with the Python [Semantic Version module](https://pypi.org/project/semantic-version/). For example, assuming your charm consumes a database over a rlation named "monitoring", you may instantiate `MySQLConsumer` as follows:
from charms.mysql_k8s.v0.mysql import MySQLConsumer
def __init__(self, *args):
super().__init__(*args)
...
self.mysql_consumer = MySQLConsumer(
self, "monitoring", {"mysql": ">=8"}
)
...
This example hard codes the consumes dictionary argument containing the minimal MySQL version required, however you may want to consider generating this dictionary by some other means, such as a `self.consumes` property in your charm. This is because the minimum required MySQL version may change when you upgrade your charm. Of course it is expected that you will keep this version string updated as you develop newer releases of your charm. If the version string can be determined at run time by inspecting the actual deployed version of your charmed application, this would be ideal.
An instantiated `MySQLConsumer` object may be used to request new databases using the `new_database()` method. This method requires no arguments unless you require multiple databases. If multiple databases are requested, you must provide a unique `name_suffix` argument. For example:
def _on_database_relation_joined(self, event):
self.mysql_consumer.new_database(name_suffix="db1")
self.mysql_consumer.new_database(name_suffix="db2")
The `address`, `port`, `databases`, and `credentials` methods can all be called
to get the relevant information from the relation data.
"""
# !/usr/bin/env python3
# Copyright 2021 Canonical Ltd.
# See LICENSE file for licensing details.
import json
import uuid
import logging
from ops.relation import ConsumerBase
LIBID = "abcdef1234" # Will change when uploding the charm to charmhub
LIBAPI = 1
LIBPATCH = 0
logger = logging.getLogger(__name__)
class MySQLConsumer(ConsumerBase):
"""
MySQLConsumer lib class
"""
def __init__(self, charm, name, consumes, multi=False):
super().__init__(charm, name, consumes, multi)
self.charm = charm
self.relation_name = name
def databases(self, rel_id=None) -> list:
"""
List of currently available databases
Returns:
list: list of database names
"""
rel = self.framework.model.get_relation(self.relation_name, rel_id)
relation_data = rel.data[rel.app]
dbs = relation_data.get("databases")
databases = json.loads(dbs) if dbs else []
return databases
def credentials(self, rel_id=None) -> dict:
"""
Dictionary of credential information to access databases
Returns:
dict: dictionary of credential information including username,
password and address
"""
rel = self.framework.model.get_relation(self.relation_name, rel_id)
relation_data = rel.data[rel.app]
data = relation_data.get("data")
data = json.loads(data) if data else {}
credentials = data.get("credentials")
return credentials
def new_database(self, rel_id=None, name_suffix=""):
"""
Request creation of an additional database
"""
if not self.charm.unit.is_leader():
return
rel = self.framework.model.get_relation(self.relation_name, rel_id)
if name_suffix:
name_suffix = "_{}".format(name_suffix)
rid = str(uuid.uuid4()).split("-")[-1]
db_name = "db_{}_{}_{}".format(rel.id, rid, name_suffix)
logger.debug("CLIENT REQUEST %s", db_name)
rel_data = rel.data[self.charm.app]
dbs = rel_data.get("databases")
dbs = json.loads(dbs) if dbs else []
dbs.append(db_name)
rel.data[self.charm.app]["databases"] = json.dumps(dbs)

View File

@ -0,0 +1,211 @@
"""Library for the ingress relation.
This library contains the Requires and Provides classes for handling
the ingress interface.
Import `IngressRequires` in your charm, with two required options:
- "self" (the charm itself)
- config_dict
`config_dict` accepts the following keys:
- service-hostname (required)
- service-name (required)
- service-port (required)
- additional-hostnames
- limit-rps
- limit-whitelist
- max-body-size
- path-routes
- retry-errors
- rewrite-enabled
- rewrite-target
- service-namespace
- session-cookie-max-age
- tls-secret-name
See [the config section](https://charmhub.io/nginx-ingress-integrator/configure) for descriptions
of each, along with the required type.
As an example, add the following to `src/charm.py`:
```
from charms.nginx_ingress_integrator.v0.ingress import IngressRequires
# In your charm's `__init__` method.
self.ingress = IngressRequires(self, {"service-hostname": self.config["external_hostname"],
"service-name": self.app.name,
"service-port": 80})
# In your charm's `config-changed` handler.
self.ingress.update_config({"service-hostname": self.config["external_hostname"]})
```
And then add the following to `metadata.yaml`:
```
requires:
ingress:
interface: ingress
```
You _must_ register the IngressRequires class as part of the `__init__` method
rather than, for instance, a config-changed event handler. This is because
doing so won't get the current relation changed event, because it wasn't
registered to handle the event (because it wasn't created in `__init__` when
the event was fired).
"""
import logging
from ops.charm import CharmEvents
from ops.framework import EventBase, EventSource, Object
from ops.model import BlockedStatus
# The unique Charmhub library identifier, never change it
LIBID = "db0af4367506491c91663468fb5caa4c"
# Increment this major API version when introducing breaking changes
LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 9
logger = logging.getLogger(__name__)
REQUIRED_INGRESS_RELATION_FIELDS = {
"service-hostname",
"service-name",
"service-port",
}
OPTIONAL_INGRESS_RELATION_FIELDS = {
"additional-hostnames",
"limit-rps",
"limit-whitelist",
"max-body-size",
"retry-errors",
"rewrite-target",
"rewrite-enabled",
"service-namespace",
"session-cookie-max-age",
"tls-secret-name",
"path-routes",
}
class IngressAvailableEvent(EventBase):
pass
class IngressCharmEvents(CharmEvents):
"""Custom charm events."""
ingress_available = EventSource(IngressAvailableEvent)
class IngressRequires(Object):
"""This class defines the functionality for the 'requires' side of the 'ingress' relation.
Hook events observed:
- relation-changed
"""
def __init__(self, charm, config_dict):
super().__init__(charm, "ingress")
self.framework.observe(charm.on["ingress"].relation_changed, self._on_relation_changed)
self.config_dict = config_dict
def _config_dict_errors(self, update_only=False):
"""Check our config dict for errors."""
blocked_message = "Error in ingress relation, check `juju debug-log`"
unknown = [
x
for x in self.config_dict
if x not in REQUIRED_INGRESS_RELATION_FIELDS | OPTIONAL_INGRESS_RELATION_FIELDS
]
if unknown:
logger.error(
"Ingress relation error, unknown key(s) in config dictionary found: %s",
", ".join(unknown),
)
self.model.unit.status = BlockedStatus(blocked_message)
return True
if not update_only:
missing = [x for x in REQUIRED_INGRESS_RELATION_FIELDS if x not in self.config_dict]
if missing:
logger.error(
"Ingress relation error, missing required key(s) in config dictionary: %s",
", ".join(missing),
)
self.model.unit.status = BlockedStatus(blocked_message)
return True
return False
def _on_relation_changed(self, event):
"""Handle the relation-changed event."""
# `self.unit` isn't available here, so use `self.model.unit`.
if self.model.unit.is_leader():
if self._config_dict_errors():
return
for key in self.config_dict:
event.relation.data[self.model.app][key] = str(self.config_dict[key])
def update_config(self, config_dict):
"""Allow for updates to relation."""
if self.model.unit.is_leader():
self.config_dict = config_dict
if self._config_dict_errors(update_only=True):
return
relation = self.model.get_relation("ingress")
if relation:
for key in self.config_dict:
relation.data[self.model.app][key] = str(self.config_dict[key])
class IngressProvides(Object):
"""This class defines the functionality for the 'provides' side of the 'ingress' relation.
Hook events observed:
- relation-changed
"""
def __init__(self, charm):
super().__init__(charm, "ingress")
# Observe the relation-changed hook event and bind
# self.on_relation_changed() to handle the event.
self.framework.observe(charm.on["ingress"].relation_changed, self._on_relation_changed)
self.charm = charm
def _on_relation_changed(self, event):
"""Handle a change to the ingress relation.
Confirm we have the fields we expect to receive."""
# `self.unit` isn't available here, so use `self.model.unit`.
if not self.model.unit.is_leader():
return
ingress_data = {
field: event.relation.data[event.app].get(field)
for field in REQUIRED_INGRESS_RELATION_FIELDS | OPTIONAL_INGRESS_RELATION_FIELDS
}
missing_fields = sorted(
[
field
for field in REQUIRED_INGRESS_RELATION_FIELDS
if ingress_data.get(field) is None
]
)
if missing_fields:
logger.error(
"Missing required data fields for ingress relation: {}".format(
", ".join(missing_fields)
)
)
self.model.unit.status = BlockedStatus(
"Missing fields for ingress: {}".format(", ".join(missing_fields))
)
# Create an event that our charm can use to decide it's okay to
# configure the ingress.
self.charm.on.ingress_available.emit()

View File

@ -75,7 +75,7 @@ LIBAPI = 0
# Increment this PATCH version before using `charmcraft publish-lib` or reset
# to 0 if you are raising the major API version
LIBPATCH = 2
LIBPATCH = 3
import logging
import requests
@ -112,6 +112,7 @@ class AMQPGoneAwayEvent(EventBase):
pass
class AMQPServerEvents(ObjectEvents):
"""Events class for `on`"""
@ -183,6 +184,16 @@ class AMQPRequires(Object):
"""Return the hostname from the AMQP relation"""
return self._amqp_rel.data[self._amqp_rel.app].get("hostname")
@property
def ssl_port(self) -> str:
"""Return the SSL port from the AMQP relation"""
return self._amqp_rel.data[self._amqp_rel.app].get("ssl_port")
@property
def ssl_ca(self) -> str:
"""Return the SSL port from the AMQP relation"""
return self._amqp_rel.data[self._amqp_rel.app].get("ssl_ca")
@property
def hostnames(self) -> List[str]:
"""Return a list of remote RMQ hosts from the AMQP relation"""

View File

@ -1,18 +1,25 @@
#!/usr/bin/env python3
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Cinder Ceph Operator Charm.
This charm provide Cinder <-> Ceph integration as part of an OpenStack deployment
This charm provide Cinder <-> Ceph integration as part
of an OpenStack deployment
"""
import logging
from ops.charm import CharmBase
from ops.framework import StoredState
from ops.charm import Object
from ops.main import main
from ops.model import ActiveStatus, BlockedStatus, WaitingStatus
from ops.model import ActiveStatus
from charms.sunbeam_rabbitmq_operator.v0.amqp import AMQPRequires
from charms.ceph.v0.ceph_client import CephClientRequires
from typing import List
@ -22,64 +29,98 @@ import advanced_sunbeam_openstack.adapters as adapters
logger = logging.getLogger(__name__)
CINDER_VOLUME_CONTAINER = 'cinder-volume'
class CinderCephAdapters(adapters.OPSRelationAdapters):
@property
def interface_map(self):
_map = super().interface_map
_map.update({
'rabbitmq': adapters.AMQPAdapter})
_map.update({"rabbitmq": adapters.AMQPAdapter})
return _map
class AMQPHandler(core.RelationHandler):
def setup_event_handler(self) -> Object:
"""Configure event handlers for an AMQP relation."""
logger.debug("Setting up AMQP event handler")
amqp = AMQPRequires(
self.charm, self.relation_name, "cinder", "openstack"
)
self.framework.observe(amqp.on.ready, self._on_amqp_ready)
return amqp
def _on_amqp_ready(self, event) -> None:
"""Handles AMQP change events."""
# Ready is only emitted when the interface considers
# that the relation is complete (indicated by a password)
self.callback_f(event)
@property
def ready(self) -> bool:
"""Handler ready for use."""
try:
return bool(self.interface.password)
except AttributeError:
return False
class CinderCephOperatorCharm(core.OSBaseOperatorCharm):
"""Cinder/Ceph Operator charm"""
# NOTE: service_name == container_name
service_name = 'cinder-volume'
service_name = "cinder-volume"
service_user = 'cinder'
service_group = 'cinder'
service_user = "cinder"
service_group = "cinder"
cinder_conf = '/etc/cinder/cinder.conf'
cinder_conf = "/etc/cinder/cinder.conf"
def __init__(self, framework):
super().__init__(
framework,
adapters=CinderCephAdapters(self)
)
super().__init__(framework, adapters=CinderCephAdapters(self))
def get_relation_handlers(self) -> List[core.RelationHandler]:
"""Relation handlers for the service."""
self.amqp = core.AMQPHandler(
self, "amqp", self.configure_charm
)
# TODO: add ceph once we've written a handler class
self.amqp = AMQPHandler(self, "amqp", self.configure_charm)
return [self.amqp]
@property
def container_configs(self) -> List[core.ContainerConfigFile]:
_cconfigs = super().container_configs
_cconfigs.extend([
_cconfigs.extend(
[
core.ContainerConfigFile(
[self.service_name],
self.cinder_conf,
self.service_user,
self.service_group
self.service_group,
)
])
]
)
return _cconfigs
def _do_bootstrap(self):
"""No-op the bootstrap method as none required"""
pass
def configure_charm(self, event) -> None:
"""Catchall handler to cconfigure charm services."""
if not self.relation_handlers_ready():
logging.debug(
"Defering configuration, charm relations not ready"
)
return
for ph in self.pebble_handlers:
if ph.pebble_ready:
ph.init_service()
for ph in self.pebble_handlers:
if not ph.service_ready:
logging.debug("Defering, container service not ready")
return
self.unit.status = ActiveStatus()
class CinderCephVictoriaOperatorCharm(CinderCephOperatorCharm):
openstack_relesae = 'victoria'
openstack_release = "victoria"
if __name__ == "__main__":

View File

@ -0,0 +1,17 @@
# This file is managed centrally. If you find the need to modify this as a
# one-off, please don't. Intead, consult #openstack-charms and ask about
# requirements management in charms via bot-control. Thank you.
charm-tools>=2.4.4
coverage>=3.6
mock>=1.2
flake8>=2.2.4,<=2.4.1
pyflakes==2.1.1
stestr>=2.2.0
requests>=2.18.4
psutil
# oslo.i18n dropped py35 support
oslo.i18n<4.0.0
git+https://github.com/openstack-charmers/zaza.git#egg=zaza
git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
pytz # workaround for 14.04 pip/tox
pyudev # for ceph-* charm unit tests (not mocked?)

View File

@ -1,66 +0,0 @@
# Copyright 2021 James Page
# See LICENSE file for licensing details.
#
# Learn more about testing at: https://juju.is/docs/sdk/testing
import unittest
from unittest.mock import Mock
from charm import CharmCinderOperatorCharm
from ops.model import ActiveStatus
from ops.testing import Harness
class TestCharm(unittest.TestCase):
def setUp(self):
self.harness = Harness(CharmCinderOperatorCharm)
self.addCleanup(self.harness.cleanup)
self.harness.begin()
def test_config_changed(self):
self.assertEqual(list(self.harness.charm._stored.things), [])
self.harness.update_config({"thing": "foo"})
self.assertEqual(list(self.harness.charm._stored.things), ["foo"])
def test_action(self):
# the harness doesn't (yet!) help much with actions themselves
action_event = Mock(params={"fail": ""})
self.harness.charm._on_fortune_action(action_event)
self.assertTrue(action_event.set_results.called)
def test_action_fail(self):
action_event = Mock(params={"fail": "fail this"})
self.harness.charm._on_fortune_action(action_event)
self.assertEqual(action_event.fail.call_args, [("fail this",)])
def test_httpbin_pebble_ready(self):
# Check the initial Pebble plan is empty
initial_plan = self.harness.get_container_pebble_plan("httpbin")
self.assertEqual(initial_plan.to_yaml(), "{}\n")
# Expected plan after Pebble ready with default config
expected_plan = {
"services": {
"httpbin": {
"override": "replace",
"summary": "httpbin",
"command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent",
"startup": "enabled",
"environment": {"thing": "🎁"},
}
},
}
# Get the httpbin container from the model
container = self.harness.model.unit.get_container("httpbin")
# Emit the PebbleReadyEvent carrying the httpbin container
self.harness.charm.on.httpbin_pebble_ready.emit(container)
# Get the plan now we've run PebbleReady
updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict()
# Check we've got the plan we expected
self.assertEqual(expected_plan, updated_plan)
# Check the service was started
service = self.harness.model.unit.get_container("httpbin").get_service("httpbin")
self.assertTrue(service.is_running())
# Ensure we set an ActiveStatus with no message
self.assertEqual(self.harness.model.unit.status, ActiveStatus())

View File

@ -0,0 +1,134 @@
# Operator charm (with zaza): tox.ini
[tox]
envlist = pep8,py3
skipsdist = True
# NOTE: Avoid build/test env pollution by not enabling sitepackages.
sitepackages = False
# NOTE: Avoid false positives by not skipping missing interpreters.
skip_missing_interpreters = False
# NOTES:
# * We avoid the new dependency resolver by pinning pip < 20.3, see
# https://github.com/pypa/pip/issues/9187
# * Pinning dependencies requires tox >= 3.2.0, see
# https://tox.readthedocs.io/en/latest/config.html#conf-requires
# * It is also necessary to pin virtualenv as a newer virtualenv would still
# lead to fetching the latest pip in the func* tox targets, see
# https://stackoverflow.com/a/38133283
requires = pip < 20.3
virtualenv < 20.0
# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
minversion = 3.2.0
[testenv]
setenv = VIRTUAL_ENV={envdir}
PYTHONHASHSEED=0
CHARM_DIR={envdir}
install_command =
pip install {opts} {packages}
commands = stestr run --slowest {posargs}
whitelist_externals =
git
add-to-archive.py
bash
charmcraft
passenv = HOME TERM CS_* OS_* TEST_*
deps = -r{toxinidir}/test-requirements.txt
[testenv:py35]
basepython = python3.5
# python3.5 is irrelevant on a focal+ charm.
commands = /bin/true
[testenv:py36]
basepython = python3.6
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py37]
basepython = python3.7
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py38]
basepython = python3.8
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py3]
basepython = python3
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:pep8]
basepython = python3
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = flake8 {posargs} src unit_tests tests
[testenv:cover]
# Technique based heavily upon
# https://github.com/openstack/nova/blob/master/tox.ini
basepython = python3
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
setenv =
{[testenv]setenv}
PYTHON=coverage run
commands =
coverage erase
stestr run --slowest {posargs}
coverage combine
coverage html -d cover
coverage xml -o cover/coverage.xml
coverage report
[coverage:run]
branch = True
concurrency = multiprocessing
parallel = True
source =
.
omit =
.tox/*
*/charmhelpers/*
unit_tests/*
[testenv:venv]
basepython = python3
commands = {posargs}
[testenv:build]
basepython = python3
deps = -r{toxinidir}/build-requirements.txt
commands =
charmcraft build
[testenv:func-noop]
basepython = python3
commands =
functest-run-suite --help
[testenv:func]
basepython = python3
commands =
functest-run-suite --keep-model
[testenv:func-smoke]
basepython = python3
commands =
functest-run-suite --keep-model --smoke
[testenv:func-dev]
basepython = python3
commands =
functest-run-suite --keep-model --dev
[testenv:func-target]
basepython = python3
commands =
functest-run-suite --keep-model --bundle {posargs}
[flake8]
# Ignore E902 because the unit_tests directory is missing in the built charm.
ignore = E402,E226,E902

View File

@ -0,0 +1,7 @@
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -0,0 +1,69 @@
#!/usr/bin/env python3
# Copyright 2021 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import sys
sys.path.append("lib") # noqa
sys.path.append("src") # noqa
from ops.testing import Harness
import charm
class _CinderCephVictoriaOperatorCharm(charm.CinderCephVictoriaOperatorCharm):
def __init__(self, framework):
self.seen_events = []
self.render_calls = []
super().__init__(framework)
def _log_event(self, event):
self.seen_events.append(type(event).__name__)
def renderer(
self,
containers,
container_configs,
template_dir,
openstack_release,
adapters,
):
self.render_calls.append(
(
containers,
container_configs,
template_dir,
openstack_release,
adapters,
)
)
def _on_service_pebble_ready(self, event):
super()._on_service_pebble_ready(event)
self._log_event(event)
class TestCinderCephOperatorCharm(unittest.TestCase):
def setUp(self):
self.harness = Harness(_CinderCephVictoriaOperatorCharm)
self.addCleanup(self.harness.cleanup)
self.harness.begin()
def test_pebble_ready_handler(self):
self.assertEqual(self.harness.charm.seen_events, [])
self.harness.container_pebble_ready("cinder-volume")
self.assertEqual(self.harness.charm.seen_events, ["PebbleReadyEvent"])