[ci] Add loadbalancer annotations as part of setup

During restarts, the k8s services of type loadbalancer
can get diferent IPs and the integration tests tries to
communicate with older IPs.
Add a step after deploying the bundle to add loadbalancer
annotations to services of type loadbalancer so that
IPs persist during restart of pods.

Pin httpx version in tox.ini due to bug [1] in lightkube

[1] https://github.com/gtsystem/lightkube/issues/78

Change-Id: I013ec1c1e9dbac3ae86c57abcd9d87a3b99f6e82

roles/k8s/tasks/main.yaml

@@ -72,3 +72,16 @@
   become: true
   changed_when: false
   failed_when: 'res.rc != 0 or "status: ready" not in res.stdout'
+
+- name: Get k8s config
+  ansible.builtin.command:
+    cmd: k8s config
+  become: true
+  register: kubeconfig
+
+- name: Copy kubeconfig to local
+  ansible.builtin.copy:
+    content: '{{ kubeconfig.stdout }}'
+    dest: '{{ ansible_env.HOME }}/kubeconfig'
+    mode: '0644'
+    owner: '{{ ansible_user }}'

tests/caas/tests.yaml

@@ -3,6 +3,7 @@ gate_bundles:
 smoke_bundles:
   - smoke
 configure:
+  - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations
   - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints
   - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
 tests:

tests/ceph/tests.yaml

@@ -4,6 +4,7 @@ smoke_bundles:
   - smoke
 # There is no storage provider at the moment so cannot run tests.
 configure:
+  - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations
   - zaza.charm_tests.noop.setup.basic_setup
 tests:
   - zaza.charm_tests.noop.tests.NoopTest

tests/core/tests.yaml

@@ -3,6 +3,7 @@ gate_bundles:
 smoke_bundles:
   - smoke
 configure:
+  - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations
   - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints
   - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
   - zaza.openstack.charm_tests.nova.setup.create_flavors

tests/local/zaza/sunbeam/charm_tests/k8s/setup.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from pathlib import (
+    Path,
+)
+
+import yaml
+from lightkube.config.kubeconfig import (
+    KubeConfig,
+)
+from lightkube.core.client import Client as KubeClient
+from lightkube.core.exceptions import (
+    ApiError,
+    ConfigError,
+)
+from lightkube.resources.core_v1 import (
+    Service,
+)
+
+METALLB_ANNOTATION = "metallb.universe.tf/loadBalancerIPs"
+
+
+def add_loadbalancer_annotations(
+    model: str = "openstack", lb_annotation: str = METALLB_ANNOTATION
+):
+    """Add loadbalancer annotations for all services of type LoadBalancer."""
+    home: Path = Path(os.environ["HOME"])
+    kubeconfig_file = home / "kubeconfig"
+
+    if not kubeconfig_file.exists():
+        logging.warning(
+            "No kubeconfig file present, not adding k8s lb service annotations."
+        )
+        return
+
+    try:
+        with kubeconfig_file.open() as f:
+            kubeconfig = KubeConfig.from_dict(yaml.safe_load(f))
+    except (AttributeError, KeyError) as e:
+        logging.warning("Error in kubeconfig content", exc_info=True)
+        return
+
+    try:
+        kube = KubeClient(kubeconfig, model, trust_env=False)
+        for service in kube.list(
+            Service, namespace=model, fields={"spec.type": "LoadBalancer"}
+        ):
+            if not service.metadata:
+                logging.warning(f"No metadata for service, {service}")
+                continue
+
+            service_name = str(service.metadata.name)
+            service_annotations = service.metadata.annotations or {}
+            if lb_annotation not in service_annotations:
+                if not service.status:
+                    logging.warning(
+                        f"k8s service {service_name!r} has no status"
+                    )
+                    continue
+                if not service.status.loadBalancer:
+                    logging.warning(
+                        f"k8s service {service_name!r} has no loadBalancer status"
+                    )
+                    continue
+                if not service.status.loadBalancer.ingress:
+                    logging.warning(
+                        f"k8s service {service_name!r} has no loadBalancer ingress"
+                    )
+                    continue
+                loadbalancer_ip = service.status.loadBalancer.ingress[0].ip
+                service_annotations[lb_annotation] = loadbalancer_ip
+                service.metadata.annotations = service_annotations
+                logging.info(
+                    f"Patching {service_name!r} to use IP {loadbalancer_ip!r}"
+                )
+                kube.patch(Service, service_name, obj=service)
+    except ConfigError:
+        logging.warning("Error creating k8s client", exc_info=True)
+    except ApiError:
+        logging.warning("Error getting services list", exc_info=True)

tests/misc/tests.yaml

@@ -3,6 +3,7 @@ gate_bundles:
 smoke_bundles:
   - smoke
 configure:
+  - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations
   - zaza.charm_tests.noop.setup.basic_setup
   # https://bugs.launchpad.net/snap-openstack/+bug/2045206
   # - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints

tests/tempest/tests.yaml

@@ -6,6 +6,7 @@ gate_bundles:
 smoke_bundles:
   - smoke
 configure:
+  - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations
   - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints
   - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
   - zaza.openstack.charm_tests.nova.setup.create_flavors

tox.ini

@@ -89,6 +89,9 @@ deps =
   git+https://github.com/openstack-charmers/zaza.git#egg=zaza
   git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
   git+https://opendev.org/openstack/tempest.git#egg=tempest
+  # Pin httpx version due to bug https://github.com/gtsystem/lightkube/issues/78
+  httpx>=0.24.0,<0.28.0
+  lightkube
 commands =
   functest-run-suite --help