diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2
index 1c2d314335..149fdee13f 100644
--- a/docker/base/Dockerfile.j2
+++ b/docker/base/Dockerfile.j2
@@ -132,6 +132,9 @@ COPY versionlock.list /etc/yum/pluginconf.d/
RUN yum install -y \
sudo \
which \
+ python \
+ python-jinja2 \
+ python-kazoo \
&& yum clean all
{% endif %}
@@ -167,6 +170,8 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com 199369E5404BD
&& apt-get dist-upgrade -y \
&& apt-get install -y --no-install-recommends \
python \
+ python-jinja2 \
+ python-kazoo \
curl \
&& apt-get clean \
&& sed -i "s|'purelib': '\$base/local/lib/python\$py_version_short/dist-packages',|'purelib': '\$base/lib/python\$py_version_short/dist-packages',|;s|'platlib': '\$platbase/local/lib/python\$py_version_short/dist-packages',|'platlib': '\$platbase/lib/python\$py_version_short/dist-packages',|;s|'headers': '\$base/local/include/python\$py_version_short/\$dist_name',|'headers': '\$base/include/python\$py_version_short/\$dist_name',|;s|'scripts': '\$base/local/bin',|'scripts': '\$base/bin',|;s|'data' : '\$base/local',|'data' : '\$base',|" /usr/lib/python2.7/distutils/command/install.py \
diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py
index cd9ccd83f7..b567e385d9 100644
--- a/docker/base/set_configs.py
+++ b/docker/base/set_configs.py
@@ -12,12 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import contextlib
import json
import logging
import os
from pwd import getpwnam
import shutil
import sys
+import urlparse
+
+from kazoo import client as kz_client
+from kazoo import exceptions as kz_exceptions
# TODO(rhallisey): add docstring.
@@ -46,7 +51,13 @@ def validate_config(config):
def validate_source(data):
source = data.get('source')
- if not os.path.exists(source):
+ if is_zk_transport(source):
+ with zk_connection(source) as zk:
+ exists = zk_path_exists(zk, source)
+ else:
+ exists = os.path.exists(source)
+
+ if not exists:
if data.get('optional'):
LOG.warn('{} does not exist, but is not required'.format(source))
return False
@@ -57,6 +68,66 @@ def validate_source(data):
return True
+def is_zk_transport(path):
+ if path.startswith('zk://'):
+ return True
+ if os.environ.get("KOLLA_ZK_HOSTS") is not None:
+ return True
+
+ return False
+
+
+@contextlib.contextmanager
+def zk_connection(url):
+ # support an environment and url
+ # if url, it should be like this:
+ # zk://<address>:<port>/<path>
+
+ zk_hosts = os.environ.get("KOLLA_ZK_HOSTS")
+ if zk_hosts is None:
+ components = urlparse.urlparse(url)
+ zk_hosts = components.netloc
+ zk = kz_client.KazooClient(hosts=zk_hosts)
+ zk.start()
+ try:
+ yield zk
+ finally:
+ zk.stop()
+
+
+def zk_path_exists(zk, path):
+ try:
+ components = urlparse.urlparse(path)
+ zk.get(components.path)
+ return True
+ except kz_exceptions.NoNodeError:
+ return False
+
+
+def zk_copy_tree(zk, src, dest):
+ """Recursively copy contents of url_source into dest."""
+ data, stat = zk.get(src)
+
+ if data:
+ dest_path = os.path.dirname(dest)
+ if not os.path.exists(dest_path):
+ LOG.info('Creating dest parent directory: {}'.format(
+ dest_path))
+ os.makedirs(dest_path)
+
+ LOG.info('Copying {} to {}'.format(src, dest))
+ with open(dest, 'w') as df:
+ df.write(data.decode("utf-8"))
+
+ try:
+ children = zk.get_children(src)
+ except kz_exceptions.NoNodeError:
+ return
+ for child in children:
+ zk_copy_tree(zk, os.path.join(src, child),
+ os.path.join(dest, child))
+
+
def copy_files(data):
dest = data.get('dest')
source = data.get('source')
@@ -68,6 +139,11 @@ def copy_files(data):
else:
os.remove(dest)
+ if is_zk_transport(source):
+ with zk_connection(source) as zk:
+ components = urlparse.urlparse(source)
+ return zk_copy_tree(zk, components.path, dest)
+
if os.path.isdir(source):
source_path = source
dest_path = dest
diff --git a/docker/base/sudoers b/docker/base/sudoers
index 76baefcb07..974f36a294 100644
--- a/docker/base/sudoers
+++ b/docker/base/sudoers
@@ -13,6 +13,6 @@ root ALL=(ALL) ALL
# anyone in the kolla group may run /usr/local/bin/kolla_set_configs as the
# root user via sudo without password confirmation
-%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs
+%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs, /usr/bin/install
#includedir /etc/sudoers.d
diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2
index 900fc3b4dd..79a6e2894c 100644
--- a/docker/openstack-base/Dockerfile.j2
+++ b/docker/openstack-base/Dockerfile.j2
@@ -81,6 +81,8 @@ RUN ln -s openstack-base-source/* /requirements \
&& pip install -U virtualenv \
&& virtualenv /var/lib/kolla/venv \
&& /var/lib/kolla/venv/bin/pip --no-cache-dir install -U -c requirements/upper-constraints.txt \
+ jinja2 \
+ kazoo \
python-barbicanclient \
python-ceilometerclient \
python-congressclient \
diff --git a/test-requirements.txt b/test-requirements.txt
index 89194d6fef..11f725250b 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -19,3 +19,4 @@ sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0
+zake>=0.1.6 # Apache-2.0
diff --git a/tests/test_set_config.py b/tests/test_set_config.py
index 1e8ac9ce62..7092895c54 100644
--- a/tests/test_set_config.py
+++ b/tests/test_set_config.py
@@ -15,8 +15,11 @@ import json
import mock
import os.path
import sys
+import tempfile
from oslotest import base
+import testscenarios
+from zake import fake_client
# nasty: to import set_config (not a part of the kolla package)
this_dir = os.path.dirname(sys.modules[__name__].__file__)
@@ -62,3 +65,58 @@ class LoadFromEnv(base.BaseTestCase):
mock.call().write(u'/bin/true'),
mock.call().__exit__(None, None, None)],
mo.mock_calls)
+
+
+class ZkCopyTest(testscenarios.WithScenarios, base.BaseTestCase):
+
+ scenarios = [
+ ('1', dict(in_paths=['a.conf'],
+ in_subtree='/',
+ expect_paths=[['a.conf']])),
+ ('2', dict(in_paths=['/a/b/c.x', '/a/b/foo.x', '/a/no.x'],
+ in_subtree='/a/b',
+ expect_paths=[['c.x'], ['foo.x']])),
+ ('3', dict(in_paths=['/a/b/c.x', '/a/z/foo.x'],
+ in_subtree='/',
+ expect_paths=[['a', 'b', 'c.x'], ['a', 'z', 'foo.x']])),
+ ]
+
+ def setUp(self):
+ super(ZkCopyTest, self).setUp()
+ self.client = fake_client.FakeClient()
+ self.client.start()
+ self.addCleanup(self.client.stop)
+ self.addCleanup(self.client.close)
+
+ def test_cp_tree(self):
+ # Note: oslotest.base cleans up all tempfiles as follows:
+ # self.useFixture(fixtures.NestedTempfile())
+ # so we don't have to.
+ temp_dir = tempfile.mkdtemp()
+
+ for path in self.in_paths:
+ self.client.create(path, 'one', makepath=True)
+ set_configs.zk_copy_tree(self.client, self.in_subtree, temp_dir)
+ for expect in self.expect_paths:
+ expect.insert(0, temp_dir)
+ expect_path = os.path.join(*expect)
+ self.assertTrue(os.path.exists(expect_path))
+
+
+class ZkExistsTest(base.BaseTestCase):
+ def setUp(self):
+ super(ZkExistsTest, self).setUp()
+ self.client = fake_client.FakeClient()
+ self.client.start()
+ self.addCleanup(self.client.stop)
+ self.addCleanup(self.client.close)
+
+ def test_path_exists_no(self):
+ self.client.create('/test/path/thing', 'one', makepath=True)
+ self.assertFalse(set_configs.zk_path_exists(self.client,
+ '/test/missing/thing'))
+
+ def test_path_exists_yes(self):
+ self.client.create('/test/path/thing', 'one', makepath=True)
+ self.assertTrue(set_configs.zk_path_exists(self.client,
+ '/test/path/thing'))