Sketch in the Context class

This adds the base Context class for striker, along with several
support functions and classes.  This also settles on the Apache v2
license.

Related to RM7484.
This commit is contained in:
Kevin L. Mitchell 2014-07-31 10:51:48 -05:00
parent d3f11c09aa
commit 4a726d124a
6 changed files with 1302 additions and 0 deletions

58
striker/common/utils.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
import os
import time
def canonicalize_path(cwd, path):
"""
Canonicalizes a path relative to a given working directory.
:param cwd: The working directory to interpret ``path`` relative
to.
:param path: The path to canonicalize. If relative, it will be
interpreted relative to ``cwd``.
:returns: The absolute path.
"""
if not os.path.isabs(path):
path = os.path.join(cwd, path)
return os.path.abspath(path)
def backoff(max_tries):
"""
A generator to perform simplified exponential backoff. Yields up
to the specified number of times, performing a ``time.sleep()``
with an exponentially increasing sleep time (starting at 1 second)
between each trial. Yields the (0-based) trial number.
:param max_tries: The maximum number of tries to attempt.
"""
# How much time will we sleep next time?
sleep = 1
for i in range(max_tries):
# Yield the trial number
yield i
# We've re-entered the loop; sleep, then increment the sleep
# time
time.sleep(sleep)
sleep <<= 1

123
striker/core/context.py Normal file
View File

@ -0,0 +1,123 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
from striker.core import environment
class Context(object):
"""
Execution context. Objects of this class contain all the basic
configuration data needed to perform a task.
"""
def __init__(self, workspace, config, logger,
debug=False, dry_run=False, **extras):
"""
Initialize a ``Context`` object.
:param workspace: The name of a temporary working directory.
The directory must exist.
:param config: An object containing configuration data. The
object should support read-only attribute-style
access to the configuration settings.
:param logger: An object compatible with ``logging.Logger``.
This will be used by all consumers of the
``Context`` to emit logging information.
:param debug: A boolean, defaulting to ``False``, indicating
whether debugging mode is active.
:param dry_run: A boolean, defaulting to ``False``, indicating
whether permanent changes should be effected.
This should be used to control whether files
are uploaded, for instance.
:param extras: Keyword arguments specifying additional data to
be stored in the context. This could be, for
instance, account data.
"""
# Store basic context data
self.workspace = workspace
self.config = config
self.logger = logger
self.debug = debug
self.dry_run = dry_run
# Extra data--things like accounts
self._extras = extras
# Environment
self._environ = None
def __getattr__(self, name):
"""
Provides access to the extra data specified to the constructor.
:param name: The name of the extra datum to retrieve.
:returns: The value of the extra datum.
"""
if name not in self._extras:
raise AttributeError("'%s' object has no attribute '%s'" %
(self.__class__.__name__, name))
return self._extras[name]
@property
def environ(self):
"""
Access the environment. The environment is a dictionary of
environment variables, but it is also a callable that can be
used to invoke shell commands.
:param cmd: The command to execute, as either a bare string or
a list of arguments. If a string, it will be
split into a list using ``shlex.split()``. Note
that use of bare strings for this argument is
discouraged.
:param capture_output: If ``True``, standard input and output
will be captured, and will be available
in the result. Defaults to ``False``.
Note that this is treated as implicitly
``True`` if the ``retry`` parameter is
provided.
:param cwd: Gives an alternate working directory from which to
run the command.
:param do_raise: If ``True`` (the default), an execution
failure will raise an exception.
:param retry: If provided, must be a callable taking one
argument. Will be called with an instance of
``ExecResult``, and can return ``True`` to
indicate that the call should be retried.
Retries are performed with an exponential
backoff controlled by ``max_tries``.
:param max_tries: The maximum number of tries to perform
before giving up, if ``retry`` is specified.
Retries are performed with an exponential
backoff: the first try is performed
immediately, and subsequent tries occur
after a sleep time that starts at one second
and is doubled for each try.
:returns: An ``ExecResult`` object containing the results of
the execution. If the return code was non-zero and
``do_raise`` is ``True``, this is the object that
will be raised.
"""
# Construct the environment if necessary
if self._environ is None:
self._environ = environment.Environment(self.logger)
return self._environ

301
striker/core/environment.py Normal file
View File

@ -0,0 +1,301 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
import os
import shlex
import shutil
import subprocess
import six
from striker.common import utils
class ExecResult(Exception):
"""
Encapsulate the results of calling a command. This class extends
``Exception`` so that it can be raised in the event of a command
failure. The command executed is available in both list (``cmd``)
and plain text (``cmd_text``) forms. If the command is executed
with ``capture_output``, the standard output (``stdout``) and
standard error (``stderr``) streams will also be available. The
command return code is available in the ``return_code`` attribute.
"""
def __init__(self, cmd, stdout, stderr, return_code):
"""
Initialize an ``ExecResult``.
:param cmd: The command, in list format.
:param stdout: The standard output from the command execution.
:param stderr: The standard error from the command execution.
:param return_code: The return code from executing the
command.
"""
# Store all the data
self.cmd = cmd
self.stdout = stdout
self.stderr = stderr
self.return_code = return_code
# Form the command text
comps = []
for comp in cmd:
# Determine if the component needs quoting
if ' ' in comp or '"' in comp or "'" in comp:
# Escape any double-quotes
parts = comp.split('"')
comp = '"%s"' % '\\"'.join(parts)
comps.append(comp)
# Save the command text
self.cmd_text = ' '.join(comps)
# Formulate the message
if return_code:
msg = ("'%s' failed with return code %s" %
(self.cmd_text, return_code))
elif stderr:
msg = "'%s' said: %s" % (self.cmd_text, stderr)
elif stdout:
msg = "'%s' said: %s" % (self.cmd_text, stdout)
else:
msg = "'%s' succeeded" % (self.cmd_text,)
# Initialize ourselves as an exception
super(ExecResult, self).__init__(msg)
def __nonzero__(self):
"""
Allows conversion of an ``ExecResult`` to boolean, based on
the command return code. If the return code was 0, the object
will be considered ``True``; otherwise, the object will be
considered ``False``.
:returns: ``True`` if the command succeeded, ``False``
otherwise.
"""
return not bool(self.return_code)
__bool__ = __nonzero__
class Environment(dict):
"""
Describes an environment that can be used for execution of
subprocesses. Virtual environments can be created by calling the
``create_venv()`` method, which returns an independent instance of
``Environment``.
"""
def __init__(self, logger, environ=None, cwd=None, venv_home=None):
"""
Initialize a new ``Environment``.
:param logger: An object compatible with ``logging.Logger``.
This will be used to emit logging information.
:param environ: A dictionary containing the environment
variables. If not given, ``os.environ`` will
be used.
:param cwd: The working directory to use. If relative, will
be interpreted relative to the current working
directory. If not given, the current working
directory will be used.
:param venv_home: The home directory for the virtual
environment.
"""
super(Environment, self).__init__(environ or os.environ)
# Save the logger
self.logger = logger
# Change to the desired working directory, then save the full
# path to it
self.cwd = os.getcwd()
if cwd:
self.chdir(cwd)
# Save the virtual environment home
self.venv_home = venv_home
def __call__(self, cmd, capture_output=False, cwd=None, do_raise=True,
retry=None, max_tries=5):
"""
Execute a command in the context of this environment.
:param cmd: The command to execute, as either a bare string or
a list of arguments. If a string, it will be
split into a list using ``shlex.split()``. Note
that use of bare strings for this argument is
discouraged.
:param capture_output: If ``True``, standard input and output
will be captured, and will be available
in the result. Defaults to ``False``.
Note that this is treated as implicitly
``True`` if the ``retry`` parameter is
provided.
:param cwd: Gives an alternate working directory from which to
run the command.
:param do_raise: If ``True`` (the default), an execution
failure will raise an exception.
:param retry: If provided, must be a callable taking one
argument. Will be called with an instance of
``ExecResult``, and can return ``True`` to
indicate that the call should be retried.
Retries are performed with an exponential
backoff controlled by ``max_tries``.
:param max_tries: The maximum number of tries to perform
before giving up, if ``retry`` is specified.
Retries are performed with an exponential
backoff: the first try is performed
immediately, and subsequent tries occur
after a sleep time that starts at one second
and is doubled for each try.
:returns: An ``ExecResult`` object containing the results of
the execution. If the return code was non-zero and
``do_raise`` is ``True``, this is the object that
will be raised.
"""
# Sanity-check arguments
if not retry or max_tries < 1:
max_tries = 1
# Determine the working directory to use
cwd = utils.canonicalize_path(self.cwd, cwd) if cwd else self.cwd
# Turn simple strings into lists of tokens
if isinstance(cmd, six.string_types):
self.logger.debug("Notice: splitting command string '%s'" %
cmd)
cmd = shlex.split(cmd)
self.logger.debug("Executing command: %r (cwd %s)" % (cmd, cwd))
# Prepare the keyword arguments for the Popen call
kwargs = {
'env': self,
'cwd': cwd,
'close_fds': True,
}
# Set up stdout and stderr
if capture_output or (retry and max_tries > 1):
kwargs.update({
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
})
# Perform the tries in a loop
for trial in utils.backoff(max_tries):
if trial:
self.logger.warn("Failure caught; retrying command "
"(try #%d)" % (trial + 1))
# Call the command
child = subprocess.Popen(cmd, **kwargs)
stdout, stderr = child.communicate()
result = ExecResult(cmd, stdout, stderr, child.returncode)
# Check if we need to retry
if retry and not result and retry(result):
continue
break
else:
# Just log a warning that we couldn't retry
self.logger.warn("Unable to retry: too many attempts")
# Raise an exception if requested
if not result and do_raise:
raise result
return result
def chdir(self, path):
"""
Change the working directory.
:param path: The path to change to. If relative, will be
interpreted relative to the current working
directory.
:returns: The new working directory.
"""
self.cwd = utils.canonicalize_path(self.cwd, path)
return self.cwd
def create_venv(self, path, rebuild=False, **kwargs):
"""
Create a new, bare virtual environment rooted at the given
directory. No packages will be installed, except what
``virtualenv`` installs. Returns a new ``Environment`` set up
for the new virtual environment, with the working directory
set to be the same as the virtual environment directory. Any
keyword arguments will override system environment variables
in the new ``Environment`` object.
:param path: The path to create the virtual environment in.
If relative, will be interpreted relative to the
current working directory.
:param rebuild: If ``True``, the virtual environment will be
rebuilt even if it already exists. If
``False`` (the default), the virtual
environment will only be rebuilt if it doesn't
already exist.
:returns: A new ``Environment`` object.
"""
# Determine the new virtual environment path
path = utils.canonicalize_path(self.cwd, path)
self.logger.debug("Preparing virtual environment %s" % path)
# Check if we need to rebuild the virtual environment
if os.path.exists(path):
if rebuild:
# Blow away the old tree
self.logger.info("Destroying old virtual environment %s" %
path)
shutil.rmtree(path)
else:
self.logger.info("Using existing virtual environment %s" %
path)
else:
# We'll need to create it
rebuild = True
# Create the new virtual environment
if rebuild:
self.logger.info("Creating virtual environment %s" % path)
self(['virtualenv', path])
# Set up the environment variables that are needed
kwargs.setdefault('VIRTUAL_ENV', path)
bindir = os.path.join(path, 'bin')
kwargs.setdefault('PATH', '%s%s%s' %
(bindir, os.pathsep, self['PATH']))
# Set up and return the new Environment
new_env = self.__class__(self.logger, environ=self, cwd=path,
venv_home=path)
new_env.update(kwargs)
return new_env

View File

@ -0,0 +1,74 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
import unittest
import mock
from striker.common import utils
import tests
class CanonicalizePathTest(unittest.TestCase):
@mock.patch('os.path.isabs', tests.fake_isabs)
@mock.patch('os.path.join', tests.fake_join)
@mock.patch('os.path.abspath', tests.fake_abspath)
def test_absolute(self):
result = utils.canonicalize_path('/foo/bar', '/bar/baz')
self.assertEqual(result, '/bar/baz')
@mock.patch('os.path.isabs', tests.fake_isabs)
@mock.patch('os.path.join', tests.fake_join)
@mock.patch('os.path.abspath', tests.fake_abspath)
def test_relative(self):
result = utils.canonicalize_path('/foo/bar', 'bar/baz')
self.assertEqual(result, '/foo/bar/bar/baz')
@mock.patch('os.path.isabs', tests.fake_isabs)
@mock.patch('os.path.join', tests.fake_join)
@mock.patch('os.path.abspath', tests.fake_abspath)
def test_relative_with_cwd(self):
result = utils.canonicalize_path('/foo/bar', './baz')
self.assertEqual(result, '/foo/bar/baz')
@mock.patch('os.path.isabs', tests.fake_isabs)
@mock.patch('os.path.join', tests.fake_join)
@mock.patch('os.path.abspath', tests.fake_abspath)
def test_relative_with_parent(self):
result = utils.canonicalize_path('/foo/bar', '../baz')
self.assertEqual(result, '/foo/baz')
class BackoffTest(unittest.TestCase):
@mock.patch('time.sleep')
def test_backoff(self, mock_sleep):
max_tries = 5
for i, trial in enumerate(utils.backoff(max_tries)):
self.assertEqual(i, trial)
if i:
mock_sleep.assert_called_once_with(1 << (i - 1))
else:
self.assertFalse(mock_sleep.called)
mock_sleep.reset_mock()
self.assertEqual(i, max_tries - 1)

View File

@ -0,0 +1,79 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
import unittest
import mock
from striker.core import context
from striker.core import environment
class ContextTest(unittest.TestCase):
def test_init_base(self):
ctxt = context.Context('/path/to/workspace', 'config', 'logger')
self.assertEqual(ctxt.workspace, '/path/to/workspace')
self.assertEqual(ctxt.config, 'config')
self.assertEqual(ctxt.logger, 'logger')
self.assertEqual(ctxt.debug, False)
self.assertEqual(ctxt.dry_run, False)
self.assertEqual(ctxt._extras, {})
self.assertEqual(ctxt._environ, None)
def test_init_alt(self):
ctxt = context.Context('/path/to/workspace', 'config', 'logger',
debug=True, dry_run=True, accounts='accounts',
other='other')
self.assertEqual(ctxt.workspace, '/path/to/workspace')
self.assertEqual(ctxt.config, 'config')
self.assertEqual(ctxt.logger, 'logger')
self.assertEqual(ctxt.debug, True)
self.assertEqual(ctxt.dry_run, True)
self.assertEqual(ctxt._extras, {
'accounts': 'accounts',
'other': 'other',
})
self.assertEqual(ctxt._environ, None)
def test_getattr_exists(self):
ctxt = context.Context('/path/to/workspace', 'config', 'logger',
attr='value')
self.assertEqual(ctxt.attr, 'value')
def test_getattr_noexist(self):
ctxt = context.Context('/path/to/workspace', 'config', 'logger',
attr='value')
self.assertRaises(AttributeError, lambda: ctxt.other)
@mock.patch.object(environment, 'Environment', return_value='environ')
def test_environ_cached(self, mock_Environment):
ctxt = context.Context('/path/to/workspace', 'config', 'logger')
ctxt._environ = 'cached'
self.assertEqual(ctxt.environ, 'cached')
self.assertEqual(ctxt._environ, 'cached')
self.assertFalse(mock_Environment.called)
@mock.patch.object(environment, 'Environment', return_value='environ')
def test_environ_uncached(self, mock_Environment):
ctxt = context.Context('/path/to/workspace', 'config', 'logger')
self.assertEqual(ctxt.environ, 'environ')
self.assertEqual(ctxt._environ, 'environ')
mock_Environment.assert_called_once_with('logger')

View File

@ -0,0 +1,667 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS
# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language
# governing permissions and limitations under the License.
import os
import subprocess
import unittest
import mock
from striker.common import utils
from striker.core import environment
import tests
class ExecResultTest(unittest.TestCase):
def test_init_success(self):
cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5']
cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5'
result = environment.ExecResult(cmd, None, None, 0)
self.assertEqual(result.cmd, cmd)
self.assertEqual(result.cmd_text, cmd_text)
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
self.assertEqual(str(result), "'%s' succeeded" % cmd_text)
def test_init_stdout(self):
cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5']
cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5'
result = environment.ExecResult(cmd, 'output', None, 0)
self.assertEqual(result.cmd, cmd)
self.assertEqual(result.cmd_text, cmd_text)
self.assertEqual(result.stdout, 'output')
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
self.assertEqual(str(result), "'%s' said: output" % cmd_text)
def test_init_stderr(self):
cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5']
cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5'
result = environment.ExecResult(cmd, 'output', 'error', 0)
self.assertEqual(result.cmd, cmd)
self.assertEqual(result.cmd_text, cmd_text)
self.assertEqual(result.stdout, 'output')
self.assertEqual(result.stderr, 'error')
self.assertEqual(result.return_code, 0)
self.assertEqual(str(result), "'%s' said: error" % cmd_text)
def test_init_failure(self):
cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5']
cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5'
result = environment.ExecResult(cmd, 'output', 'error', 5)
self.assertEqual(result.cmd, cmd)
self.assertEqual(result.cmd_text, cmd_text)
self.assertEqual(result.stdout, 'output')
self.assertEqual(result.stderr, 'error')
self.assertEqual(result.return_code, 5)
self.assertEqual(str(result), "'%s' failed with return code 5" %
cmd_text)
def test_true(self):
result = environment.ExecResult(['cmd'], None, None, 0)
self.assertTrue(result)
def test_false(self):
result = environment.ExecResult(['cmd'], None, None, 1)
self.assertFalse(result)
class EnvironmentTest(unittest.TestCase):
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
def test_init_base(self, mock_chdir, mock_getcwd):
env = environment.Environment('logger')
self.assertEqual(env, {'TEST_VAR1': '1', 'TEST_VAR2': '2'})
self.assertEqual(env.logger, 'logger')
self.assertEqual(env.cwd, '/some/path')
self.assertEqual(env.venv_home, None)
self.assertFalse(mock_chdir.called)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
def test_init_alt(self, mock_chdir, mock_getcwd):
environ = {
'TEST_VAR3': '3',
'TEST_VAR4': '4',
}
env = environment.Environment('logger', environ, '/other/path',
'/venv/home')
self.assertEqual(env, environ)
self.assertEqual(env.logger, 'logger')
self.assertEqual(env.cwd, '/some/path')
self.assertEqual(env.venv_home, '/venv/home')
mock_chdir.assert_called_once_with('/other/path')
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os.path, 'join', tests.fake_join)
@mock.patch.object(os, 'pathsep', ':')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': (None, None),
}))
def test_call_basic(self, mock_Popen, mock_backoff, mock_canonicalize_path,
mock_chdir, mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
result = env(['test', 'one', 'two'])
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': (None, None),
}))
def test_call_string(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir, mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
result = env("test one two")
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Notice: splitting command string 'test one two'"),
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 2)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': (None, None),
}))
def test_call_cwd(self, mock_Popen, mock_backoff, mock_canonicalize_path,
mock_chdir, mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
result = env(['test', 'one', 'two'], cwd='/other/path')
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
mock_canonicalize_path.assert_called_once_with(
'/some/path', '/other/path')
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/canon/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /canon/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': ('output', 'error'),
}))
def test_call_capture(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir, mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
result = env(['test', 'one', 'two'], capture_output=True)
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, 'output')
self.assertEqual(result.stderr, 'error')
self.assertEqual(result.return_code, 0)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 1,
'communicate.return_value': (None, None),
}))
def test_call_failure_raise(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
try:
result = env(['test', 'one', 'two'])
except environment.ExecResult as exc:
self.assertEqual(exc.cmd, ['test', 'one', 'two'])
self.assertEqual(exc.stdout, None)
self.assertEqual(exc.stderr, None)
self.assertEqual(exc.return_code, 1)
else:
self.fail("Expected ExecResult to be raised")
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 1,
'communicate.return_value': (None, None),
}))
def test_call_failure_noraise(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
result = env(['test', 'one', 'two'], do_raise=False)
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 1)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': ('output', 'error'),
}))
def test_call_retry_success(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
retry = mock.Mock(return_value=True)
result = env(['test', 'one', 'two'], retry=retry)
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, 'output')
self.assertEqual(result.stderr, 'error')
self.assertEqual(result.return_code, 0)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(5)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
self.assertFalse(retry.called)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': (None, None),
}))
def test_call_retry_success_badretries(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
retry = mock.Mock(return_value=True)
result = env(['test', 'one', 'two'], retry=retry, max_tries=-1)
self.assertEqual(result.cmd, ['test', 'one', 'two'])
self.assertEqual(result.stdout, None)
self.assertEqual(result.stderr, None)
self.assertEqual(result.return_code, 0)
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(1)
mock_Popen.assert_called_once_with(
['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
])
self.assertEqual(len(logger.method_calls), 1)
self.assertFalse(retry.called)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0, 1, 2, 3, 4, 5, 6])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': ('output', 'error'),
}))
def test_call_retry_withtries(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
retry = mock.Mock(return_value=True)
exec_results = [
mock.Mock(__nonzero__=mock.Mock(return_value=False),
__bool__=mock.Mock(return_value=False)),
mock.Mock(__nonzero__=mock.Mock(return_value=False),
__bool__=mock.Mock(return_value=False)),
mock.Mock(__nonzero__=mock.Mock(return_value=True),
__bool__=mock.Mock(return_value=True)),
]
with mock.patch.object(environment, 'ExecResult',
side_effect=exec_results) as mock_ExecResult:
result = env(['test', 'one', 'two'], retry=retry, max_tries=7)
self.assertEqual(result, exec_results[-1])
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(7)
mock_Popen.assert_has_calls([
mock.call(['test', 'one', 'two'], env=env, cwd='/some/path',
close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE),
mock.call(['test', 'one', 'two'], env=env, cwd='/some/path',
close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE),
mock.call(['test', 'one', 'two'], env=env, cwd='/some/path',
close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE),
])
self.assertEqual(mock_Popen.call_count, 3)
mock_ExecResult.assert_has_calls([
mock.call(['test', 'one', 'two'], 'output', 'error', 0),
mock.call(['test', 'one', 'two'], 'output', 'error', 0),
mock.call(['test', 'one', 'two'], 'output', 'error', 0),
])
self.assertEqual(mock_ExecResult.call_count, 3)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
mock.call.warn('Failure caught; retrying command (try #2)'),
mock.call.warn('Failure caught; retrying command (try #3)'),
])
self.assertEqual(len(logger.method_calls), 3)
retry.assert_has_calls([mock.call(res) for res in exec_results[:-1]])
self.assertEqual(retry.call_count, len(exec_results) - 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
@mock.patch.object(utils, 'backoff', return_value=[0, 1])
@mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{
'returncode': 0,
'communicate.return_value': ('output', 'error'),
}))
def test_call_retry_withtries_failure(self, mock_Popen, mock_backoff,
mock_canonicalize_path, mock_chdir,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
retry = mock.Mock(return_value=True)
exec_results = [
mock.Mock(__nonzero__=mock.Mock(return_value=False),
__bool__=mock.Mock(return_value=False)),
mock.Mock(__nonzero__=mock.Mock(return_value=False),
__bool__=mock.Mock(return_value=False)),
mock.Mock(__nonzero__=mock.Mock(return_value=True),
__bool__=mock.Mock(return_value=True)),
]
with mock.patch.object(environment, 'ExecResult',
side_effect=exec_results) as mock_ExecResult:
result = env(['test', 'one', 'two'], retry=retry, max_tries=2,
do_raise=False)
self.assertEqual(result, exec_results[-2])
self.assertFalse(mock_canonicalize_path.called)
mock_backoff.assert_called_once_with(2)
mock_Popen.assert_has_calls([
mock.call(['test', 'one', 'two'], env=env, cwd='/some/path',
close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE),
mock.call(['test', 'one', 'two'], env=env, cwd='/some/path',
close_fds=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE),
])
self.assertEqual(mock_Popen.call_count, 2)
mock_ExecResult.assert_has_calls([
mock.call(['test', 'one', 'two'], 'output', 'error', 0),
mock.call(['test', 'one', 'two'], 'output', 'error', 0),
])
self.assertEqual(mock_ExecResult.call_count, 2)
logger.assert_has_calls([
mock.call.debug(
"Executing command: ['test', 'one', 'two'] (cwd /some/path)"),
mock.call.warn('Failure caught; retrying command (try #2)'),
mock.call.warn('Unable to retry: too many attempts'),
])
self.assertEqual(len(logger.method_calls), 3)
retry.assert_has_calls([mock.call(res) for res in exec_results[:-2]])
self.assertEqual(retry.call_count, len(exec_results) - 1)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
def test_chdir(self, mock_canonicalize_path, mock_getcwd):
with mock.patch.object(environment.Environment, 'chdir'):
env = environment.Environment('logger')
result = env.chdir('test/directory')
self.assertEqual(result, '/canon/path')
self.assertEqual(env.cwd, '/canon/path')
mock_canonicalize_path.assert_called_once_with(
'/some/path', 'test/directory')
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2',
PATH='/bin')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(os.path, 'exists', return_value=False)
@mock.patch('shutil.rmtree')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(environment.Environment, '__call__')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
def test_create_venv_basic(self, mock_canonicalize_path, mock_call,
mock_chdir, mock_rmtree, mock_exists,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
expected = dict(env)
expected.update({
'VIRTUAL_ENV': '/canon/path',
'PATH': '/canon/path/bin:/bin',
})
mock_chdir.reset_mock()
new_env = env.create_venv('venv/dir')
self.assertNotEqual(id(new_env), id(env))
self.assertTrue(isinstance(new_env, environment.Environment))
self.assertEqual(new_env, expected)
self.assertEqual(new_env.logger, logger)
self.assertEqual(new_env.cwd, '/some/path')
self.assertEqual(new_env.venv_home, '/canon/path')
mock_canonicalize_path.assert_called_once_with(
'/some/path', 'venv/dir')
mock_exists.assert_called_once_with('/canon/path')
self.assertFalse(mock_rmtree.called)
mock_call.assert_called_once_with(['virtualenv', '/canon/path'])
mock_chdir.assert_called_once_with('/canon/path')
logger.assert_has_calls([
mock.call.debug('Preparing virtual environment /canon/path'),
mock.call.info('Creating virtual environment /canon/path'),
])
self.assertEqual(len(logger.method_calls), 2)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2',
PATH='/bin')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(os.path, 'exists', return_value=False)
@mock.patch('shutil.rmtree')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(environment.Environment, '__call__')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
def test_create_venv_update(self, mock_canonicalize_path, mock_call,
mock_chdir, mock_rmtree, mock_exists,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
expected = dict(env)
expected.update({
'VIRTUAL_ENV': 'bah',
'PATH': '/canon/path/bin:/bin',
'a': 'foo',
})
mock_chdir.reset_mock()
new_env = env.create_venv('venv/dir', VIRTUAL_ENV='bah', a='foo')
self.assertNotEqual(id(new_env), id(env))
self.assertTrue(isinstance(new_env, environment.Environment))
self.assertEqual(new_env, expected)
self.assertEqual(new_env.logger, logger)
self.assertEqual(new_env.cwd, '/some/path')
self.assertEqual(new_env.venv_home, '/canon/path')
mock_canonicalize_path.assert_called_once_with(
'/some/path', 'venv/dir')
mock_exists.assert_called_once_with('/canon/path')
self.assertFalse(mock_rmtree.called)
mock_call.assert_called_once_with(['virtualenv', '/canon/path'])
mock_chdir.assert_called_once_with('/canon/path')
logger.assert_has_calls([
mock.call.debug('Preparing virtual environment /canon/path'),
mock.call.info('Creating virtual environment /canon/path'),
])
self.assertEqual(len(logger.method_calls), 2)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2',
PATH='/bin')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(os.path, 'exists', return_value=True)
@mock.patch('shutil.rmtree')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(environment.Environment, '__call__')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
def test_create_venv_exists(self, mock_canonicalize_path, mock_call,
mock_chdir, mock_rmtree, mock_exists,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
expected = dict(env)
expected.update({
'VIRTUAL_ENV': '/canon/path',
'PATH': '/canon/path/bin:/bin',
})
mock_chdir.reset_mock()
new_env = env.create_venv('venv/dir')
self.assertNotEqual(id(new_env), id(env))
self.assertTrue(isinstance(new_env, environment.Environment))
self.assertEqual(new_env, expected)
self.assertEqual(new_env.logger, logger)
self.assertEqual(new_env.cwd, '/some/path')
self.assertEqual(new_env.venv_home, '/canon/path')
mock_canonicalize_path.assert_called_once_with(
'/some/path', 'venv/dir')
mock_exists.assert_called_once_with('/canon/path')
self.assertFalse(mock_rmtree.called)
self.assertFalse(mock_call.called)
mock_chdir.assert_called_once_with('/canon/path')
logger.assert_has_calls([
mock.call.debug('Preparing virtual environment /canon/path'),
mock.call.info('Using existing virtual environment /canon/path'),
])
self.assertEqual(len(logger.method_calls), 2)
@mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2',
PATH='/bin')
@mock.patch.object(os, 'getcwd', return_value='/some/path')
@mock.patch.object(os.path, 'exists', return_value=True)
@mock.patch('shutil.rmtree')
@mock.patch.object(environment.Environment, 'chdir')
@mock.patch.object(environment.Environment, '__call__')
@mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path')
def test_create_venv_rebuild(self, mock_canonicalize_path, mock_call,
mock_chdir, mock_rmtree, mock_exists,
mock_getcwd):
logger = mock.Mock()
env = environment.Environment(logger)
expected = dict(env)
expected.update({
'VIRTUAL_ENV': '/canon/path',
'PATH': '/canon/path/bin:/bin',
})
mock_chdir.reset_mock()
new_env = env.create_venv('venv/dir', True)
self.assertNotEqual(id(new_env), id(env))
self.assertTrue(isinstance(new_env, environment.Environment))
self.assertEqual(new_env, expected)
self.assertEqual(new_env.logger, logger)
self.assertEqual(new_env.cwd, '/some/path')
self.assertEqual(new_env.venv_home, '/canon/path')
mock_canonicalize_path.assert_called_once_with(
'/some/path', 'venv/dir')
mock_exists.assert_called_once_with('/canon/path')
mock_rmtree.assert_called_once_with('/canon/path')
mock_call.assert_called_once_with(['virtualenv', '/canon/path'])
mock_chdir.assert_called_once_with('/canon/path')
logger.assert_has_calls([
mock.call.debug('Preparing virtual environment /canon/path'),
mock.call.info('Destroying old virtual environment /canon/path'),
mock.call.info('Creating virtual environment /canon/path'),
])
self.assertEqual(len(logger.method_calls), 3)