openstack/mistral
Code Issues Proposed changes

Merge "Centralize session creation and authorization from OS clients"

Browse Source
This commit is contained in:
Jenkins 2017-06-30 11:27:22 +00:00 committed by Gerrit Code Review
commit b19d871415
7 changed files with 131 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
mistral/actions/openstack/actions.py
View File

@ -42,9 +42,8 @@ def _try_import(module_name):
try:
return importutils.try_import(module_name)
except Exception as e:
msg = 'Unable to load module "%s". %s' % (module_name, e.message)
msg = 'Unable to load module "%s". %s' % (module_name, str(e))
LOG.error(msg)
print('ERROR [%s] %s' % (__name__, msg))
return None
@ -72,43 +71,24 @@ zaqarclient = _try_import('zaqarclient.queues.v2.client')
class NovaAction(base.OpenStackAction):
_service_name = 'nova'
_service_type = 'compute'
def _create_client(self, context):
@classmethod
def _get_client_class(cls):
return novaclient.Client
def _create_client(self, context):
LOG.debug("Nova action security context: %s" % context)
keystone_endpoint = keystone_utils.get_keystone_endpoint_v2()
nova_endpoint = self.get_service_endpoint()
client = novaclient.Client(
2,
username=None,
api_key=None,
endpoint_type=CONF.openstack_actions.os_actions_endpoint_type,
service_type='compute',
auth_token=context.auth_token,
tenant_id=context.project_id,
region_name=nova_endpoint.region,
auth_url=keystone_endpoint.url,
insecure=context.insecure
)
client.client.management_url = keystone_utils.format_url(
nova_endpoint.url,
{'tenant_id': context.project_id}
)
return client
return novaclient.Client(2, **self.get_session_and_auth(context))
@classmethod
def _get_fake_client(cls):
return novaclient.Client(2)
return cls._get_client_class()(2)
class GlanceAction(base.OpenStackAction):
_service_name = 'glance'
_service_type = 'image'
@classmethod
def _get_client_class(cls):
@ -123,8 +103,7 @@ class GlanceAction(base.OpenStackAction):
return self._get_client_class()(
glance_endpoint.url,
region_name=glance_endpoint.region,
token=context.auth_token,
insecure=context.insecure
**self.get_session_and_auth(context)
)
@classmethod
@ -134,6 +113,8 @@ class GlanceAction(base.OpenStackAction):
class KeystoneAction(base.OpenStackAction):
_service_type = 'identity'
@classmethod
def _get_client_class(cls):
return keystoneclient.Client
@ -142,26 +123,15 @@ class KeystoneAction(base.OpenStackAction):
LOG.debug("Keystone action security context: %s" % context)
# TODO(akovi) cacert is deprecated in favor of session
# TODO(akovi) this piece of code should be refactored
# TODO(akovi) to follow the new guide lines
kwargs = {
'token': context.auth_token,
'auth_url': context.auth_uri,
'project_id': context.project_id,
'cacert': context.auth_cacert,
'insecure': context.insecure
}
kwargs = self.get_session_and_auth(context)
# In case of trust-scoped token explicitly pass endpoint parameter.
if (context.is_trust_scoped
or keystone_utils.is_token_trust_scoped(context.auth_token)):
kwargs['endpoint'] = context.auth_uri
# NOTE(akovi): the endpoint in the token messes up
# keystone. The auth parameter should not be provided for
# these operations.
kwargs.pop('auth')
client = self._get_client_class()(**kwargs)
client.management_url = context.auth_uri
return client
@classmethod
@ -179,7 +149,7 @@ class KeystoneAction(base.OpenStackAction):
class CeilometerAction(base.OpenStackAction):
_service_name = 'ceilometer'
_service_type = 'metering'
@classmethod
def _get_client_class(cls):
@ -210,7 +180,7 @@ class CeilometerAction(base.OpenStackAction):
class HeatAction(base.OpenStackAction):
_service_name = 'heat'
_service_type = 'orchestration'
@classmethod
def _get_client_class(cls):
@ -233,9 +203,7 @@ class HeatAction(base.OpenStackAction):
return self._get_client_class()(
endpoint_url,
region_name=heat_endpoint.region,
token=context.auth_token,
username=context.user_name,
insecure=context.insecure
**self.get_session_and_auth(context)
)
@classmethod
@ -244,7 +212,7 @@ class HeatAction(base.OpenStackAction):
class NeutronAction(base.OpenStackAction):
_service_name = 'neutron'
_service_type = 'network'
@classmethod
def _get_client_class(cls):
@ -306,6 +274,7 @@ class CinderAction(base.OpenStackAction):
class MistralAction(base.OpenStackAction):
_service_type = 'workflowv2'
@classmethod
def _get_client_class(cls):
@ -315,17 +284,10 @@ class MistralAction(base.OpenStackAction):
LOG.debug("Mistral action security context: %s" % context)
mistral_endpoint = keystone_utils.get_endpoint_for_project(
'mistral'
)
mistral_url = mistral_endpoint.url
session_and_auth = self.get_session_and_auth(context)
return self._get_client_class()(
mistral_url=mistral_url,
auth_token=context.auth_token,
project_id=context.project_id,
user_id=context.user_id,
insecure=context.insecure
mistral_url=session_and_auth['auth'].endpoint,
**session_and_auth
)
@classmethod
@ -501,9 +463,9 @@ class ZaqarAction(base.OpenStackAction):
def wrap(*args, **kwargs):
return method(client, *args, **kwargs)
args = inspect_utils.get_arg_list_as_str(method)
arguments = inspect_utils.get_arg_list_as_str(method)
# Remove client
wrap.__arguments__ = args.split(', ', 1)[1]
wrap.__arguments__ = arguments.split(', ', 1)[1]
return wrap
@ -511,6 +473,9 @@ class ZaqarAction(base.OpenStackAction):
def queue_messages(client, queue_name, **params):
"""Gets a list of messages from the queue.
:param client: the Zaqar client
:type client: zaqarclient.queues.client
:param queue_name: Name of the target queue.
:type queue_name: `six.string_type`
@ -528,6 +493,9 @@ class ZaqarAction(base.OpenStackAction):
def queue_post(client, queue_name, messages):
"""Posts one or more messages to a queue.
:param client: the Zaqar client
:type client: zaqarclient.queues.client
:param queue_name: Name of the target queue.
:type queue_name: `six.string_type`
@ -545,6 +513,9 @@ class ZaqarAction(base.OpenStackAction):
def queue_pop(client, queue_name, count=1):
"""Pop `count` messages from the queue.
:param client: the Zaqar client
:type client: zaqarclient.queues.client
:param queue_name: Name of the target queue.
:type queue_name: `six.string_type`
@ -604,10 +575,10 @@ class BarbicanAction(base.OpenStackAction):
def wrap(*args, **kwargs):
return method(client, *args, **kwargs)
args = inspect_utils.get_arg_list_as_str(method)
arguments = inspect_utils.get_arg_list_as_str(method)
# Remove client.
wrap.__arguments__ = args.split(', ', 1)[1]
wrap.__arguments__ = arguments.split(', ', 1)[1]
return wrap
@ -621,6 +592,9 @@ class BarbicanAction(base.OpenStackAction):
mode=None, expiration=None):
"""Create and Store a secret in Barbican.
:param client: the Zaqar client
:type client: zaqarclient.queues.client
:param name: A friendly name for the Secret
:type name: string
@ -805,13 +779,13 @@ class SenlinAction(base.OpenStackAction):
insecure=context.insecure
)
@classmethod
def _get_fake_client(cls):
return cls._get_client_class()("http://127.0.0.1:8778")
@classmethod
def _get_fake_client(cls):
return cls._get_client_class()("http://127.0.0.1:8778")
class AodhAction(base.OpenStackAction):
_service_name = 'aodh'
_service_type = 'alarming'
@classmethod
def _get_client_class(cls):
@ -842,7 +816,7 @@ class AodhAction(base.OpenStackAction):
class GnocchiAction(base.OpenStackAction):
_service_name = 'gnocchi'
_service_type = 'metric'
@classmethod
def _get_client_class(cls):

60
mistral/actions/openstack/base.py
View File

@ -16,16 +16,12 @@ import abc
import inspect
import traceback
from cachetools import LRUCache
from oslo_log import log
from mistral import exceptions as exc
from mistral.utils.openstack import keystone as keystone_utils
from mistral_lib import actions
from threading import Lock
LOG = log.getLogger(__name__)
@ -37,23 +33,22 @@ class OpenStackAction(actions.Action):
"""
_kwargs_for_run = {}
client_method_name = None
_clients = LRUCache(100)
_lock = Lock()
_service_name = None
_service_type = None
_client_class = None
def __init__(self, **kwargs):
self._kwargs_for_run = kwargs
self.action_region = self._kwargs_for_run.pop('action_region', None)
@abc.abstractmethod
def _create_client(self):
"""Creates client required for action operation"""
pass
def _create_client(self, context):
"""Creates client required for action operation."""
return None
@classmethod
def _get_client_class(cls):
return None
return cls._client_class
@classmethod
def _get_client_method(cls, client):
@ -86,40 +81,20 @@ class OpenStackAction(actions.Action):
(e.g. Nova, Glance, Heat, Keystone etc)
"""
# TODO(d0ugal): Caching has caused some security problems and
# regressions in Mistral. It is disabled for now and
# will be revisited in Ocata. See:
# https://bugs.launchpad.net/mistral/+bug/1627689
return self._create_client(context)
client_class = self.__class__.__name__
# Colon character is reserved (rfc3986) which avoids key collisions.
key = client_class + ':' + context.project_id
def get_session_and_auth(self, context):
"""Get keystone session and auth parameters.
def create_cached_client():
new_client = self._create_client(context)
new_client._mistral_ctx_expires_at = context.expires_at
:param context: the action context
:return: dict that can be used to initialize service clients
"""
with self._lock:
self._clients[key] = new_client
return new_client
with self._lock:
client = self._clients.get(key)
if client is None:
return create_cached_client()
if keystone_utils.will_expire_soon(client._mistral_ctx_expires_at):
LOG.debug("cache expiring soon, will refresh client")
return create_cached_client()
LOG.debug("cache not expiring soon, will return cached client")
return client
return keystone_utils.get_session_and_auth(
service_name=self._service_name,
service_type=self._service_type,
region_name=self.action_region,
context=context)
def get_service_endpoint(self):
"""Get OpenStack service endpoint.
@ -150,7 +125,10 @@ class OpenStackAction(actions.Action):
# where the issue comes from.
LOG.warning(traceback.format_exc())
e_str = '%s: %s' % (type(e), e.message)
if hasattr(e, 'message'):
e_str = '%s: %s' % (type(e), e.message)
else:
e_str = str(e)
raise exc.ActionException(
"%s.%s failed: %s" %

20
mistral/services/periodic.py
View File

@ -35,11 +35,11 @@ _periodic_tasks = {}
class MistralPeriodicTasks(periodic_task.PeriodicTasks):
@periodic_task.periodic_task(spacing=1, run_immediately=True)
def process_cron_triggers_v2(self, ctx):
for t in triggers.get_next_cron_triggers():
LOG.debug("Processing cron trigger: %s" % t)
for trigger in triggers.get_next_cron_triggers():
LOG.debug("Processing cron trigger: %s" % trigger)
# Setup admin context before schedule triggers.
ctx = security.create_context(t.trust_id, t.project_id)
ctx = security.create_context(trigger.trust_id, trigger.project_id)
auth_ctx.set_ctx(ctx)
@ -48,25 +48,27 @@ class MistralPeriodicTasks(periodic_task.PeriodicTasks):
try:
# Try to advance the cron trigger next_execution_time and
# remaining_executions if relevant.
modified = advance_cron_trigger(t)
modified = advance_cron_trigger(trigger)
# If cron trigger was not already modified by another engine.
if modified:
LOG.debug(
"Starting workflow '%s' by cron trigger '%s'",
t.workflow.name, t.name
trigger.workflow.name, trigger.name
)
rpc.get_engine_client().start_workflow(
t.workflow.name,
t.workflow_input,
trigger.workflow.name,
trigger.workflow_input,
description="Workflow execution created "
"by cron trigger.",
**t.workflow_params
**trigger.workflow_params
)
except Exception:
# Log and continue to next cron trigger.
LOG.exception("Failed to process cron trigger %s" % str(t))
LOG.exception(
"Failed to process cron trigger %s" % str(trigger)
)
finally:
auth_ctx.set_ctx(None)

3
mistral/services/security.py
View File

@ -95,6 +95,7 @@ def delete_trust(trust_id):
def add_trust_id(secure_object_values):
if cfg.CONF.pecan.auth_enable:
trust = create_trust()
secure_object_values.update({
'trust_id': create_trust().id
'trust_id': trust.id
})

8
mistral/services/triggers.py
View File

@ -108,7 +108,7 @@ def create_cron_trigger(name, workflow_name, workflow_input,
wf_spec.__class__.__name__
)
values = {
trigger_parameters = {
'name': name,
'pattern': pattern,
'first_execution_time': first_time,
@ -121,13 +121,13 @@ def create_cron_trigger(name, workflow_name, workflow_input,
'scope': 'private'
}
security.add_trust_id(values)
security.add_trust_id(trigger_parameters)
try:
trig = db_api.create_cron_trigger(values)
trig = db_api.create_cron_trigger(trigger_parameters)
except Exception:
# Delete trust before raising exception.
security.delete_trust(values.get('trust_id'))
security.delete_trust(trigger_parameters.get('trust_id'))
raise
return trig

112
mistral/tests/unit/actions/openstack/test_openstack_actions.py
View File

@ -15,7 +15,6 @@
import mock
from mistral.actions.openstack import actions
from mistral import context as ctx
from oslotest import base
@ -38,117 +37,6 @@ class OpenStackActionTest(base.BaseTestCase):
self.assertTrue(mocked().servers.get.called)
mocked().servers.get.assert_called_once_with(server="1234-abcd")
@mock.patch('mistral.actions.openstack.actions.keystone_utils.get_'
'keystone_endpoint_v2')
@mock.patch('mistral.actions.openstack.actions.keystone_utils.get_'
'endpoint_for_project')
@mock.patch('mistral.actions.openstack.actions.novaclient')
def test_nova_action_config_endpoint(self, mock_novaclient,
mock_nova_endpoint,
mock_ks_endpoint_v2):
test_ctx = ctx.MistralContext(
user=None,
tenant='1234',
project_name='admin',
auth_token=None,
is_admin=False,
# set year to 3016 in order for token to always be valid
expires_at='3016-07-13T18:34:22.000000Z',
insecure=False
)
# attributes mirror keystone Endpoint object exactly
# (with endpoint type public)
keystone_attrs = {
'url': 'http://192.0.2.1:5000/v2.0',
'enabled': True,
'id': 'b1ddf133fa6e491c8ee13701be97db2d',
'interface': 'public',
'links': {
u'self': u'http://192.0.2.1:5000/v3/endpoints/'
'b1ddf133fa6e491c8ee13701be97db2d'
},
'region': 'regionOne',
'region_id': 'regionOne',
'service_id': '8f4afc75cd584d5cb381f68a9db80147',
}
keystone_endpoint = FakeEndpoint(**keystone_attrs)
nova_attrs = {
'url': 'http://192.0.2.1:8774/v2/%(tenant_id)s',
'enabled': True,
'id': '5bb51b33c9984513b52b6a3e85154305',
'interface': 'public',
'links': {
u'self': u'http://192.0.2.1:5000/v3/endpoints/'
'5bb51b33c9984513b52b6a3e85154305'
},
'region': 'regionOne',
'region_id': 'regionOne',
'service_id': '1af46173f37848edb65bd4962ed2d09d',
}
nova_endpoint = FakeEndpoint(**nova_attrs)
mock_ks_endpoint_v2.return_value(keystone_endpoint)
mock_nova_endpoint.return_value(nova_endpoint)
method_name = "servers.get"
action_class = actions.NovaAction
action_class.client_method_name = method_name
params = {'server': '1234-abcd'}
action = action_class(**params)
action.run(test_ctx)
mock_novaclient.Client.assert_called_once_with(
2,
username=None,
api_key=None,
endpoint_type='public',
service_type='compute',
auth_token=test_ctx.auth_token,
tenant_id=test_ctx.project_id,
region_name=mock_nova_endpoint().region,
auth_url=mock_ks_endpoint_v2().url,
insecure=test_ctx.insecure
)
self.assertTrue(mock_novaclient.Client().servers.get.called)
mock_novaclient.Client().servers.get.assert_called_once_with(
server="1234-abcd")
# Repeat test in order to validate cache.
mock_novaclient.reset_mock()
action.run(test_ctx)
# TODO(d0ugal): Uncomment the following line when caching is fixed.
# mock_novaclient.Client.assert_not_called()
mock_novaclient.Client().servers.get.assert_called_with(
server="1234-abcd")
# Repeat again with different context for cache testing.
test_ctx.project_name = 'service'
test_ctx.project_id = '1235'
mock_novaclient.reset_mock()
action.run(test_ctx)
mock_novaclient.Client.assert_called_once_with(
2,
username=None,
api_key=None,
endpoint_type='public',
service_type='compute',
auth_token=test_ctx.auth_token,
tenant_id=test_ctx.project_id,
region_name=mock_nova_endpoint().region,
auth_url=mock_ks_endpoint_v2().url,
insecure=test_ctx.insecure
)
self.assertTrue(mock_novaclient.Client().servers.get.called)
mock_novaclient.Client().servers.get.assert_called_once_with(
server="1234-abcd")
@mock.patch.object(actions.GlanceAction, '_get_client')
def test_glance_action(self, mocked):
mock_ctx = mock.Mock()

52
mistral/utils/openstack/keystone.py
View File

@ -15,6 +15,7 @@
import keystoneauth1.identity.generic as auth_plugins
from keystoneauth1 import session as ks_session
from keystoneauth1.token_endpoint import Token
from keystoneclient import service_catalog as ks_service_catalog
from keystoneclient.v3 import client as ks_client
from keystoneclient.v3 import endpoints as ks_endpoints
@ -44,6 +45,52 @@ def client():
return cl
def _determine_verify(ctx):
if ctx.insecure:
return False
elif ctx.auth_cacert:
return ctx.auth_cacert
else:
return True
def get_session_and_auth(context, **kwargs):
"""Get session and auth parameters
:param context: action context
:return: dict to be used as kwargs for client serviceinitialization
"""
if not context:
raise AssertionError('context is mandatory')
project_endpoint = get_endpoint_for_project(**kwargs)
endpoint = format_url(
project_endpoint.url,
{
'tenant_id': context.project_id,
'project_id': context.project_id
}
)
auth = Token(endpoint=endpoint, token=context.auth_token)
auth_uri = context.auth_uri or CONF.keystone_authtoken.auth_uri
ks_auth = Token(
endpoint=auth_uri,
token=context.auth_token
)
session = ks_session.Session(
auth=ks_auth,
verify=_determine_verify(context)
)
return {
"session": session,
"auth": auth
}
def _admin_client(trust_id=None, project_name=None):
auth_url = CONF.keystone_authtoken.auth_uri
@ -144,10 +191,11 @@ def obtain_service_catalog(ctx):
)
trust_client = client_for_trusts(ctx.trust_id)
response = trust_client.tokens.get_token_data(
token_data = trust_client.tokens.get_token_data(
token,
include_catalog=True
)['token']
)
response = token_data['token']
else:
response = ctx.service_catalog