From 93993a0cedbe2105d7481fda0b1f83dee0a63fe4 Mon Sep 17 00:00:00 2001 From: Peter Wang Date: Mon, 27 Feb 2017 19:29:59 +0800 Subject: [PATCH] VNX: Add QoS support This patch adds QoS support by leveraging the NQM on the VNX. The supported qos specs are: * maxBWS, maximum bandwidth in MiB * maxIOPS, maximum IOPS DocImpact Implements: blueprint add-vnx-qos-support Change-Id: Iec7ce170e152bff0429de1922b9ccc26822007c8 --- .../volume/drivers/dell_emc/vnx/fake_enum.py | 9 ++ .../dell_emc/vnx/fake_storops/__init__.py | 3 + .../drivers/dell_emc/vnx/mocked_cinder.yaml | 17 ++++ .../drivers/dell_emc/vnx/mocked_vnx.yaml | 56 ++++++++++++ .../volume/drivers/dell_emc/vnx/res_mock.py | 5 +- .../drivers/dell_emc/vnx/test_adapter.py | 31 +++++-- .../drivers/dell_emc/vnx/test_client.py | 32 +++++++ .../volume/drivers/dell_emc/vnx/test_utils.py | 27 +++++- cinder/volume/drivers/dell_emc/vnx/adapter.py | 5 +- cinder/volume/drivers/dell_emc/vnx/client.py | 85 ++++++++++++++++++- cinder/volume/drivers/dell_emc/vnx/common.py | 5 ++ cinder/volume/drivers/dell_emc/vnx/driver.py | 7 +- cinder/volume/drivers/dell_emc/vnx/utils.py | 27 ++++++ .../vnx-qos-support-7057196782e2c388.yaml | 3 + 14 files changed, 294 insertions(+), 18 deletions(-) create mode 100644 releasenotes/notes/vnx-qos-support-7057196782e2c388.yaml diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_enum.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_enum.py index 40a755bc467..9a20687b6ec 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_enum.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_enum.py @@ -117,3 +117,12 @@ class Enum(enum.Enum): @classmethod def enum_name(cls): return cls.__name__ + + +class VNXCtrlMethod(object): + LIMIT_CTRL = 'limit' + + def __init__(self, method, metric, value): + self.method = method + self.metric = metric + self.value = value diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_storops/__init__.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_storops/__init__.py index 516f3a658f1..b50af7e37b7 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_storops/__init__.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/fake_storops/__init__.py @@ -74,3 +74,6 @@ class VNXMirrorImageState(VNXEnum): INCOMPLETE = 'Incomplete' LOCAL_ONLY = 'Local Only' EMPTY = 'Empty' + + +VNXCtrlMethod = fake_enum.VNXCtrlMethod diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_cinder.yaml b/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_cinder.yaml index 68aa325bd97..40377303df0 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_cinder.yaml +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_cinder.yaml @@ -83,6 +83,15 @@ test_create_volume_error: *test_create_volume test_create_thick_volume: *test_create_volume +test_create_volume_with_qos: + volume: + _type: 'volume' + _properties: + <<: *volume_base_properties + name: "volume_with_qos" + volume_type_id: + _uuid: volume_type_id + test_migrate_volume: volume: *volume_base @@ -468,6 +477,14 @@ test_calc_migrate_and_provision_image_cache: test_calc_migrate_and_provision: volume: *volume_base +test_get_backend_qos_specs: + volume: + _type: 'volume' + _properties: + <<: *volume_base_properties + volume_type_id: + _uuid: volume_type_id + ########################################################### # TestClient ########################################################### diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_vnx.yaml b/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_vnx.yaml index c0572051976..a0db1db9e22 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_vnx.yaml +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/mocked_vnx.yaml @@ -1011,6 +1011,53 @@ test_get_lun_id_without_provider_location: _methods: get_lun: *test_get_lun_id_without_provider_location +test_get_ioclass: + ioclass_false: &ioclass_false + _properties: + existed: False + ioclass_true: &ioclass_true + _properties: + existed: True + _methods: + add_lun: + vnx: + _methods: + get_ioclass: *ioclass_false + create_ioclass: *ioclass_true + +test_create_ioclass_iops: + vnx: + _methods: + create_ioclass: *ioclass_true + +test_create_ioclass_bws: + vnx: + _methods: + create_ioclass: *ioclass_true + +test_create_policy: + policy: &policy + _properties: + state: "Running" + existed: False + _methods: + add_class: + run_policy: + vnx: + _methods: + get_policy: *policy + create_policy: *policy + +test_get_running_policy: + vnx: + _methods: + get_policy: [*policy, *policy] + +test_add_lun_to_ioclass: + vnx: + _methods: + get_ioclass: *ioclass_true + ########################################################### # TestCommonAdapter ########################################################### @@ -1020,6 +1067,15 @@ test_create_volume_error: *test_create_lun_error test_create_thick_volume: *test_create_lun +test_create_volume_with_qos: + vnx: + _properties: + <<: *vnx_base_prop + _methods: + get_pool: *pool_test_create_lun + get_ioclass: *ioclass_true + get_policy: [*policy] + test_migrate_volume: lun: &src_lun_1 _properties: diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/res_mock.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/res_mock.py index 944a849dcdd..db889ca00f0 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/res_mock.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/res_mock.py @@ -16,6 +16,7 @@ import mock import six +from cinder import context from cinder.tests.unit.consistencygroup import fake_cgsnapshot from cinder.tests.unit.consistencygroup import fake_consistencygroup from cinder.tests.unit import fake_constants @@ -100,7 +101,7 @@ def _fake_volume_wrapper(*args, **kwargs): expected_attrs_key = {'volume_attachment': 'volume_attachment', 'volume_metadata': 'metadata'} return fake_volume.fake_volume_obj( - None, + context.get_admin_context(), expected_attrs=[ v for (k, v) in expected_attrs_key.items() if k in kwargs], **kwargs) @@ -111,7 +112,7 @@ def _fake_cg_wrapper(*args, **kwargs): def _fake_snapshot_wrapper(*args, **kwargs): - return fake_snapshot.fake_snapshot_obj(None, + return fake_snapshot.fake_snapshot_obj('fake_context', expected_attrs=( ['volume'] if 'volume' in kwargs else None), diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_adapter.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_adapter.py index 10f05ff8698..2cc499055f7 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_adapter.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_adapter.py @@ -56,15 +56,19 @@ class TestCommonAdapter(test.TestCase): def test_create_volume(self, vnx_common, _ignore, mocked_input): volume = mocked_input['volume'] volume.host.split('#')[1] - model_update = vnx_common.create_volume(volume) - self.assertEqual('False', model_update.get('metadata')['snapcopy']) + with mock.patch.object(vnx_utils, 'get_backend_qos_specs', + return_value=None): + model_update = vnx_common.create_volume(volume) + self.assertEqual('False', model_update.get('metadata')['snapcopy']) @res_mock.mock_driver_input @res_mock.patch_common_adapter def test_create_volume_error(self, vnx_common, _ignore, mocked_input): - self.assertRaises(storops_ex.VNXCreateLunError, - vnx_common.create_volume, - mocked_input['volume']) + def inner(): + with mock.patch.object(vnx_utils, 'get_backend_qos_specs', + return_value=None): + vnx_common.create_volume(mocked_input['volume']) + self.assertRaises(storops_ex.VNXCreateLunError, inner) @utils.patch_extra_specs({'provisioning:type': 'thick'}) @res_mock.mock_driver_input @@ -72,10 +76,24 @@ class TestCommonAdapter(test.TestCase): def test_create_thick_volume(self, vnx_common, _ignore, mocked_input): volume = mocked_input['volume'] expected_pool = volume.host.split('#')[1] - vnx_common.create_volume(volume) + with mock.patch.object(vnx_utils, 'get_backend_qos_specs', + return_value=None): + vnx_common.create_volume(volume) vnx_common.client.vnx.get_pool.assert_called_with( name=expected_pool) + @utils.patch_extra_specs({'provisioning:type': 'thin'}) + @res_mock.mock_driver_input + @res_mock.patch_common_adapter + def test_create_volume_with_qos(self, vnx_common, _ignore, mocked_input): + volume = mocked_input['volume'] + with mock.patch.object(vnx_utils, 'get_backend_qos_specs', + return_value={'id': 'test', + 'maxBWS': 100, + 'maxIOPS': 123}): + model_update = vnx_common.create_volume(volume) + self.assertEqual('False', model_update.get('metadata')['snapcopy']) + @res_mock.mock_driver_input @res_mock.patch_common_adapter def test_migrate_volume(self, vnx_common, mocked, cinder_input): @@ -317,6 +335,7 @@ class TestCommonAdapter(test.TestCase): self.assertEqual(2, len(pool_stats)) for stat in pool_stats: self.assertTrue(stat['fast_cache_enabled']) + self.assertTrue(stat['QoS_support']) self.assertIn(stat['pool_name'], [pools[0].name, pools[1].name]) self.assertFalse(stat['replication_enabled']) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_client.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_client.py index 412fe60a4a3..8871e0b1995 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_client.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_client.py @@ -476,3 +476,35 @@ class TestClient(test.TestCase): lun_id = client.get_lun_id(cinder_input['volume']) self.assertIsInstance(lun_id, int) self.assertEqual(mocked['lun'].lun_id, lun_id) + + @res_mock.patch_client + def test_get_ioclass(self, client, mocked): + qos_specs = {'id': 'qos', vnx_common.QOS_MAX_IOPS: 10, + vnx_common.QOS_MAX_BWS: 100} + ioclasses = client.get_ioclass(qos_specs) + self.assertEqual(2, len(ioclasses)) + + @res_mock.patch_client + def test_create_ioclass_iops(self, client, mocked): + ioclass = client.create_ioclass_iops('test', 1000) + self.assertIsNotNone(ioclass) + + @res_mock.patch_client + def test_create_ioclass_bws(self, client, mocked): + ioclass = client.create_ioclass_bws('test', 100) + self.assertIsNotNone(ioclass) + + @res_mock.patch_client + def test_create_policy(self, client, mocked): + policy = client.create_policy('policy_name') + self.assertIsNotNone(policy) + + @res_mock.patch_client + def test_get_running_policy(self, client, mocked): + policy, is_new = client.get_running_policy() + self.assertEqual(policy.state in ['Running', 'Measuring'], True) + self.assertFalse(is_new) + + @res_mock.patch_client + def test_add_lun_to_ioclass(self, client, mocked): + client.add_lun_to_ioclass('test_ioclass', 1) diff --git a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_utils.py b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_utils.py index c6d860e00e0..385d88f37d0 100644 --- a/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_utils.py +++ b/cinder/tests/unit/volume/drivers/dell_emc/vnx/test_utils.py @@ -21,7 +21,7 @@ from cinder.tests.unit.volume.drivers.dell_emc.vnx import fake_exception \ from cinder.tests.unit.volume.drivers.dell_emc.vnx import fake_storops \ as storops from cinder.tests.unit.volume.drivers.dell_emc.vnx import res_mock -from cinder.tests.unit.volume.drivers.dell_emc.vnx import utils +from cinder.tests.unit.volume.drivers.dell_emc.vnx import utils as ut_utils from cinder.volume.drivers.dell_emc.vnx import common from cinder.volume.drivers.dell_emc.vnx import utils as vnx_utils @@ -172,7 +172,7 @@ class TestUtils(test.TestCase): 'wwn2_2': ['wwnt_1', 'wwnt_3']}, itor_tgt_map) - @utils.patch_group_specs(' True') + @ut_utils.patch_group_specs(' True') @res_mock.mock_driver_input def test_require_consistent_group_snapshot_enabled(self, input): driver = FakeDriver() @@ -210,3 +210,26 @@ class TestUtils(test.TestCase): self.assertEqual(vnx_utils.is_async_migrate_enabled(volume), async_migrate) self.assertEqual(provision.name, 'THICK') + + @ut_utils.patch_extra_specs({}) + @res_mock.mock_driver_input + def test_get_backend_qos_specs(self, cinder_input): + volume = mock.Mock() + volume.volume_type.qos_specs = mock.Mock() + volume.volume_type.qos_specs.__getitem__ = mock.Mock(return_value=None) + r = vnx_utils.get_backend_qos_specs(volume) + self.assertIsNone(r) + + volume.volume_type.qos_specs.__getitem__ = mock.Mock( + return_value={'consumer': 'frontend'}) + r = vnx_utils.get_backend_qos_specs(volume) + self.assertIsNone(r) + + volume.volume_type.qos_specs.__getitem__ = mock.Mock( + return_value={'id': 'test', 'consumer': 'back-end', + 'specs': {common.QOS_MAX_BWS: 100, + common.QOS_MAX_IOPS: 10}}) + r = vnx_utils.get_backend_qos_specs(volume) + self.assertIsNotNone(r) + self.assertEqual(100, r[common.QOS_MAX_BWS]) + self.assertEqual(10, r[common.QOS_MAX_IOPS]) diff --git a/cinder/volume/drivers/dell_emc/vnx/adapter.py b/cinder/volume/drivers/dell_emc/vnx/adapter.py index 6d72847c337..5cb3de6a3e5 100644 --- a/cinder/volume/drivers/dell_emc/vnx/adapter.py +++ b/cinder/volume/drivers/dell_emc/vnx/adapter.py @@ -237,11 +237,13 @@ class CommonAdapter(object): 'provision': provision, 'tier': tier}) + qos_specs = utils.get_backend_qos_specs(volume) cg_id = volume.group_id lun = self.client.create_lun( pool, volume_name, volume_size, provision, tier, cg_id, - ignore_thresholds=self.config.ignore_pool_full_threshold) + ignore_thresholds=self.config.ignore_pool_full_threshold, + qos_specs=qos_specs) location = self._build_provider_location( lun_type='lun', lun_id=lun.lun_id, @@ -747,6 +749,7 @@ class CommonAdapter(object): stats['consistent_group_snapshot_enabled']) pool_stats['max_over_subscription_ratio'] = ( self.max_over_subscription_ratio) + pool_stats['QoS_support'] = True # Add replication v2.1 support self.append_replication_stats(pool_stats) pools_stats.append(pool_stats) diff --git a/cinder/volume/drivers/dell_emc/vnx/client.py b/cinder/volume/drivers/dell_emc/vnx/client.py index 21d74118cff..12858335873 100644 --- a/cinder/volume/drivers/dell_emc/vnx/client.py +++ b/cinder/volume/drivers/dell_emc/vnx/client.py @@ -50,9 +50,8 @@ class Condition(object): # Quick exit wait_until when the lun is other state to avoid # long-time timeout. msg = (_('Volume %(name)s was created in VNX, ' - 'but in %(state)s state.') - % {'name': lun.name, - 'state': lun_state}) + 'but in %(state)s state.') % { + 'name': lun.name, 'state': lun_state}) raise exception.VolumeBackendAPIException(data=msg) @staticmethod @@ -98,7 +97,8 @@ class Client(object): LOG.info('PQueue[%s] starts now.', queue_path) def create_lun(self, pool, name, size, provision, - tier, cg_id=None, ignore_thresholds=False): + tier, cg_id=None, ignore_thresholds=False, + qos_specs=None): pool = self.vnx.get_pool(name=pool) try: lun = pool.create_lun(lun_name=name, @@ -113,6 +113,14 @@ class Client(object): if cg_id: cg = self.vnx.get_cg(name=cg_id) cg.add_member(lun) + ioclasses = self.get_ioclass(qos_specs) + if ioclasses: + policy, is_new = self.get_running_policy() + for one in ioclasses: + one.add_lun(lun) + policy.add_class(one) + if is_new: + policy.run_policy() return lun def get_lun(self, name=None, lun_id=None): @@ -598,3 +606,72 @@ class Client(object): lun = self.get_lun(name=lun_name) utils.update_res_without_poll(lun) return lun.pool_name + + def get_ioclass(self, qos_specs): + ioclasses = [] + if qos_specs is not None: + prefix = qos_specs['id'] + max_bws = qos_specs[common.QOS_MAX_BWS] + max_iops = qos_specs[common.QOS_MAX_IOPS] + if max_bws: + name = '%(prefix)s-bws-%(max)s' % { + 'prefix': prefix, 'max': max_bws} + class_bws = self.vnx.get_ioclass(name=name) + if not class_bws.existed: + class_bws = self.create_ioclass_bws(name, + max_bws) + ioclasses.append(class_bws) + if max_iops: + name = '%(prefix)s-iops-%(max)s' % { + 'prefix': prefix, 'max': max_iops} + class_iops = self.vnx.get_ioclass(name=name) + if not class_iops.existed: + class_iops = self.create_ioclass_iops(name, + max_iops) + ioclasses.append(class_iops) + return ioclasses + + def create_ioclass_iops(self, name, max_iops): + """Creates a ioclass by IOPS.""" + max_iops = int(max_iops) + ctrl_method = storops.VNXCtrlMethod( + method=storops.VNXCtrlMethod.LIMIT_CTRL, + metric='tt', value=max_iops) + ioclass = self.vnx.create_ioclass(name=name, iotype='rw', + ctrlmethod=ctrl_method) + return ioclass + + def create_ioclass_bws(self, name, max_bws): + """Creates a ioclass by bandwidth in MiB.""" + max_bws = int(max_bws) + ctrl_method = storops.VNXCtrlMethod( + method=storops.VNXCtrlMethod.LIMIT_CTRL, + metric='bw', value=max_bws) + ioclass = self.vnx.create_ioclass(name=name, iotype='rw', + ctrlmethod=ctrl_method) + return ioclass + + def create_policy(self, policy_name): + """Creates the policy and starts it.""" + policy = self.vnx.get_policy(name=policy_name) + if not policy.existed: + LOG.info('Creating the policy: %s', policy_name) + policy = self.vnx.create_policy(name=policy_name) + return policy + + def get_running_policy(self): + """Returns the only running/measuring policy on VNX. + + .. note: VNX only allows one running policy. + """ + policies = self.vnx.get_policy() + policies = list(filter(lambda p: p.state == "Running" or p.state == + "Measuring", policies)) + if len(policies) >= 1: + return policies[0], False + else: + return self.create_policy("vnx_policy"), True + + def add_lun_to_ioclass(self, ioclass_name, lun_id): + ioclass = self.vnx.get_ioclass(name=ioclass_name) + ioclass.add_lun(lun_id) diff --git a/cinder/volume/drivers/dell_emc/vnx/common.py b/cinder/volume/drivers/dell_emc/vnx/common.py index ad94645422c..f4c45c2184e 100644 --- a/cinder/volume/drivers/dell_emc/vnx/common.py +++ b/cinder/volume/drivers/dell_emc/vnx/common.py @@ -41,6 +41,11 @@ INTERVAL_60_SEC = 60 SNAP_EXPIRATION_HOUR = '1h' +BACKEND_QOS_CONSUMERS = frozenset(['back-end', 'both']) +QOS_MAX_IOPS = 'maxIOPS' +QOS_MAX_BWS = 'maxBWS' + + VNX_OPTS = [ cfg.StrOpt('storage_vnx_authentication_type', default='global', diff --git a/cinder/volume/drivers/dell_emc/vnx/driver.py b/cinder/volume/drivers/dell_emc/vnx/driver.py index e14bcbd2cf4..962cd88f693 100644 --- a/cinder/volume/drivers/dell_emc/vnx/driver.py +++ b/cinder/volume/drivers/dell_emc/vnx/driver.py @@ -70,11 +70,12 @@ class VNXDriver(driver.ManageableVD, Configurable migration rate support 8.0.0 - New VNX Cinder driver 9.0.0 - Use asynchronous migration for cloning - 10.0.1 - Extend SMP size before aync migration when cloning from an - image cache volume + 10.0.0 - Extend SMP size before aync migration when cloning from an + image cache volume + 10.1.0 - Add QoS support """ - VERSION = '10.00.01' + VERSION = '10.01.00' VENDOR = 'Dell EMC' # ThirdPartySystems wiki page CI_WIKI_NAME = "EMC_VNX_CI" diff --git a/cinder/volume/drivers/dell_emc/vnx/utils.py b/cinder/volume/drivers/dell_emc/vnx/utils.py index 9d3c699cc6c..f7ceea32e78 100644 --- a/cinder/volume/drivers/dell_emc/vnx/utils.py +++ b/cinder/volume/drivers/dell_emc/vnx/utils.py @@ -380,3 +380,30 @@ def calc_migrate_and_provision(volume): else: specs = common.ExtraSpecs.from_volume(volume) return is_async_migrate_enabled(volume), specs.provision + + +def get_backend_qos_specs(volume): + qos_specs = volume.volume_type.qos_specs + if qos_specs is None: + return None + + qos_specs = qos_specs['qos_specs'] + if qos_specs is None: + return None + + consumer = qos_specs['consumer'] + # Front end QoS specs are handled by nova. Just ignore them here. + if consumer not in common.BACKEND_QOS_CONSUMERS: + return None + + max_iops = qos_specs['specs'].get(common.QOS_MAX_IOPS) + max_bws = qos_specs['specs'].get(common.QOS_MAX_BWS) + + if max_iops is None and max_bws is None: + return None + + return { + 'id': qos_specs['id'], + common.QOS_MAX_IOPS: max_iops, + common.QOS_MAX_BWS: max_bws, + } diff --git a/releasenotes/notes/vnx-qos-support-7057196782e2c388.yaml b/releasenotes/notes/vnx-qos-support-7057196782e2c388.yaml new file mode 100644 index 00000000000..f328c618358 --- /dev/null +++ b/releasenotes/notes/vnx-qos-support-7057196782e2c388.yaml @@ -0,0 +1,3 @@ +--- +features: + - Adds QoS support for VNX Cinder driver.