Merge "Retype support for CloudByte iSCSI cinder driver"

cinder/tests/unit/test_cloudbyte.py

@@ -25,9 +25,12 @@ import mock
 import testtools
 from testtools import matchers
 
+from cinder import context
 from cinder import exception
 from cinder.volume import configuration as conf
 from cinder.volume.drivers.cloudbyte import cloudbyte
+from cinder.volume import qos_specs
+from cinder.volume import volume_types
 
 # A fake list account response of cloudbyte's elasticenter
 FAKE_LIST_ACCOUNT_RESPONSE = """{ "listAccountResponse" : {
@@ -650,6 +653,7 @@ class CloudByteISCSIDriverTestCase(testtools.TestCase):
     def setUp(self):
         super(CloudByteISCSIDriverTestCase, self).setUp()
         self._configure_driver()
+        self.ctxt = context.get_admin_context()
 
     def _configure_driver(self):
 
@@ -753,6 +757,14 @@ class CloudByteISCSIDriverTestCase(testtools.TestCase):
 
         return MAP_COMMAND_TO_FAKE_RESPONSE[cmd]
 
+    def _fake_api_req_to_list_filesystem(
+            self, cmd, params, version='1.0'):
+        """This is a side effect function."""
+        if cmd == 'listFileSystem':
+            return {"listFilesystemResponse": {"filesystem": [{}]}}
+
+        return MAP_COMMAND_TO_FAKE_RESPONSE[cmd]
+
     def _side_effect_api_req_to_list_vol_iscsi_service(
             self, cmd, params, version='1.0'):
         """This is a side effect function."""
@@ -824,6 +836,20 @@ class CloudByteISCSIDriverTestCase(testtools.TestCase):
 
         return volume_id
 
+    def _fake_get_volume_type(self, ctxt, type_id):
+        fake_type = {'qos_specs_id': 'fake-id',
+                     'extra_specs': {'qos:iops': '100000'},
+                     'id': 'fake-volume-type-id'}
+
+        return fake_type
+
+    def _fake_get_qos_spec(self, ctxt, spec_id):
+        fake_qos_spec = {'id': 'fake-qos-spec-id',
+                         'specs': {'iops': '1000',
+                                   'graceallowed': 'true',
+                                   'readonly': 'true'}}
+        return fake_qos_spec
+
     @mock.patch.object(cloudbyte.CloudByteISCSIDriver,
                        '_execute_and_get_response_details')
     def test_api_request_for_cloudbyte(self, mock_conn):
@@ -1030,7 +1056,8 @@ class CloudByteISCSIDriverTestCase(testtools.TestCase):
 
         volume = {
             'id': fake_volume_id,
-            'size': 22
+            'size': 22,
+            'volume_type_id': None
         }
 
         # Test - I
@@ -1469,3 +1496,84 @@ class CloudByteISCSIDriverTestCase(testtools.TestCase):
                 "backend API: No response was received from CloudByte "
                 "storage list tsm API call."):
             self.driver.get_volume_stats(refresh=True)
+
+    @mock.patch.object(cloudbyte.CloudByteISCSIDriver,
+                       '_api_request_for_cloudbyte')
+    @mock.patch.object(volume_types,
+                       'get_volume_type')
+    @mock.patch.object(qos_specs,
+                       'get_qos_specs')
+    def test_retype(self, get_qos_spec, get_volume_type, mock_api_req):
+
+        # prepare the input test data
+        fake_new_type = {'id': 'fake-new-type-id'}
+        fake_volume_id = self._get_fake_volume_id()
+
+        volume = {
+            'id': 'SomeID',
+            'provider_id': fake_volume_id
+        }
+
+        # configure the mocks with respective side-effects
+        mock_api_req.side_effect = self._side_effect_api_req
+        get_qos_spec.side_effect = self._fake_get_qos_spec
+        get_volume_type.side_effect = self._fake_get_volume_type
+
+        self.assertTrue(self.driver.retype(self.ctxt,
+                                           volume,
+                                           fake_new_type, None, None))
+
+        # assert the invoked api calls
+        self.assertEqual(3, mock_api_req.call_count)
+
+    @mock.patch.object(cloudbyte.CloudByteISCSIDriver,
+                       '_api_request_for_cloudbyte')
+    @mock.patch.object(volume_types,
+                       'get_volume_type')
+    @mock.patch.object(qos_specs,
+                       'get_qos_specs')
+    def test_retype_without_provider_id(self, get_qos_spec, get_volume_type,
+                                        mock_api_req):
+
+        # prepare the input test data
+        fake_new_type = {'id': 'fake-new-type-id'}
+        volume = {'id': 'SomeID'}
+
+        # configure the mocks with respective side-effects
+        mock_api_req.side_effect = self._side_effect_api_req
+        get_qos_spec.side_effect = self._fake_get_qos_spec
+        get_volume_type.side_effect = self._fake_get_volume_type
+
+        # Now run the test & assert the exception
+        self.assertRaises(exception.VolumeBackendAPIException,
+                          self.driver.retype,
+                          self.ctxt, volume, fake_new_type, None, None)
+
+    @mock.patch.object(cloudbyte.CloudByteISCSIDriver,
+                       '_api_request_for_cloudbyte')
+    @mock.patch.object(volume_types,
+                       'get_volume_type')
+    @mock.patch.object(qos_specs,
+                       'get_qos_specs')
+    def test_retype_without_filesystem(self, get_qos_spec, get_volume_type,
+                                       mock_api_req):
+
+        # prepare the input test data
+        fake_new_type = {'id': 'fake-new-type-id'}
+        fake_volume_id = self._get_fake_volume_id()
+
+        volume = {
+            'id': 'SomeID',
+            'provider_id': fake_volume_id
+        }
+
+        # configure the mocks with respective side-effects
+        mock_api_req.side_effect = self._side_effect_api_req
+        get_qos_spec.side_effect = self._fake_get_qos_spec
+        get_volume_type.side_effect = self._fake_get_volume_type
+        mock_api_req.side_effect = self._fake_api_req_to_list_filesystem
+
+        # Now run the test & assert the exception
+        self.assertRaises(exception.VolumeBackendAPIException,
+                          self.driver.retype,
+                          self.ctxt, volume, fake_new_type, None, None)

cinder/volume/drivers/cloudbyte/cloudbyte.py

@@ -23,10 +23,13 @@ import six
 from six.moves import http_client
 from six.moves import urllib
 
+from cinder import context
 from cinder import exception
 from cinder.i18n import _, _LE, _LI
 from cinder.volume.drivers.cloudbyte import options
 from cinder.volume.drivers.san import san
+from cinder.volume import qos_specs
+from cinder.volume import volume_types
 
 LOG = logging.getLogger(__name__)
 
@@ -39,9 +42,10 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         1.1.0 - Add chap support and minor bug fixes
         1.1.1 - Add wait logic for delete volumes
         1.1.2 - Update ig to None before delete volume
+        1.2.0 - Add retype support
     """
 
-    VERSION = '1.1.2'
+    VERSION = '1.2.0'
     volume_stats = {}
 
     def __init__(self, *args, **kwargs):
@@ -50,6 +54,8 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
             options.cloudbyte_add_qosgroup_opts)
         self.configuration.append_config_values(
             options.cloudbyte_create_volume_opts)
+        self.configuration.append_config_values(
+            options.cloudbyte_update_volume_opts)
         self.configuration.append_config_values(
             options.cloudbyte_connection_opts)
         self.cb_use_chap = self.configuration.use_chap_auth
@@ -182,35 +188,25 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         data = self._api_request_for_cloudbyte("listTsm", params)
         return data
 
-    def _override_params(self, default_dict, filtered_user_dict):
+    def _add_qos_group_request(self, volume, tsmid, volume_name,
-        """Override the default config values with user provided values."""
+                               qos_group_params):
-
+        # Prepare the user input params
-        if filtered_user_dict is None:
+        params = {
-            # Nothing to override
+            "name": "QoS_" + volume_name,
-            return default_dict
+            "tsmid": tsmid
-
+        }
-        for key, value in default_dict.items():
-            # Fill the user dict with default options based on condition
-            if filtered_user_dict.get(key) is None and value is not None:
-                filtered_user_dict[key] = value
-
-        return filtered_user_dict
-
-    def _add_qos_group_request(self, volume, tsmid, volume_name):
         # Get qos related params from configuration
-        params = self.configuration.cb_add_qosgroup
+        params.update(self.configuration.cb_add_qosgroup)
 
-        if params is None:
+        # Override the default configuration by qos specs
-            params = {}
+        if qos_group_params:
-
+            params.update(qos_group_params)
-        params['name'] = "QoS_" + volume_name
-        params['tsmid'] = tsmid
 
         data = self._api_request_for_cloudbyte("addQosGroup", params)
         return data
 
     def _create_volume_request(self, volume, datasetid, qosgroupid,
-                               tsmid, volume_name):
+                               tsmid, volume_name, file_system_params):
 
         size = volume.get('size')
         quotasize = six.text_type(size) + "G"
@@ -225,8 +221,11 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         }
 
         # Get the additional params from configuration
-        params = self._override_params(self.configuration.cb_create_volume,
+        params.update(self.configuration.cb_create_volume)
-                                       params)
+
+        # Override the default configuration by qos specs
+        if file_system_params:
+            params.update(file_system_params)
 
         data = self._api_request_for_cloudbyte("createVolume", params)
         return data
@@ -756,8 +755,40 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
                   {'vol': volume_id,
                    'ig': ig_name})
 
+    def _get_qos_by_volume_type(self, ctxt, type_id):
+        """Get the properties which can be QoS or file system related."""
+
+        update_qos_group_params = {}
+        update_file_system_params = {}
+
+        volume_type = volume_types.get_volume_type(ctxt, type_id)
+        qos_specs_id = volume_type.get('qos_specs_id')
+        extra_specs = volume_type.get('extra_specs')
+
+        if qos_specs_id is not None:
+            specs = qos_specs.get_qos_specs(ctxt, qos_specs_id)['specs']
+
+            # Override extra specs with specs
+            # Hence specs will prefer QoS than extra specs
+            extra_specs.update(specs)
+
+        for key, value in extra_specs.items():
+            if ':' in key:
+                fields = key.split(':')
+                key = fields[1]
+
+            if key in self.configuration.cb_update_qos_group:
+                update_qos_group_params[key] = value
+
+            elif key in self.configuration.cb_update_file_system:
+                update_file_system_params[key] = value
+
+        return update_qos_group_params, update_file_system_params
+
     def create_volume(self, volume):
 
+        qos_group_params = {}
+        file_system_params = {}
         tsm_name = self.configuration.cb_tsm_name
         account_name = self.configuration.cb_account_name
 
@@ -767,6 +798,13 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         # Set backend storage volume name using OpenStack volume id
         cb_volume_name = volume['id'].replace("-", "")
 
+        ctxt = context.get_admin_context()
+        type_id = volume['volume_type_id']
+
+        if type_id is not None:
+            qos_group_params, file_system_params = (
+                self._get_qos_by_volume_type(ctxt, type_id))
+
         LOG.debug("Will create a volume [%(cb_vol)s] in TSM [%(tsm)s] "
                   "at CloudByte storage w.r.t "
                   "OpenStack volume [%(stack_vol)s].",
@@ -781,7 +819,7 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         LOG.debug("Creating qos group for CloudByte volume [%s].",
                   cb_volume_name)
         qos_data = self._add_qos_group_request(
-            volume, tsm_details.get('tsmid'), cb_volume_name)
+            volume, tsm_details.get('tsmid'), cb_volume_name, qos_group_params)
 
         # Extract the qos group id from response
         qosgroupid = qos_data['addqosgroupresponse']['qosgroup']['id']
@@ -792,7 +830,7 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
         # Send a create volume request to CloudByte API
         vol_data = self._create_volume_request(
             volume, tsm_details.get('datasetid'), qosgroupid,
-            tsm_details.get('tsmid'), cb_volume_name)
+            tsm_details.get('tsmid'), cb_volume_name, file_system_params)
 
         # Since create volume is an async call;
         # need to confirm the creation before proceeding further
@@ -1134,3 +1172,51 @@ class CloudByteISCSIDriver(san.SanISCSIDriver):
             self.volume_stats = data
 
         return self.volume_stats
+
+    def retype(self, ctxt, volume, new_type, diff, host):
+        """Retypes a volume, QoS and file system update is only done."""
+
+        cb_volume_id = volume.get('provider_id')
+
+        if cb_volume_id is None:
+            message = _("Provider information w.r.t CloudByte storage "
+                        "was not found for OpenStack "
+                        "volume [%s].") % volume['id']
+
+            raise exception.VolumeBackendAPIException(message)
+
+        update_qos_group_params, update_file_system_params = (
+            self._get_qos_by_volume_type(ctxt, new_type['id']))
+
+        if update_qos_group_params:
+            list_file_sys_params = {'id': cb_volume_id}
+            response = self._api_request_for_cloudbyte(
+                'listFileSystem', list_file_sys_params)
+
+            response = response['listFilesystemResponse']
+            cb_volume_list = response['filesystem']
+            cb_volume = cb_volume_list[0]
+
+            if not cb_volume:
+                msg = (_("Volume [%(cb_vol)s] was not found at "
+                         "CloudByte storage corresponding to OpenStack "
+                         "volume [%(ops_vol)s].") %
+                       {'cb_vol': cb_volume_id, 'ops_vol': volume['id']})
+
+                raise exception.VolumeBackendAPIException(data=msg)
+
+            update_qos_group_params['id'] = cb_volume.get('groupid')
+
+            self._api_request_for_cloudbyte(
+                'updateQosGroup', update_qos_group_params)
+
+        if update_file_system_params:
+            update_file_system_params['id'] = cb_volume_id
+            self._api_request_for_cloudbyte(
+                'updateFileSystem', update_file_system_params)
+
+        LOG.info(_LI("Successfully updated CloudByte volume [%(cb_vol)s] "
+                     "corresponding to OpenStack volume [%(ops_vol)s]."),
+                 {'cb_vol': cb_volume_id, 'ops_vol': volume['id']})
+
+        return True

cinder/volume/drivers/cloudbyte/options.py

@@ -85,7 +85,18 @@ cloudbyte_create_volume_opts = [
                 help="These values will be used for CloudByte storage's "
                      "createVolume API call."), ]
 
+cloudbyte_update_volume_opts = [
+    cfg.ListOpt('cb_update_qos_group',
+                default=["iops", "latency", "graceallowed"],
+                help="These values will be used for CloudByte storage's "
+                     "updateQosGroup API call."),
+    cfg.ListOpt('cb_update_file_system',
+                default=["compression", "sync", "noofcopies", "readonly"],
+                help="These values will be used for CloudByte storage's "
+                     "updateFileSystem API call."), ]
+
 CONF = cfg.CONF
 CONF.register_opts(cloudbyte_add_qosgroup_opts)
 CONF.register_opts(cloudbyte_create_volume_opts)
 CONF.register_opts(cloudbyte_connection_opts)
+CONF.register_opts(cloudbyte_update_volume_opts)