diff --git a/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py b/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py index cb37e3ae557..b463bf903db 100644 --- a/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py +++ b/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py @@ -77,6 +77,7 @@ class StorwizeSVCManagementSimulator(object): self._rcrelationship_list = {} self._partnership_list = {} self._partnershipcandidate_list = {} + self._rcconsistgrp_list = {} self._system_list = {'storwize-svc-sim': {'id': '0123456789ABCDEF', 'name': 'storwize-svc-sim'}, 'aux-svc-sim': {'id': 'ABCDEF0123456789', @@ -160,7 +161,13 @@ class StorwizeSVCManagementSimulator(object): 'CMMVC5963E': ('', 'CMMVC5963E No direction has been defined.'), 'CMMVC5713E': ('', 'CMMVC5713E Some parameters are mutually ' 'exclusive.'), - + 'CMMVC5804E': ('', 'CMMVC5804E The action failed because an ' + 'object that was specified in the command ' + 'does not exist.'), + 'CMMVC6065E': ('', 'CMMVC6065E The action failed as the object ' + 'is not in a group.'), + 'CMMVC9012E': ('', 'CMMVC9012E The copy type differs from other ' + 'copies already in the consistency group.'), } self._fc_transitions = {'begin': {'make': 'idle_or_copied'}, 'idle_or_copied': {'prepare': 'preparing', @@ -238,6 +245,45 @@ class StorwizeSVCManagementSimulator(object): 'delete': 'end', 'delete_force': 'end'}, } + self._rccg_transitions = {'empty': {'add': 'inconsistent_stopped', + 'delete': 'end', + 'delete_force': 'end'}, + 'inconsistent_stopped': + {'start': 'inconsistent_copying', + 'stop': 'inconsistent_stopped', + 'delete': 'end', + 'delete_force': 'end'}, + 'inconsistent_copying': { + 'wait': 'consistent_synchronized', + 'start': 'inconsistent_copying', + 'stop': 'inconsistent_stopped', + 'delete': 'end', + 'delete_force': 'end'}, + 'consistent_synchronized': { + 'start': 'consistent_synchronized', + 'stop': 'consistent_stopped', + 'stop_access': 'idling', + 'delete': 'end', + 'delete_force': 'end'}, + 'consistent_stopped': + {'start': 'consistent_synchronized', + 'stop': 'consistent_stopped', + 'delete': 'end', + 'delete_force': 'end'}, + 'consistent_copying': { + 'start': 'consistent_copying', + 'stop': 'consistent_stopped', + 'stop_access': 'idling', + 'delete': 'end', + 'delete_force': 'end'}, + 'end': None, + 'idling': { + 'start': 'inconsistent_copying', + 'stop': 'inconsistent_stopped', + 'stop_access': 'idling', + 'delete': 'end', + 'delete_force': 'end'}, + } def _state_transition(self, function, fcmap): if (function == 'wait' and @@ -302,6 +348,7 @@ class StorwizeSVCManagementSimulator(object): 'force', 'nohdr', 'nofmtdisk', + 'noconsistgrp', 'global', 'access', 'start' @@ -1666,6 +1713,7 @@ port_speed!N/A for cg_id in self._fcconsistgrp_list.keys(): if self._fcconsistgrp_list[cg_id]['name'] == kwargs['obj']: fcconsistgrp = self._fcconsistgrp_list[cg_id] + break rows = [] rows.append(['id', six.text_type(cg_id)]) rows.append(['name', fcconsistgrp['name']]) @@ -2114,6 +2162,62 @@ port_speed!N/A else: return self._errors['CMMVC5753E'] + def _cmd_chrcrelationship(self, **kwargs): + if 'obj' not in kwargs: + return self._errors['CMMVC5701E'] + id_num = kwargs['obj'] + + try: + rcrel = self._rcrelationship_list[id_num] + except KeyError: + return self._errors['CMMVC5753E'] + + remove_from_rccg = True if 'noconsistgrp' in kwargs else False + add_to_rccg = True if 'consistgrp' in kwargs else False + if remove_from_rccg: + if rcrel['consistency_group_name']: + rccg_name = rcrel['consistency_group_name'] + else: + return self._errors['CMMVC6065E'] + elif add_to_rccg: + rccg_name = (kwargs['consistgrp'].strip('\'\"') + if 'consistgrp' in kwargs else None) + else: + return self._chrcrelationship_attr(**kwargs) + + try: + rccg = self._rcconsistgrp_list[rccg_name] + except KeyError: + return self._errors['CMMVC5753E'] + + if remove_from_rccg: + rcrel['consistency_group_name'] = '' + rcrel['consistency_group_id'] = '' + + if int(rccg['relationship_count']) > 0: + rccg['relationship_count'] = str( + int(rccg['relationship_count']) - 1) + if rccg['relationship_count'] == '0': + rccg['state'] = 'empty' + rccg['copy_type'] = 'empty_group' + else: + if rccg['copy_type'] == 'empty_group': + rccg['copy_type'] = rcrel['copy_type'] + elif rccg['copy_type'] != rcrel['copy_type']: + return self._errors['CMMVC9012E'] + + rcrel['consistency_group_name'] = rccg['name'] + rcrel['consistency_group_id'] = rccg['id'] + rccg['relationship_count'] = str( + int(rccg['relationship_count']) + 1) + if rccg['state'] == 'empty': + rccg['state'] = rcrel['state'] + rccg['primary'] = rcrel['primary'] + rccg['cycling_mode'] = rcrel['cycling_mode'] + rccg['cycle_period_seconds'] = rcrel['cycle_period_seconds'] + + return '', '' + def _cmd_rmrcrelationship(self, **kwargs): if 'obj' not in kwargs: return self._errors['CMMVC5701E'] @@ -2136,7 +2240,7 @@ port_speed!N/A return ('', '') - def _cmd_chrcrelationship(self, **kwargs): + def _chrcrelationship_attr(self, **kwargs): if 'obj' not in kwargs: return self._errors['CMMVC5707E'] id_num = kwargs['obj'] @@ -2198,6 +2302,208 @@ port_speed!N/A except Exception: return self._errors['CMMVC5982E'] + def _rccg_state_transition(self, function, rccg): + if (function == 'wait' and + 'wait' not in self._rccg_transitions[rccg['state']]): + return ('', '') + + if rccg['state'] == 'inconsistent_copying' and function == 'wait': + if rccg['cycling_mode'] == storwize_const.GMCV_MULTI: + rccg['state'] = storwize_const.REP_CONSIS_COPYING + else: + rccg['state'] = storwize_const.REP_CONSIS_SYNC + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + rcrel_info['progress'] = '100' + rcrel_info['state'] = rccg['state'] + return ('', '') + else: + try: + curr_state = rccg['state'] + rccg['state'] = self._rccg_transitions[curr_state][function] + return ('', '') + except Exception: + return self._errors['CMMVC5982E'] + + def _cmd_mkrcconsistgrp(self, **kwargs): + master_sys = self._system_list['storwize-svc-sim'] + aux_sys = self._system_list['aux-svc-sim'] + if 'cluster' not in kwargs: + return self._errors['CMMVC5707E'] + aux_cluster = kwargs['cluster'].strip('\'\"') + if aux_cluster != aux_sys['name']: + return self._errors['CMMVC5754E'] + + rccg_info = {} + rccg_info['id'] = self._find_unused_id(self._rcconsistgrp_list) + + if 'name' in kwargs: + rccg_info['name'] = kwargs['name'].strip('\'\"') + else: + rccg_info['name'] = self.driver._get_rccg_name(None, + rccg_info['id']) + rccg_info['master_cluster_id'] = master_sys['id'] + rccg_info['master_cluster_name'] = master_sys['name'] + rccg_info['aux_cluster_id'] = aux_sys['id'] + rccg_info['aux_cluster_name'] = aux_sys['name'] + + rccg_info['primary'] = '' + rccg_info['state'] = 'empty' + rccg_info['relationship_count'] = '0' + + rccg_info['freeze_time'] = '' + rccg_info['status'] = '' + rccg_info['sync'] = '' + rccg_info['copy_type'] = 'empty_group' + rccg_info['cycling_mode'] = '' + rccg_info['cycle_period_seconds'] = '300' + self._rcconsistgrp_list[rccg_info['name']] = rccg_info + + return('RC Consistency Group, id [' + rccg_info['id'] + + '], successfully created', '') + + def _cmd_lsrcconsistgrp(self, **kwargs): + rows = [] + + if 'obj' not in kwargs: + rows.append(['id', 'name', 'master_cluster_id', + 'master_cluster_name', 'aux_cluster_id', + 'aux_cluster_name', 'primary', 'state', + 'relationship_count', 'copy_type', + 'cycling_mode', 'freeze_time']) + for rccg_info in self._rcconsistgrp_list.values(): + rows.append([rccg_info['id'], rccg_info['name'], + rccg_info['master_cluster_id'], + rccg_info['master_cluster_name'], + rccg_info['aux_cluster_id'], + rccg_info['aux_cluster_name'], + rccg_info['primary'], rccg_info['state'], + rccg_info['relationship_count'], + rccg_info['copy_type'], rccg_info['cycling_mode'], + rccg_info['freeze_time']]) + return self._print_info_cmd(rows=rows, **kwargs) + else: + try: + rccg_info = self._rcconsistgrp_list[kwargs['obj']] + except KeyError: + return self._errors['CMMVC5804E'] + + rows = [] + rows.append(['id', rccg_info['id']]) + rows.append(['name', rccg_info['name']]) + rows.append(['master_cluster_id', rccg_info['master_cluster_id']]) + rows.append(['master_cluster_name', + rccg_info['master_cluster_name']]) + rows.append(['aux_cluster_id', rccg_info['aux_cluster_id']]) + rows.append(['aux_cluster_name', rccg_info['aux_cluster_name']]) + rows.append(['primary', rccg_info['primary']]) + rows.append(['state', rccg_info['state']]) + rows.append(['relationship_count', + rccg_info['relationship_count']]) + rows.append(['freeze_time', rccg_info['freeze_time']]) + rows.append(['status', rccg_info['status']]) + rows.append(['sync', rccg_info['sync']]) + rows.append(['copy_type', rccg_info['copy_type']]) + rows.append(['cycling_mode', rccg_info['cycling_mode']]) + rows.append(['cycle_period_seconds', + rccg_info['cycle_period_seconds']]) + + if 'delim' in kwargs: + for index in range(len(rows)): + rows[index] = kwargs['delim'].join(rows[index]) + return ('%s' % '\n'.join(rows), '') + + def _cmd_startrcconsistgrp(self, **kwargs): + if 'obj' not in kwargs: + return self._errors['CMMVC5701E'] + id_num = kwargs['obj'] + + primary = (kwargs['primary'].strip('\'\"') if 'primary' + in kwargs else None) + try: + rccg = self._rcconsistgrp_list[id_num] + except KeyError: + return self._errors['CMMVC5753E'] + + if rccg['state'] == 'idling' and not primary: + return self._errors['CMMVC5963E'] + + self._rccg_state_transition('start', rccg) + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg: + self._rc_state_transition('start', rcrel_info) + if primary: + self._rcconsistgrp_list[id_num]['primary'] = primary + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + rcrel_info['primary'] = primary + return ('', '') + + def _cmd_stoprcconsistgrp(self, **kwargs): + if 'obj' not in kwargs: + return self._errors['CMMVC5701E'] + id_num = kwargs['obj'] + force_access = True if 'access' in kwargs else False + + try: + rccg = self._rcconsistgrp_list[id_num] + except KeyError: + return self._errors['CMMVC5753E'] + + function = 'stop_access' if force_access else 'stop' + self._rccg_state_transition(function, rccg) + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + self._rc_state_transition(function, rcrel_info) + if force_access: + self._rcconsistgrp_list[id_num]['primary'] = '' + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + rcrel_info['primary'] = '' + return ('', '') + + def _cmd_switchrcconsistgrp(self, **kwargs): + if 'obj' not in kwargs: + return self._errors['CMMVC5707E'] + id_num = kwargs['obj'] + + try: + rccg = self._rcconsistgrp_list[id_num] + except KeyError: + return self._errors['CMMVC5753E'] + + if (rccg['state'] == storwize_const.REP_CONSIS_SYNC or + (rccg['cycling_mode'] == storwize_const.GMCV_MULTI and + rccg['state'] == storwize_const.REP_CONSIS_COPYING)): + rccg['primary'] = kwargs['primary'] + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + rcrel_info['primary'] = kwargs['primary'] + return ('', '') + else: + return self._errors['CMMVC5753E'] + + def _cmd_rmrcconsistgrp(self, **kwargs): + if 'obj' not in kwargs: + return self._errors['CMMVC5701E'] + rccg_name = kwargs['obj'].strip('\'\"') + force = True if 'force' in kwargs else False + + try: + rccg = self._rcconsistgrp_list[rccg_name] + except KeyError: + return self._errors['CMMVC5804E'] + + function = 'delete_force' if force else 'delete' + self._rccg_state_transition(function, rccg) + if rccg['state'] == 'end': + for rcrel_info in self._rcrelationship_list.values(): + if rcrel_info['consistency_group_name'] == rccg['name']: + rcrel_info['consistency_group_name'] = '' + rcrel_info['consistency_group_id'] = '' + del self._rcconsistgrp_list[rccg_name] + return ('', '') + def _cmd_lspartnershipcandidate(self, **kwargs): rows = [None] * 4 master_sys = self._system_list['storwize-svc-sim'] @@ -5026,10 +5332,12 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): # Test groups operation #### @mock.patch('cinder.volume.utils.is_group_a_cg_snapshot_type') + @mock.patch('cinder.volume.utils.is_group_a_type') def test_storwize_group_create_with_replication( - self, is_grp_a_cg_snapshot_type): + self, is_grp_a_cg_rep_type, is_grp_a_cg_snapshot_type): """Test group create.""" - is_grp_a_cg_snapshot_type.side_effect = True + is_grp_a_cg_snapshot_type.side_effect = [True, True, False, False] + is_grp_a_cg_rep_type.side_effect = [True, True] spec = {'replication_enabled': ' True', 'replication_type': ' metro'} rep_type_ref = volume_types.create(self.ctxt, 'rep_type', spec) @@ -5041,30 +5349,49 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): self.assertEqual(fields.GroupStatus.ERROR, model_update['status']) - self.assertFalse(is_grp_a_cg_snapshot_type.called) + spec = {'replication_enabled': ' False'} + non_rep_type_ref = volume_types.create(self.ctxt, 'non_rep_type', spec) + non_rep_group = testutils.create_group( + self.ctxt, group_type_id=fake.GROUP_TYPE_ID, + volume_type_ids=[non_rep_type_ref['id']]) + + model_update = self.driver.create_group(self.ctxt, non_rep_group) + self.assertEqual(fields.GroupStatus.ERROR, + model_update['status']) @mock.patch('cinder.volume.utils.is_group_a_cg_snapshot_type') - def test_storwize_group_create(self, is_grp_a_cg_snapshot_type): + @mock.patch('cinder.volume.utils.is_group_a_type') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + 'create_rccg') + def test_storwize_group_create(self, create_rccg, is_grp_a_cg_rep_type, + is_grp_a_cg_snapshot_type): """Test group create.""" - is_grp_a_cg_snapshot_type.side_effect = [False, True] + is_grp_a_cg_snapshot_type.side_effect = [False, True, True] + is_grp_a_cg_rep_type.side_effect = [False, False] group = mock.MagicMock() self.assertRaises(NotImplementedError, self.driver.create_group, self.ctxt, group) model_update = self.driver.create_group(self.ctxt, group) + self.assertFalse(create_rccg.called) self.assertEqual(fields.GroupStatus.AVAILABLE, model_update['status']) @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall', new=testutils.ZeroIntervalLoopingCall) @mock.patch('cinder.volume.utils.is_group_a_cg_snapshot_type') - def test_storwize_delete_group(self, is_grp_a_cg_snapshot_type): + @mock.patch('cinder.volume.utils.is_group_a_type') + @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, + '_delete_replication_grp') + def test_storwize_delete_group(self, _del_rep_grp, is_grp_a_cg_rep_type, + is_grp_a_cg_snapshot_type): is_grp_a_cg_snapshot_type.side_effect = [False, True] + is_grp_a_cg_rep_type.side_effect = [False, False] type_ref = volume_types.create(self.ctxt, 'testtype', None) group = testutils.create_group(self.ctxt, group_type_id=fake.GROUP_TYPE_ID, - volume_type_id=type_ref['id']) + volume_type_ids=[type_ref['id']]) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) @@ -5075,25 +5402,35 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): self.ctxt, group, volumes) model_update = self.driver.delete_group(self.ctxt, group, volumes) + self.assertFalse(_del_rep_grp.called) self.assertEqual(fields.GroupStatus.DELETED, model_update[0]['status']) for volume in model_update[1]: self.assertEqual('deleted', volume['status']) @mock.patch('cinder.volume.utils.is_group_a_cg_snapshot_type') - def test_storwize_group_update(self, is_grp_a_cg_snapshot_type): + @mock.patch('cinder.volume.utils.is_group_a_type') + @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, + '_update_replication_grp') + def test_storwize_group_update(self, _update_rep_grp, is_grp_a_cg_rep_type, + is_grp_a_cg_snapshot_type): """Test group update.""" - is_grp_a_cg_snapshot_type.side_effect = [False, True] + is_grp_a_cg_snapshot_type.side_effect = [False, True, True, False] + is_grp_a_cg_rep_type.side_effect = [False, False, True, True] group = mock.MagicMock() self.assertRaises(NotImplementedError, self.driver.update_group, self.ctxt, group, None, None) (model_update, add_volumes_update, remove_volumes_update) = self.driver.update_group(self.ctxt, group) + self.assertFalse(_update_rep_grp.called) self.assertIsNone(model_update) self.assertIsNone(add_volumes_update) self.assertIsNone(remove_volumes_update) + self.driver.update_group(self.ctxt, group) + self.assertTrue(_update_rep_grp.called) + @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall', new=testutils.ZeroIntervalLoopingCall) @mock.patch('cinder.volume.utils.is_group_a_cg_snapshot_type') @@ -5102,7 +5439,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): type_ref = volume_types.create(self.ctxt, 'testtype', None) group = testutils.create_group(self.ctxt, group_type_id=fake.GROUP_TYPE_ID, - volume_type_id=type_ref['id']) + volume_type_ids=[type_ref['id']]) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) @@ -5131,7 +5468,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): type_ref = volume_types.create(self.ctxt, 'testtype', None) group = testutils.create_group(self.ctxt, group_type_id=fake.GROUP_TYPE_ID, - volume_type_id=type_ref['id']) + volume_type_ids=[type_ref['id']]) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) self._create_volume(volume_type_id=type_ref['id'], group_id=group.id) @@ -5154,19 +5491,29 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): def test_storwize_create_group_from_src_invalid(self): # Invalid input case for create group from src type_ref = volume_types.create(self.ctxt, 'testtype', None) - spec = {'consistent_group_snapshot_enabled': ' True'} - cg_type_ref = group_types.create(self.ctxt, 'cg_type', spec) + cg_spec = {'consistent_group_snapshot_enabled': ' True'} + rccg_spec = {'consistent_group_replication_enabled': ' True'} + cg_type_ref = group_types.create(self.ctxt, 'cg_type', cg_spec) + rccg_type_ref = group_types.create(self.ctxt, 'rccg_type', rccg_spec) vg_type_ref = group_types.create(self.ctxt, 'vg_type', None) # create group in db - group = self._create_group_in_db(volume_type_id=type_ref.id, + group = self._create_group_in_db(volume_type_ids=[type_ref.id], group_type_id=vg_type_ref.id) self.assertRaises(NotImplementedError, self.driver.create_group_from_src, self.ctxt, group, None, None, None, None, None) - group = self._create_group_in_db(volume_type_id=type_ref.id, + group = self._create_group_in_db(volume_type_ids=[type_ref.id], + group_type_id=rccg_type_ref.id) + vol1 = testutils.create_volume(self.ctxt, volume_type_id=type_ref.id, + group_id=group.id) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.create_group_from_src, + self.ctxt, group, [vol1]) + + group = self._create_group_in_db(volume_type_ids=[type_ref.id], group_type_id=cg_type_ref.id) # create volumes in db @@ -5176,7 +5523,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): group_id=group.id) volumes = [vol1, vol2] - source_cg = self._create_group_in_db(volume_type_id=type_ref.id, + source_cg = self._create_group_in_db(volume_type_ids=[type_ref.id], group_type_id=cg_type_ref.id) # Add volumes to source CG @@ -5250,7 +5597,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): cg_type_ref = group_types.create(self.ctxt, 'cg_type', spec) pool = _get_test_pool() # Create cg in db - group = self._create_group_in_db(volume_type_id=type_ref.id, + group = self._create_group_in_db(volume_type_ids=[type_ref.id], group_type_id=cg_type_ref.id) # Create volumes in db testutils.create_volume(self.ctxt, volume_type_id=type_ref.id, @@ -5263,7 +5610,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): self.ctxt.elevated(), group.id) # Create source CG - source_cg = self._create_group_in_db(volume_type_id=type_ref.id, + source_cg = self._create_group_in_db(volume_type_ids=[type_ref.id], group_type_id=cg_type_ref.id) # Add volumes to source CG self._create_volume(volume_type_id=type_ref.id, @@ -6222,6 +6569,7 @@ class StorwizeSVCReplicationTestCase(test.TestCase): self.driver.do_setup(None) self.driver.check_for_setup_error() self._create_test_volume_types() + self.rccg_type = self._create_consistent_rep_grp_type() def _set_flag(self, flag, value): group = self.driver.configuration.config_group @@ -6237,14 +6585,17 @@ class StorwizeSVCReplicationTestCase(test.TestCase): is_vol_defined = self.driver._helpers.is_vdisk_defined(name) self.assertEqual(exists, is_vol_defined) - def _generate_vol_info(self, vol_type=None, size=1): + def _generate_vol_info(self, vol_type=None, **kwargs): pool = _get_test_pool() volume_type = vol_type if vol_type else self.non_replica_type - prop = {'size': size, + prop = {'size': 1, 'volume_type_id': volume_type.id, 'host': 'openstack@svc#%s' % pool } - vol = testutils.create_volume(self.ctxt, **prop) + for p in prop.keys(): + if p not in kwargs: + kwargs[p] = prop[p] + vol = testutils.create_volume(self.ctxt, **kwargs) return vol def _generate_snap_info(self, vol_id): @@ -6304,11 +6655,27 @@ class StorwizeSVCReplicationTestCase(test.TestCase): True, rep_type=storwize_const.GMCV, cycle_period_seconds="86401") self.non_replica_type = self._create_replica_volume_type(False) - def _create_test_volume(self, rep_type): - volume = self._generate_vol_info(rep_type) + def _create_test_volume(self, rep_type, **kwargs): + volume = self._generate_vol_info(rep_type, **kwargs) model_update = self.driver.create_volume(volume) return volume, model_update + def _create_consistent_rep_grp_type(self): + rccg_spec = {'consistent_group_replication_enabled': ' True'} + rccg_type_ref = group_types.create(self.ctxt, 'cg_type', rccg_spec) + rccg_type = objects.GroupType.get_by_id(self.ctxt, rccg_type_ref['id']) + return rccg_type + + def _create_test_rccg(self, rccg_type, vol_type_ids): + # create group in db + group = testutils.create_group(self.ctxt, + volume_type_ids=vol_type_ids, + group_type_id=rccg_type.id) + if self.rccg_type == rccg_type: + group.replication_status = fields.ReplicationStatus.ENABLED + self.driver.create_group(self.ctxt, group) + return group + def _get_vdisk_uid(self, vdisk_name): vdisk_properties, _err = self.sim._cmd_lsvdisk(obj=vdisk_name, delim='!') @@ -6752,7 +7119,8 @@ class StorwizeSVCReplicationTestCase(test.TestCase): self.driver.do_setup(self.ctxt) volume, model_update = self._create_test_volume(self.non_replica_type) - self.assertIsNone(model_update) + self.assertEqual(fields.ReplicationStatus.NOT_CAPABLE, + model_update['replication_status']) # Retype to mm replica host = {'host': 'openstack@svc#openstack'} @@ -6770,8 +7138,8 @@ class StorwizeSVCReplicationTestCase(test.TestCase): # Create non-replica volume volume, model_update = self._create_test_volume(self.non_replica_type) - self.assertIsNone(model_update) - + self.assertEqual(fields.ReplicationStatus.NOT_CAPABLE, + model_update['replication_status']) # Retype to gmcv replica host = {'host': 'openstack@svc#openstack'} diff, _equal = volume_types.volume_types_diff( @@ -7090,7 +7458,8 @@ class StorwizeSVCReplicationTestCase(test.TestCase): non_replica_vol, model_update = self._create_test_volume( self.non_replica_type) - self.assertIsNone(model_update) + self.assertEqual(fields.ReplicationStatus.NOT_CAPABLE, + model_update['replication_status']) volumes = [volume, non_replica_vol, gmcv_volume] # Delete volume in failover state @@ -7262,10 +7631,7 @@ class StorwizeSVCReplicationTestCase(test.TestCase): @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, '_update_volume_stats') - @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, - '_update_storwize_state') def test_storwize_failover_host_replica_volumes(self, - update_storwize_state, update_volume_stats): self.driver.configuration.set_override('replication_device', [self.rep_target]) @@ -7280,53 +7646,159 @@ class StorwizeSVCReplicationTestCase(test.TestCase): gm_vol, model_update = self._create_test_volume(self.gm_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) + gm_vol['status'] = 'in-use' + + # Create global replication volume. + gm_vol1, model_update = self._create_test_volume(self.gm_type) + gm_vol1['status'] = 'in-use' + gm_vol1['previous_status'] = 'in-use' + + gm_vol2, model_update = self._create_test_volume(self.gm_type) + gm_vol2['status'] = 'in-use' + gm_vol2['previous_status'] = 'available' # Create gmcv volume. gmcv_vol, model_update = self._create_test_volume( self.gmcv_with_cps600_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) + gmcv_vol['status'] = 'available' + gmcv_vol['previous_status'] = 'in-use' - volumes = [mm_vol, gm_vol, gmcv_vol] - expected_list = [ - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': mm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': gm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': gmcv_vol['id']}] + volumes = [mm_vol, gm_vol, gm_vol1, gm_vol2, gmcv_vol] + expected_list = [{'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol1['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol2['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gmcv_vol['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gmcv_vol['id']} + ] - target_id, volume_list, __ = self.driver.failover_host( - self.ctxt, volumes, self.rep_target['backend_id'], []) + group1 = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + group2 = self._create_test_rccg(self.rccg_type, [self.gm_type.id]) + mm_vol1, model_update = self._create_test_volume( + self.mm_type, group_id=group1.id, status='available') + mm_vol2, model_update = self._create_test_volume( + self.mm_type, group_id=group1.id, status='in-use') + gm_vol3, model_update = self._create_test_volume( + self.gm_type, group_id=group2.id, + status='available', previous_status='in-use') + vols1 = [mm_vol1, mm_vol2] + self.driver.update_group(self.ctxt, group1, vols1, []) + mm_vol1.group = group1 + mm_vol2.group = group1 + group1.volumes = objects.VolumeList.get_all_by_generic_group(self.ctxt, + group1.id) + vols2 = [gm_vol3] + self.driver.update_group(self.ctxt, group2, vols2, []) + gm_vol3.group = group2 + group2.volumes = objects.VolumeList.get_all_by_generic_group(self.ctxt, + group2.id) + rccg_name = self.driver._get_rccg_name(group1) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + rccg_name = self.driver._get_rccg_name(group2) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + volumes.extend(vols1) + volumes.extend(vols2) + expected_list1 = [{'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol1['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol2['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol3['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol3['id']}] + expected_list.extend(expected_list1) + grp_expected = [{'group_id': group1.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'status': fields.GroupStatus.AVAILABLE}}, + {'group_id': group2.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'status': fields.GroupStatus.AVAILABLE}} + ] + + target_id, volume_list, groups_update = self.driver.failover_host( + self.ctxt, volumes, self.rep_target['backend_id'], + [group1, group2]) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual(expected_list, volume_list) + self.assertEqual(grp_expected, groups_update) self.assertEqual(self.driver._active_backend_id, target_id) self.assertEqual(self.driver._aux_backend_helpers, self.driver._helpers) self.assertEqual([self.driver._replica_target['pool_name']], self.driver._get_backend_pools()) - self.assertTrue(update_storwize_state.called) + self.assertEqual(self.driver._state, self.driver._aux_state) self.assertTrue(update_volume_stats.called) - self.driver.delete_volume(mm_vol) - self.driver.delete_volume(gm_vol) self.driver.delete_volume(gmcv_vol) - target_id, volume_list, __ = self.driver.failover_host( + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, None, []) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual([], volume_list) + self.assertEqual([], groups_update) + + self.driver.delete_volume(mm_vol) + self.driver.delete_volume(gm_vol) + self.driver.delete_volume(gm_vol1) + self.driver.delete_volume(gm_vol2) + self.driver.delete_volume(gmcv_vol) + self.driver.delete_group(self.ctxt, group1, vols1) + self.driver.delete_group(self.ctxt, group2, vols2) @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, '_update_volume_stats') - @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, - '_update_storwize_state') def test_storwize_failover_host_normal_volumes(self, - update_storwize_state, update_volume_stats): self.driver.configuration.set_override('replication_device', [self.rep_target]) @@ -7336,19 +7808,20 @@ class StorwizeSVCReplicationTestCase(test.TestCase): mm_vol, model_update = self._create_test_volume(self.mm_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) - mm_vol['status'] = 'in-use' + mm_vol['status'] = 'error' # Create gmcv replication. gmcv_vol, model_update = self._create_test_volume( self.gmcv_with_cps600_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) - gmcv_vol['status'] = 'in-use' + gmcv_vol['status'] = 'error' # Create non-replication volume. non_replica_vol, model_update = self._create_test_volume( self.non_replica_type) - self.assertIsNone(model_update) + self.assertEqual(fields.ReplicationStatus.NOT_CAPABLE, + model_update['replication_status']) non_replica_vol['status'] = 'error' volumes = [mm_vol, gmcv_vol, non_replica_vol] @@ -7368,23 +7841,25 @@ class StorwizeSVCReplicationTestCase(test.TestCase): 'volume_id': non_replica_vol['id']}, ] - target_id, volume_list, __ = self.driver.failover_host( + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, self.rep_target['backend_id'], []) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual(expected_list, volume_list) + self.assertEqual([], groups_update) self.assertEqual(self.driver._active_backend_id, target_id) self.assertEqual(self.driver._aux_backend_helpers, self.driver._helpers) self.assertEqual([self.driver._replica_target['pool_name']], self.driver._get_backend_pools()) - self.assertTrue(update_storwize_state.called) + self.assertEqual(self.driver._state, self.driver._aux_state) self.assertTrue(update_volume_stats.called) - target_id, volume_list, __ = self.driver.failover_host( + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, None, []) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual([], volume_list) + self.assertEqual([], groups_update) # Delete non-replicate volume in a failover state self.assertRaises(exception.VolumeDriverException, self.driver.delete_volume, @@ -7404,25 +7879,21 @@ class StorwizeSVCReplicationTestCase(test.TestCase): stop_relationship, switch_relationship): replica_obj = self.driver._get_replica_obj(storwize_const.METRO) - fake_vol = {'id': '21345678-1234-5678-1234-567812345683', - 'name': 'fake-volume'} - target_vol = storwize_const.REPLICA_AUX_VOL_PREFIX + fake_vol['name'] + mm_vol, model_update = self._create_test_volume(self.mm_type) + target_vol = storwize_const.REPLICA_AUX_VOL_PREFIX + mm_vol.name context = mock.Mock get_relationship_info.side_effect = [{ 'aux_vdisk_name': 'replica-12345678-1234-5678-1234-567812345678', 'name': 'RC_name'}] switch_relationship.side_effect = exception.VolumeDriverException - replica_obj.failover_volume_host(context, fake_vol) + replica_obj.failover_volume_host(context, mm_vol) get_relationship_info.assert_called_once_with(target_vol) switch_relationship.assert_called_once_with('RC_name') stop_relationship.assert_called_once_with(target_vol, access=True) @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, '_update_volume_stats') - @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, - '_update_storwize_state') def test_storwize_failback_replica_volumes(self, - update_storwize_state, update_volume_stats): self.driver.configuration.set_override('replication_device', [self.rep_target]) @@ -7437,6 +7908,16 @@ class StorwizeSVCReplicationTestCase(test.TestCase): gm_vol, model_update = self._create_test_volume(self.gm_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) + gm_vol['status'] = 'in-use' + gm_vol['previous_status'] = '' + + gm_vol1, model_update = self._create_test_volume(self.gm_type) + gm_vol1['status'] = 'in-use' + gm_vol1['previous_status'] = 'in-use' + + gm_vol2, model_update = self._create_test_volume(self.gm_type) + gm_vol2['status'] = 'in-use' + gm_vol2['previous_status'] = 'available' # Create gmcv replication. gmcv_vol, model_update = self._create_test_volume( @@ -7444,64 +7925,221 @@ class StorwizeSVCReplicationTestCase(test.TestCase): self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) - volumes = [gm_vol, mm_vol, gmcv_vol] - failover_expect = [ - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': gm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': mm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': gmcv_vol['id']}] + volumes = [mm_vol, gm_vol, gm_vol1, gm_vol2, gmcv_vol] + failover_expect = [{'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol1['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol2['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gmcv_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gmcv_vol['id']} + ] + group1 = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + group2 = self._create_test_rccg(self.rccg_type, [self.gm_type.id]) + mm_vol1, model_update = self._create_test_volume( + self.mm_type, group_id=group1.id, status='available') + mm_vol2, model_update = self._create_test_volume( + self.mm_type, group_id=group1.id, status='in-use') + gm_vol3, model_update = self._create_test_volume( + self.gm_type, group_id=group2.id, + status='available', previous_status='in-use') + vols1 = [mm_vol1, mm_vol2] + self.driver.update_group(self.ctxt, group1, vols1, []) + mm_vol1.group = group1 + mm_vol2.group = group1 + group1.volumes = objects.VolumeList.get_all_by_generic_group(self.ctxt, + group1.id) + vols2 = [gm_vol3] + self.driver.update_group(self.ctxt, group2, vols2, []) + gm_vol3.group = group2 + group2.volumes = objects.VolumeList.get_all_by_generic_group(self.ctxt, + group2.id) + rccg_name = self.driver._get_rccg_name(group1) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + rccg_name = self.driver._get_rccg_name(group2) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + volumes.extend(vols1) + volumes.extend(vols2) + expected_list1 = [{'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol1['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': mm_vol2['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol3['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol3['id']}] + failover_expect.extend(expected_list1) + grp_expected = [{'group_id': group1.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'status': fields.GroupStatus.AVAILABLE}}, + {'group_id': group2.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'status': fields.GroupStatus.AVAILABLE}} + ] - failback_expect = [ - {'updates': - {'replication_status': fields.ReplicationStatus.ENABLED, - 'status': 'available'}, - 'volume_id': gm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.ENABLED, - 'status': 'available'}, - 'volume_id': mm_vol['id']}, - {'updates': - {'replication_status': fields.ReplicationStatus.ENABLED, - 'status': 'available'}, - 'volume_id': gmcv_vol['id']}] # Already failback - target_id, volume_list, __ = self.driver.failover_host( - self.ctxt, volumes, 'default', []) + target_id, volume_list, groups_update = self.driver.failover_host( + self.ctxt, volumes, 'default', [group1, group2]) self.assertIsNone(target_id) self.assertEqual([], volume_list) + self.assertEqual([], groups_update) # fail over operation - target_id, volume_list, __ = self.driver.failover_host( - self.ctxt, volumes, self.rep_target['backend_id'], []) + target_id, volume_list, groups_update = self.driver.failover_host( + self.ctxt, volumes, self.rep_target['backend_id'], + [group1, group2]) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual(failover_expect, volume_list) - self.assertTrue(update_storwize_state.called) + self.assertEqual(grp_expected, groups_update) + self.assertEqual(self.driver._state, self.driver._aux_state) self.assertTrue(update_volume_stats.called) + mm_vol['status'] = 'available' + mm_vol['previous_status'] = 'available' + gm_vol['status'] = 'available' + gm_vol['previous_status'] = 'in-use' + gm_vol1['status'] = 'in-use' + gm_vol1['previous_status'] = 'in-use' + gm_vol2['status'] = 'available' + gm_vol2['previous_status'] = 'in-use' + gmcv_vol['status'] = 'available' + gmcv_vol['previous_status'] = '' + failback_expect = [{'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': mm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': gm_vol['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': gm_vol1['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': gm_vol2['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': gmcv_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gmcv_vol['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': 'available', + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol1['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': 'in-use', + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': mm_vol2['id']}, + {'updates': {'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': 'available', + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol3['id']}] + grp_expected = [{'group_id': group1.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.ENABLED, + 'status': fields.GroupStatus.AVAILABLE}}, + {'group_id': group2.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.ENABLED, + 'status': fields.GroupStatus.AVAILABLE}} + ] # fail back operation - target_id, volume_list, __ = self.driver.failover_host( - self.ctxt, volumes, 'default', []) + target_id, volume_list, groups_update = self.driver.failover_host( + self.ctxt, volumes, 'default', [group1, group2]) self.assertEqual('default', target_id) self.assertEqual(failback_expect, volume_list) + self.assertEqual(grp_expected, groups_update) + self.assertIsNone(self.driver._active_backend_id) self.assertEqual(SVC_POOLS, self.driver._get_backend_pools()) - self.assertTrue(update_storwize_state.called) + self.assertEqual(self.driver._state, self.driver._master_state) self.assertTrue(update_volume_stats.called) self.driver.delete_volume(mm_vol) self.driver.delete_volume(gm_vol) + self.driver.delete_volume(gm_vol1) + self.driver.delete_volume(gm_vol2) self.driver.delete_volume(gmcv_vol) + self.driver.delete_group(self.ctxt, group1, vols1) + self.driver.delete_group(self.ctxt, group2, vols2) @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, '_update_volume_stats') - @mock.patch.object(storwize_svc_common.StorwizeSVCCommonDriver, - '_update_storwize_state') def test_storwize_failback_normal_volumes(self, - update_storwize_state, update_volume_stats): self.driver.configuration.set_override('replication_device', @@ -7512,7 +8150,9 @@ class StorwizeSVCReplicationTestCase(test.TestCase): mm_vol, model_update = self._create_test_volume(self.mm_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) - mm_vol['status'] = 'in-use' + self.assertEqual('enabled', model_update['replication_status']) + mm_vol['status'] = 'error' + gm_vol, model_update = self._create_test_volume(self.gm_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) @@ -7523,94 +8163,72 @@ class StorwizeSVCReplicationTestCase(test.TestCase): self.gmcv_default_type) self.assertEqual(fields.ReplicationStatus.ENABLED, model_update['replication_status']) - gmcv_vol['status'] = 'in-use' - - # Create non-replication volume. - non_replica_vol1, model_update = self._create_test_volume( - self.non_replica_type) - self.assertIsNone(model_update) - non_replica_vol2, model_update = self._create_test_volume( - self.non_replica_type) - self.assertIsNone(model_update) - non_replica_vol1['status'] = 'error' - non_replica_vol2['status'] = 'available' - - volumes = [mm_vol, gmcv_vol, non_replica_vol1, - non_replica_vol2, gm_vol] + gmcv_vol['status'] = 'error' + volumes = [mm_vol, gmcv_vol, gm_vol] rep_data0 = json.dumps({'previous_status': mm_vol['status']}) rep_data1 = json.dumps({'previous_status': gmcv_vol['status']}) - rep_data2 = json.dumps({'previous_status': non_replica_vol1['status']}) - rep_data3 = json.dumps({'previous_status': non_replica_vol2['status']}) - failover_expect = [ - {'updates': - {'replication_status': fields.ReplicationStatus.FAILED_OVER}, - 'volume_id': gm_vol['id']}, - {'updates': {'status': 'error', - 'replication_driver_data': rep_data0}, - 'volume_id': mm_vol['id']}, - {'updates': {'status': 'error', - 'replication_driver_data': rep_data1}, - 'volume_id': gmcv_vol['id']}, - {'updates': {'status': 'error', - 'replication_driver_data': rep_data2}, - 'volume_id': non_replica_vol1['id']}, - {'updates': {'status': 'error', - 'replication_driver_data': rep_data3}, - 'volume_id': non_replica_vol2['id']}] + + failover_expect = [{'updates': {'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': gm_vol['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + 'volume_id': gm_vol['id']}, + {'updates': {'status': 'error', + 'replication_driver_data': rep_data0}, + 'volume_id': mm_vol['id']}, + {'updates': {'status': 'error', + 'replication_driver_data': rep_data1}, + 'volume_id': gmcv_vol['id']}] # Already failback - target_id, volume_list, __ = self.driver.failover_host( + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, 'default', []) self.assertIsNone(target_id) self.assertEqual([], volume_list) + self.assertEqual([], groups_update) # fail over operation - target_id, volume_list, __ = self.driver.failover_host( + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, self.rep_target['backend_id'], []) self.assertEqual(self.rep_target['backend_id'], target_id) self.assertEqual(failover_expect, volume_list) - self.assertTrue(update_storwize_state.called) + self.assertEqual([], groups_update) + self.assertEqual(self.driver._state, self.driver._aux_state) self.assertTrue(update_volume_stats.called) # fail back operation mm_vol['replication_driver_data'] = json.dumps( - {'previous_status': 'in-use'}) - gmcv_vol['replication_driver_data'] = json.dumps( - {'previous_status': 'in-use'}) - non_replica_vol1['replication_driver_data'] = json.dumps( {'previous_status': 'error'}) - non_replica_vol2['replication_driver_data'] = json.dumps( - {'previous_status': 'available'}) + gmcv_vol['replication_driver_data'] = json.dumps( + {'previous_status': 'error'}) gm_vol['status'] = 'in-use' - rep_data4 = json.dumps({'previous_status': gm_vol['status']}) - failback_expect = [{'updates': {'status': 'in-use', + gm_vol['previous_status'] = 'in-use' + failback_expect = [{'updates': {'replication_status': 'enabled', + 'previous_status': gm_vol['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + 'volume_id': gm_vol['id']}, + {'updates': {'status': 'error', 'replication_driver_data': ''}, 'volume_id': mm_vol['id']}, - {'updates': {'status': 'in-use', - 'replication_driver_data': ''}, - 'volume_id': gmcv_vol['id']}, {'updates': {'status': 'error', 'replication_driver_data': ''}, - 'volume_id': non_replica_vol1['id']}, - {'updates': {'status': 'available', - 'replication_driver_data': ''}, - 'volume_id': non_replica_vol2['id']}, - {'updates': {'status': 'error', - 'replication_driver_data': rep_data4}, - 'volume_id': gm_vol['id']}] - target_id, volume_list, __ = self.driver.failover_host( + 'volume_id': gmcv_vol['id']}] + target_id, volume_list, groups_update = self.driver.failover_host( self.ctxt, volumes, 'default', []) self.assertEqual('default', target_id) self.assertEqual(failback_expect, volume_list) + self.assertEqual([], groups_update) self.assertIsNone(self.driver._active_backend_id) self.assertEqual(SVC_POOLS, self.driver._get_backend_pools()) - self.assertTrue(update_storwize_state.called) + self.assertEqual(self.driver._state, self.driver._master_state) self.assertTrue(update_volume_stats.called) self.driver.delete_volume(mm_vol) self.driver.delete_volume(gmcv_vol) - self.driver.delete_volume(non_replica_vol1) - self.driver.delete_volume(non_replica_vol2) @mock.patch.object(storwize_svc_common.StorwizeHelpers, 'get_system_info') @@ -7796,3 +8414,598 @@ class StorwizeSVCReplicationTestCase(test.TestCase): self.assertRaises(exception.VolumeBackendAPIException, self.driver._wait_replica_vol_ready, self.ctxt, gmcv_vol) + + # Replication groups operation + def test_storwize_rep_group_create(self): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + # create group in db + group = testutils.create_group(self.ctxt, + volume_type_ids=[self.mm_type.id], + group_type_id=self.rccg_type.id) + + model_update = self.driver.create_group(self.ctxt, group) + self.assertEqual(fields.GroupStatus.AVAILABLE, model_update['status']) + self.assertEqual(fields.ReplicationStatus.ENABLED, + model_update['replication_status']) + rccg_name = self.driver._get_rccg_name(group) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual(rccg['name'], rccg_name) + self.driver.delete_group(self.ctxt, group, []) + + def test_storwize_rep_group_delete(self): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + mm_vol1, model_update = self._create_test_volume(self.mm_type) + mm_vol2, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1, mm_vol2] + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + self.driver.update_group(self.ctxt, group, vols, []) + (model_update, volumes_model_update) = self.driver.delete_group( + self.ctxt, group, vols) + for vol in vols: + self.assertFalse(self.driver._helpers.is_vdisk_defined(vol.name)) + self.assertIsNone(self.driver._helpers.get_rccg( + self.driver._get_rccg_name(group))) + for vol_update in volumes_model_update: + self.assertEqual(fields.GroupStatus.DELETED, vol_update['status']) + self.assertEqual(fields.GroupStatus.DELETED, model_update['status']) + + def test_storwize_rep_group_update(self): + """Test group update.""" + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + mm_vol, model_update = self._create_test_volume(self.mm_type) + gm_vol, model_update = self._create_test_volume(self.gm_type) + add_vols = [mm_vol, gm_vol] + (model_update, add_volumes_update, + remove_volumes_update) = self.driver.update_group( + self.ctxt, group, add_vols, []) + self.assertEqual(model_update['status'], fields.GroupStatus.ERROR) + self.assertIsNone(add_volumes_update) + self.assertIsNone(remove_volumes_update) + self.driver.delete_group(self.ctxt, group, add_vols) + + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + rccg_name = self.driver._get_rccg_name(group) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + mm_vol2, model_update = self._create_test_volume(self.mm_type) + mm_vol3, model_update = self._create_test_volume(self.mm_type) + mm_vol4, model_update = self._create_test_volume(self.mm_type) + + add_vols = [mm_vol1, mm_vol2] + (model_update, add_volumes_update, + remove_volumes_update) = self.driver.update_group( + self.ctxt, group, add_vols, []) + self.assertEqual( + rccg_name, + self.driver._helpers.get_rccg_info(mm_vol1.name)['name']) + self.assertEqual( + rccg_name, + self.driver._helpers.get_rccg_info(mm_vol2.name)['name']) + self.assertEqual(model_update['status'], fields.GroupStatus.AVAILABLE) + self.assertIsNone(add_volumes_update) + self.assertIsNone(remove_volumes_update) + + add_vols = [mm_vol3, mm_vol4] + rmv_vols = [mm_vol1, mm_vol2] + (model_update, add_volumes_update, + remove_volumes_update) = self.driver.update_group( + self.ctxt, group, add_volumes=add_vols, remove_volumes=rmv_vols) + self.assertIsNone(self.driver._helpers.get_rccg_info(mm_vol1.name)) + self.assertIsNone(self.driver._helpers.get_rccg_info(mm_vol2.name)) + self.assertEqual( + rccg_name, + self.driver._helpers.get_rccg_info(mm_vol3.name)['name']) + self.assertEqual( + rccg_name, + self.driver._helpers.get_rccg_info(mm_vol4.name)['name']) + self.assertEqual(model_update['status'], fields.GroupStatus.AVAILABLE) + self.assertIsNone(add_volumes_update) + self.assertIsNone(remove_volumes_update) + self.driver.delete_group(self.ctxt, group, [mm_vol1, mm_vol2, + mm_vol3, mm_vol4]) + + @mock.patch.object(storwize_svc_common.StorwizeSSH, + 'startrcconsistgrp') + def test_storwize_enable_replication_error(self, startrccg): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + rccg_name = self.driver._get_rccg_name(group) + exp_mod_upd = {'replication_status': fields.ReplicationStatus.ENABLED} + exp_mod_upd_err = {'replication_status': + fields.ReplicationStatus.ERROR} + # enable replicaion on empty group + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, []) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual([], volumes_update) + self.assertFalse(startrccg.called) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1] + self.driver.update_group(self.ctxt, group, vols, []) + exp_vols_upd = [ + {'id': mm_vol1['id'], + 'replication_status': exp_mod_upd['replication_status']}] + exp_vols_upd_err = [ + {'id': mm_vol1['id'], + 'replication_status': exp_mod_upd_err['replication_status']}] + + with mock.patch.object(storwize_svc_common.StorwizeSSH, + 'lsrcconsistgrp', + side_effect=[None, {'primary': 'master', + 'relationship_count': '1'}, + {'primary': 'aux', + 'relationship_count': '1'}, + {'primary': 'master', + 'relationship_count': '1'}]): + startrccg.side_effect = [ + None, None, + exception.VolumeBackendAPIException(data='CMMVC6372W')] + + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual(exp_vols_upd_err, volumes_update) + self.assertFalse(startrccg.called) + + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd, model_update) + self.assertEqual(exp_vols_upd, volumes_update) + startrccg.assert_called_with(rccg_name, 'master') + + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd, model_update) + self.assertEqual(exp_vols_upd, volumes_update) + startrccg.assert_called_with(rccg_name, 'aux') + + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual(exp_vols_upd_err, volumes_update) + + def test_storwize_enable_replication(self): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + mm_vol2, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1, mm_vol2] + expect_model_update = {'replication_status': + fields.ReplicationStatus.ENABLED} + expect_vols_update = [ + {'id': mm_vol1['id'], + 'replication_status': expect_model_update['replication_status']}, + {'id': mm_vol2['id'], + 'replication_status': expect_model_update['replication_status']} + ] + self.driver.update_group(self.ctxt, group, vols, []) + model_update, volumes_update = self.driver.enable_replication( + self.ctxt, group, vols) + self.assertEqual(expect_model_update, model_update) + self.assertEqual(expect_vols_update, volumes_update) + rccg_name = self.driver._get_rccg_name(group) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual(rccg['primary'], 'master') + self.assertIn(rccg['state'], ['inconsistent_copying', + 'consistent_synchronized']) + self.driver.delete_group(self.ctxt, group, vols) + + @mock.patch.object(storwize_svc_common.StorwizeSSH, + 'stoprcconsistgrp') + def test_storwize_disable_replication_error(self, stoprccg): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + rccg_name = self.driver._get_rccg_name(group) + exp_mod_upd = {'replication_status': fields.ReplicationStatus.DISABLED} + exp_mod_upd_err = {'replication_status': + fields.ReplicationStatus.ERROR} + # disable replicarion on empty group + model_update, volumes_update = self.driver.disable_replication( + self.ctxt, group, []) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual([], volumes_update) + self.assertFalse(stoprccg.called) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1] + self.driver.update_group(self.ctxt, group, vols, []) + exp_vols_upd = [ + {'id': mm_vol1['id'], + 'replication_status': exp_mod_upd['replication_status']}] + exp_vols_upd_err = [ + {'id': mm_vol1['id'], + 'replication_status': exp_mod_upd_err['replication_status']}] + + with mock.patch.object(storwize_svc_common.StorwizeSSH, + 'lsrcconsistgrp', + side_effect=[None, {'name': rccg_name, + 'relationship_count': '1'}, + {'name': rccg_name, + 'relationship_count': '1'}]): + stoprccg.side_effect = [ + None, exception.VolumeBackendAPIException(data='CMMVC6372W')] + + model_update, volumes_update = self.driver.disable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual(exp_vols_upd_err, volumes_update) + self.assertFalse(stoprccg.called) + + model_update, volumes_update = self.driver.disable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd, model_update) + self.assertEqual(exp_vols_upd, volumes_update) + stoprccg.assert_called_with(rccg_name, False) + + model_update, volumes_update = self.driver.disable_replication( + self.ctxt, group, vols) + self.assertEqual(exp_mod_upd_err, model_update) + self.assertEqual(exp_vols_upd_err, volumes_update) + + def test_storwize_disable_replication(self): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + mm_vol2, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1, mm_vol2] + expect_model_update = {'replication_status': + fields.ReplicationStatus.DISABLED} + expect_vols_update = [ + {'id': mm_vol1['id'], + 'replication_status': expect_model_update['replication_status']}, + {'id': mm_vol2['id'], + 'replication_status': expect_model_update['replication_status']} + ] + self.driver.update_group(self.ctxt, group, vols, []) + model_update, volumes_update = self.driver.disable_replication( + self.ctxt, group, vols) + self.assertEqual(expect_model_update, model_update) + self.assertEqual(expect_vols_update, volumes_update) + rccg_name = self.driver._get_rccg_name(group) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertIn(rccg['state'], ['inconsistent_stopped', + 'consistent_stopped']) + self.driver.delete_group(self.ctxt, group, vols) + + def test_storwize_failover_group_error(self): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + # Create metro mirror replication. + mm_vol1, model_update = self._create_test_volume(self.mm_type) + mm_vol2, model_update = self._create_test_volume(self.mm_type) + vols = [mm_vol1, mm_vol2] + + self.driver._replica_enabled = False + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, group, + vols, self.rep_target['backend_id']) + self.driver._replica_enabled = True + self.assertRaises(exception.InvalidReplicationTarget, + self.driver.failover_replication, self.ctxt, group, + vols, self.fake_target['backend_id']) + + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, group, + vols, self.rep_target['backend_id']) + + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, group, + vols, storwize_const.FAILBACK_VALUE) + + with mock.patch.object(storwize_svc_common.StorwizeHelpers, + 'get_system_info') as get_sys_info: + get_sys_info.side_effect = [ + exception.VolumeBackendAPIException(data='CMMVC6071E'), + exception.VolumeBackendAPIException(data='CMMVC6071E')] + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, + group, vols, self.rep_target['backend_id']) + + self.driver._active_backend_id = self.rep_target['backend_id'] + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, + group, vols, 'default') + with mock.patch.object(storwize_svc_common.StorwizeSSH, + 'lsrcconsistgrp', side_effect=[None]): + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, + group, vols, self.rep_target['backend_id']) + self.driver.delete_group(self.ctxt, group, vols) + + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + 'switch_rccg') + def test_storwize_failover_group_without_action(self, switchrccg): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + mm_vol1, model_update = self._create_test_volume(self.mm_type) + self.driver.update_group(self.ctxt, group, [mm_vol1], []) + rccg_name = self.driver._get_rccg_name(group) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + + rccg['primary'] = 'aux' + model_update = self.driver._rep_grp_failover(self.ctxt, rccg, group) + self.assertIsNone(model_update) + self.assertFalse(switchrccg.called) + + rccg['primary'] = 'master' + model_update = self.driver._rep_grp_failback(self.ctxt, rccg, group) + self.assertIsNone(model_update) + self.assertFalse(switchrccg.called) + + self.driver.delete_group(self.ctxt, group, []) + + @ddt.data(({'replication_enabled': ' True', + 'replication_type': ' metro'}, 'test_rep_metro'), + ({'replication_enabled': ' True', + 'replication_type': ' gmcv'}, 'test_rep_gmcv')) + @ddt.unpack + def test_storwize_failover_replica_group(self, spec, type_name): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + type_ref = volume_types.create(self.ctxt, type_name, spec) + rep_type = objects.VolumeType.get_by_id(self.ctxt, type_ref['id']) + group = self._create_test_rccg(self.rccg_type, [rep_type.id]) + vol1, model_update = self._create_test_volume(rep_type) + vol2, model_update = self._create_test_volume(rep_type) + vol2['status'] = 'in-use' + vol3, model_update = self._create_test_volume(rep_type) + vol3['status'] = 'available' + vol3['previous_status'] = 'in-use' + vols = [vol1, vol2, vol3] + self.driver.update_group(self.ctxt, group, vols, []) + rccg_name = self.driver._get_rccg_name(group) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + expected_list = [{'id': vol1['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol1['status'], + 'status': 'available', + 'attach_status': fields.VolumeAttachStatus.DETACHED}, + {'id': vol2['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol2['status'], + 'status': 'available', + 'attach_status': fields.VolumeAttachStatus.DETACHED}, + {'id': vol3['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol3['status'], + 'status': 'in-use', + 'attach_status': fields.VolumeAttachStatus.ATTACHED}] + + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, self.rep_target['backend_id']) + self.assertEqual( + {'replication_status': fields.ReplicationStatus.FAILED_OVER}, + model_update) + self.assertEqual(expected_list, volumes_model_update) + self.assertIsNone(self.driver._active_backend_id) + self.assertEqual(self.driver._master_backend_helpers, + self.driver._helpers) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('aux', rccg['primary']) + + group.replication_status = fields.ReplicationStatus.FAILED_OVER + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, None) + self.assertIsNone(model_update) + self.assertIsNone(volumes_model_update) + self.assertIsNone(self.driver._active_backend_id) + self.assertEqual(self.driver._master_backend_helpers, + self.driver._helpers) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('aux', rccg['primary']) + + self.driver.delete_group(self.ctxt, group, vols) + + @mock.patch.object(storwize_svc_common.StorwizeSSH, + 'switchrcconsistgrp') + def test_failover_replica_group_by_force_access(self, switchrcconsistgrp): + self.driver.do_setup(self.ctxt) + group = self._create_test_rccg(self.rccg_type, [self.mm_type.id]) + mm_vol1, model_update = self._create_test_volume(self.mm_type) + self.driver.update_group(self.ctxt, group, [mm_vol1], []) + rccg_name = self.driver._get_rccg_name(group) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + switchrcconsistgrp.side_effect = [ + exception.VolumeBackendAPIException(data='CMMVC6071E'), + exception.VolumeBackendAPIException(data='CMMVC6071E')] + with mock.patch.object(storwize_svc_common.StorwizeSSH, + 'startrcconsistgrp') as startrcconsistgrp: + self.driver.failover_replication(self.ctxt, group, [mm_vol1], None) + switchrcconsistgrp.assert_called_once_with(rccg_name, True) + startrcconsistgrp.assert_called_once_with(rccg_name, 'aux') + + with mock.patch.object(storwize_svc_common.StorwizeSSH, + 'stoprcconsistgrp') as stoprccg: + stoprccg.side_effect = exception.VolumeBackendAPIException( + data='CMMVC6071E') + self.assertRaises(exception.UnableToFailOver, + self.driver.failover_replication, self.ctxt, + group, [mm_vol1], self.rep_target['backend_id']) + self.driver.delete_group(self.ctxt, group, [mm_vol1]) + + @ddt.data(({'replication_enabled': ' True', + 'replication_type': ' metro'}, 'test_rep_metro'), + ({'replication_enabled': ' True', + 'replication_type': ' gmcv'}, 'test_rep_gmcv_default')) + @ddt.unpack + def test_storwize_failback_replica_group(self, spec, type_name): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + type_ref = volume_types.create(self.ctxt, type_name, spec) + rep_type = objects.VolumeType.get_by_id(self.ctxt, type_ref['id']) + group = self._create_test_rccg(self.rccg_type, [rep_type.id]) + vol1, model_update = self._create_test_volume(rep_type) + vol2, model_update = self._create_test_volume(rep_type) + vol2['status'] = 'in-use' + vol3, model_update = self._create_test_volume(rep_type) + vol3['status'] = 'available' + vol3['previous_status'] = 'in-use' + vols = [vol1, vol2, vol3] + self.driver.update_group(self.ctxt, group, vols, []) + rccg_name = self.driver._get_rccg_name(group) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + failover_expect = [{'id': vol1['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol1['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + {'id': vol2['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol2['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + {'id': vol3['id'], + 'replication_status': + fields.ReplicationStatus.FAILED_OVER, + 'previous_status': vol3['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}] + + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, self.rep_target['backend_id']) + self.assertEqual( + {'replication_status': fields.ReplicationStatus.FAILED_OVER}, + model_update) + self.assertEqual(failover_expect, volumes_model_update) + self.assertIsNone(self.driver._active_backend_id) + self.assertEqual(self.driver._master_backend_helpers, + self.driver._helpers) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('aux', rccg['primary']) + + group.replication_status = fields.ReplicationStatus.FAILED_OVER + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, None) + self.assertIsNone(model_update) + self.assertIsNone(volumes_model_update) + self.assertIsNone(self.driver._active_backend_id) + self.assertEqual(self.driver._master_backend_helpers, + self.driver._helpers) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('aux', rccg['primary']) + self.sim._rccg_state_transition('wait', + self.sim._rcconsistgrp_list[rccg_name]) + + vol1['status'] = 'available' + vol1['previous_status'] = 'available' + vol2['status'] = 'available' + vol2['previous_status'] = 'in-use' + vol3['status'] = 'in-use' + vol3['previous_status'] = 'in-use' + failback_expect = [{'id': vol1['id'], + 'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': vol1['status'], + 'status': 'available', + 'attach_status': + fields.VolumeAttachStatus.DETACHED}, + {'id': vol2['id'], + 'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': vol2['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}, + {'id': vol3['id'], + 'replication_status': + fields.ReplicationStatus.ENABLED, + 'previous_status': vol3['status'], + 'status': 'in-use', + 'attach_status': + fields.VolumeAttachStatus.ATTACHED}] + self.driver._active_backend_id = self.rep_target['backend_id'] + + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, 'default') + self.assertEqual( + {'replication_status': fields.ReplicationStatus.ENABLED}, + model_update) + self.assertEqual(failback_expect, volumes_model_update) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('master', rccg['primary']) + + group.replication_status = fields.ReplicationStatus.ENABLED + model_update, volumes_model_update = self.driver.failover_replication( + self.ctxt, group, vols, 'default') + self.assertIsNone(model_update) + self.assertIsNone(volumes_model_update) + rccg = self.driver._helpers.get_rccg(rccg_name) + self.assertEqual('master', rccg['primary']) + + self.driver.delete_group(self.ctxt, group, vols) + + @mock.patch.object(storwize_svc_common.StorwizeSSH, + 'lsrcconsistgrp') + @mock.patch.object(storwize_svc_common.StorwizeSSH, + 'startrcconsistgrp') + def test_sync_replica_group_with_aux(self, startrccg, lsrccg): + self.driver.configuration.set_override('replication_device', + [self.rep_target]) + self.driver.do_setup(self.ctxt) + rccg_name = 'fakerccg' + + sync_state = {'state': storwize_const.REP_CONSIS_SYNC, + 'primary': 'fake', 'relationship_count': '1'} + + sync_copying_state = {'state': storwize_const.REP_CONSIS_COPYING, + 'primary': 'fake', 'relationship_count': '1'} + + disconn_state = {'state': storwize_const.REP_IDL_DISC, + 'primary': 'master', 'relationship_count': '1'} + + stop_state = {'state': storwize_const.REP_CONSIS_STOP, + 'primary': 'aux', 'relationship_count': '1'} + lsrccg.side_effect = [None, sync_state, sync_copying_state, + disconn_state, stop_state] + + self.driver._sync_with_aux_grp(self.ctxt, rccg_name) + self.assertFalse(startrccg.called) + + self.driver._sync_with_aux_grp(self.ctxt, rccg_name) + self.assertFalse(startrccg.called) + + self.driver._sync_with_aux_grp(self.ctxt, rccg_name) + self.assertFalse(startrccg.called) + + self.driver._sync_with_aux_grp(self.ctxt, rccg_name) + startrccg.assert_called_once_with(rccg_name, 'master') + + self.driver._sync_with_aux_grp(self.ctxt, rccg_name) + startrccg.assert_called_with(rccg_name, 'aux') diff --git a/cinder/volume/drivers/ibm/storwize_svc/replication.py b/cinder/volume/drivers/ibm/storwize_svc/replication.py index faa3e20b426..561e55630b2 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/replication.py +++ b/cinder/volume/drivers/ibm/storwize_svc/replication.py @@ -24,7 +24,6 @@ import six from cinder import exception from cinder.i18n import _ -from cinder.objects import fields from cinder import ssh_utils from cinder import utils from cinder.volume.drivers.ibm.storwize_svc import storwize_const @@ -101,8 +100,7 @@ class StorwizeSVCReplicationGlobalMirror(StorwizeSVCReplication): rel_info = self.target_helpers.get_relationship_info(target_vol) # Reverse the role of the primary and secondary volumes self.target_helpers.switch_relationship(rel_info['name']) - return {'replication_status': - fields.ReplicationStatus.FAILED_OVER} + return except Exception as e: LOG.exception('Unable to fail-over the volume %(id)s to the ' 'secondary back-end by switchrcrelationship ' @@ -113,8 +111,13 @@ class StorwizeSVCReplicationGlobalMirror(StorwizeSVCReplication): try: self.target_helpers.stop_relationship(target_vol, access=True) - return {'replication_status': - fields.ReplicationStatus.FAILED_OVER} + try: + self.target_helpers.start_relationship(target_vol, 'aux') + except exception.VolumeBackendAPIException as e: + LOG.error( + 'Error running startrcrelationship due to %(err)s.', + {'err': e}) + return except Exception as e: msg = (_('Unable to fail-over the volume %(id)s to the ' 'secondary back-end, error: %(error)s') % @@ -129,9 +132,7 @@ class StorwizeSVCReplicationGlobalMirror(StorwizeSVCReplication): try: self.target_helpers.switch_relationship(rel_info['name'], aux=False) - return {'replication_status': - fields.ReplicationStatus.ENABLED, - 'status': 'available'} + return except Exception as e: msg = (_('Unable to fail-back the volume:%(vol)s to the ' 'master back-end, error:%(error)s') % @@ -253,11 +254,14 @@ class StorwizeSVCReplicationGMCV(StorwizeSVCReplicationGlobalMirror): {'vref': vref['name']}) # Make the aux volume writeable. try: - self.target_helpers.stop_relationship( - storwize_const.REPLICA_AUX_VOL_PREFIX + vref['name'], - access=True) - return {'replication_status': - fields.ReplicationStatus.FAILED_OVER} + tgt_volume = storwize_const.REPLICA_AUX_VOL_PREFIX + vref.name + self.target_helpers.stop_relationship(tgt_volume, access=True) + try: + self.target_helpers.start_relationship(tgt_volume, 'aux') + except exception.VolumeBackendAPIException as e: + LOG.error('Error running startrcrelationship due to %(err)s.', + {'err': e}) + return except Exception as e: msg = (_('Unable to fail-over the volume %(id)s to the ' 'secondary back-end, error: %(error)s') % @@ -274,9 +278,7 @@ class StorwizeSVCReplicationGMCV(StorwizeSVCReplicationGlobalMirror): try: self.target_helpers.stop_relationship(tgt_volume, access=True) self.target_helpers.start_relationship(tgt_volume, 'master') - return {'replication_status': - fields.ReplicationStatus.ENABLED, - 'status': 'available'} + return except Exception as e: msg = (_('Unable to fail-back the volume:%(vol)s to the ' 'master back-end, error:%(error)s') % diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_const.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_const.py index b97994ba246..8216c12ea99 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_const.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_const.py @@ -40,9 +40,14 @@ FAILBACK_VALUE = 'default' DEFAULT_RC_TIMEOUT = 3600 * 24 * 7 DEFAULT_RC_INTERVAL = 5 +DEFAULT_RCCG_TIMEOUT = 60 * 30 +DEFAULT_RCCG_INTERVAL = 2 + REPLICA_AUX_VOL_PREFIX = 'aux_' REPLICA_CHG_VOL_PREFIX = 'chg_' +RCCG_PREFIX = 'rccg-' + # remote mirror copy status REP_CONSIS_SYNC = 'consistent_synchronized' REP_CONSIS_COPYING = 'consistent_copying' diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py index ff3609340ba..dee7bc614f2 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py @@ -376,6 +376,58 @@ class StorwizeSSH(object): ssh_cmd = ['svcinfo', 'lsrcrelationship', '-delim', '!', rc_rel] return self.run_ssh_info(ssh_cmd) + # replication cg + def chrcrelationship(self, relationship, rccg=None): + ssh_cmd = ['svctask', 'chrcrelationship'] + if rccg: + ssh_cmd.extend(['-consistgrp', rccg]) + else: + ssh_cmd.extend(['-noconsistgrp']) + ssh_cmd.append(relationship) + self.run_ssh_assert_no_output(ssh_cmd) + + def lsrcconsistgrp(self, rccg): + ssh_cmd = ['svcinfo', 'lsrcconsistgrp', '-delim', '!', rccg] + try: + return self.run_ssh_info(ssh_cmd)[0] + except exception.VolumeBackendAPIException as ex: + LOG.warning("Failed to get rcconsistgrp %(rccg)s info. " + "Exception: %(ex)s.", {'rccg': rccg, + 'ex': ex}) + return None + + def mkrcconsistgrp(self, rccg, system): + ssh_cmd = ['svctask', 'mkrcconsistgrp', '-name', rccg, + '-cluster', system] + return self.run_ssh_check_created(ssh_cmd) + + def rmrcconsistgrp(self, rccg, force=True): + ssh_cmd = ['svctask', 'rmrcconsistgrp'] + if force: + ssh_cmd += ['-force'] + ssh_cmd += ['"%s"' % rccg] + return self.run_ssh_assert_no_output(ssh_cmd) + + def startrcconsistgrp(self, rccg, primary=None): + ssh_cmd = ['svctask', 'startrcconsistgrp', '-force'] + if primary: + ssh_cmd.extend(['-primary', primary]) + ssh_cmd.append(rccg) + self.run_ssh_assert_no_output(ssh_cmd) + + def stoprcconsistgrp(self, rccg, access=False): + ssh_cmd = ['svctask', 'stoprcconsistgrp'] + if access: + ssh_cmd.append('-access') + ssh_cmd.append(rccg) + self.run_ssh_assert_no_output(ssh_cmd) + + def switchrcconsistgrp(self, rccg, aux=True): + primary = 'aux' if aux else 'master' + ssh_cmd = ['svctask', 'switchrcconsistgrp', '-primary', + primary, rccg] + self.run_ssh_assert_no_output(ssh_cmd) + def lspartnership(self, system_name): key_value = 'name=%s' % system_name ssh_cmd = ['svcinfo', 'lspartnership', '-filtervalue', @@ -1490,8 +1542,8 @@ class StorwizeHelpers(object): for snapshot in snapshots: snapshots_model_update.append( {'id': snapshot['id'], - 'status': model_update['status']}) - + 'status': model_update['status'], + 'replication_status': fields.ReplicationStatus.NOT_CAPABLE}) return model_update, snapshots_model_update def delete_consistgrp_snapshots(self, fc_consistgrp, snapshots): @@ -1587,8 +1639,11 @@ class StorwizeHelpers(object): {'id': cgId}) if volumes: for volume in volumes: - volume_model_updates.append({'id': volume['id'], - 'status': status}) + volume_model_updates.append({ + 'id': volume['id'], + 'status': status, + 'replication_status': + fields.ReplicationStatus.NOT_CAPABLE}) else: LOG.info("No volume found for CG: %(cg)s.", {'cg': cgId}) @@ -1798,7 +1853,7 @@ class StorwizeHelpers(object): if not vol_attrs or not vol_attrs['RC_name']: LOG.info("Unable to get remote copy information for " "volume %s", volume_name) - return + return None relationship = self.ssh.lsrcrelationship(vol_attrs['RC_name']) return relationship[0] if len(relationship) > 0 else None @@ -1826,6 +1881,42 @@ class StorwizeHelpers(object): def switch_relationship(self, relationship, aux=True): self.ssh.switchrelationship(relationship, aux) + # replication cg + def chrcrelationship(self, relationship, rccg=None): + self.ssh.chrcrelationship(relationship, rccg) + + def get_rccg(self, rccg): + return self.ssh.lsrcconsistgrp(rccg) + + def create_rccg(self, rccg, system): + self.ssh.mkrcconsistgrp(rccg, system) + + def delete_rccg(self, rccg): + if self.ssh.lsrcconsistgrp(rccg): + self.ssh.rmrcconsistgrp(rccg) + + def start_rccg(self, rccg, primary=None): + self.ssh.startrcconsistgrp(rccg, primary) + + def stop_rccg(self, rccg, access=False): + self.ssh.stoprcconsistgrp(rccg, access) + + def switch_rccg(self, rccg, aux=True): + self.ssh.switchrcconsistgrp(rccg, aux) + + def get_rccg_info(self, volume_name): + vol_attrs = self.get_vdisk_attributes(volume_name) + if not vol_attrs or not vol_attrs['RC_name']: + LOG.warning("Unable to get remote copy information for " + "volume %s", volume_name) + return None + + rcrel = self.ssh.lsrcrelationship(vol_attrs['RC_name']) + if len(rcrel) > 0 and rcrel[0]['consistency_group_name']: + return self.ssh.lsrcconsistgrp(rcrel[0]['consistency_group_name']) + else: + return None + def get_partnership_info(self, system_name): partnership = self.ssh.lspartnership(system_name) return partnership[0] if len(partnership) > 0 else None @@ -2196,14 +2287,23 @@ class StorwizeSVCCommonDriver(san.SanDriver, self._vdiskcopyops = {} self._vdiskcopyops_loop = None self.protocol = None - self._state = {'storage_nodes': {}, - 'enabled_protocols': set(), - 'compression_enabled': False, - 'available_iogrps': [], - 'system_name': None, - 'system_id': None, - 'code_level': None, - } + self._master_state = {'storage_nodes': {}, + 'enabled_protocols': set(), + 'compression_enabled': False, + 'available_iogrps': [], + 'system_name': None, + 'system_id': None, + 'code_level': None, + } + self._state = self._master_state + self._aux_state = {'storage_nodes': {}, + 'enabled_protocols': set(), + 'compression_enabled': False, + 'available_iogrps': [], + 'system_name': None, + 'system_id': None, + 'code_level': None, + } self._active_backend_id = kwargs.get('active_backend_id') # This dictionary is used to map each replication target to certain @@ -2235,9 +2335,6 @@ class StorwizeSVCCommonDriver(san.SanDriver, # v2.1 replication setup self._get_storwize_config() - # Update the storwize state - self._update_storwize_state() - # Validate that the pool exists self._validate_pools_exist() @@ -2262,41 +2359,38 @@ class StorwizeSVCCommonDriver(san.SanDriver, self._vdiskcopyops_loop.start(interval=self.VDISKCOPYOPS_INTERVAL) LOG.debug('leave: do_setup') - def _update_storwize_state(self): + def _update_storwize_state(self, state, helper): # Get storage system name, id, and code level - self._state.update(self._helpers.get_system_info()) + state.update(helper.get_system_info()) # Check if compression is supported - self._state['compression_enabled'] = (self._helpers. - compression_enabled()) + state['compression_enabled'] = helper.compression_enabled() # Get the available I/O groups - self._state['available_iogrps'] = (self._helpers. - get_available_io_groups()) + state['available_iogrps'] = helper.get_available_io_groups() # Get the iSCSI and FC names of the Storwize/SVC nodes - self._state['storage_nodes'] = self._helpers.get_node_info() + state['storage_nodes'] = helper.get_node_info() # Add the iSCSI IP addresses and WWPNs to the storage node info - self._helpers.add_iscsi_ip_addrs(self._state['storage_nodes']) - self._helpers.add_fc_wwpns(self._state['storage_nodes'], - self._state['code_level']) + helper.add_iscsi_ip_addrs(state['storage_nodes']) + helper.add_fc_wwpns(state['storage_nodes'], state['code_level']) # For each node, check what connection modes it supports. Delete any # nodes that do not support any types (may be partially configured). to_delete = [] - for k, node in self._state['storage_nodes'].items(): + for k, node in state['storage_nodes'].items(): if ((len(node['ipv4']) or len(node['ipv6'])) and len(node['iscsi_name'])): node['enabled_protocols'].append('iSCSI') - self._state['enabled_protocols'].add('iSCSI') + state['enabled_protocols'].add('iSCSI') if len(node['WWPN']): node['enabled_protocols'].append('FC') - self._state['enabled_protocols'].add('FC') + state['enabled_protocols'].add('FC') if not len(node['enabled_protocols']): to_delete.append(k) for delkey in to_delete: - del self._state['storage_nodes'][delkey] + del state['storage_nodes'][delkey] def _get_backend_pools(self): if not self._active_backend_id: @@ -2510,7 +2604,8 @@ class StorwizeSVCCommonDriver(san.SanDriver, if opts['qos']: self._helpers.add_vdisk_qos(volume['name'], opts['qos']) - model_update = None + model_update = {'replication_status': + fields.ReplicationStatus.NOT_CAPABLE} if rep_type: replica_obj = self._get_replica_obj(rep_type) @@ -2615,13 +2710,17 @@ class StorwizeSVCCommonDriver(san.SanDriver, self._helpers.add_vdisk_qos(volume['name'], opts['qos']) ctxt = context.get_admin_context() + model_update = {'replication_status': + fields.ReplicationStatus.NOT_CAPABLE} rep_type = self._get_volume_replicated_type(ctxt, volume) if rep_type: self._validate_replication_enabled() replica_obj = self._get_replica_obj(rep_type) replica_obj.volume_replication_setup(ctxt, volume) - return {'replication_status': fields.ReplicationStatus.ENABLED} + model_update = {'replication_status': + fields.ReplicationStatus.ENABLED} + return model_update def create_cloned_volume(self, tgt_volume, src_volume): """Creates a clone of the specified volume.""" @@ -2650,13 +2749,17 @@ class StorwizeSVCCommonDriver(san.SanDriver, self._helpers.add_vdisk_qos(tgt_volume['name'], opts['qos']) ctxt = context.get_admin_context() + model_update = {'replication_status': + fields.ReplicationStatus.NOT_CAPABLE} rep_type = self._get_volume_replicated_type(ctxt, tgt_volume) if rep_type: self._validate_replication_enabled() replica_obj = self._get_replica_obj(rep_type) replica_obj.volume_replication_setup(ctxt, tgt_volume) - return {'replication_status': fields.ReplicationStatus.ENABLED} + model_update = {'replication_status': + fields.ReplicationStatus.ENABLED} + return model_update def extend_volume(self, volume, new_size): self._extend_volume_op(volume, new_size) @@ -2850,13 +2953,13 @@ class StorwizeSVCCommonDriver(san.SanDriver, if storwize_const.FAILBACK_VALUE == secondary_id: # In this case the administrator would like to fail back. - secondary_id, volumes_update = self._replication_failback(context, - volumes) + secondary_id, volumes_update, groups_update = self._host_failback( + context, volumes, groups) elif (secondary_id == self._replica_target['backend_id'] or secondary_id is None): # In this case the administrator would like to fail over. - secondary_id, volumes_update = self._replication_failover(context, - volumes) + secondary_id, volumes_update, groups_update = self._host_failover( + context, volumes, groups) else: msg = (_("Invalid secondary id %s.") % secondary_id) LOG.error(msg) @@ -2864,16 +2967,16 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.debug('leave: failover_host: secondary_id=%(id)s', {'id': secondary_id}) - return secondary_id, volumes_update, [] + return secondary_id, volumes_update, groups_update - def _replication_failback(self, ctxt, volumes): + def _host_failback(self, ctxt, volumes, groups): """Fail back all the volume on the secondary backend.""" - volumes_update = [] + groups_update = [] if not self._active_backend_id: LOG.info("Host has been failed back. doesn't need " "to fail back again") - return None, volumes_update + return None, volumes_update, groups_update try: self._master_backend_helpers.get_system_info() @@ -2882,27 +2985,31 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.error(msg) raise exception.UnableToFailOver(reason=msg) - unrep_volumes, rep_volumes = self._classify_volume(ctxt, volumes) + bypass_volumes, rep_volumes = self._classify_volume(ctxt, volumes) # start synchronize from aux volume to master volume self._sync_with_aux(ctxt, rep_volumes) + self._sync_replica_groups(ctxt, groups) self._wait_replica_ready(ctxt, rep_volumes) + self._wait_replica_groups_ready(ctxt, groups) rep_volumes_update = self._failback_replica_volumes(ctxt, rep_volumes) volumes_update.extend(rep_volumes_update) - unrep_volumes_update = self._failover_unreplicated_volume( - unrep_volumes) - volumes_update.extend(unrep_volumes_update) + rep_vols_in_grp_update, groups_update = self._failback_replica_groups( + ctxt, groups) + volumes_update.extend(rep_vols_in_grp_update) + + bypass_volumes_update = self._bypass_volume_process(bypass_volumes) + volumes_update.extend(bypass_volumes_update) self._helpers = self._master_backend_helpers self._active_backend_id = None + self._state = self._master_state - # Update the storwize state - self._update_storwize_state() self._update_volume_stats() - return storwize_const.FAILBACK_VALUE, volumes_update + return storwize_const.FAILBACK_VALUE, volumes_update, groups_update def _failback_replica_volumes(self, ctxt, rep_volumes): LOG.debug('enter: _failback_replica_volumes') @@ -2936,7 +3043,16 @@ class StorwizeSVCCommonDriver(san.SanDriver, 'state': rep_info['state'], 'primary': rep_info['primary']}) try: - model_updates = replica_obj.replication_failback(volume) + replica_obj.replication_failback(volume) + vol_status = volume.previous_status or 'available' + vol_attach_status = (fields.VolumeAttachStatus.ATTACHED + if vol_status == 'in-use' + else fields.VolumeAttachStatus.DETACHED) + model_updates = { + 'replication_status': fields.ReplicationStatus.ENABLED, + 'previous_status': volume.status, + 'status': vol_status, + 'attach_status': vol_attach_status} volumes_update.append( {'volume_id': volume['id'], 'updates': model_updates}) @@ -2953,9 +3069,9 @@ class StorwizeSVCCommonDriver(san.SanDriver, {'volumes_update': volumes_update}) return volumes_update - def _failover_unreplicated_volume(self, unreplicated_vols): + def _bypass_volume_process(self, bypass_vols): volumes_update = [] - for vol in unreplicated_vols: + for vol in bypass_vols: if vol.replication_driver_data: rep_data = json.loads(vol.replication_driver_data) update_status = rep_data['previous_status'] @@ -2971,6 +3087,62 @@ class StorwizeSVCCommonDriver(san.SanDriver, return volumes_update + def _failback_replica_groups(self, ctxt, groups): + volumes_update = [] + groups_update = [] + for grp in groups: + rccg_name = self._get_rccg_name(grp) + rccg = self._helpers.get_rccg(rccg_name) + if rccg: + try: + grp_update = self._rep_grp_failback(ctxt, rccg, + grp, sync_grp=False) + grp_rep_status = (grp_update['replication_status'] + if grp_update else None) + except Exception as ex: + LOG.error('Fail to failback group %(grp)s during host ' + 'failback due to error: %(error)s', + {'grp': grp.id, 'error': ex}) + grp_rep_status = fields.ReplicationStatus.ERROR + else: + LOG.error('Fail to failback group %(grp)s during host ' + 'failback due to group does not exist on backend', + {'grp': grp.id}) + grp_rep_status = fields.ReplicationStatus.ERROR + + # Update all the volumes' status in that group + if grp_rep_status: + for vol in grp.volumes: + vol_update = {'volume_id': vol.id, + 'updates': + {'replication_status': grp_rep_status, + 'previous_status': vol.status}} + if grp_rep_status == fields.ReplicationStatus.ENABLED: + vol_update['updates']['status'] = (vol.previous_status + or 'available') + else: + vol_update['updates']['status'] = 'error' + vol_update['updates']['attach_status'] = ( + fields.VolumeAttachStatus.ATTACHED if + vol_update['updates']['status'] == 'in-use' + else fields.VolumeAttachStatus.DETACHED) + volumes_update.append(vol_update) + grp_status = (fields.GroupStatus.AVAILABLE + if grp_rep_status == + fields.ReplicationStatus.ENABLED + else fields.GroupStatus.ERROR) + grp_update = {'group_id': grp.id, + 'updates': {'replication_status': grp_rep_status, + 'status': grp_status}} + groups_update.append(grp_update) + else: + grp_update = {'group_id': grp.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.ENABLED}} + groups_update.append(grp_update) + return volumes_update, groups_update + def _sync_with_aux(self, ctxt, volumes): LOG.debug('enter: _sync_with_aux ') try: @@ -3068,12 +3240,23 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.debug('leave: _wait_replica_vol_ready: volume=%(volume)s', {'volume': volume}) - def _replication_failover(self, ctxt, volumes): + def _sync_replica_groups(self, ctxt, groups): + for grp in groups: + rccg_name = self._get_rccg_name(grp) + self._sync_with_aux_grp(ctxt, rccg_name) + + def _wait_replica_groups_ready(self, ctxt, groups): + for grp in groups: + rccg_name = self._get_rccg_name(grp) + self._wait_replica_grp_ready(ctxt, rccg_name) + + def _host_failover(self, ctxt, volumes, groups): volumes_update = [] + groups_update = [] if self._active_backend_id: LOG.info("Host has been failed over to %s", self._active_backend_id) - return self._active_backend_id, volumes_update + return self._active_backend_id, volumes_update, groups_update try: self._aux_backend_helpers.get_system_info() @@ -3083,23 +3266,25 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.error(msg) raise exception.UnableToFailOver(reason=msg) - unrep_volumes, rep_volumes = self._classify_volume(ctxt, volumes) + bypass_volumes, rep_volumes = self._classify_volume(ctxt, volumes) rep_volumes_update = self._failover_replica_volumes(ctxt, rep_volumes) volumes_update.extend(rep_volumes_update) - unrep_volumes_update = self._failover_unreplicated_volume( - unrep_volumes) - volumes_update.extend(unrep_volumes_update) + rep_vols_in_grp_update, groups_update = self._failover_replica_groups( + ctxt, groups) + volumes_update.extend(rep_vols_in_grp_update) + + bypass_volumes_update = self._bypass_volume_process(bypass_volumes) + volumes_update.extend(bypass_volumes_update) self._helpers = self._aux_backend_helpers self._active_backend_id = self._replica_target['backend_id'] self._secondary_pools = [self._replica_target['pool_name']] + self._state = self._aux_state - # Update the storwize state - self._update_storwize_state() self._update_volume_stats() - return self._active_backend_id, volumes_update + return self._active_backend_id, volumes_update, groups_update def _failover_replica_volumes(self, ctxt, rep_volumes): LOG.debug('enter: _failover_replica_volumes') @@ -3120,11 +3305,11 @@ class StorwizeSVCCommonDriver(san.SanDriver, fields.ReplicationStatus.FAILOVER_ERROR, 'status': 'error'}}) LOG.error('_failover_replica_volumes: no rc-' - 'releationship is established for master:' - '%(master)s. Please re-establish the rc-' + 'releationship is established for volume:' + '%(volume)s. Please re-establish the rc-' 'relationship and synchronize the volumes on' ' backend storage.', - {'master': volume['name']}) + {'volume': volume.name}) continue LOG.debug('_failover_replica_volumes: vol=%(vol)s, ' 'master_vol=%(master_vol)s, aux_vol=%(aux_vol)s, ' @@ -3134,7 +3319,16 @@ class StorwizeSVCCommonDriver(san.SanDriver, 'aux_vol': rep_info['aux_vdisk_name'], 'state': rep_info['state'], 'primary': rep_info['primary']}) - model_updates = replica_obj.failover_volume_host(ctxt, volume) + replica_obj.failover_volume_host(ctxt, volume) + vol_status = volume.previous_status or 'available' + vol_attach_status = (fields.VolumeAttachStatus.ATTACHED + if vol_status == 'in-use' + else fields.VolumeAttachStatus.DETACHED) + model_updates = { + 'replication_status': fields.ReplicationStatus.FAILED_OVER, + 'previous_status': volume.status, + 'status': vol_status, + 'attach_status': vol_attach_status} volumes_update.append( {'volume_id': volume['id'], 'updates': model_updates}) @@ -3151,18 +3345,77 @@ class StorwizeSVCCommonDriver(san.SanDriver, {'volumes_update': volumes_update}) return volumes_update + def _failover_replica_groups(self, ctxt, groups): + volumes_update = [] + groups_update = [] + for grp in groups: + rccg_name = self._get_rccg_name(grp) + rccg = self._helpers.get_rccg(rccg_name) + if rccg: + try: + grp_update = self._rep_grp_failover(ctxt, rccg, grp) + grp_rep_status = (grp_update['replication_status'] + if grp_update else None) + except Exception as ex: + LOG.error('Fail to failover group %(grp)s during host ' + 'failover due to error: %(error)s', + {'grp': grp.id, 'error': ex}) + grp_rep_status = fields.ReplicationStatus.ERROR + else: + LOG.error('Fail to failover group %(grp)s during host ' + 'failover due to group does not exist on backend', + {'grp': grp.id}) + grp_rep_status = fields.ReplicationStatus.ERROR + + # Update all the volumes' status in that group + if grp_rep_status: + for vol in grp.volumes: + vol_update = {'volume_id': vol.id, + 'updates': + {'replication_status': grp_rep_status, + 'previous_status': vol.status}} + if grp_rep_status == fields.ReplicationStatus.FAILED_OVER: + vol_update['updates']['status'] = (vol.previous_status + or 'available') + else: + vol_update['updates']['status'] = 'error' + vol_update['updates']['attach_status'] = ( + fields.VolumeAttachStatus.ATTACHED if + vol_update['updates']['status'] == 'in-use' + else fields.VolumeAttachStatus.DETACHED) + volumes_update.append(vol_update) + grp_status = (fields.GroupStatus.AVAILABLE + if grp_rep_status == + fields.ReplicationStatus.FAILED_OVER + else fields.GroupStatus.ERROR) + grp_update = {'group_id': grp.id, + 'updates': {'replication_status': grp_rep_status, + 'status': grp_status}} + groups_update.append(grp_update) + else: + grp_update = {'group_id': grp.id, + 'updates': + {'replication_status': + fields.ReplicationStatus.FAILED_OVER}} + groups_update.append(grp_update) + + return volumes_update, groups_update + def _classify_volume(self, ctxt, volumes): - normal_volumes = [] + bypass_volumes = [] replica_volumes = [] for v in volumes: volume_type = self._get_volume_replicated_type(ctxt, v) - if volume_type and v['status'] == 'available': + grp = v.group + if grp and utils.is_group_a_type( + grp, "consistent_group_replication_enabled"): + continue + elif volume_type and v.status in ['available', 'in-use']: replica_volumes.append(v) else: - normal_volumes.append(v) - - return normal_volumes, replica_volumes + bypass_volumes.append(v) + return bypass_volumes, replica_volumes def _get_replica_obj(self, rep_type): replica_manager = self.replica_manager[ @@ -3222,10 +3475,13 @@ class StorwizeSVCCommonDriver(san.SanDriver, return replication_type def _get_storwize_config(self): + # Update the storwize state + self._update_storwize_state(self._master_state, self._helpers) self._do_replication_setup() if self._active_backend_id and self._replica_target: self._helpers = self._aux_backend_helpers + self._state = self._aux_state self._replica_enabled = (True if (self._helpers.replication_licensed() and self._replica_target) else False) @@ -3282,6 +3538,327 @@ class StorwizeSVCCommonDriver(san.SanDriver, self._aux_backend_helpers = rep_manager.get_target_helpers() self.replica_manager[target['backend_id']] = rep_manager self._replica_target = target + self._update_storwize_state(self._aux_state, self._aux_backend_helpers) + + # Replication Group (Tiramisu) + @cinder_utils.trace + def enable_replication(self, context, group, volumes): + """Enables replication for a group and volumes in the group.""" + model_update = {'replication_status': fields.ReplicationStatus.ENABLED} + volumes_update = [] + rccg_name = self._get_rccg_name(group) + rccg = self._helpers.get_rccg(rccg_name) + if rccg and rccg['relationship_count'] != '0': + try: + if rccg['primary'] == 'aux': + self._helpers.start_rccg(rccg_name, primary='aux') + else: + self._helpers.start_rccg(rccg_name, primary='master') + except exception.VolumeBackendAPIException as err: + LOG.error("Failed to enable group replication on %(rccg)s. " + "Exception: %(exception)s.", + {'rccg': rccg_name, 'exception': err}) + model_update[ + 'replication_status'] = fields.ReplicationStatus.ERROR + else: + if rccg: + LOG.error("Enable replication on empty group %(rccg)s is " + "forbidden.", {'rccg': rccg['name']}) + else: + LOG.error("Failed to enable group replication: %(grp)s does " + "not exist in backend.", {'grp': group.id}) + model_update['replication_status'] = fields.ReplicationStatus.ERROR + + for vol in volumes: + volumes_update.append( + {'id': vol.id, + 'replication_status': model_update['replication_status']}) + return model_update, volumes_update + + @cinder_utils.trace + def disable_replication(self, context, group, volumes): + """Disables replication for a group and volumes in the group.""" + model_update = { + 'replication_status': fields.ReplicationStatus.DISABLED} + volumes_update = [] + rccg_name = self._get_rccg_name(group) + rccg = self._helpers.get_rccg(rccg_name) + if rccg and rccg['relationship_count'] != '0': + try: + self._helpers.stop_rccg(rccg_name) + except exception.VolumeBackendAPIException as err: + LOG.error("Failed to disable group replication on %(rccg)s. " + "Exception: %(exception)s.", + {'rccg': rccg_name, 'exception': err}) + model_update[ + 'replication_status'] = fields.ReplicationStatus.ERROR + else: + if rccg: + LOG.error("Disable replication on empty group %(rccg)s is " + "forbidden.", {'rccg': rccg['name']}) + else: + LOG.error("Failed to disable group replication: %(grp)s does " + "not exist in backend.", {'grp': group.id}) + model_update['replication_status'] = fields.ReplicationStatus.ERROR + + for vol in volumes: + volumes_update.append( + {'id': vol.id, + 'replication_status': model_update['replication_status']}) + return model_update, volumes_update + + @cinder_utils.trace + def failover_replication(self, context, group, volumes, + secondary_backend_id=None): + """Fails over replication for a group and volumes in the group.""" + volumes_model_update = [] + model_update = {} + if not self._replica_enabled: + msg = _("Replication is not properly enabled on backend.") + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + rccg_name = self._get_rccg_name(group) + rccg = self._helpers.get_rccg(rccg_name) + if not rccg: + msg = (_("Replication group %s does not exist on " + "backend.") % rccg_name) + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + if storwize_const.FAILBACK_VALUE == secondary_backend_id: + # In this case the administrator would like to group fail back. + model_update = self._rep_grp_failback(context, rccg, group) + elif (secondary_backend_id == self._replica_target['backend_id'] + or secondary_backend_id is None): + # In this case the administrator would like to group fail over. + model_update = self._rep_grp_failover(context, rccg, group) + else: + msg = (_("Invalid secondary id %s.") % secondary_backend_id) + LOG.error(msg) + raise exception.InvalidReplicationTarget(reason=msg) + + if not model_update: + return None, None + + for vol in volumes: + vol_status = vol.previous_status or 'available' + vol_attach_status = (fields.VolumeAttachStatus.ATTACHED + if vol_status == 'in-use' + else fields.VolumeAttachStatus.DETACHED) + volume_model_update = {'id': vol.id, + 'replication_status': + model_update['replication_status'], + 'previous_status': vol.status, + 'status': vol_status, + 'attach_status': vol_attach_status} + volumes_model_update.append(volume_model_update) + return model_update, volumes_model_update + + def _rep_grp_failback(self, ctxt, rccg, group, sync_grp=True): + """Fail back all the volume in the replication group.""" + model_update = { + 'replication_status': fields.ReplicationStatus.ENABLED} + + try: + self._master_backend_helpers.get_system_info() + except Exception as ex: + msg = (_("Unable to failback group %(rccg)s due to primary is not " + "reachable. error=%(error)s"), + {'rccg': rccg['name'], 'error': ex}) + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + if rccg['relationship_count'] == '0': + msg = (_("Unable to failback empty group %(rccg)s"), + {'rccg': rccg['name']}) + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + if rccg['primary'] == 'master': + LOG.info("Do not need to fail back group %(rccg)s again due to " + "primary is already master.", {'rccg': rccg['name']}) + return None + + if sync_grp: + self._sync_with_aux_grp(ctxt, rccg['name']) + self._wait_replica_grp_ready(ctxt, rccg['name']) + + if rccg['cycling_mode'] == 'multi': + # This is a gmcv replication group + try: + self._aux_backend_helpers.stop_rccg(rccg['name'], access=True) + self._aux_backend_helpers.start_rccg(rccg['name'], + primary='master') + return model_update + except exception.VolumeBackendAPIException as e: + msg = (_('Unable to fail over the group %(rccg)s to the aux ' + 'back-end, error: %(error)s') % + {"rccg": rccg['name'], "error": e}) + LOG.exception(msg) + raise exception.UnableToFailOver(reason=msg) + else: + try: + self._helpers.switch_rccg(rccg['name'], aux=False) + except exception.VolumeBackendAPIException as e: + msg = (_('Unable to fail back the group %(rccg)s, error: ' + '%(error)s') % {"rccg": rccg['name'], "error": e}) + LOG.exception(msg) + raise exception.UnableToFailOver(reason=msg) + return model_update + + def _rep_grp_failover(self, ctxt, rccg, group): + """Fail over all the volume in the replication group.""" + model_update = { + 'replication_status': fields.ReplicationStatus.FAILED_OVER} + try: + self._aux_backend_helpers.get_system_info() + except Exception as ex: + msg = (_("Unable to failover group %(rccg)s due to replication " + "target is not reachable. error=%(error)s"), + {'rccg': rccg['name'], 'error': ex}) + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + if rccg['relationship_count'] == '0': + msg = (_("Unable to failover group %(rccg)s due to it is an " + "empty group."), {'rccg': rccg['name']}) + LOG.error(msg) + raise exception.UnableToFailOver(reason=msg) + + if rccg['primary'] == 'aux': + LOG.info("Do not need to fail over group %(rccg)s again due to " + "primary is already aux.", {'rccg': rccg['name']}) + return None + + if rccg['cycling_mode'] == 'multi': + # This is a gmcv replication group + try: + self._aux_backend_helpers.stop_rccg(rccg['name'], access=True) + self._sync_with_aux_grp(ctxt, rccg['name']) + return model_update + except exception.VolumeBackendAPIException as e: + msg = (_('Unable to fail over the group %(rccg)s to the aux ' + 'back-end, error: %(error)s') % + {"rccg": rccg['name'], "error": e}) + LOG.exception(msg) + raise exception.UnableToFailOver(reason=msg) + else: + try: + # Reverse the role of the primary and secondary volumes + self._helpers.switch_rccg(rccg['name'], aux=True) + return model_update + except exception.VolumeBackendAPIException as e: + LOG.exception('Unable to fail over the group %(rccg)s to the ' + 'aux back-end by switchrcconsistgrp command, ' + 'error: %(error)s', + {"rccg": rccg['name'], "error": e}) + # If the switch command fail, try to make the aux group + # writeable again. + try: + self._aux_backend_helpers.stop_rccg(rccg['name'], + access=True) + self._sync_with_aux_grp(ctxt, rccg['name']) + return model_update + except exception.VolumeBackendAPIException as e: + msg = (_('Unable to fail over the group %(rccg)s to the ' + 'aux back-end, error: %(error)s') % + {"rccg": rccg['name'], "error": e}) + LOG.exception(msg) + raise exception.UnableToFailOver(reason=msg) + + @cinder_utils.trace + def _sync_with_aux_grp(self, ctxt, rccg_name): + try: + rccg = self._helpers.get_rccg(rccg_name) + if rccg and rccg['relationship_count'] != '0': + if (rccg['state'] not in + [storwize_const.REP_CONSIS_SYNC, + storwize_const.REP_CONSIS_COPYING]): + if rccg['primary'] == 'master': + self._helpers.start_rccg(rccg_name, primary='master') + else: + self._helpers.start_rccg(rccg_name, primary='aux') + else: + LOG.warning('group %(grp)s is not in sync.') + except exception.VolumeBackendAPIException as ex: + LOG.warning('Fail to copy data from aux group %(rccg)s to master ' + 'group. Please recheck the relationship and ' + 'synchronize the group on backend storage. error=' + '%(error)s', {'rccg': rccg['name'], 'error': ex}) + + def _wait_replica_grp_ready(self, ctxt, rccg_name): + LOG.debug('_wait_replica_grp_ready: group=%(rccg)s', + {'rccg': rccg_name}) + + def _replica_grp_ready(): + rccg = self._helpers.get_rccg(rccg_name) + if not rccg: + msg = (_('_replica_grp_ready: no group %(rccg)s exists on the ' + 'backend. Please re-create the rccg and synchronize' + 'the volumes on backend storage.'), + {'rccg': rccg_name}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + if rccg['relationship_count'] == '0': + return True + LOG.debug('_replica_grp_ready: group: %(rccg)s: state=%(state)s, ' + 'primary=%(primary)s', + {'rccg': rccg['name'], 'state': rccg['state'], + 'primary': rccg['primary']}) + if rccg['state'] in [storwize_const.REP_CONSIS_SYNC, + storwize_const.REP_CONSIS_COPYING]: + return True + if rccg['state'] == storwize_const.REP_IDL_DISC: + msg = (_('Wait synchronize failed. group: %(rccg)s') % + {'rccg': rccg_name}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + return False + try: + self._helpers._wait_for_a_condition( + _replica_grp_ready, + timeout=storwize_const.DEFAULT_RCCG_TIMEOUT, + interval=storwize_const.DEFAULT_RCCG_INTERVAL, + raise_exception=True) + except Exception as ex: + LOG.error('_wait_replica_grp_ready: wait for group %(rccg)s ' + 'synchronization failed due to ' + 'error: %(err)s.', {'rccg': rccg_name, + 'err': ex}) + + def get_replication_error_status(self, context, groups): + """Returns error info for replicated groups and its volumes. + + The failover/failback only happens manually, no need to update the + status. + """ + return [], [] + + def _get_vol_sys_info(self, volume): + tgt_vol = volume.name + backend_helper = self._helpers + node_state = self._state + grp = volume.group + if grp and utils.is_group_a_type( + grp, "consistent_group_replication_enabled"): + if (grp.replication_status == + fields.ReplicationStatus.FAILED_OVER): + tgt_vol = (storwize_const.REPLICA_AUX_VOL_PREFIX + + volume.name) + backend_helper = self._aux_backend_helpers + node_state = self._aux_state + else: + backend_helper = self._master_backend_helpers + node_state = self._master_state + elif self._active_backend_id: + ctxt = context.get_admin_context() + rep_type = self._get_volume_replicated_type(ctxt, volume) + if rep_type: + tgt_vol = (storwize_const.REPLICA_AUX_VOL_PREFIX + + volume.name) + + return tgt_vol, backend_helper, node_state def migrate_volume(self, ctxt, volume, host): """Migrate directly if source and dest are managed by same storage. @@ -3702,7 +4279,8 @@ class StorwizeSVCCommonDriver(san.SanDriver, 'backend_pool': pool}) raise exception.ManageExistingVolumeTypeMismatch(reason=msg) - model_update = {} + model_update = {'replication_status': + fields.ReplicationStatus.NOT_CAPABLE} self._helpers.rename_vdisk(vdisk['name'], volume['name']) if vol_rep_type: aux_vol = storwize_const.REPLICA_AUX_VOL_PREFIX + volume['name'] @@ -3760,6 +4338,11 @@ class StorwizeSVCCommonDriver(san.SanDriver, return self._stats + @staticmethod + def _get_rccg_name(group, grp_id=None): + group_id = group.id if group else grp_id + return storwize_const.RCCG_PREFIX + group_id[0:4] + '-' + group_id[-5:] + # Add CG capability to generic volume groups def create_group(self, context, group): """Creates a group. @@ -3770,23 +4353,55 @@ class StorwizeSVCCommonDriver(san.SanDriver, """ LOG.debug("Creating group.") + + # we'll rely on the generic group implementation if it is + # not a consistency group and not a consistency replication + # request. + if (not utils.is_group_a_cg_snapshot_type(group) and not + utils.is_group_a_type(group, + "consistent_group_replication_enabled")): + raise NotImplementedError() + model_update = {'status': fields.GroupStatus.AVAILABLE} - for vol_type_id in group.volume_type_ids: - replication_type = self._get_volume_replicated_type( - context, None, vol_type_id) - if replication_type: - # An unsupported configuration - LOG.error('Unable to create group: create group with ' - 'replication volume type is not supported.') - model_update = {'status': fields.GroupStatus.ERROR} - return model_update - if utils.is_group_a_cg_snapshot_type(group): - return {'status': fields.GroupStatus.AVAILABLE} - # we'll rely on the generic group implementation if it is not a - # consistency group request. - raise NotImplementedError() + for vol_type_id in group.volume_type_ids: + replication_type = self._get_volume_replicated_type( + context, None, vol_type_id) + if replication_type: + # An unsupported configuration + LOG.error('Unable to create group: create consistent ' + 'snapshot group with replication volume type is ' + 'not supported.') + model_update = {'status': fields.GroupStatus.ERROR} + return model_update + + if utils.is_group_a_type(group, + "consistent_group_replication_enabled"): + for vol_type_id in group.volume_type_ids: + replication_type = self._get_volume_replicated_type( + context, None, vol_type_id) + if not replication_type: + # An unsupported configuration + LOG.error('Unable to create group: create consistent ' + 'replication group with non-replication volume' + ' type is not supported.') + model_update = {'status': fields.GroupStatus.ERROR} + return model_update + + rccg_name = self._get_rccg_name(group) + try: + tgt_sys = self._aux_backend_helpers.get_system_info() + self._helpers.create_rccg( + rccg_name, tgt_sys.get('system_name')) + model_update.update({'replication_status': + fields.ReplicationStatus.ENABLED}) + except exception.VolumeBackendAPIException as err: + LOG.error("Failed to create rccg %(rccg)s. " + "Exception: %(exception)s.", + {'rccg': rccg_name, 'exception': err}) + model_update = {'status': fields.GroupStatus.ERROR} + return model_update def delete_group(self, context, group, volumes): """Deletes a group. @@ -3797,28 +4412,36 @@ class StorwizeSVCCommonDriver(san.SanDriver, :returns: model_update, volumes_model_update """ LOG.debug("Deleting group.") - if not utils.is_group_a_cg_snapshot_type(group): - # we'll rely on the generic group implementation if it is - # not a consistency group request. + + # we'll rely on the generic group implementation if it is + # not a consistency group and not a consistency replication + # request. + if (not utils.is_group_a_cg_snapshot_type(group) and not + utils.is_group_a_type(group, + "consistent_group_replication_enabled")): raise NotImplementedError() model_update = {'status': fields.GroupStatus.DELETED} volumes_model_update = [] - - for volume in volumes: - try: - self._helpers.delete_vdisk(volume['name'], True) - volumes_model_update.append( - {'id': volume['id'], 'status': 'deleted'}) - except exception.VolumeBackendAPIException as err: - model_update['status'] = ( - fields.GroupStatus.ERROR_DELETING) - LOG.error("Failed to delete the volume %(vol)s of CG. " - "Exception: %(exception)s.", - {'vol': volume['name'], 'exception': err}) - volumes_model_update.append( - {'id': volume['id'], - 'status': fields.GroupStatus.ERROR_DELETING}) + if utils.is_group_a_type(group, + "consistent_group_replication_enabled"): + model_update, volumes_model_update = self._delete_replication_grp( + group, volumes) + else: + for volume in volumes: + try: + self._helpers.delete_vdisk(volume.name, True) + volumes_model_update.append( + {'id': volume.id, 'status': 'deleted'}) + except exception.VolumeBackendAPIException as err: + model_update['status'] = ( + fields.GroupStatus.ERROR_DELETING) + LOG.error("Failed to delete the volume %(vol)s of CG. " + "Exception: %(exception)s.", + {'vol': volume.name, 'exception': err}) + volumes_model_update.append( + {'id': volume.id, + 'status': fields.GroupStatus.ERROR_DELETING}) return model_update, volumes_model_update @@ -3834,12 +4457,21 @@ class StorwizeSVCCommonDriver(san.SanDriver, """ LOG.debug("Updating group.") - if utils.is_group_a_cg_snapshot_type(group): - return None, None, None # we'll rely on the generic group implementation if it is not a - # consistency group request. - raise NotImplementedError() + # consistency group request and not consistency replication request. + if (not utils.is_group_a_cg_snapshot_type(group) and not + utils.is_group_a_type(group, + "consistent_group_replication_enabled")): + raise NotImplementedError() + + if utils.is_group_a_type(group, + "consistent_group_replication_enabled"): + return self._update_replication_grp(context, group, add_volumes, + remove_volumes) + + if utils.is_group_a_cg_snapshot_type(group): + return None, None, None def create_group_from_src(self, context, group, volumes, group_snapshot=None, snapshots=None, @@ -3856,6 +4488,15 @@ class StorwizeSVCCommonDriver(san.SanDriver, :returns: model_update, volumes_model_update """ LOG.debug('Enter: create_group_from_src.') + + if utils.is_group_a_type(group, + "consistent_group_replication_enabled"): + # An unsupported configuration + msg = _('Unable to create replication group: create replication ' + 'group from a replication group is not supported.') + LOG.exception(msg) + raise exception.VolumeBackendAPIException(data=msg) + if not utils.is_group_a_cg_snapshot_type(group): # we'll rely on the generic volume groups implementation if it is # not a consistency group request. @@ -3873,7 +4514,6 @@ class StorwizeSVCCommonDriver(san.SanDriver, error_msg = _("create_group_from_src must be creating from a " "group snapshot, or a source group.") raise exception.InvalidInput(reason=error_msg) - LOG.debug('create_group_from_src: cg_name %(cg_name)s' ' %(sources)s', {'cg_name': cg_name, 'sources': sources}) self._helpers.create_fc_consistgrp(cg_name) @@ -3974,6 +4614,7 @@ class StorwizeSVCCommonDriver(san.SanDriver, data['replication'] = self._replica_enabled data['replication_enabled'] = self._replica_enabled data['replication_targets'] = self._get_replication_targets() + data['consistent_group_replication_enabled'] = True self._stats = data def _build_pool_stats(self, pool): @@ -4026,7 +4667,8 @@ class StorwizeSVCCommonDriver(san.SanDriver, 'replication_enabled': self._replica_enabled, 'replication_type': self._supported_replica_types, 'replication_targets': self._get_replication_targets(), - 'replication_count': len(self._get_replication_targets()) + 'replication_count': len(self._get_replication_targets()), + 'consistent_group_replication_enabled': True }) except exception.VolumeBackendAPIException: @@ -4059,3 +4701,106 @@ class StorwizeSVCCommonDriver(san.SanDriver, raise exception.ManageExistingInvalidReference(existing_ref=ref, reason=reason) return vdisk + + def _delete_replication_grp(self, group, volumes): + model_update = {'status': fields.GroupStatus.DELETED} + volumes_model_update = [] + rccg_name = self._get_rccg_name(group) + try: + self._helpers.delete_rccg(rccg_name) + except exception.VolumeBackendAPIException as err: + LOG.error("Failed to delete rccg %(rccg)s. " + "Exception: %(exception)s.", + {'rccg': rccg_name, 'exception': err}) + model_update = {'status': fields.GroupStatus.ERROR_DELETING} + + for volume in volumes: + try: + self._master_backend_helpers.delete_rc_volume(volume.name) + self._aux_backend_helpers.delete_rc_volume(volume.name, + target_vol=True) + volumes_model_update.append( + {'id': volume.id, 'status': 'deleted'}) + except exception.VolumeDriverException as err: + model_update['status'] = ( + fields.GroupStatus.ERROR_DELETING) + LOG.error("Failed to delete the volume %(vol)s of CG. " + "Exception: %(exception)s.", + {'vol': volume.name, 'exception': err}) + volumes_model_update.append( + {'id': volume.id, + 'status': fields.GroupStatus.ERROR_DELETING}) + return model_update, volumes_model_update + + def _update_replication_grp(self, context, group, + add_volumes, remove_volumes): + model_update = {'status': fields.GroupStatus.AVAILABLE} + LOG.info("Update replication group: %(group)s. ", {'group': group.id}) + + rccg_name = self._get_rccg_name(group) + rccg = self._helpers.get_rccg(rccg_name) + if not rccg: + LOG.error("Failed to update group: %(grp)s does not exist in " + "backend.", {'grp': group.id}) + model_update['status'] = fields.GroupStatus.ERROR + return model_update, None, None + + # Add remote copy relationship to rccg + for volume in add_volumes: + try: + rcrel = self._helpers.get_relationship_info(volume.name) + if not rcrel: + LOG.error("Failed to update group: remote copy " + "relationship of %(vol)s does not exist in " + "backend.", {'vol': volume.id}) + model_update['status'] = fields.GroupStatus.ERROR + elif (rccg['copy_type'] != 'empty_group' and + (rccg['copy_type'] != rcrel['copy_type'] or + rccg['state'] != rcrel['state'] or + rccg['cycling_mode'] != rcrel['cycling_mode'] or + (rccg['cycle_period_seconds'] != + rcrel['cycle_period_seconds']))): + LOG.error("Failed to update rccg %(rccg)s: remote copy " + "type of %(vol)s is %(vol_rc_type)s, the rccg " + "type is %(rccg_type)s. rcrel state is " + "%(rcrel_state)s, rccg state is %(rccg_state)s. " + "rcrel cycling mode is %(rcrel_cmode)s, rccg " + "cycling mode is %(rccg_cmode)s. rcrel cycling " + "period is %(rcrel_period)s, rccg cycling " + "period is %(rccg_period)s. ", + {'rccg': rccg_name, + 'vol': volume.id, + 'vol_rc_type': rcrel['copy_type'], + 'rccg_type': rccg['copy_type'], + 'rcrel_state': rcrel['state'], + 'rccg_state': rccg['state'], + 'rcrel_cmode': rcrel['cycling_mode'], + 'rccg_cmode': rccg['cycling_mode'], + 'rcrel_period': rcrel['cycle_period_seconds'], + 'rccg_period': rccg['cycle_period_seconds']}) + model_update['status'] = fields.GroupStatus.ERROR + else: + self._helpers.chrcrelationship(rcrel['name'], rccg_name) + except exception.VolumeBackendAPIException as err: + model_update['status'] = fields.GroupStatus.ERROR + LOG.error("Failed to add the remote copy of volume %(vol)s to " + "group. Exception: %(exception)s.", + {'vol': volume.name, 'exception': err}) + + # Remove remote copy relationship from rccg + for volume in remove_volumes: + try: + rcrel = self._helpers.get_relationship_info(volume.name) + if not rcrel: + LOG.error("Failed to update group: remote copy " + "relationship of %(vol)s does not exist in " + "backend.", {'vol': volume.id}) + model_update['status'] = fields.GroupStatus.ERROR + else: + self._helpers.chrcrelationship(rcrel['name']) + except exception.VolumeBackendAPIException as err: + model_update['status'] = fields.GroupStatus.ERROR + LOG.error("Failed to remove the remote copy of volume %(vol)s " + "from group. Exception: %(exception)s.", + {'vol': volume.name, 'exception': err}) + return model_update, None, None diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_fc.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_fc.py index 9f5cf4beca5..850bbd4b1c9 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_fc.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_fc.py @@ -92,9 +92,10 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): 2.2 - Add CG capability to generic volume groups 2.2.1 - Add vdisk mirror/stretch cluster support 2.2.2 - Add npiv support + 2.2.3 - Add replication group support """ - VERSION = "2.2.2" + VERSION = "2.2.3" # ThirdPartySystems wiki page CI_WIKI_NAME = "IBM_STORAGE_CI" @@ -136,15 +137,16 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): """ LOG.debug('enter: initialize_connection: volume %(vol)s with connector' ' %(conn)s', {'vol': volume['id'], 'conn': connector}) - volume_name = self._get_target_vol(volume) + volume_name, backend_helper, node_state = self._get_vol_sys_info( + volume) # Check if a host object is defined for this host name - host_name = self._helpers.get_host_from_connector(connector) + host_name = backend_helper.get_host_from_connector(connector) if host_name is None: # Host does not exist - add a new host to Storwize/SVC - host_name = self._helpers.create_host(connector) + host_name = backend_helper.create_host(connector) - volume_attributes = self._helpers.get_vdisk_attributes(volume_name) + volume_attributes = backend_helper.get_vdisk_attributes(volume_name) if volume_attributes is None: msg = (_('initialize_connection: Failed to get attributes' ' for volume %s.') % volume_name) @@ -152,8 +154,8 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): raise exception.VolumeDriverException(message=msg) multihostmap = self.configuration.storwize_svc_multihostmap_enabled - lun_id = self._helpers.map_vol_to_host(volume_name, host_name, - multihostmap) + lun_id = backend_helper.map_vol_to_host(volume_name, host_name, + multihostmap) try: preferred_node = volume_attributes['preferred_node_id'] IO_group = volume_attributes['IO_group_id'] @@ -168,7 +170,7 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): # Get preferred node and other nodes in I/O group preferred_node_entry = None io_group_nodes = [] - for node in self._state['storage_nodes'].values(): + for node in node_state['storage_nodes'].values(): if node['id'] == preferred_node: preferred_node_entry = node if node['IO_group'] == IO_group: @@ -192,19 +194,19 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): properties['target_lun'] = lun_id properties['volume_id'] = volume['id'] - conn_wwpns = self._helpers.get_conn_fc_wwpns(host_name) + conn_wwpns = backend_helper.get_conn_fc_wwpns(host_name) # If conn_wwpns is empty, then that means that there were # no target ports with visibility to any of the initiators # so we return all target ports. if len(conn_wwpns) == 0: - for node in self._state['storage_nodes'].values(): + for node in node_state['storage_nodes'].values(): # The Storwize/svc release 7.7.0.0 introduced NPIV feature, # Different commands be used to get the wwpns for host I/O - if self._state['code_level'] < (7, 7, 0, 0): + if node_state['code_level'] < (7, 7, 0, 0): conn_wwpns.extend(node['WWPN']) else: - npiv_wwpns = self._helpers.get_npiv_wwpns( + npiv_wwpns = backend_helper.get_npiv_wwpns( node_id=node['id'], host_io="yes") conn_wwpns.extend(npiv_wwpns) @@ -218,8 +220,11 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): # specific for z/VM, refer to cinder bug 1323993 if "zvm_fcp" in connector: properties['zvm_fcp'] = connector['zvm_fcp'] - except Exception: + except Exception as ex: with excutils.save_and_reraise_exception(): + LOG.error('initialize_connection: Failed to export volume ' + '%(vol)s due to %(ex)s.', {'vol': volume.name, + 'ex': ex}) self._do_terminate_connection(volume, connector) LOG.error('initialize_connection: Failed ' 'to collect return ' @@ -272,7 +277,8 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): """ LOG.debug('enter: terminate_connection: volume %(vol)s with connector' ' %(conn)s', {'vol': volume['id'], 'conn': connector}) - vol_name = self._get_target_vol(volume) + vol_name, backend_helper, node_state = self._get_vol_sys_info(volume) + info = {} if 'host' in connector: # get host according to FC protocol @@ -282,7 +288,7 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): info = {'driver_volume_type': 'fibre_channel', 'data': {}} - host_name = self._helpers.get_host_from_connector( + host_name = backend_helper.get_host_from_connector( connector, volume_name=vol_name) if host_name is None: msg = (_('terminate_connection: Failed to get host name from' @@ -294,11 +300,11 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): host_name = None # Unmap volumes, if hostname is None, need to get value from vdiskmap - host_name = self._helpers.unmap_vol_from_host(vol_name, host_name) + host_name = backend_helper.unmap_vol_from_host(vol_name, host_name) # Host_name could be none if host_name: - resp = self._helpers.check_host_mapped_vols(host_name) + resp = backend_helper.check_host_mapped_vols(host_name) if not len(resp): LOG.info("Need to remove FC Zone, building initiator " "target map.") @@ -308,14 +314,14 @@ class StorwizeSVCFCDriver(storwize_common.StorwizeSVCCommonDriver): # Returning all target_wwpns in storage_nodes, since # we cannot determine which wwpns are logged in during # a VM deletion. - for node in self._state['storage_nodes'].values(): + for node in node_state['storage_nodes'].values(): target_wwpns.extend(node['WWPN']) init_targ_map = (self._make_initiator_target_map (connector['wwpns'], target_wwpns)) info['data'] = {'initiator_target_map': init_targ_map} # No volume mapped to the host, delete host from array - self._helpers.delete_host(host_name) + backend_helper.delete_host(host_name) LOG.debug('leave: terminate_connection: volume %(vol)s with ' 'connector %(conn)s', {'vol': volume['id'], diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_iscsi.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_iscsi.py index 1af2fb827d6..09cc1083c42 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_iscsi.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_iscsi.py @@ -91,9 +91,10 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): 2.1.1 - Update replication to version 2.1 2.2 - Add CG capability to generic volume groups 2.2.1 - Add vdisk mirror/stretch cluster support + 2.2.2 - Add replication group support """ - VERSION = "2.2.1" + VERSION = "2.2.2" # ThirdPartySystems wiki page CI_WIKI_NAME = "IBM_STORAGE_CI" @@ -133,25 +134,26 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): """ LOG.debug('enter: initialize_connection: volume %(vol)s with connector' ' %(conn)s', {'vol': volume['id'], 'conn': connector}) - volume_name = self._get_target_vol(volume) + volume_name, backend_helper, node_state = self._get_vol_sys_info( + volume) # Check if a host object is defined for this host name - host_name = self._helpers.get_host_from_connector(connector, - iscsi=True) + host_name = backend_helper.get_host_from_connector(connector, + iscsi=True) if host_name is None: # Host does not exist - add a new host to Storwize/SVC - host_name = self._helpers.create_host(connector, iscsi=True) + host_name = backend_helper.create_host(connector, iscsi=True) - chap_secret = self._helpers.get_chap_secret_for_host(host_name) + chap_secret = backend_helper.get_chap_secret_for_host(host_name) chap_enabled = self.configuration.storwize_svc_iscsi_chap_enabled if chap_enabled and chap_secret is None: - chap_secret = self._helpers.add_chap_secret_to_host(host_name) + chap_secret = backend_helper.add_chap_secret_to_host(host_name) elif not chap_enabled and chap_secret: LOG.warning('CHAP secret exists for host but CHAP is disabled.') multihostmap = self.configuration.storwize_svc_multihostmap_enabled - lun_id = self._helpers.map_vol_to_host(volume_name, host_name, - multihostmap) + lun_id = backend_helper.map_vol_to_host(volume_name, host_name, + multihostmap) try: properties = self._get_single_iscsi_data(volume, connector, @@ -159,9 +161,14 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): multipath = connector.get('multipath', False) if multipath: properties = self._get_multi_iscsi_data(volume, connector, - lun_id, properties) - except Exception: + lun_id, properties, + backend_helper, + node_state) + except Exception as ex: with excutils.save_and_reraise_exception(): + LOG.error('initialize_connection: Failed to export volume ' + '%(vol)s due to %(ex)s.', {'vol': volume.name, + 'ex': ex}) self._do_terminate_connection(volume, connector) LOG.error('initialize_connection: Failed ' 'to collect return ' @@ -182,8 +189,9 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): {'vol': volume['id'], 'conn': connector, 'lun_id': lun_id}) - volume_name = self._get_target_vol(volume) - volume_attributes = self._helpers.get_vdisk_attributes(volume_name) + volume_name, backend_helper, node_state = self._get_vol_sys_info( + volume) + volume_attributes = backend_helper.get_vdisk_attributes(volume_name) if volume_attributes is None: msg = (_('_get_single_iscsi_data: Failed to get attributes' ' for volume %s.') % volume_name) @@ -204,7 +212,7 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): # Get preferred node and other nodes in I/O group preferred_node_entry = None io_group_nodes = [] - for node in self._state['storage_nodes'].values(): + for node in node_state['storage_nodes'].values(): if self.protocol not in node['enabled_protocols']: continue @@ -252,14 +260,15 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): 'prop': properties}) return properties - def _get_multi_iscsi_data(self, volume, connector, lun_id, properties): + def _get_multi_iscsi_data(self, volume, connector, lun_id, properties, + backend_helper, node_state): LOG.debug('enter: _get_multi_iscsi_data: volume %(vol)s with ' 'connector %(conn)s lun_id %(lun_id)s', {'vol': volume.id, 'conn': connector, 'lun_id': lun_id}) try: - resp = self._helpers.ssh.lsportip() + resp = backend_helper.ssh.lsportip() except Exception as ex: msg = (_('_get_multi_iscsi_data: Failed to ' 'get port ip because of exception: ' @@ -270,7 +279,7 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): properties['target_iqns'] = [] properties['target_portals'] = [] properties['target_luns'] = [] - for node in self._state['storage_nodes'].values(): + for node in node_state['storage_nodes'].values(): for ip_data in resp: if ip_data['node_id'] != node['id']: continue @@ -329,15 +338,15 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): """ LOG.debug('enter: terminate_connection: volume %(vol)s with connector' ' %(conn)s', {'vol': volume['id'], 'conn': connector}) - vol_name = self._get_target_vol(volume) + vol_name, backend_helper, node_state = self._get_vol_sys_info(volume) info = {} if 'host' in connector: # get host according to iSCSI protocol info = {'driver_volume_type': 'iscsi', 'data': {}} - host_name = self._helpers.get_host_from_connector(connector, - iscsi=True) + host_name = backend_helper.get_host_from_connector(connector, + iscsi=True) if host_name is None: msg = (_('terminate_connection: Failed to get host name from' ' connector.')) @@ -348,13 +357,13 @@ class StorwizeSVCISCSIDriver(storwize_common.StorwizeSVCCommonDriver): host_name = None # Unmap volumes, if hostname is None, need to get value from vdiskmap - host_name = self._helpers.unmap_vol_from_host(vol_name, host_name) + host_name = backend_helper.unmap_vol_from_host(vol_name, host_name) # Host_name could be none if host_name: - resp = self._helpers.check_host_mapped_vols(host_name) + resp = backend_helper.check_host_mapped_vols(host_name) if not len(resp): - self._helpers.delete_host(host_name) + backend_helper.delete_host(host_name) LOG.debug('leave: terminate_connection: volume %(vol)s with ' 'connector %(conn)s', {'vol': volume['id'], diff --git a/releasenotes/notes/storwize-cg-replication-b038ff0d39fe909f.yaml b/releasenotes/notes/storwize-cg-replication-b038ff0d39fe909f.yaml new file mode 100644 index 00000000000..e063c6a05c5 --- /dev/null +++ b/releasenotes/notes/storwize-cg-replication-b038ff0d39fe909f.yaml @@ -0,0 +1,3 @@ +--- +features: + - Add consistent replication group support in Storwize Cinder driver.