From 59ddeabacabdbbd9e396d1d62ae65c1fdfa3f455 Mon Sep 17 00:00:00 2001 From: zengyingzhe Date: Sat, 22 Apr 2017 15:33:52 +0800 Subject: [PATCH] Use requests lib for Huawei array connection Updates Huawei driver logic to use requests lib for Huawei array connection instead. Change-Id: I397e4edde4d82baceb324c1fe9896b1c0f914194 --- .../drivers/huawei/test_huawei_drivers.py | 103 +++++++++++---- cinder/volume/drivers/huawei/rest_client.py | 123 +++++++++--------- 2 files changed, 140 insertions(+), 86 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py b/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py index 422ace3ee27..dc22da1ebc1 100644 --- a/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py +++ b/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py @@ -19,6 +19,7 @@ import ddt import json import mock import re +import requests import tempfile import unittest from xml.dom import minidom @@ -1214,10 +1215,10 @@ FAKE_GET_METROROUP_ID_RESPONSE = """ # mock login info map MAP_COMMAND_TO_FAKE_RESPONSE = {} -MAP_COMMAND_TO_FAKE_RESPONSE['/xx/sessions'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/xx/sessions/POST'] = ( FAKE_GET_LOGIN_STORAGE_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/sessions'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/sessions/DELETE'] = ( FAKE_LOGIN_OUT_STORAGE_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUN_MIGRATION/POST'] = ( @@ -1230,11 +1231,11 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/LUN_MIGRATION/11/DELETE'] = ( FAKE_COMMON_SUCCESS_RESPONSE) # mock storage info map -MAP_COMMAND_TO_FAKE_RESPONSE['/storagepool'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/storagepool/GET'] = ( FAKE_STORAGE_POOL_RESPONSE) # mock lun info map -MAP_COMMAND_TO_FAKE_RESPONSE['/lun'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lun/POST'] = ( FAKE_LUN_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/lun/11/GET'] = ( @@ -1290,10 +1291,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/associate?TYPE=27&ASSOCIATEOBJTYPE=256' MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup?range=[0-8191]/GET'] = ( FAKE_QUERY_LUN_GROUP_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/POST'] = ( FAKE_QUERY_LUN_GROUP_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate/POST'] = ( FAKE_QUERY_LUN_GROUP_ASSOCIAT_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUNGroup/11/DELETE'] = ( @@ -1343,14 +1344,14 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate?ID=12&ASSOCIATEOBJTYPE=11' FAKE_COMMON_SUCCESS_RESPONSE) # mock snapshot info map -MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/POST'] = ( FAKE_CREATE_SNAPSHOT_INFO_RESPONSE) # mock snapshot info map MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/11/GET'] = ( FAKE_GET_SNAPSHOT_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/activate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/activate/POST'] = ( FAKE_COMMON_SUCCESS_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/stop/PUT'] = ( @@ -1375,10 +1376,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/11/PUT'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/active/11/PUT'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/POST'] = ( FAKE_QOS_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) # mock iscsi info map @@ -1392,13 +1393,13 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/eth_port/associate?TYPE=213&ASSOCIATEOBJTYPE' '=257&ASSOCIATEOBJID=11/GET'] = ( FAKE_GET_ETH_ASSOCIATE_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/iscsidevicename'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/iscsidevicename/GET'] = ( FAKE_GET_ISCSI_DEVICE_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator?range=[0-256]/GET'] = ( FAKE_ISCSI_INITIATOR_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/GET'] = ( FAKE_ISCSI_INITIATOR_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/POST'] = ( @@ -1427,13 +1428,13 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/host/1/DELETE'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/host/1/GET'] = ( FAKE_GET_HOST_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/host'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/host/POST'] = ( FAKE_CREATE_HOST_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup?range=[0-8191]/GET'] = ( FAKE_GET_ALL_HOST_GROUP_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/GET'] = ( FAKE_GET_HOST_GROUP_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/host/associate?TYPE=14&ID=0' @@ -1457,11 +1458,11 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/host/associate?TYPE=21&' FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/associate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/associate/POST'] = ( FAKE_COMMON_SUCCESS_RESPONSE) # mock copy info map -MAP_COMMAND_TO_FAKE_RESPONSE['/luncopy'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/luncopy/POST'] = ( FAKE_GET_LUN_COPY_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUNCOPY?range=[0-1023]/GET'] = ( @@ -1477,7 +1478,7 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/LUNCOPY/0/DELETE'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview?range=[0-8191]/GET'] = ( FAKE_GET_MAPPING_VIEW_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview/POST'] = ( FAKE_GET_MAPPING_VIEW_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview/PUT'] = ( @@ -1561,7 +1562,7 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/portgroup?range=[0-8191]&TYPE=257/GET'] = ( FAKE_PORT_GROUP_RESPONSE) # mock system info map -MAP_COMMAND_TO_FAKE_RESPONSE['/system//GET'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/system/GET'] = ( FAKE_SYSTEM_VERSION_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/fc_initiator?range=[0-256]/GET'] = ( @@ -1594,7 +1595,7 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/0/GET'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/REMOVE_ASSOCIATE/PUT'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/cachepartition/0/GET'] = ( @@ -1633,10 +1634,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/HyperMetroPair/synchronize_hcpair/PUT'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror?range=[0-8191]/GET'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/smartcachepool/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/smartcachepool/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) FAKE_GET_PORTG_BY_VIEW = """ @@ -2130,7 +2131,7 @@ class FakeClient(rest_client.RestClient): def add_lun_to_cache(self, lunid, cache_id): pass - def do_call(self, url=False, data=None, method=None, calltimeout=4, + def do_call(self, url, data, method, calltimeout=4, log_filter_flag=False): url = url.replace('http://192.0.2.69:8082/deviceManager/rest', '') command = url.replace('/210235G7J20000000000/', '') @@ -2322,7 +2323,8 @@ class HuaweiTestBase(test.TestCase): "COPYSPEED": actual_speed, "LUNCOPYTYPE": "1", "SOURCELUN": "INVALID;fake_src_lun;INVALID;INVALID;INVALID", - "TARGETLUN": "INVALID;fake_tgt_lun;INVALID;INVALID;INVALID"} + "TARGETLUN": "INVALID;fake_tgt_lun;INVALID;INVALID;INVALID"}, + 'POST' ) @@ -5310,3 +5312,58 @@ class HuaweiConfTestCase(test.TestCase): fakefile = open(self.conf.cinder_huawei_conf_file, 'w') fakefile.write(doc.toprettyxml(indent='')) fakefile.close() + + +@ddt.ddt +class HuaweiRestClientTestCase(test.TestCase): + def setUp(self): + super(HuaweiRestClientTestCase, self).setUp() + config = mock.Mock(spec=conf.Configuration) + huawei_conf = FakeHuaweiConf(config, 'iSCSI') + huawei_conf.update_config_value() + self.client = rest_client.RestClient( + config, config.san_address, config.san_user, config.san_password) + + def test_init_http_head(self): + self.client.init_http_head() + self.assertIsNone(self.client.url) + self.assertEqual("keep-alive", + self.client.session.headers["Connection"]) + self.assertEqual("application/json", + self.client.session.headers["Content-Type"]) + self.assertEqual(False, self.client.session.verify) + + @ddt.data('POST', 'PUT', 'GET', 'DELETE') + def test_do_call_method(self, method): + self.client.init_http_head() + + if method: + mock_func = self.mock_object(self.client.session, method.lower()) + else: + mock_func = self.mock_object(self.client.session, 'post') + + self.client.do_call("http://fake-rest-url", None, method) + mock_func.assert_called_once_with("http://fake-rest-url", + timeout=constants.SOCKET_TIMEOUT) + + def test_do_call_method_invalid(self): + self.assertRaises(exception.VolumeBackendAPIException, + self.client.do_call, + "http://fake-rest-url", None, 'fake-method') + + def test_do_call_http_error(self): + self.client.init_http_head() + + fake_res = requests.Response() + fake_res.reason = 'something wrong' + fake_res.status_code = 500 + fake_res.url = "http://fake-rest-url" + + self.mock_object(self.client.session, 'post', return_value=fake_res) + res = self.client.do_call("http://fake-rest-url", None, 'POST') + + expected = {"error": {"code": 500, + "description": + '500 Server Error: something wrong for ' + 'url: http://fake-rest-url'}} + self.assertEqual(expected, res) diff --git a/cinder/volume/drivers/huawei/rest_client.py b/cinder/volume/drivers/huawei/rest_client.py index e4ebeadaaa3..66d61af4a91 100644 --- a/cinder/volume/drivers/huawei/rest_client.py +++ b/cinder/volume/drivers/huawei/rest_client.py @@ -15,14 +15,12 @@ import json import re +import requests import six -import socket import time from oslo_log import log as logging from oslo_utils import excutils -from six.moves import http_cookiejar -from six.moves import urllib from cinder import exception from cinder.i18n import _ @@ -41,7 +39,6 @@ class RestClient(object): self.san_address = san_address self.san_user = san_user self.san_password = san_password - self.init_http_head() self.storage_pools = kwargs.get('storage_pools', self.configuration.storage_pools) self.iscsi_info = kwargs.get('iscsi_info', @@ -49,17 +46,19 @@ class RestClient(object): self.iscsi_default_target_ip = kwargs.get( 'iscsi_default_target_ip', self.configuration.iscsi_default_target_ip) - - def init_http_head(self): - self.cookie = http_cookiejar.CookieJar() + self.session = None self.url = None self.device_id = None - self.headers = { - "Connection": "keep-alive", - "Content-Type": "application/json", - } - def do_call(self, url=None, data=None, method=None, + def init_http_head(self): + self.url = None + self.session = requests.Session() + self.session.headers.update({ + "Connection": "keep-alive", + "Content-Type": "application/json"}) + self.session.verify = False + + def do_call(self, url, data, method, calltimeout=constants.SOCKET_TIMEOUT, log_filter_flag=False): """Send requests to Huawei storage server. @@ -68,44 +67,42 @@ class RestClient(object): """ if self.url: url = self.url + url - handler = urllib.request.HTTPCookieProcessor(self.cookie) - opener = urllib.request.build_opener(handler) - urllib.request.install_opener(opener) - res_json = None + + kwargs = {'timeout': calltimeout} + if data: + kwargs['data'] = json.dumps(data) + + if method in ('POST', 'PUT', 'GET', 'DELETE'): + func = getattr(self.session, method.lower()) + else: + msg = _("Request method %s is invalid.") % method + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) try: - socket.setdefaulttimeout(calltimeout) - if data: - data = json.dumps(data) - req = urllib.request.Request(url, data, self.headers) - if method: - req.get_method = lambda: method - # all URLs begin with hardcoded values - res = urllib.request.urlopen(req).read().decode("utf-8") # nosec - - if not log_filter_flag: - LOG.info('\n\n\n\nRequest URL: %(url)s\n\n' - 'Call Method: %(method)s\n\n' - 'Request Data: %(data)s\n\n' - 'Response Data:%(res)s\n\n', - {'url': url, - 'method': method, - 'data': data, - 'res': res}) - + res = func(url, **kwargs) except Exception as err: - LOG.error('Bad response from server: %(url)s.' - ' Error: %(err)s', {'url': url, 'err': err}) - json_msg = ('{"error":{"code": %s,"description": "Connect to ' - 'server error."}}') % constants.ERROR_CONNECT_TO_SERVER - res_json = json.loads(json_msg) - return res_json + LOG.exception('Bad response from server: %(url)s.' + ' Error: %(err)s', {'url': url, 'err': err}) + return {"error": {"code": constants.ERROR_CONNECT_TO_SERVER, + "description": "Connect to server error."}} try: - res_json = json.loads(res) - except Exception as err: - LOG.error('JSON transfer error: %s.', err) - raise + res.raise_for_status() + except requests.HTTPError as exc: + return {"error": {"code": exc.response.status_code, + "description": six.text_type(exc)}} + + res_json = res.json() + if not log_filter_flag: + LOG.info('\n\n\n\nRequest URL: %(url)s\n\n' + 'Call Method: %(method)s\n\n' + 'Request Data: %(data)s\n\n' + 'Response Data:%(res)s\n\n', + {'url': url, + 'method': method, + 'data': data, + 'res': res_json}) return res_json @@ -118,7 +115,7 @@ class RestClient(object): "password": self.san_password, "scope": "0"} self.init_http_head() - result = self.do_call(url, data, + result = self.do_call(url, data, 'POST', calltimeout=constants.LOGIN_SOCKET_TIMEOUT, log_filter_flag=True) @@ -132,7 +129,7 @@ class RestClient(object): device_id = result['data']['deviceid'] self.device_id = device_id self.url = item_url + device_id - self.headers['iBaseToken'] = result['data']['iBaseToken'] + self.session.headers['iBaseToken'] = result['data']['iBaseToken'] if (result['data']['accountstate'] in (constants.PWD_EXPIRED, constants.PWD_RESET)): self.logout() @@ -207,7 +204,7 @@ class RestClient(object): # Set the mirror switch always on lun_params['MIRRORPOLICY'] = '1' url = "/lun" - result = self.call(url, lun_params) + result = self.call(url, lun_params, 'POST') if result['error']['code'] == constants.ERROR_VOLUME_ALREADY_EXIST: lun_id = self.get_lun_id_by_name(lun_params['NAME']) if lun_id: @@ -242,7 +239,7 @@ class RestClient(object): def get_all_pools(self): url = "/storagepool" - result = self.call(url, None, log_filter_flag=True) + result = self.call(url, None, "GET", log_filter_flag=True) msg = _('Query resource pool error.') self._assert_rest_result(result, msg) self._assert_data_in_result(result, msg) @@ -308,7 +305,7 @@ class RestClient(object): data = ({"SNAPSHOTLIST": snapshot_id} if type(snapshot_id) in (list, tuple) else {"SNAPSHOTLIST": [snapshot_id]}) - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Activate snapshot error.')) def create_snapshot(self, lun_id, snapshot_name, snapshot_description): @@ -318,7 +315,7 @@ class RestClient(object): "PARENTTYPE": "11", "DESCRIPTION": snapshot_description, "PARENTID": lun_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create snapshot error.') self._assert_rest_result(result, msg) @@ -391,7 +388,7 @@ class RestClient(object): % srclunid), "TARGETLUN": ("INVALID;%s;INVALID;INVALID;INVALID" % tgtlunid)} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create luncopy error.') self._assert_rest_result(result, msg) @@ -522,7 +519,7 @@ class RestClient(object): def _get_iscsi_tgt_port(self): url = "/iscsidevicename" - result = self.call(url, None) + result = self.call(url, None, 'GET') msg = _('Get iSCSI target port error.') self._assert_rest_result(result, msg) @@ -586,7 +583,7 @@ class RestClient(object): def _create_hostgroup(self, hostgroup_name): url = "/hostgroup" data = {"TYPE": "14", "NAME": hostgroup_name} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create hostgroup error.') self._assert_rest_result(result, msg) @@ -600,7 +597,7 @@ class RestClient(object): "APPTYPE": '0', "GROUPTYPE": '0', "NAME": lungroup_name} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create lungroup error.') self._assert_rest_result(result, msg) @@ -705,7 +702,7 @@ class RestClient(object): "NAME": hostname, "OPERATIONSYSTEM": "0", "DESCRIPTION": host_name_before_hash} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add new host error.')) if 'data' in result: @@ -747,7 +744,7 @@ class RestClient(object): "ASSOCIATEOBJTYPE": "21", "ASSOCIATEOBJID": host_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Associate host to hostgroup ' 'error.')) @@ -758,7 +755,7 @@ class RestClient(object): data = {"ID": lungroup_id, "ASSOCIATEOBJTYPE": lun_type, "ASSOCIATEOBJID": lun_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Associate lun to lungroup error.')) def remove_lun_from_lungroup(self, lungroup_id, lun_id, @@ -920,7 +917,7 @@ class RestClient(object): def _add_mapping_view(self, name): url = "/mappingview" data = {"NAME": name, "TYPE": "245"} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add mapping view error.')) return result['data']['ID'] @@ -1433,9 +1430,9 @@ class RestClient(object): "CYCLESET": "[1,2,3,4,5,6,0]", } data.update(qos) - url = "/ioclass/" + url = "/ioclass" - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Create QoS policy error.')) return result['data']['ID'] @@ -1735,7 +1732,7 @@ class RestClient(object): self._assert_rest_result(result, _('Add lun to cache error.')) def get_array_info(self): - url = "/system/" + url = "/system" result = self.call(url, None, "GET", log_filter_flag=True) self._assert_rest_result(result, _('Get array info error.')) return result.get('data', None) @@ -1870,7 +1867,7 @@ class RestClient(object): url = '/fc_initiator/' data = {"TYPE": '223', "ID": ininame} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add fc initiator to array error.')) def ensure_fc_initiator_added(self, initiator_name, host_id):