From 019d7f5cda388c3053e369ca70902d03b72cca22 Mon Sep 17 00:00:00 2001 From: Paul Luse Date: Mon, 17 Mar 2014 17:34:35 -0700 Subject: [PATCH] Add Storage Policy Support to list_endpoints This patch makes list_endpoints policy aware so that the object is looked up in the right ring and its actual locations returned to the client. DocImpact Implements: blueprint storage-policies Change-Id: I56d4b0f4f321a4c72b11ec44d868a194d02ea3a3 --- swift/common/middleware/list_endpoints.py | 30 ++++- .../common/middleware/test_list_endpoints.py | 105 +++++++++++++----- 2 files changed, 102 insertions(+), 33 deletions(-) diff --git a/swift/common/middleware/list_endpoints.py b/swift/common/middleware/list_endpoints.py index 72e179d49b..fbe3fad1da 100644 --- a/swift/common/middleware/list_endpoints.py +++ b/swift/common/middleware/list_endpoints.py @@ -61,6 +61,8 @@ from swift.common.ring import Ring from swift.common.utils import json, get_logger, split_path from swift.common.swob import Request, Response from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed +from swift.common.storage_policy import POLICIES +from swift.proxy.controllers.base import get_container_info class ListEndpointsMiddleware(object): @@ -79,17 +81,24 @@ class ListEndpointsMiddleware(object): def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='endpoints') - swift_dir = conf.get('swift_dir', '/etc/swift') - self.account_ring = Ring(swift_dir, ring_name='account') - self.container_ring = Ring(swift_dir, ring_name='container') - self.object_ring = Ring(swift_dir, ring_name='object') + self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.account_ring = Ring(self.swift_dir, ring_name='account') + self.container_ring = Ring(self.swift_dir, ring_name='container') self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/') if not self.endpoints_path.endswith('/'): self.endpoints_path += '/' + def get_object_ring(self, policy_idx): + """ + Get the ring object to use to handle a request based on its policy. + + :policy_idx: policy index as defined in swift.conf + :returns: appropriate ring object + """ + return POLICIES.get_object_ring(policy_idx, self.swift_dir) + def __call__(self, env, start_response): request = Request(env) - if not request.path.startswith(self.endpoints_path): return self.app(env, start_response) @@ -112,7 +121,16 @@ class ListEndpointsMiddleware(object): obj = unquote(obj) if obj is not None: - partition, nodes = self.object_ring.get_nodes( + # remove 'endpoints' from call to get_container_info + stripped = request.environ + if stripped['PATH_INFO'][:len(self.endpoints_path)] == \ + self.endpoints_path: + stripped['PATH_INFO'] = "/v1/" + \ + stripped['PATH_INFO'][len(self.endpoints_path):] + container_info = get_container_info( + stripped, self.app, swift_source='LE') + obj_ring = self.get_object_ring(container_info['storage_policy']) + partition, nodes = obj_ring.get_nodes( account, container, obj) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}/{obj}' diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py index 218394ad00..52c3dc60ca 100644 --- a/test/unit/common/middleware/test_list_endpoints.py +++ b/test/unit/common/middleware/test_list_endpoints.py @@ -19,10 +19,13 @@ from tempfile import mkdtemp from shutil import rmtree import os +import mock from swift.common import ring, utils -from swift.common.utils import json +from swift.common.utils import json, split_path from swift.common.swob import Request, Response from swift.common.middleware import list_endpoints +from swift.common.storage_policy import StoragePolicy, POLICIES +from test.unit import patch_policies class FakeApp(object): @@ -34,6 +37,8 @@ def start_response(*args): pass +@patch_policies([StoragePolicy(0, 'zero', False), + StoragePolicy(1, 'one', True)]) class TestListEndpoints(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' @@ -43,6 +48,9 @@ class TestListEndpoints(unittest.TestCase): accountgz = os.path.join(self.testdir, 'account.ring.gz') containergz = os.path.join(self.testdir, 'container.ring.gz') objectgz = os.path.join(self.testdir, 'object.ring.gz') + objectgz_1 = os.path.join(self.testdir, 'object-1.ring.gz') + self.policy_to_test = 0 + self.expected_path = ('v1', 'a', 'c', 'o1') # Let's make the rings slightly different so we can test # that the correct ring is consulted (e.g. we don't consult @@ -59,6 +67,10 @@ class TestListEndpoints(unittest.TestCase): array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]), array.array('H', [3, 4, 3, 4])] + intended_replica2part2dev_id_o_1 = [ + array.array('H', [1, 0, 1, 0]), + array.array('H', [1, 0, 1, 0]), + array.array('H', [4, 3, 4, 3])] intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'device': 'sda1'}, @@ -79,6 +91,8 @@ class TestListEndpoints(unittest.TestCase): intended_devs, intended_part_shift).save(containergz) ring.RingData(intended_replica2part2dev_id_o, intended_devs, intended_part_shift).save(objectgz) + ring.RingData(intended_replica2part2dev_id_o_1, + intended_devs, intended_part_shift).save(objectgz_1) self.app = FakeApp() self.list_endpoints = list_endpoints.filter_factory( @@ -87,6 +101,26 @@ class TestListEndpoints(unittest.TestCase): def tearDown(self): rmtree(self.testdir, ignore_errors=1) + def FakeGetInfo(self, env, app, swift_source=None): + info = {'status': 0, 'sync_key': None, 'meta': {}, + 'cors': {'allow_origin': None, 'expose_headers': None, + 'max_age': None}, 'sysmeta': {}, 'read_acl': None, + 'object_count': None, 'write_acl': None, 'versions': None, + 'bytes': None} + info['storage_policy'] = self.policy_to_test + (version, account, container, unused) = \ + split_path(env['PATH_INFO'], 3, 4, True) + self.assertEquals((version, account, container, unused), + self.expected_path) + return info + + def test_get_object_ring(self): + self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0), + ring.Ring), True) + self.assertEquals(isinstance(self.list_endpoints.get_object_ring(1), + ring.Ring), True) + self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99) + def test_get_endpoint(self): # Expected results for objects taken from test_ring # Expected results for others computed by manually invoking @@ -100,6 +134,23 @@ class TestListEndpoints(unittest.TestCase): "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) + # test policies with default endpoint name + expected = [[ + "http://10.1.1.1:6000/sdb1/1/a/c/o1", + "http://10.1.2.2:6000/sdd1/1/a/c/o1"], [ + "http://10.1.1.1:6000/sda1/1/a/c/o1", + "http://10.1.2.1:6000/sdc1/1/a/c/o1" + ]] + PATCHGI = 'swift.common.middleware.list_endpoints.get_container_info' + for pol in POLICIES: + self.policy_to_test = pol.idx + with mock.patch(PATCHGI, self.FakeGetInfo): + resp = Request.blank('/endpoints/a/c/o1').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), expected[pol.idx]) + # Here, 'o1/' is the object name. resp = Request.blank('/endpoints/a/c/o1/').get_response( self.list_endpoints) @@ -166,33 +217,33 @@ class TestListEndpoints(unittest.TestCase): self.assertEquals(resp.status, '200 OK') self.assertEquals(resp.body, 'FakeApp') - # test custom path with trailing slash - custom_path_le = list_endpoints.filter_factory({ - 'swift_dir': self.testdir, - 'list_endpoints_path': '/some/another/path/' - })(self.app) - resp = Request.blank('/some/another/path/a/c/o1') \ - .get_response(custom_path_le) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'application/json') - self.assertEquals(json.loads(resp.body), [ - "http://10.1.1.1:6000/sdb1/1/a/c/o1", - "http://10.1.2.2:6000/sdd1/1/a/c/o1" - ]) + # test policies with custom endpoint name + for pol in POLICIES: + # test custom path with trailing slash + custom_path_le = list_endpoints.filter_factory({ + 'swift_dir': self.testdir, + 'list_endpoints_path': '/some/another/path/' + })(self.app) + self.policy_to_test = pol.idx + with mock.patch(PATCHGI, self.FakeGetInfo): + resp = Request.blank('/some/another/path/a/c/o1') \ + .get_response(custom_path_le) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), expected[pol.idx]) - # test ustom path without trailing slash - custom_path_le = list_endpoints.filter_factory({ - 'swift_dir': self.testdir, - 'list_endpoints_path': '/some/another/path' - })(self.app) - resp = Request.blank('/some/another/path/a/c/o1') \ - .get_response(custom_path_le) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'application/json') - self.assertEquals(json.loads(resp.body), [ - "http://10.1.1.1:6000/sdb1/1/a/c/o1", - "http://10.1.2.2:6000/sdd1/1/a/c/o1" - ]) + # test ustom path without trailing slash + custom_path_le = list_endpoints.filter_factory({ + 'swift_dir': self.testdir, + 'list_endpoints_path': '/some/another/path' + })(self.app) + self.policy_to_test = pol.idx + with mock.patch(PATCHGI, self.FakeGetInfo): + resp = Request.blank('/some/another/path/a/c/o1') \ + .get_response(custom_path_le) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), expected[pol.idx]) if __name__ == '__main__':