openstack/swift
Code Issues Proposed changes

Add Storage Policy Support to list_endpoints

Browse Source 
This patch makes list_endpoints policy aware so that the object is
looked up in the right ring and its actual locations returned to the
client.

DocImpact
Implements: blueprint storage-policies
Change-Id: I56d4b0f4f321a4c72b11ec44d868a194d02ea3a3
This commit is contained in:
Paul Luse 2014-03-17 17:34:35 -07:00 committed by Clay Gerrard
parent ad2a9cefe5
commit 019d7f5cda
2 changed files with 102 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
swift/common/middleware/list_endpoints.py
View File

@ -61,6 +61,8 @@ from swift.common.ring import Ring
from swift.common.utils import json, get_logger, split_path from swift.common.utils import json, get_logger, split_path
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
from swift.common.storage_policy import POLICIES
from swift.proxy.controllers.base import get_container_info
class ListEndpointsMiddleware(object): class ListEndpointsMiddleware(object):
@ -79,17 +81,24 @@ class ListEndpointsMiddleware(object):
def __init__(self, app, conf): def __init__(self, app, conf):
self.app = app self.app = app
self.logger = get_logger(conf, log_route='endpoints') self.logger = get_logger(conf, log_route='endpoints')
swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = Ring(swift_dir, ring_name='account') self.account_ring = Ring(self.swift_dir, ring_name='account')
self.container_ring = Ring(swift_dir, ring_name='container') self.container_ring = Ring(self.swift_dir, ring_name='container')
self.object_ring = Ring(swift_dir, ring_name='object')
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/') self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
if not self.endpoints_path.endswith('/'): if not self.endpoints_path.endswith('/'):
self.endpoints_path += '/' self.endpoints_path += '/'
def get_object_ring(self, policy_idx):
"""
Get the ring object to use to handle a request based on its policy.
:policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def __call__(self, env, start_response): def __call__(self, env, start_response):
request = Request(env) request = Request(env)
if not request.path.startswith(self.endpoints_path): if not request.path.startswith(self.endpoints_path):
return self.app(env, start_response) return self.app(env, start_response)
@ -112,7 +121,16 @@ class ListEndpointsMiddleware(object):
obj = unquote(obj) obj = unquote(obj)
if obj is not None: if obj is not None:
partition, nodes = self.object_ring.get_nodes( # remove 'endpoints' from call to get_container_info
stripped = request.environ
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
self.endpoints_path:
stripped['PATH_INFO'] = "/v1/" + \
stripped['PATH_INFO'][len(self.endpoints_path):]
container_info = get_container_info(
stripped, self.app, swift_source='LE')
obj_ring = self.get_object_ring(container_info['storage_policy'])
partition, nodes = obj_ring.get_nodes(
account, container, obj) account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}/{obj}' '{account}/{container}/{obj}'

105
test/unit/common/middleware/test_list_endpoints.py
View File

@ -19,10 +19,13 @@ from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
import os import os
import mock
from swift.common import ring, utils from swift.common import ring, utils
from swift.common.utils import json from swift.common.utils import json, split_path
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
from swift.common.middleware import list_endpoints from swift.common.middleware import list_endpoints
from swift.common.storage_policy import StoragePolicy, POLICIES
from test.unit import patch_policies
class FakeApp(object): class FakeApp(object):
@ -34,6 +37,8 @@ def start_response(*args):
pass pass
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
class TestListEndpoints(unittest.TestCase): class TestListEndpoints(unittest.TestCase):
def setUp(self): def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_SUFFIX = 'endcap'
@ -43,6 +48,9 @@ class TestListEndpoints(unittest.TestCase):
accountgz = os.path.join(self.testdir, 'account.ring.gz') accountgz = os.path.join(self.testdir, 'account.ring.gz')
containergz = os.path.join(self.testdir, 'container.ring.gz') containergz = os.path.join(self.testdir, 'container.ring.gz')
objectgz = os.path.join(self.testdir, 'object.ring.gz') objectgz = os.path.join(self.testdir, 'object.ring.gz')
objectgz_1 = os.path.join(self.testdir, 'object-1.ring.gz')
self.policy_to_test = 0
self.expected_path = ('v1', 'a', 'c', 'o1')
# Let's make the rings slightly different so we can test # Let's make the rings slightly different so we can test
# that the correct ring is consulted (e.g. we don't consult # that the correct ring is consulted (e.g. we don't consult
@ -59,6 +67,10 @@ class TestListEndpoints(unittest.TestCase):
array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]),
array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]),
array.array('H', [3, 4, 3, 4])] array.array('H', [3, 4, 3, 4])]
intended_replica2part2dev_id_o_1 = [
array.array('H', [1, 0, 1, 0]),
array.array('H', [1, 0, 1, 0]),
array.array('H', [4, 3, 4, 3])]
intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000, 'ip': '10.1.1.1', 'port': 6000,
'device': 'sda1'}, 'device': 'sda1'},
@ -79,6 +91,8 @@ class TestListEndpoints(unittest.TestCase):
intended_devs, intended_part_shift).save(containergz) intended_devs, intended_part_shift).save(containergz)
ring.RingData(intended_replica2part2dev_id_o, ring.RingData(intended_replica2part2dev_id_o,
intended_devs, intended_part_shift).save(objectgz) intended_devs, intended_part_shift).save(objectgz)
ring.RingData(intended_replica2part2dev_id_o_1,
intended_devs, intended_part_shift).save(objectgz_1)
self.app = FakeApp() self.app = FakeApp()
self.list_endpoints = list_endpoints.filter_factory( self.list_endpoints = list_endpoints.filter_factory(
@ -87,6 +101,26 @@ class TestListEndpoints(unittest.TestCase):
def tearDown(self): def tearDown(self):
rmtree(self.testdir, ignore_errors=1) rmtree(self.testdir, ignore_errors=1)
def FakeGetInfo(self, env, app, swift_source=None):
info = {'status': 0, 'sync_key': None, 'meta': {},
'cors': {'allow_origin': None, 'expose_headers': None,
'max_age': None}, 'sysmeta': {}, 'read_acl': None,
'object_count': None, 'write_acl': None, 'versions': None,
'bytes': None}
info['storage_policy'] = self.policy_to_test
(version, account, container, unused) = \
split_path(env['PATH_INFO'], 3, 4, True)
self.assertEquals((version, account, container, unused),
self.expected_path)
return info
def test_get_object_ring(self):
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0),
ring.Ring), True)
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(1),
ring.Ring), True)
self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99)
def test_get_endpoint(self): def test_get_endpoint(self):
# Expected results for objects taken from test_ring # Expected results for objects taken from test_ring
# Expected results for others computed by manually invoking # Expected results for others computed by manually invoking
@ -100,6 +134,23 @@ class TestListEndpoints(unittest.TestCase):
"http://10.1.2.2:6000/sdd1/1/a/c/o1" "http://10.1.2.2:6000/sdd1/1/a/c/o1"
]) ])
# test policies with default endpoint name
expected = [[
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"], [
"http://10.1.1.1:6000/sda1/1/a/c/o1",
"http://10.1.2.1:6000/sdc1/1/a/c/o1"
]]
PATCHGI = 'swift.common.middleware.list_endpoints.get_container_info'
for pol in POLICIES:
self.policy_to_test = pol.idx
with mock.patch(PATCHGI, self.FakeGetInfo):
resp = Request.blank('/endpoints/a/c/o1').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
# Here, 'o1/' is the object name. # Here, 'o1/' is the object name.
resp = Request.blank('/endpoints/a/c/o1/').get_response( resp = Request.blank('/endpoints/a/c/o1/').get_response(
self.list_endpoints) self.list_endpoints)
@ -166,33 +217,33 @@ class TestListEndpoints(unittest.TestCase):
self.assertEquals(resp.status, '200 OK') self.assertEquals(resp.status, '200 OK')
self.assertEquals(resp.body, 'FakeApp') self.assertEquals(resp.body, 'FakeApp')
# test custom path with trailing slash # test policies with custom endpoint name
custom_path_le = list_endpoints.filter_factory({ for pol in POLICIES:
'swift_dir': self.testdir, # test custom path with trailing slash
'list_endpoints_path': '/some/another/path/' custom_path_le = list_endpoints.filter_factory({
})(self.app) 'swift_dir': self.testdir,
resp = Request.blank('/some/another/path/a/c/o1') \ 'list_endpoints_path': '/some/another/path/'
.get_response(custom_path_le) })(self.app)
self.assertEquals(resp.status_int, 200) self.policy_to_test = pol.idx
self.assertEquals(resp.content_type, 'application/json') with mock.patch(PATCHGI, self.FakeGetInfo):
self.assertEquals(json.loads(resp.body), [ resp = Request.blank('/some/another/path/a/c/o1') \
"http://10.1.1.1:6000/sdb1/1/a/c/o1", .get_response(custom_path_le)
"http://10.1.2.2:6000/sdd1/1/a/c/o1" self.assertEquals(resp.status_int, 200)
]) self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
# test ustom path without trailing slash # test ustom path without trailing slash
custom_path_le = list_endpoints.filter_factory({ custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir, 'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path' 'list_endpoints_path': '/some/another/path'
})(self.app) })(self.app)
resp = Request.blank('/some/another/path/a/c/o1') \ self.policy_to_test = pol.idx
.get_response(custom_path_le) with mock.patch(PATCHGI, self.FakeGetInfo):
self.assertEquals(resp.status_int, 200) resp = Request.blank('/some/another/path/a/c/o1') \
self.assertEquals(resp.content_type, 'application/json') .get_response(custom_path_le)
self.assertEquals(json.loads(resp.body), [ self.assertEquals(resp.status_int, 200)
"http://10.1.1.1:6000/sdb1/1/a/c/o1", self.assertEquals(resp.content_type, 'application/json')
"http://10.1.2.2:6000/sdd1/1/a/c/o1" self.assertEquals(json.loads(resp.body), expected[pol.idx])
])
if __name__ == '__main__': if __name__ == '__main__':