Add Storage Policy Support to list_endpoints
This patch makes list_endpoints policy aware so that the object is looked up in the right ring and its actual locations returned to the client. DocImpact Implements: blueprint storage-policies Change-Id: I56d4b0f4f321a4c72b11ec44d868a194d02ea3a3
This commit is contained in:
Paul Luse
Clay Gerrard
parent
ad2a9cefe5
commit
019d7f5cda
@ -61,6 +61,8 @@ from swift.common.ring import Ring
|
||||
from swift.common.utils import json, get_logger, split_path
|
||||
from swift.common.swob import Request, Response
|
||||
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.proxy.controllers.base import get_container_info
|
||||
|
||||
|
||||
class ListEndpointsMiddleware(object):
|
||||
@ -79,17 +81,24 @@ class ListEndpointsMiddleware(object):
|
||||
def __init__(self, app, conf):
|
||||
self.app = app
|
||||
self.logger = get_logger(conf, log_route='endpoints')
|
||||
swift_dir = conf.get('swift_dir', '/etc/swift')
|
||||
self.account_ring = Ring(swift_dir, ring_name='account')
|
||||
self.container_ring = Ring(swift_dir, ring_name='container')
|
||||
self.object_ring = Ring(swift_dir, ring_name='object')
|
||||
self.swift_dir = conf.get('swift_dir', '/etc/swift')
|
||||
self.account_ring = Ring(self.swift_dir, ring_name='account')
|
||||
self.container_ring = Ring(self.swift_dir, ring_name='container')
|
||||
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
|
||||
if not self.endpoints_path.endswith('/'):
|
||||
self.endpoints_path += '/'
|
||||
|
||||
def get_object_ring(self, policy_idx):
|
||||
"""
|
||||
Get the ring object to use to handle a request based on its policy.
|
||||
|
||||
:policy_idx: policy index as defined in swift.conf
|
||||
:returns: appropriate ring object
|
||||
"""
|
||||
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
request = Request(env)
|
||||
|
||||
if not request.path.startswith(self.endpoints_path):
|
||||
return self.app(env, start_response)
|
||||
|
||||
@ -112,7 +121,16 @@ class ListEndpointsMiddleware(object):
|
||||
obj = unquote(obj)
|
||||
|
||||
if obj is not None:
|
||||
partition, nodes = self.object_ring.get_nodes(
|
||||
# remove 'endpoints' from call to get_container_info
|
||||
stripped = request.environ
|
||||
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
|
||||
self.endpoints_path:
|
||||
stripped['PATH_INFO'] = "/v1/" + \
|
||||
stripped['PATH_INFO'][len(self.endpoints_path):]
|
||||
container_info = get_container_info(
|
||||
stripped, self.app, swift_source='LE')
|
||||
obj_ring = self.get_object_ring(container_info['storage_policy'])
|
||||
partition, nodes = obj_ring.get_nodes(
|
||||
account, container, obj)
|
||||
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
|
||||
'{account}/{container}/{obj}'
|
||||
|
||||
@ -19,10 +19,13 @@ from tempfile import mkdtemp
|
||||
from shutil import rmtree
|
||||
|
||||
import os
|
||||
import mock
|
||||
from swift.common import ring, utils
|
||||
from swift.common.utils import json
|
||||
from swift.common.utils import json, split_path
|
||||
from swift.common.swob import Request, Response
|
||||
from swift.common.middleware import list_endpoints
|
||||
from swift.common.storage_policy import StoragePolicy, POLICIES
|
||||
from test.unit import patch_policies
|
||||
|
||||
|
||||
class FakeApp(object):
|
||||
@ -34,6 +37,8 @@ def start_response(*args):
|
||||
pass
|
||||
|
||||
|
||||
@patch_policies([StoragePolicy(0, 'zero', False),
|
||||
StoragePolicy(1, 'one', True)])
|
||||
class TestListEndpoints(unittest.TestCase):
|
||||
def setUp(self):
|
||||
utils.HASH_PATH_SUFFIX = 'endcap'
|
||||
@ -43,6 +48,9 @@ class TestListEndpoints(unittest.TestCase):
|
||||
accountgz = os.path.join(self.testdir, 'account.ring.gz')
|
||||
containergz = os.path.join(self.testdir, 'container.ring.gz')
|
||||
objectgz = os.path.join(self.testdir, 'object.ring.gz')
|
||||
objectgz_1 = os.path.join(self.testdir, 'object-1.ring.gz')
|
||||
self.policy_to_test = 0
|
||||
self.expected_path = ('v1', 'a', 'c', 'o1')
|
||||
|
||||
# Let's make the rings slightly different so we can test
|
||||
# that the correct ring is consulted (e.g. we don't consult
|
||||
@ -59,6 +67,10 @@ class TestListEndpoints(unittest.TestCase):
|
||||
array.array('H', [0, 1, 0, 1]),
|
||||
array.array('H', [0, 1, 0, 1]),
|
||||
array.array('H', [3, 4, 3, 4])]
|
||||
intended_replica2part2dev_id_o_1 = [
|
||||
array.array('H', [1, 0, 1, 0]),
|
||||
array.array('H', [1, 0, 1, 0]),
|
||||
array.array('H', [4, 3, 4, 3])]
|
||||
intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0,
|
||||
'ip': '10.1.1.1', 'port': 6000,
|
||||
'device': 'sda1'},
|
||||
@ -79,6 +91,8 @@ class TestListEndpoints(unittest.TestCase):
|
||||
intended_devs, intended_part_shift).save(containergz)
|
||||
ring.RingData(intended_replica2part2dev_id_o,
|
||||
intended_devs, intended_part_shift).save(objectgz)
|
||||
ring.RingData(intended_replica2part2dev_id_o_1,
|
||||
intended_devs, intended_part_shift).save(objectgz_1)
|
||||
|
||||
self.app = FakeApp()
|
||||
self.list_endpoints = list_endpoints.filter_factory(
|
||||
@ -87,6 +101,26 @@ class TestListEndpoints(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
rmtree(self.testdir, ignore_errors=1)
|
||||
|
||||
def FakeGetInfo(self, env, app, swift_source=None):
|
||||
info = {'status': 0, 'sync_key': None, 'meta': {},
|
||||
'cors': {'allow_origin': None, 'expose_headers': None,
|
||||
'max_age': None}, 'sysmeta': {}, 'read_acl': None,
|
||||
'object_count': None, 'write_acl': None, 'versions': None,
|
||||
'bytes': None}
|
||||
info['storage_policy'] = self.policy_to_test
|
||||
(version, account, container, unused) = \
|
||||
split_path(env['PATH_INFO'], 3, 4, True)
|
||||
self.assertEquals((version, account, container, unused),
|
||||
self.expected_path)
|
||||
return info
|
||||
|
||||
def test_get_object_ring(self):
|
||||
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0),
|
||||
ring.Ring), True)
|
||||
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(1),
|
||||
ring.Ring), True)
|
||||
self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99)
|
||||
|
||||
def test_get_endpoint(self):
|
||||
# Expected results for objects taken from test_ring
|
||||
# Expected results for others computed by manually invoking
|
||||
@ -100,6 +134,23 @@ class TestListEndpoints(unittest.TestCase):
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
|
||||
])
|
||||
|
||||
# test policies with default endpoint name
|
||||
expected = [[
|
||||
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"], [
|
||||
"http://10.1.1.1:6000/sda1/1/a/c/o1",
|
||||
"http://10.1.2.1:6000/sdc1/1/a/c/o1"
|
||||
]]
|
||||
PATCHGI = 'swift.common.middleware.list_endpoints.get_container_info'
|
||||
for pol in POLICIES:
|
||||
self.policy_to_test = pol.idx
|
||||
with mock.patch(PATCHGI, self.FakeGetInfo):
|
||||
resp = Request.blank('/endpoints/a/c/o1').get_response(
|
||||
self.list_endpoints)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), expected[pol.idx])
|
||||
|
||||
# Here, 'o1/' is the object name.
|
||||
resp = Request.blank('/endpoints/a/c/o1/').get_response(
|
||||
self.list_endpoints)
|
||||
@ -166,33 +217,33 @@ class TestListEndpoints(unittest.TestCase):
|
||||
self.assertEquals(resp.status, '200 OK')
|
||||
self.assertEquals(resp.body, 'FakeApp')
|
||||
|
||||
# test custom path with trailing slash
|
||||
custom_path_le = list_endpoints.filter_factory({
|
||||
'swift_dir': self.testdir,
|
||||
'list_endpoints_path': '/some/another/path/'
|
||||
})(self.app)
|
||||
resp = Request.blank('/some/another/path/a/c/o1') \
|
||||
.get_response(custom_path_le)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), [
|
||||
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
|
||||
])
|
||||
# test policies with custom endpoint name
|
||||
for pol in POLICIES:
|
||||
# test custom path with trailing slash
|
||||
custom_path_le = list_endpoints.filter_factory({
|
||||
'swift_dir': self.testdir,
|
||||
'list_endpoints_path': '/some/another/path/'
|
||||
})(self.app)
|
||||
self.policy_to_test = pol.idx
|
||||
with mock.patch(PATCHGI, self.FakeGetInfo):
|
||||
resp = Request.blank('/some/another/path/a/c/o1') \
|
||||
.get_response(custom_path_le)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), expected[pol.idx])
|
||||
|
||||
# test ustom path without trailing slash
|
||||
custom_path_le = list_endpoints.filter_factory({
|
||||
'swift_dir': self.testdir,
|
||||
'list_endpoints_path': '/some/another/path'
|
||||
})(self.app)
|
||||
resp = Request.blank('/some/another/path/a/c/o1') \
|
||||
.get_response(custom_path_le)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), [
|
||||
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
|
||||
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
|
||||
])
|
||||
# test ustom path without trailing slash
|
||||
custom_path_le = list_endpoints.filter_factory({
|
||||
'swift_dir': self.testdir,
|
||||
'list_endpoints_path': '/some/another/path'
|
||||
})(self.app)
|
||||
self.policy_to_test = pol.idx
|
||||
with mock.patch(PATCHGI, self.FakeGetInfo):
|
||||
resp = Request.blank('/some/another/path/a/c/o1') \
|
||||
.get_response(custom_path_le)
|
||||
self.assertEquals(resp.status_int, 200)
|
||||
self.assertEquals(resp.content_type, 'application/json')
|
||||
self.assertEquals(json.loads(resp.body), expected[pol.idx])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user