diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index f3506476af..59c5cc02aa 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -629,14 +629,17 @@ use = egg:swift#bulk use = egg:swift#slo # max_manifest_segments = 1000 # max_manifest_size = 2097152 -# min_segment_size = 1048576 -# Start rate-limiting SLO segment serving after the Nth segment of a +# +# Rate limiting applies only to segments smaller than this size (bytes). +# rate_limit_under_size = 1048576 +# +# Start rate-limiting SLO segment serving after the Nth small segment of a # segmented object. # rate_limit_after_segment = 10 # # Once segment rate-limiting kicks in for an object, limit segments served # to N per second. 0 means no rate-limiting. -# rate_limit_segments_per_sec = 0 +# rate_limit_segments_per_sec = 1 # # Time limit on GET requests (seconds) # max_get_time = 86400 diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 49aa44ecd2..a5ab1085b2 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -57,12 +57,11 @@ The format of the list will be: "range": "1048576-2097151"}, ...] The number of object segments is limited to a configurable amount, default -1000. Each segment, except for the final one, must be at least 1 megabyte -(configurable). On upload, the middleware will head every segment passed in to -verify: +1000. Each segment must be at least 1 byte. On upload, the middleware will +head every segment passed in to verify: 1. the segment exists (i.e. the HEAD was successful); - 2. the segment meets minimum size requirements (if not the last segment); + 2. the segment meets minimum size requirements; 3. if the user provided a non-null etag, the etag matches; 4. if the user provided a non-null size_bytes, the size_bytes matches; and 5. if the user provided a range, it is a singular, syntactically correct range @@ -121,8 +120,9 @@ finally bytes 2095104 through 2097152 (i.e., the last 2048 bytes) of .. note:: - The minimum sized range is min_segment_size, which by - default is 1048576 (1MB). + + The minimum sized range is 1 byte. This is the same as the minimum + segment size. ------------------------- @@ -221,7 +221,7 @@ from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk -DEFAULT_MIN_SEGMENT_SIZE = 1024 * 1024 # 1 MiB +DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB DEFAULT_MAX_MANIFEST_SEGMENTS = 1000 DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB @@ -231,7 +231,7 @@ OPTIONAL_SLO_KEYS = set(['range']) ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS -def parse_and_validate_input(req_body, req_path, min_segment_size): +def parse_and_validate_input(req_body, req_path): """ Given a request body, parses it and returns a list of dictionaries. @@ -269,7 +269,6 @@ def parse_and_validate_input(req_body, req_path, min_segment_size): vrs, account, _junk = split_path(req_path, 3, 3, True) errors = [] - num_segs = len(parsed_data) for seg_index, seg_dict in enumerate(parsed_data): if not isinstance(seg_dict, dict): errors.append("Index %d: not a JSON object" % seg_index) @@ -315,10 +314,10 @@ def parse_and_validate_input(req_body, req_path, min_segment_size): except (TypeError, ValueError): errors.append("Index %d: invalid size_bytes" % seg_index) continue - if (seg_size < min_segment_size and seg_index < num_segs - 1): - errors.append("Index %d: too small; each segment, except " - "the last, must be at least %d bytes." - % (seg_index, min_segment_size)) + if seg_size < 1: + errors.append("Index %d: too small; each segment must be " + "at least 1 byte." + % (seg_index,)) continue obj_path = '/'.join(['', vrs, account, seg_dict['path'].lstrip('/')]) @@ -662,10 +661,17 @@ class SloGetContext(WSGIContext): plain_listing_iter = self._segment_listing_iterator( req, ver, account, segments) + def is_small_segment((seg_dict, start_byte, end_byte)): + start = 0 if start_byte is None else start_byte + end = int(seg_dict['bytes']) - 1 if end_byte is None else end_byte + is_small = (end - start + 1) < self.slo.rate_limit_under_size + return is_small + ratelimited_listing_iter = RateLimitedIterator( plain_listing_iter, self.slo.rate_limit_segments_per_sec, - limit_after=self.slo.rate_limit_after_segment) + limit_after=self.slo.rate_limit_after_segment, + ratelimit_if=is_small_segment) # self._segment_listing_iterator gives us 3-tuples of (segment dict, # start byte, end byte), but SegmentedIterable wants (obj path, etag, @@ -716,7 +722,7 @@ class StaticLargeObject(object): :param conf: The configuration dict for the middleware. """ - def __init__(self, app, conf, min_segment_size=DEFAULT_MIN_SEGMENT_SIZE, + def __init__(self, app, conf, max_manifest_segments=DEFAULT_MAX_MANIFEST_SEGMENTS, max_manifest_size=DEFAULT_MAX_MANIFEST_SIZE): self.conf = conf @@ -724,12 +730,13 @@ class StaticLargeObject(object): self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = max_manifest_segments self.max_manifest_size = max_manifest_size - self.min_segment_size = min_segment_size self.max_get_time = int(self.conf.get('max_get_time', 86400)) + self.rate_limit_under_size = int(self.conf.get( + 'rate_limit_under_size', DEFAULT_RATE_LIMIT_UNDER_SIZE)) self.rate_limit_after_segment = int(self.conf.get( 'rate_limit_after_segment', '10')) self.rate_limit_segments_per_sec = int(self.conf.get( - 'rate_limit_segments_per_sec', '0')) + 'rate_limit_segments_per_sec', '1')) self.bulk_deleter = Bulk(app, {}, logger=self.logger) def handle_multipart_get_or_head(self, req, start_response): @@ -783,7 +790,7 @@ class StaticLargeObject(object): raise HTTPLengthRequired(request=req) parsed_data = parse_and_validate_input( req.body_file.read(self.max_manifest_size), - req.path, self.min_segment_size) + req.path) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: @@ -812,6 +819,7 @@ class StaticLargeObject(object): new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') + if obj_path != last_obj_path: last_obj_path = obj_path head_seg_resp = \ @@ -840,12 +848,10 @@ class StaticLargeObject(object): seg_dict['range'] = '%d-%d' % (rng[0], rng[1] - 1) segment_length = rng[1] - rng[0] - if segment_length < self.min_segment_size and \ - index < len(parsed_data) - 1: + if segment_length < 1: problem_segments.append( [quote(obj_name), - 'Too small; each segment, except the last, must be ' - 'at least %d bytes.' % self.min_segment_size]) + 'Too small; each segment must be at least 1 byte.']) total_size += segment_length if seg_dict['size_bytes'] is not None and \ seg_dict['size_bytes'] != head_seg_resp.content_length: @@ -1045,18 +1051,17 @@ def filter_factory(global_conf, **local_conf): DEFAULT_MAX_MANIFEST_SEGMENTS)) max_manifest_size = int(conf.get('max_manifest_size', DEFAULT_MAX_MANIFEST_SIZE)) - min_segment_size = int(conf.get('min_segment_size', - DEFAULT_MIN_SEGMENT_SIZE)) register_swift_info('slo', max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, - min_segment_size=min_segment_size) + # this used to be configurable; report it as 1 for + # clients that might still care + min_segment_size=1) def slo_filter(app): return StaticLargeObject( app, conf, max_manifest_segments=max_manifest_segments, - max_manifest_size=max_manifest_size, - min_segment_size=min_segment_size) + max_manifest_size=max_manifest_size) return slo_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index b5bb03a3ab..8991855a50 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1044,22 +1044,27 @@ class RateLimitedIterator(object): this many elements; default is 0 (rate limit immediately) """ - def __init__(self, iterable, elements_per_second, limit_after=0): + def __init__(self, iterable, elements_per_second, limit_after=0, + ratelimit_if=lambda _junk: True): self.iterator = iter(iterable) self.elements_per_second = elements_per_second self.limit_after = limit_after self.running_time = 0 + self.ratelimit_if = ratelimit_if def __iter__(self): return self def next(self): - if self.limit_after > 0: - self.limit_after -= 1 - else: - self.running_time = ratelimit_sleep(self.running_time, - self.elements_per_second) - return next(self.iterator) + next_value = next(self.iterator) + + if self.ratelimit_if(next_value): + if self.limit_after > 0: + self.limit_after -= 1 + else: + self.running_time = ratelimit_sleep(self.running_time, + self.elements_per_second) + return next_value class GreenthreadSafeIterator(object): diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index bc6ad50fdd..1387a773b4 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -56,7 +56,7 @@ class FakeSwift(object): self.container_ring = FakeRing() self.get_object_ring = lambda policy_index: FakeRing() - def _get_response(self, method, path): + def _find_response(self, method, path): resp = self._responses[(method, path)] if isinstance(resp, list): try: @@ -84,16 +84,17 @@ class FakeSwift(object): self.swift_sources.append(env.get('swift.source')) try: - resp_class, raw_headers, body = self._get_response(method, path) + resp_class, raw_headers, body = self._find_response(method, path) headers = swob.HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): - resp_class, raw_headers, body = self._get_response( + resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = swob.HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: - resp_class, raw_headers, body = self._get_response('GET', path) + resp_class, raw_headers, body = self._find_response( + 'GET', path) body = None headers = swob.HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index f1bafe11cf..3d8788f6af 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -55,8 +55,8 @@ def md5hex(s): class SloTestCase(unittest.TestCase): def setUp(self): self.app = FakeSwift() - self.slo = slo.filter_factory({})(self.app) - self.slo.min_segment_size = 1 + slo_conf = {'rate_limit_under_size': '0'} + self.slo = slo.filter_factory(slo_conf)(self.app) self.slo.logger = self.app.logger def call_app(self, req, app=None, expect_exception=False): @@ -120,18 +120,14 @@ class TestSloMiddleware(SloTestCase): resp.startswith('X-Static-Large-Object is a reserved header')) def _put_bogus_slo(self, manifest_text, - manifest_path='/v1/a/c/the-manifest', - min_segment_size=1): + manifest_path='/v1/a/c/the-manifest'): with self.assertRaises(HTTPException) as catcher: - slo.parse_and_validate_input(manifest_text, manifest_path, - min_segment_size) + slo.parse_and_validate_input(manifest_text, manifest_path) self.assertEqual(400, catcher.exception.status_int) return catcher.exception.body - def _put_slo(self, manifest_text, manifest_path='/v1/a/c/the-manifest', - min_segment_size=1): - return slo.parse_and_validate_input(manifest_text, manifest_path, - min_segment_size) + def _put_slo(self, manifest_text, manifest_path='/v1/a/c/the-manifest'): + return slo.parse_and_validate_input(manifest_text, manifest_path) def test_bogus_input(self): self.assertEqual('Manifest must be valid JSON.\n', @@ -248,19 +244,18 @@ class TestSloMiddleware(SloTestCase): def test_bogus_input_undersize_segment(self): self.assertEqual( - "Index 1: too small; each segment, except the last, " - "must be at least 1000 bytes.\n" - "Index 2: too small; each segment, except the last, " - "must be at least 1000 bytes.\n", + "Index 1: too small; each segment " + "must be at least 1 byte.\n" + "Index 2: too small; each segment " + "must be at least 1 byte.\n", self._put_bogus_slo( json.dumps([ - {'path': u'/c/s1', 'etag': 'a', 'size_bytes': 1000}, - {'path': u'/c/s2', 'etag': 'b', 'size_bytes': 999}, - {'path': u'/c/s3', 'etag': 'c', 'size_bytes': 998}, + {'path': u'/c/s1', 'etag': 'a', 'size_bytes': 1}, + {'path': u'/c/s2', 'etag': 'b', 'size_bytes': 0}, + {'path': u'/c/s3', 'etag': 'c', 'size_bytes': 0}, # No error for this one since size_bytes is unspecified {'path': u'/c/s4', 'etag': 'd', 'size_bytes': None}, - {'path': u'/c/s5', 'etag': 'e', 'size_bytes': 996}]), - min_segment_size=1000)) + {'path': u'/c/s5', 'etag': 'e', 'size_bytes': 1000}]))) def test_valid_input(self): data = json.dumps( @@ -268,19 +263,19 @@ class TestSloMiddleware(SloTestCase): 'size_bytes': 100}]) self.assertEqual( '/cont/object', - slo.parse_and_validate_input(data, '/v1/a/cont/man', 1)[0]['path']) + slo.parse_and_validate_input(data, '/v1/a/cont/man')[0]['path']) data = json.dumps( [{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100, 'range': '0-40'}]) - parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man', 1) + parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man') self.assertEqual('/cont/object', parsed[0]['path']) self.assertEqual([(0, 40)], parsed[0]['range'].ranges) data = json.dumps( [{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': None, 'range': '0-40'}]) - parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man', 1) + parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man') self.assertEqual('/cont/object', parsed[0]['path']) self.assertEqual(None, parsed[0]['size_bytes']) self.assertEqual([(0, 40)], parsed[0]['range'].ranges) @@ -316,6 +311,11 @@ class TestSloPutManifest(SloTestCase): swob.HTTPOk, {'Content-Length': '10', 'Etag': 'etagoftheobjectsegment'}, None) + self.app.register( + 'HEAD', '/v1/AUTH_test/cont/empty_object', + swob.HTTPOk, + {'Content-Length': '0', 'Etag': 'etagoftheobjectsegment'}, + None) self.app.register( 'HEAD', u'/v1/AUTH_test/cont/あ_1', swob.HTTPOk, @@ -340,11 +340,17 @@ class TestSloPutManifest(SloTestCase): {'Content-Length': '2', 'Etag': 'b', 'Last-Modified': 'Fri, 01 Feb 2012 20:38:36 GMT'}, None) + + _manifest_json = json.dumps( + [{'name': '/checktest/a_5', 'hash': md5hex("a" * 5), + 'content_type': 'text/plain', 'bytes': '5'}]) self.app.register( 'GET', '/v1/AUTH_test/checktest/slob', swob.HTTPOk, - {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag'}, - None) + {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag', + 'Content-Type': 'cat/picture;swift_bytes=12345', + 'Content-Length': len(_manifest_json)}, + _manifest_json) self.app.register( 'PUT', '/v1/AUTH_test/checktest/man_3', swob.HTTPCreated, {}, None) @@ -367,21 +373,6 @@ class TestSloPutManifest(SloTestCase): pass self.assertEqual(e.status_int, 413) - with patch.object(self.slo, 'min_segment_size', 1000): - test_json_data_2obj = json.dumps( - [{'path': '/cont/small_object1', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}, - {'path': '/cont/small_object2', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank('/v1/a/c/o', body=test_json_data_2obj) - try: - self.slo.handle_multipart_put(req, fake_start_response) - except HTTPException as e: - pass - self.assertEqual(e.status_int, 400) - req = Request.blank('/v1/a/c/o', headers={'X-Copy-From': 'lala'}) try: self.slo.handle_multipart_put(req, fake_start_response) @@ -411,49 +402,29 @@ class TestSloPutManifest(SloTestCase): self.slo(req.environ, my_fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) - def test_handle_multipart_put_success_allow_small_last_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank( - '/v1/AUTH_test/c/man?multipart-manifest=put', - environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, - body=test_json_data) - self.assertTrue('X-Static-Large-Object' not in req.headers) - self.slo(req.environ, fake_start_response) - self.assertTrue('X-Static-Large-Object' in req.headers) + def test_handle_multipart_put_disallow_empty_first_segment(self): + test_json_data = json.dumps([{'path': '/cont/object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 0}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}]) + req = Request.blank('/v1/a/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as catcher: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(catcher.exception.status_int, 400) - def test_handle_multipart_put_success_allow_only_one_small_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank( - '/v1/AUTH_test/c/man?multipart-manifest=put', - environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, - body=test_json_data) - self.assertTrue('X-Static-Large-Object' not in req.headers) - self.slo(req.environ, fake_start_response) - self.assertTrue('X-Static-Large-Object' in req.headers) - - def test_handle_multipart_put_disallow_small_first_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}]) - req = Request.blank('/v1/a/c/o', body=test_json_data) - try: - self.slo.handle_multipart_put(req, fake_start_response) - except HTTPException as e: - pass - self.assertEqual(e.status_int, 400) + def test_handle_multipart_put_disallow_empty_last_segment(self): + test_json_data = json.dumps([{'path': '/cont/object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 0}]) + req = Request.blank('/v1/a/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as catcher: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(catcher.exception.status_int, 400) def test_handle_multipart_put_success_unicode(self): test_json_data = json.dumps([{'path': u'/cont/object\u2661', @@ -543,7 +514,7 @@ class TestSloPutManifest(SloTestCase): {'path': '/checktest/badreq', 'etag': 'a', 'size_bytes': '1'}, {'path': '/checktest/b_2', 'etag': 'not-b', 'size_bytes': '2'}, {'path': '/checktest/slob', 'etag': 'not-slob', - 'size_bytes': '2'}]) + 'size_bytes': '12345'}]) req = Request.blank( '/v1/AUTH_test/checktest/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, @@ -553,6 +524,7 @@ class TestSloPutManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(self.app.call_count, 5) errors = json.loads(body)['Errors'] + self.assertEqual(len(errors), 5) self.assertEqual(errors[0][0], '/checktest/a_1') self.assertEqual(errors[0][1], 'Size Mismatch') @@ -587,35 +559,33 @@ class TestSloPutManifest(SloTestCase): self.assertEqual(2, manifest_data[1]['bytes']) def test_handle_multipart_put_skip_size_check_still_uses_min_size(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': None}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}]) - req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) - with self.assertRaises(HTTPException) as cm: - self.slo.handle_multipart_put(req, fake_start_response) - self.assertEqual(cm.exception.status_int, 400) + test_json_data = json.dumps([{'path': '/cont/empty_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': None}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}]) + req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as cm: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(cm.exception.status_int, 400) def test_handle_multipart_put_skip_size_check_no_early_bailout(self): - with patch.object(self.slo, 'min_segment_size', 50): - # The first is too small (it's 10 bytes but min size is 50), and - # the second has a bad etag. Make sure both errors show up in - # the response. - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': None}, - {'path': '/cont/object2', - 'etag': 'wrong wrong wrong', - 'size_bytes': 100}]) - req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) - with self.assertRaises(HTTPException) as cm: - self.slo.handle_multipart_put(req, fake_start_response) - self.assertEqual(cm.exception.status_int, 400) - self.assertIn('at least 50 bytes', cm.exception.body) - self.assertIn('Etag Mismatch', cm.exception.body) + # The first is too small (it's 0 bytes), and + # the second has a bad etag. Make sure both errors show up in + # the response. + test_json_data = json.dumps([{'path': '/cont/empty_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': None}, + {'path': '/cont/object2', + 'etag': 'wrong wrong wrong', + 'size_bytes': 100}]) + req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as cm: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(cm.exception.status_int, 400) + self.assertIn('at least 1 byte', cm.exception.body) + self.assertIn('Etag Mismatch', cm.exception.body) def test_handle_multipart_put_skip_etag_check(self): good_data = json.dumps( @@ -1126,6 +1096,46 @@ class TestSloGetManifest(SloTestCase): swob.HTTPOk, {'Content-Length': '20', 'Etag': md5hex('d' * 20)}, 'd' * 20) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/e_25', + swob.HTTPOk, {'Content-Length': '25', + 'Etag': md5hex('e' * 25)}, + 'e' * 25) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/f_30', + swob.HTTPOk, {'Content-Length': '30', + 'Etag': md5hex('f' * 30)}, + 'f' * 30) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/g_35', + swob.HTTPOk, {'Content-Length': '35', + 'Etag': md5hex('g' * 35)}, + 'g' * 35) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/h_40', + swob.HTTPOk, {'Content-Length': '40', + 'Etag': md5hex('h' * 40)}, + 'h' * 40) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/i_45', + swob.HTTPOk, {'Content-Length': '45', + 'Etag': md5hex('i' * 45)}, + 'i' * 45) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/j_50', + swob.HTTPOk, {'Content-Length': '50', + 'Etag': md5hex('j' * 50)}, + 'j' * 50) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/k_55', + swob.HTTPOk, {'Content-Length': '55', + 'Etag': md5hex('k' * 55)}, + 'k' * 55) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/l_60', + swob.HTTPOk, {'Content-Length': '60', + 'Etag': md5hex('l' * 60)}, + 'l' * 60) _bc_manifest_json = json.dumps( [{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10', @@ -1156,6 +1166,39 @@ class TestSloGetManifest(SloTestCase): 'Etag': md5(_abcd_manifest_json).hexdigest()}, _abcd_manifest_json) + _abcdefghijkl_manifest_json = json.dumps( + [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), + 'content_type': 'text/plain', 'bytes': '5'}, + {'name': '/gettest/b_10', 'hash': md5hex("b" * 10), + 'content_type': 'text/plain', 'bytes': '10'}, + {'name': '/gettest/c_15', 'hash': md5hex("c" * 15), + 'content_type': 'text/plain', 'bytes': '15'}, + {'name': '/gettest/d_20', 'hash': md5hex("d" * 20), + 'content_type': 'text/plain', 'bytes': '20'}, + {'name': '/gettest/e_25', 'hash': md5hex("e" * 25), + 'content_type': 'text/plain', 'bytes': '25'}, + {'name': '/gettest/f_30', 'hash': md5hex("f" * 30), + 'content_type': 'text/plain', 'bytes': '30'}, + {'name': '/gettest/g_35', 'hash': md5hex("g" * 35), + 'content_type': 'text/plain', 'bytes': '35'}, + {'name': '/gettest/h_40', 'hash': md5hex("h" * 40), + 'content_type': 'text/plain', 'bytes': '40'}, + {'name': '/gettest/i_45', 'hash': md5hex("i" * 45), + 'content_type': 'text/plain', 'bytes': '45'}, + {'name': '/gettest/j_50', 'hash': md5hex("j" * 50), + 'content_type': 'text/plain', 'bytes': '50'}, + {'name': '/gettest/k_55', 'hash': md5hex("k" * 55), + 'content_type': 'text/plain', 'bytes': '55'}, + {'name': '/gettest/l_60', 'hash': md5hex("l" * 60), + 'content_type': 'text/plain', 'bytes': '60'}]) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/manifest-abcdefghijkl', + swob.HTTPOk, { + 'Content-Type': 'application/json', + 'X-Static-Large-Object': 'true', + 'Etag': md5(_abcdefghijkl_manifest_json).hexdigest()}, + _abcdefghijkl_manifest_json) + self.manifest_abcd_etag = md5hex( md5hex("a" * 5) + md5hex(md5hex("b" * 10) + md5hex("c" * 15)) + md5hex("d" * 20)) @@ -1361,6 +1404,65 @@ class TestSloGetManifest(SloTestCase): 'bytes=0-14,0-14', 'bytes=0-19,0-19']) + def test_get_manifest_ratelimiting(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcdefghijkl', + environ={'REQUEST_METHOD': 'GET'}) + + the_time = [time.time()] + sleeps = [] + + def mock_time(): + return the_time[0] + + def mock_sleep(duration): + sleeps.append(duration) + the_time[0] += duration + + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 999999999), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0, 2.0, 2.0]) + + # give the client the first 4 segments without ratelimiting; we'll + # sleep less + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 999999999), \ + patch.object(self.slo, 'rate_limit_after_segment', 4): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + + # ratelimit segments under 35 bytes; this affects a-f + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 35), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0]) + + # ratelimit segments under 36 bytes; this now affects a-g, netting + # us one more sleep than before + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 36), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + def test_if_none_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', @@ -2508,8 +2610,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) - self.assertEqual(swift_info['slo'].get('min_segment_size'), - mware.min_segment_size) + self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) self.assertEqual(swift_info['slo'].get('max_manifest_size'), mware.max_manifest_size) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 8f5f82eff0..f68e6b3b78 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -3912,6 +3912,26 @@ class TestRateLimitedIterator(unittest.TestCase): # first element. self.assertEqual(len(got), 11) + def test_rate_limiting_sometimes(self): + + def testfunc(): + limited_iterator = utils.RateLimitedIterator( + range(9999), 100, + ratelimit_if=lambda item: item % 23 != 0) + got = [] + started_at = time.time() + try: + while time.time() - started_at < 0.5: + got.append(next(limited_iterator)) + except StopIteration: + pass + return got + + got = self.run_under_pseudo_time(testfunc) + # we'd get 51 without the ratelimit_if, but because 0, 23 and 46 + # weren't subject to ratelimiting, we get 54 instead + self.assertEqual(len(got), 54) + def test_limit_after(self): def testfunc():