diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst index ceabfd527e..7413911e87 100644 --- a/doc/source/overview_container_sync.rst +++ b/doc/source/overview_container_sync.rst @@ -45,6 +45,12 @@ synchronization key. are being synced, then you should follow the instructions for :ref:`symlink_container_sync_client_config` to be compatible with symlinks. + Be aware that symlinks may be synced before their targets even if they are + in the same container and were created after the target objects. In such + cases, a GET for the symlink will fail with a ``404 Not Found`` error. If + the target has been overwritten, a GET may produce an older version (for + dynamic links) or a ``409 Conflict`` error (for static links). + -------------------------- Configuring Container Sync -------------------------- diff --git a/swift/common/middleware/container_sync.py b/swift/common/middleware/container_sync.py index f005a40a43..bde33ca70c 100644 --- a/swift/common/middleware/container_sync.py +++ b/swift/common/middleware/container_sync.py @@ -15,6 +15,7 @@ import os +from swift.common.constraints import valid_api_version from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify from swift.common.utils import ( @@ -67,8 +68,27 @@ class ContainerSync(object): @wsgify def __call__(self, req): + if req.path == '/info': + # Ensure /info requests get the freshest results + self.register_info() + return self.app + + try: + (version, acc, cont, obj) = req.split_path(3, 4, True) + bad_path = False + except ValueError: + bad_path = True + + # use of bad_path bool is to avoid recursive tracebacks + if bad_path or not valid_api_version(version): + return self.app + + # validate container-sync metdata update + info = get_container_info( + req.environ, self.app, swift_source='CS') + sync_to = req.headers.get('x-container-sync-to') + if not self.allow_full_urls: - sync_to = req.headers.get('x-container-sync-to') if sync_to and not sync_to.startswith('//'): raise HTTPBadRequest( body='Full URLs are not allowed for X-Container-Sync-To ' @@ -90,8 +110,6 @@ class ContainerSync(object): req.environ.setdefault('swift.log_info', []).append( 'cs:no-local-realm-key') else: - info = get_container_info( - req.environ, self.app, swift_source='CS') user_key = info.get('sync_key') if not user_key: req.environ.setdefault('swift.log_info', []).append( @@ -134,10 +152,9 @@ class ContainerSync(object): # syntax and might be synced before its segments, so stop SLO # middleware from performing the usual manifest validation. req.environ['swift.slo_override'] = True + # Similar arguments for static symlinks + req.environ['swift.symlink_override'] = True - if req.path == '/info': - # Ensure /info requests get the freshest results - self.register_info() return self.app diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 0dc75419e9..1854e712ad 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -506,6 +506,12 @@ class SymlinkObjectContext(WSGIContext): def _validate_etag_and_update_sysmeta(self, req, symlink_target_path, etag): + if req.environ.get('swift.symlink_override'): + req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag + req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \ + req.headers[TGT_BYTES_SYMLINK_HDR] + return + # next we'll make sure the E-Tag matches a real object new_req = make_subrequest( req.environ, path=wsgi_quote(symlink_target_path), method='HEAD', diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 87da0aa5fc..831864aba1 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -179,6 +179,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'status': status_int, 'read_acl': headers.get('x-container-read'), 'write_acl': headers.get('x-container-write'), + 'sync_to': headers.get('x-container-sync-to'), 'sync_key': headers.get('x-container-sync-key'), 'object_count': headers.get('x-container-object-count'), 'bytes': headers.get('x-container-bytes-used'), diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index 6d8156088e..3f87f9b57f 100644 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -147,6 +147,9 @@ class TestContainerFailures(ReplProbeTest): def run_test(num_locks, catch_503): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) + # Get the container info into memcache (so no stray + # get_container_info calls muck up our timings) + client.get_container(self.url, self.token, container) db_files = self._get_container_db_files(container) db_conns = [] for i in range(num_locks): diff --git a/test/probe/test_container_sync.py b/test/probe/test_container_sync.py index f04941c444..96f63065c6 100644 --- a/test/probe/test_container_sync.py +++ b/test/probe/test_container_sync.py @@ -562,6 +562,182 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): self.url, self.token, dest_container, symlink_name) self.assertEqual(target_body, actual_target_body) + def test_sync_static_symlink_different_container(self): + source_container, dest_container = self._setup_synced_containers() + + symlink_cont = 'symlink-container-%s' % uuid.uuid4() + client.put_container(self.url, self.token, symlink_cont) + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, symlink_cont, target_name, + target_body) + + # upload a regular object + regular_name = 'regular-%s' % uuid.uuid4() + regular_body = b'regular body' + client.put_object( + self.url, self.token, source_container, regular_name, + regular_body) + + # static symlink + target_path = '%s/%s' % (symlink_cont, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + content_location = resp_headers['content-location'] + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, regular_name) + self.assertEqual(regular_body, actual_target_body) + + # static symlink gets synced, too + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + self.assertEqual(content_location, resp_headers['content-location']) + + def test_sync_busted_static_symlink_different_container(self): + source_container, dest_container = self._setup_synced_containers() + + symlink_cont = 'symlink-container-%s' % uuid.uuid4() + client.put_container(self.url, self.token, symlink_cont) + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, symlink_cont, target_name, + target_body) + + # upload a regular object + regular_name = 'regular-%s' % uuid.uuid4() + regular_body = b'regular body' + client.put_object( + self.url, self.token, source_container, regular_name, + regular_body) + + # static symlink + target_path = '%s/%s' % (symlink_cont, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + content_location = resp_headers['content-location'] + + # Break the link + client.put_object( + self.url, self.token, symlink_cont, target_name, + b'something else') + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, regular_name) + self.assertEqual(regular_body, actual_target_body) + + # static symlink gets synced, too, even though the target's different! + with self.assertRaises(ClientException) as cm: + client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(409, cm.exception.http_status) + resp_headers = cm.exception.http_response_headers + self.assertIn('content-location', resp_headers) + self.assertEqual(content_location, resp_headers['content-location']) + + def test_sync_static_symlink(self): + source_container, dest_container = self._setup_synced_containers() + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, source_container, target_name, + target_body) + + # static symlink + target_path = '%s/%s' % (source_container, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, target_name) + self.assertEqual(target_body, actual_target_body) + + # and static link too + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + if __name__ == "__main__": unittest.main() diff --git a/test/unit/common/middleware/test_container_sync.py b/test/unit/common/middleware/test_container_sync.py index ec030d7490..15e33dce55 100644 --- a/test/unit/common/middleware/test_container_sync.py +++ b/test/unit/common/middleware/test_container_sync.py @@ -216,6 +216,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertIn('cs:invalid-sig', req.environ.get('swift.log_info')) self.assertNotIn('swift.authorize_override', req.environ) self.assertNotIn('swift.slo_override', req.environ) + self.assertNotIn('swift.symlink_override', req.environ) def test_valid_sig(self): ts = '1455221706.726999_0123456789abcdef' @@ -235,6 +236,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual(ts, resp.headers['X-Timestamp']) self.assertIn('swift.authorize_override', req.environ) self.assertIn('swift.slo_override', req.environ) + self.assertIn('swift.symlink_override', req.environ) def test_valid_sig2(self): sig = self.sync.realms_conf.get_sig( @@ -250,6 +252,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertIn('cs:valid', req.environ.get('swift.log_info')) self.assertIn('swift.authorize_override', req.environ) self.assertIn('swift.slo_override', req.environ) + self.assertIn('swift.symlink_override', req.environ) def test_info(self): req = swob.Request.blank('/info') diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py index 44ba9b2cf3..c008e23c98 100644 --- a/test/unit/common/middleware/test_symlink.py +++ b/test/unit/common/middleware/test_symlink.py @@ -243,6 +243,21 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase): # ... we better have a body! self.assertIn(b'Internal Error', body) + def test_symlink_simple_put_to_non_existing_object_override(self): + self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPNotFound, {}) + self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={ + 'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Etag': 'some-tgt-etag', + # this header isn't normally sent with PUT + 'X-Symlink-Target-Bytes': '13', + }, body='') + # this can be set in container_sync + req.environ['swift.symlink_override'] = True + status, headers, body = self.call_sym(req) + self.assertEqual(status, '201 Created') + def test_symlink_put_with_prevalidated_etag(self): self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) req = Request.blank('/v1/a/c/symlink', method='PUT', headers={ diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 2f97380bc9..e2aa93487f 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -639,6 +639,8 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp['status'], 404) self.assertIsNone(resp['read_acl']) self.assertIsNone(resp['write_acl']) + self.assertIsNone(resp['sync_key']) + self.assertIsNone(resp['sync_to']) def test_headers_to_container_info_meta(self): headers = {'X-Container-Meta-Whatevs': 14, @@ -662,11 +664,14 @@ class TestFuncs(unittest.TestCase): 'x-container-read': 'readvalue', 'x-container-write': 'writevalue', 'x-container-sync-key': 'keyvalue', + 'x-container-sync-to': '//r/c/a/c', 'x-container-meta-access-control-allow-origin': 'here', } resp = headers_to_container_info(headers.items(), 200) self.assertEqual(resp['read_acl'], 'readvalue') self.assertEqual(resp['write_acl'], 'writevalue') + self.assertEqual(resp['sync_key'], 'keyvalue') + self.assertEqual(resp['sync_to'], '//r/c/a/c') self.assertEqual(resp['cors']['allow_origin'], 'here') headers['x-unused-header'] = 'blahblahblah'