github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/pfs_middleware/tests/test_pfs_middleware.py (about) 1 # Copyright (c) 2015-2021, NVIDIA CORPORATION. 2 # SPDX-License-Identifier: Apache-2.0 3 4 5 import base64 6 import collections 7 import hashlib 8 import json 9 import mock 10 import unittest 11 from io import BytesIO 12 from swift.common import swob 13 from xml.etree import ElementTree 14 15 import pfs_middleware.middleware as mware 16 import pfs_middleware.bimodal_checker as bimodal_checker 17 from . import helpers 18 19 20 class FakeLogger(object): 21 def critical(fmt, *args): 22 pass 23 24 def exception(fmt, *args): 25 pass 26 27 def error(fmt, *args): 28 pass 29 30 def warn(fmt, *args): 31 pass 32 33 def info(fmt, *args): 34 pass 35 36 def debug(fmt, *args, **kwargs): 37 pass 38 39 40 class TestHelpers(unittest.TestCase): 41 def test_deserialize_metadata(self): 42 self.assertEqual(mware.deserialize_metadata(None), {}) 43 self.assertEqual(mware.deserialize_metadata(''), {}) 44 self.assertEqual(mware.deserialize_metadata('{}'), {}) 45 self.assertEqual(mware.deserialize_metadata('{"foo": "bar"}'), 46 {"foo": "bar"}) 47 self.assertEqual(mware.deserialize_metadata( 48 '{"unicode-\\u1234":"meta \\ud83c\\udf34"}' 49 ), { 50 # NB: WSGI strings 51 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', 52 }) 53 54 def test_serialize_metadata(self): 55 self.assertEqual(mware.serialize_metadata({}), '{}') 56 self.assertEqual( 57 mware.serialize_metadata({ 58 # NB: WSGI strings 59 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', 60 }), 61 # But it comes out as good Unicode 62 '{"unicode-\\u1234": "meta \\ud83c\\udf34"}') 63 64 65 class BaseMiddlewareTest(unittest.TestCase): 66 # no test cases in here, just common setup and utility functions 67 def setUp(self): 68 super(BaseMiddlewareTest, self).setUp() 69 self.app = helpers.FakeProxy() 70 self.pfs = mware.PfsMiddleware(self.app, { 71 'bypass_mode': 'read-only', 72 }, FakeLogger()) 73 self.bimodal_checker = bimodal_checker.BimodalChecker(self.pfs, { 74 'bimodal_recheck_interval': 'inf', # avoid timing dependencies 75 }, FakeLogger()) 76 self.bimodal_accounts = {"AUTH_test"} 77 78 self.app.register('HEAD', '/v1/AUTH_test', 204, 79 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 80 '') 81 82 self.swift_info = { 83 # some stuff omitted 84 "slo": { 85 "max_manifest_segments": 1000, 86 "max_manifest_size": 2097152, 87 "min_segment_size": 1}, 88 "swift": { 89 "account_autocreate": True, 90 "account_listing_limit": 9876, # note: not default 91 "allow_account_management": True, 92 "container_listing_limit": 6543, # note: not default 93 "extra_header_count": 0, 94 "max_account_name_length": 128, 95 "max_container_name_length": 256, 96 "max_file_size": 5368709122, 97 "max_header_size": 8192, 98 "max_meta_count": 90, 99 "max_meta_name_length": 128, 100 "max_meta_overall_size": 4096, 101 "max_meta_value_length": 256, 102 "max_object_name_length": 1024, 103 "policies": [{ 104 "aliases": "default", 105 "default": True, 106 "name": "default", 107 }, { 108 "aliases": "not-default", 109 "name": "not-default", 110 }], 111 "strict_cors_mode": True, 112 "version": "2.9.1.dev47" 113 }, 114 "tempauth": {"account_acls": True}} 115 116 self.app.register( 117 'GET', '/info', 118 200, {'Content-Type': 'application/json'}, 119 json.dumps(self.swift_info)) 120 121 for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): 122 self.app.register( 123 method, '/v1/AUTH_test//o', 124 412, {'Content-Type': 'text/html'}, 'Bad URL') 125 126 self.fake_rpc = helpers.FakeJsonRpc() 127 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 128 lambda *_: self.fake_rpc) 129 patcher.start() 130 self.addCleanup(patcher.stop) 131 132 # For the sake of the listing format tests, assume old Swift 133 patcher = mock.patch( 134 'pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', False) 135 patcher.start() 136 self.addCleanup(patcher.stop) 137 138 def fake_RpcIsAccountBimodal(request): 139 return { 140 "error": None, 141 "result": { 142 "IsBimodal": 143 request["AccountName"] in self.bimodal_accounts, 144 "ActivePeerPrivateIPAddr": "127.0.0.1", 145 }} 146 147 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 148 fake_RpcIsAccountBimodal) 149 150 def call_app(self, req, app=None, expect_exception=False): 151 # Normally this happens in eventlet.wsgi.HttpProtocol.get_environ(). 152 req.environ.setdefault('CONTENT_TYPE', None) 153 154 if app is None: 155 app = self.app 156 157 status = [None] 158 headers = [None] 159 160 def start_response(s, h, ei=None): 161 status[0] = s 162 headers[0] = swob.HeaderKeyDict(h) 163 164 body_iter = app(req.environ, start_response) 165 body = b'' 166 caught_exc = None 167 try: 168 try: 169 for chunk in body_iter: 170 body += chunk 171 finally: 172 # WSGI says we have to do this. Plus, if we don't, then 173 # our leak detector gets false positives. 174 if callable(getattr(body_iter, 'close', None)): 175 body_iter.close() 176 except Exception as exc: 177 if expect_exception: 178 caught_exc = exc 179 else: 180 raise 181 182 if expect_exception: 183 return status[0], headers[0], body, caught_exc 184 else: 185 return status[0], headers[0], body 186 187 def call_pfs(self, req, **kwargs): 188 return self.call_app(req, app=self.bimodal_checker, **kwargs) 189 190 191 class TestAccountGet(BaseMiddlewareTest): 192 def setUp(self): 193 super(TestAccountGet, self).setUp() 194 195 self.app.register( 196 'HEAD', '/v1/AUTH_test', 197 204, 198 {'X-Account-Meta-Flavor': 'cherry', 199 'X-Account-Sysmeta-Shipping-Class': 'ultraslow', 200 'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 201 '') 202 203 def mock_RpcGetAccount(_): 204 return { 205 "error": None, 206 "result": { 207 "ModificationTime": 1498766381451119000, 208 "AccountEntries": [{ 209 "Basename": "chickens", 210 "ModificationTime": 1510958440808682000, 211 }, { 212 "Basename": "cows", 213 "ModificationTime": 1510958450657045000, 214 }, { 215 "Basename": "goats", 216 "ModificationTime": 1510958452544251000, 217 }, { 218 "Basename": "pigs", 219 "ModificationTime": 1510958459200130000, 220 }], 221 }} 222 223 self.fake_rpc.register_handler( 224 "Server.RpcGetAccount", mock_RpcGetAccount) 225 226 def test_headers(self): 227 req = swob.Request.blank("/v1/AUTH_test") 228 status, headers, _ = self.call_pfs(req) 229 self.assertEqual(status, '200 OK') 230 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 231 self.assertEqual(headers.get("X-Timestamp"), "1498766381.45112") 232 233 # These we just lie about for now 234 self.assertEqual( 235 headers.get("X-Account-Object-Count"), "0") 236 self.assertEqual( 237 headers.get("X-Account-Storage-Policy-Default-Object-Count"), "0") 238 self.assertEqual( 239 headers.get("X-Account-Bytes-Used"), "0") 240 self.assertEqual( 241 headers.get("X-Account-Storage-Policy-Default-Bytes-Used"), "0") 242 243 # We pretend all our containers are in the default storage policy. 244 self.assertEqual( 245 headers.get("X-Account-Container-Count"), "4") 246 self.assertEqual( 247 headers.get("X-Account-Storage-Policy-Default-Container-Count"), 248 "4") 249 250 def test_escape_hatch(self): 251 self.app.register( 252 'GET', '/v1/AUTH_test', 200, {}, 253 '000000000000DACA\n000000000000DACC\n') 254 255 # Using path 256 req = swob.Request.blank("/proxyfs/AUTH_test", environ={ 257 'swift_owner': True}) 258 status, _, body = self.call_pfs(req) 259 self.assertEqual(status, '200 OK') 260 self.assertEqual(body, b'000000000000DACA\n000000000000DACC\n') 261 262 # Non-bypass 263 req = swob.Request.blank("/v1/AUTH_test", environ={ 264 'swift_owner': True}) 265 status, _, body = self.call_pfs(req) 266 self.assertEqual(status, '200 OK') 267 self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') 268 269 req = swob.Request.blank("/proxyfs/AUTH_test", method='PUT', 270 environ={'swift_owner': True}) 271 status, _, _ = self.call_pfs(req) 272 self.assertEqual(status, '405 Method Not Allowed') 273 274 # Non-owner 275 req = swob.Request.blank("/proxyfs/AUTH_test") 276 status, _, body = self.call_pfs(req) 277 self.assertEqual(status, '200 OK') 278 self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') 279 280 def test_text(self): 281 req = swob.Request.blank("/v1/AUTH_test") 282 status, headers, body = self.call_pfs(req) 283 self.assertEqual(status, '200 OK') 284 self.assertEqual(body, b"chickens\ncows\ngoats\npigs\n") 285 286 def test_json(self): 287 self.maxDiff = None 288 289 req = swob.Request.blank("/v1/AUTH_test?format=json") 290 status, headers, body = self.call_pfs(req) 291 self.assertEqual(status, '200 OK') 292 self.assertEqual(json.loads(body), [{ 293 "bytes": 0, 294 "count": 0, 295 "last_modified": "2017-11-17T22:40:40.808682", 296 "name": "chickens", 297 }, { 298 "bytes": 0, 299 "count": 0, 300 "last_modified": "2017-11-17T22:40:50.657045", 301 "name": "cows", 302 }, { 303 "bytes": 0, 304 "count": 0, 305 "last_modified": "2017-11-17T22:40:52.544251", 306 "name": "goats", 307 }, { 308 "bytes": 0, 309 "count": 0, 310 "last_modified": "2017-11-17T22:40:59.200130", 311 "name": "pigs" 312 }]) 313 314 def test_xml(self): 315 req = swob.Request.blank("/v1/AUTH_test?format=xml") 316 status, headers, body = self.call_pfs(req) 317 self.assertEqual(status, '200 OK') 318 self.assertEqual(headers["Content-Type"], 319 "application/xml; charset=utf-8") 320 self.assertTrue(body.startswith( 321 b"""<?xml version='1.0' encoding='utf-8'?>""")) 322 323 root_node = ElementTree.fromstring(body) 324 self.assertEqual(root_node.tag, 'account') 325 self.assertEqual(root_node.attrib["name"], 'AUTH_test') 326 327 containers = list(root_node) 328 self.assertEqual(containers[0].tag, 'container') 329 330 # The XML account listing doesn't use XML attributes for data, but 331 # rather a sequence of tags like <name>X</name> <bytes>Y</bytes> ... 332 con_attr_tags = list(containers[0]) 333 self.assertEqual(len(con_attr_tags), 4) 334 335 name_node = con_attr_tags[0] 336 self.assertEqual(name_node.tag, 'name') 337 self.assertEqual(name_node.text, 'chickens') 338 self.assertEqual(name_node.attrib, {}) # nothing extra in there 339 340 count_node = con_attr_tags[1] 341 self.assertEqual(count_node.tag, 'count') 342 self.assertEqual(count_node.text, '0') 343 self.assertEqual(count_node.attrib, {}) 344 345 bytes_node = con_attr_tags[2] 346 self.assertEqual(bytes_node.tag, 'bytes') 347 self.assertEqual(bytes_node.text, '0') 348 self.assertEqual(bytes_node.attrib, {}) 349 350 lm_node = con_attr_tags[3] 351 self.assertEqual(lm_node.tag, 'last_modified') 352 self.assertEqual(lm_node.text, '2017-11-17T22:40:40.808682') 353 self.assertEqual(lm_node.attrib, {}) 354 355 def test_metadata(self): 356 req = swob.Request.blank("/v1/AUTH_test") 357 status, headers, body = self.call_pfs(req) 358 self.assertEqual(headers.get("X-Account-Meta-Flavor"), "cherry") 359 self.assertEqual(headers.get("X-Account-Sysmeta-Shipping-Class"), 360 "ultraslow") 361 362 def test_account_acl(self): 363 def do_test(acl, expected_header, swift_owner): 364 head_resp_hdrs = {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'} 365 if acl is not None: 366 head_resp_hdrs['X-Account-Sysmeta-Core-Access-Control'] = acl 367 self.app.register( 368 'HEAD', '/v1/AUTH_test', 204, head_resp_hdrs, '') 369 req = swob.Request.blank("/v1/AUTH_test", 370 environ={"swift_owner": swift_owner}) 371 status, headers, body = self.call_pfs(req) 372 self.assertEqual(status, "200 OK") 373 actual_header = headers.get("X-Account-Access-Control") 374 self.assertEqual(expected_header, actual_header) 375 376 # swift_owner 377 do_test(None, None, True) 378 do_test('', None, True) 379 do_test('not a dict', None, True) 380 do_test('{"admin": ["someone"]}', '{"admin":["someone"]}', True) 381 # not swift_owner 382 do_test(None, None, False) 383 do_test('', None, False) 384 do_test('not a dict', None, False) 385 do_test('{"admin": ["someone"]}', None, False) 386 387 def test_marker(self): 388 req = swob.Request.blank("/v1/AUTH_test?marker=mk") 389 status, headers, body = self.call_pfs(req) 390 self.assertEqual(status, '200 OK') 391 self.assertEqual(2, len(self.fake_rpc.calls)) 392 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 393 # relevant to what we're testing here 394 self.assertEqual(self.fake_rpc.calls[1][1][0]['Marker'], 'mk') 395 396 def test_end_marker(self): 397 req = swob.Request.blank("/v1/AUTH_test?end_marker=mk") 398 status, headers, body = self.call_pfs(req) 399 self.assertEqual(status, '200 OK') 400 self.assertEqual(2, len(self.fake_rpc.calls)) 401 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 402 # relevant to what we're testing here 403 self.assertEqual(self.fake_rpc.calls[1][1][0]['EndMarker'], 'mk') 404 405 def test_limit(self): 406 req = swob.Request.blank("/v1/AUTH_test?limit=101") 407 status, headers, body = self.call_pfs(req) 408 self.assertEqual(status, '200 OK') 409 self.assertEqual(2, len(self.fake_rpc.calls)) 410 self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 101) 411 412 def test_default_limit(self): 413 req = swob.Request.blank("/v1/AUTH_test") 414 status, headers, body = self.call_pfs(req) 415 self.assertEqual(status, '200 OK') 416 self.assertEqual(2, len(self.fake_rpc.calls)) 417 # value from GET /info 418 self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 9876) 419 420 def test_not_found(self): 421 self.app.register('HEAD', '/v1/AUTH_missing', 404, {}, '') 422 self.app.register('GET', '/v1/AUTH_missing', 404, {}, '') 423 req = swob.Request.blank("/v1/AUTH_missing") 424 status, headers, body = self.call_pfs(req) 425 self.assertEqual(status, '404 Not Found') 426 427 def test_rpc_error(self): 428 def broken_RpcGetAccount(_): 429 return { 430 "error": "errno: 123", # meaningless errno 431 "result": None} 432 433 self.fake_rpc.register_handler( 434 "Server.RpcGetAccount", broken_RpcGetAccount) 435 req = swob.Request.blank("/v1/AUTH_test") 436 status, headers, body = self.call_pfs(req) 437 self.assertEqual(status, '500 Internal Error') 438 439 def test_empty_last_page(self): 440 def last_page_RpcGetAccount(_): 441 return { 442 "error": None, 443 # None, not [], for mysterious reasons 444 "result": { 445 "ModificationTime": 1510966502886466000, 446 "AccountEntries": None, 447 }} 448 449 self.fake_rpc.register_handler( 450 "Server.RpcGetAccount", last_page_RpcGetAccount) 451 req = swob.Request.blank("/v1/AUTH_test?marker=zzz") 452 status, headers, body = self.call_pfs(req) 453 self.assertEqual(status, '204 No Content') 454 self.assertEqual(body, b'') 455 456 def test_spaces(self): 457 self.bimodal_accounts.add('AUTH_test with spaces') 458 self.app.register('HEAD', '/v1/AUTH_test with spaces', 204, 459 {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 460 '') 461 req = swob.Request.blank("/v1/AUTH_test with spaces") 462 status, headers, body = self.call_pfs(req) 463 self.assertEqual(status, '200 OK') 464 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 465 '/v1/AUTH_test with spaces') 466 467 def test_empty(self): 468 def mock_RpcGetAccount(_): 469 return { 470 "error": None, 471 "result": { 472 "ModificationTime": 1510966489413995000, 473 "AccountEntries": []}} 474 475 self.fake_rpc.register_handler( 476 "Server.RpcGetAccount", mock_RpcGetAccount) 477 req = swob.Request.blank("/v1/AUTH_test") 478 status, headers, body = self.call_pfs(req) 479 self.assertEqual(status, '204 No Content') 480 481 482 class TestAccountHead(BaseMiddlewareTest): 483 def setUp(self): 484 super(TestAccountHead, self).setUp() 485 486 self.app.register( 487 'HEAD', '/v1/AUTH_test', 488 204, 489 {'X-Account-Meta-Beans': 'lots of', 490 'X-Account-Sysmeta-Proxyfs-Bimodal': 'true'}, 491 '') 492 493 def test_indicator_header(self): 494 req = swob.Request.blank("/v1/AUTH_test", 495 environ={"REQUEST_METHOD": "HEAD"}) 496 status, headers, body = self.call_pfs(req) 497 self.assertEqual(status, '204 No Content') 498 self.assertEqual(headers.get("ProxyFS-Enabled"), "yes") 499 self.assertEqual(body, b'') 500 501 def test_in_transit(self): 502 503 def fake_RpcIsAccountBimodal(request): 504 return { 505 "error": None, 506 "result": { 507 "IsBimodal": True, 508 "ActivePeerPrivateIPAddr": ""}} 509 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 510 fake_RpcIsAccountBimodal) 511 512 req = swob.Request.blank("/v1/AUTH_test", 513 environ={"REQUEST_METHOD": "HEAD"}) 514 status, _, _ = self.call_pfs(req) 515 self.assertEqual(status, '503 Service Unavailable') 516 517 518 class TestObjectGet(BaseMiddlewareTest): 519 def setUp(self): 520 super(TestObjectGet, self).setUp() 521 522 def dummy_rpc(request): 523 return {"error": None, "result": {}} 524 525 self.fake_rpc.register_handler("Server.RpcRenewLease", dummy_rpc) 526 self.fake_rpc.register_handler("Server.RpcReleaseLease", dummy_rpc) 527 528 def test_info_passthrough(self): 529 self.app.register( 530 'GET', '/info', 200, {}, '{"stuff": "yes"}') 531 532 req = swob.Request.blank('/info') 533 status, headers, body = self.call_pfs(req) 534 self.assertEqual(status, '200 OK') 535 self.assertEqual(body, b'{"stuff": "yes"}') 536 537 def test_non_bimodal_account(self): 538 self.app.register( 539 'HEAD', '/v1/AUTH_unimodal', 204, {}, '') 540 self.app.register( 541 'GET', '/v1/AUTH_unimodal/c/o', 200, {}, 'squirrel') 542 543 req = swob.Request.blank('/v1/AUTH_unimodal/c/o') 544 status, headers, body = self.call_pfs(req) 545 self.assertEqual(status, '200 OK') 546 self.assertEqual(body, b'squirrel') 547 548 def test_GET_basic(self): 549 # ProxyFS log segments look a lot like actual file contents followed 550 # by a bunch of fairly opaque binary data plus some JSON-looking 551 # bits. The actual bytes in the file here are the 8 that spell 552 # "burritos"; the rest of the bytes are there so we can omit them in 553 # the response to the user. 554 self.app.register( 555 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', 556 200, {}, 557 ("burritos\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 558 559 def mock_RpcGetObject(get_object_req): 560 self.assertEqual(get_object_req['VirtPath'], 561 "/v1/AUTH_test/notes/lunch") 562 self.assertEqual(get_object_req['ReadEntsIn'], []) 563 564 return { 565 "error": None, 566 "result": { 567 "FileSize": 8, 568 "Metadata": "", 569 "InodeNumber": 1245, 570 "NumWrites": 2424, 571 "ModificationTime": 1481152134331862558, 572 "IsDir": False, 573 "LeaseId": "prominority-sarcocyst", 574 "ReadEntsOut": [{ 575 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 576 "Name/0000000000c11fbd"), 577 "Offset": 0, 578 "Length": 8}]}} 579 580 self.fake_rpc.register_handler( 581 "Server.RpcGetObject", mock_RpcGetObject) 582 583 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 584 status, headers, body = self.call_pfs(req) 585 586 self.assertEqual(status, '200 OK') 587 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 588 self.assertEqual(headers["Last-Modified"], 589 "Wed, 07 Dec 2016 23:08:55 GMT") 590 self.assertEqual(headers["ETag"], 591 mware.construct_etag("AUTH_test", 1245, 2424)) 592 self.assertEqual(body, b'burritos') 593 594 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=on', 595 environ={'swift_owner': True}) 596 status, headers, body = self.call_pfs(req) 597 598 self.assertEqual(status, '200 OK') 599 self.assertEqual(headers['Content-Type'], 'application/json') 600 self.assertEqual(headers['X-Object-Content-Type'], 601 'application/octet-stream') 602 self.assertEqual(headers['X-Object-Content-Length'], '8') 603 self.assertIn('ETag', headers) 604 self.assertIn('Last-Modified', headers) 605 self.assertEqual(json.loads(body), [{ 606 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 607 "/0000000000c11fbd"), 608 "Offset": 0, 609 "Length": 8, 610 }]) 611 612 # Can explicitly say you *don't* want the read plan 613 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=no', 614 environ={'swift_owner': True}) 615 status, headers, body = self.call_pfs(req) 616 self.assertEqual(status, '200 OK') 617 self.assertEqual(body, b'burritos') 618 619 # Can handle Range requests, too 620 def mock_RpcGetObject(get_object_req): 621 self.assertEqual(get_object_req['VirtPath'], 622 "/v1/AUTH_test/notes/lunch") 623 self.assertEqual(get_object_req['ReadEntsIn'], [ 624 {"Len": 2, "Offset": 2}]) 625 626 return { 627 "error": None, 628 "result": { 629 "FileSize": 8, 630 "Metadata": "", 631 "InodeNumber": 1245, 632 "NumWrites": 2424, 633 "ModificationTime": 1481152134331862558, 634 "IsDir": False, 635 "LeaseId": "prominority-sarcocyst", 636 "ReadEntsOut": [ 637 { 638 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 639 "Name/0000000000c11fbd"), 640 "Offset": 587, 641 "Length": 1 642 }, 643 { 644 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 645 "Name/0000000000c11798"), 646 "Offset": 25, 647 "Length": 1 648 }, 649 ]}} 650 651 self.fake_rpc.register_handler( 652 "Server.RpcGetObject", mock_RpcGetObject) 653 654 # Present-but-blank query param is truthy 655 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan', 656 headers={'Range': 'bytes=2-3'}, 657 environ={'swift_owner': True}) 658 status, headers, body = self.call_pfs(req) 659 660 self.assertEqual(status, '200 OK') 661 self.assertEqual(headers['Content-Type'], 'application/json') 662 self.assertEqual(headers['X-Object-Content-Type'], 663 'application/octet-stream') 664 self.assertEqual(headers['X-Object-Content-Length'], '8') 665 self.assertIn('ETag', headers) 666 self.assertIn('Last-Modified', headers) 667 self.assertEqual(json.loads(body), [ 668 { 669 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 670 "/0000000000c11fbd"), 671 "Offset": 587, 672 "Length": 1, 673 }, 674 { 675 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 676 "/0000000000c11798"), 677 "Offset": 25, 678 "Length": 1, 679 }, 680 ]) 681 682 def test_GET_slo_manifest(self): 683 self.app.register( 684 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', 685 200, {}, 686 ("blah[]\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 687 688 def mock_RpcGetObject(get_object_req): 689 self.assertEqual(get_object_req['VirtPath'], 690 "/v1/AUTH_test/notes/lunch") 691 self.assertEqual(get_object_req['ReadEntsIn'], []) 692 693 return { 694 "error": None, 695 "result": { 696 "FileSize": 2, 697 "Metadata": base64.b64encode( 698 json.dumps({ 699 'X-Object-Sysmeta-Slo-Etag': 'some etag', 700 'X-Object-Sysmeta-Slo-Size': '0', 701 'Content-Type': 'text/plain;swift_bytes=0', 702 }).encode('ascii')).decode('ascii'), 703 "InodeNumber": 1245, 704 "NumWrites": 2424, 705 "ModificationTime": 1481152134331862558, 706 "IsDir": False, 707 "LeaseId": "prominority-sarcocyst", 708 "ReadEntsOut": [{ 709 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 710 "Name/0000000000c11fbd"), 711 "Offset": 4, 712 "Length": 2}]}} 713 714 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 715 716 self.fake_rpc.register_handler( 717 "Server.RpcGetObject", mock_RpcGetObject) 718 status, headers, body = self.call_pfs(req) 719 720 self.assertEqual(status, '200 OK') 721 self.assertEqual(headers, { 722 'Content-Type': 'text/plain', # no swift_bytes! 723 'Content-Length': '2', 724 # ^^^ needs to be actual size, but slo may/will use vvv for HEADs 725 'X-Object-Sysmeta-Slo-Size': '0', 726 'Etag': '"pfsv2/AUTH_test/000004DD/00000978-32"', 727 # slo may/will use vvv to fix up ^^^ 728 'X-Object-Sysmeta-Slo-Etag': 'some etag', 729 'Accept-Ranges': 'bytes', 730 'X-Timestamp': '1481152134.33186', 731 'Last-Modified': 'Wed, 07 Dec 2016 23:08:55 GMT', 732 }) 733 self.assertEqual(body, b'[]') 734 735 def test_GET_authed(self): 736 self.app.register( 737 'GET', '/v1/AUTH_test/InternalContainerName/0000000001178995', 738 200, {}, 739 ("burr\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 740 self.app.register( 741 'GET', '/v1/AUTH_test/InternalContainerName/0000000004181255', 742 200, {}, 743 ("itos\x62\x6f\x6f\x74{\"more\": \"junk\"}\x00\x00\x00")) 744 745 def mock_RpcHead(head_req): 746 return { 747 "error": None, 748 "result": { 749 "Metadata": None, 750 "ModificationTime": 1488414932080993000, 751 "FileSize": 0, 752 "IsDir": True, 753 "InodeNumber": 6283109, 754 "NumWrites": 0, 755 }} 756 757 def mock_RpcGetObject(get_object_req): 758 self.assertEqual(get_object_req['VirtPath'], 759 "/v1/AUTH_test/notes/lunch") 760 self.assertEqual(get_object_req['ReadEntsIn'], []) 761 762 return { 763 "error": None, 764 "result": { 765 "FileSize": 8, 766 "Metadata": "", 767 "InodeNumber": 1245, 768 "NumWrites": 2424, 769 "ModificationTime": 1481152134331862558, 770 "IsDir": False, 771 "LeaseId": "a65b5591b90fe6035e669f1f216502d2", 772 "ReadEntsOut": [{ 773 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 774 "Name/0000000001178995"), 775 "Offset": 0, 776 "Length": 4}, 777 {"ObjectPath": ("/v1/AUTH_test/InternalContainer" 778 "Name/0000000004181255"), 779 "Offset": 0, 780 "Length": 4}]}} 781 782 def auth_callback(req): 783 if "InternalContainerName" in req.path: 784 return swob.HTTPForbidden(body="lol nope", request=req) 785 786 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 787 req.environ["swift.authorize"] = auth_callback 788 789 self.fake_rpc.register_handler( 790 "Server.RpcGetObject", mock_RpcGetObject) 791 self.fake_rpc.register_handler( 792 "Server.RpcHead", mock_RpcHead) 793 status, headers, body = self.call_pfs(req) 794 795 self.assertEqual(status, '200 OK') 796 self.assertEqual(headers["Last-Modified"], 797 "Wed, 07 Dec 2016 23:08:55 GMT") 798 self.assertEqual(headers["ETag"], 799 mware.construct_etag("AUTH_test", 1245, 2424)) 800 self.assertEqual(body, b'burritos') 801 802 def test_GET_sparse(self): 803 # This log segment, except for the obvious fake metadata at the end, 804 # was obtained by really doing this: 805 # 806 # with open(filename, 'w') as fh: 807 # fh.write('sparse') 808 # fh.seek(10006) 809 # fh.write('file') 810 # 811 # Despite its appearance, this is really what the underlying log 812 # segment for a sparse file looks like, at least sometimes. 813 self.app.register( 814 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000D0', 815 200, {}, 816 ("sparsefile\x57\x18\xa0\xf3-junkety-junk")) 817 818 def mock_RpcGetObject(get_object_req): 819 self.assertEqual(get_object_req['VirtPath'], 820 "/v1/AUTH_test/c/sparse-file") 821 self.assertEqual(get_object_req['ReadEntsIn'], []) 822 823 return { 824 "error": None, 825 "result": { 826 "FileSize": 10010, 827 "Metadata": "", 828 "InodeNumber": 1245, 829 "NumWrites": 2424, 830 "ModificationTime": 1481152134331862558, 831 "IsDir": False, 832 "LeaseId": "6840595b3370f109dc8ed388b41800a4", 833 "ReadEntsOut": [{ 834 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 835 "Name/00000000000000D0"), 836 "Offset": 0, 837 "Length": 6 838 }, { 839 "ObjectPath": "", # empty path means zero-fill 840 "Offset": 0, 841 "Length": 10000, 842 }, { 843 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 844 "Name/00000000000000D0"), 845 "Offset": 6, 846 "Length": 4}]}} 847 848 req = swob.Request.blank('/v1/AUTH_test/c/sparse-file') 849 850 self.fake_rpc.register_handler( 851 "Server.RpcGetObject", mock_RpcGetObject) 852 status, headers, body = self.call_pfs(req) 853 854 self.assertEqual(status, "200 OK") 855 self.assertEqual(body, b"sparse" + (b"\x00" * 10000) + b"file") 856 857 def test_GET_multiple_segments(self): 858 # Typically, a GET request will include data from multiple log 859 # segments. Small files written all at once might fit in a single 860 # log segment, but that's not always true. 861 self.app.register( 862 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 863 200, {}, 864 ("There once was an X from place B,\n\xff\xea\x00junk")) 865 self.app.register( 866 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a3', 867 200, {}, 868 ("That satisfied predicate P.\n\xff\xea\x00junk")) 869 self.app.register( 870 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a4', 871 200, {}, 872 ("He or she did thing A,\n\xff\xea\x00junk")) 873 self.app.register( 874 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000e0', 875 200, {}, 876 ("In an adjective way,\n\xff\xea\x00junk")) 877 self.app.register( 878 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000108', 879 200, {}, 880 ("Resulting in circumstance C.\xff\xea\x00junk")) 881 882 def mock_RpcGetObject(get_object_req): 883 self.assertEqual(get_object_req['VirtPath'], 884 "/v1/AUTH_test/limericks/generic") 885 self.assertEqual(get_object_req['ReadEntsIn'], []) 886 887 return { 888 "error": None, 889 "result": { 890 "FileSize": 134, 891 "Metadata": "", 892 "InodeNumber": 1245, 893 "NumWrites": 2424, 894 "ModificationTime": 1481152134331862558, 895 "IsDir": False, 896 "LeaseId": "who cares", 897 "ReadEntsOut": [{ 898 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 899 "Name/0000000000000001"), 900 "Offset": 0, 901 "Length": 34, 902 }, { 903 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 904 "Name/00000000000000a3"), 905 "Offset": 0, 906 "Length": 28, 907 }, { 908 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 909 "Name/00000000000000a4"), 910 "Offset": 0, 911 "Length": 23, 912 }, { 913 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 914 "Name/00000000000000e0"), 915 "Offset": 0, 916 "Length": 21, 917 }, { 918 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 919 "Name/0000000000000108"), 920 "Offset": 0, 921 "Length": 28, 922 }]}} 923 924 req = swob.Request.blank('/v1/AUTH_test/limericks/generic') 925 926 self.fake_rpc.register_handler( 927 "Server.RpcGetObject", mock_RpcGetObject) 928 status, headers, body = self.call_pfs(req) 929 930 self.assertEqual(status, '200 OK') 931 self.assertEqual(body, ( 932 b"There once was an X from place B,\n" 933 b"That satisfied predicate P.\n" 934 b"He or she did thing A,\n" 935 b"In an adjective way,\n" 936 b"Resulting in circumstance C.")) 937 938 def test_GET_conditional_if_match(self): 939 obj_etag = "42d0f073592cdef8f602cda59fbb270e" 940 obj_body = b"annet-pentahydric-nuculoid-defiber" 941 942 self.app.register( 943 'GET', '/v1/AUTH_test/InternalContainerName/000000000097830c', 944 200, {}, 945 "annet-pentahydric-nuculoid-defiber") 946 947 def mock_RpcGetObject(get_object_req): 948 return { 949 "error": None, 950 "result": { 951 "FileSize": 8, 952 "Metadata": base64.b64encode( 953 json.dumps({ 954 mware.ORIGINAL_MD5_HEADER: "123:%s" % obj_etag, 955 }).encode('ascii')).decode('ascii'), 956 "InodeNumber": 1245, 957 "NumWrites": 123, 958 "ModificationTime": 1511222561631497000, 959 "IsDir": False, 960 "LeaseId": "dontcare", 961 "ReadEntsOut": [{ 962 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 963 "Name/000000000097830c"), 964 "Offset": 0, 965 "Length": len(obj_body)}]}} 966 967 self.fake_rpc.register_handler( 968 "Server.RpcGetObject", mock_RpcGetObject) 969 970 # matches 971 req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', 972 headers={"If-Match": obj_etag}) 973 status, _, body = self.call_pfs(req) 974 self.assertEqual(status, '200 OK') 975 self.assertEqual(body, obj_body) 976 977 # doesn't match 978 req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', 979 headers={"If-Match": obj_etag + "abc"}) 980 status, _, body = self.call_pfs(req) 981 self.assertEqual(status, '412 Precondition Failed') 982 983 def test_GET_range(self): 984 # Typically, a GET response will include data from multiple log 985 # segments. Small files written all at once might fit in a single 986 # log segment, but most files won't. 987 self.app.register( 988 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 989 200, {}, 990 ("Caerphilly, Cheddar, Cheshire, \xff\xea\x00junk")) 991 self.app.register( 992 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 993 200, {}, 994 ("Duddleswell, Dunlop, Coquetdale, \xff\xea\x00junk")) 995 self.app.register( 996 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000003', 997 200, {}, 998 ("Derby, Gloucester, Wensleydale\xff\xea\x00junk")) 999 1000 def mock_RpcGetObject(get_object_req): 1001 self.assertEqual(get_object_req['VirtPath'], 1002 "/v1/AUTH_test/cheeses/UK") 1003 self.assertEqual(get_object_req['ReadEntsIn'], 1004 [{"Offset": 21, "Len": 49}]) 1005 1006 return { 1007 "error": None, 1008 "result": { 1009 "FileSize": 94, 1010 "Metadata": "", 1011 "InodeNumber": 1245, 1012 "NumWrites": 2424, 1013 "ModificationTime": 1481152134331862558, 1014 "IsDir": False, 1015 "LeaseId": "982938", 1016 "ReadEntsOut": [{ 1017 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1018 "Name/0000000000000001"), 1019 "Offset": 21, 1020 "Length": 10, 1021 }, { 1022 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1023 "Name/0000000000000002"), 1024 "Offset": 0, 1025 "Length": 33, 1026 }, { 1027 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1028 "Name/0000000000000003"), 1029 "Offset": 0, 1030 "Length": 5}]}} 1031 1032 req = swob.Request.blank('/v1/AUTH_test/cheeses/UK', 1033 headers={"Range": "bytes=21-69"}) 1034 1035 self.fake_rpc.register_handler( 1036 "Server.RpcGetObject", mock_RpcGetObject) 1037 status, headers, body = self.call_pfs(req) 1038 1039 self.assertEqual(status, '206 Partial Content') 1040 self.assertEqual(headers.get('Content-Range'), "bytes 21-69/94") 1041 self.assertEqual( 1042 body, b'Cheshire, Duddleswell, Dunlop, Coquetdale, Derby') 1043 1044 def test_GET_range_suffix(self): 1045 self.app.register( 1046 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1047 200, {}, 1048 ("hydrogen, helium, \xff\xea\x00junk")) 1049 self.app.register( 1050 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1051 200, {}, 1052 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1053 1054 def mock_RpcGetObject(get_object_req): 1055 self.assertEqual(get_object_req['VirtPath'], 1056 "/v1/AUTH_test/c/elements") 1057 self.assertEqual(get_object_req['ReadEntsIn'], 1058 [{"Offset": None, "Len": 10}]) 1059 1060 return { 1061 "error": None, 1062 "result": { 1063 "FileSize": 94, 1064 "Metadata": "", 1065 "InodeNumber": 1245, 1066 "NumWrites": 2424, 1067 "ModificationTime": 1481152134331862558, 1068 "IsDir": False, 1069 "LeaseId": "fc00:752b:5cca:a544:2d41:3177:2c71:85ae", 1070 "ReadEntsOut": [{ 1071 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1072 "Name/0000000000000002"), 1073 "Offset": 35, 1074 "Length": 10}]}} 1075 1076 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1077 headers={"Range": "bytes=-10"}) 1078 1079 self.fake_rpc.register_handler( 1080 "Server.RpcGetObject", mock_RpcGetObject) 1081 status, headers, body = self.call_pfs(req) 1082 1083 self.assertEqual(status, '206 Partial Content') 1084 self.assertEqual(headers.get('Content-Range'), "bytes 84-93/94") 1085 self.assertEqual(body, b"nitrogen, ") 1086 1087 def test_GET_range_prefix(self): 1088 self.app.register( 1089 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1090 200, {}, 1091 ("hydrogen, helium, \xff\xea\x00junk")) 1092 self.app.register( 1093 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1094 200, {}, 1095 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1096 1097 def mock_RpcGetObject(get_object_req): 1098 self.assertEqual(get_object_req['VirtPath'], 1099 "/v1/AUTH_test/c/elements") 1100 self.assertEqual(get_object_req['ReadEntsIn'], 1101 [{"Offset": 40, "Len": None}]) 1102 1103 return { 1104 "error": None, 1105 "result": { 1106 "FileSize": 62, 1107 "Metadata": "", 1108 "InodeNumber": 1245, 1109 "NumWrites": 2424, 1110 "ModificationTime": 1481152134331862558, 1111 "IsDir": False, 1112 "LeaseId": "", 1113 "ReadEntsOut": [{ 1114 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1115 "Name/0000000000000002"), 1116 "Offset": 22, 1117 "Length": 23}]}} 1118 1119 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1120 headers={"Range": "bytes=40-"}) 1121 1122 self.fake_rpc.register_handler( 1123 "Server.RpcGetObject", mock_RpcGetObject) 1124 status, headers, body = self.call_pfs(req) 1125 1126 self.assertEqual(status, '206 Partial Content') 1127 self.assertEqual(headers.get('Content-Range'), "bytes 40-61/62") 1128 self.assertEqual(body, b"ron, carbon, nitrogen, ") 1129 1130 def test_GET_range_unsatisfiable(self): 1131 self.app.register( 1132 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1133 200, {}, 1134 ("hydrogen, helium, \xff\xea\x00junk")) 1135 self.app.register( 1136 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1137 200, {}, 1138 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1139 1140 def mock_RpcGetObject(get_object_req): 1141 self.assertEqual(get_object_req['VirtPath'], 1142 "/v1/AUTH_test/c/elements") 1143 self.assertEqual(get_object_req['ReadEntsIn'], 1144 [{"Offset": 4000, "Len": None}]) 1145 1146 return { 1147 "error": None, 1148 "result": { 1149 "FileSize": 62, 1150 "Metadata": "", 1151 "InodeNumber": 1245, 1152 "NumWrites": 2426, 1153 "ModificationTime": 1481152134331862558, 1154 "IsDir": False, 1155 "LeaseId": "borkbork", 1156 "ReadEntsOut": None}} 1157 1158 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1159 headers={"Range": "bytes=4000-"}) 1160 1161 self.fake_rpc.register_handler( 1162 "Server.RpcGetObject", mock_RpcGetObject) 1163 status, headers, body = self.call_pfs(req) 1164 1165 self.assertEqual(status, '416 Requested Range Not Satisfiable') 1166 self.assertEqual(headers.get('Content-Range'), 'bytes */62') 1167 self.assertEqual(headers.get('ETag'), 1168 mware.construct_etag("AUTH_test", 1245, 2426)) 1169 self.assertEqual(headers.get('Last-Modified'), 1170 'Wed, 07 Dec 2016 23:08:55 GMT') 1171 self.assertEqual(headers.get('X-Timestamp'), 1172 '1481152134.33186') 1173 1174 def test_GET_multiple_ranges(self): 1175 self.app.register( 1176 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1177 200, {}, "abcd1234efgh5678") 1178 1179 def mock_RpcGetObject(get_object_req): 1180 self.assertEqual(get_object_req['VirtPath'], 1181 "/v1/AUTH_test/c/crud") 1182 self.assertEqual(get_object_req['ReadEntsIn'], [ 1183 {'Len': 3, 'Offset': 2}, 1184 {'Len': 3, 'Offset': 6}, 1185 {'Len': 3, 'Offset': 10}, 1186 ]) 1187 1188 return { 1189 "error": None, 1190 "result": { 1191 "FileSize": 16, 1192 "Metadata": "", 1193 "InodeNumber": 1245, 1194 "NumWrites": 2424, 1195 "ModificationTime": 1481152134331862558, 1196 "IsDir": False, 1197 "LeaseId": "e1885b511fa445d18b1d447a5606a06d", 1198 "ReadEntsOut": [{ 1199 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1200 "Name/0000000000000001"), 1201 "Offset": 2, 1202 "Length": 3, 1203 }, { 1204 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1205 "Name/0000000000000001"), 1206 "Offset": 6, 1207 "Length": 3, 1208 }, { 1209 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1210 "Name/0000000000000001"), 1211 "Offset": 10, 1212 "Length": 3, 1213 }]}} 1214 1215 self.fake_rpc.register_handler( 1216 "Server.RpcGetObject", mock_RpcGetObject) 1217 1218 req = swob.Request.blank('/v1/AUTH_test/c/crud', 1219 headers={"Range": "bytes=2-4,6-8,10-12"}) 1220 1221 # Lock down the MIME boundary so it doesn't change on every test run 1222 with mock.patch('random.randint', 1223 lambda u, l: 0xf0a9157cb1757bfb124aef22fee31051): 1224 status, headers, body = self.call_pfs(req) 1225 1226 self.assertEqual(status, '206 Partial Content') 1227 self.assertEqual( 1228 headers.get('Content-Type'), 1229 'multipart/byteranges;boundary=f0a9157cb1757bfb124aef22fee31051') 1230 self.assertEqual( 1231 body, 1232 (b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1233 b'Content-Type: application/octet-stream\r\n' 1234 b'Content-Range: bytes 2-4/16\r\n' 1235 b'\r\n' 1236 b'cd1\r\n' 1237 b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1238 b'Content-Type: application/octet-stream\r\n' 1239 b'Content-Range: bytes 6-8/16\r\n' 1240 b'\r\n' 1241 b'34e\r\n' 1242 b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1243 b'Content-Type: application/octet-stream\r\n' 1244 b'Content-Range: bytes 10-12/16\r\n' 1245 b'\r\n' 1246 b'gh5\r\n' 1247 b'--f0a9157cb1757bfb124aef22fee31051--')) 1248 1249 def test_GET_metadata(self): 1250 self.app.register( 1251 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1252 200, {}, "abcd1234efgh5678") 1253 1254 def mock_RpcGetObject(get_object_req): 1255 self.assertEqual(get_object_req['VirtPath'], 1256 "/v1/AUTH_test/c/crud") 1257 1258 return { 1259 "error": None, 1260 "result": { 1261 "FileSize": 16, 1262 "Metadata": base64.b64encode(json.dumps({ 1263 "X-Object-Meta-Cow": "moo", 1264 }).encode('ascii')).decode('ascii'), 1265 "InodeNumber": 1245, 1266 "NumWrites": 2424, 1267 "ModificationTime": 1481152134331862558, 1268 "IsDir": False, 1269 "LeaseId": "fe57a6ed-758a-23fb-f7d4-9683aee07c0e", 1270 "ReadEntsOut": [{ 1271 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1272 "Name/0000000000000001"), 1273 "Offset": 0, 1274 "Length": 16}]}} 1275 1276 req = swob.Request.blank('/v1/AUTH_test/c/crud') 1277 1278 self.fake_rpc.register_handler( 1279 "Server.RpcGetObject", mock_RpcGetObject) 1280 status, headers, body = self.call_pfs(req) 1281 1282 self.assertEqual(status, '200 OK') 1283 self.assertEqual(headers.get("X-Object-Meta-Cow"), "moo") 1284 self.assertEqual(body, b'abcd1234efgh5678') 1285 1286 def test_GET_bad_path(self): 1287 bad_paths = [ 1288 '/v1/AUTH_test/c/..', 1289 '/v1/AUTH_test/c/../o', 1290 '/v1/AUTH_test/c/o/..', 1291 '/v1/AUTH_test/c/.', 1292 '/v1/AUTH_test/c/./o', 1293 '/v1/AUTH_test/c/o/.', 1294 '/v1/AUTH_test/c//o', 1295 '/v1/AUTH_test/c/o//', 1296 '/v1/AUTH_test/c/o/', 1297 '/v1/AUTH_test/c/' + ('x' * 256), 1298 ] 1299 for path in bad_paths: 1300 req = swob.Request.blank(path) 1301 status, headers, body = self.call_pfs(req) 1302 self.assertEqual(status, '404 Not Found', 1303 'Got %s for %s' % (status, path)) 1304 1305 req.environ['REQUEST_METHOD'] = 'HEAD' 1306 status, headers, body = self.call_pfs(req) 1307 self.assertEqual(status, '404 Not Found', 1308 'Got %s for %s' % (status, path)) 1309 1310 path = '/v1/AUTH_test//o' 1311 req = swob.Request.blank(path, headers={'Content-Length': '0'}) 1312 for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): 1313 req.environ['REQUEST_METHOD'] = method 1314 status, headers, body = self.call_pfs(req) 1315 self.assertEqual(status, '412 Precondition Failed', 1316 'Got %s for %s %s' % (status, method, path)) 1317 1318 def test_GET_not_found(self): 1319 def mock_RpcGetObject(get_object_req): 1320 self.assertEqual(get_object_req['VirtPath'], 1321 "/v1/AUTH_test/c/missing") 1322 1323 return { 1324 "error": "errno: 2", 1325 "result": None} 1326 1327 req = swob.Request.blank('/v1/AUTH_test/c/missing') 1328 self.fake_rpc.register_handler( 1329 "Server.RpcGetObject", mock_RpcGetObject) 1330 status, headers, body = self.call_pfs(req) 1331 self.assertEqual(status, '404 Not Found') 1332 1333 def test_GET_file_as_dir(self): 1334 # Subdirectories of files don't exist, but asking for one returns a 1335 # different error code than asking for a file that could exist but 1336 # doesn't. 1337 def mock_RpcGetObject(get_object_req): 1338 self.assertEqual(get_object_req['VirtPath'], 1339 "/v1/AUTH_test/c/thing.txt/kitten.png") 1340 1341 return { 1342 "error": "errno: 20", 1343 "result": None} 1344 1345 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png') 1346 self.fake_rpc.register_handler( 1347 "Server.RpcGetObject", mock_RpcGetObject) 1348 status, headers, body = self.call_pfs(req) 1349 self.assertEqual(status, '404 Not Found') 1350 1351 def test_GET_weird_error(self): 1352 def mock_RpcGetObject(get_object_req): 1353 self.assertEqual(get_object_req['VirtPath'], 1354 "/v1/AUTH_test/c/superborked") 1355 1356 return { 1357 "error": "errno: 23", # ENFILE (too many open files in system) 1358 "result": None} 1359 1360 req = swob.Request.blank('/v1/AUTH_test/c/superborked') 1361 self.fake_rpc.register_handler( 1362 "Server.RpcGetObject", mock_RpcGetObject) 1363 status, headers, body = self.call_pfs(req) 1364 self.assertEqual(status, '500 Internal Error') 1365 1366 def test_GET_zero_byte(self): 1367 def mock_RpcGetObject(get_object_req): 1368 self.assertEqual(get_object_req['VirtPath'], 1369 "/v1/AUTH_test/c/empty") 1370 return { 1371 "error": None, 1372 "result": {"FileSize": 0, "ReadEntsOut": None, "Metadata": "", 1373 "IsDir": False, 1374 "LeaseId": "3d73d2bcf39224df00d5ccd912d92c82", 1375 "InodeNumber": 1245, "NumWrites": 2424, 1376 "ModificationTime": 1481152134331862558}} 1377 1378 self.fake_rpc.register_handler( 1379 "Server.RpcGetObject", mock_RpcGetObject) 1380 1381 req = swob.Request.blank('/v1/AUTH_test/c/empty') 1382 status, headers, body = self.call_pfs(req) 1383 self.assertEqual(status, '200 OK') 1384 self.assertEqual(headers["Content-Length"], "0") 1385 1386 def test_GET_dir(self): 1387 def mock_RpcGetObject(get_object_req): 1388 self.assertEqual(get_object_req['VirtPath'], 1389 "/v1/AUTH_test/c/a-dir") 1390 return { 1391 "error": None, 1392 "result": { 1393 "Metadata": "", 1394 "ModificationTime": 1479173168018879490, 1395 "FileSize": 0, 1396 "IsDir": True, 1397 "InodeNumber": 1254, 1398 "NumWrites": 896, 1399 "ReadEntsOut": None, 1400 "LeaseId": "borkbork", 1401 }} 1402 1403 self.fake_rpc.register_handler( 1404 "Server.RpcGetObject", mock_RpcGetObject) 1405 1406 req = swob.Request.blank('/v1/AUTH_test/c/a-dir') 1407 status, headers, body = self.call_pfs(req) 1408 self.assertEqual(status, '200 OK') 1409 self.assertEqual(headers["Content-Length"], "0") 1410 self.assertEqual(headers["Content-Type"], "application/directory") 1411 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 1412 1413 def test_GET_special_chars(self): 1414 self.app.register( 1415 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000123', 1416 200, {}, "abcd1234efgh5678") 1417 1418 def mock_RpcGetObject(get_object_req): 1419 return { 1420 "error": None, 1421 "result": { 1422 "FileSize": 8, 1423 "Metadata": "", 1424 "InodeNumber": 1245, 1425 "NumWrites": 2424, 1426 "ModificationTime": 1481152134331862558, 1427 "IsDir": False, 1428 "LeaseId": "a7ec296d3f3c39ef95407789c436f5f8", 1429 "ReadEntsOut": [{ 1430 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1431 "Name/0000000000000123"), 1432 "Offset": 0, 1433 "Length": 16}]}} 1434 1435 req = swob.Request.blank('/v1/AUTH_test/c o n/o b j') 1436 1437 self.fake_rpc.register_handler( 1438 "Server.RpcGetObject", mock_RpcGetObject) 1439 status, headers, body = self.call_pfs(req) 1440 1441 self.assertEqual(status, '200 OK') 1442 self.assertEqual(body, b'abcd1234efgh5678') 1443 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1444 '/v1/AUTH_test/c o n/o b j') 1445 1446 def test_md5_etag(self): 1447 self.app.register( 1448 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', 1449 200, {}, "stuff stuff stuff") 1450 1451 def mock_RpcGetObject(_): 1452 return { 1453 "error": None, 1454 "result": { 1455 "Metadata": base64.b64encode(json.dumps({ 1456 mware.ORIGINAL_MD5_HEADER: 1457 "3:25152b9f7ca24b61eec895be4e89a950", 1458 }).encode('ascii')).decode('ascii'), 1459 "ModificationTime": 1506039770222591000, 1460 "IsDir": False, 1461 "FileSize": 17, 1462 "IsDir": False, 1463 "InodeNumber": 1433230, 1464 "NumWrites": 3, 1465 "LeaseId": "leaseid", 1466 "ReadEntsOut": [{ 1467 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1468 "Name/0000000000000456"), 1469 "Offset": 0, 1470 "Length": 13}]}} 1471 1472 self.fake_rpc.register_handler( 1473 "Server.RpcGetObject", mock_RpcGetObject) 1474 1475 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png") 1476 status, headers, body = self.call_pfs(req) 1477 self.assertEqual(status, '200 OK') 1478 self.assertEqual(headers["Etag"], "25152b9f7ca24b61eec895be4e89a950") 1479 1480 def test_lease_maintenance(self): 1481 self.app.register( 1482 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', 1483 200, {}, "some contents") 1484 1485 expected_lease_id = "f1abb6c7fc4b4463b04f8313d71986b7" 1486 1487 def mock_RpcGetObject(get_object_req): 1488 return { 1489 "error": None, 1490 "result": { 1491 "FileSize": 13, 1492 "Metadata": "", 1493 "InodeNumber": 7677424, 1494 "NumWrites": 2325461, 1495 "ModificationTime": 1488841810471415000, 1496 "IsDir": False, 1497 "LeaseId": expected_lease_id, 1498 "ReadEntsOut": [{ 1499 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1500 "Name/0000000000000456"), 1501 "Offset": 0, 1502 "Length": 13}]}} 1503 1504 req = swob.Request.blank('/v1/AUTH_test/some/thing') 1505 1506 self.fake_rpc.register_handler( 1507 "Server.RpcGetObject", mock_RpcGetObject) 1508 status, headers, body = self.call_pfs(req) 1509 1510 self.assertEqual(status, '200 OK') 1511 self.assertEqual(body, b'some contents') 1512 1513 called_rpcs = [c[0] for c in self.fake_rpc.calls] 1514 1515 # There's at least one RpcRenewLease, but perhaps there's more: it's 1516 # time-based, and we aren't mocking out time in this test. 1517 self.assertEqual(called_rpcs[0], 'Server.RpcIsAccountBimodal') 1518 self.assertEqual(called_rpcs[1], 'Server.RpcGetObject') 1519 self.assertEqual(called_rpcs[-2], 'Server.RpcRenewLease') 1520 self.assertEqual(called_rpcs[-1], 'Server.RpcReleaseLease') 1521 1522 # make sure we got the right lease ID 1523 self.assertEqual(self.fake_rpc.calls[-2][1][0]['LeaseId'], 1524 expected_lease_id) 1525 self.assertEqual(self.fake_rpc.calls[-1][1][0]['LeaseId'], 1526 expected_lease_id) 1527 1528 1529 class TestContainerHead(BaseMiddlewareTest): 1530 def setUp(self): 1531 super(TestContainerHead, self).setUp() 1532 1533 self.serialized_container_metadata = "" 1534 1535 # All these tests run against the same container. 1536 def mock_RpcHead(head_container_req): 1537 return { 1538 "error": None, 1539 "result": { 1540 "Metadata": base64.b64encode( 1541 self.serialized_container_metadata.encode('ascii')), 1542 "ModificationTime": 1479240397189581131, 1543 "IsDir": False, 1544 "FileSize": 0, 1545 "InodeNumber": 2718, 1546 "NumWrites": 0, 1547 }} 1548 1549 self.fake_rpc.register_handler( 1550 "Server.RpcHead", mock_RpcHead) 1551 1552 def test_special_chars(self): 1553 req = swob.Request.blank("/v1/AUTH_test/a container", 1554 environ={"REQUEST_METHOD": "HEAD"}) 1555 status, headers, body = self.call_pfs(req) 1556 self.assertEqual(status, '204 No Content') 1557 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1558 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1559 '/v1/AUTH_test/a container') 1560 1561 def test_content_type(self): 1562 req = swob.Request.blank("/v1/AUTH_test/a-container?format=xml", 1563 environ={"REQUEST_METHOD": "HEAD"}) 1564 status, headers, _ = self.call_pfs(req) 1565 self.assertEqual(status, '204 No Content') # sanity check 1566 self.assertEqual(headers["Content-Type"], 1567 "application/xml; charset=utf-8") 1568 1569 req = swob.Request.blank("/v1/AUTH_test/a-container?format=json", 1570 environ={"REQUEST_METHOD": "HEAD"}) 1571 status, headers, _ = self.call_pfs(req) 1572 self.assertEqual(status, '204 No Content') # sanity check 1573 self.assertEqual(headers["Content-Type"], 1574 "application/json; charset=utf-8") 1575 1576 req = swob.Request.blank("/v1/AUTH_test/a-container?format=plain", 1577 environ={"REQUEST_METHOD": "HEAD"}) 1578 status, headers, _ = self.call_pfs(req) 1579 self.assertEqual(status, '204 No Content') # sanity check 1580 self.assertEqual(headers["Content-Type"], 1581 "text/plain; charset=utf-8") 1582 1583 with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', 1584 True): 1585 status, headers, _ = self.call_pfs(req) 1586 self.assertEqual(status, '204 No Content') # sanity check 1587 self.assertEqual(headers["Content-Type"], 1588 "application/json; charset=utf-8") 1589 1590 def test_no_meta(self): 1591 self.serialized_container_metadata = "" 1592 1593 req = swob.Request.blank("/v1/AUTH_test/a-container", 1594 environ={"REQUEST_METHOD": "HEAD"}) 1595 status, headers, body = self.call_pfs(req) 1596 self.assertEqual(status, '204 No Content') 1597 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1598 self.assertEqual(headers["X-Container-Object-Count"], "0") 1599 self.assertEqual(headers["X-Container-Bytes-Used"], "0") 1600 self.assertEqual(headers["X-Storage-Policy"], "default") 1601 self.assertEqual(headers["Last-Modified"], 1602 "Tue, 15 Nov 2016 20:06:38 GMT") 1603 self.assertEqual(headers["X-Timestamp"], "1479240397.18958") 1604 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1605 '/v1/AUTH_test/a-container') 1606 1607 def test_bogus_meta(self): 1608 self.serialized_container_metadata = "{[{[{[{[{[[(((!" 1609 1610 req = swob.Request.blank("/v1/AUTH_test/a-container", 1611 environ={"REQUEST_METHOD": "HEAD"}) 1612 status, headers, body = self.call_pfs(req) 1613 self.assertEqual(status, '204 No Content') 1614 1615 def _check_meta(self, expected_headers, swift_owner=False): 1616 container_metadata = { 1617 "X-Container-Read": "xcr", 1618 "X-Container-Write": "xcw", 1619 "X-Container-Sync-Key": "sync-key", 1620 "X-Container-Sync-To": "sync-to", 1621 "X-Container-Meta-Temp-Url-Key": "tuk", 1622 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1623 "X-Container-Sysmeta-Fish": "cod", 1624 "X-Container-Meta-Fish": "trout"} 1625 self.serialized_container_metadata = json.dumps(container_metadata) 1626 1627 req = swob.Request.blank("/v1/AUTH_test/a-container", 1628 environ={"REQUEST_METHOD": "HEAD", 1629 "swift_owner": swift_owner}) 1630 status, headers, body = self.call_pfs(req) 1631 self.assertEqual(status, '204 No Content') 1632 for k, v in expected_headers.items(): 1633 self.assertIn(k, headers) 1634 self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % 1635 (v, headers[k], v)) 1636 for k in container_metadata: 1637 self.assertFalse(k in headers and k not in expected_headers, 1638 "Found unexpected header %r" % k) 1639 1640 def test_meta_swift_owner(self): 1641 self._check_meta({ 1642 "X-Container-Read": "xcr", 1643 "X-Container-Write": "xcw", 1644 "X-Container-Sync-Key": "sync-key", 1645 "X-Container-Sync-To": "sync-to", 1646 "X-Container-Meta-Temp-Url-Key": "tuk", 1647 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1648 "X-Container-Sysmeta-Fish": "cod", 1649 "X-Container-Meta-Fish": "trout"}, 1650 swift_owner=True) 1651 1652 def test_meta_not_swift_owner(self): 1653 self._check_meta({ 1654 "X-Container-Sysmeta-Fish": "cod", 1655 "X-Container-Meta-Fish": "trout"}) 1656 1657 def test_not_found(self): 1658 def mock_RpcHead(head_container_req): 1659 self.assertEqual(head_container_req['VirtPath'], 1660 '/v1/AUTH_test/a-container') 1661 return {"error": "errno: 2", "result": None} 1662 1663 self.fake_rpc.register_handler( 1664 "Server.RpcHead", mock_RpcHead) 1665 1666 req = swob.Request.blank("/v1/AUTH_test/a-container", 1667 environ={"REQUEST_METHOD": "HEAD"}) 1668 status, headers, body = self.call_pfs(req) 1669 self.assertEqual(status, '404 Not Found') 1670 1671 def test_other_error(self): 1672 def mock_RpcHead(head_container_req): 1673 self.assertEqual(head_container_req['VirtPath'], 1674 '/v1/AUTH_test/a-container') 1675 return {"error": "errno: 7581", "result": None} 1676 1677 self.fake_rpc.register_handler( 1678 "Server.RpcHead", mock_RpcHead) 1679 1680 req = swob.Request.blank("/v1/AUTH_test/a-container", 1681 environ={"REQUEST_METHOD": "HEAD"}) 1682 status, headers, body = self.call_pfs(req) 1683 self.assertEqual(status, '500 Internal Error') 1684 1685 1686 class TestContainerGet(BaseMiddlewareTest): 1687 def setUp(self): 1688 super(TestContainerGet, self).setUp() 1689 self.serialized_container_metadata = "" 1690 1691 # All these tests run against the same container. 1692 def mock_RpcGetContainer(get_container_req): 1693 return { 1694 "error": None, 1695 "result": { 1696 "Metadata": base64.b64encode( 1697 self.serialized_container_metadata.encode('ascii')), 1698 "ModificationTime": 1510790796076041000, 1699 "ContainerEntries": [{ 1700 "Basename": "images", 1701 "FileSize": 0, 1702 "ModificationTime": 1471915816359209849, 1703 "IsDir": True, 1704 "InodeNumber": 2489682, 1705 "NumWrites": 0, 1706 "Metadata": "", 1707 }, { 1708 "Basename": u"images/\xE1vocado.png", 1709 "FileSize": 70, 1710 "ModificationTime": 1471915816859209471, 1711 "IsDir": False, 1712 "InodeNumber": 9213768, 1713 "NumWrites": 2, 1714 "Metadata": base64.b64encode(json.dumps({ 1715 "Content-Type": u"snack/m\xEDllenial" + 1716 u";swift_bytes=3503770", 1717 }).encode('ascii')), 1718 }, { 1719 "Basename": "images/banana.png", 1720 "FileSize": 2189865, 1721 # has fractional seconds = 0 to cover edge cases 1722 "ModificationTime": 1471915873000000000, 1723 "IsDir": False, 1724 "InodeNumber": 8878410, 1725 "NumWrites": 2, 1726 "Metadata": "", 1727 }, { 1728 "Basename": "images/cherimoya.png", 1729 "FileSize": 1636662, 1730 "ModificationTime": 1471915917767421311, 1731 "IsDir": False, 1732 "InodeNumber": 8879064, 1733 "NumWrites": 2, 1734 # Note: this has NumWrites=2, but the original MD5 1735 # starts with "1:", so it is stale and must not be 1736 # used. 1737 "Metadata": base64.b64encode(json.dumps({ 1738 mware.ORIGINAL_MD5_HEADER: 1739 "1:552528fbf2366f8a4711ac0a3875188b", 1740 }).encode('ascii')), 1741 }, { 1742 "Basename": "images/durian.png", 1743 "FileSize": 8414281, 1744 "ModificationTime": 1471985233074909930, 1745 "IsDir": False, 1746 "InodeNumber": 5807979, 1747 "NumWrites": 3, 1748 "Metadata": base64.b64encode(json.dumps({ 1749 mware.ORIGINAL_MD5_HEADER: 1750 "3:34f99f7784c573541e11e5ad66f065c8", 1751 }).encode('ascii')), 1752 }, { 1753 "Basename": "images/elderberry.png", 1754 "FileSize": 3178293, 1755 "ModificationTime": 1471985240833932653, 1756 "IsDir": False, 1757 "InodeNumber": 4974021, 1758 "NumWrites": 1, 1759 "Metadata": "", 1760 }]}} 1761 1762 self.fake_rpc.register_handler( 1763 "Server.RpcGetContainer", mock_RpcGetContainer) 1764 1765 def test_text(self): 1766 req = swob.Request.blank('/v1/AUTH_test/a-container') 1767 status, headers, body = self.call_pfs(req) 1768 1769 self.assertEqual(status, '200 OK') 1770 self.assertEqual(headers["Content-Type"], "text/plain; charset=utf-8") 1771 self.assertEqual(body, (b"images\n" 1772 b"images/\xC3\xA1vocado.png\n" 1773 b"images/banana.png\n" 1774 b"images/cherimoya.png\n" 1775 b"images/durian.png\n" 1776 b"images/elderberry.png\n")) 1777 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1778 '/v1/AUTH_test/a-container') 1779 1780 with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', 1781 True): 1782 status, headers, body = self.call_pfs(req) 1783 self.assertEqual(status, '200 OK') # sanity check 1784 self.assertEqual(headers["Content-Type"], 1785 "application/json; charset=utf-8") 1786 json.loads(body) 1787 1788 def test_dlo(self): 1789 req = swob.Request.blank('/v1/AUTH_test/a-container', 1790 environ={'swift.source': 'DLO'}) 1791 status, headers, body = self.call_pfs(req) 1792 1793 self.assertEqual(status, '200 OK') 1794 self.assertEqual(headers["Content-Type"], 1795 "application/json; charset=utf-8") 1796 json.loads(body) # doesn't crash 1797 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1798 '/v1/AUTH_test/a-container') 1799 1800 def _check_metadata(self, expected_headers, swift_owner=False): 1801 container_metadata = { 1802 "X-Container-Read": "xcr", 1803 "X-Container-Write": "xcw", 1804 "X-Container-Sync-Key": "sync-key", 1805 "X-Container-Sync-To": "sync-to", 1806 "X-Container-Meta-Temp-Url-Key": "tuk", 1807 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1808 "X-Container-Sysmeta-Fish": "tilefish", 1809 "X-Container-Meta-Fish": "haddock"} 1810 self.serialized_container_metadata = json.dumps(container_metadata) 1811 req = swob.Request.blank('/v1/AUTH_test/a-container', 1812 environ={"REQUEST_METHOD": "GET", 1813 "swift_owner": swift_owner}) 1814 status, headers, body = self.call_pfs(req) 1815 1816 self.assertEqual(status, '200 OK') 1817 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1818 self.assertEqual(headers["X-Container-Object-Count"], "0") 1819 self.assertEqual(headers["X-Container-Bytes-Used"], "0") 1820 self.assertEqual(headers["X-Storage-Policy"], "default") 1821 self.assertEqual(headers["X-Timestamp"], "1510790796.07604") 1822 self.assertEqual(headers["Last-Modified"], 1823 "Thu, 16 Nov 2017 00:06:37 GMT") 1824 for k, v in expected_headers.items(): 1825 self.assertIn(k, headers) 1826 self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % 1827 (v, headers[k], v)) 1828 for k in container_metadata: 1829 self.assertFalse(k in headers and k not in expected_headers, 1830 "Found unexpected header %r" % k) 1831 1832 def test_metadata_swift_owner(self): 1833 self._check_metadata({ 1834 "X-Container-Read": "xcr", 1835 "X-Container-Write": "xcw", 1836 "X-Container-Sync-Key": "sync-key", 1837 "X-Container-Sync-To": "sync-to", 1838 "X-Container-Meta-Temp-Url-Key": "tuk", 1839 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1840 "X-Container-Sysmeta-Fish": "tilefish", 1841 "X-Container-Meta-Fish": "haddock"}, 1842 swift_owner=True) 1843 1844 def test_metadata_not_swift_owner(self): 1845 self._check_metadata({ 1846 "X-Container-Sysmeta-Fish": "tilefish", 1847 "X-Container-Meta-Fish": "haddock"}) 1848 1849 def test_bogus_metadata(self): 1850 self.serialized_container_metadata = "{<xml?" 1851 req = swob.Request.blank('/v1/AUTH_test/a-container') 1852 status, headers, body = self.call_pfs(req) 1853 1854 self.assertEqual(status, '200 OK') 1855 1856 def test_json(self): 1857 req = swob.Request.blank('/v1/AUTH_test/a-container', 1858 headers={"Accept": "application/json"}) 1859 status, headers, body = self.call_pfs(req) 1860 1861 self.assertEqual(status, '200 OK') 1862 self.assertEqual(headers["Content-Type"], 1863 "application/json; charset=utf-8") 1864 resp_data = json.loads(body) 1865 self.assertIsInstance(resp_data, list) 1866 self.assertEqual(len(resp_data), 6) 1867 self.assertEqual(resp_data[0], { 1868 "name": "images", 1869 "bytes": 0, 1870 "content_type": "application/directory", 1871 "hash": "d41d8cd98f00b204e9800998ecf8427e", 1872 "last_modified": "2016-08-23T01:30:16.359210"}) 1873 self.assertEqual(resp_data[1], { 1874 "name": u"images/\xE1vocado.png", 1875 "bytes": 3503770, 1876 "content_type": u"snack/m\xEDllenial", 1877 "hash": mware.construct_etag( 1878 "AUTH_test", 9213768, 2), 1879 "last_modified": "2016-08-23T01:30:16.859210"}) 1880 self.assertEqual(resp_data[2], { 1881 "name": "images/banana.png", 1882 "bytes": 2189865, 1883 "content_type": "image/png", 1884 "hash": mware.construct_etag( 1885 "AUTH_test", 8878410, 2), 1886 "last_modified": "2016-08-23T01:31:13.000000"}) 1887 self.assertEqual(resp_data[3], { 1888 "name": "images/cherimoya.png", 1889 "bytes": 1636662, 1890 "content_type": "image/png", 1891 "hash": mware.construct_etag( 1892 "AUTH_test", 8879064, 2), 1893 "last_modified": "2016-08-23T01:31:57.767421"}) 1894 self.assertEqual(resp_data[4], { 1895 "name": "images/durian.png", 1896 "bytes": 8414281, 1897 "content_type": "image/png", 1898 "hash": "34f99f7784c573541e11e5ad66f065c8", 1899 "last_modified": "2016-08-23T20:47:13.074910"}) 1900 self.assertEqual(resp_data[5], { 1901 "name": "images/elderberry.png", 1902 "bytes": 3178293, 1903 "content_type": "image/png", 1904 "hash": mware.construct_etag( 1905 "AUTH_test", 4974021, 1), 1906 "last_modified": "2016-08-23T20:47:20.833933"}) 1907 1908 def test_json_with_delimiter(self): 1909 req = swob.Request.blank( 1910 '/v1/AUTH_test/a-container?prefix=&delimiter=/', 1911 headers={"Accept": "application/json"}) 1912 status, headers, body = self.call_pfs(req) 1913 1914 self.assertEqual(status, '200 OK') 1915 self.assertEqual(headers["Content-Type"], 1916 "application/json; charset=utf-8") 1917 resp_data = json.loads(body) 1918 self.assertIsInstance(resp_data, list) 1919 self.assertEqual(len(resp_data), 7) 1920 self.assertEqual(resp_data[0], { 1921 "name": "images", 1922 "bytes": 0, 1923 "content_type": "application/directory", 1924 "hash": "d41d8cd98f00b204e9800998ecf8427e", 1925 "last_modified": "2016-08-23T01:30:16.359210"}) 1926 self.assertEqual(resp_data[1], { 1927 "subdir": "images/"}) 1928 self.assertEqual(resp_data[2], { 1929 "name": u"images/\xE1vocado.png", 1930 "bytes": 3503770, 1931 "content_type": u"snack/m\xEDllenial", 1932 "hash": mware.construct_etag( 1933 "AUTH_test", 9213768, 2), 1934 "last_modified": "2016-08-23T01:30:16.859210"}) 1935 self.assertEqual(resp_data[3], { 1936 "name": "images/banana.png", 1937 "bytes": 2189865, 1938 "content_type": "image/png", 1939 "hash": mware.construct_etag( 1940 "AUTH_test", 8878410, 2), 1941 "last_modified": "2016-08-23T01:31:13.000000"}) 1942 self.assertEqual(resp_data[4], { 1943 "name": "images/cherimoya.png", 1944 "bytes": 1636662, 1945 "content_type": "image/png", 1946 "hash": mware.construct_etag( 1947 "AUTH_test", 8879064, 2), 1948 "last_modified": "2016-08-23T01:31:57.767421"}) 1949 self.assertEqual(resp_data[5], { 1950 "name": "images/durian.png", 1951 "bytes": 8414281, 1952 "content_type": "image/png", 1953 "hash": "34f99f7784c573541e11e5ad66f065c8", 1954 "last_modified": "2016-08-23T20:47:13.074910"}) 1955 self.assertEqual(resp_data[6], { 1956 "name": "images/elderberry.png", 1957 "bytes": 3178293, 1958 "content_type": "image/png", 1959 "hash": mware.construct_etag( 1960 "AUTH_test", 4974021, 1), 1961 "last_modified": "2016-08-23T20:47:20.833933"}) 1962 1963 def test_json_query_param(self): 1964 req = swob.Request.blank('/v1/AUTH_test/a-container?format=json') 1965 status, headers, body = self.call_pfs(req) 1966 1967 self.assertEqual(status, '200 OK') 1968 self.assertEqual(headers["Content-Type"], 1969 "application/json; charset=utf-8") 1970 json.loads(body) # doesn't crash 1971 1972 # TODO: The tests don't seem to access the code from the same path as 1973 # regular requests, so we're testing a code path that is usually not 1974 # reachable. Should we just get rid of that part of the code? 1975 # For now, delimiter support is non-existent from the tests' point of view 1976 # when requesting XML output. 1977 def test_xml(self): 1978 req = swob.Request.blank('/v1/AUTH_test/a-container', 1979 headers={"Accept": "text/xml"}) 1980 status, headers, body = self.call_pfs(req) 1981 1982 self.assertEqual(status, '200 OK') 1983 self.assertEqual(headers["Content-Type"], 1984 "text/xml; charset=utf-8") 1985 self.assertTrue(body.startswith( 1986 b"""<?xml version='1.0' encoding='utf-8'?>""")) 1987 1988 root_node = ElementTree.fromstring(body) 1989 self.assertEqual(root_node.tag, 'container') 1990 self.assertEqual(root_node.attrib["name"], 'a-container') 1991 1992 objects = list(root_node) 1993 self.assertEqual(6, len(objects)) 1994 self.assertEqual(objects[0].tag, 'object') 1995 1996 # The XML container listing doesn't use XML attributes for data, but 1997 # rather a sequence of tags like <name>X</name> <hash>Y</hash> ... 1998 # 1999 # We do an exhaustive check of one object's attributes, then 2000 # spot-check the rest of the listing for brevity's sake. 2001 obj_attr_tags = list(objects[1]) 2002 self.assertEqual(len(obj_attr_tags), 5) 2003 2004 name_node = obj_attr_tags[0] 2005 self.assertEqual(name_node.tag, 'name') 2006 self.assertEqual(name_node.text, u'images/\xE1vocado.png') 2007 self.assertEqual(name_node.attrib, {}) # nothing extra in there 2008 2009 hash_node = obj_attr_tags[1] 2010 self.assertEqual(hash_node.tag, 'hash') 2011 self.assertEqual(hash_node.text, mware.construct_etag( 2012 "AUTH_test", 9213768, 2)) 2013 self.assertEqual(hash_node.attrib, {}) 2014 2015 bytes_node = obj_attr_tags[2] 2016 self.assertEqual(bytes_node.tag, 'bytes') 2017 self.assertEqual(bytes_node.text, '3503770') 2018 self.assertEqual(bytes_node.attrib, {}) 2019 2020 content_type_node = obj_attr_tags[3] 2021 self.assertEqual(content_type_node.tag, 'content_type') 2022 self.assertEqual(content_type_node.text, u'snack/m\xEDllenial') 2023 self.assertEqual(content_type_node.attrib, {}) 2024 2025 last_modified_node = obj_attr_tags[4] 2026 self.assertEqual(last_modified_node.tag, 'last_modified') 2027 self.assertEqual(last_modified_node.text, '2016-08-23T01:30:16.859210') 2028 self.assertEqual(last_modified_node.attrib, {}) 2029 2030 # Make sure the directory has the right type 2031 obj_attr_tags = list(objects[0]) 2032 self.assertEqual(len(obj_attr_tags), 5) 2033 2034 name_node = obj_attr_tags[0] 2035 self.assertEqual(name_node.tag, 'name') 2036 self.assertEqual(name_node.text, 'images') 2037 2038 hash_node = obj_attr_tags[1] 2039 self.assertEqual(hash_node.tag, 'hash') 2040 self.assertEqual(hash_node.text, "d41d8cd98f00b204e9800998ecf8427e") 2041 self.assertEqual(hash_node.attrib, {}) 2042 2043 content_type_node = obj_attr_tags[3] 2044 self.assertEqual(content_type_node.tag, 'content_type') 2045 self.assertEqual(content_type_node.text, 'application/directory') 2046 2047 # Check the names are correct 2048 all_names = [list(tag)[0].text for tag in objects] 2049 self.assertEqual( 2050 ["images", u"images/\xE1vocado.png", "images/banana.png", 2051 "images/cherimoya.png", "images/durian.png", 2052 "images/elderberry.png"], 2053 all_names) 2054 2055 def test_xml_alternate_mime_type(self): 2056 req = swob.Request.blank('/v1/AUTH_test/a-container', 2057 headers={"Accept": "application/xml"}) 2058 status, headers, body = self.call_pfs(req) 2059 2060 self.assertEqual(status, '200 OK') 2061 self.assertEqual(headers["Content-Type"], 2062 "application/xml; charset=utf-8") 2063 self.assertTrue(body.startswith( 2064 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2065 2066 def test_xml_query_param(self): 2067 req = swob.Request.blank('/v1/AUTH_test/a-container?format=xml') 2068 status, headers, body = self.call_pfs(req) 2069 2070 self.assertEqual(status, '200 OK') 2071 self.assertEqual(headers["Content-Type"], 2072 "application/xml; charset=utf-8") 2073 self.assertTrue(body.startswith( 2074 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2075 2076 def test_xml_special_chars(self): 2077 req = swob.Request.blank('/v1/AUTH_test/c o n', 2078 headers={"Accept": "text/xml"}) 2079 status, headers, body = self.call_pfs(req) 2080 2081 self.assertEqual(status, '200 OK') 2082 self.assertEqual(headers["Content-Type"], 2083 "text/xml; charset=utf-8") 2084 self.assertTrue(body.startswith( 2085 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2086 2087 root_node = ElementTree.fromstring(body) 2088 self.assertEqual(root_node.tag, 'container') 2089 self.assertEqual(root_node.attrib["name"], 'c o n') 2090 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 2091 '/v1/AUTH_test/c o n') 2092 2093 def test_marker(self): 2094 req = swob.Request.blank('/v1/AUTH_test/a-container?marker=sharpie') 2095 status, _, _ = self.call_pfs(req) 2096 self.assertEqual(status, '200 OK') 2097 2098 rpc_calls = self.fake_rpc.calls 2099 self.assertEqual(len(rpc_calls), 2) 2100 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2101 # relevant to what we're testing here 2102 rpc_method, rpc_args = rpc_calls[1] 2103 # sanity check 2104 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2105 self.assertEqual(rpc_args[0]["Marker"], "sharpie") 2106 2107 def test_end_marker(self): 2108 req = swob.Request.blank( 2109 '/v1/AUTH_test/a-container?end_marker=whiteboard') 2110 status, _, _ = self.call_pfs(req) 2111 self.assertEqual(status, '200 OK') 2112 2113 rpc_calls = self.fake_rpc.calls 2114 self.assertEqual(len(rpc_calls), 2) 2115 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2116 # relevant to what we're testing here 2117 rpc_method, rpc_args = rpc_calls[1] 2118 # sanity check 2119 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2120 self.assertEqual(rpc_args[0]["EndMarker"], "whiteboard") 2121 2122 def test_prefix(self): 2123 req = swob.Request.blank('/v1/AUTH_test/a-container?prefix=cow') 2124 status, _, _ = self.call_pfs(req) 2125 self.assertEqual(status, '200 OK') 2126 2127 rpc_calls = self.fake_rpc.calls 2128 self.assertEqual(len(rpc_calls), 2) 2129 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2130 # relevant to what we're testing here 2131 rpc_method, rpc_args = rpc_calls[1] 2132 # sanity check 2133 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2134 self.assertEqual(rpc_args[0]["Prefix"], "cow") 2135 2136 def test_delimiter(self): 2137 req = swob.Request.blank('/v1/AUTH_test/a-container?delimiter=/') 2138 status, _, _ = self.call_pfs(req) 2139 self.assertEqual(status, '200 OK') 2140 2141 rpc_calls = self.fake_rpc.calls 2142 self.assertEqual(len(rpc_calls), 2) 2143 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2144 # relevant to what we're testing here 2145 rpc_method, rpc_args = rpc_calls[1] 2146 # sanity check 2147 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2148 self.assertEqual(rpc_args[0]["Delimiter"], "/") 2149 2150 def test_default_limit(self): 2151 req = swob.Request.blank('/v1/AUTH_test/a-container') 2152 status, _, _ = self.call_pfs(req) 2153 self.assertEqual(status, '200 OK') 2154 2155 rpc_calls = self.fake_rpc.calls 2156 self.assertEqual(len(rpc_calls), 2) 2157 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2158 # relevant to what we're testing here 2159 rpc_method, rpc_args = rpc_calls[1] 2160 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) 2161 2162 def test_valid_user_supplied_limit(self): 2163 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=150') 2164 status, _, _ = self.call_pfs(req) 2165 self.assertEqual(status, '200 OK') 2166 2167 rpc_calls = self.fake_rpc.calls 2168 self.assertEqual(len(rpc_calls), 2) 2169 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2170 # relevant to what we're testing here 2171 rpc_method, rpc_args = rpc_calls[1] 2172 self.assertEqual(rpc_args[0]["MaxEntries"], 150) 2173 2174 def test_zero_supplied_limit(self): 2175 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=0') 2176 status, _, _ = self.call_pfs(req) 2177 self.assertEqual(status, '200 OK') 2178 2179 rpc_calls = self.fake_rpc.calls 2180 self.assertEqual(len(rpc_calls), 2) 2181 2182 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2183 # relevant to what we're testing here 2184 rpc_method, rpc_args = rpc_calls[1] 2185 self.assertEqual(rpc_args[0]["MaxEntries"], 0) 2186 2187 def test_negative_supplied_limit(self): 2188 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=-1') 2189 status, _, _ = self.call_pfs(req) 2190 self.assertEqual(status, '200 OK') 2191 2192 rpc_calls = self.fake_rpc.calls 2193 self.assertEqual(len(rpc_calls), 2) 2194 2195 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2196 # relevant to what we're testing here 2197 rpc_method, rpc_args = rpc_calls[1] 2198 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2199 2200 def test_overlarge_supplied_limits(self): 2201 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=6544') 2202 status, _, _ = self.call_pfs(req) 2203 self.assertEqual(status, '412 Precondition Failed') 2204 2205 rpc_calls = self.fake_rpc.calls 2206 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2207 # relevant to what we're testing here 2208 self.assertEqual(len(rpc_calls), 1) 2209 2210 def test_bogus_user_supplied_limit(self): 2211 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=chihuahua') 2212 status, _, _ = self.call_pfs(req) 2213 self.assertEqual(status, '200 OK') 2214 2215 rpc_calls = self.fake_rpc.calls 2216 self.assertEqual(len(rpc_calls), 2) 2217 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2218 # relevant to what we're testing here 2219 rpc_method, rpc_args = rpc_calls[1] 2220 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2221 2222 def test_default_limit_matches_proxy_server(self): 2223 req = swob.Request.blank('/v1/AUTH_test/a-container') 2224 status, _, _ = self.call_pfs(req) 2225 self.assertEqual(status, '200 OK') 2226 2227 rpc_calls = self.fake_rpc.calls 2228 self.assertEqual(len(rpc_calls), 2) 2229 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2230 # relevant to what we're testing here 2231 rpc_method, rpc_args = rpc_calls[1] 2232 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2233 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2234 2235 def test_GET_bad_path(self): 2236 bad_paths = [ 2237 '/v1/AUTH_test/..', 2238 '/v1/AUTH_test/.', 2239 '/v1/AUTH_test/' + ('x' * 256), 2240 ] 2241 for path in bad_paths: 2242 req = swob.Request.blank(path) 2243 status, headers, body = self.call_pfs(req) 2244 self.assertEqual(status, '404 Not Found', 2245 'Got %s for %s' % (status, path)) 2246 2247 req.environ['REQUEST_METHOD'] = 'HEAD' 2248 status, headers, body = self.call_pfs(req) 2249 self.assertEqual(status, '404 Not Found', 2250 'Got %s for %s' % (status, path)) 2251 2252 def test_not_found(self): 2253 def mock_RpcGetContainer_error(get_container_req): 2254 self.assertEqual(get_container_req['VirtPath'], 2255 '/v1/AUTH_test/a-container') 2256 2257 return {"error": "errno: 2", "result": None} 2258 2259 self.fake_rpc.register_handler( 2260 "Server.RpcGetContainer", mock_RpcGetContainer_error) 2261 2262 req = swob.Request.blank('/v1/AUTH_test/a-container') 2263 status, _, _ = self.call_pfs(req) 2264 self.assertEqual(status, '404 Not Found') 2265 2266 def test_other_error(self): 2267 def mock_RpcGetContainer_error(get_container_req): 2268 self.assertEqual(get_container_req['VirtPath'], 2269 '/v1/AUTH_test/a-container') 2270 2271 return {"error": "errno: 1661", "result": None} 2272 2273 self.fake_rpc.register_handler( 2274 "Server.RpcGetContainer", mock_RpcGetContainer_error) 2275 2276 req = swob.Request.blank('/v1/AUTH_test/a-container') 2277 status, _, _ = self.call_pfs(req) 2278 self.assertEqual(status, '500 Internal Error') 2279 2280 2281 class TestContainerGetDelimiter(BaseMiddlewareTest): 2282 def setUp(self): 2283 super(TestContainerGetDelimiter, self).setUp() 2284 self.serialized_container_metadata = "" 2285 2286 def _mock_RpcGetContainerDelimiterNoTrailingSlash(self, get_container_req): 2287 return { 2288 "error": None, 2289 "result": { 2290 "Metadata": base64.b64encode( 2291 self.serialized_container_metadata.encode('ascii')), 2292 "ModificationTime": 1510790796076041000, 2293 "ContainerEntries": [{ 2294 "Basename": "images", 2295 "FileSize": 0, 2296 "ModificationTime": 1471915816359209849, 2297 "IsDir": True, 2298 "InodeNumber": 2489682, 2299 "NumWrites": 0, 2300 "Metadata": "", 2301 }]}} 2302 2303 def _mock_RpcGetContainerDelimiterWithTrailingSlash(self, 2304 get_container_req): 2305 return { 2306 "error": None, 2307 "result": { 2308 "Metadata": base64.b64encode( 2309 self.serialized_container_metadata.encode('ascii')), 2310 "ModificationTime": 1510790796076041000, 2311 "ContainerEntries": [{ 2312 "Basename": "images/avocado.png", 2313 "FileSize": 70, 2314 "ModificationTime": 1471915816859209471, 2315 "IsDir": False, 2316 "InodeNumber": 9213768, 2317 "NumWrites": 2, 2318 "Metadata": base64.b64encode(json.dumps({ 2319 "Content-Type": "snack/millenial" + 2320 ";swift_bytes=3503770", 2321 }).encode('ascii')), 2322 }, { 2323 "Basename": "images/banana.png", 2324 "FileSize": 2189865, 2325 # has fractional seconds = 0 to cover edge cases 2326 "ModificationTime": 1471915873000000000, 2327 "IsDir": False, 2328 "InodeNumber": 8878410, 2329 "NumWrites": 2, 2330 "Metadata": "", 2331 }, { 2332 "Basename": "images/cherimoya.png", 2333 "FileSize": 1636662, 2334 "ModificationTime": 1471915917767421311, 2335 "IsDir": False, 2336 "InodeNumber": 8879064, 2337 "NumWrites": 2, 2338 # Note: this has NumWrites=2, but the original MD5 2339 # starts with "1:", so it is stale and must not be 2340 # used. 2341 "Metadata": base64.b64encode(json.dumps({ 2342 mware.ORIGINAL_MD5_HEADER: 2343 "1:552528fbf2366f8a4711ac0a3875188b", 2344 }).encode('ascii')), 2345 }, { 2346 "Basename": "images/durian.png", 2347 "FileSize": 8414281, 2348 "ModificationTime": 1471985233074909930, 2349 "IsDir": False, 2350 "InodeNumber": 5807979, 2351 "NumWrites": 3, 2352 "Metadata": base64.b64encode(json.dumps({ 2353 mware.ORIGINAL_MD5_HEADER: 2354 "3:34f99f7784c573541e11e5ad66f065c8", 2355 }).encode('ascii')), 2356 }, { 2357 "Basename": "images/elderberry.png", 2358 "FileSize": 3178293, 2359 "ModificationTime": 1471985240833932653, 2360 "IsDir": False, 2361 "InodeNumber": 4974021, 2362 "NumWrites": 1, 2363 "Metadata": "", 2364 }]}} 2365 2366 def test_json_no_trailing_slash(self): 2367 self.fake_rpc.register_handler( 2368 "Server.RpcGetContainer", 2369 self._mock_RpcGetContainerDelimiterNoTrailingSlash) 2370 2371 req = swob.Request.blank( 2372 '/v1/AUTH_test/a-container?prefix=images&delimiter=/', 2373 headers={"Accept": "application/json"}) 2374 status, headers, body = self.call_pfs(req) 2375 2376 self.assertEqual(status, '200 OK') 2377 self.assertEqual(headers["Content-Type"], 2378 "application/json; charset=utf-8") 2379 resp_data = json.loads(body) 2380 self.assertIsInstance(resp_data, list) 2381 self.assertEqual(len(resp_data), 2) 2382 self.assertEqual(resp_data[0], { 2383 "name": "images", 2384 "bytes": 0, 2385 "content_type": "application/directory", 2386 "hash": "d41d8cd98f00b204e9800998ecf8427e", 2387 "last_modified": "2016-08-23T01:30:16.359210"}) 2388 self.assertEqual(resp_data[1], { 2389 "subdir": "images/"}) 2390 2391 def test_json_with_trailing_slash(self): 2392 self.fake_rpc.register_handler( 2393 "Server.RpcGetContainer", 2394 self._mock_RpcGetContainerDelimiterWithTrailingSlash) 2395 2396 req = swob.Request.blank( 2397 '/v1/AUTH_test/a-container?prefix=images/&delimiter=/', 2398 headers={"Accept": "application/json"}) 2399 status, headers, body = self.call_pfs(req) 2400 2401 self.assertEqual(status, '200 OK') 2402 self.assertEqual(headers["Content-Type"], 2403 "application/json; charset=utf-8") 2404 resp_data = json.loads(body) 2405 self.assertIsInstance(resp_data, list) 2406 self.assertEqual(len(resp_data), 5) 2407 self.assertEqual(resp_data[0], { 2408 "name": "images/avocado.png", 2409 "bytes": 3503770, 2410 "content_type": "snack/millenial", 2411 "hash": mware.construct_etag( 2412 "AUTH_test", 9213768, 2), 2413 "last_modified": "2016-08-23T01:30:16.859210"}) 2414 self.assertEqual(resp_data[1], { 2415 "name": "images/banana.png", 2416 "bytes": 2189865, 2417 "content_type": "image/png", 2418 "hash": mware.construct_etag( 2419 "AUTH_test", 8878410, 2), 2420 "last_modified": "2016-08-23T01:31:13.000000"}) 2421 self.assertEqual(resp_data[2], { 2422 "name": "images/cherimoya.png", 2423 "bytes": 1636662, 2424 "content_type": "image/png", 2425 "hash": mware.construct_etag( 2426 "AUTH_test", 8879064, 2), 2427 "last_modified": "2016-08-23T01:31:57.767421"}) 2428 self.assertEqual(resp_data[3], { 2429 "name": "images/durian.png", 2430 "bytes": 8414281, 2431 "content_type": "image/png", 2432 "hash": "34f99f7784c573541e11e5ad66f065c8", 2433 "last_modified": "2016-08-23T20:47:13.074910"}) 2434 self.assertEqual(resp_data[4], { 2435 "name": "images/elderberry.png", 2436 "bytes": 3178293, 2437 "content_type": "image/png", 2438 "hash": mware.construct_etag( 2439 "AUTH_test", 4974021, 1), 2440 "last_modified": "2016-08-23T20:47:20.833933"}) 2441 2442 2443 class TestContainerPost(BaseMiddlewareTest): 2444 def test_missing_container(self): 2445 def mock_RpcHead(_): 2446 return {"error": "errno: 2", "result": None} 2447 2448 self.fake_rpc.register_handler( 2449 "Server.RpcHead", mock_RpcHead) 2450 2451 req = swob.Request.blank( 2452 "/v1/AUTH_test/new-con", 2453 environ={"REQUEST_METHOD": "POST"}, 2454 headers={"X-Container-Meta-One-Fish": "two fish"}) 2455 status, _, _ = self.call_pfs(req) 2456 self.assertEqual("404 Not Found", status) 2457 2458 rpc_calls = self.fake_rpc.calls 2459 self.assertEqual(2, len(rpc_calls)) 2460 2461 method, args = rpc_calls[0] 2462 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2463 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2464 2465 method, args = rpc_calls[1] 2466 self.assertEqual(method, "Server.RpcHead") 2467 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2468 2469 def test_existing_container(self): 2470 old_meta = json.dumps({ 2471 "X-Container-Read": "xcr", 2472 "X-Container-Meta-One-Fish": "no fish"}) 2473 2474 def mock_RpcHead(_): 2475 return { 2476 "error": None, 2477 "result": { 2478 "Metadata": base64.b64encode( 2479 old_meta.encode('ascii')).decode('ascii'), 2480 "ModificationTime": 1482280565956671142, 2481 "FileSize": 0, 2482 "IsDir": True, 2483 "InodeNumber": 6515443, 2484 "NumWrites": 0}} 2485 2486 self.fake_rpc.register_handler( 2487 "Server.RpcHead", mock_RpcHead) 2488 2489 self.fake_rpc.register_handler( 2490 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 2491 2492 req = swob.Request.blank( 2493 "/v1/AUTH_test/new-con", 2494 environ={"REQUEST_METHOD": "POST"}, 2495 headers={"X-Container-Meta-One-Fish": "two fish"}) 2496 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2497 status, _, _ = self.call_pfs(req) 2498 self.assertEqual("204 No Content", status) 2499 cic.assert_called() 2500 2501 rpc_calls = self.fake_rpc.calls 2502 self.assertEqual(3, len(rpc_calls)) 2503 2504 method, args = rpc_calls[0] 2505 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2506 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2507 2508 method, args = rpc_calls[1] 2509 self.assertEqual(method, "Server.RpcHead") 2510 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2511 2512 method, args = rpc_calls[2] 2513 self.assertEqual(method, "Server.RpcPost") 2514 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2515 self.assertEqual( 2516 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 2517 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 2518 self.assertEqual(new_meta["X-Container-Meta-One-Fish"], "two fish") 2519 self.assertEqual(new_meta["X-Container-Read"], "xcr") 2520 2521 2522 class TestContainerPut(BaseMiddlewareTest): 2523 def setUp(self): 2524 super(TestContainerPut, self).setUp() 2525 2526 # This only returns success/failure, not any interesting data 2527 def mock_RpcPutContainer(_): 2528 return {"error": None, "result": {}} 2529 2530 self.fake_rpc.register_handler( 2531 "Server.RpcPutContainer", mock_RpcPutContainer) 2532 2533 def test_new_container(self): 2534 def mock_RpcHead(_): 2535 return {"error": "errno: 2", "result": None} 2536 2537 self.fake_rpc.register_handler( 2538 "Server.RpcHead", mock_RpcHead) 2539 2540 req = swob.Request.blank( 2541 "/v1/AUTH_test/new-con", 2542 environ={"REQUEST_METHOD": "PUT"}, 2543 headers={"X-Container-Meta-Red-Fish": "blue fish"}) 2544 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2545 status, _, _ = self.call_pfs(req) 2546 self.assertEqual("201 Created", status) 2547 cic.assert_called() 2548 2549 rpc_calls = self.fake_rpc.calls 2550 self.assertEqual(3, len(rpc_calls)) 2551 2552 method, args = rpc_calls[0] 2553 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2554 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2555 2556 method, args = rpc_calls[1] 2557 self.assertEqual(method, "Server.RpcHead") 2558 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2559 2560 method, args = rpc_calls[2] 2561 self.assertEqual(method, "Server.RpcPutContainer") 2562 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2563 self.assertEqual(args[0]["OldMetadata"], "") 2564 self.assertEqual( 2565 base64.b64decode(args[0]["NewMetadata"]).decode('ascii'), 2566 json.dumps({"X-Container-Meta-Red-Fish": "blue fish"})) 2567 2568 def test_PUT_bad_path(self): 2569 bad_container_paths = [ 2570 '/v1/AUTH_test/..', 2571 '/v1/AUTH_test/.', 2572 '/v1/AUTH_test/' + ('x' * 256), 2573 ] 2574 for path in bad_container_paths: 2575 req = swob.Request.blank(path, 2576 environ={"REQUEST_METHOD": "PUT", 2577 "wsgi.input": BytesIO(b""), 2578 "CONTENT_LENGTH": "0"}) 2579 status, headers, body = self.call_pfs(req) 2580 self.assertEqual(status, '400 Bad Request', 2581 'Got %s for %s' % (status, path)) 2582 2583 req.environ['REQUEST_METHOD'] = 'POST' 2584 status, headers, body = self.call_pfs(req) 2585 self.assertEqual(status, '404 Not Found', 2586 'Got %s for %s' % (status, path)) 2587 2588 req.environ['REQUEST_METHOD'] = 'DELETE' 2589 status, headers, body = self.call_pfs(req) 2590 self.assertEqual(status, '404 Not Found', 2591 'Got %s for %s' % (status, path)) 2592 2593 def test_name_too_long(self): 2594 def mock_RpcHead(_): 2595 return {"error": "errno: 2", "result": None} 2596 2597 self.fake_rpc.register_handler( 2598 "Server.RpcHead", mock_RpcHead) 2599 2600 self.swift_info['swift']['max_container_name_length'] = 50 2601 acceptable_name = 'A' * ( 2602 self.swift_info['swift']['max_container_name_length']) 2603 too_long_name = 'A' * ( 2604 self.swift_info['swift']['max_container_name_length'] + 1) 2605 self.app.register( 2606 'GET', '/info', 2607 200, {'Content-Type': 'application/json'}, 2608 json.dumps(self.swift_info)) 2609 2610 req = swob.Request.blank( 2611 "/v1/AUTH_test/%s" % acceptable_name, 2612 environ={"REQUEST_METHOD": "PUT"}) 2613 status, _, _ = self.call_pfs(req) 2614 self.assertEqual("201 Created", status) 2615 2616 req = swob.Request.blank( 2617 "/v1/AUTH_test/%s" % too_long_name, 2618 environ={"REQUEST_METHOD": "PUT"}) 2619 status, _, _ = self.call_pfs(req) 2620 self.assertEqual("400 Bad Request", status) 2621 2622 def test_name_too_long_posix(self): 2623 def mock_RpcHead(_): 2624 return {"error": "errno: 2", "result": None} 2625 2626 self.fake_rpc.register_handler( 2627 "Server.RpcHead", mock_RpcHead) 2628 2629 posix_limit = mware.NAME_MAX 2630 self.swift_info['swift']['max_container_name_length'] = posix_limit * 2 2631 self.app.register( 2632 'GET', '/info', 2633 200, {'Content-Type': 'application/json'}, 2634 json.dumps(self.swift_info)) 2635 2636 too_long_name = 'A' * (posix_limit + 1) 2637 req = swob.Request.blank( 2638 "/v1/AUTH_test/%s" % too_long_name, 2639 environ={"REQUEST_METHOD": "PUT"}) 2640 status, _, _ = self.call_pfs(req) 2641 self.assertEqual("400 Bad Request", status) 2642 2643 def _check_existing_container(self, req_headers, expected_meta, 2644 swift_owner=False): 2645 old_meta = json.dumps({ 2646 "X-Container-Read": "xcr", 2647 "X-Container-Write": "xcw", 2648 "X-Container-Sync-Key": "sync-key", 2649 "X-Container-Sync-To": "sync-to", 2650 "X-Container-Meta-Temp-Url-Key": "tuk", 2651 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 2652 "X-Container-Meta-Red-Fish": "dead fish"}) 2653 2654 def mock_RpcHead(_): 2655 return { 2656 "error": None, 2657 "result": { 2658 "Metadata": base64.b64encode( 2659 old_meta.encode('ascii')).decode('ascii'), 2660 "ModificationTime": 1482270529646747881, 2661 "FileSize": 0, 2662 "IsDir": True, 2663 "InodeNumber": 8054914, 2664 "NumWrites": 0}} 2665 2666 self.fake_rpc.register_handler( 2667 "Server.RpcHead", mock_RpcHead) 2668 2669 req = swob.Request.blank( 2670 "/v1/AUTH_test/new-con", 2671 environ={"REQUEST_METHOD": "PUT", 'swift_owner': swift_owner}, 2672 headers=req_headers) 2673 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2674 status, _, _ = self.call_pfs(req) 2675 self.assertEqual("202 Accepted", status) 2676 cic.assert_called() 2677 2678 rpc_calls = self.fake_rpc.calls 2679 self.assertEqual(3, len(rpc_calls)) 2680 2681 method, args = rpc_calls[0] 2682 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2683 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2684 2685 method, args = rpc_calls[1] 2686 self.assertEqual(method, "Server.RpcHead") 2687 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2688 2689 method, args = rpc_calls[2] 2690 self.assertEqual(method, "Server.RpcPutContainer") 2691 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2692 self.assertEqual( 2693 base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) 2694 new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) 2695 self.assertEqual(expected_meta, new_meta) 2696 2697 def test_existing_container_swift_owner(self): 2698 self._check_existing_container( 2699 {"X-Container-Read": "xcr-new", 2700 "X-Container-Write": "xcw-new", 2701 "X-Container-Sync-Key": "sync-key-new", 2702 "X-Container-Sync-To": "sync-to-new", 2703 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2704 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2705 "X-Container-Meta-Red-Fish": "blue fish"}, 2706 {"X-Container-Read": "xcr-new", 2707 "X-Container-Write": "xcw-new", 2708 "X-Container-Sync-Key": "sync-key-new", 2709 "X-Container-Sync-To": "sync-to-new", 2710 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2711 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2712 "X-Container-Meta-Red-Fish": "blue fish"}, 2713 swift_owner=True 2714 ) 2715 2716 def test_existing_container_not_swift_owner(self): 2717 self._check_existing_container( 2718 {"X-Container-Read": "xcr-new", 2719 "X-Container-Write": "xcw-new", 2720 "X-Container-Sync-Key": "sync-key-new", 2721 "X-Container-Sync-To": "sync-to-new", 2722 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2723 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2724 "X-Container-Meta-Red-Fish": "blue fish"}, 2725 {"X-Container-Read": "xcr", 2726 "X-Container-Write": "xcw", 2727 "X-Container-Sync-Key": "sync-key", 2728 "X-Container-Sync-To": "sync-to", 2729 "X-Container-Meta-Temp-Url-Key": "tuk", 2730 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 2731 "X-Container-Meta-Red-Fish": "blue fish"} 2732 ) 2733 2734 def _check_metadata_removal(self, headers, expected_meta, 2735 swift_owner=False): 2736 old_meta = json.dumps({ 2737 "X-Container-Read": "xcr", 2738 "X-Container-Write": "xcw", 2739 "X-Container-Sync-Key": "sync-key", 2740 "X-Container-Sync-To": "sync-to", 2741 "X-Versions-Location": "loc", 2742 "X-Container-Meta-Box": "cardboard" 2743 }) 2744 2745 def mock_RpcHead(_): 2746 return { 2747 "error": None, 2748 "result": { 2749 "Metadata": base64.b64encode( 2750 old_meta.encode('ascii')).decode('ascii'), 2751 "ModificationTime": 1511224700123739000, 2752 "FileSize": 0, 2753 "IsDir": True, 2754 "InodeNumber": 8017342, 2755 "NumWrites": 0}} 2756 2757 self.fake_rpc.register_handler( 2758 "Server.RpcHead", mock_RpcHead) 2759 2760 req = swob.Request.blank( 2761 "/v1/AUTH_test/new-con", 2762 environ={"REQUEST_METHOD": "PUT", "swift_owner": swift_owner}, 2763 headers=headers) 2764 status, _, _ = self.call_pfs(req) 2765 self.assertEqual("202 Accepted", status) 2766 2767 rpc_calls = self.fake_rpc.calls 2768 method, args = rpc_calls[-1] 2769 self.assertEqual(method, "Server.RpcPutContainer") 2770 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2771 self.assertEqual( 2772 base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) 2773 new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) 2774 self.assertEqual(expected_meta, new_meta) 2775 2776 def test_metadata_removal_swift_owner(self): 2777 self._check_metadata_removal( 2778 {"X-Container-Read": "", 2779 "X-Container-Write": "", 2780 "X-Container-Sync-Key": "", 2781 "X-Container-Sync-To": "", 2782 "X-Versions-Location": "", 2783 "X-Container-Meta-Box": ""}, 2784 {}, swift_owner=True) 2785 2786 def test_metadata_removal_with_remove_swift_owner(self): 2787 self._check_metadata_removal( 2788 {"X-Remove-Container-Read": "", 2789 "X-Remove-Container-Write": "", 2790 "X-Remove-Container-Sync-Key": "", 2791 "X-Remove-Container-Sync-To": "", 2792 "X-Remove-Versions-Location": "", 2793 "X-Remove-Container-Meta-Box": ""}, 2794 {}, swift_owner=True) 2795 2796 def test_metadata_removal_not_swift_owner(self): 2797 self._check_metadata_removal( 2798 {"X-Container-Read": "", 2799 "X-Container-Write": "", 2800 "X-Container-Sync-Key": "", 2801 "X-Container-Sync-To": "", 2802 "X-Versions-Location": "", 2803 "X-Container-Meta-Box": ""}, 2804 {"X-Container-Read": "xcr", 2805 "X-Container-Write": "xcw", 2806 "X-Container-Sync-Key": "sync-key", 2807 "X-Container-Sync-To": "sync-to"}) 2808 2809 def test_metadata_removal_with_remove_not_swift_owner(self): 2810 self._check_metadata_removal( 2811 {"X-Remove-Container-Read": "", 2812 "X-Remove-Container-Write": "", 2813 "X-Remove-Container-Sync-Key": "", 2814 "X-Remove-Container-Sync-To": "", 2815 "X-Remove-Versions-Location": "", 2816 "X-Remove-Container-Meta-Box": ""}, 2817 {"X-Container-Read": "xcr", 2818 "X-Container-Write": "xcw", 2819 "X-Container-Sync-Key": "sync-key", 2820 "X-Container-Sync-To": "sync-to"}) 2821 2822 2823 class TestContainerDelete(BaseMiddlewareTest): 2824 def test_success(self): 2825 def mock_RpcDelete(_): 2826 return {"error": None, "result": {}} 2827 2828 self.fake_rpc.register_handler( 2829 "Server.RpcDelete", mock_RpcDelete) 2830 2831 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2832 environ={"REQUEST_METHOD": "DELETE"}) 2833 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2834 status, _, _ = self.call_pfs(req) 2835 self.assertEqual("204 No Content", status) 2836 self.assertNotIn("Accept-Ranges", req.headers) 2837 self.assertEqual(2, len(self.fake_rpc.calls)) 2838 self.assertEqual("/v1/AUTH_test/empty-con", 2839 self.fake_rpc.calls[1][1][0]["VirtPath"]) 2840 cic.assert_called() 2841 2842 def test_special_chars(self): 2843 def mock_RpcDelete(_): 2844 return {"error": None, "result": {}} 2845 2846 self.fake_rpc.register_handler( 2847 "Server.RpcDelete", mock_RpcDelete) 2848 2849 req = swob.Request.blank("/v1/AUTH_test/e m p t y", 2850 environ={"REQUEST_METHOD": "DELETE"}) 2851 status, _, _ = self.call_pfs(req) 2852 self.assertEqual("204 No Content", status) 2853 self.assertEqual("/v1/AUTH_test/e m p t y", 2854 self.fake_rpc.calls[1][1][0]["VirtPath"]) 2855 2856 def test_not_found(self): 2857 def mock_RpcDelete(_): 2858 return {"error": "errno: 2", "result": None} 2859 2860 self.fake_rpc.register_handler( 2861 "Server.RpcDelete", mock_RpcDelete) 2862 2863 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2864 environ={"REQUEST_METHOD": "DELETE"}) 2865 status, _, _ = self.call_pfs(req) 2866 self.assertEqual("404 Not Found", status) 2867 2868 def test_not_empty(self): 2869 def mock_RpcDelete(_): 2870 return {"error": "errno: 39", "result": None} 2871 2872 self.fake_rpc.register_handler( 2873 "Server.RpcDelete", mock_RpcDelete) 2874 2875 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2876 environ={"REQUEST_METHOD": "DELETE"}) 2877 status, _, _ = self.call_pfs(req) 2878 self.assertEqual("409 Conflict", status) 2879 2880 def test_other_error(self): 2881 def mock_RpcDelete(_): 2882 return {"error": "errno: 987654321", "result": None} 2883 2884 self.fake_rpc.register_handler( 2885 "Server.RpcDelete", mock_RpcDelete) 2886 2887 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2888 environ={"REQUEST_METHOD": "DELETE"}) 2889 status, _, _ = self.call_pfs(req) 2890 self.assertEqual("500 Internal Error", status) 2891 2892 2893 class TestObjectPut(BaseMiddlewareTest): 2894 def setUp(self): 2895 super(TestObjectPut, self).setUp() 2896 2897 # These mocks act as though everything was successful. Failure tests 2898 # can override the relevant mocks in the individual test cases. 2899 def mock_RpcHead(head_container_req): 2900 # Empty container, but it exists. That's enough for testing 2901 # object PUT. 2902 return { 2903 "error": None, 2904 "result": { 2905 "Metadata": "", 2906 "ModificationTime": 14792389930244718933, 2907 "FileSize": 0, 2908 "IsDir": True, 2909 "InodeNumber": 1828, 2910 "NumWrites": 893, 2911 }} 2912 2913 put_loc_count = collections.defaultdict(int) 2914 2915 def mock_RpcPutLocation(put_location_req): 2916 # Give a different sequence of physical paths for each object 2917 # name 2918 virt_path = put_location_req["VirtPath"] 2919 obj_name = hashlib.sha1( 2920 virt_path.encode('utf8')).hexdigest().upper() 2921 phys_path = "/v1/AUTH_test/PhysContainer_1/" + obj_name 2922 if put_loc_count[virt_path] > 0: 2923 phys_path += "-%02x" % put_loc_count[virt_path] 2924 put_loc_count[virt_path] += 1 2925 2926 # Someone's probably about to PUT an object there, so let's set 2927 # up the mock to allow it. Doing it here ensures that the 2928 # location comes out of this RPC and nowhere else. 2929 2930 self.app.register('PUT', phys_path, 201, {}, "") 2931 2932 return { 2933 "error": None, 2934 "result": {"PhysPath": phys_path}} 2935 2936 def mock_RpcPutComplete(put_complete_req): 2937 return {"error": None, "result": { 2938 "ModificationTime": 12345, 2939 "InodeNumber": 678, 2940 "NumWrites": 9}} 2941 2942 def mock_RpcMiddlewareMkdir(middleware_mkdir_req): 2943 return {"error": None, "result": { 2944 "ModificationTime": 1504652321749543000, 2945 "InodeNumber": 9268022, 2946 "NumWrites": 0}} 2947 2948 self.fake_rpc.register_handler( 2949 "Server.RpcHead", mock_RpcHead) 2950 self.fake_rpc.register_handler( 2951 "Server.RpcPutLocation", mock_RpcPutLocation) 2952 self.fake_rpc.register_handler( 2953 "Server.RpcPutComplete", mock_RpcPutComplete) 2954 self.fake_rpc.register_handler( 2955 "Server.RpcMiddlewareMkdir", mock_RpcMiddlewareMkdir) 2956 2957 def test_basic(self): 2958 wsgi_input = BytesIO(b"sparkleberry-displeasurably") 2959 cl = str(len(wsgi_input.getvalue())) 2960 2961 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 2962 environ={"REQUEST_METHOD": "PUT", 2963 "wsgi.input": wsgi_input, 2964 "CONTENT_LENGTH": cl}) 2965 status, headers, body = self.call_pfs(req) 2966 self.assertEqual(status, '201 Created') 2967 self.assertEqual(headers["ETag"], 2968 hashlib.md5(wsgi_input.getvalue()).hexdigest()) 2969 2970 rpc_calls = self.fake_rpc.calls 2971 self.assertEqual(len(rpc_calls), 4) 2972 2973 method, args = rpc_calls[0] 2974 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2975 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2976 2977 method, args = rpc_calls[1] 2978 self.assertEqual(method, "Server.RpcHead") 2979 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/a-container") 2980 2981 method, args = rpc_calls[2] 2982 self.assertEqual(method, "Server.RpcPutLocation") 2983 self.assertEqual(args[0]["VirtPath"], 2984 "/v1/AUTH_test/a-container/an-object") 2985 2986 method, args = rpc_calls[3] 2987 expected_phys_path = ("/v1/AUTH_test/PhysContainer_1/" 2988 "80D184B041B9BF0C2EE8D55D8DC9797BF7129E13") 2989 self.assertEqual(method, "Server.RpcPutComplete") 2990 self.assertEqual(args[0]["VirtPath"], 2991 "/v1/AUTH_test/a-container/an-object") 2992 self.assertEqual(args[0]["PhysPaths"], [expected_phys_path]) 2993 self.assertEqual(args[0]["PhysLengths"], [len(wsgi_input.getvalue())]) 2994 2995 def test_directory(self): 2996 req = swob.Request.blank( 2997 "/v1/AUTH_test/a-container/a-dir", 2998 environ={"REQUEST_METHOD": "PUT"}, 2999 headers={"Content-Length": 0, 3000 "Content-Type": "application/directory", 3001 "X-Object-Sysmeta-Abc": "DEF"}, 3002 body="") 3003 3004 # directories always return the hard coded value for EMPTY_OBJECT_ETAG 3005 status, headers, body = self.call_pfs(req) 3006 self.assertEqual(status, '201 Created') 3007 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 3008 3009 rpc_calls = self.fake_rpc.calls 3010 self.assertEqual(len(rpc_calls), 3) 3011 3012 method, args = rpc_calls[2] 3013 self.assertEqual(method, "Server.RpcMiddlewareMkdir") 3014 self.assertEqual(args[0]["VirtPath"], 3015 "/v1/AUTH_test/a-container/a-dir") 3016 3017 serialized_metadata = args[0]["Metadata"] 3018 metadata = json.loads(base64.b64decode(serialized_metadata)) 3019 self.assertEqual(metadata.get("X-Object-Sysmeta-Abc"), "DEF") 3020 3021 def test_modification_time(self): 3022 def mock_RpcPutComplete(put_complete_req): 3023 return {"error": None, "result": { 3024 "ModificationTime": 1481311245635845000, 3025 "InodeNumber": 4116394, 3026 "NumWrites": 1}} 3027 3028 self.fake_rpc.register_handler( 3029 "Server.RpcPutComplete", mock_RpcPutComplete) 3030 3031 wsgi_input = BytesIO(b"Rhodothece-cholesterinuria") 3032 cl = str(len(wsgi_input.getvalue())) 3033 3034 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3035 environ={"REQUEST_METHOD": "PUT", 3036 "wsgi.input": wsgi_input, 3037 "CONTENT_LENGTH": cl}) 3038 status, headers, body = self.call_pfs(req) 3039 self.assertEqual(status, "201 Created") 3040 self.assertEqual(headers["Last-Modified"], 3041 "Fri, 09 Dec 2016 19:20:46 GMT") 3042 3043 def test_special_chars(self): 3044 wsgi_input = BytesIO(b"pancreas-mystagogically") 3045 cl = str(len(wsgi_input.getvalue())) 3046 3047 req = swob.Request.blank("/v1/AUTH_test/c o n/o b j", 3048 environ={"REQUEST_METHOD": "PUT", 3049 "wsgi.input": wsgi_input, 3050 "CONTENT_LENGTH": cl}) 3051 3052 status, headers, body = self.call_pfs(req) 3053 self.assertEqual(status, '201 Created') 3054 3055 rpc_calls = self.fake_rpc.calls 3056 self.assertEqual(len(rpc_calls), 4) 3057 3058 method, args = rpc_calls[0] 3059 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3060 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3061 3062 method, args = rpc_calls[1] 3063 self.assertEqual(method, "Server.RpcHead") 3064 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/c o n") 3065 3066 method, args = rpc_calls[2] 3067 self.assertEqual(method, "Server.RpcPutLocation") 3068 self.assertEqual(args[0]["VirtPath"], 3069 "/v1/AUTH_test/c o n/o b j") 3070 3071 method, args = rpc_calls[3] 3072 self.assertEqual(method, "Server.RpcPutComplete") 3073 self.assertEqual(args[0]["VirtPath"], 3074 "/v1/AUTH_test/c o n/o b j") 3075 3076 def test_PUT_bad_path(self): 3077 bad_container_paths = [ 3078 '/v1/AUTH_test/../o', 3079 '/v1/AUTH_test/./o', 3080 '/v1/AUTH_test/' + ('x' * 256) + '/o', 3081 ] 3082 for path in bad_container_paths: 3083 req = swob.Request.blank(path, 3084 environ={"REQUEST_METHOD": "PUT", 3085 "wsgi.input": BytesIO(b""), 3086 "CONTENT_LENGTH": "0"}) 3087 status, headers, body = self.call_pfs(req) 3088 self.assertEqual(status, '404 Not Found', 3089 'Got %s for %s' % (status, path)) 3090 3091 req.environ['REQUEST_METHOD'] = 'POST' 3092 status, headers, body = self.call_pfs(req) 3093 self.assertEqual(status, '404 Not Found', 3094 'Got %s for %s' % (status, path)) 3095 3096 req.environ['REQUEST_METHOD'] = 'DELETE' 3097 status, headers, body = self.call_pfs(req) 3098 self.assertEqual(status, '404 Not Found', 3099 'Got %s for %s' % (status, path)) 3100 3101 bad_paths = [ 3102 '/v1/AUTH_test/c/..', 3103 '/v1/AUTH_test/c/../o', 3104 '/v1/AUTH_test/c/o/..', 3105 '/v1/AUTH_test/c/.', 3106 '/v1/AUTH_test/c/./o', 3107 '/v1/AUTH_test/c/o/.', 3108 '/v1/AUTH_test/c//o', 3109 '/v1/AUTH_test/c/o//', 3110 '/v1/AUTH_test/c/o/', 3111 '/v1/AUTH_test/c/' + ('x' * 256), 3112 ] 3113 for path in bad_paths: 3114 req = swob.Request.blank(path, 3115 environ={"REQUEST_METHOD": "PUT", 3116 "wsgi.input": BytesIO(b""), 3117 "CONTENT_LENGTH": "0"}) 3118 status, headers, body = self.call_pfs(req) 3119 self.assertEqual(status, '400 Bad Request', 3120 'Got %s for %s' % (status, path)) 3121 3122 req.environ['REQUEST_METHOD'] = 'POST' 3123 status, headers, body = self.call_pfs(req) 3124 self.assertEqual(status, '404 Not Found', 3125 'Got %s for %s' % (status, path)) 3126 3127 req.environ['REQUEST_METHOD'] = 'DELETE' 3128 status, headers, body = self.call_pfs(req) 3129 self.assertEqual(status, '404 Not Found', 3130 'Got %s for %s' % (status, path)) 3131 3132 def test_big(self): 3133 wsgi_input = BytesIO(b'A' * 100 + b'B' * 100 + b'C' * 75) 3134 self.pfs.max_log_segment_size = 100 3135 3136 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3137 environ={"REQUEST_METHOD": "PUT", 3138 "wsgi.input": wsgi_input}, 3139 headers={"X-Trans-Id": "big-txid", 3140 "Content-Length": "275"}) 3141 status, headers, body = self.call_pfs(req) 3142 self.assertEqual(status, '201 Created') 3143 self.assertEqual(headers["Content-Type"], "application/octet-stream") 3144 3145 rpc_calls = self.fake_rpc.calls 3146 self.assertEqual(len(rpc_calls), 6) 3147 3148 method, args = rpc_calls[0] 3149 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3150 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3151 3152 method, args = rpc_calls[1] 3153 self.assertEqual(method, "Server.RpcHead") 3154 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") 3155 3156 # 3 calls to RpcPutLocation since this was spread across 3 log 3157 # segments 3158 method, args = rpc_calls[2] 3159 self.assertEqual(method, "Server.RpcPutLocation") 3160 self.assertEqual(args[0]["VirtPath"], 3161 "/v1/AUTH_test/con/obj") 3162 3163 method, args = rpc_calls[3] 3164 self.assertEqual(method, "Server.RpcPutLocation") 3165 self.assertEqual(args[0]["VirtPath"], 3166 "/v1/AUTH_test/con/obj") 3167 3168 method, args = rpc_calls[4] 3169 self.assertEqual(method, "Server.RpcPutLocation") 3170 self.assertEqual(args[0]["VirtPath"], 3171 "/v1/AUTH_test/con/obj") 3172 3173 method, args = rpc_calls[5] 3174 self.assertEqual(method, "Server.RpcPutComplete") 3175 self.assertEqual(args[0]["VirtPath"], 3176 "/v1/AUTH_test/con/obj") 3177 pre = "/v1/AUTH_test/PhysContainer_1/" 3178 self.assertEqual( 3179 args[0]["PhysPaths"], 3180 [pre + "1550057D8B0039185EB6184C599C940E51953403", 3181 pre + "1550057D8B0039185EB6184C599C940E51953403-01", 3182 pre + "1550057D8B0039185EB6184C599C940E51953403-02"]) 3183 self.assertEqual(args[0]["PhysLengths"], [100, 100, 75]) 3184 3185 # check the txids as well 3186 put_calls = [c for c in self.app.calls if c[0] == 'PUT'] 3187 self.assertEqual( 3188 "big-txid-000", put_calls[0][2]["X-Trans-Id"]) # 1st PUT 3189 self.assertEqual( 3190 "big-txid-001", put_calls[1][2]["X-Trans-Id"]) # 2nd PUT 3191 self.assertEqual( 3192 "big-txid-002", put_calls[2][2]["X-Trans-Id"]) # 3rd PUT 3193 3194 # If we sent the original Content-Length, the first PUT would fail. 3195 # At some point, we should send the correct Content-Length value 3196 # when we can compute it, but for now, we just send nothing. 3197 self.assertNotIn("Content-Length", put_calls[0][2]) # 1st PUT 3198 self.assertNotIn("Content-Length", put_calls[1][2]) # 2nd PUT 3199 self.assertNotIn("Content-Length", put_calls[2][2]) # 3rd PUT 3200 3201 def test_big_exact_multiple(self): 3202 wsgi_input = BytesIO(b'A' * 100 + b'B' * 100) 3203 cl = str(len(wsgi_input.getvalue())) 3204 self.pfs.max_log_segment_size = 100 3205 3206 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3207 environ={"REQUEST_METHOD": "PUT", 3208 "wsgi.input": wsgi_input, 3209 "CONTENT_LENGTH": cl}, 3210 headers={"X-Trans-Id": "big-txid"}) 3211 status, headers, body = self.call_pfs(req) 3212 self.assertEqual(status, '201 Created') 3213 self.assertEqual(headers["Content-Type"], "application/octet-stream") 3214 3215 rpc_calls = self.fake_rpc.calls 3216 self.assertEqual(len(rpc_calls), 5) 3217 3218 method, args = rpc_calls[0] 3219 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3220 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3221 3222 method, args = rpc_calls[1] 3223 self.assertEqual(method, "Server.RpcHead") 3224 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") 3225 3226 # 2 calls to RpcPutLocation since this was spread across 2 log 3227 # segments. We didn't make a 0-length log segment and try to splice 3228 # that in. 3229 method, args = rpc_calls[2] 3230 self.assertEqual(method, "Server.RpcPutLocation") 3231 self.assertEqual(args[0]["VirtPath"], 3232 "/v1/AUTH_test/con/obj") 3233 3234 method, args = rpc_calls[3] 3235 self.assertEqual(method, "Server.RpcPutLocation") 3236 self.assertEqual(args[0]["VirtPath"], 3237 "/v1/AUTH_test/con/obj") 3238 3239 method, args = rpc_calls[4] 3240 self.assertEqual(method, "Server.RpcPutComplete") 3241 self.assertEqual(args[0]["VirtPath"], 3242 "/v1/AUTH_test/con/obj") 3243 pre = "/v1/AUTH_test/PhysContainer_1/" 3244 self.assertEqual( 3245 args[0]["PhysPaths"], 3246 [pre + "1550057D8B0039185EB6184C599C940E51953403", 3247 pre + "1550057D8B0039185EB6184C599C940E51953403-01"]) 3248 self.assertEqual(args[0]["PhysLengths"], [100, 100]) 3249 3250 def test_missing_container(self): 3251 def mock_RpcHead_not_found(head_container_req): 3252 # This is what you get for no-such-container. 3253 return { 3254 "error": "errno: 2", 3255 "result": None} 3256 3257 self.fake_rpc.register_handler( 3258 "Server.RpcHead", mock_RpcHead_not_found) 3259 3260 wsgi_input = BytesIO(b"toxicum-brickcroft") 3261 3262 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3263 environ={"REQUEST_METHOD": "PUT", 3264 "wsgi.input": wsgi_input}) 3265 status, headers, body = self.call_pfs(req) 3266 self.assertEqual(status, "404 Not Found") 3267 3268 def test_metadata(self): 3269 wsgi_input = BytesIO(b"extranean-paleophysiology") 3270 cl = str(len(wsgi_input.getvalue())) 3271 3272 headers_in = { 3273 "X-Object-Meta-Color": "puce", 3274 "X-Object-Sysmeta-Flavor": "bbq", 3275 3276 "Content-Disposition": "recycle when no longer needed", 3277 "Content-Encoding": "quadruple rot13", 3278 "Content-Type": "application/eggplant", 3279 # NB: we'll never actually see these two together, but it's fine 3280 # for this test since we're only looking at which headers get 3281 # saved and which don't. 3282 "X-Object-Manifest": "pre/fix", 3283 "X-Static-Large-Object": "yes", 3284 3285 # These are not included 3286 "X-Timestamp": "1473968446.11364", 3287 "X-Object-Qmeta-Shape": "trapezoidal", 3288 } 3289 3290 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3291 headers=headers_in, 3292 environ={"REQUEST_METHOD": "PUT", 3293 "wsgi.input": wsgi_input, 3294 "CONTENT_LENGTH": cl}) 3295 status, headers, body = self.call_pfs(req) 3296 self.assertEqual(status, '201 Created') 3297 3298 serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] 3299 metadata = json.loads(base64.b64decode(serialized_metadata)) 3300 self.assertEqual(metadata.get("X-Object-Meta-Color"), "puce") 3301 self.assertEqual(metadata.get("X-Object-Sysmeta-Flavor"), "bbq") 3302 self.assertEqual(metadata.get("X-Object-Manifest"), "pre/fix") 3303 self.assertEqual(metadata.get("X-Static-Large-Object"), "yes") 3304 self.assertEqual(metadata.get("Content-Disposition"), 3305 "recycle when no longer needed") 3306 self.assertEqual(metadata.get("Content-Encoding"), "quadruple rot13") 3307 self.assertEqual(metadata.get("Content-Type"), "application/eggplant") 3308 3309 def test_directory_in_the_way(self): 3310 # If "thing.txt" is a nonempty directory, we get an error that the 3311 # middleware turns into a 409 Conflict response. 3312 wsgi_input = BytesIO(b"Celestine-malleal") 3313 cl = str(len(wsgi_input.getvalue())) 3314 3315 def mock_RpcPutComplete_isdir(head_container_req): 3316 # This is what you get when there's a nonempty directory in 3317 # place of your file. 3318 return { 3319 "error": "errno: 39", 3320 "result": None} 3321 3322 self.fake_rpc.register_handler( 3323 "Server.RpcPutComplete", mock_RpcPutComplete_isdir) 3324 3325 req = swob.Request.blank("/v1/AUTH_test/a-container/d1/d2/thing.txt", 3326 environ={"REQUEST_METHOD": "PUT", 3327 "wsgi.input": wsgi_input, 3328 "CONTENT_LENGTH": cl}) 3329 status, headers, body = self.call_pfs(req) 3330 self.assertEqual(status, '409 Conflict') 3331 3332 def test_file_in_the_way(self): 3333 # If "thing.txt" is a nonempty directory, we get an error that the 3334 # middleware turns into a 409 Conflict response. 3335 wsgi_input = BytesIO(b"Celestine-malleal") 3336 cl = str(len(wsgi_input.getvalue())) 3337 3338 def mock_RpcPutComplete_notdir(head_container_req): 3339 # This is what you get when there's a file where your path 3340 # contains a subdirectory. 3341 return { 3342 "error": "errno: 20", 3343 "result": None} 3344 3345 self.fake_rpc.register_handler( 3346 "Server.RpcPutComplete", mock_RpcPutComplete_notdir) 3347 3348 req = swob.Request.blank("/v1/AUTH_test/a-container/a-file/thing.txt", 3349 environ={"REQUEST_METHOD": "PUT", 3350 "wsgi.input": wsgi_input, 3351 "CONTENT_LENGTH": cl}) 3352 status, headers, body = self.call_pfs(req) 3353 self.assertEqual(status, '409 Conflict') 3354 3355 def test_stripping_bad_headers(self): 3356 # Someday, we'll have to figure out how to expire objects in 3357 # proxyfs. For now, though, we remove X-Delete-At and X-Delete-After 3358 # because having a log segment expire will do bad things to our 3359 # file's integrity. 3360 # 3361 # We strip ETag because Swift will treat that as the expected MD5 of 3362 # the log segment, and if we split across multiple log segments, 3363 # then any user-supplied ETag will be wrong. Also, this makes 3364 # POST-as-COPY work despite ProxyFS's ETag values not being MD5 3365 # checksums. 3366 wsgi_input = BytesIO(b"extranean-paleophysiology") 3367 cl = str(len(wsgi_input.getvalue())) 3368 3369 headers_in = {"X-Delete-After": 86400, 3370 "ETag": hashlib.md5(wsgi_input.getvalue()).hexdigest()} 3371 3372 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3373 headers=headers_in, 3374 environ={"REQUEST_METHOD": "PUT", 3375 "wsgi.input": wsgi_input, 3376 "CONTENT_LENGTH": cl}) 3377 status, headers, body = self.call_pfs(req) 3378 self.assertEqual(status, '201 Created') 3379 3380 serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] 3381 metadata = json.loads(base64.b64decode(serialized_metadata)) 3382 # it didn't get saved in metadata (not that it matters too much) 3383 self.assertNotIn("X-Delete-At", metadata) 3384 self.assertNotIn("X-Delete-After", metadata) 3385 self.assertNotIn("ETag", metadata) 3386 3387 # it didn't make it to Swift (this is important) 3388 put_method, put_path, put_headers = self.app.calls[-1] 3389 self.assertEqual(put_method, 'PUT') # sanity check 3390 self.assertNotIn("X-Delete-At", put_headers) 3391 self.assertNotIn("X-Delete-After", put_headers) 3392 self.assertNotIn("ETag", put_headers) 3393 3394 def test_etag_checking(self): 3395 wsgi_input = BytesIO(b"unsplashed-comprest") 3396 right_etag = hashlib.md5(wsgi_input.getvalue()).hexdigest() 3397 wrong_etag = hashlib.md5(wsgi_input.getvalue() + b"abc").hexdigest() 3398 non_checksum_etag = "pfsv2/AUTH_test/2226116/4341333-32" 3399 cl = str(len(wsgi_input.getvalue())) 3400 3401 wsgi_input.seek(0) 3402 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3403 environ={"REQUEST_METHOD": "PUT", 3404 "wsgi.input": wsgi_input}, 3405 headers={"ETag": right_etag, 3406 "Content-Length": cl}) 3407 status, headers, body = self.call_pfs(req) 3408 self.assertEqual(status, '201 Created') 3409 3410 wsgi_input.seek(0) 3411 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3412 environ={"REQUEST_METHOD": "PUT", 3413 "wsgi.input": wsgi_input}, 3414 headers={"ETag": wrong_etag, 3415 "Content-Length": cl}) 3416 status, headers, body = self.call_pfs(req) 3417 self.assertEqual(status, '422 Unprocessable Entity') 3418 3419 wsgi_input.seek(0) 3420 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3421 environ={"REQUEST_METHOD": "PUT", 3422 "wsgi.input": wsgi_input}, 3423 headers={"ETag": non_checksum_etag, 3424 "Content-Length": cl}) 3425 status, headers, body = self.call_pfs(req) 3426 self.assertEqual(status, '201 Created') 3427 3428 def test_etag_checking_dir(self): 3429 req = swob.Request.blank( 3430 "/v1/AUTH_test/a-container/a-dir-object", 3431 environ={"REQUEST_METHOD": "PUT"}, 3432 headers={"ETag": hashlib.md5(b"x").hexdigest(), 3433 "Content-Length": 0, 3434 "Content-Type": "application/directory"}) 3435 status, headers, body = self.call_pfs(req) 3436 self.assertEqual(status, '422 Unprocessable Entity') 3437 3438 req = swob.Request.blank( 3439 "/v1/AUTH_test/a-container/a-dir-object", 3440 environ={"REQUEST_METHOD": "PUT"}, 3441 headers={"ETag": hashlib.md5(b"").hexdigest(), 3442 "Content-Length": 0, 3443 "Content-Type": "application/directory"}) 3444 status, headers, body = self.call_pfs(req) 3445 self.assertEqual(status, '201 Created') 3446 3447 req = swob.Request.blank( 3448 "/v1/AUTH_test/a-container/a-dir-object", 3449 environ={"REQUEST_METHOD": "PUT"}, 3450 headers={"Content-Length": 0, 3451 "Content-Type": "application/directory"}) 3452 status, headers, body = self.call_pfs(req) 3453 self.assertEqual(status, '201 Created') 3454 3455 3456 class TestObjectPost(BaseMiddlewareTest): 3457 def test_missing_object(self): 3458 def mock_RpcHead(_): 3459 return {"error": "errno: 2", "result": None} 3460 3461 self.fake_rpc.register_handler( 3462 "Server.RpcHead", mock_RpcHead) 3463 3464 req = swob.Request.blank( 3465 "/v1/AUTH_test/con/obj", 3466 environ={"REQUEST_METHOD": "POST"}, 3467 headers={"X-Object-Meta-One-Fish": "two fish"}) 3468 status, _, _ = self.call_pfs(req) 3469 self.assertEqual("404 Not Found", status) 3470 3471 rpc_calls = self.fake_rpc.calls 3472 self.assertEqual(2, len(rpc_calls)) 3473 3474 method, args = rpc_calls[0] 3475 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3476 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3477 3478 method, args = rpc_calls[1] 3479 self.assertEqual(method, "Server.RpcHead") 3480 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3481 3482 def test_file_as_dir(self): 3483 # Subdirectories of files don't exist, but asking for one returns a 3484 # different error code than asking for a file that could exist but 3485 # doesn't. 3486 def mock_RpcHead(head_object_req): 3487 self.assertEqual(head_object_req['VirtPath'], 3488 "/v1/AUTH_test/c/thing.txt/kitten.png") 3489 3490 return { 3491 "error": "errno: 20", 3492 "result": None} 3493 3494 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', 3495 method='POST') 3496 self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) 3497 status, headers, body = self.call_pfs(req) 3498 self.assertEqual(status, '404 Not Found') 3499 3500 def test_existing_object(self): 3501 old_meta = json.dumps({ 3502 "Content-Type": "application/fishy", 3503 mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", 3504 "X-Object-Meta-One-Fish": "two fish"}) 3505 3506 def mock_RpcHead(_): 3507 return { 3508 "error": None, 3509 "result": { 3510 "Metadata": base64.b64encode( 3511 old_meta.encode('ascii')).decode('ascii'), 3512 "ModificationTime": 1482345542483719281, 3513 "FileSize": 551155, 3514 "IsDir": False, 3515 "InodeNumber": 6519913, 3516 "NumWrites": 1}} 3517 3518 self.fake_rpc.register_handler( 3519 "Server.RpcHead", mock_RpcHead) 3520 3521 self.fake_rpc.register_handler( 3522 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3523 3524 req = swob.Request.blank( 3525 "/v1/AUTH_test/con/obj", 3526 environ={"REQUEST_METHOD": "POST"}, 3527 headers={"X-Object-Meta-Red-Fish": "blue fish"}) 3528 status, headers, _ = self.call_pfs(req) 3529 3530 # For reference, the result of a real object POST request. The 3531 # request contained new metadata items, but notice that those are 3532 # not reflected in the response. 3533 # 3534 # HTTP/1.1 202 Accepted 3535 # Content-Length: 76 3536 # Content-Type: text/html; charset=UTF-8 3537 # X-Trans-Id: tx65d5632fe4a548999ee9e-005a39781b 3538 # X-Openstack-Request-Id: tx65d5632fe4a548999ee9e-005a39781b 3539 # Date: Tue, 19 Dec 2017 20:35:39 GMT 3540 self.assertEqual("202 Accepted", status) 3541 self.assertNotIn('Last-Modified', headers) 3542 self.assertNotIn('Etag', headers) 3543 self.assertEqual("0", headers["Content-Length"]) 3544 self.assertEqual("text/html; charset=UTF-8", headers["Content-Type"]) 3545 # Date and X-Trans-Id are added in by other parts of the WSGI stack 3546 3547 rpc_calls = self.fake_rpc.calls 3548 self.assertEqual(3, len(rpc_calls)) 3549 3550 method, args = rpc_calls[0] 3551 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3552 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3553 3554 method, args = rpc_calls[1] 3555 self.assertEqual(method, "Server.RpcHead") 3556 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3557 3558 method, args = rpc_calls[2] 3559 self.assertEqual(method, "Server.RpcPost") 3560 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3561 self.assertEqual( 3562 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3563 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3564 self.assertEqual(new_meta["X-Object-Meta-Red-Fish"], "blue fish") 3565 self.assertEqual(new_meta["Content-Type"], "application/fishy") 3566 self.assertNotIn("X-Object-Meta-One-Fish", new_meta) 3567 3568 def test_preservation(self): 3569 old_meta = json.dumps({ 3570 "Content-Type": "application/fishy", 3571 mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", 3572 "X-Static-Large-Object": "true", 3573 "X-Object-Manifest": "solo/duet", 3574 "X-Object-Sysmeta-Dog": "collie", 3575 "X-Object-Meta-Fish": "perch"}) 3576 3577 def mock_RpcHead(_): 3578 return { 3579 "error": None, 3580 "result": { 3581 "Metadata": base64.b64encode( 3582 old_meta.encode('ascii')).decode('ascii'), 3583 "ModificationTime": 1510873171878460000, 3584 "FileSize": 7748115, 3585 "IsDir": False, 3586 "InodeNumber": 3741569, 3587 "NumWrites": 1}} 3588 3589 self.fake_rpc.register_handler( 3590 "Server.RpcHead", mock_RpcHead) 3591 3592 self.fake_rpc.register_handler( 3593 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3594 3595 req = swob.Request.blank( 3596 "/v1/AUTH_test/con/obj", 3597 environ={"REQUEST_METHOD": "POST"}, 3598 headers={"X-Object-Meta-Fish": "trout"}) 3599 status, _, _ = self.call_pfs(req) 3600 self.assertEqual("202 Accepted", status) # sanity check 3601 3602 method, args = self.fake_rpc.calls[2] 3603 self.assertEqual(method, "Server.RpcPost") 3604 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3605 self.assertEqual(new_meta["Content-Type"], "application/fishy") 3606 self.assertEqual(new_meta["X-Object-Sysmeta-Dog"], "collie") 3607 self.assertEqual(new_meta["X-Static-Large-Object"], "true") 3608 self.assertEqual(new_meta["X-Object-Meta-Fish"], "trout") 3609 self.assertNotIn("X-Object-Manifest", new_meta) 3610 3611 def test_change_content_type(self): 3612 old_meta = json.dumps({"Content-Type": "old/type"}) 3613 3614 def mock_RpcHead(_): 3615 return { 3616 "error": None, 3617 "result": { 3618 "Metadata": base64.b64encode( 3619 old_meta.encode('ascii')).decode('ascii'), 3620 "ModificationTime": 1482345542483719281, 3621 "FileSize": 551155, 3622 "IsDir": False, 3623 "InodeNumber": 6519913, 3624 "NumWrites": 381}} 3625 3626 self.fake_rpc.register_handler( 3627 "Server.RpcHead", mock_RpcHead) 3628 3629 self.fake_rpc.register_handler( 3630 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3631 3632 req = swob.Request.blank( 3633 "/v1/AUTH_test/con/obj", 3634 environ={"REQUEST_METHOD": "POST"}, 3635 headers={"Content-Type": "new/type"}) 3636 status, _, _ = self.call_pfs(req) 3637 self.assertEqual("202 Accepted", status) 3638 3639 rpc_calls = self.fake_rpc.calls 3640 3641 method, args = rpc_calls[2] 3642 self.assertEqual(method, "Server.RpcPost") 3643 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3644 self.assertEqual( 3645 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3646 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3647 self.assertEqual(new_meta["Content-Type"], "new/type") 3648 3649 def test_metadata_blanks(self): 3650 old_meta = json.dumps({"X-Object-Meta-Color": "red"}) 3651 3652 def mock_RpcHead(_): 3653 return { 3654 "error": None, 3655 "result": { 3656 "Metadata": base64.b64encode( 3657 old_meta.encode('ascii')).decode('ascii'), 3658 "ModificationTime": 1482345542483719281, 3659 "FileSize": 551155, 3660 "IsDir": False, 3661 "InodeNumber": 6519913, 3662 "NumWrites": 381}} 3663 3664 self.fake_rpc.register_handler( 3665 "Server.RpcHead", mock_RpcHead) 3666 3667 self.fake_rpc.register_handler( 3668 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3669 3670 req = swob.Request.blank( 3671 "/v1/AUTH_test/con/obj", 3672 environ={"REQUEST_METHOD": "POST"}, 3673 headers={"X-Object-Meta-Color": ""}) 3674 status, _, _ = self.call_pfs(req) 3675 self.assertEqual("202 Accepted", status) 3676 3677 rpc_calls = self.fake_rpc.calls 3678 3679 method, args = rpc_calls[2] 3680 self.assertEqual(method, "Server.RpcPost") 3681 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3682 self.assertEqual( 3683 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3684 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3685 self.assertNotIn("X-Object-Meta-Color", new_meta) 3686 3687 3688 class TestObjectDelete(BaseMiddlewareTest): 3689 def test_success(self): 3690 def fake_RpcDelete(delete_request): 3691 return {"error": None, "result": {}} 3692 3693 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3694 3695 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3696 environ={"REQUEST_METHOD": "DELETE"}) 3697 3698 status, _, _ = self.call_pfs(req) 3699 self.assertEqual(status, "204 No Content") 3700 3701 # the first call is a call to RpcIsAccountBimodal 3702 self.assertEqual(len(self.fake_rpc.calls), 2) 3703 self.assertEqual(self.fake_rpc.calls[1][1][0]["VirtPath"], 3704 "/v1/AUTH_test/con/obj") 3705 3706 def test_not_found(self): 3707 def fake_RpcDelete(delete_request): 3708 return {"error": "errno: 2", # NotFoundError / ENOENT 3709 "result": None} 3710 3711 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3712 3713 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3714 environ={"REQUEST_METHOD": "DELETE"}) 3715 3716 status, _, _ = self.call_pfs(req) 3717 self.assertEqual(status, "404 Not Found") 3718 3719 def test_no_such_dir(self): 3720 def fake_RpcDelete(delete_request): 3721 return {"error": "errno: 20", # NotDirError 3722 "result": None} 3723 3724 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3725 3726 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3727 environ={"REQUEST_METHOD": "DELETE"}) 3728 3729 status, _, _ = self.call_pfs(req) 3730 self.assertEqual(status, "404 Not Found") 3731 3732 def test_not_empty(self): 3733 def fake_RpcDelete(delete_request): 3734 return {"error": "errno: 39", # NotEmptyError / ENOTEMPTY 3735 "result": None} 3736 3737 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3738 3739 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3740 environ={"REQUEST_METHOD": "DELETE"}) 3741 3742 status, _, _ = self.call_pfs(req) 3743 self.assertEqual(status, "409 Conflict") 3744 3745 def test_other_failure(self): 3746 def fake_RpcDelete(delete_request): 3747 return {"error": "errno: 9", # BadFileError / EBADF 3748 "result": None} 3749 3750 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3751 3752 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3753 environ={"REQUEST_METHOD": "DELETE"}) 3754 3755 status, _, _ = self.call_pfs(req) 3756 self.assertEqual(status, "500 Internal Error") 3757 3758 3759 class TestObjectHead(BaseMiddlewareTest): 3760 def setUp(self): 3761 super(TestObjectHead, self).setUp() 3762 3763 self.serialized_object_metadata = "" 3764 3765 # All these tests run against the same object. 3766 def mock_RpcHead(head_object_req): 3767 self.assertEqual(head_object_req['VirtPath'], 3768 '/v1/AUTH_test/c/an-object.png') 3769 3770 if self.serialized_object_metadata: 3771 md = base64.b64encode( 3772 self.serialized_object_metadata.encode('ascii') 3773 ).decode('ascii') 3774 else: 3775 md = "" 3776 3777 return { 3778 "error": None, 3779 "result": { 3780 "Metadata": md, 3781 "ModificationTime": 1479173168018879490, 3782 "FileSize": 2641863, 3783 "IsDir": False, 3784 "InodeNumber": 4591, 3785 "NumWrites": 874, 3786 }} 3787 3788 self.fake_rpc.register_handler( 3789 "Server.RpcHead", mock_RpcHead) 3790 3791 # For reference: the result of a real HEAD request 3792 # 3793 # swift@saio:~/swift$ curl -I -H "X-Auth-Token: $TOKEN" \ 3794 # http://localhost:8080/v1/AUTH_test/s/tox.ini 3795 # HTTP/1.1 200 OK 3796 # Content-Length: 3954 3797 # Content-Type: application/octet-stream 3798 # Accept-Ranges: bytes 3799 # Last-Modified: Mon, 14 Nov 2016 23:29:00 GMT 3800 # Etag: ceffb3138058597bd7f8a09bdd3865d0 3801 # X-Timestamp: 1479166139.38308 3802 # X-Object-Meta-Mtime: 1476487400.000000 3803 # X-Trans-Id: txeaa367b1809a4583af3c8-00582a4a6c 3804 # Date: Mon, 14 Nov 2016 23:36:12 GMT 3805 # 3806 3807 def test_file_as_dir(self): 3808 # Subdirectories of files don't exist, but asking for one returns a 3809 # different error code than asking for a file that could exist but 3810 # doesn't. 3811 def mock_RpcHead(head_object_req): 3812 self.assertEqual(head_object_req['VirtPath'], 3813 "/v1/AUTH_test/c/thing.txt/kitten.png") 3814 3815 return { 3816 "error": "errno: 20", 3817 "result": None} 3818 3819 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', 3820 method='HEAD') 3821 self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) 3822 status, headers, body = self.call_pfs(req) 3823 self.assertEqual(status, '404 Not Found') 3824 3825 def test_no_meta(self): 3826 self.serialized_object_metadata = "" 3827 3828 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3829 environ={"REQUEST_METHOD": "HEAD"}) 3830 status, headers, body = self.call_pfs(req) 3831 self.assertEqual(status, '200 OK') 3832 3833 self.assertEqual(headers["Content-Length"], "2641863") 3834 self.assertEqual(headers["Content-Type"], "image/png") 3835 self.assertEqual(headers["Accept-Ranges"], "bytes") 3836 self.assertEqual(headers["ETag"], 3837 '"pfsv2/AUTH_test/000011EF/0000036A-32"') 3838 self.assertEqual(headers["Last-Modified"], 3839 "Tue, 15 Nov 2016 01:26:09 GMT") 3840 self.assertEqual(headers["X-Timestamp"], "1479173168.01888") 3841 3842 def test_explicit_content_type(self): 3843 self.serialized_object_metadata = json.dumps({ 3844 "Content-Type": "Pegasus/inartistic"}) 3845 3846 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3847 environ={"REQUEST_METHOD": "HEAD"}) 3848 status, headers, body = self.call_pfs(req) 3849 self.assertEqual(status, '200 OK') 3850 self.assertEqual(headers["Content-Type"], "Pegasus/inartistic") 3851 3852 def test_bogus_meta(self): 3853 self.serialized_object_metadata = "{[{[{[{[{[[(((!" 3854 3855 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3856 environ={"REQUEST_METHOD": "HEAD"}) 3857 status, headers, body = self.call_pfs(req) 3858 self.assertEqual(status, '200 OK') 3859 3860 def test_meta(self): 3861 self.serialized_object_metadata = json.dumps({ 3862 "X-Object-Sysmeta-Fish": "cod", 3863 "X-Object-Meta-Fish": "trout"}) 3864 3865 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3866 environ={"REQUEST_METHOD": "HEAD"}) 3867 status, headers, body = self.call_pfs(req) 3868 self.assertEqual(status, '200 OK') 3869 self.assertEqual(headers["X-Object-Sysmeta-Fish"], "cod") 3870 self.assertEqual(headers["X-Object-Meta-Fish"], "trout") 3871 3872 def test_none_meta(self): 3873 # Sometimes we get a null in the response instead of an empty 3874 # string. No idea why. 3875 self.serialized_object_metadata = None 3876 3877 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3878 environ={"REQUEST_METHOD": "HEAD"}) 3879 status, headers, body = self.call_pfs(req) 3880 self.assertEqual(status, '200 OK') 3881 3882 def test_special_chars(self): 3883 def mock_RpcHead(head_object_req): 3884 self.assertEqual(head_object_req['VirtPath'], 3885 '/v1/AUTH_test/c/a cat.png') 3886 return {"error": "errno: 2", "result": None} 3887 3888 self.fake_rpc.register_handler( 3889 "Server.RpcHead", mock_RpcHead) 3890 3891 req = swob.Request.blank("/v1/AUTH_test/c/a cat.png", 3892 environ={"REQUEST_METHOD": "HEAD"}) 3893 status, headers, body = self.call_pfs(req) 3894 self.assertEqual(status, '404 Not Found') 3895 3896 def test_not_found(self): 3897 def mock_RpcHead(head_object_req): 3898 self.assertEqual(head_object_req['VirtPath'], 3899 '/v1/AUTH_test/c/an-object.png') 3900 return {"error": "errno: 2", "result": None} 3901 3902 self.fake_rpc.register_handler( 3903 "Server.RpcHead", mock_RpcHead) 3904 3905 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3906 environ={"REQUEST_METHOD": "HEAD"}) 3907 status, headers, body = self.call_pfs(req) 3908 self.assertEqual(status, '404 Not Found') 3909 3910 def test_other_error(self): 3911 def mock_RpcHead(head_object_req): 3912 self.assertEqual(head_object_req['VirtPath'], 3913 '/v1/AUTH_test/c/an-object.png') 3914 return {"error": "errno: 7581", "result": None} 3915 3916 self.fake_rpc.register_handler( 3917 "Server.RpcHead", mock_RpcHead) 3918 3919 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3920 environ={"REQUEST_METHOD": "HEAD"}) 3921 status, headers, body = self.call_pfs(req) 3922 self.assertEqual(status, '500 Internal Error') 3923 3924 def test_md5_etag(self): 3925 self.serialized_object_metadata = json.dumps({ 3926 "Content-Type": "Pegasus/inartistic", 3927 mware.ORIGINAL_MD5_HEADER: "1:b61d068208b52f4acbd618860d30faae", 3928 }) 3929 3930 def mock_RpcHead(head_object_req): 3931 md = base64.b64encode( 3932 self.serialized_object_metadata.encode('ascii') 3933 ).decode('ascii') 3934 return { 3935 "error": None, 3936 "result": { 3937 "Metadata": md, 3938 "ModificationTime": 1506039770222591000, 3939 "FileSize": 3397331, 3940 "IsDir": False, 3941 "InodeNumber": 1433230, 3942 "NumWrites": 1, 3943 }} 3944 3945 self.fake_rpc.register_handler( 3946 "Server.RpcHead", mock_RpcHead) 3947 3948 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3949 environ={"REQUEST_METHOD": "HEAD"}) 3950 status, headers, body = self.call_pfs(req) 3951 self.assertEqual(status, '200 OK') 3952 self.assertEqual(headers["Etag"], "b61d068208b52f4acbd618860d30faae") 3953 3954 def test_conditional_if_match(self): 3955 obj_etag = "c2185d19ada5f5c3aa47d6b55dc912df" 3956 3957 self.serialized_object_metadata = json.dumps({ 3958 mware.ORIGINAL_MD5_HEADER: "1:%s" % obj_etag, 3959 }) 3960 3961 def mock_RpcHead(head_object_req): 3962 md = base64.b64encode( 3963 self.serialized_object_metadata.encode('ascii') 3964 ).decode('ascii') 3965 return { 3966 "error": None, 3967 "result": { 3968 "Metadata": md, 3969 "ModificationTime": 1511223407565078000, 3970 "FileSize": 152874, 3971 "IsDir": False, 3972 "InodeNumber": 9566555, 3973 "NumWrites": 1, 3974 }} 3975 3976 self.fake_rpc.register_handler( 3977 "Server.RpcHead", mock_RpcHead) 3978 3979 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3980 environ={"REQUEST_METHOD": "HEAD"}, 3981 headers={"If-Match": obj_etag}) 3982 # matches 3983 status, _, _ = self.call_pfs(req) 3984 self.assertEqual(status, '200 OK') 3985 3986 # doesn't match 3987 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3988 environ={"REQUEST_METHOD": "HEAD"}, 3989 headers={"If-Match": obj_etag + "XYZZY"}) 3990 status, _, _ = self.call_pfs(req) 3991 self.assertEqual(status, '412 Precondition Failed') 3992 3993 3994 class TestObjectHeadDir(BaseMiddlewareTest): 3995 def test_dir(self): 3996 def mock_RpcHead(head_object_req): 3997 self.assertEqual(head_object_req['VirtPath'], 3998 '/v1/AUTH_test/c/a-dir') 3999 4000 return { 4001 "error": None, 4002 "result": { 4003 "Metadata": "", 4004 "ModificationTime": 1479173168018879490, 4005 "FileSize": 0, 4006 "IsDir": True, 4007 "InodeNumber": 1254, 4008 "NumWrites": 896, 4009 }} 4010 4011 self.fake_rpc.register_handler( 4012 "Server.RpcHead", mock_RpcHead) 4013 req = swob.Request.blank("/v1/AUTH_test/c/a-dir", 4014 environ={"REQUEST_METHOD": "HEAD"}) 4015 status, headers, body = self.call_pfs(req) 4016 self.assertEqual(status, '200 OK') 4017 self.assertEqual(headers["Content-Length"], "0") 4018 self.assertEqual(headers["Content-Type"], "application/directory") 4019 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 4020 self.assertEqual(headers["Last-Modified"], 4021 "Tue, 15 Nov 2016 01:26:09 GMT") 4022 4023 4024 class TestObjectCoalesce(BaseMiddlewareTest): 4025 def setUp(self): 4026 super(TestObjectCoalesce, self).setUp() 4027 4028 def mock_RpcHead(head_container_req): 4029 return { 4030 "error": None, 4031 "result": { 4032 "Metadata": "", 4033 "ModificationTime": 1485814697697650000, 4034 "FileSize": 0, 4035 "IsDir": True, 4036 "InodeNumber": 1828, 4037 "NumWrites": 893, 4038 }} 4039 4040 # this handler gets overwritten by the first test that 4041 # registers a "Server.RpcHead" 4042 self.fake_rpc.register_handler( 4043 "Server.RpcHead", mock_RpcHead) 4044 4045 def test_success(self): 4046 4047 # a map from an object's "virtual path" to its metadata 4048 self.obj_metadata = {} 4049 4050 def mock_RpcHead(head_req): 4051 '''Return the object informattion for the new coalesced object. This 4052 assumes that COALESCE operation has already created it. 4053 ''' 4054 4055 resp = { 4056 "error": None, 4057 "result": { 4058 "Metadata": "", 4059 "ModificationTime": 1488323796002909000, 4060 "FileSize": 80 * 1024 * 1024, 4061 "IsDir": False, 4062 "InodeNumber": 283253, 4063 "NumWrites": 5, 4064 } 4065 } 4066 virt_path = head_req['VirtPath'] 4067 if self.obj_metadata[virt_path] is not None: 4068 resp['result']['Metadata'] = self.obj_metadata[virt_path] 4069 return resp 4070 4071 self.fake_rpc.register_handler( 4072 "Server.RpcHead", mock_RpcHead) 4073 4074 def mock_RpcCoalesce(coalesce_req): 4075 4076 # if there's metadata for the new object, save it to return later 4077 if coalesce_req['NewMetaData'] != "": 4078 virt_path = coalesce_req['VirtPath'] 4079 self.obj_metadata[virt_path] = coalesce_req['NewMetaData'] 4080 4081 numWrites = len(coalesce_req['ElementAccountRelativePaths']) 4082 return { 4083 "error": None, 4084 "result": { 4085 "ModificationTime": 1488323796002909000, 4086 "InodeNumber": 283253, 4087 "NumWrites": numWrites, 4088 }} 4089 4090 self.fake_rpc.register_handler( 4091 "Server.RpcCoalesce", mock_RpcCoalesce) 4092 4093 # have the coalesce request suppply the headers that would 4094 # come from s3api for a "complete multi-part upload" request 4095 request_headers = { 4096 'X-Object-Sysmeta-S3Api-Acl': 4097 '{"Owner":"fc",' + 4098 '"Grant":[{"Grantee":"fc","Permission":"FULL_CONTROL"}]}', 4099 'X-Object-Sysmeta-S3Api-Etag': 4100 'cb45770d6cf51effdfb2ea35322459c3-205', 4101 'X-Object-Sysmeta-Slo-Etag': '363d958f0f4c8501a50408a728ba5599', 4102 'X-Object-Sysmeta-Slo-Size': '1073741824', 4103 'X-Object-Sysmeta-Container-Update-Override-Etag': 4104 '10340ab593ac8c32290a278e36d1f8df; ' + 4105 's3_etag=cb45770d6cf51effdfb2ea35322459c3-205; ' + 4106 'slo_etag=363d958f0f4c8501a50408a728ba5599', 4107 'X-Static-Large-Object': 'True', 4108 'Content-Type': 4109 'application/x-www-form-urlencoded; ' + 4110 'charset=utf-8;swift_bytes=1073741824', 4111 'Etag': '10340ab593ac8c32290a278e36d1f8df', 4112 'Date': 'Mon, 01 Apr 2019 22:53:31 GMT', 4113 'Host': 'sm-p1.swiftstack.org', 4114 'Accept': 'application/json', 4115 'Content-Length': '18356', 4116 'X-Auth-Token': None, 4117 } 4118 request_body = json.dumps({ 4119 "elements": [ 4120 "c1/seg1a", 4121 "c1/seg1b", 4122 "c2/seg space 2a", 4123 "c2/seg space 2b", 4124 "c3/seg3", 4125 ]}).encode('ascii') 4126 req = swob.Request.blank( 4127 "/v1/AUTH_test/con/obj", 4128 headers=request_headers, 4129 environ={"REQUEST_METHOD": "COALESCE", 4130 "wsgi.input": BytesIO(request_body)}) 4131 4132 status, headers, body = self.call_pfs(req) 4133 self.assertEqual(status, '201 Created') 4134 self.assertEqual(headers["Etag"], '10340ab593ac8c32290a278e36d1f8df') 4135 4136 # The first call is a call to RpcIsAccountBimodal, the last is the one 4137 # we care about: RpcCoalesce. There *could* be some intervening calls 4138 # to RpcHead to authorize read/write access to the segments, but since 4139 # there's no authorize callback installed, we skip it. 4140 self.assertEqual([method for method, args in self.fake_rpc.calls], [ 4141 "Server.RpcIsAccountBimodal", 4142 "Server.RpcCoalesce", 4143 ]) 4144 method, args = self.fake_rpc.calls[-1] 4145 self.assertEqual(method, "Server.RpcCoalesce") 4146 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 4147 self.assertEqual(args[0]["ElementAccountRelativePaths"], [ 4148 "c1/seg1a", 4149 "c1/seg1b", 4150 "c2/seg space 2a", 4151 "c2/seg space 2b", 4152 "c3/seg3", 4153 ]) 4154 4155 # verify that the metadata was munged correctly 4156 # (SLO headers stripped out, etc.) 4157 req = swob.Request.blank( 4158 "/v1/AUTH_test/con/obj", 4159 environ={"REQUEST_METHOD": "HEAD"}) 4160 status, headers, body = self.call_pfs(req) 4161 self.assertEqual(status, '200 OK') 4162 self.assertEqual(body, b'') 4163 self.assertEqual(headers["Etag"], 4164 '10340ab593ac8c32290a278e36d1f8df') 4165 self.assertIn('X-Object-Sysmeta-S3Api-Acl', headers) 4166 self.assertIn('X-Object-Sysmeta-S3Api-Etag', headers) 4167 self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', 4168 headers) 4169 self.assertNotIn('X-Static-Large-Object', headers) 4170 self.assertNotIn('X-Object-Sysmeta-Slo-Etag', headers) 4171 self.assertNotIn('X-Object-Sysmeta-Slo-Size', headers) 4172 4173 def test_not_authed(self): 4174 def mock_RpcHead(get_container_req): 4175 path = get_container_req['VirtPath'] 4176 return { 4177 "error": None, 4178 "result": { 4179 "Metadata": base64.b64encode(json.dumps({ 4180 "X-Container-Read": path + '\x00read-acl', 4181 "X-Container-Write": path + '\x00write-acl', 4182 }).encode('ascii')).decode('ascii'), 4183 "ModificationTime": 1479240451156825194, 4184 "FileSize": 0, 4185 "IsDir": True, 4186 "InodeNumber": 1255, 4187 "NumWrites": 897, 4188 }} 4189 4190 self.fake_rpc.register_handler( 4191 "Server.RpcHead", mock_RpcHead) 4192 4193 acls = [] 4194 4195 def auth_cb(req): 4196 acls.append(req.acl) 4197 # write access for the target, plus 3 distince element containers 4198 # each needing both read and write checks -- fail the last 4199 if len(acls) >= 7: 4200 return swob.HTTPForbidden(request=req) 4201 4202 request_body = json.dumps({ 4203 "elements": [ 4204 "c1/seg1a", 4205 "c1/seg1b", 4206 "c2/seg space 2a", 4207 "c2/seg space 2b", 4208 "c3/seg3", 4209 ]}).encode('ascii') 4210 req = swob.Request.blank( 4211 "/v1/AUTH_test/con/obj", 4212 environ={"REQUEST_METHOD": "COALESCE", 4213 "wsgi.input": BytesIO(request_body), 4214 "swift.authorize": auth_cb}) 4215 status, headers, body = self.call_pfs(req) 4216 # The first call is a call to RpcIsAccountBimodal, then a bunch of 4217 # RpcHead calls as we authorize the four containers involved. Since 4218 # we fail the authorization, we never make it to RpcCoalesce. 4219 self.assertEqual([method for method, args in self.fake_rpc.calls], [ 4220 "Server.RpcIsAccountBimodal", 4221 "Server.RpcHead", 4222 "Server.RpcHead", 4223 "Server.RpcHead", 4224 "Server.RpcHead", 4225 ]) 4226 container_paths = [ 4227 "/v1/AUTH_test/con", 4228 "/v1/AUTH_test/c1", 4229 "/v1/AUTH_test/c2", 4230 "/v1/AUTH_test/c3", 4231 ] 4232 self.assertEqual(container_paths, [ 4233 args[0]['VirtPath'] for method, args in self.fake_rpc.calls[1:]]) 4234 self.assertEqual(status, '403 Forbidden') 4235 self.assertEqual(acls, [ 4236 "/v1/AUTH_test/con\x00write-acl", 4237 "/v1/AUTH_test/c1\x00read-acl", 4238 "/v1/AUTH_test/c1\x00write-acl", 4239 "/v1/AUTH_test/c2\x00read-acl", 4240 "/v1/AUTH_test/c2\x00write-acl", 4241 "/v1/AUTH_test/c3\x00read-acl", 4242 "/v1/AUTH_test/c3\x00write-acl", 4243 ]) 4244 4245 def test_malformed_json(self): 4246 request_body = b"{{{[[[(((" 4247 req = swob.Request.blank( 4248 "/v1/AUTH_test/con/obj", 4249 environ={"REQUEST_METHOD": "COALESCE", 4250 "wsgi.input": BytesIO(request_body)}) 4251 status, headers, body = self.call_pfs(req) 4252 self.assertEqual(status, '400 Bad Request') 4253 4254 def test_incorrect_json(self): 4255 request_body = b"{}" 4256 req = swob.Request.blank( 4257 "/v1/AUTH_test/con/obj", 4258 environ={"REQUEST_METHOD": "COALESCE", 4259 "wsgi.input": BytesIO(request_body)}) 4260 status, headers, body = self.call_pfs(req) 4261 self.assertEqual(status, '400 Bad Request') 4262 4263 def test_incorrect_json_wrong_type(self): 4264 request_body = b"[]" 4265 req = swob.Request.blank( 4266 "/v1/AUTH_test/con/obj", 4267 environ={"REQUEST_METHOD": "COALESCE", 4268 "wsgi.input": BytesIO(request_body)}) 4269 status, headers, body = self.call_pfs(req) 4270 self.assertEqual(status, '400 Bad Request') 4271 4272 def test_incorrect_json_wrong_elements_type(self): 4273 request_body = json.dumps({"elements": {1: 2}}).encode('ascii') 4274 req = swob.Request.blank( 4275 "/v1/AUTH_test/con/obj", 4276 environ={"REQUEST_METHOD": "COALESCE", 4277 "wsgi.input": BytesIO(request_body)}) 4278 status, headers, body = self.call_pfs(req) 4279 self.assertEqual(status, '400 Bad Request') 4280 4281 def test_incorrect_json_wrong_element_type(self): 4282 request_body = json.dumps({"elements": [1, "two", {}]}).encode('ascii') 4283 req = swob.Request.blank( 4284 "/v1/AUTH_test/con/obj", 4285 environ={"REQUEST_METHOD": "COALESCE", 4286 "wsgi.input": BytesIO(request_body)}) 4287 status, headers, body = self.call_pfs(req) 4288 self.assertEqual(status, '400 Bad Request') 4289 4290 def test_incorrect_json_subtle(self): 4291 request_body = b'["elements"]' 4292 req = swob.Request.blank( 4293 "/v1/AUTH_test/con/obj", 4294 environ={"REQUEST_METHOD": "COALESCE", 4295 "wsgi.input": BytesIO(request_body)}) 4296 status, headers, body = self.call_pfs(req) 4297 self.assertEqual(status, '400 Bad Request') 4298 4299 def test_too_big(self): 4300 request_body = b'{' * (self.pfs.max_coalesce_request_size + 1) 4301 req = swob.Request.blank( 4302 "/v1/AUTH_test/con/obj", 4303 environ={"REQUEST_METHOD": "COALESCE", 4304 "wsgi.input": BytesIO(request_body)}) 4305 status, headers, body = self.call_pfs(req) 4306 self.assertEqual(status, '413 Request Entity Too Large') 4307 4308 def test_too_many_elements(self): 4309 request_body = json.dumps({ 4310 "elements": ["/c/o"] * (self.pfs.max_coalesce + 1), 4311 }).encode('ascii') 4312 req = swob.Request.blank( 4313 "/v1/AUTH_test/con/obj", 4314 environ={"REQUEST_METHOD": "COALESCE", 4315 "wsgi.input": BytesIO(request_body)}) 4316 status, headers, body = self.call_pfs(req) 4317 self.assertEqual(status, '413 Request Entity Too Large') 4318 4319 def test_not_found(self): 4320 # Note: this test covers all sorts of thing-not-found errors, as the 4321 # Server.RpcCoalesce remote procedure does not distinguish between 4322 # them. In particular, this covers the case when an element is not 4323 # found as well as the case when the destination container is not 4324 # found. 4325 def mock_RpcCoalesce(coalese_req): 4326 return { 4327 "error": "errno: 2", 4328 "result": None, 4329 } 4330 4331 self.fake_rpc.register_handler( 4332 "Server.RpcCoalesce", mock_RpcCoalesce) 4333 4334 request_body = json.dumps({ 4335 "elements": [ 4336 "some/stuff", 4337 ]}).encode('ascii') 4338 req = swob.Request.blank( 4339 "/v1/AUTH_test/con/obj", 4340 environ={"REQUEST_METHOD": "COALESCE", 4341 "wsgi.input": BytesIO(request_body)}) 4342 status, headers, body = self.call_pfs(req) 4343 self.assertEqual(status, '404 Not Found') 4344 4345 def test_coalesce_directory(self): 4346 # This happens when one of the named elements is a directory. 4347 def mock_RpcCoalesce(coalese_req): 4348 return { 4349 "error": "errno: 21", 4350 "result": None, 4351 } 4352 4353 self.fake_rpc.register_handler( 4354 "Server.RpcCoalesce", mock_RpcCoalesce) 4355 4356 request_body = json.dumps({ 4357 "elements": [ 4358 "some/stuff", 4359 ]}).encode('ascii') 4360 req = swob.Request.blank( 4361 "/v1/AUTH_test/con/obj", 4362 environ={"REQUEST_METHOD": "COALESCE", 4363 "wsgi.input": BytesIO(request_body)}) 4364 status, headers, body = self.call_pfs(req) 4365 self.assertEqual(status, '409 Conflict') 4366 4367 def test_coalesce_file_instead_of_dir(self): 4368 # This happens when one of the named elements is not a regular file. 4369 # Yes, you get IsDirError for not-a-file, even if it's a symlink. 4370 def mock_RpcCoalesce(coalese_req): 4371 return { 4372 "error": "errno: 20", 4373 "result": None, 4374 } 4375 4376 self.fake_rpc.register_handler( 4377 "Server.RpcCoalesce", mock_RpcCoalesce) 4378 4379 request_body = json.dumps({ 4380 "elements": [ 4381 "some/stuff", 4382 ]}).encode('ascii') 4383 req = swob.Request.blank( 4384 "/v1/AUTH_test/con/obj", 4385 environ={"REQUEST_METHOD": "COALESCE", 4386 "wsgi.input": BytesIO(request_body)}) 4387 status, headers, body = self.call_pfs(req) 4388 self.assertEqual(status, '409 Conflict') 4389 4390 def test_element_has_multiple_links(self): 4391 # If a file has multiple links to it (hard links, not symlinks), 4392 # then we can't coalesce it. 4393 def mock_RpcCoalesce(coalese_req): 4394 return { 4395 "error": "errno: 31", 4396 "result": None, 4397 } 4398 4399 self.fake_rpc.register_handler( 4400 "Server.RpcCoalesce", mock_RpcCoalesce) 4401 4402 request_body = json.dumps({ 4403 "elements": [ 4404 "some/stuff", 4405 ]}).encode('ascii') 4406 req = swob.Request.blank( 4407 "/v1/AUTH_test/con/obj", 4408 environ={"REQUEST_METHOD": "COALESCE", 4409 "wsgi.input": BytesIO(request_body)}) 4410 status, headers, body = self.call_pfs(req) 4411 self.assertEqual(status, '409 Conflict') 4412 4413 def test_other_error(self): 4414 # If a file has multiple links to it (hard links, not symlinks), 4415 # then we can't coalesce it. 4416 def mock_RpcCoalesce(coalese_req): 4417 return { 4418 "error": "errno: 1159268", 4419 "result": None, 4420 } 4421 4422 self.fake_rpc.register_handler( 4423 "Server.RpcCoalesce", mock_RpcCoalesce) 4424 4425 request_body = json.dumps({ 4426 "elements": [ 4427 "thing/one", 4428 "thing/two", 4429 ]}).encode('ascii') 4430 req = swob.Request.blank( 4431 "/v1/AUTH_test/con/obj", 4432 environ={"REQUEST_METHOD": "COALESCE", 4433 "wsgi.input": BytesIO(request_body)}) 4434 status, headers, body = self.call_pfs(req) 4435 self.assertEqual(status, '500 Internal Error') 4436 4437 4438 class TestAuth(BaseMiddlewareTest): 4439 def setUp(self): 4440 super(TestAuth, self).setUp() 4441 4442 def mock_RpcHead(get_container_req): 4443 return { 4444 "error": None, 4445 "result": { 4446 "Metadata": base64.b64encode(json.dumps({ 4447 "X-Container-Read": "the-x-con-read", 4448 "X-Container-Write": "the-x-con-write", 4449 }).encode('ascii')).decode('ascii'), 4450 "ModificationTime": 1479240451156825194, 4451 "FileSize": 0, 4452 "IsDir": True, 4453 "InodeNumber": 1255, 4454 "NumWrites": 897, 4455 }} 4456 4457 self.fake_rpc.register_handler( 4458 "Server.RpcHead", mock_RpcHead) 4459 4460 def test_auth_callback_args(self): 4461 want_x_container_read = (('GET', '/v1/AUTH_test/con/obj'), 4462 ('HEAD', '/v1/AUTH_test/con/obj'), 4463 ('GET', '/v1/AUTH_test/con'), 4464 ('HEAD', '/v1/AUTH_test/con')) 4465 4466 want_x_container_write = (('PUT', '/v1/AUTH_test/con/obj'), 4467 ('POST', '/v1/AUTH_test/con/obj'), 4468 ('DELETE', '/v1/AUTH_test/con/obj')) 4469 got_acls = [] 4470 4471 def capture_acl_and_deny(request): 4472 got_acls.append(request.acl) 4473 return swob.HTTPForbidden(request=request) 4474 4475 for method, path in want_x_container_read: 4476 req = swob.Request.blank( 4477 path, environ={'REQUEST_METHOD': method, 4478 'swift.authorize': capture_acl_and_deny}) 4479 status, _, _ = self.call_pfs(req) 4480 self.assertEqual(status, '403 Forbidden') 4481 self.assertEqual( 4482 got_acls, ["the-x-con-read"] * len(want_x_container_read)) 4483 4484 del got_acls[:] 4485 for method, path in want_x_container_write: 4486 req = swob.Request.blank( 4487 path, environ={'REQUEST_METHOD': method, 4488 'swift.authorize': capture_acl_and_deny}) 4489 status, _, _ = self.call_pfs(req) 4490 self.assertEqual(status, '403 Forbidden') 4491 self.assertEqual( 4492 got_acls, ["the-x-con-write"] * len(want_x_container_write)) 4493 4494 def test_auth_override(self): 4495 def auth_nope(request): 4496 return swob.HTTPForbidden(request=request) 4497 4498 req = swob.Request.blank( 4499 "/v1/AUTH_test/con", 4500 environ={'REQUEST_METHOD': 'HEAD', 4501 'swift.authorize': auth_nope, 4502 'swift.authorize_override': True}) 4503 status, _, _ = self.call_pfs(req) 4504 self.assertEqual(status, '403 Forbidden') 4505 4506 def test_auth_bypass(self): 4507 def auth_nope(request): 4508 return swob.HTTPForbidden(request=request) 4509 4510 # pfs is re-entrant when getting ACLs from what may be other accounts. 4511 req = swob.Request.blank( 4512 "/v1/AUTH_test/con", 4513 environ={'REQUEST_METHOD': 'HEAD', 4514 'swift.authorize': auth_nope, 4515 'swift.authorize_override': True, 4516 'swift.source': 'PFS'}) 4517 status, _, _ = self.call_pfs(req) 4518 self.assertEqual(status, '204 No Content') 4519 4520 def test_auth_allowed(self): 4521 def auth_its_fine(request): 4522 return None 4523 4524 req = swob.Request.blank( 4525 "/v1/AUTH_test/con", 4526 environ={'REQUEST_METHOD': 'HEAD', 4527 'swift.authorize': auth_its_fine}) 4528 status, _, _ = self.call_pfs(req) 4529 self.assertEqual(status, '204 No Content') 4530 4531 4532 class TestEtagHandling(unittest.TestCase): 4533 '''Test that mung_etags()/unmung_etags() are inverse functions (more 4534 or less), that the unmung_etags() correctly unmungs the current 4535 disk layout for the header, and that best_possible_etag() returns 4536 the correct etag value. 4537 ''' 4538 4539 # Both the header names and the way the values are calculated and 4540 # the way they are formatted are part of the "disk layout". If we 4541 # change them, we have to consider handling old data. 4542 # 4543 # Names and values duplicated here so this test will break if they 4544 # are changed. 4545 HEADER = "X-Object-Sysmeta-ProxyFS-Initial-MD5" 4546 4547 ETAG_HEADERS = { 4548 "ORIGINAL_MD5": { 4549 "name": "X-Object-Sysmeta-ProxyFS-Initial-MD5", 4550 "value": "5484c2634aa61c69fc02ef5400a61c94", 4551 "num_writes": 7, 4552 "munged_value": "7:5484c2634aa61c69fc02ef5400a61c94" 4553 }, 4554 "S3API_ETAG": { 4555 "name": "X-Object-Sysmeta-S3Api-Etag", 4556 "value": "cb0dc66d591395cdf93555dafd4145ad", 4557 "num_writes": 3, 4558 "munged_value": "3:cb0dc66d591395cdf93555dafd4145ad" 4559 }, 4560 "LISTING_ETAG_OVERRIDE": { 4561 "name": "X-Object-Sysmeta-Container-Update-Override-Etag", 4562 "value": "dbca19e0c46aa9b10e8de2f5856abc86", 4563 "num_writes": 9, 4564 "munged_value": "9:dbca19e0c46aa9b10e8de2f5856abc86" 4565 }, 4566 } 4567 4568 EMPTY_OBJECT_ETAG = "d41d8cd98f00b204e9800998ecf8427e" 4569 4570 def test_mung_etags(self): 4571 '''Verify that mung_etags() mungs each of the etag header values to 4572 the correct on-disk version with num_writes and unmung_etags() 4573 recovers the original value. 4574 ''' 4575 for header in self.ETAG_HEADERS.keys(): 4576 4577 name = self.ETAG_HEADERS[header]["name"] 4578 value = self.ETAG_HEADERS[header]["value"] 4579 num_writes = self.ETAG_HEADERS[header]["num_writes"] 4580 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4581 4582 obj_metadata = {name: value} 4583 mware.mung_etags(obj_metadata, None, num_writes) 4584 4585 # The handling of ORIGINAL_MD5_HEADER is a bit strange. 4586 # If an etag value is passed to mung_etags() as the 2nd 4587 # argument then the header is created and assigned the 4588 # munged version of that etag value. But if its None than 4589 # an ORIGINAL_MD5_HEADER, if present, is passed through 4590 # unmolested (but will be dropped by the unmung code if 4591 # the format is wrong or num_writes does not match, which 4592 # is likely). 4593 if header == "ORIGINAL_MD5": 4594 self.assertEqual(obj_metadata[name], value) 4595 else: 4596 self.assertEqual(obj_metadata[name], munged_value) 4597 4598 # same test but with an etag value passed to mung_etags() 4599 # instead of None 4600 md5_etag_value = self.ETAG_HEADERS["ORIGINAL_MD5"]["value"] 4601 4602 obj_metadata = {name: value} 4603 mware.mung_etags(obj_metadata, md5_etag_value, num_writes) 4604 self.assertEqual(obj_metadata[name], munged_value) 4605 4606 # and verify that it gets unmunged to the original value 4607 mware.unmung_etags(obj_metadata, num_writes) 4608 self.assertEqual(obj_metadata[name], value) 4609 4610 def test_unmung_etags(self): 4611 '''Verify that unmung_etags() recovers the correct value for 4612 each ETag header and discards header values where num_writes 4613 is incorrect or the value is corrupt. 4614 ''' 4615 # build metadata with all of the etag headers 4616 orig_obj_metadata = {} 4617 for header in self.ETAG_HEADERS.keys(): 4618 4619 name = self.ETAG_HEADERS[header]["name"] 4620 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4621 orig_obj_metadata[name] = munged_value 4622 4623 # unmung_etags() should strip out headers with stale 4624 # num_writes so only the one for header remains 4625 for header in self.ETAG_HEADERS.keys(): 4626 4627 name = self.ETAG_HEADERS[header]["name"] 4628 value = self.ETAG_HEADERS[header]["value"] 4629 num_writes = self.ETAG_HEADERS[header]["num_writes"] 4630 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4631 4632 obj_metadata = orig_obj_metadata.copy() 4633 mware.unmung_etags(obj_metadata, num_writes) 4634 4635 # header should be the only one left 4636 for hdr2 in self.ETAG_HEADERS.keys(): 4637 4638 if hdr2 == header: 4639 self.assertEqual(obj_metadata[name], value) 4640 else: 4641 self.assertTrue( 4642 self.ETAG_HEADERS[hdr2]["name"] not in obj_metadata) 4643 4644 # verify mung_etags() takes it back to the munged value 4645 if header != "ORIGINAL_MD5": 4646 mware.mung_etags(obj_metadata, None, num_writes) 4647 self.assertEqual(obj_metadata[name], munged_value) 4648 4649 # Try to unmung on-disk headers with corrupt values. Instead 4650 # of a panic it should simply elide the headers (and possibly 4651 # log the corrupt value, which it doesn't currently do). 4652 for bad_value in ("counterfact-preformative", 4653 "magpie:interfollicular"): 4654 4655 obj_metadata = {} 4656 for header in self.ETAG_HEADERS.keys(): 4657 4658 name = self.ETAG_HEADERS[header]["name"] 4659 obj_metadata[name] = bad_value 4660 4661 mware.unmung_etags(obj_metadata, 0) 4662 self.assertEqual(obj_metadata, {}) 4663 4664 def test_best_possible_etag(self): 4665 '''Test that best_possible_etag() returns the best ETag. 4666 ''' 4667 4668 # Start with metadata that consists of all possible ETag 4669 # headers and verify that best_possible_headers() chooses the 4670 # right ETag based on the circumstance. 4671 obj_metadata = {} 4672 for header in self.ETAG_HEADERS.keys(): 4673 4674 name = self.ETAG_HEADERS[header]["name"] 4675 value = self.ETAG_HEADERS[header]["value"] 4676 obj_metadata[name] = value 4677 4678 # directories always return the same etag 4679 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, 4680 is_dir=True) 4681 self.assertEqual(etag, self.EMPTY_OBJECT_ETAG) 4682 4683 # a container listing should return the Container Listing Override ETag 4684 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, 4685 container_listing=True) 4686 self.assertEqual(etag, 4687 self.ETAG_HEADERS["LISTING_ETAG_OVERRIDE"]["value"]) 4688 4689 # if its not a directory and its not a container list, then 4690 # the function should return the original MD5 ETag (the 4691 # S3API_ETAG is returned as a header, but not as an ETag). 4692 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) 4693 self.assertEqual(etag, 4694 self.ETAG_HEADERS["ORIGINAL_MD5"]["value"]) 4695 4696 # if none of the headers provide the correct ETag then 4697 # best_possible_etag() will construct a unique one (which also 4698 # should not be changed without handling "old data", i.e. 4699 # its part of the "disk layout"). 4700 del obj_metadata[self.ETAG_HEADERS["ORIGINAL_MD5"]["name"]] 4701 4702 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) 4703 self.assertEqual(etag, 4704 '"pfsv2/AUTH_test/0000002A/00000007-32"') 4705 4706 4707 class TestProxyfsMethod(BaseMiddlewareTest): 4708 def test_non_swift_owner(self): 4709 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS') 4710 status, _, _ = self.call_pfs(req) 4711 self.assertEqual(status, '405 Method Not Allowed') 4712 4713 def test_bad_content_type(self): 4714 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', 4715 environ={'swift_owner': True}, body=b'') 4716 status, _, body = self.call_pfs(req) 4717 self.assertEqual(status, '415 Unsupported Media Type') 4718 self.assertEqual( 4719 body, b'RPC body must have Content-Type application/json') 4720 4721 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', 4722 headers={'Content-Type': 'text/plain'}, 4723 environ={'swift_owner': True}, body=b'') 4724 status, _, body = self.call_pfs(req) 4725 self.assertEqual(status, '415 Unsupported Media Type') 4726 self.assertEqual(body, (b'RPC body must have Content-Type ' 4727 b'application/json, not text/plain')) 4728 4729 def test_exactly_one_payload(self): 4730 def expect_one_payload(body): 4731 req = swob.Request.blank( 4732 "/v1/AUTH_test", method='PROXYFS', 4733 headers={'Content-Type': 'application/json'}, 4734 environ={'swift_owner': True}, body=body) 4735 status, _, body = self.call_pfs(req) 4736 self.assertEqual(status, '400 Bad Request') 4737 self.assertEqual(body, b'Expected exactly one JSON payload') 4738 4739 expect_one_payload(b'') 4740 expect_one_payload(b'\n') 4741 one_payload = json.dumps({ 4742 'jsonrpc': '2.0', 4743 'method': 'Server.RpcPing', 4744 'params': [{}], 4745 }) 4746 expect_one_payload(one_payload + '\n' + one_payload) 4747 4748 def test_bad_body(self): 4749 def expect_parse_error(req_body): 4750 req = swob.Request.blank( 4751 "/v1/AUTH_test", method='PROXYFS', 4752 headers={'Content-Type': 'application/json'}, 4753 environ={'swift_owner': True}, body=req_body) 4754 status, _, body = self.call_pfs(req) 4755 self.assertEqual(status, '400 Bad Request') 4756 exp_start = (b'Could not parse/validate JSON payload #0 ' 4757 b'%s:' % req_body) 4758 self.assertEqual(body[:len(exp_start)], exp_start) 4759 4760 expect_parse_error(b'[') 4761 expect_parse_error(b'{') 4762 expect_parse_error(b'[]') 4763 expect_parse_error(b'{}') 4764 4765 expect_parse_error(json.dumps({ 4766 'jsonrpc': '1.0', 4767 'method': 'Server.PingReq', 4768 'params': [{}], 4769 }).encode('ascii')) 4770 4771 expect_parse_error(json.dumps({ 4772 'jsonrpc': '2.0', 4773 'method': 'asdf', 4774 'params': [{}], 4775 }).encode('ascii')) 4776 4777 expect_parse_error(json.dumps({ 4778 'jsonrpc': '2.0', 4779 'method': 'Server.PingReq', 4780 'params': [{}, {}], 4781 }).encode('ascii')) 4782 expect_parse_error(json.dumps({ 4783 'jsonrpc': '2.0', 4784 'method': 'Server.PingReq', 4785 'params': [], 4786 }).encode('ascii')) 4787 expect_parse_error(json.dumps({ 4788 'jsonrpc': '2.0', 4789 'method': 'Server.PingReq', 4790 'params': [[]], 4791 }).encode('ascii')) 4792 expect_parse_error(json.dumps({ 4793 'jsonrpc': '2.0', 4794 'method': 'Server.PingReq', 4795 'params': {}, 4796 }).encode('ascii')) 4797 4798 def test_success(self): 4799 def mock_RpcGetAccount(params): 4800 self.assertEqual(params, { 4801 "AccountName": "AUTH_test", 4802 }) 4803 return { 4804 "error": None, 4805 "result": { 4806 "ModificationTime": 1498766381451119000, 4807 "AccountEntries": [{ 4808 "Basename": "chickens", 4809 "ModificationTime": 1510958440808682000, 4810 }, { 4811 "Basename": "cows", 4812 "ModificationTime": 1510958450657045000, 4813 }, { 4814 "Basename": "goats", 4815 "ModificationTime": 1510958452544251000, 4816 }, { 4817 "Basename": "pigs", 4818 "ModificationTime": 1510958459200130000, 4819 }], 4820 }} 4821 4822 self.fake_rpc.register_handler( 4823 "Server.RpcGetAccount", mock_RpcGetAccount) 4824 4825 req = swob.Request.blank( 4826 "/v1/AUTH_test", method='PROXYFS', 4827 headers={'Content-Type': 'application/json'}, 4828 environ={'swift_owner': True}, body=json.dumps({ 4829 'jsonrpc': '2.0', 4830 'method': 'Server.RpcGetAccount', 4831 'params': [{}], 4832 })) 4833 status, headers, body = self.call_pfs(req) 4834 self.assertEqual(status, '200 OK', body) 4835 self.assertEqual(headers.get('content-type'), 'application/json') 4836 resp = json.loads(body) 4837 self.assertIsNotNone(resp.pop('id', None)) 4838 self.assertEqual(resp, { 4839 "error": None, 4840 "result": { 4841 "ModificationTime": 1498766381451119000, 4842 "AccountEntries": [{ 4843 "Basename": "chickens", 4844 "ModificationTime": 1510958440808682000, 4845 }, { 4846 "Basename": "cows", 4847 "ModificationTime": 1510958450657045000, 4848 }, { 4849 "Basename": "goats", 4850 "ModificationTime": 1510958452544251000, 4851 }, { 4852 "Basename": "pigs", 4853 "ModificationTime": 1510958459200130000, 4854 }] 4855 } 4856 }) 4857 4858 def test_error(self): 4859 def mock_RpcGetAccount(params): 4860 self.assertEqual(params, { 4861 "AccountName": "AUTH_test", 4862 "some": "args", 4863 }) 4864 return {"error": "errno 2"} 4865 4866 self.fake_rpc.register_handler( 4867 "Server.RpcGetAccount", mock_RpcGetAccount) 4868 4869 req = swob.Request.blank( 4870 "/v1/AUTH_test", method='PROXYFS', 4871 headers={'Content-Type': 'application/json'}, 4872 environ={'swift_owner': True}, body=json.dumps({ 4873 'jsonrpc': '2.0', 4874 'method': 'Server.RpcGetAccount', 4875 'params': [{'some': 'args'}], 4876 })) 4877 status, headers, body = self.call_pfs(req) 4878 self.assertEqual(status, '200 OK', body) 4879 self.assertEqual(headers.get('content-type'), 'application/json') 4880 resp = json.loads(body) 4881 self.assertIsNotNone(resp.pop('id', None)) 4882 self.assertEqual(resp, {"error": "errno 2"})