github.com/swiftstack/proxyfs@v0.0.0-20201223034610-5434d919416e/pfs_middleware/tests/test_pfs_middleware.py (about) 1 # Copyright (c) 2016-2017 SwiftStack, Inc. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 # implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import base64 17 import collections 18 import hashlib 19 import json 20 import mock 21 import unittest 22 from io import BytesIO 23 from swift.common import swob 24 from xml.etree import ElementTree 25 26 import pfs_middleware.middleware as mware 27 import pfs_middleware.bimodal_checker as bimodal_checker 28 from . import helpers 29 30 31 class FakeLogger(object): 32 def critical(fmt, *args): 33 pass 34 35 def exception(fmt, *args): 36 pass 37 38 def error(fmt, *args): 39 pass 40 41 def warn(fmt, *args): 42 pass 43 44 def info(fmt, *args): 45 pass 46 47 def debug(fmt, *args, **kwargs): 48 pass 49 50 51 class TestHelpers(unittest.TestCase): 52 def test_deserialize_metadata(self): 53 self.assertEqual(mware.deserialize_metadata(None), {}) 54 self.assertEqual(mware.deserialize_metadata(''), {}) 55 self.assertEqual(mware.deserialize_metadata('{}'), {}) 56 self.assertEqual(mware.deserialize_metadata('{"foo": "bar"}'), 57 {"foo": "bar"}) 58 self.assertEqual(mware.deserialize_metadata( 59 '{"unicode-\\u1234":"meta \\ud83c\\udf34"}' 60 ), { 61 # NB: WSGI strings 62 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', 63 }) 64 65 def test_serialize_metadata(self): 66 self.assertEqual(mware.serialize_metadata({}), '{}') 67 self.assertEqual( 68 mware.serialize_metadata({ 69 # NB: WSGI strings 70 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', 71 }), 72 # But it comes out as good Unicode 73 '{"unicode-\\u1234": "meta \\ud83c\\udf34"}') 74 75 76 class BaseMiddlewareTest(unittest.TestCase): 77 # no test cases in here, just common setup and utility functions 78 def setUp(self): 79 super(BaseMiddlewareTest, self).setUp() 80 self.app = helpers.FakeProxy() 81 self.pfs = mware.PfsMiddleware(self.app, { 82 'bypass_mode': 'read-only', 83 }, FakeLogger()) 84 self.bimodal_checker = bimodal_checker.BimodalChecker(self.pfs, { 85 'bimodal_recheck_interval': 'inf', # avoid timing dependencies 86 }, FakeLogger()) 87 self.bimodal_accounts = {"AUTH_test"} 88 89 self.app.register('HEAD', '/v1/AUTH_test', 204, 90 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 91 '') 92 93 self.swift_info = { 94 # some stuff omitted 95 "slo": { 96 "max_manifest_segments": 1000, 97 "max_manifest_size": 2097152, 98 "min_segment_size": 1}, 99 "swift": { 100 "account_autocreate": True, 101 "account_listing_limit": 9876, # note: not default 102 "allow_account_management": True, 103 "container_listing_limit": 6543, # note: not default 104 "extra_header_count": 0, 105 "max_account_name_length": 128, 106 "max_container_name_length": 256, 107 "max_file_size": 5368709122, 108 "max_header_size": 8192, 109 "max_meta_count": 90, 110 "max_meta_name_length": 128, 111 "max_meta_overall_size": 4096, 112 "max_meta_value_length": 256, 113 "max_object_name_length": 1024, 114 "policies": [{ 115 "aliases": "default", 116 "default": True, 117 "name": "default", 118 }, { 119 "aliases": "not-default", 120 "name": "not-default", 121 }], 122 "strict_cors_mode": True, 123 "version": "2.9.1.dev47" 124 }, 125 "tempauth": {"account_acls": True}} 126 127 self.app.register( 128 'GET', '/info', 129 200, {'Content-Type': 'application/json'}, 130 json.dumps(self.swift_info)) 131 132 for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): 133 self.app.register( 134 method, '/v1/AUTH_test//o', 135 412, {'Content-Type': 'text/html'}, 'Bad URL') 136 137 self.fake_rpc = helpers.FakeJsonRpc() 138 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 139 lambda *_: self.fake_rpc) 140 patcher.start() 141 self.addCleanup(patcher.stop) 142 143 # For the sake of the listing format tests, assume old Swift 144 patcher = mock.patch( 145 'pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', False) 146 patcher.start() 147 self.addCleanup(patcher.stop) 148 149 def fake_RpcIsAccountBimodal(request): 150 return { 151 "error": None, 152 "result": { 153 "IsBimodal": 154 request["AccountName"] in self.bimodal_accounts, 155 "ActivePeerPrivateIPAddr": "127.0.0.1", 156 }} 157 158 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 159 fake_RpcIsAccountBimodal) 160 161 def call_app(self, req, app=None, expect_exception=False): 162 # Normally this happens in eventlet.wsgi.HttpProtocol.get_environ(). 163 req.environ.setdefault('CONTENT_TYPE', None) 164 165 if app is None: 166 app = self.app 167 168 status = [None] 169 headers = [None] 170 171 def start_response(s, h, ei=None): 172 status[0] = s 173 headers[0] = swob.HeaderKeyDict(h) 174 175 body_iter = app(req.environ, start_response) 176 body = b'' 177 caught_exc = None 178 try: 179 try: 180 for chunk in body_iter: 181 body += chunk 182 finally: 183 # WSGI says we have to do this. Plus, if we don't, then 184 # our leak detector gets false positives. 185 if callable(getattr(body_iter, 'close', None)): 186 body_iter.close() 187 except Exception as exc: 188 if expect_exception: 189 caught_exc = exc 190 else: 191 raise 192 193 if expect_exception: 194 return status[0], headers[0], body, caught_exc 195 else: 196 return status[0], headers[0], body 197 198 def call_pfs(self, req, **kwargs): 199 return self.call_app(req, app=self.bimodal_checker, **kwargs) 200 201 202 class TestAccountGet(BaseMiddlewareTest): 203 def setUp(self): 204 super(TestAccountGet, self).setUp() 205 206 self.app.register( 207 'HEAD', '/v1/AUTH_test', 208 204, 209 {'X-Account-Meta-Flavor': 'cherry', 210 'X-Account-Sysmeta-Shipping-Class': 'ultraslow', 211 'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 212 '') 213 214 def mock_RpcGetAccount(_): 215 return { 216 "error": None, 217 "result": { 218 "ModificationTime": 1498766381451119000, 219 "AccountEntries": [{ 220 "Basename": "chickens", 221 "ModificationTime": 1510958440808682000, 222 }, { 223 "Basename": "cows", 224 "ModificationTime": 1510958450657045000, 225 }, { 226 "Basename": "goats", 227 "ModificationTime": 1510958452544251000, 228 }, { 229 "Basename": "pigs", 230 "ModificationTime": 1510958459200130000, 231 }], 232 }} 233 234 self.fake_rpc.register_handler( 235 "Server.RpcGetAccount", mock_RpcGetAccount) 236 237 def test_headers(self): 238 req = swob.Request.blank("/v1/AUTH_test") 239 status, headers, _ = self.call_pfs(req) 240 self.assertEqual(status, '200 OK') 241 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 242 self.assertEqual(headers.get("X-Timestamp"), "1498766381.45112") 243 244 # These we just lie about for now 245 self.assertEqual( 246 headers.get("X-Account-Object-Count"), "0") 247 self.assertEqual( 248 headers.get("X-Account-Storage-Policy-Default-Object-Count"), "0") 249 self.assertEqual( 250 headers.get("X-Account-Bytes-Used"), "0") 251 self.assertEqual( 252 headers.get("X-Account-Storage-Policy-Default-Bytes-Used"), "0") 253 254 # We pretend all our containers are in the default storage policy. 255 self.assertEqual( 256 headers.get("X-Account-Container-Count"), "4") 257 self.assertEqual( 258 headers.get("X-Account-Storage-Policy-Default-Container-Count"), 259 "4") 260 261 def test_escape_hatch(self): 262 self.app.register( 263 'GET', '/v1/AUTH_test', 200, {}, 264 '000000000000DACA\n000000000000DACC\n') 265 266 # Using path 267 req = swob.Request.blank("/proxyfs/AUTH_test", environ={ 268 'swift_owner': True}) 269 status, _, body = self.call_pfs(req) 270 self.assertEqual(status, '200 OK') 271 self.assertEqual(body, b'000000000000DACA\n000000000000DACC\n') 272 273 # Non-bypass 274 req = swob.Request.blank("/v1/AUTH_test", environ={ 275 'swift_owner': True}) 276 status, _, body = self.call_pfs(req) 277 self.assertEqual(status, '200 OK') 278 self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') 279 280 req = swob.Request.blank("/proxyfs/AUTH_test", method='PUT', 281 environ={'swift_owner': True}) 282 status, _, _ = self.call_pfs(req) 283 self.assertEqual(status, '405 Method Not Allowed') 284 285 # Non-owner 286 req = swob.Request.blank("/proxyfs/AUTH_test") 287 status, _, body = self.call_pfs(req) 288 self.assertEqual(status, '200 OK') 289 self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') 290 291 def test_text(self): 292 req = swob.Request.blank("/v1/AUTH_test") 293 status, headers, body = self.call_pfs(req) 294 self.assertEqual(status, '200 OK') 295 self.assertEqual(body, b"chickens\ncows\ngoats\npigs\n") 296 297 def test_json(self): 298 self.maxDiff = None 299 300 req = swob.Request.blank("/v1/AUTH_test?format=json") 301 status, headers, body = self.call_pfs(req) 302 self.assertEqual(status, '200 OK') 303 self.assertEqual(json.loads(body), [{ 304 "bytes": 0, 305 "count": 0, 306 "last_modified": "2017-11-17T22:40:40.808682", 307 "name": "chickens", 308 }, { 309 "bytes": 0, 310 "count": 0, 311 "last_modified": "2017-11-17T22:40:50.657045", 312 "name": "cows", 313 }, { 314 "bytes": 0, 315 "count": 0, 316 "last_modified": "2017-11-17T22:40:52.544251", 317 "name": "goats", 318 }, { 319 "bytes": 0, 320 "count": 0, 321 "last_modified": "2017-11-17T22:40:59.200130", 322 "name": "pigs" 323 }]) 324 325 def test_xml(self): 326 req = swob.Request.blank("/v1/AUTH_test?format=xml") 327 status, headers, body = self.call_pfs(req) 328 self.assertEqual(status, '200 OK') 329 self.assertEqual(headers["Content-Type"], 330 "application/xml; charset=utf-8") 331 self.assertTrue(body.startswith( 332 b"""<?xml version='1.0' encoding='utf-8'?>""")) 333 334 root_node = ElementTree.fromstring(body) 335 self.assertEqual(root_node.tag, 'account') 336 self.assertEqual(root_node.attrib["name"], 'AUTH_test') 337 338 containers = list(root_node) 339 self.assertEqual(containers[0].tag, 'container') 340 341 # The XML account listing doesn't use XML attributes for data, but 342 # rather a sequence of tags like <name>X</name> <bytes>Y</bytes> ... 343 con_attr_tags = list(containers[0]) 344 self.assertEqual(len(con_attr_tags), 4) 345 346 name_node = con_attr_tags[0] 347 self.assertEqual(name_node.tag, 'name') 348 self.assertEqual(name_node.text, 'chickens') 349 self.assertEqual(name_node.attrib, {}) # nothing extra in there 350 351 count_node = con_attr_tags[1] 352 self.assertEqual(count_node.tag, 'count') 353 self.assertEqual(count_node.text, '0') 354 self.assertEqual(count_node.attrib, {}) 355 356 bytes_node = con_attr_tags[2] 357 self.assertEqual(bytes_node.tag, 'bytes') 358 self.assertEqual(bytes_node.text, '0') 359 self.assertEqual(bytes_node.attrib, {}) 360 361 lm_node = con_attr_tags[3] 362 self.assertEqual(lm_node.tag, 'last_modified') 363 self.assertEqual(lm_node.text, '2017-11-17T22:40:40.808682') 364 self.assertEqual(lm_node.attrib, {}) 365 366 def test_metadata(self): 367 req = swob.Request.blank("/v1/AUTH_test") 368 status, headers, body = self.call_pfs(req) 369 self.assertEqual(headers.get("X-Account-Meta-Flavor"), "cherry") 370 self.assertEqual(headers.get("X-Account-Sysmeta-Shipping-Class"), 371 "ultraslow") 372 373 def test_account_acl(self): 374 def do_test(acl, expected_header, swift_owner): 375 head_resp_hdrs = {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'} 376 if acl is not None: 377 head_resp_hdrs['X-Account-Sysmeta-Core-Access-Control'] = acl 378 self.app.register( 379 'HEAD', '/v1/AUTH_test', 204, head_resp_hdrs, '') 380 req = swob.Request.blank("/v1/AUTH_test", 381 environ={"swift_owner": swift_owner}) 382 status, headers, body = self.call_pfs(req) 383 self.assertEqual(status, "200 OK") 384 actual_header = headers.get("X-Account-Access-Control") 385 self.assertEqual(expected_header, actual_header) 386 387 # swift_owner 388 do_test(None, None, True) 389 do_test('', None, True) 390 do_test('not a dict', None, True) 391 do_test('{"admin": ["someone"]}', '{"admin":["someone"]}', True) 392 # not swift_owner 393 do_test(None, None, False) 394 do_test('', None, False) 395 do_test('not a dict', None, False) 396 do_test('{"admin": ["someone"]}', None, False) 397 398 def test_marker(self): 399 req = swob.Request.blank("/v1/AUTH_test?marker=mk") 400 status, headers, body = self.call_pfs(req) 401 self.assertEqual(status, '200 OK') 402 self.assertEqual(2, len(self.fake_rpc.calls)) 403 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 404 # relevant to what we're testing here 405 self.assertEqual(self.fake_rpc.calls[1][1][0]['Marker'], 'mk') 406 407 def test_end_marker(self): 408 req = swob.Request.blank("/v1/AUTH_test?end_marker=mk") 409 status, headers, body = self.call_pfs(req) 410 self.assertEqual(status, '200 OK') 411 self.assertEqual(2, len(self.fake_rpc.calls)) 412 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 413 # relevant to what we're testing here 414 self.assertEqual(self.fake_rpc.calls[1][1][0]['EndMarker'], 'mk') 415 416 def test_limit(self): 417 req = swob.Request.blank("/v1/AUTH_test?limit=101") 418 status, headers, body = self.call_pfs(req) 419 self.assertEqual(status, '200 OK') 420 self.assertEqual(2, len(self.fake_rpc.calls)) 421 self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 101) 422 423 def test_default_limit(self): 424 req = swob.Request.blank("/v1/AUTH_test") 425 status, headers, body = self.call_pfs(req) 426 self.assertEqual(status, '200 OK') 427 self.assertEqual(2, len(self.fake_rpc.calls)) 428 # value from GET /info 429 self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 9876) 430 431 def test_not_found(self): 432 self.app.register('HEAD', '/v1/AUTH_missing', 404, {}, '') 433 self.app.register('GET', '/v1/AUTH_missing', 404, {}, '') 434 req = swob.Request.blank("/v1/AUTH_missing") 435 status, headers, body = self.call_pfs(req) 436 self.assertEqual(status, '404 Not Found') 437 438 def test_rpc_error(self): 439 def broken_RpcGetAccount(_): 440 return { 441 "error": "errno: 123", # meaningless errno 442 "result": None} 443 444 self.fake_rpc.register_handler( 445 "Server.RpcGetAccount", broken_RpcGetAccount) 446 req = swob.Request.blank("/v1/AUTH_test") 447 status, headers, body = self.call_pfs(req) 448 self.assertEqual(status, '500 Internal Error') 449 450 def test_empty_last_page(self): 451 def last_page_RpcGetAccount(_): 452 return { 453 "error": None, 454 # None, not [], for mysterious reasons 455 "result": { 456 "ModificationTime": 1510966502886466000, 457 "AccountEntries": None, 458 }} 459 460 self.fake_rpc.register_handler( 461 "Server.RpcGetAccount", last_page_RpcGetAccount) 462 req = swob.Request.blank("/v1/AUTH_test?marker=zzz") 463 status, headers, body = self.call_pfs(req) 464 self.assertEqual(status, '204 No Content') 465 self.assertEqual(body, b'') 466 467 def test_spaces(self): 468 self.bimodal_accounts.add('AUTH_test with spaces') 469 self.app.register('HEAD', '/v1/AUTH_test with spaces', 204, 470 {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 471 '') 472 req = swob.Request.blank("/v1/AUTH_test with spaces") 473 status, headers, body = self.call_pfs(req) 474 self.assertEqual(status, '200 OK') 475 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 476 '/v1/AUTH_test with spaces') 477 478 def test_empty(self): 479 def mock_RpcGetAccount(_): 480 return { 481 "error": None, 482 "result": { 483 "ModificationTime": 1510966489413995000, 484 "AccountEntries": []}} 485 486 self.fake_rpc.register_handler( 487 "Server.RpcGetAccount", mock_RpcGetAccount) 488 req = swob.Request.blank("/v1/AUTH_test") 489 status, headers, body = self.call_pfs(req) 490 self.assertEqual(status, '204 No Content') 491 492 493 class TestAccountHead(BaseMiddlewareTest): 494 def setUp(self): 495 super(TestAccountHead, self).setUp() 496 497 self.app.register( 498 'HEAD', '/v1/AUTH_test', 499 204, 500 {'X-Account-Meta-Beans': 'lots of', 501 'X-Account-Sysmeta-Proxyfs-Bimodal': 'true'}, 502 '') 503 504 def test_indicator_header(self): 505 req = swob.Request.blank("/v1/AUTH_test", 506 environ={"REQUEST_METHOD": "HEAD"}) 507 status, headers, body = self.call_pfs(req) 508 self.assertEqual(status, '204 No Content') 509 self.assertEqual(headers.get("ProxyFS-Enabled"), "yes") 510 self.assertEqual(body, b'') 511 512 def test_in_transit(self): 513 514 def fake_RpcIsAccountBimodal(request): 515 return { 516 "error": None, 517 "result": { 518 "IsBimodal": True, 519 "ActivePeerPrivateIPAddr": ""}} 520 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 521 fake_RpcIsAccountBimodal) 522 523 req = swob.Request.blank("/v1/AUTH_test", 524 environ={"REQUEST_METHOD": "HEAD"}) 525 status, _, _ = self.call_pfs(req) 526 self.assertEqual(status, '503 Service Unavailable') 527 528 529 class TestObjectGet(BaseMiddlewareTest): 530 def setUp(self): 531 super(TestObjectGet, self).setUp() 532 533 def dummy_rpc(request): 534 return {"error": None, "result": {}} 535 536 self.fake_rpc.register_handler("Server.RpcRenewLease", dummy_rpc) 537 self.fake_rpc.register_handler("Server.RpcReleaseLease", dummy_rpc) 538 539 def test_info_passthrough(self): 540 self.app.register( 541 'GET', '/info', 200, {}, '{"stuff": "yes"}') 542 543 req = swob.Request.blank('/info') 544 status, headers, body = self.call_pfs(req) 545 self.assertEqual(status, '200 OK') 546 self.assertEqual(body, b'{"stuff": "yes"}') 547 548 def test_non_bimodal_account(self): 549 self.app.register( 550 'HEAD', '/v1/AUTH_unimodal', 204, {}, '') 551 self.app.register( 552 'GET', '/v1/AUTH_unimodal/c/o', 200, {}, 'squirrel') 553 554 req = swob.Request.blank('/v1/AUTH_unimodal/c/o') 555 status, headers, body = self.call_pfs(req) 556 self.assertEqual(status, '200 OK') 557 self.assertEqual(body, b'squirrel') 558 559 def test_GET_basic(self): 560 # ProxyFS log segments look a lot like actual file contents followed 561 # by a bunch of fairly opaque binary data plus some JSON-looking 562 # bits. The actual bytes in the file here are the 8 that spell 563 # "burritos"; the rest of the bytes are there so we can omit them in 564 # the response to the user. 565 self.app.register( 566 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', 567 200, {}, 568 ("burritos\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 569 570 def mock_RpcGetObject(get_object_req): 571 self.assertEqual(get_object_req['VirtPath'], 572 "/v1/AUTH_test/notes/lunch") 573 self.assertEqual(get_object_req['ReadEntsIn'], []) 574 575 return { 576 "error": None, 577 "result": { 578 "FileSize": 8, 579 "Metadata": "", 580 "InodeNumber": 1245, 581 "NumWrites": 2424, 582 "ModificationTime": 1481152134331862558, 583 "IsDir": False, 584 "LeaseId": "prominority-sarcocyst", 585 "ReadEntsOut": [{ 586 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 587 "Name/0000000000c11fbd"), 588 "Offset": 0, 589 "Length": 8}]}} 590 591 self.fake_rpc.register_handler( 592 "Server.RpcGetObject", mock_RpcGetObject) 593 594 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 595 status, headers, body = self.call_pfs(req) 596 597 self.assertEqual(status, '200 OK') 598 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 599 self.assertEqual(headers["Last-Modified"], 600 "Wed, 07 Dec 2016 23:08:55 GMT") 601 self.assertEqual(headers["ETag"], 602 mware.construct_etag("AUTH_test", 1245, 2424)) 603 self.assertEqual(body, b'burritos') 604 605 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=on', 606 environ={'swift_owner': True}) 607 status, headers, body = self.call_pfs(req) 608 609 self.assertEqual(status, '200 OK') 610 self.assertEqual(headers['Content-Type'], 'application/json') 611 self.assertEqual(headers['X-Object-Content-Type'], 612 'application/octet-stream') 613 self.assertEqual(headers['X-Object-Content-Length'], '8') 614 self.assertIn('ETag', headers) 615 self.assertIn('Last-Modified', headers) 616 self.assertEqual(json.loads(body), [{ 617 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 618 "/0000000000c11fbd"), 619 "Offset": 0, 620 "Length": 8, 621 }]) 622 623 # Can explicitly say you *don't* want the read plan 624 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=no', 625 environ={'swift_owner': True}) 626 status, headers, body = self.call_pfs(req) 627 self.assertEqual(status, '200 OK') 628 self.assertEqual(body, b'burritos') 629 630 # Can handle Range requests, too 631 def mock_RpcGetObject(get_object_req): 632 self.assertEqual(get_object_req['VirtPath'], 633 "/v1/AUTH_test/notes/lunch") 634 self.assertEqual(get_object_req['ReadEntsIn'], [ 635 {"Len": 2, "Offset": 2}]) 636 637 return { 638 "error": None, 639 "result": { 640 "FileSize": 8, 641 "Metadata": "", 642 "InodeNumber": 1245, 643 "NumWrites": 2424, 644 "ModificationTime": 1481152134331862558, 645 "IsDir": False, 646 "LeaseId": "prominority-sarcocyst", 647 "ReadEntsOut": [ 648 { 649 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 650 "Name/0000000000c11fbd"), 651 "Offset": 587, 652 "Length": 1 653 }, 654 { 655 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 656 "Name/0000000000c11798"), 657 "Offset": 25, 658 "Length": 1 659 }, 660 ]}} 661 662 self.fake_rpc.register_handler( 663 "Server.RpcGetObject", mock_RpcGetObject) 664 665 # Present-but-blank query param is truthy 666 req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan', 667 headers={'Range': 'bytes=2-3'}, 668 environ={'swift_owner': True}) 669 status, headers, body = self.call_pfs(req) 670 671 self.assertEqual(status, '200 OK') 672 self.assertEqual(headers['Content-Type'], 'application/json') 673 self.assertEqual(headers['X-Object-Content-Type'], 674 'application/octet-stream') 675 self.assertEqual(headers['X-Object-Content-Length'], '8') 676 self.assertIn('ETag', headers) 677 self.assertIn('Last-Modified', headers) 678 self.assertEqual(json.loads(body), [ 679 { 680 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 681 "/0000000000c11fbd"), 682 "Offset": 587, 683 "Length": 1, 684 }, 685 { 686 "ObjectPath": ("/v1/AUTH_test/InternalContainerName" 687 "/0000000000c11798"), 688 "Offset": 25, 689 "Length": 1, 690 }, 691 ]) 692 693 def test_GET_slo_manifest(self): 694 self.app.register( 695 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', 696 200, {}, 697 ("blah[]\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 698 699 def mock_RpcGetObject(get_object_req): 700 self.assertEqual(get_object_req['VirtPath'], 701 "/v1/AUTH_test/notes/lunch") 702 self.assertEqual(get_object_req['ReadEntsIn'], []) 703 704 return { 705 "error": None, 706 "result": { 707 "FileSize": 2, 708 "Metadata": base64.b64encode( 709 json.dumps({ 710 'X-Object-Sysmeta-Slo-Etag': 'some etag', 711 'X-Object-Sysmeta-Slo-Size': '0', 712 'Content-Type': 'text/plain;swift_bytes=0', 713 }).encode('ascii')).decode('ascii'), 714 "InodeNumber": 1245, 715 "NumWrites": 2424, 716 "ModificationTime": 1481152134331862558, 717 "IsDir": False, 718 "LeaseId": "prominority-sarcocyst", 719 "ReadEntsOut": [{ 720 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 721 "Name/0000000000c11fbd"), 722 "Offset": 4, 723 "Length": 2}]}} 724 725 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 726 727 self.fake_rpc.register_handler( 728 "Server.RpcGetObject", mock_RpcGetObject) 729 status, headers, body = self.call_pfs(req) 730 731 self.assertEqual(status, '200 OK') 732 self.assertEqual(headers, { 733 'Content-Type': 'text/plain', # no swift_bytes! 734 'Content-Length': '2', 735 # ^^^ needs to be actual size, but slo may/will use vvv for HEADs 736 'X-Object-Sysmeta-Slo-Size': '0', 737 'Etag': '"pfsv2/AUTH_test/000004DD/00000978-32"', 738 # slo may/will use vvv to fix up ^^^ 739 'X-Object-Sysmeta-Slo-Etag': 'some etag', 740 'Accept-Ranges': 'bytes', 741 'X-Timestamp': '1481152134.33186', 742 'Last-Modified': 'Wed, 07 Dec 2016 23:08:55 GMT', 743 }) 744 self.assertEqual(body, b'[]') 745 746 def test_GET_authed(self): 747 self.app.register( 748 'GET', '/v1/AUTH_test/InternalContainerName/0000000001178995', 749 200, {}, 750 ("burr\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) 751 self.app.register( 752 'GET', '/v1/AUTH_test/InternalContainerName/0000000004181255', 753 200, {}, 754 ("itos\x62\x6f\x6f\x74{\"more\": \"junk\"}\x00\x00\x00")) 755 756 def mock_RpcHead(head_req): 757 return { 758 "error": None, 759 "result": { 760 "Metadata": None, 761 "ModificationTime": 1488414932080993000, 762 "FileSize": 0, 763 "IsDir": True, 764 "InodeNumber": 6283109, 765 "NumWrites": 0, 766 }} 767 768 def mock_RpcGetObject(get_object_req): 769 self.assertEqual(get_object_req['VirtPath'], 770 "/v1/AUTH_test/notes/lunch") 771 self.assertEqual(get_object_req['ReadEntsIn'], []) 772 773 return { 774 "error": None, 775 "result": { 776 "FileSize": 8, 777 "Metadata": "", 778 "InodeNumber": 1245, 779 "NumWrites": 2424, 780 "ModificationTime": 1481152134331862558, 781 "IsDir": False, 782 "LeaseId": "a65b5591b90fe6035e669f1f216502d2", 783 "ReadEntsOut": [{ 784 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 785 "Name/0000000001178995"), 786 "Offset": 0, 787 "Length": 4}, 788 {"ObjectPath": ("/v1/AUTH_test/InternalContainer" 789 "Name/0000000004181255"), 790 "Offset": 0, 791 "Length": 4}]}} 792 793 def auth_callback(req): 794 if "InternalContainerName" in req.path: 795 return swob.HTTPForbidden(body="lol nope", request=req) 796 797 req = swob.Request.blank('/v1/AUTH_test/notes/lunch') 798 req.environ["swift.authorize"] = auth_callback 799 800 self.fake_rpc.register_handler( 801 "Server.RpcGetObject", mock_RpcGetObject) 802 self.fake_rpc.register_handler( 803 "Server.RpcHead", mock_RpcHead) 804 status, headers, body = self.call_pfs(req) 805 806 self.assertEqual(status, '200 OK') 807 self.assertEqual(headers["Last-Modified"], 808 "Wed, 07 Dec 2016 23:08:55 GMT") 809 self.assertEqual(headers["ETag"], 810 mware.construct_etag("AUTH_test", 1245, 2424)) 811 self.assertEqual(body, b'burritos') 812 813 def test_GET_sparse(self): 814 # This log segment, except for the obvious fake metadata at the end, 815 # was obtained by really doing this: 816 # 817 # with open(filename, 'w') as fh: 818 # fh.write('sparse') 819 # fh.seek(10006) 820 # fh.write('file') 821 # 822 # Despite its appearance, this is really what the underlying log 823 # segment for a sparse file looks like, at least sometimes. 824 self.app.register( 825 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000D0', 826 200, {}, 827 ("sparsefile\x57\x18\xa0\xf3-junkety-junk")) 828 829 def mock_RpcGetObject(get_object_req): 830 self.assertEqual(get_object_req['VirtPath'], 831 "/v1/AUTH_test/c/sparse-file") 832 self.assertEqual(get_object_req['ReadEntsIn'], []) 833 834 return { 835 "error": None, 836 "result": { 837 "FileSize": 10010, 838 "Metadata": "", 839 "InodeNumber": 1245, 840 "NumWrites": 2424, 841 "ModificationTime": 1481152134331862558, 842 "IsDir": False, 843 "LeaseId": "6840595b3370f109dc8ed388b41800a4", 844 "ReadEntsOut": [{ 845 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 846 "Name/00000000000000D0"), 847 "Offset": 0, 848 "Length": 6 849 }, { 850 "ObjectPath": "", # empty path means zero-fill 851 "Offset": 0, 852 "Length": 10000, 853 }, { 854 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 855 "Name/00000000000000D0"), 856 "Offset": 6, 857 "Length": 4}]}} 858 859 req = swob.Request.blank('/v1/AUTH_test/c/sparse-file') 860 861 self.fake_rpc.register_handler( 862 "Server.RpcGetObject", mock_RpcGetObject) 863 status, headers, body = self.call_pfs(req) 864 865 self.assertEqual(status, "200 OK") 866 self.assertEqual(body, b"sparse" + (b"\x00" * 10000) + b"file") 867 868 def test_GET_multiple_segments(self): 869 # Typically, a GET request will include data from multiple log 870 # segments. Small files written all at once might fit in a single 871 # log segment, but that's not always true. 872 self.app.register( 873 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 874 200, {}, 875 ("There once was an X from place B,\n\xff\xea\x00junk")) 876 self.app.register( 877 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a3', 878 200, {}, 879 ("That satisfied predicate P.\n\xff\xea\x00junk")) 880 self.app.register( 881 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a4', 882 200, {}, 883 ("He or she did thing A,\n\xff\xea\x00junk")) 884 self.app.register( 885 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000e0', 886 200, {}, 887 ("In an adjective way,\n\xff\xea\x00junk")) 888 self.app.register( 889 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000108', 890 200, {}, 891 ("Resulting in circumstance C.\xff\xea\x00junk")) 892 893 def mock_RpcGetObject(get_object_req): 894 self.assertEqual(get_object_req['VirtPath'], 895 "/v1/AUTH_test/limericks/generic") 896 self.assertEqual(get_object_req['ReadEntsIn'], []) 897 898 return { 899 "error": None, 900 "result": { 901 "FileSize": 134, 902 "Metadata": "", 903 "InodeNumber": 1245, 904 "NumWrites": 2424, 905 "ModificationTime": 1481152134331862558, 906 "IsDir": False, 907 "LeaseId": "who cares", 908 "ReadEntsOut": [{ 909 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 910 "Name/0000000000000001"), 911 "Offset": 0, 912 "Length": 34, 913 }, { 914 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 915 "Name/00000000000000a3"), 916 "Offset": 0, 917 "Length": 28, 918 }, { 919 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 920 "Name/00000000000000a4"), 921 "Offset": 0, 922 "Length": 23, 923 }, { 924 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 925 "Name/00000000000000e0"), 926 "Offset": 0, 927 "Length": 21, 928 }, { 929 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 930 "Name/0000000000000108"), 931 "Offset": 0, 932 "Length": 28, 933 }]}} 934 935 req = swob.Request.blank('/v1/AUTH_test/limericks/generic') 936 937 self.fake_rpc.register_handler( 938 "Server.RpcGetObject", mock_RpcGetObject) 939 status, headers, body = self.call_pfs(req) 940 941 self.assertEqual(status, '200 OK') 942 self.assertEqual(body, ( 943 b"There once was an X from place B,\n" 944 b"That satisfied predicate P.\n" 945 b"He or she did thing A,\n" 946 b"In an adjective way,\n" 947 b"Resulting in circumstance C.")) 948 949 def test_GET_conditional_if_match(self): 950 obj_etag = "42d0f073592cdef8f602cda59fbb270e" 951 obj_body = b"annet-pentahydric-nuculoid-defiber" 952 953 self.app.register( 954 'GET', '/v1/AUTH_test/InternalContainerName/000000000097830c', 955 200, {}, 956 "annet-pentahydric-nuculoid-defiber") 957 958 def mock_RpcGetObject(get_object_req): 959 return { 960 "error": None, 961 "result": { 962 "FileSize": 8, 963 "Metadata": base64.b64encode( 964 json.dumps({ 965 mware.ORIGINAL_MD5_HEADER: "123:%s" % obj_etag, 966 }).encode('ascii')).decode('ascii'), 967 "InodeNumber": 1245, 968 "NumWrites": 123, 969 "ModificationTime": 1511222561631497000, 970 "IsDir": False, 971 "LeaseId": "dontcare", 972 "ReadEntsOut": [{ 973 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 974 "Name/000000000097830c"), 975 "Offset": 0, 976 "Length": len(obj_body)}]}} 977 978 self.fake_rpc.register_handler( 979 "Server.RpcGetObject", mock_RpcGetObject) 980 981 # matches 982 req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', 983 headers={"If-Match": obj_etag}) 984 status, _, body = self.call_pfs(req) 985 self.assertEqual(status, '200 OK') 986 self.assertEqual(body, obj_body) 987 988 # doesn't match 989 req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', 990 headers={"If-Match": obj_etag + "abc"}) 991 status, _, body = self.call_pfs(req) 992 self.assertEqual(status, '412 Precondition Failed') 993 994 def test_GET_range(self): 995 # Typically, a GET response will include data from multiple log 996 # segments. Small files written all at once might fit in a single 997 # log segment, but most files won't. 998 self.app.register( 999 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1000 200, {}, 1001 ("Caerphilly, Cheddar, Cheshire, \xff\xea\x00junk")) 1002 self.app.register( 1003 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1004 200, {}, 1005 ("Duddleswell, Dunlop, Coquetdale, \xff\xea\x00junk")) 1006 self.app.register( 1007 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000003', 1008 200, {}, 1009 ("Derby, Gloucester, Wensleydale\xff\xea\x00junk")) 1010 1011 def mock_RpcGetObject(get_object_req): 1012 self.assertEqual(get_object_req['VirtPath'], 1013 "/v1/AUTH_test/cheeses/UK") 1014 self.assertEqual(get_object_req['ReadEntsIn'], 1015 [{"Offset": 21, "Len": 49}]) 1016 1017 return { 1018 "error": None, 1019 "result": { 1020 "FileSize": 94, 1021 "Metadata": "", 1022 "InodeNumber": 1245, 1023 "NumWrites": 2424, 1024 "ModificationTime": 1481152134331862558, 1025 "IsDir": False, 1026 "LeaseId": "982938", 1027 "ReadEntsOut": [{ 1028 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1029 "Name/0000000000000001"), 1030 "Offset": 21, 1031 "Length": 10, 1032 }, { 1033 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1034 "Name/0000000000000002"), 1035 "Offset": 0, 1036 "Length": 33, 1037 }, { 1038 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1039 "Name/0000000000000003"), 1040 "Offset": 0, 1041 "Length": 5}]}} 1042 1043 req = swob.Request.blank('/v1/AUTH_test/cheeses/UK', 1044 headers={"Range": "bytes=21-69"}) 1045 1046 self.fake_rpc.register_handler( 1047 "Server.RpcGetObject", mock_RpcGetObject) 1048 status, headers, body = self.call_pfs(req) 1049 1050 self.assertEqual(status, '206 Partial Content') 1051 self.assertEqual(headers.get('Content-Range'), "bytes 21-69/94") 1052 self.assertEqual( 1053 body, b'Cheshire, Duddleswell, Dunlop, Coquetdale, Derby') 1054 1055 def test_GET_range_suffix(self): 1056 self.app.register( 1057 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1058 200, {}, 1059 ("hydrogen, helium, \xff\xea\x00junk")) 1060 self.app.register( 1061 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1062 200, {}, 1063 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1064 1065 def mock_RpcGetObject(get_object_req): 1066 self.assertEqual(get_object_req['VirtPath'], 1067 "/v1/AUTH_test/c/elements") 1068 self.assertEqual(get_object_req['ReadEntsIn'], 1069 [{"Offset": None, "Len": 10}]) 1070 1071 return { 1072 "error": None, 1073 "result": { 1074 "FileSize": 94, 1075 "Metadata": "", 1076 "InodeNumber": 1245, 1077 "NumWrites": 2424, 1078 "ModificationTime": 1481152134331862558, 1079 "IsDir": False, 1080 "LeaseId": "fc00:752b:5cca:a544:2d41:3177:2c71:85ae", 1081 "ReadEntsOut": [{ 1082 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1083 "Name/0000000000000002"), 1084 "Offset": 35, 1085 "Length": 10}]}} 1086 1087 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1088 headers={"Range": "bytes=-10"}) 1089 1090 self.fake_rpc.register_handler( 1091 "Server.RpcGetObject", mock_RpcGetObject) 1092 status, headers, body = self.call_pfs(req) 1093 1094 self.assertEqual(status, '206 Partial Content') 1095 self.assertEqual(headers.get('Content-Range'), "bytes 84-93/94") 1096 self.assertEqual(body, b"nitrogen, ") 1097 1098 def test_GET_range_prefix(self): 1099 self.app.register( 1100 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1101 200, {}, 1102 ("hydrogen, helium, \xff\xea\x00junk")) 1103 self.app.register( 1104 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1105 200, {}, 1106 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1107 1108 def mock_RpcGetObject(get_object_req): 1109 self.assertEqual(get_object_req['VirtPath'], 1110 "/v1/AUTH_test/c/elements") 1111 self.assertEqual(get_object_req['ReadEntsIn'], 1112 [{"Offset": 40, "Len": None}]) 1113 1114 return { 1115 "error": None, 1116 "result": { 1117 "FileSize": 62, 1118 "Metadata": "", 1119 "InodeNumber": 1245, 1120 "NumWrites": 2424, 1121 "ModificationTime": 1481152134331862558, 1122 "IsDir": False, 1123 "LeaseId": "", 1124 "ReadEntsOut": [{ 1125 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1126 "Name/0000000000000002"), 1127 "Offset": 22, 1128 "Length": 23}]}} 1129 1130 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1131 headers={"Range": "bytes=40-"}) 1132 1133 self.fake_rpc.register_handler( 1134 "Server.RpcGetObject", mock_RpcGetObject) 1135 status, headers, body = self.call_pfs(req) 1136 1137 self.assertEqual(status, '206 Partial Content') 1138 self.assertEqual(headers.get('Content-Range'), "bytes 40-61/62") 1139 self.assertEqual(body, b"ron, carbon, nitrogen, ") 1140 1141 def test_GET_range_unsatisfiable(self): 1142 self.app.register( 1143 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1144 200, {}, 1145 ("hydrogen, helium, \xff\xea\x00junk")) 1146 self.app.register( 1147 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', 1148 200, {}, 1149 ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) 1150 1151 def mock_RpcGetObject(get_object_req): 1152 self.assertEqual(get_object_req['VirtPath'], 1153 "/v1/AUTH_test/c/elements") 1154 self.assertEqual(get_object_req['ReadEntsIn'], 1155 [{"Offset": 4000, "Len": None}]) 1156 1157 return { 1158 "error": None, 1159 "result": { 1160 "FileSize": 62, 1161 "Metadata": "", 1162 "InodeNumber": 1245, 1163 "NumWrites": 2426, 1164 "ModificationTime": 1481152134331862558, 1165 "IsDir": False, 1166 "LeaseId": "borkbork", 1167 "ReadEntsOut": None}} 1168 1169 req = swob.Request.blank('/v1/AUTH_test/c/elements', 1170 headers={"Range": "bytes=4000-"}) 1171 1172 self.fake_rpc.register_handler( 1173 "Server.RpcGetObject", mock_RpcGetObject) 1174 status, headers, body = self.call_pfs(req) 1175 1176 self.assertEqual(status, '416 Requested Range Not Satisfiable') 1177 self.assertEqual(headers.get('Content-Range'), 'bytes */62') 1178 self.assertEqual(headers.get('ETag'), 1179 mware.construct_etag("AUTH_test", 1245, 2426)) 1180 self.assertEqual(headers.get('Last-Modified'), 1181 'Wed, 07 Dec 2016 23:08:55 GMT') 1182 self.assertEqual(headers.get('X-Timestamp'), 1183 '1481152134.33186') 1184 1185 def test_GET_multiple_ranges(self): 1186 self.app.register( 1187 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1188 200, {}, "abcd1234efgh5678") 1189 1190 def mock_RpcGetObject(get_object_req): 1191 self.assertEqual(get_object_req['VirtPath'], 1192 "/v1/AUTH_test/c/crud") 1193 self.assertEqual(get_object_req['ReadEntsIn'], [ 1194 {'Len': 3, 'Offset': 2}, 1195 {'Len': 3, 'Offset': 6}, 1196 {'Len': 3, 'Offset': 10}, 1197 ]) 1198 1199 return { 1200 "error": None, 1201 "result": { 1202 "FileSize": 16, 1203 "Metadata": "", 1204 "InodeNumber": 1245, 1205 "NumWrites": 2424, 1206 "ModificationTime": 1481152134331862558, 1207 "IsDir": False, 1208 "LeaseId": "e1885b511fa445d18b1d447a5606a06d", 1209 "ReadEntsOut": [{ 1210 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1211 "Name/0000000000000001"), 1212 "Offset": 2, 1213 "Length": 3, 1214 }, { 1215 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1216 "Name/0000000000000001"), 1217 "Offset": 6, 1218 "Length": 3, 1219 }, { 1220 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1221 "Name/0000000000000001"), 1222 "Offset": 10, 1223 "Length": 3, 1224 }]}} 1225 1226 self.fake_rpc.register_handler( 1227 "Server.RpcGetObject", mock_RpcGetObject) 1228 1229 req = swob.Request.blank('/v1/AUTH_test/c/crud', 1230 headers={"Range": "bytes=2-4,6-8,10-12"}) 1231 1232 # Lock down the MIME boundary so it doesn't change on every test run 1233 with mock.patch('random.randint', 1234 lambda u, l: 0xf0a9157cb1757bfb124aef22fee31051): 1235 status, headers, body = self.call_pfs(req) 1236 1237 self.assertEqual(status, '206 Partial Content') 1238 self.assertEqual( 1239 headers.get('Content-Type'), 1240 'multipart/byteranges;boundary=f0a9157cb1757bfb124aef22fee31051') 1241 self.assertEqual( 1242 body, 1243 (b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1244 b'Content-Type: application/octet-stream\r\n' 1245 b'Content-Range: bytes 2-4/16\r\n' 1246 b'\r\n' 1247 b'cd1\r\n' 1248 b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1249 b'Content-Type: application/octet-stream\r\n' 1250 b'Content-Range: bytes 6-8/16\r\n' 1251 b'\r\n' 1252 b'34e\r\n' 1253 b'--f0a9157cb1757bfb124aef22fee31051\r\n' 1254 b'Content-Type: application/octet-stream\r\n' 1255 b'Content-Range: bytes 10-12/16\r\n' 1256 b'\r\n' 1257 b'gh5\r\n' 1258 b'--f0a9157cb1757bfb124aef22fee31051--')) 1259 1260 def test_GET_metadata(self): 1261 self.app.register( 1262 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', 1263 200, {}, "abcd1234efgh5678") 1264 1265 def mock_RpcGetObject(get_object_req): 1266 self.assertEqual(get_object_req['VirtPath'], 1267 "/v1/AUTH_test/c/crud") 1268 1269 return { 1270 "error": None, 1271 "result": { 1272 "FileSize": 16, 1273 "Metadata": base64.b64encode(json.dumps({ 1274 "X-Object-Meta-Cow": "moo", 1275 }).encode('ascii')).decode('ascii'), 1276 "InodeNumber": 1245, 1277 "NumWrites": 2424, 1278 "ModificationTime": 1481152134331862558, 1279 "IsDir": False, 1280 "LeaseId": "fe57a6ed-758a-23fb-f7d4-9683aee07c0e", 1281 "ReadEntsOut": [{ 1282 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1283 "Name/0000000000000001"), 1284 "Offset": 0, 1285 "Length": 16}]}} 1286 1287 req = swob.Request.blank('/v1/AUTH_test/c/crud') 1288 1289 self.fake_rpc.register_handler( 1290 "Server.RpcGetObject", mock_RpcGetObject) 1291 status, headers, body = self.call_pfs(req) 1292 1293 self.assertEqual(status, '200 OK') 1294 self.assertEqual(headers.get("X-Object-Meta-Cow"), "moo") 1295 self.assertEqual(body, b'abcd1234efgh5678') 1296 1297 def test_GET_bad_path(self): 1298 bad_paths = [ 1299 '/v1/AUTH_test/c/..', 1300 '/v1/AUTH_test/c/../o', 1301 '/v1/AUTH_test/c/o/..', 1302 '/v1/AUTH_test/c/.', 1303 '/v1/AUTH_test/c/./o', 1304 '/v1/AUTH_test/c/o/.', 1305 '/v1/AUTH_test/c//o', 1306 '/v1/AUTH_test/c/o//', 1307 '/v1/AUTH_test/c/o/', 1308 '/v1/AUTH_test/c/' + ('x' * 256), 1309 ] 1310 for path in bad_paths: 1311 req = swob.Request.blank(path) 1312 status, headers, body = self.call_pfs(req) 1313 self.assertEqual(status, '404 Not Found', 1314 'Got %s for %s' % (status, path)) 1315 1316 req.environ['REQUEST_METHOD'] = 'HEAD' 1317 status, headers, body = self.call_pfs(req) 1318 self.assertEqual(status, '404 Not Found', 1319 'Got %s for %s' % (status, path)) 1320 1321 path = '/v1/AUTH_test//o' 1322 req = swob.Request.blank(path, headers={'Content-Length': '0'}) 1323 for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): 1324 req.environ['REQUEST_METHOD'] = method 1325 status, headers, body = self.call_pfs(req) 1326 self.assertEqual(status, '412 Precondition Failed', 1327 'Got %s for %s %s' % (status, method, path)) 1328 1329 def test_GET_not_found(self): 1330 def mock_RpcGetObject(get_object_req): 1331 self.assertEqual(get_object_req['VirtPath'], 1332 "/v1/AUTH_test/c/missing") 1333 1334 return { 1335 "error": "errno: 2", 1336 "result": None} 1337 1338 req = swob.Request.blank('/v1/AUTH_test/c/missing') 1339 self.fake_rpc.register_handler( 1340 "Server.RpcGetObject", mock_RpcGetObject) 1341 status, headers, body = self.call_pfs(req) 1342 self.assertEqual(status, '404 Not Found') 1343 1344 def test_GET_file_as_dir(self): 1345 # Subdirectories of files don't exist, but asking for one returns a 1346 # different error code than asking for a file that could exist but 1347 # doesn't. 1348 def mock_RpcGetObject(get_object_req): 1349 self.assertEqual(get_object_req['VirtPath'], 1350 "/v1/AUTH_test/c/thing.txt/kitten.png") 1351 1352 return { 1353 "error": "errno: 20", 1354 "result": None} 1355 1356 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png') 1357 self.fake_rpc.register_handler( 1358 "Server.RpcGetObject", mock_RpcGetObject) 1359 status, headers, body = self.call_pfs(req) 1360 self.assertEqual(status, '404 Not Found') 1361 1362 def test_GET_weird_error(self): 1363 def mock_RpcGetObject(get_object_req): 1364 self.assertEqual(get_object_req['VirtPath'], 1365 "/v1/AUTH_test/c/superborked") 1366 1367 return { 1368 "error": "errno: 23", # ENFILE (too many open files in system) 1369 "result": None} 1370 1371 req = swob.Request.blank('/v1/AUTH_test/c/superborked') 1372 self.fake_rpc.register_handler( 1373 "Server.RpcGetObject", mock_RpcGetObject) 1374 status, headers, body = self.call_pfs(req) 1375 self.assertEqual(status, '500 Internal Error') 1376 1377 def test_GET_zero_byte(self): 1378 def mock_RpcGetObject(get_object_req): 1379 self.assertEqual(get_object_req['VirtPath'], 1380 "/v1/AUTH_test/c/empty") 1381 return { 1382 "error": None, 1383 "result": {"FileSize": 0, "ReadEntsOut": None, "Metadata": "", 1384 "IsDir": False, 1385 "LeaseId": "3d73d2bcf39224df00d5ccd912d92c82", 1386 "InodeNumber": 1245, "NumWrites": 2424, 1387 "ModificationTime": 1481152134331862558}} 1388 1389 self.fake_rpc.register_handler( 1390 "Server.RpcGetObject", mock_RpcGetObject) 1391 1392 req = swob.Request.blank('/v1/AUTH_test/c/empty') 1393 status, headers, body = self.call_pfs(req) 1394 self.assertEqual(status, '200 OK') 1395 self.assertEqual(headers["Content-Length"], "0") 1396 1397 def test_GET_dir(self): 1398 def mock_RpcGetObject(get_object_req): 1399 self.assertEqual(get_object_req['VirtPath'], 1400 "/v1/AUTH_test/c/a-dir") 1401 return { 1402 "error": None, 1403 "result": { 1404 "Metadata": "", 1405 "ModificationTime": 1479173168018879490, 1406 "FileSize": 0, 1407 "IsDir": True, 1408 "InodeNumber": 1254, 1409 "NumWrites": 896, 1410 "ReadEntsOut": None, 1411 "LeaseId": "borkbork", 1412 }} 1413 1414 self.fake_rpc.register_handler( 1415 "Server.RpcGetObject", mock_RpcGetObject) 1416 1417 req = swob.Request.blank('/v1/AUTH_test/c/a-dir') 1418 status, headers, body = self.call_pfs(req) 1419 self.assertEqual(status, '200 OK') 1420 self.assertEqual(headers["Content-Length"], "0") 1421 self.assertEqual(headers["Content-Type"], "application/directory") 1422 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 1423 1424 def test_GET_special_chars(self): 1425 self.app.register( 1426 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000123', 1427 200, {}, "abcd1234efgh5678") 1428 1429 def mock_RpcGetObject(get_object_req): 1430 return { 1431 "error": None, 1432 "result": { 1433 "FileSize": 8, 1434 "Metadata": "", 1435 "InodeNumber": 1245, 1436 "NumWrites": 2424, 1437 "ModificationTime": 1481152134331862558, 1438 "IsDir": False, 1439 "LeaseId": "a7ec296d3f3c39ef95407789c436f5f8", 1440 "ReadEntsOut": [{ 1441 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1442 "Name/0000000000000123"), 1443 "Offset": 0, 1444 "Length": 16}]}} 1445 1446 req = swob.Request.blank('/v1/AUTH_test/c o n/o b j') 1447 1448 self.fake_rpc.register_handler( 1449 "Server.RpcGetObject", mock_RpcGetObject) 1450 status, headers, body = self.call_pfs(req) 1451 1452 self.assertEqual(status, '200 OK') 1453 self.assertEqual(body, b'abcd1234efgh5678') 1454 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1455 '/v1/AUTH_test/c o n/o b j') 1456 1457 def test_md5_etag(self): 1458 self.app.register( 1459 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', 1460 200, {}, "stuff stuff stuff") 1461 1462 def mock_RpcGetObject(_): 1463 return { 1464 "error": None, 1465 "result": { 1466 "Metadata": base64.b64encode(json.dumps({ 1467 mware.ORIGINAL_MD5_HEADER: 1468 "3:25152b9f7ca24b61eec895be4e89a950", 1469 }).encode('ascii')).decode('ascii'), 1470 "ModificationTime": 1506039770222591000, 1471 "IsDir": False, 1472 "FileSize": 17, 1473 "IsDir": False, 1474 "InodeNumber": 1433230, 1475 "NumWrites": 3, 1476 "LeaseId": "leaseid", 1477 "ReadEntsOut": [{ 1478 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1479 "Name/0000000000000456"), 1480 "Offset": 0, 1481 "Length": 13}]}} 1482 1483 self.fake_rpc.register_handler( 1484 "Server.RpcGetObject", mock_RpcGetObject) 1485 1486 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png") 1487 status, headers, body = self.call_pfs(req) 1488 self.assertEqual(status, '200 OK') 1489 self.assertEqual(headers["Etag"], "25152b9f7ca24b61eec895be4e89a950") 1490 1491 def test_lease_maintenance(self): 1492 self.app.register( 1493 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', 1494 200, {}, "some contents") 1495 1496 expected_lease_id = "f1abb6c7fc4b4463b04f8313d71986b7" 1497 1498 def mock_RpcGetObject(get_object_req): 1499 return { 1500 "error": None, 1501 "result": { 1502 "FileSize": 13, 1503 "Metadata": "", 1504 "InodeNumber": 7677424, 1505 "NumWrites": 2325461, 1506 "ModificationTime": 1488841810471415000, 1507 "IsDir": False, 1508 "LeaseId": expected_lease_id, 1509 "ReadEntsOut": [{ 1510 "ObjectPath": ("/v1/AUTH_test/InternalContainer" 1511 "Name/0000000000000456"), 1512 "Offset": 0, 1513 "Length": 13}]}} 1514 1515 req = swob.Request.blank('/v1/AUTH_test/some/thing') 1516 1517 self.fake_rpc.register_handler( 1518 "Server.RpcGetObject", mock_RpcGetObject) 1519 status, headers, body = self.call_pfs(req) 1520 1521 self.assertEqual(status, '200 OK') 1522 self.assertEqual(body, b'some contents') 1523 1524 called_rpcs = [c[0] for c in self.fake_rpc.calls] 1525 1526 # There's at least one RpcRenewLease, but perhaps there's more: it's 1527 # time-based, and we aren't mocking out time in this test. 1528 self.assertEqual(called_rpcs[0], 'Server.RpcIsAccountBimodal') 1529 self.assertEqual(called_rpcs[1], 'Server.RpcGetObject') 1530 self.assertEqual(called_rpcs[-2], 'Server.RpcRenewLease') 1531 self.assertEqual(called_rpcs[-1], 'Server.RpcReleaseLease') 1532 1533 # make sure we got the right lease ID 1534 self.assertEqual(self.fake_rpc.calls[-2][1][0]['LeaseId'], 1535 expected_lease_id) 1536 self.assertEqual(self.fake_rpc.calls[-1][1][0]['LeaseId'], 1537 expected_lease_id) 1538 1539 1540 class TestContainerHead(BaseMiddlewareTest): 1541 def setUp(self): 1542 super(TestContainerHead, self).setUp() 1543 1544 self.serialized_container_metadata = "" 1545 1546 # All these tests run against the same container. 1547 def mock_RpcHead(head_container_req): 1548 return { 1549 "error": None, 1550 "result": { 1551 "Metadata": base64.b64encode( 1552 self.serialized_container_metadata.encode('ascii')), 1553 "ModificationTime": 1479240397189581131, 1554 "IsDir": False, 1555 "FileSize": 0, 1556 "InodeNumber": 2718, 1557 "NumWrites": 0, 1558 }} 1559 1560 self.fake_rpc.register_handler( 1561 "Server.RpcHead", mock_RpcHead) 1562 1563 def test_special_chars(self): 1564 req = swob.Request.blank("/v1/AUTH_test/a container", 1565 environ={"REQUEST_METHOD": "HEAD"}) 1566 status, headers, body = self.call_pfs(req) 1567 self.assertEqual(status, '204 No Content') 1568 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1569 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1570 '/v1/AUTH_test/a container') 1571 1572 def test_content_type(self): 1573 req = swob.Request.blank("/v1/AUTH_test/a-container?format=xml", 1574 environ={"REQUEST_METHOD": "HEAD"}) 1575 status, headers, _ = self.call_pfs(req) 1576 self.assertEqual(status, '204 No Content') # sanity check 1577 self.assertEqual(headers["Content-Type"], 1578 "application/xml; charset=utf-8") 1579 1580 req = swob.Request.blank("/v1/AUTH_test/a-container?format=json", 1581 environ={"REQUEST_METHOD": "HEAD"}) 1582 status, headers, _ = self.call_pfs(req) 1583 self.assertEqual(status, '204 No Content') # sanity check 1584 self.assertEqual(headers["Content-Type"], 1585 "application/json; charset=utf-8") 1586 1587 req = swob.Request.blank("/v1/AUTH_test/a-container?format=plain", 1588 environ={"REQUEST_METHOD": "HEAD"}) 1589 status, headers, _ = self.call_pfs(req) 1590 self.assertEqual(status, '204 No Content') # sanity check 1591 self.assertEqual(headers["Content-Type"], 1592 "text/plain; charset=utf-8") 1593 1594 with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', 1595 True): 1596 status, headers, _ = self.call_pfs(req) 1597 self.assertEqual(status, '204 No Content') # sanity check 1598 self.assertEqual(headers["Content-Type"], 1599 "application/json; charset=utf-8") 1600 1601 def test_no_meta(self): 1602 self.serialized_container_metadata = "" 1603 1604 req = swob.Request.blank("/v1/AUTH_test/a-container", 1605 environ={"REQUEST_METHOD": "HEAD"}) 1606 status, headers, body = self.call_pfs(req) 1607 self.assertEqual(status, '204 No Content') 1608 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1609 self.assertEqual(headers["X-Container-Object-Count"], "0") 1610 self.assertEqual(headers["X-Container-Bytes-Used"], "0") 1611 self.assertEqual(headers["X-Storage-Policy"], "default") 1612 self.assertEqual(headers["Last-Modified"], 1613 "Tue, 15 Nov 2016 20:06:38 GMT") 1614 self.assertEqual(headers["X-Timestamp"], "1479240397.18958") 1615 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1616 '/v1/AUTH_test/a-container') 1617 1618 def test_bogus_meta(self): 1619 self.serialized_container_metadata = "{[{[{[{[{[[(((!" 1620 1621 req = swob.Request.blank("/v1/AUTH_test/a-container", 1622 environ={"REQUEST_METHOD": "HEAD"}) 1623 status, headers, body = self.call_pfs(req) 1624 self.assertEqual(status, '204 No Content') 1625 1626 def _check_meta(self, expected_headers, swift_owner=False): 1627 container_metadata = { 1628 "X-Container-Read": "xcr", 1629 "X-Container-Write": "xcw", 1630 "X-Container-Sync-Key": "sync-key", 1631 "X-Container-Sync-To": "sync-to", 1632 "X-Container-Meta-Temp-Url-Key": "tuk", 1633 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1634 "X-Container-Sysmeta-Fish": "cod", 1635 "X-Container-Meta-Fish": "trout"} 1636 self.serialized_container_metadata = json.dumps(container_metadata) 1637 1638 req = swob.Request.blank("/v1/AUTH_test/a-container", 1639 environ={"REQUEST_METHOD": "HEAD", 1640 "swift_owner": swift_owner}) 1641 status, headers, body = self.call_pfs(req) 1642 self.assertEqual(status, '204 No Content') 1643 for k, v in expected_headers.items(): 1644 self.assertIn(k, headers) 1645 self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % 1646 (v, headers[k], v)) 1647 for k in container_metadata: 1648 self.assertFalse(k in headers and k not in expected_headers, 1649 "Found unexpected header %r" % k) 1650 1651 def test_meta_swift_owner(self): 1652 self._check_meta({ 1653 "X-Container-Read": "xcr", 1654 "X-Container-Write": "xcw", 1655 "X-Container-Sync-Key": "sync-key", 1656 "X-Container-Sync-To": "sync-to", 1657 "X-Container-Meta-Temp-Url-Key": "tuk", 1658 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1659 "X-Container-Sysmeta-Fish": "cod", 1660 "X-Container-Meta-Fish": "trout"}, 1661 swift_owner=True) 1662 1663 def test_meta_not_swift_owner(self): 1664 self._check_meta({ 1665 "X-Container-Sysmeta-Fish": "cod", 1666 "X-Container-Meta-Fish": "trout"}) 1667 1668 def test_not_found(self): 1669 def mock_RpcHead(head_container_req): 1670 self.assertEqual(head_container_req['VirtPath'], 1671 '/v1/AUTH_test/a-container') 1672 return {"error": "errno: 2", "result": None} 1673 1674 self.fake_rpc.register_handler( 1675 "Server.RpcHead", mock_RpcHead) 1676 1677 req = swob.Request.blank("/v1/AUTH_test/a-container", 1678 environ={"REQUEST_METHOD": "HEAD"}) 1679 status, headers, body = self.call_pfs(req) 1680 self.assertEqual(status, '404 Not Found') 1681 1682 def test_other_error(self): 1683 def mock_RpcHead(head_container_req): 1684 self.assertEqual(head_container_req['VirtPath'], 1685 '/v1/AUTH_test/a-container') 1686 return {"error": "errno: 7581", "result": None} 1687 1688 self.fake_rpc.register_handler( 1689 "Server.RpcHead", mock_RpcHead) 1690 1691 req = swob.Request.blank("/v1/AUTH_test/a-container", 1692 environ={"REQUEST_METHOD": "HEAD"}) 1693 status, headers, body = self.call_pfs(req) 1694 self.assertEqual(status, '500 Internal Error') 1695 1696 1697 class TestContainerGet(BaseMiddlewareTest): 1698 def setUp(self): 1699 super(TestContainerGet, self).setUp() 1700 self.serialized_container_metadata = "" 1701 1702 # All these tests run against the same container. 1703 def mock_RpcGetContainer(get_container_req): 1704 return { 1705 "error": None, 1706 "result": { 1707 "Metadata": base64.b64encode( 1708 self.serialized_container_metadata.encode('ascii')), 1709 "ModificationTime": 1510790796076041000, 1710 "ContainerEntries": [{ 1711 "Basename": "images", 1712 "FileSize": 0, 1713 "ModificationTime": 1471915816359209849, 1714 "IsDir": True, 1715 "InodeNumber": 2489682, 1716 "NumWrites": 0, 1717 "Metadata": "", 1718 }, { 1719 "Basename": u"images/\xE1vocado.png", 1720 "FileSize": 70, 1721 "ModificationTime": 1471915816859209471, 1722 "IsDir": False, 1723 "InodeNumber": 9213768, 1724 "NumWrites": 2, 1725 "Metadata": base64.b64encode(json.dumps({ 1726 "Content-Type": u"snack/m\xEDllenial" + 1727 u";swift_bytes=3503770", 1728 }).encode('ascii')), 1729 }, { 1730 "Basename": "images/banana.png", 1731 "FileSize": 2189865, 1732 # has fractional seconds = 0 to cover edge cases 1733 "ModificationTime": 1471915873000000000, 1734 "IsDir": False, 1735 "InodeNumber": 8878410, 1736 "NumWrites": 2, 1737 "Metadata": "", 1738 }, { 1739 "Basename": "images/cherimoya.png", 1740 "FileSize": 1636662, 1741 "ModificationTime": 1471915917767421311, 1742 "IsDir": False, 1743 "InodeNumber": 8879064, 1744 "NumWrites": 2, 1745 # Note: this has NumWrites=2, but the original MD5 1746 # starts with "1:", so it is stale and must not be 1747 # used. 1748 "Metadata": base64.b64encode(json.dumps({ 1749 mware.ORIGINAL_MD5_HEADER: 1750 "1:552528fbf2366f8a4711ac0a3875188b", 1751 }).encode('ascii')), 1752 }, { 1753 "Basename": "images/durian.png", 1754 "FileSize": 8414281, 1755 "ModificationTime": 1471985233074909930, 1756 "IsDir": False, 1757 "InodeNumber": 5807979, 1758 "NumWrites": 3, 1759 "Metadata": base64.b64encode(json.dumps({ 1760 mware.ORIGINAL_MD5_HEADER: 1761 "3:34f99f7784c573541e11e5ad66f065c8", 1762 }).encode('ascii')), 1763 }, { 1764 "Basename": "images/elderberry.png", 1765 "FileSize": 3178293, 1766 "ModificationTime": 1471985240833932653, 1767 "IsDir": False, 1768 "InodeNumber": 4974021, 1769 "NumWrites": 1, 1770 "Metadata": "", 1771 }]}} 1772 1773 self.fake_rpc.register_handler( 1774 "Server.RpcGetContainer", mock_RpcGetContainer) 1775 1776 def test_text(self): 1777 req = swob.Request.blank('/v1/AUTH_test/a-container') 1778 status, headers, body = self.call_pfs(req) 1779 1780 self.assertEqual(status, '200 OK') 1781 self.assertEqual(headers["Content-Type"], "text/plain; charset=utf-8") 1782 self.assertEqual(body, (b"images\n" 1783 b"images/\xC3\xA1vocado.png\n" 1784 b"images/banana.png\n" 1785 b"images/cherimoya.png\n" 1786 b"images/durian.png\n" 1787 b"images/elderberry.png\n")) 1788 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1789 '/v1/AUTH_test/a-container') 1790 1791 with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', 1792 True): 1793 status, headers, body = self.call_pfs(req) 1794 self.assertEqual(status, '200 OK') # sanity check 1795 self.assertEqual(headers["Content-Type"], 1796 "application/json; charset=utf-8") 1797 json.loads(body) 1798 1799 def test_dlo(self): 1800 req = swob.Request.blank('/v1/AUTH_test/a-container', 1801 environ={'swift.source': 'DLO'}) 1802 status, headers, body = self.call_pfs(req) 1803 1804 self.assertEqual(status, '200 OK') 1805 self.assertEqual(headers["Content-Type"], 1806 "application/json; charset=utf-8") 1807 json.loads(body) # doesn't crash 1808 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 1809 '/v1/AUTH_test/a-container') 1810 1811 def _check_metadata(self, expected_headers, swift_owner=False): 1812 container_metadata = { 1813 "X-Container-Read": "xcr", 1814 "X-Container-Write": "xcw", 1815 "X-Container-Sync-Key": "sync-key", 1816 "X-Container-Sync-To": "sync-to", 1817 "X-Container-Meta-Temp-Url-Key": "tuk", 1818 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1819 "X-Container-Sysmeta-Fish": "tilefish", 1820 "X-Container-Meta-Fish": "haddock"} 1821 self.serialized_container_metadata = json.dumps(container_metadata) 1822 req = swob.Request.blank('/v1/AUTH_test/a-container', 1823 environ={"REQUEST_METHOD": "GET", 1824 "swift_owner": swift_owner}) 1825 status, headers, body = self.call_pfs(req) 1826 1827 self.assertEqual(status, '200 OK') 1828 self.assertEqual(headers.get("Accept-Ranges"), "bytes") 1829 self.assertEqual(headers["X-Container-Object-Count"], "0") 1830 self.assertEqual(headers["X-Container-Bytes-Used"], "0") 1831 self.assertEqual(headers["X-Storage-Policy"], "default") 1832 self.assertEqual(headers["X-Timestamp"], "1510790796.07604") 1833 self.assertEqual(headers["Last-Modified"], 1834 "Thu, 16 Nov 2017 00:06:37 GMT") 1835 for k, v in expected_headers.items(): 1836 self.assertIn(k, headers) 1837 self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % 1838 (v, headers[k], v)) 1839 for k in container_metadata: 1840 self.assertFalse(k in headers and k not in expected_headers, 1841 "Found unexpected header %r" % k) 1842 1843 def test_metadata_swift_owner(self): 1844 self._check_metadata({ 1845 "X-Container-Read": "xcr", 1846 "X-Container-Write": "xcw", 1847 "X-Container-Sync-Key": "sync-key", 1848 "X-Container-Sync-To": "sync-to", 1849 "X-Container-Meta-Temp-Url-Key": "tuk", 1850 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 1851 "X-Container-Sysmeta-Fish": "tilefish", 1852 "X-Container-Meta-Fish": "haddock"}, 1853 swift_owner=True) 1854 1855 def test_metadata_not_swift_owner(self): 1856 self._check_metadata({ 1857 "X-Container-Sysmeta-Fish": "tilefish", 1858 "X-Container-Meta-Fish": "haddock"}) 1859 1860 def test_bogus_metadata(self): 1861 self.serialized_container_metadata = "{<xml?" 1862 req = swob.Request.blank('/v1/AUTH_test/a-container') 1863 status, headers, body = self.call_pfs(req) 1864 1865 self.assertEqual(status, '200 OK') 1866 1867 def test_json(self): 1868 req = swob.Request.blank('/v1/AUTH_test/a-container', 1869 headers={"Accept": "application/json"}) 1870 status, headers, body = self.call_pfs(req) 1871 1872 self.assertEqual(status, '200 OK') 1873 self.assertEqual(headers["Content-Type"], 1874 "application/json; charset=utf-8") 1875 resp_data = json.loads(body) 1876 self.assertIsInstance(resp_data, list) 1877 self.assertEqual(len(resp_data), 6) 1878 self.assertEqual(resp_data[0], { 1879 "name": "images", 1880 "bytes": 0, 1881 "content_type": "application/directory", 1882 "hash": "d41d8cd98f00b204e9800998ecf8427e", 1883 "last_modified": "2016-08-23T01:30:16.359210"}) 1884 self.assertEqual(resp_data[1], { 1885 "name": u"images/\xE1vocado.png", 1886 "bytes": 3503770, 1887 "content_type": u"snack/m\xEDllenial", 1888 "hash": mware.construct_etag( 1889 "AUTH_test", 9213768, 2), 1890 "last_modified": "2016-08-23T01:30:16.859210"}) 1891 self.assertEqual(resp_data[2], { 1892 "name": "images/banana.png", 1893 "bytes": 2189865, 1894 "content_type": "image/png", 1895 "hash": mware.construct_etag( 1896 "AUTH_test", 8878410, 2), 1897 "last_modified": "2016-08-23T01:31:13.000000"}) 1898 self.assertEqual(resp_data[3], { 1899 "name": "images/cherimoya.png", 1900 "bytes": 1636662, 1901 "content_type": "image/png", 1902 "hash": mware.construct_etag( 1903 "AUTH_test", 8879064, 2), 1904 "last_modified": "2016-08-23T01:31:57.767421"}) 1905 self.assertEqual(resp_data[4], { 1906 "name": "images/durian.png", 1907 "bytes": 8414281, 1908 "content_type": "image/png", 1909 "hash": "34f99f7784c573541e11e5ad66f065c8", 1910 "last_modified": "2016-08-23T20:47:13.074910"}) 1911 self.assertEqual(resp_data[5], { 1912 "name": "images/elderberry.png", 1913 "bytes": 3178293, 1914 "content_type": "image/png", 1915 "hash": mware.construct_etag( 1916 "AUTH_test", 4974021, 1), 1917 "last_modified": "2016-08-23T20:47:20.833933"}) 1918 1919 def test_json_with_delimiter(self): 1920 req = swob.Request.blank( 1921 '/v1/AUTH_test/a-container?prefix=&delimiter=/', 1922 headers={"Accept": "application/json"}) 1923 status, headers, body = self.call_pfs(req) 1924 1925 self.assertEqual(status, '200 OK') 1926 self.assertEqual(headers["Content-Type"], 1927 "application/json; charset=utf-8") 1928 resp_data = json.loads(body) 1929 self.assertIsInstance(resp_data, list) 1930 self.assertEqual(len(resp_data), 7) 1931 self.assertEqual(resp_data[0], { 1932 "name": "images", 1933 "bytes": 0, 1934 "content_type": "application/directory", 1935 "hash": "d41d8cd98f00b204e9800998ecf8427e", 1936 "last_modified": "2016-08-23T01:30:16.359210"}) 1937 self.assertEqual(resp_data[1], { 1938 "subdir": "images/"}) 1939 self.assertEqual(resp_data[2], { 1940 "name": u"images/\xE1vocado.png", 1941 "bytes": 3503770, 1942 "content_type": u"snack/m\xEDllenial", 1943 "hash": mware.construct_etag( 1944 "AUTH_test", 9213768, 2), 1945 "last_modified": "2016-08-23T01:30:16.859210"}) 1946 self.assertEqual(resp_data[3], { 1947 "name": "images/banana.png", 1948 "bytes": 2189865, 1949 "content_type": "image/png", 1950 "hash": mware.construct_etag( 1951 "AUTH_test", 8878410, 2), 1952 "last_modified": "2016-08-23T01:31:13.000000"}) 1953 self.assertEqual(resp_data[4], { 1954 "name": "images/cherimoya.png", 1955 "bytes": 1636662, 1956 "content_type": "image/png", 1957 "hash": mware.construct_etag( 1958 "AUTH_test", 8879064, 2), 1959 "last_modified": "2016-08-23T01:31:57.767421"}) 1960 self.assertEqual(resp_data[5], { 1961 "name": "images/durian.png", 1962 "bytes": 8414281, 1963 "content_type": "image/png", 1964 "hash": "34f99f7784c573541e11e5ad66f065c8", 1965 "last_modified": "2016-08-23T20:47:13.074910"}) 1966 self.assertEqual(resp_data[6], { 1967 "name": "images/elderberry.png", 1968 "bytes": 3178293, 1969 "content_type": "image/png", 1970 "hash": mware.construct_etag( 1971 "AUTH_test", 4974021, 1), 1972 "last_modified": "2016-08-23T20:47:20.833933"}) 1973 1974 def test_json_query_param(self): 1975 req = swob.Request.blank('/v1/AUTH_test/a-container?format=json') 1976 status, headers, body = self.call_pfs(req) 1977 1978 self.assertEqual(status, '200 OK') 1979 self.assertEqual(headers["Content-Type"], 1980 "application/json; charset=utf-8") 1981 json.loads(body) # doesn't crash 1982 1983 # TODO: The tests don't seem to access the code from the same path as 1984 # regular requests, so we're testing a code path that is usually not 1985 # reachable. Should we just get rid of that part of the code? 1986 # For now, delimiter support is non-existent from the tests' point of view 1987 # when requesting XML output. 1988 def test_xml(self): 1989 req = swob.Request.blank('/v1/AUTH_test/a-container', 1990 headers={"Accept": "text/xml"}) 1991 status, headers, body = self.call_pfs(req) 1992 1993 self.assertEqual(status, '200 OK') 1994 self.assertEqual(headers["Content-Type"], 1995 "text/xml; charset=utf-8") 1996 self.assertTrue(body.startswith( 1997 b"""<?xml version='1.0' encoding='utf-8'?>""")) 1998 1999 root_node = ElementTree.fromstring(body) 2000 self.assertEqual(root_node.tag, 'container') 2001 self.assertEqual(root_node.attrib["name"], 'a-container') 2002 2003 objects = list(root_node) 2004 self.assertEqual(6, len(objects)) 2005 self.assertEqual(objects[0].tag, 'object') 2006 2007 # The XML container listing doesn't use XML attributes for data, but 2008 # rather a sequence of tags like <name>X</name> <hash>Y</hash> ... 2009 # 2010 # We do an exhaustive check of one object's attributes, then 2011 # spot-check the rest of the listing for brevity's sake. 2012 obj_attr_tags = list(objects[1]) 2013 self.assertEqual(len(obj_attr_tags), 5) 2014 2015 name_node = obj_attr_tags[0] 2016 self.assertEqual(name_node.tag, 'name') 2017 self.assertEqual(name_node.text, u'images/\xE1vocado.png') 2018 self.assertEqual(name_node.attrib, {}) # nothing extra in there 2019 2020 hash_node = obj_attr_tags[1] 2021 self.assertEqual(hash_node.tag, 'hash') 2022 self.assertEqual(hash_node.text, mware.construct_etag( 2023 "AUTH_test", 9213768, 2)) 2024 self.assertEqual(hash_node.attrib, {}) 2025 2026 bytes_node = obj_attr_tags[2] 2027 self.assertEqual(bytes_node.tag, 'bytes') 2028 self.assertEqual(bytes_node.text, '3503770') 2029 self.assertEqual(bytes_node.attrib, {}) 2030 2031 content_type_node = obj_attr_tags[3] 2032 self.assertEqual(content_type_node.tag, 'content_type') 2033 self.assertEqual(content_type_node.text, u'snack/m\xEDllenial') 2034 self.assertEqual(content_type_node.attrib, {}) 2035 2036 last_modified_node = obj_attr_tags[4] 2037 self.assertEqual(last_modified_node.tag, 'last_modified') 2038 self.assertEqual(last_modified_node.text, '2016-08-23T01:30:16.859210') 2039 self.assertEqual(last_modified_node.attrib, {}) 2040 2041 # Make sure the directory has the right type 2042 obj_attr_tags = list(objects[0]) 2043 self.assertEqual(len(obj_attr_tags), 5) 2044 2045 name_node = obj_attr_tags[0] 2046 self.assertEqual(name_node.tag, 'name') 2047 self.assertEqual(name_node.text, 'images') 2048 2049 hash_node = obj_attr_tags[1] 2050 self.assertEqual(hash_node.tag, 'hash') 2051 self.assertEqual(hash_node.text, "d41d8cd98f00b204e9800998ecf8427e") 2052 self.assertEqual(hash_node.attrib, {}) 2053 2054 content_type_node = obj_attr_tags[3] 2055 self.assertEqual(content_type_node.tag, 'content_type') 2056 self.assertEqual(content_type_node.text, 'application/directory') 2057 2058 # Check the names are correct 2059 all_names = [list(tag)[0].text for tag in objects] 2060 self.assertEqual( 2061 ["images", u"images/\xE1vocado.png", "images/banana.png", 2062 "images/cherimoya.png", "images/durian.png", 2063 "images/elderberry.png"], 2064 all_names) 2065 2066 def test_xml_alternate_mime_type(self): 2067 req = swob.Request.blank('/v1/AUTH_test/a-container', 2068 headers={"Accept": "application/xml"}) 2069 status, headers, body = self.call_pfs(req) 2070 2071 self.assertEqual(status, '200 OK') 2072 self.assertEqual(headers["Content-Type"], 2073 "application/xml; charset=utf-8") 2074 self.assertTrue(body.startswith( 2075 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2076 2077 def test_xml_query_param(self): 2078 req = swob.Request.blank('/v1/AUTH_test/a-container?format=xml') 2079 status, headers, body = self.call_pfs(req) 2080 2081 self.assertEqual(status, '200 OK') 2082 self.assertEqual(headers["Content-Type"], 2083 "application/xml; charset=utf-8") 2084 self.assertTrue(body.startswith( 2085 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2086 2087 def test_xml_special_chars(self): 2088 req = swob.Request.blank('/v1/AUTH_test/c o n', 2089 headers={"Accept": "text/xml"}) 2090 status, headers, body = self.call_pfs(req) 2091 2092 self.assertEqual(status, '200 OK') 2093 self.assertEqual(headers["Content-Type"], 2094 "text/xml; charset=utf-8") 2095 self.assertTrue(body.startswith( 2096 b"""<?xml version='1.0' encoding='utf-8'?>""")) 2097 2098 root_node = ElementTree.fromstring(body) 2099 self.assertEqual(root_node.tag, 'container') 2100 self.assertEqual(root_node.attrib["name"], 'c o n') 2101 self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], 2102 '/v1/AUTH_test/c o n') 2103 2104 def test_marker(self): 2105 req = swob.Request.blank('/v1/AUTH_test/a-container?marker=sharpie') 2106 status, _, _ = self.call_pfs(req) 2107 self.assertEqual(status, '200 OK') 2108 2109 rpc_calls = self.fake_rpc.calls 2110 self.assertEqual(len(rpc_calls), 2) 2111 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2112 # relevant to what we're testing here 2113 rpc_method, rpc_args = rpc_calls[1] 2114 # sanity check 2115 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2116 self.assertEqual(rpc_args[0]["Marker"], "sharpie") 2117 2118 def test_end_marker(self): 2119 req = swob.Request.blank( 2120 '/v1/AUTH_test/a-container?end_marker=whiteboard') 2121 status, _, _ = self.call_pfs(req) 2122 self.assertEqual(status, '200 OK') 2123 2124 rpc_calls = self.fake_rpc.calls 2125 self.assertEqual(len(rpc_calls), 2) 2126 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2127 # relevant to what we're testing here 2128 rpc_method, rpc_args = rpc_calls[1] 2129 # sanity check 2130 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2131 self.assertEqual(rpc_args[0]["EndMarker"], "whiteboard") 2132 2133 def test_prefix(self): 2134 req = swob.Request.blank('/v1/AUTH_test/a-container?prefix=cow') 2135 status, _, _ = self.call_pfs(req) 2136 self.assertEqual(status, '200 OK') 2137 2138 rpc_calls = self.fake_rpc.calls 2139 self.assertEqual(len(rpc_calls), 2) 2140 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2141 # relevant to what we're testing here 2142 rpc_method, rpc_args = rpc_calls[1] 2143 # sanity check 2144 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2145 self.assertEqual(rpc_args[0]["Prefix"], "cow") 2146 2147 def test_delimiter(self): 2148 req = swob.Request.blank('/v1/AUTH_test/a-container?delimiter=/') 2149 status, _, _ = self.call_pfs(req) 2150 self.assertEqual(status, '200 OK') 2151 2152 rpc_calls = self.fake_rpc.calls 2153 self.assertEqual(len(rpc_calls), 2) 2154 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2155 # relevant to what we're testing here 2156 rpc_method, rpc_args = rpc_calls[1] 2157 # sanity check 2158 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2159 self.assertEqual(rpc_args[0]["Delimiter"], "/") 2160 2161 def test_default_limit(self): 2162 req = swob.Request.blank('/v1/AUTH_test/a-container') 2163 status, _, _ = self.call_pfs(req) 2164 self.assertEqual(status, '200 OK') 2165 2166 rpc_calls = self.fake_rpc.calls 2167 self.assertEqual(len(rpc_calls), 2) 2168 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2169 # relevant to what we're testing here 2170 rpc_method, rpc_args = rpc_calls[1] 2171 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) 2172 2173 def test_valid_user_supplied_limit(self): 2174 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=150') 2175 status, _, _ = self.call_pfs(req) 2176 self.assertEqual(status, '200 OK') 2177 2178 rpc_calls = self.fake_rpc.calls 2179 self.assertEqual(len(rpc_calls), 2) 2180 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2181 # relevant to what we're testing here 2182 rpc_method, rpc_args = rpc_calls[1] 2183 self.assertEqual(rpc_args[0]["MaxEntries"], 150) 2184 2185 def test_zero_supplied_limit(self): 2186 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=0') 2187 status, _, _ = self.call_pfs(req) 2188 self.assertEqual(status, '200 OK') 2189 2190 rpc_calls = self.fake_rpc.calls 2191 self.assertEqual(len(rpc_calls), 2) 2192 2193 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2194 # relevant to what we're testing here 2195 rpc_method, rpc_args = rpc_calls[1] 2196 self.assertEqual(rpc_args[0]["MaxEntries"], 0) 2197 2198 def test_negative_supplied_limit(self): 2199 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=-1') 2200 status, _, _ = self.call_pfs(req) 2201 self.assertEqual(status, '200 OK') 2202 2203 rpc_calls = self.fake_rpc.calls 2204 self.assertEqual(len(rpc_calls), 2) 2205 2206 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2207 # relevant to what we're testing here 2208 rpc_method, rpc_args = rpc_calls[1] 2209 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2210 2211 def test_overlarge_supplied_limits(self): 2212 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=6544') 2213 status, _, _ = self.call_pfs(req) 2214 self.assertEqual(status, '412 Precondition Failed') 2215 2216 rpc_calls = self.fake_rpc.calls 2217 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2218 # relevant to what we're testing here 2219 self.assertEqual(len(rpc_calls), 1) 2220 2221 def test_bogus_user_supplied_limit(self): 2222 req = swob.Request.blank('/v1/AUTH_test/a-container?limit=chihuahua') 2223 status, _, _ = self.call_pfs(req) 2224 self.assertEqual(status, '200 OK') 2225 2226 rpc_calls = self.fake_rpc.calls 2227 self.assertEqual(len(rpc_calls), 2) 2228 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2229 # relevant to what we're testing here 2230 rpc_method, rpc_args = rpc_calls[1] 2231 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2232 2233 def test_default_limit_matches_proxy_server(self): 2234 req = swob.Request.blank('/v1/AUTH_test/a-container') 2235 status, _, _ = self.call_pfs(req) 2236 self.assertEqual(status, '200 OK') 2237 2238 rpc_calls = self.fake_rpc.calls 2239 self.assertEqual(len(rpc_calls), 2) 2240 # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not 2241 # relevant to what we're testing here 2242 rpc_method, rpc_args = rpc_calls[1] 2243 self.assertEqual(rpc_method, "Server.RpcGetContainer") 2244 self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value 2245 2246 def test_GET_bad_path(self): 2247 bad_paths = [ 2248 '/v1/AUTH_test/..', 2249 '/v1/AUTH_test/.', 2250 '/v1/AUTH_test/' + ('x' * 256), 2251 ] 2252 for path in bad_paths: 2253 req = swob.Request.blank(path) 2254 status, headers, body = self.call_pfs(req) 2255 self.assertEqual(status, '404 Not Found', 2256 'Got %s for %s' % (status, path)) 2257 2258 req.environ['REQUEST_METHOD'] = 'HEAD' 2259 status, headers, body = self.call_pfs(req) 2260 self.assertEqual(status, '404 Not Found', 2261 'Got %s for %s' % (status, path)) 2262 2263 def test_not_found(self): 2264 def mock_RpcGetContainer_error(get_container_req): 2265 self.assertEqual(get_container_req['VirtPath'], 2266 '/v1/AUTH_test/a-container') 2267 2268 return {"error": "errno: 2", "result": None} 2269 2270 self.fake_rpc.register_handler( 2271 "Server.RpcGetContainer", mock_RpcGetContainer_error) 2272 2273 req = swob.Request.blank('/v1/AUTH_test/a-container') 2274 status, _, _ = self.call_pfs(req) 2275 self.assertEqual(status, '404 Not Found') 2276 2277 def test_other_error(self): 2278 def mock_RpcGetContainer_error(get_container_req): 2279 self.assertEqual(get_container_req['VirtPath'], 2280 '/v1/AUTH_test/a-container') 2281 2282 return {"error": "errno: 1661", "result": None} 2283 2284 self.fake_rpc.register_handler( 2285 "Server.RpcGetContainer", mock_RpcGetContainer_error) 2286 2287 req = swob.Request.blank('/v1/AUTH_test/a-container') 2288 status, _, _ = self.call_pfs(req) 2289 self.assertEqual(status, '500 Internal Error') 2290 2291 2292 class TestContainerGetDelimiter(BaseMiddlewareTest): 2293 def setUp(self): 2294 super(TestContainerGetDelimiter, self).setUp() 2295 self.serialized_container_metadata = "" 2296 2297 def _mock_RpcGetContainerDelimiterNoTrailingSlash(self, get_container_req): 2298 return { 2299 "error": None, 2300 "result": { 2301 "Metadata": base64.b64encode( 2302 self.serialized_container_metadata.encode('ascii')), 2303 "ModificationTime": 1510790796076041000, 2304 "ContainerEntries": [{ 2305 "Basename": "images", 2306 "FileSize": 0, 2307 "ModificationTime": 1471915816359209849, 2308 "IsDir": True, 2309 "InodeNumber": 2489682, 2310 "NumWrites": 0, 2311 "Metadata": "", 2312 }]}} 2313 2314 def _mock_RpcGetContainerDelimiterWithTrailingSlash(self, 2315 get_container_req): 2316 return { 2317 "error": None, 2318 "result": { 2319 "Metadata": base64.b64encode( 2320 self.serialized_container_metadata.encode('ascii')), 2321 "ModificationTime": 1510790796076041000, 2322 "ContainerEntries": [{ 2323 "Basename": "images/avocado.png", 2324 "FileSize": 70, 2325 "ModificationTime": 1471915816859209471, 2326 "IsDir": False, 2327 "InodeNumber": 9213768, 2328 "NumWrites": 2, 2329 "Metadata": base64.b64encode(json.dumps({ 2330 "Content-Type": "snack/millenial" + 2331 ";swift_bytes=3503770", 2332 }).encode('ascii')), 2333 }, { 2334 "Basename": "images/banana.png", 2335 "FileSize": 2189865, 2336 # has fractional seconds = 0 to cover edge cases 2337 "ModificationTime": 1471915873000000000, 2338 "IsDir": False, 2339 "InodeNumber": 8878410, 2340 "NumWrites": 2, 2341 "Metadata": "", 2342 }, { 2343 "Basename": "images/cherimoya.png", 2344 "FileSize": 1636662, 2345 "ModificationTime": 1471915917767421311, 2346 "IsDir": False, 2347 "InodeNumber": 8879064, 2348 "NumWrites": 2, 2349 # Note: this has NumWrites=2, but the original MD5 2350 # starts with "1:", so it is stale and must not be 2351 # used. 2352 "Metadata": base64.b64encode(json.dumps({ 2353 mware.ORIGINAL_MD5_HEADER: 2354 "1:552528fbf2366f8a4711ac0a3875188b", 2355 }).encode('ascii')), 2356 }, { 2357 "Basename": "images/durian.png", 2358 "FileSize": 8414281, 2359 "ModificationTime": 1471985233074909930, 2360 "IsDir": False, 2361 "InodeNumber": 5807979, 2362 "NumWrites": 3, 2363 "Metadata": base64.b64encode(json.dumps({ 2364 mware.ORIGINAL_MD5_HEADER: 2365 "3:34f99f7784c573541e11e5ad66f065c8", 2366 }).encode('ascii')), 2367 }, { 2368 "Basename": "images/elderberry.png", 2369 "FileSize": 3178293, 2370 "ModificationTime": 1471985240833932653, 2371 "IsDir": False, 2372 "InodeNumber": 4974021, 2373 "NumWrites": 1, 2374 "Metadata": "", 2375 }]}} 2376 2377 def test_json_no_trailing_slash(self): 2378 self.fake_rpc.register_handler( 2379 "Server.RpcGetContainer", 2380 self._mock_RpcGetContainerDelimiterNoTrailingSlash) 2381 2382 req = swob.Request.blank( 2383 '/v1/AUTH_test/a-container?prefix=images&delimiter=/', 2384 headers={"Accept": "application/json"}) 2385 status, headers, body = self.call_pfs(req) 2386 2387 self.assertEqual(status, '200 OK') 2388 self.assertEqual(headers["Content-Type"], 2389 "application/json; charset=utf-8") 2390 resp_data = json.loads(body) 2391 self.assertIsInstance(resp_data, list) 2392 self.assertEqual(len(resp_data), 2) 2393 self.assertEqual(resp_data[0], { 2394 "name": "images", 2395 "bytes": 0, 2396 "content_type": "application/directory", 2397 "hash": "d41d8cd98f00b204e9800998ecf8427e", 2398 "last_modified": "2016-08-23T01:30:16.359210"}) 2399 self.assertEqual(resp_data[1], { 2400 "subdir": "images/"}) 2401 2402 def test_json_with_trailing_slash(self): 2403 self.fake_rpc.register_handler( 2404 "Server.RpcGetContainer", 2405 self._mock_RpcGetContainerDelimiterWithTrailingSlash) 2406 2407 req = swob.Request.blank( 2408 '/v1/AUTH_test/a-container?prefix=images/&delimiter=/', 2409 headers={"Accept": "application/json"}) 2410 status, headers, body = self.call_pfs(req) 2411 2412 self.assertEqual(status, '200 OK') 2413 self.assertEqual(headers["Content-Type"], 2414 "application/json; charset=utf-8") 2415 resp_data = json.loads(body) 2416 self.assertIsInstance(resp_data, list) 2417 self.assertEqual(len(resp_data), 5) 2418 self.assertEqual(resp_data[0], { 2419 "name": "images/avocado.png", 2420 "bytes": 3503770, 2421 "content_type": "snack/millenial", 2422 "hash": mware.construct_etag( 2423 "AUTH_test", 9213768, 2), 2424 "last_modified": "2016-08-23T01:30:16.859210"}) 2425 self.assertEqual(resp_data[1], { 2426 "name": "images/banana.png", 2427 "bytes": 2189865, 2428 "content_type": "image/png", 2429 "hash": mware.construct_etag( 2430 "AUTH_test", 8878410, 2), 2431 "last_modified": "2016-08-23T01:31:13.000000"}) 2432 self.assertEqual(resp_data[2], { 2433 "name": "images/cherimoya.png", 2434 "bytes": 1636662, 2435 "content_type": "image/png", 2436 "hash": mware.construct_etag( 2437 "AUTH_test", 8879064, 2), 2438 "last_modified": "2016-08-23T01:31:57.767421"}) 2439 self.assertEqual(resp_data[3], { 2440 "name": "images/durian.png", 2441 "bytes": 8414281, 2442 "content_type": "image/png", 2443 "hash": "34f99f7784c573541e11e5ad66f065c8", 2444 "last_modified": "2016-08-23T20:47:13.074910"}) 2445 self.assertEqual(resp_data[4], { 2446 "name": "images/elderberry.png", 2447 "bytes": 3178293, 2448 "content_type": "image/png", 2449 "hash": mware.construct_etag( 2450 "AUTH_test", 4974021, 1), 2451 "last_modified": "2016-08-23T20:47:20.833933"}) 2452 2453 2454 class TestContainerPost(BaseMiddlewareTest): 2455 def test_missing_container(self): 2456 def mock_RpcHead(_): 2457 return {"error": "errno: 2", "result": None} 2458 2459 self.fake_rpc.register_handler( 2460 "Server.RpcHead", mock_RpcHead) 2461 2462 req = swob.Request.blank( 2463 "/v1/AUTH_test/new-con", 2464 environ={"REQUEST_METHOD": "POST"}, 2465 headers={"X-Container-Meta-One-Fish": "two fish"}) 2466 status, _, _ = self.call_pfs(req) 2467 self.assertEqual("404 Not Found", status) 2468 2469 rpc_calls = self.fake_rpc.calls 2470 self.assertEqual(2, len(rpc_calls)) 2471 2472 method, args = rpc_calls[0] 2473 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2474 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2475 2476 method, args = rpc_calls[1] 2477 self.assertEqual(method, "Server.RpcHead") 2478 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2479 2480 def test_existing_container(self): 2481 old_meta = json.dumps({ 2482 "X-Container-Read": "xcr", 2483 "X-Container-Meta-One-Fish": "no fish"}) 2484 2485 def mock_RpcHead(_): 2486 return { 2487 "error": None, 2488 "result": { 2489 "Metadata": base64.b64encode( 2490 old_meta.encode('ascii')).decode('ascii'), 2491 "ModificationTime": 1482280565956671142, 2492 "FileSize": 0, 2493 "IsDir": True, 2494 "InodeNumber": 6515443, 2495 "NumWrites": 0}} 2496 2497 self.fake_rpc.register_handler( 2498 "Server.RpcHead", mock_RpcHead) 2499 2500 self.fake_rpc.register_handler( 2501 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 2502 2503 req = swob.Request.blank( 2504 "/v1/AUTH_test/new-con", 2505 environ={"REQUEST_METHOD": "POST"}, 2506 headers={"X-Container-Meta-One-Fish": "two fish"}) 2507 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2508 status, _, _ = self.call_pfs(req) 2509 self.assertEqual("204 No Content", status) 2510 cic.assert_called() 2511 2512 rpc_calls = self.fake_rpc.calls 2513 self.assertEqual(3, len(rpc_calls)) 2514 2515 method, args = rpc_calls[0] 2516 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2517 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2518 2519 method, args = rpc_calls[1] 2520 self.assertEqual(method, "Server.RpcHead") 2521 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2522 2523 method, args = rpc_calls[2] 2524 self.assertEqual(method, "Server.RpcPost") 2525 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2526 self.assertEqual( 2527 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 2528 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 2529 self.assertEqual(new_meta["X-Container-Meta-One-Fish"], "two fish") 2530 self.assertEqual(new_meta["X-Container-Read"], "xcr") 2531 2532 2533 class TestContainerPut(BaseMiddlewareTest): 2534 def setUp(self): 2535 super(TestContainerPut, self).setUp() 2536 2537 # This only returns success/failure, not any interesting data 2538 def mock_RpcPutContainer(_): 2539 return {"error": None, "result": {}} 2540 2541 self.fake_rpc.register_handler( 2542 "Server.RpcPutContainer", mock_RpcPutContainer) 2543 2544 def test_new_container(self): 2545 def mock_RpcHead(_): 2546 return {"error": "errno: 2", "result": None} 2547 2548 self.fake_rpc.register_handler( 2549 "Server.RpcHead", mock_RpcHead) 2550 2551 req = swob.Request.blank( 2552 "/v1/AUTH_test/new-con", 2553 environ={"REQUEST_METHOD": "PUT"}, 2554 headers={"X-Container-Meta-Red-Fish": "blue fish"}) 2555 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2556 status, _, _ = self.call_pfs(req) 2557 self.assertEqual("201 Created", status) 2558 cic.assert_called() 2559 2560 rpc_calls = self.fake_rpc.calls 2561 self.assertEqual(3, len(rpc_calls)) 2562 2563 method, args = rpc_calls[0] 2564 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2565 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2566 2567 method, args = rpc_calls[1] 2568 self.assertEqual(method, "Server.RpcHead") 2569 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2570 2571 method, args = rpc_calls[2] 2572 self.assertEqual(method, "Server.RpcPutContainer") 2573 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2574 self.assertEqual(args[0]["OldMetadata"], "") 2575 self.assertEqual( 2576 base64.b64decode(args[0]["NewMetadata"]).decode('ascii'), 2577 json.dumps({"X-Container-Meta-Red-Fish": "blue fish"})) 2578 2579 def test_PUT_bad_path(self): 2580 bad_container_paths = [ 2581 '/v1/AUTH_test/..', 2582 '/v1/AUTH_test/.', 2583 '/v1/AUTH_test/' + ('x' * 256), 2584 ] 2585 for path in bad_container_paths: 2586 req = swob.Request.blank(path, 2587 environ={"REQUEST_METHOD": "PUT", 2588 "wsgi.input": BytesIO(b""), 2589 "CONTENT_LENGTH": "0"}) 2590 status, headers, body = self.call_pfs(req) 2591 self.assertEqual(status, '400 Bad Request', 2592 'Got %s for %s' % (status, path)) 2593 2594 req.environ['REQUEST_METHOD'] = 'POST' 2595 status, headers, body = self.call_pfs(req) 2596 self.assertEqual(status, '404 Not Found', 2597 'Got %s for %s' % (status, path)) 2598 2599 req.environ['REQUEST_METHOD'] = 'DELETE' 2600 status, headers, body = self.call_pfs(req) 2601 self.assertEqual(status, '404 Not Found', 2602 'Got %s for %s' % (status, path)) 2603 2604 def test_name_too_long(self): 2605 def mock_RpcHead(_): 2606 return {"error": "errno: 2", "result": None} 2607 2608 self.fake_rpc.register_handler( 2609 "Server.RpcHead", mock_RpcHead) 2610 2611 self.swift_info['swift']['max_container_name_length'] = 50 2612 acceptable_name = 'A' * ( 2613 self.swift_info['swift']['max_container_name_length']) 2614 too_long_name = 'A' * ( 2615 self.swift_info['swift']['max_container_name_length'] + 1) 2616 self.app.register( 2617 'GET', '/info', 2618 200, {'Content-Type': 'application/json'}, 2619 json.dumps(self.swift_info)) 2620 2621 req = swob.Request.blank( 2622 "/v1/AUTH_test/%s" % acceptable_name, 2623 environ={"REQUEST_METHOD": "PUT"}) 2624 status, _, _ = self.call_pfs(req) 2625 self.assertEqual("201 Created", status) 2626 2627 req = swob.Request.blank( 2628 "/v1/AUTH_test/%s" % too_long_name, 2629 environ={"REQUEST_METHOD": "PUT"}) 2630 status, _, _ = self.call_pfs(req) 2631 self.assertEqual("400 Bad Request", status) 2632 2633 def test_name_too_long_posix(self): 2634 def mock_RpcHead(_): 2635 return {"error": "errno: 2", "result": None} 2636 2637 self.fake_rpc.register_handler( 2638 "Server.RpcHead", mock_RpcHead) 2639 2640 posix_limit = mware.NAME_MAX 2641 self.swift_info['swift']['max_container_name_length'] = posix_limit * 2 2642 self.app.register( 2643 'GET', '/info', 2644 200, {'Content-Type': 'application/json'}, 2645 json.dumps(self.swift_info)) 2646 2647 too_long_name = 'A' * (posix_limit + 1) 2648 req = swob.Request.blank( 2649 "/v1/AUTH_test/%s" % too_long_name, 2650 environ={"REQUEST_METHOD": "PUT"}) 2651 status, _, _ = self.call_pfs(req) 2652 self.assertEqual("400 Bad Request", status) 2653 2654 def _check_existing_container(self, req_headers, expected_meta, 2655 swift_owner=False): 2656 old_meta = json.dumps({ 2657 "X-Container-Read": "xcr", 2658 "X-Container-Write": "xcw", 2659 "X-Container-Sync-Key": "sync-key", 2660 "X-Container-Sync-To": "sync-to", 2661 "X-Container-Meta-Temp-Url-Key": "tuk", 2662 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 2663 "X-Container-Meta-Red-Fish": "dead fish"}) 2664 2665 def mock_RpcHead(_): 2666 return { 2667 "error": None, 2668 "result": { 2669 "Metadata": base64.b64encode( 2670 old_meta.encode('ascii')).decode('ascii'), 2671 "ModificationTime": 1482270529646747881, 2672 "FileSize": 0, 2673 "IsDir": True, 2674 "InodeNumber": 8054914, 2675 "NumWrites": 0}} 2676 2677 self.fake_rpc.register_handler( 2678 "Server.RpcHead", mock_RpcHead) 2679 2680 req = swob.Request.blank( 2681 "/v1/AUTH_test/new-con", 2682 environ={"REQUEST_METHOD": "PUT", 'swift_owner': swift_owner}, 2683 headers=req_headers) 2684 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2685 status, _, _ = self.call_pfs(req) 2686 self.assertEqual("202 Accepted", status) 2687 cic.assert_called() 2688 2689 rpc_calls = self.fake_rpc.calls 2690 self.assertEqual(3, len(rpc_calls)) 2691 2692 method, args = rpc_calls[0] 2693 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2694 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2695 2696 method, args = rpc_calls[1] 2697 self.assertEqual(method, "Server.RpcHead") 2698 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2699 2700 method, args = rpc_calls[2] 2701 self.assertEqual(method, "Server.RpcPutContainer") 2702 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2703 self.assertEqual( 2704 base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) 2705 new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) 2706 self.assertEqual(expected_meta, new_meta) 2707 2708 def test_existing_container_swift_owner(self): 2709 self._check_existing_container( 2710 {"X-Container-Read": "xcr-new", 2711 "X-Container-Write": "xcw-new", 2712 "X-Container-Sync-Key": "sync-key-new", 2713 "X-Container-Sync-To": "sync-to-new", 2714 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2715 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2716 "X-Container-Meta-Red-Fish": "blue fish"}, 2717 {"X-Container-Read": "xcr-new", 2718 "X-Container-Write": "xcw-new", 2719 "X-Container-Sync-Key": "sync-key-new", 2720 "X-Container-Sync-To": "sync-to-new", 2721 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2722 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2723 "X-Container-Meta-Red-Fish": "blue fish"}, 2724 swift_owner=True 2725 ) 2726 2727 def test_existing_container_not_swift_owner(self): 2728 self._check_existing_container( 2729 {"X-Container-Read": "xcr-new", 2730 "X-Container-Write": "xcw-new", 2731 "X-Container-Sync-Key": "sync-key-new", 2732 "X-Container-Sync-To": "sync-to-new", 2733 "X-Container-Meta-Temp-Url-Key": "tuk-new", 2734 "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", 2735 "X-Container-Meta-Red-Fish": "blue fish"}, 2736 {"X-Container-Read": "xcr", 2737 "X-Container-Write": "xcw", 2738 "X-Container-Sync-Key": "sync-key", 2739 "X-Container-Sync-To": "sync-to", 2740 "X-Container-Meta-Temp-Url-Key": "tuk", 2741 "X-Container-Meta-Temp-Url-Key-2": "tuk2", 2742 "X-Container-Meta-Red-Fish": "blue fish"} 2743 ) 2744 2745 def _check_metadata_removal(self, headers, expected_meta, 2746 swift_owner=False): 2747 old_meta = json.dumps({ 2748 "X-Container-Read": "xcr", 2749 "X-Container-Write": "xcw", 2750 "X-Container-Sync-Key": "sync-key", 2751 "X-Container-Sync-To": "sync-to", 2752 "X-Versions-Location": "loc", 2753 "X-Container-Meta-Box": "cardboard" 2754 }) 2755 2756 def mock_RpcHead(_): 2757 return { 2758 "error": None, 2759 "result": { 2760 "Metadata": base64.b64encode( 2761 old_meta.encode('ascii')).decode('ascii'), 2762 "ModificationTime": 1511224700123739000, 2763 "FileSize": 0, 2764 "IsDir": True, 2765 "InodeNumber": 8017342, 2766 "NumWrites": 0}} 2767 2768 self.fake_rpc.register_handler( 2769 "Server.RpcHead", mock_RpcHead) 2770 2771 req = swob.Request.blank( 2772 "/v1/AUTH_test/new-con", 2773 environ={"REQUEST_METHOD": "PUT", "swift_owner": swift_owner}, 2774 headers=headers) 2775 status, _, _ = self.call_pfs(req) 2776 self.assertEqual("202 Accepted", status) 2777 2778 rpc_calls = self.fake_rpc.calls 2779 method, args = rpc_calls[-1] 2780 self.assertEqual(method, "Server.RpcPutContainer") 2781 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") 2782 self.assertEqual( 2783 base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) 2784 new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) 2785 self.assertEqual(expected_meta, new_meta) 2786 2787 def test_metadata_removal_swift_owner(self): 2788 self._check_metadata_removal( 2789 {"X-Container-Read": "", 2790 "X-Container-Write": "", 2791 "X-Container-Sync-Key": "", 2792 "X-Container-Sync-To": "", 2793 "X-Versions-Location": "", 2794 "X-Container-Meta-Box": ""}, 2795 {}, swift_owner=True) 2796 2797 def test_metadata_removal_with_remove_swift_owner(self): 2798 self._check_metadata_removal( 2799 {"X-Remove-Container-Read": "", 2800 "X-Remove-Container-Write": "", 2801 "X-Remove-Container-Sync-Key": "", 2802 "X-Remove-Container-Sync-To": "", 2803 "X-Remove-Versions-Location": "", 2804 "X-Remove-Container-Meta-Box": ""}, 2805 {}, swift_owner=True) 2806 2807 def test_metadata_removal_not_swift_owner(self): 2808 self._check_metadata_removal( 2809 {"X-Container-Read": "", 2810 "X-Container-Write": "", 2811 "X-Container-Sync-Key": "", 2812 "X-Container-Sync-To": "", 2813 "X-Versions-Location": "", 2814 "X-Container-Meta-Box": ""}, 2815 {"X-Container-Read": "xcr", 2816 "X-Container-Write": "xcw", 2817 "X-Container-Sync-Key": "sync-key", 2818 "X-Container-Sync-To": "sync-to"}) 2819 2820 def test_metadata_removal_with_remove_not_swift_owner(self): 2821 self._check_metadata_removal( 2822 {"X-Remove-Container-Read": "", 2823 "X-Remove-Container-Write": "", 2824 "X-Remove-Container-Sync-Key": "", 2825 "X-Remove-Container-Sync-To": "", 2826 "X-Remove-Versions-Location": "", 2827 "X-Remove-Container-Meta-Box": ""}, 2828 {"X-Container-Read": "xcr", 2829 "X-Container-Write": "xcw", 2830 "X-Container-Sync-Key": "sync-key", 2831 "X-Container-Sync-To": "sync-to"}) 2832 2833 2834 class TestContainerDelete(BaseMiddlewareTest): 2835 def test_success(self): 2836 def mock_RpcDelete(_): 2837 return {"error": None, "result": {}} 2838 2839 self.fake_rpc.register_handler( 2840 "Server.RpcDelete", mock_RpcDelete) 2841 2842 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2843 environ={"REQUEST_METHOD": "DELETE"}) 2844 with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: 2845 status, _, _ = self.call_pfs(req) 2846 self.assertEqual("204 No Content", status) 2847 self.assertNotIn("Accept-Ranges", req.headers) 2848 self.assertEqual(2, len(self.fake_rpc.calls)) 2849 self.assertEqual("/v1/AUTH_test/empty-con", 2850 self.fake_rpc.calls[1][1][0]["VirtPath"]) 2851 cic.assert_called() 2852 2853 def test_special_chars(self): 2854 def mock_RpcDelete(_): 2855 return {"error": None, "result": {}} 2856 2857 self.fake_rpc.register_handler( 2858 "Server.RpcDelete", mock_RpcDelete) 2859 2860 req = swob.Request.blank("/v1/AUTH_test/e m p t y", 2861 environ={"REQUEST_METHOD": "DELETE"}) 2862 status, _, _ = self.call_pfs(req) 2863 self.assertEqual("204 No Content", status) 2864 self.assertEqual("/v1/AUTH_test/e m p t y", 2865 self.fake_rpc.calls[1][1][0]["VirtPath"]) 2866 2867 def test_not_found(self): 2868 def mock_RpcDelete(_): 2869 return {"error": "errno: 2", "result": None} 2870 2871 self.fake_rpc.register_handler( 2872 "Server.RpcDelete", mock_RpcDelete) 2873 2874 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2875 environ={"REQUEST_METHOD": "DELETE"}) 2876 status, _, _ = self.call_pfs(req) 2877 self.assertEqual("404 Not Found", status) 2878 2879 def test_not_empty(self): 2880 def mock_RpcDelete(_): 2881 return {"error": "errno: 39", "result": None} 2882 2883 self.fake_rpc.register_handler( 2884 "Server.RpcDelete", mock_RpcDelete) 2885 2886 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2887 environ={"REQUEST_METHOD": "DELETE"}) 2888 status, _, _ = self.call_pfs(req) 2889 self.assertEqual("409 Conflict", status) 2890 2891 def test_other_error(self): 2892 def mock_RpcDelete(_): 2893 return {"error": "errno: 987654321", "result": None} 2894 2895 self.fake_rpc.register_handler( 2896 "Server.RpcDelete", mock_RpcDelete) 2897 2898 req = swob.Request.blank("/v1/AUTH_test/empty-con", 2899 environ={"REQUEST_METHOD": "DELETE"}) 2900 status, _, _ = self.call_pfs(req) 2901 self.assertEqual("500 Internal Error", status) 2902 2903 2904 class TestObjectPut(BaseMiddlewareTest): 2905 def setUp(self): 2906 super(TestObjectPut, self).setUp() 2907 2908 # These mocks act as though everything was successful. Failure tests 2909 # can override the relevant mocks in the individual test cases. 2910 def mock_RpcHead(head_container_req): 2911 # Empty container, but it exists. That's enough for testing 2912 # object PUT. 2913 return { 2914 "error": None, 2915 "result": { 2916 "Metadata": "", 2917 "ModificationTime": 14792389930244718933, 2918 "FileSize": 0, 2919 "IsDir": True, 2920 "InodeNumber": 1828, 2921 "NumWrites": 893, 2922 }} 2923 2924 put_loc_count = collections.defaultdict(int) 2925 2926 def mock_RpcPutLocation(put_location_req): 2927 # Give a different sequence of physical paths for each object 2928 # name 2929 virt_path = put_location_req["VirtPath"] 2930 obj_name = hashlib.sha1( 2931 virt_path.encode('utf8')).hexdigest().upper() 2932 phys_path = "/v1/AUTH_test/PhysContainer_1/" + obj_name 2933 if put_loc_count[virt_path] > 0: 2934 phys_path += "-%02x" % put_loc_count[virt_path] 2935 put_loc_count[virt_path] += 1 2936 2937 # Someone's probably about to PUT an object there, so let's set 2938 # up the mock to allow it. Doing it here ensures that the 2939 # location comes out of this RPC and nowhere else. 2940 2941 self.app.register('PUT', phys_path, 201, {}, "") 2942 2943 return { 2944 "error": None, 2945 "result": {"PhysPath": phys_path}} 2946 2947 def mock_RpcPutComplete(put_complete_req): 2948 return {"error": None, "result": { 2949 "ModificationTime": 12345, 2950 "InodeNumber": 678, 2951 "NumWrites": 9}} 2952 2953 def mock_RpcMiddlewareMkdir(middleware_mkdir_req): 2954 return {"error": None, "result": { 2955 "ModificationTime": 1504652321749543000, 2956 "InodeNumber": 9268022, 2957 "NumWrites": 0}} 2958 2959 self.fake_rpc.register_handler( 2960 "Server.RpcHead", mock_RpcHead) 2961 self.fake_rpc.register_handler( 2962 "Server.RpcPutLocation", mock_RpcPutLocation) 2963 self.fake_rpc.register_handler( 2964 "Server.RpcPutComplete", mock_RpcPutComplete) 2965 self.fake_rpc.register_handler( 2966 "Server.RpcMiddlewareMkdir", mock_RpcMiddlewareMkdir) 2967 2968 def test_basic(self): 2969 wsgi_input = BytesIO(b"sparkleberry-displeasurably") 2970 cl = str(len(wsgi_input.getvalue())) 2971 2972 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 2973 environ={"REQUEST_METHOD": "PUT", 2974 "wsgi.input": wsgi_input, 2975 "CONTENT_LENGTH": cl}) 2976 status, headers, body = self.call_pfs(req) 2977 self.assertEqual(status, '201 Created') 2978 self.assertEqual(headers["ETag"], 2979 hashlib.md5(wsgi_input.getvalue()).hexdigest()) 2980 2981 rpc_calls = self.fake_rpc.calls 2982 self.assertEqual(len(rpc_calls), 4) 2983 2984 method, args = rpc_calls[0] 2985 self.assertEqual(method, "Server.RpcIsAccountBimodal") 2986 self.assertEqual(args[0]["AccountName"], "AUTH_test") 2987 2988 method, args = rpc_calls[1] 2989 self.assertEqual(method, "Server.RpcHead") 2990 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/a-container") 2991 2992 method, args = rpc_calls[2] 2993 self.assertEqual(method, "Server.RpcPutLocation") 2994 self.assertEqual(args[0]["VirtPath"], 2995 "/v1/AUTH_test/a-container/an-object") 2996 2997 method, args = rpc_calls[3] 2998 expected_phys_path = ("/v1/AUTH_test/PhysContainer_1/" 2999 "80D184B041B9BF0C2EE8D55D8DC9797BF7129E13") 3000 self.assertEqual(method, "Server.RpcPutComplete") 3001 self.assertEqual(args[0]["VirtPath"], 3002 "/v1/AUTH_test/a-container/an-object") 3003 self.assertEqual(args[0]["PhysPaths"], [expected_phys_path]) 3004 self.assertEqual(args[0]["PhysLengths"], [len(wsgi_input.getvalue())]) 3005 3006 def test_directory(self): 3007 req = swob.Request.blank( 3008 "/v1/AUTH_test/a-container/a-dir", 3009 environ={"REQUEST_METHOD": "PUT"}, 3010 headers={"Content-Length": 0, 3011 "Content-Type": "application/directory", 3012 "X-Object-Sysmeta-Abc": "DEF"}, 3013 body="") 3014 3015 # directories always return the hard coded value for EMPTY_OBJECT_ETAG 3016 status, headers, body = self.call_pfs(req) 3017 self.assertEqual(status, '201 Created') 3018 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 3019 3020 rpc_calls = self.fake_rpc.calls 3021 self.assertEqual(len(rpc_calls), 3) 3022 3023 method, args = rpc_calls[2] 3024 self.assertEqual(method, "Server.RpcMiddlewareMkdir") 3025 self.assertEqual(args[0]["VirtPath"], 3026 "/v1/AUTH_test/a-container/a-dir") 3027 3028 serialized_metadata = args[0]["Metadata"] 3029 metadata = json.loads(base64.b64decode(serialized_metadata)) 3030 self.assertEqual(metadata.get("X-Object-Sysmeta-Abc"), "DEF") 3031 3032 def test_modification_time(self): 3033 def mock_RpcPutComplete(put_complete_req): 3034 return {"error": None, "result": { 3035 "ModificationTime": 1481311245635845000, 3036 "InodeNumber": 4116394, 3037 "NumWrites": 1}} 3038 3039 self.fake_rpc.register_handler( 3040 "Server.RpcPutComplete", mock_RpcPutComplete) 3041 3042 wsgi_input = BytesIO(b"Rhodothece-cholesterinuria") 3043 cl = str(len(wsgi_input.getvalue())) 3044 3045 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3046 environ={"REQUEST_METHOD": "PUT", 3047 "wsgi.input": wsgi_input, 3048 "CONTENT_LENGTH": cl}) 3049 status, headers, body = self.call_pfs(req) 3050 self.assertEqual(status, "201 Created") 3051 self.assertEqual(headers["Last-Modified"], 3052 "Fri, 09 Dec 2016 19:20:46 GMT") 3053 3054 def test_special_chars(self): 3055 wsgi_input = BytesIO(b"pancreas-mystagogically") 3056 cl = str(len(wsgi_input.getvalue())) 3057 3058 req = swob.Request.blank("/v1/AUTH_test/c o n/o b j", 3059 environ={"REQUEST_METHOD": "PUT", 3060 "wsgi.input": wsgi_input, 3061 "CONTENT_LENGTH": cl}) 3062 3063 status, headers, body = self.call_pfs(req) 3064 self.assertEqual(status, '201 Created') 3065 3066 rpc_calls = self.fake_rpc.calls 3067 self.assertEqual(len(rpc_calls), 4) 3068 3069 method, args = rpc_calls[0] 3070 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3071 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3072 3073 method, args = rpc_calls[1] 3074 self.assertEqual(method, "Server.RpcHead") 3075 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/c o n") 3076 3077 method, args = rpc_calls[2] 3078 self.assertEqual(method, "Server.RpcPutLocation") 3079 self.assertEqual(args[0]["VirtPath"], 3080 "/v1/AUTH_test/c o n/o b j") 3081 3082 method, args = rpc_calls[3] 3083 self.assertEqual(method, "Server.RpcPutComplete") 3084 self.assertEqual(args[0]["VirtPath"], 3085 "/v1/AUTH_test/c o n/o b j") 3086 3087 def test_PUT_bad_path(self): 3088 bad_container_paths = [ 3089 '/v1/AUTH_test/../o', 3090 '/v1/AUTH_test/./o', 3091 '/v1/AUTH_test/' + ('x' * 256) + '/o', 3092 ] 3093 for path in bad_container_paths: 3094 req = swob.Request.blank(path, 3095 environ={"REQUEST_METHOD": "PUT", 3096 "wsgi.input": BytesIO(b""), 3097 "CONTENT_LENGTH": "0"}) 3098 status, headers, body = self.call_pfs(req) 3099 self.assertEqual(status, '404 Not Found', 3100 'Got %s for %s' % (status, path)) 3101 3102 req.environ['REQUEST_METHOD'] = 'POST' 3103 status, headers, body = self.call_pfs(req) 3104 self.assertEqual(status, '404 Not Found', 3105 'Got %s for %s' % (status, path)) 3106 3107 req.environ['REQUEST_METHOD'] = 'DELETE' 3108 status, headers, body = self.call_pfs(req) 3109 self.assertEqual(status, '404 Not Found', 3110 'Got %s for %s' % (status, path)) 3111 3112 bad_paths = [ 3113 '/v1/AUTH_test/c/..', 3114 '/v1/AUTH_test/c/../o', 3115 '/v1/AUTH_test/c/o/..', 3116 '/v1/AUTH_test/c/.', 3117 '/v1/AUTH_test/c/./o', 3118 '/v1/AUTH_test/c/o/.', 3119 '/v1/AUTH_test/c//o', 3120 '/v1/AUTH_test/c/o//', 3121 '/v1/AUTH_test/c/o/', 3122 '/v1/AUTH_test/c/' + ('x' * 256), 3123 ] 3124 for path in bad_paths: 3125 req = swob.Request.blank(path, 3126 environ={"REQUEST_METHOD": "PUT", 3127 "wsgi.input": BytesIO(b""), 3128 "CONTENT_LENGTH": "0"}) 3129 status, headers, body = self.call_pfs(req) 3130 self.assertEqual(status, '400 Bad Request', 3131 'Got %s for %s' % (status, path)) 3132 3133 req.environ['REQUEST_METHOD'] = 'POST' 3134 status, headers, body = self.call_pfs(req) 3135 self.assertEqual(status, '404 Not Found', 3136 'Got %s for %s' % (status, path)) 3137 3138 req.environ['REQUEST_METHOD'] = 'DELETE' 3139 status, headers, body = self.call_pfs(req) 3140 self.assertEqual(status, '404 Not Found', 3141 'Got %s for %s' % (status, path)) 3142 3143 def test_big(self): 3144 wsgi_input = BytesIO(b'A' * 100 + b'B' * 100 + b'C' * 75) 3145 self.pfs.max_log_segment_size = 100 3146 3147 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3148 environ={"REQUEST_METHOD": "PUT", 3149 "wsgi.input": wsgi_input}, 3150 headers={"X-Trans-Id": "big-txid", 3151 "Content-Length": "275"}) 3152 status, headers, body = self.call_pfs(req) 3153 self.assertEqual(status, '201 Created') 3154 self.assertEqual(headers["Content-Type"], "application/octet-stream") 3155 3156 rpc_calls = self.fake_rpc.calls 3157 self.assertEqual(len(rpc_calls), 6) 3158 3159 method, args = rpc_calls[0] 3160 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3161 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3162 3163 method, args = rpc_calls[1] 3164 self.assertEqual(method, "Server.RpcHead") 3165 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") 3166 3167 # 3 calls to RpcPutLocation since this was spread across 3 log 3168 # segments 3169 method, args = rpc_calls[2] 3170 self.assertEqual(method, "Server.RpcPutLocation") 3171 self.assertEqual(args[0]["VirtPath"], 3172 "/v1/AUTH_test/con/obj") 3173 3174 method, args = rpc_calls[3] 3175 self.assertEqual(method, "Server.RpcPutLocation") 3176 self.assertEqual(args[0]["VirtPath"], 3177 "/v1/AUTH_test/con/obj") 3178 3179 method, args = rpc_calls[4] 3180 self.assertEqual(method, "Server.RpcPutLocation") 3181 self.assertEqual(args[0]["VirtPath"], 3182 "/v1/AUTH_test/con/obj") 3183 3184 method, args = rpc_calls[5] 3185 self.assertEqual(method, "Server.RpcPutComplete") 3186 self.assertEqual(args[0]["VirtPath"], 3187 "/v1/AUTH_test/con/obj") 3188 pre = "/v1/AUTH_test/PhysContainer_1/" 3189 self.assertEqual( 3190 args[0]["PhysPaths"], 3191 [pre + "1550057D8B0039185EB6184C599C940E51953403", 3192 pre + "1550057D8B0039185EB6184C599C940E51953403-01", 3193 pre + "1550057D8B0039185EB6184C599C940E51953403-02"]) 3194 self.assertEqual(args[0]["PhysLengths"], [100, 100, 75]) 3195 3196 # check the txids as well 3197 put_calls = [c for c in self.app.calls if c[0] == 'PUT'] 3198 self.assertEqual( 3199 "big-txid-000", put_calls[0][2]["X-Trans-Id"]) # 1st PUT 3200 self.assertEqual( 3201 "big-txid-001", put_calls[1][2]["X-Trans-Id"]) # 2nd PUT 3202 self.assertEqual( 3203 "big-txid-002", put_calls[2][2]["X-Trans-Id"]) # 3rd PUT 3204 3205 # If we sent the original Content-Length, the first PUT would fail. 3206 # At some point, we should send the correct Content-Length value 3207 # when we can compute it, but for now, we just send nothing. 3208 self.assertNotIn("Content-Length", put_calls[0][2]) # 1st PUT 3209 self.assertNotIn("Content-Length", put_calls[1][2]) # 2nd PUT 3210 self.assertNotIn("Content-Length", put_calls[2][2]) # 3rd PUT 3211 3212 def test_big_exact_multiple(self): 3213 wsgi_input = BytesIO(b'A' * 100 + b'B' * 100) 3214 cl = str(len(wsgi_input.getvalue())) 3215 self.pfs.max_log_segment_size = 100 3216 3217 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3218 environ={"REQUEST_METHOD": "PUT", 3219 "wsgi.input": wsgi_input, 3220 "CONTENT_LENGTH": cl}, 3221 headers={"X-Trans-Id": "big-txid"}) 3222 status, headers, body = self.call_pfs(req) 3223 self.assertEqual(status, '201 Created') 3224 self.assertEqual(headers["Content-Type"], "application/octet-stream") 3225 3226 rpc_calls = self.fake_rpc.calls 3227 self.assertEqual(len(rpc_calls), 5) 3228 3229 method, args = rpc_calls[0] 3230 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3231 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3232 3233 method, args = rpc_calls[1] 3234 self.assertEqual(method, "Server.RpcHead") 3235 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") 3236 3237 # 2 calls to RpcPutLocation since this was spread across 2 log 3238 # segments. We didn't make a 0-length log segment and try to splice 3239 # that in. 3240 method, args = rpc_calls[2] 3241 self.assertEqual(method, "Server.RpcPutLocation") 3242 self.assertEqual(args[0]["VirtPath"], 3243 "/v1/AUTH_test/con/obj") 3244 3245 method, args = rpc_calls[3] 3246 self.assertEqual(method, "Server.RpcPutLocation") 3247 self.assertEqual(args[0]["VirtPath"], 3248 "/v1/AUTH_test/con/obj") 3249 3250 method, args = rpc_calls[4] 3251 self.assertEqual(method, "Server.RpcPutComplete") 3252 self.assertEqual(args[0]["VirtPath"], 3253 "/v1/AUTH_test/con/obj") 3254 pre = "/v1/AUTH_test/PhysContainer_1/" 3255 self.assertEqual( 3256 args[0]["PhysPaths"], 3257 [pre + "1550057D8B0039185EB6184C599C940E51953403", 3258 pre + "1550057D8B0039185EB6184C599C940E51953403-01"]) 3259 self.assertEqual(args[0]["PhysLengths"], [100, 100]) 3260 3261 def test_missing_container(self): 3262 def mock_RpcHead_not_found(head_container_req): 3263 # This is what you get for no-such-container. 3264 return { 3265 "error": "errno: 2", 3266 "result": None} 3267 3268 self.fake_rpc.register_handler( 3269 "Server.RpcHead", mock_RpcHead_not_found) 3270 3271 wsgi_input = BytesIO(b"toxicum-brickcroft") 3272 3273 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3274 environ={"REQUEST_METHOD": "PUT", 3275 "wsgi.input": wsgi_input}) 3276 status, headers, body = self.call_pfs(req) 3277 self.assertEqual(status, "404 Not Found") 3278 3279 def test_metadata(self): 3280 wsgi_input = BytesIO(b"extranean-paleophysiology") 3281 cl = str(len(wsgi_input.getvalue())) 3282 3283 headers_in = { 3284 "X-Object-Meta-Color": "puce", 3285 "X-Object-Sysmeta-Flavor": "bbq", 3286 3287 "Content-Disposition": "recycle when no longer needed", 3288 "Content-Encoding": "quadruple rot13", 3289 "Content-Type": "application/eggplant", 3290 # NB: we'll never actually see these two together, but it's fine 3291 # for this test since we're only looking at which headers get 3292 # saved and which don't. 3293 "X-Object-Manifest": "pre/fix", 3294 "X-Static-Large-Object": "yes", 3295 3296 # These are not included 3297 "X-Timestamp": "1473968446.11364", 3298 "X-Object-Qmeta-Shape": "trapezoidal", 3299 } 3300 3301 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3302 headers=headers_in, 3303 environ={"REQUEST_METHOD": "PUT", 3304 "wsgi.input": wsgi_input, 3305 "CONTENT_LENGTH": cl}) 3306 status, headers, body = self.call_pfs(req) 3307 self.assertEqual(status, '201 Created') 3308 3309 serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] 3310 metadata = json.loads(base64.b64decode(serialized_metadata)) 3311 self.assertEqual(metadata.get("X-Object-Meta-Color"), "puce") 3312 self.assertEqual(metadata.get("X-Object-Sysmeta-Flavor"), "bbq") 3313 self.assertEqual(metadata.get("X-Object-Manifest"), "pre/fix") 3314 self.assertEqual(metadata.get("X-Static-Large-Object"), "yes") 3315 self.assertEqual(metadata.get("Content-Disposition"), 3316 "recycle when no longer needed") 3317 self.assertEqual(metadata.get("Content-Encoding"), "quadruple rot13") 3318 self.assertEqual(metadata.get("Content-Type"), "application/eggplant") 3319 3320 def test_directory_in_the_way(self): 3321 # If "thing.txt" is a nonempty directory, we get an error that the 3322 # middleware turns into a 409 Conflict response. 3323 wsgi_input = BytesIO(b"Celestine-malleal") 3324 cl = str(len(wsgi_input.getvalue())) 3325 3326 def mock_RpcPutComplete_isdir(head_container_req): 3327 # This is what you get when there's a nonempty directory in 3328 # place of your file. 3329 return { 3330 "error": "errno: 39", 3331 "result": None} 3332 3333 self.fake_rpc.register_handler( 3334 "Server.RpcPutComplete", mock_RpcPutComplete_isdir) 3335 3336 req = swob.Request.blank("/v1/AUTH_test/a-container/d1/d2/thing.txt", 3337 environ={"REQUEST_METHOD": "PUT", 3338 "wsgi.input": wsgi_input, 3339 "CONTENT_LENGTH": cl}) 3340 status, headers, body = self.call_pfs(req) 3341 self.assertEqual(status, '409 Conflict') 3342 3343 def test_file_in_the_way(self): 3344 # If "thing.txt" is a nonempty directory, we get an error that the 3345 # middleware turns into a 409 Conflict response. 3346 wsgi_input = BytesIO(b"Celestine-malleal") 3347 cl = str(len(wsgi_input.getvalue())) 3348 3349 def mock_RpcPutComplete_notdir(head_container_req): 3350 # This is what you get when there's a file where your path 3351 # contains a subdirectory. 3352 return { 3353 "error": "errno: 20", 3354 "result": None} 3355 3356 self.fake_rpc.register_handler( 3357 "Server.RpcPutComplete", mock_RpcPutComplete_notdir) 3358 3359 req = swob.Request.blank("/v1/AUTH_test/a-container/a-file/thing.txt", 3360 environ={"REQUEST_METHOD": "PUT", 3361 "wsgi.input": wsgi_input, 3362 "CONTENT_LENGTH": cl}) 3363 status, headers, body = self.call_pfs(req) 3364 self.assertEqual(status, '409 Conflict') 3365 3366 def test_stripping_bad_headers(self): 3367 # Someday, we'll have to figure out how to expire objects in 3368 # proxyfs. For now, though, we remove X-Delete-At and X-Delete-After 3369 # because having a log segment expire will do bad things to our 3370 # file's integrity. 3371 # 3372 # We strip ETag because Swift will treat that as the expected MD5 of 3373 # the log segment, and if we split across multiple log segments, 3374 # then any user-supplied ETag will be wrong. Also, this makes 3375 # POST-as-COPY work despite ProxyFS's ETag values not being MD5 3376 # checksums. 3377 wsgi_input = BytesIO(b"extranean-paleophysiology") 3378 cl = str(len(wsgi_input.getvalue())) 3379 3380 headers_in = {"X-Delete-After": 86400, 3381 "ETag": hashlib.md5(wsgi_input.getvalue()).hexdigest()} 3382 3383 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3384 headers=headers_in, 3385 environ={"REQUEST_METHOD": "PUT", 3386 "wsgi.input": wsgi_input, 3387 "CONTENT_LENGTH": cl}) 3388 status, headers, body = self.call_pfs(req) 3389 self.assertEqual(status, '201 Created') 3390 3391 serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] 3392 metadata = json.loads(base64.b64decode(serialized_metadata)) 3393 # it didn't get saved in metadata (not that it matters too much) 3394 self.assertNotIn("X-Delete-At", metadata) 3395 self.assertNotIn("X-Delete-After", metadata) 3396 self.assertNotIn("ETag", metadata) 3397 3398 # it didn't make it to Swift (this is important) 3399 put_method, put_path, put_headers = self.app.calls[-1] 3400 self.assertEqual(put_method, 'PUT') # sanity check 3401 self.assertNotIn("X-Delete-At", put_headers) 3402 self.assertNotIn("X-Delete-After", put_headers) 3403 self.assertNotIn("ETag", put_headers) 3404 3405 def test_etag_checking(self): 3406 wsgi_input = BytesIO(b"unsplashed-comprest") 3407 right_etag = hashlib.md5(wsgi_input.getvalue()).hexdigest() 3408 wrong_etag = hashlib.md5(wsgi_input.getvalue() + b"abc").hexdigest() 3409 non_checksum_etag = "pfsv2/AUTH_test/2226116/4341333-32" 3410 cl = str(len(wsgi_input.getvalue())) 3411 3412 wsgi_input.seek(0) 3413 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3414 environ={"REQUEST_METHOD": "PUT", 3415 "wsgi.input": wsgi_input}, 3416 headers={"ETag": right_etag, 3417 "Content-Length": cl}) 3418 status, headers, body = self.call_pfs(req) 3419 self.assertEqual(status, '201 Created') 3420 3421 wsgi_input.seek(0) 3422 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3423 environ={"REQUEST_METHOD": "PUT", 3424 "wsgi.input": wsgi_input}, 3425 headers={"ETag": wrong_etag, 3426 "Content-Length": cl}) 3427 status, headers, body = self.call_pfs(req) 3428 self.assertEqual(status, '422 Unprocessable Entity') 3429 3430 wsgi_input.seek(0) 3431 req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", 3432 environ={"REQUEST_METHOD": "PUT", 3433 "wsgi.input": wsgi_input}, 3434 headers={"ETag": non_checksum_etag, 3435 "Content-Length": cl}) 3436 status, headers, body = self.call_pfs(req) 3437 self.assertEqual(status, '201 Created') 3438 3439 def test_etag_checking_dir(self): 3440 req = swob.Request.blank( 3441 "/v1/AUTH_test/a-container/a-dir-object", 3442 environ={"REQUEST_METHOD": "PUT"}, 3443 headers={"ETag": hashlib.md5(b"x").hexdigest(), 3444 "Content-Length": 0, 3445 "Content-Type": "application/directory"}) 3446 status, headers, body = self.call_pfs(req) 3447 self.assertEqual(status, '422 Unprocessable Entity') 3448 3449 req = swob.Request.blank( 3450 "/v1/AUTH_test/a-container/a-dir-object", 3451 environ={"REQUEST_METHOD": "PUT"}, 3452 headers={"ETag": hashlib.md5(b"").hexdigest(), 3453 "Content-Length": 0, 3454 "Content-Type": "application/directory"}) 3455 status, headers, body = self.call_pfs(req) 3456 self.assertEqual(status, '201 Created') 3457 3458 req = swob.Request.blank( 3459 "/v1/AUTH_test/a-container/a-dir-object", 3460 environ={"REQUEST_METHOD": "PUT"}, 3461 headers={"Content-Length": 0, 3462 "Content-Type": "application/directory"}) 3463 status, headers, body = self.call_pfs(req) 3464 self.assertEqual(status, '201 Created') 3465 3466 3467 class TestObjectPost(BaseMiddlewareTest): 3468 def test_missing_object(self): 3469 def mock_RpcHead(_): 3470 return {"error": "errno: 2", "result": None} 3471 3472 self.fake_rpc.register_handler( 3473 "Server.RpcHead", mock_RpcHead) 3474 3475 req = swob.Request.blank( 3476 "/v1/AUTH_test/con/obj", 3477 environ={"REQUEST_METHOD": "POST"}, 3478 headers={"X-Object-Meta-One-Fish": "two fish"}) 3479 status, _, _ = self.call_pfs(req) 3480 self.assertEqual("404 Not Found", status) 3481 3482 rpc_calls = self.fake_rpc.calls 3483 self.assertEqual(2, len(rpc_calls)) 3484 3485 method, args = rpc_calls[0] 3486 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3487 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3488 3489 method, args = rpc_calls[1] 3490 self.assertEqual(method, "Server.RpcHead") 3491 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3492 3493 def test_file_as_dir(self): 3494 # Subdirectories of files don't exist, but asking for one returns a 3495 # different error code than asking for a file that could exist but 3496 # doesn't. 3497 def mock_RpcHead(head_object_req): 3498 self.assertEqual(head_object_req['VirtPath'], 3499 "/v1/AUTH_test/c/thing.txt/kitten.png") 3500 3501 return { 3502 "error": "errno: 20", 3503 "result": None} 3504 3505 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', 3506 method='POST') 3507 self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) 3508 status, headers, body = self.call_pfs(req) 3509 self.assertEqual(status, '404 Not Found') 3510 3511 def test_existing_object(self): 3512 old_meta = json.dumps({ 3513 "Content-Type": "application/fishy", 3514 mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", 3515 "X-Object-Meta-One-Fish": "two fish"}) 3516 3517 def mock_RpcHead(_): 3518 return { 3519 "error": None, 3520 "result": { 3521 "Metadata": base64.b64encode( 3522 old_meta.encode('ascii')).decode('ascii'), 3523 "ModificationTime": 1482345542483719281, 3524 "FileSize": 551155, 3525 "IsDir": False, 3526 "InodeNumber": 6519913, 3527 "NumWrites": 1}} 3528 3529 self.fake_rpc.register_handler( 3530 "Server.RpcHead", mock_RpcHead) 3531 3532 self.fake_rpc.register_handler( 3533 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3534 3535 req = swob.Request.blank( 3536 "/v1/AUTH_test/con/obj", 3537 environ={"REQUEST_METHOD": "POST"}, 3538 headers={"X-Object-Meta-Red-Fish": "blue fish"}) 3539 status, headers, _ = self.call_pfs(req) 3540 3541 # For reference, the result of a real object POST request. The 3542 # request contained new metadata items, but notice that those are 3543 # not reflected in the response. 3544 # 3545 # HTTP/1.1 202 Accepted 3546 # Content-Length: 76 3547 # Content-Type: text/html; charset=UTF-8 3548 # X-Trans-Id: tx65d5632fe4a548999ee9e-005a39781b 3549 # X-Openstack-Request-Id: tx65d5632fe4a548999ee9e-005a39781b 3550 # Date: Tue, 19 Dec 2017 20:35:39 GMT 3551 self.assertEqual("202 Accepted", status) 3552 self.assertNotIn('Last-Modified', headers) 3553 self.assertNotIn('Etag', headers) 3554 self.assertEqual("0", headers["Content-Length"]) 3555 self.assertEqual("text/html; charset=UTF-8", headers["Content-Type"]) 3556 # Date and X-Trans-Id are added in by other parts of the WSGI stack 3557 3558 rpc_calls = self.fake_rpc.calls 3559 self.assertEqual(3, len(rpc_calls)) 3560 3561 method, args = rpc_calls[0] 3562 self.assertEqual(method, "Server.RpcIsAccountBimodal") 3563 self.assertEqual(args[0]["AccountName"], "AUTH_test") 3564 3565 method, args = rpc_calls[1] 3566 self.assertEqual(method, "Server.RpcHead") 3567 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3568 3569 method, args = rpc_calls[2] 3570 self.assertEqual(method, "Server.RpcPost") 3571 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3572 self.assertEqual( 3573 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3574 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3575 self.assertEqual(new_meta["X-Object-Meta-Red-Fish"], "blue fish") 3576 self.assertEqual(new_meta["Content-Type"], "application/fishy") 3577 self.assertNotIn("X-Object-Meta-One-Fish", new_meta) 3578 3579 def test_preservation(self): 3580 old_meta = json.dumps({ 3581 "Content-Type": "application/fishy", 3582 mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", 3583 "X-Static-Large-Object": "true", 3584 "X-Object-Manifest": "solo/duet", 3585 "X-Object-Sysmeta-Dog": "collie", 3586 "X-Object-Meta-Fish": "perch"}) 3587 3588 def mock_RpcHead(_): 3589 return { 3590 "error": None, 3591 "result": { 3592 "Metadata": base64.b64encode( 3593 old_meta.encode('ascii')).decode('ascii'), 3594 "ModificationTime": 1510873171878460000, 3595 "FileSize": 7748115, 3596 "IsDir": False, 3597 "InodeNumber": 3741569, 3598 "NumWrites": 1}} 3599 3600 self.fake_rpc.register_handler( 3601 "Server.RpcHead", mock_RpcHead) 3602 3603 self.fake_rpc.register_handler( 3604 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3605 3606 req = swob.Request.blank( 3607 "/v1/AUTH_test/con/obj", 3608 environ={"REQUEST_METHOD": "POST"}, 3609 headers={"X-Object-Meta-Fish": "trout"}) 3610 status, _, _ = self.call_pfs(req) 3611 self.assertEqual("202 Accepted", status) # sanity check 3612 3613 method, args = self.fake_rpc.calls[2] 3614 self.assertEqual(method, "Server.RpcPost") 3615 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3616 self.assertEqual(new_meta["Content-Type"], "application/fishy") 3617 self.assertEqual(new_meta["X-Object-Sysmeta-Dog"], "collie") 3618 self.assertEqual(new_meta["X-Static-Large-Object"], "true") 3619 self.assertEqual(new_meta["X-Object-Meta-Fish"], "trout") 3620 self.assertNotIn("X-Object-Manifest", new_meta) 3621 3622 def test_change_content_type(self): 3623 old_meta = json.dumps({"Content-Type": "old/type"}) 3624 3625 def mock_RpcHead(_): 3626 return { 3627 "error": None, 3628 "result": { 3629 "Metadata": base64.b64encode( 3630 old_meta.encode('ascii')).decode('ascii'), 3631 "ModificationTime": 1482345542483719281, 3632 "FileSize": 551155, 3633 "IsDir": False, 3634 "InodeNumber": 6519913, 3635 "NumWrites": 381}} 3636 3637 self.fake_rpc.register_handler( 3638 "Server.RpcHead", mock_RpcHead) 3639 3640 self.fake_rpc.register_handler( 3641 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3642 3643 req = swob.Request.blank( 3644 "/v1/AUTH_test/con/obj", 3645 environ={"REQUEST_METHOD": "POST"}, 3646 headers={"Content-Type": "new/type"}) 3647 status, _, _ = self.call_pfs(req) 3648 self.assertEqual("202 Accepted", status) 3649 3650 rpc_calls = self.fake_rpc.calls 3651 3652 method, args = rpc_calls[2] 3653 self.assertEqual(method, "Server.RpcPost") 3654 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3655 self.assertEqual( 3656 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3657 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3658 self.assertEqual(new_meta["Content-Type"], "new/type") 3659 3660 def test_metadata_blanks(self): 3661 old_meta = json.dumps({"X-Object-Meta-Color": "red"}) 3662 3663 def mock_RpcHead(_): 3664 return { 3665 "error": None, 3666 "result": { 3667 "Metadata": base64.b64encode( 3668 old_meta.encode('ascii')).decode('ascii'), 3669 "ModificationTime": 1482345542483719281, 3670 "FileSize": 551155, 3671 "IsDir": False, 3672 "InodeNumber": 6519913, 3673 "NumWrites": 381}} 3674 3675 self.fake_rpc.register_handler( 3676 "Server.RpcHead", mock_RpcHead) 3677 3678 self.fake_rpc.register_handler( 3679 "Server.RpcPost", lambda *a: {"error": None, "result": {}}) 3680 3681 req = swob.Request.blank( 3682 "/v1/AUTH_test/con/obj", 3683 environ={"REQUEST_METHOD": "POST"}, 3684 headers={"X-Object-Meta-Color": ""}) 3685 status, _, _ = self.call_pfs(req) 3686 self.assertEqual("202 Accepted", status) 3687 3688 rpc_calls = self.fake_rpc.calls 3689 3690 method, args = rpc_calls[2] 3691 self.assertEqual(method, "Server.RpcPost") 3692 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 3693 self.assertEqual( 3694 base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) 3695 new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) 3696 self.assertNotIn("X-Object-Meta-Color", new_meta) 3697 3698 3699 class TestObjectDelete(BaseMiddlewareTest): 3700 def test_success(self): 3701 def fake_RpcDelete(delete_request): 3702 return {"error": None, "result": {}} 3703 3704 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3705 3706 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3707 environ={"REQUEST_METHOD": "DELETE"}) 3708 3709 status, _, _ = self.call_pfs(req) 3710 self.assertEqual(status, "204 No Content") 3711 3712 # the first call is a call to RpcIsAccountBimodal 3713 self.assertEqual(len(self.fake_rpc.calls), 2) 3714 self.assertEqual(self.fake_rpc.calls[1][1][0]["VirtPath"], 3715 "/v1/AUTH_test/con/obj") 3716 3717 def test_not_found(self): 3718 def fake_RpcDelete(delete_request): 3719 return {"error": "errno: 2", # NotFoundError / ENOENT 3720 "result": None} 3721 3722 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3723 3724 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3725 environ={"REQUEST_METHOD": "DELETE"}) 3726 3727 status, _, _ = self.call_pfs(req) 3728 self.assertEqual(status, "404 Not Found") 3729 3730 def test_no_such_dir(self): 3731 def fake_RpcDelete(delete_request): 3732 return {"error": "errno: 20", # NotDirError 3733 "result": None} 3734 3735 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3736 3737 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3738 environ={"REQUEST_METHOD": "DELETE"}) 3739 3740 status, _, _ = self.call_pfs(req) 3741 self.assertEqual(status, "404 Not Found") 3742 3743 def test_not_empty(self): 3744 def fake_RpcDelete(delete_request): 3745 return {"error": "errno: 39", # NotEmptyError / ENOTEMPTY 3746 "result": None} 3747 3748 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3749 3750 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3751 environ={"REQUEST_METHOD": "DELETE"}) 3752 3753 status, _, _ = self.call_pfs(req) 3754 self.assertEqual(status, "409 Conflict") 3755 3756 def test_other_failure(self): 3757 def fake_RpcDelete(delete_request): 3758 return {"error": "errno: 9", # BadFileError / EBADF 3759 "result": None} 3760 3761 self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) 3762 3763 req = swob.Request.blank("/v1/AUTH_test/con/obj", 3764 environ={"REQUEST_METHOD": "DELETE"}) 3765 3766 status, _, _ = self.call_pfs(req) 3767 self.assertEqual(status, "500 Internal Error") 3768 3769 3770 class TestObjectHead(BaseMiddlewareTest): 3771 def setUp(self): 3772 super(TestObjectHead, self).setUp() 3773 3774 self.serialized_object_metadata = "" 3775 3776 # All these tests run against the same object. 3777 def mock_RpcHead(head_object_req): 3778 self.assertEqual(head_object_req['VirtPath'], 3779 '/v1/AUTH_test/c/an-object.png') 3780 3781 if self.serialized_object_metadata: 3782 md = base64.b64encode( 3783 self.serialized_object_metadata.encode('ascii') 3784 ).decode('ascii') 3785 else: 3786 md = "" 3787 3788 return { 3789 "error": None, 3790 "result": { 3791 "Metadata": md, 3792 "ModificationTime": 1479173168018879490, 3793 "FileSize": 2641863, 3794 "IsDir": False, 3795 "InodeNumber": 4591, 3796 "NumWrites": 874, 3797 }} 3798 3799 self.fake_rpc.register_handler( 3800 "Server.RpcHead", mock_RpcHead) 3801 3802 # For reference: the result of a real HEAD request 3803 # 3804 # swift@saio:~/swift$ curl -I -H "X-Auth-Token: $TOKEN" \ 3805 # http://localhost:8080/v1/AUTH_test/s/tox.ini 3806 # HTTP/1.1 200 OK 3807 # Content-Length: 3954 3808 # Content-Type: application/octet-stream 3809 # Accept-Ranges: bytes 3810 # Last-Modified: Mon, 14 Nov 2016 23:29:00 GMT 3811 # Etag: ceffb3138058597bd7f8a09bdd3865d0 3812 # X-Timestamp: 1479166139.38308 3813 # X-Object-Meta-Mtime: 1476487400.000000 3814 # X-Trans-Id: txeaa367b1809a4583af3c8-00582a4a6c 3815 # Date: Mon, 14 Nov 2016 23:36:12 GMT 3816 # 3817 3818 def test_file_as_dir(self): 3819 # Subdirectories of files don't exist, but asking for one returns a 3820 # different error code than asking for a file that could exist but 3821 # doesn't. 3822 def mock_RpcHead(head_object_req): 3823 self.assertEqual(head_object_req['VirtPath'], 3824 "/v1/AUTH_test/c/thing.txt/kitten.png") 3825 3826 return { 3827 "error": "errno: 20", 3828 "result": None} 3829 3830 req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', 3831 method='HEAD') 3832 self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) 3833 status, headers, body = self.call_pfs(req) 3834 self.assertEqual(status, '404 Not Found') 3835 3836 def test_no_meta(self): 3837 self.serialized_object_metadata = "" 3838 3839 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3840 environ={"REQUEST_METHOD": "HEAD"}) 3841 status, headers, body = self.call_pfs(req) 3842 self.assertEqual(status, '200 OK') 3843 3844 self.assertEqual(headers["Content-Length"], "2641863") 3845 self.assertEqual(headers["Content-Type"], "image/png") 3846 self.assertEqual(headers["Accept-Ranges"], "bytes") 3847 self.assertEqual(headers["ETag"], 3848 '"pfsv2/AUTH_test/000011EF/0000036A-32"') 3849 self.assertEqual(headers["Last-Modified"], 3850 "Tue, 15 Nov 2016 01:26:09 GMT") 3851 self.assertEqual(headers["X-Timestamp"], "1479173168.01888") 3852 3853 def test_explicit_content_type(self): 3854 self.serialized_object_metadata = json.dumps({ 3855 "Content-Type": "Pegasus/inartistic"}) 3856 3857 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3858 environ={"REQUEST_METHOD": "HEAD"}) 3859 status, headers, body = self.call_pfs(req) 3860 self.assertEqual(status, '200 OK') 3861 self.assertEqual(headers["Content-Type"], "Pegasus/inartistic") 3862 3863 def test_bogus_meta(self): 3864 self.serialized_object_metadata = "{[{[{[{[{[[(((!" 3865 3866 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3867 environ={"REQUEST_METHOD": "HEAD"}) 3868 status, headers, body = self.call_pfs(req) 3869 self.assertEqual(status, '200 OK') 3870 3871 def test_meta(self): 3872 self.serialized_object_metadata = json.dumps({ 3873 "X-Object-Sysmeta-Fish": "cod", 3874 "X-Object-Meta-Fish": "trout"}) 3875 3876 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3877 environ={"REQUEST_METHOD": "HEAD"}) 3878 status, headers, body = self.call_pfs(req) 3879 self.assertEqual(status, '200 OK') 3880 self.assertEqual(headers["X-Object-Sysmeta-Fish"], "cod") 3881 self.assertEqual(headers["X-Object-Meta-Fish"], "trout") 3882 3883 def test_none_meta(self): 3884 # Sometimes we get a null in the response instead of an empty 3885 # string. No idea why. 3886 self.serialized_object_metadata = None 3887 3888 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3889 environ={"REQUEST_METHOD": "HEAD"}) 3890 status, headers, body = self.call_pfs(req) 3891 self.assertEqual(status, '200 OK') 3892 3893 def test_special_chars(self): 3894 def mock_RpcHead(head_object_req): 3895 self.assertEqual(head_object_req['VirtPath'], 3896 '/v1/AUTH_test/c/a cat.png') 3897 return {"error": "errno: 2", "result": None} 3898 3899 self.fake_rpc.register_handler( 3900 "Server.RpcHead", mock_RpcHead) 3901 3902 req = swob.Request.blank("/v1/AUTH_test/c/a cat.png", 3903 environ={"REQUEST_METHOD": "HEAD"}) 3904 status, headers, body = self.call_pfs(req) 3905 self.assertEqual(status, '404 Not Found') 3906 3907 def test_not_found(self): 3908 def mock_RpcHead(head_object_req): 3909 self.assertEqual(head_object_req['VirtPath'], 3910 '/v1/AUTH_test/c/an-object.png') 3911 return {"error": "errno: 2", "result": None} 3912 3913 self.fake_rpc.register_handler( 3914 "Server.RpcHead", mock_RpcHead) 3915 3916 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3917 environ={"REQUEST_METHOD": "HEAD"}) 3918 status, headers, body = self.call_pfs(req) 3919 self.assertEqual(status, '404 Not Found') 3920 3921 def test_other_error(self): 3922 def mock_RpcHead(head_object_req): 3923 self.assertEqual(head_object_req['VirtPath'], 3924 '/v1/AUTH_test/c/an-object.png') 3925 return {"error": "errno: 7581", "result": None} 3926 3927 self.fake_rpc.register_handler( 3928 "Server.RpcHead", mock_RpcHead) 3929 3930 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3931 environ={"REQUEST_METHOD": "HEAD"}) 3932 status, headers, body = self.call_pfs(req) 3933 self.assertEqual(status, '500 Internal Error') 3934 3935 def test_md5_etag(self): 3936 self.serialized_object_metadata = json.dumps({ 3937 "Content-Type": "Pegasus/inartistic", 3938 mware.ORIGINAL_MD5_HEADER: "1:b61d068208b52f4acbd618860d30faae", 3939 }) 3940 3941 def mock_RpcHead(head_object_req): 3942 md = base64.b64encode( 3943 self.serialized_object_metadata.encode('ascii') 3944 ).decode('ascii') 3945 return { 3946 "error": None, 3947 "result": { 3948 "Metadata": md, 3949 "ModificationTime": 1506039770222591000, 3950 "FileSize": 3397331, 3951 "IsDir": False, 3952 "InodeNumber": 1433230, 3953 "NumWrites": 1, 3954 }} 3955 3956 self.fake_rpc.register_handler( 3957 "Server.RpcHead", mock_RpcHead) 3958 3959 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3960 environ={"REQUEST_METHOD": "HEAD"}) 3961 status, headers, body = self.call_pfs(req) 3962 self.assertEqual(status, '200 OK') 3963 self.assertEqual(headers["Etag"], "b61d068208b52f4acbd618860d30faae") 3964 3965 def test_conditional_if_match(self): 3966 obj_etag = "c2185d19ada5f5c3aa47d6b55dc912df" 3967 3968 self.serialized_object_metadata = json.dumps({ 3969 mware.ORIGINAL_MD5_HEADER: "1:%s" % obj_etag, 3970 }) 3971 3972 def mock_RpcHead(head_object_req): 3973 md = base64.b64encode( 3974 self.serialized_object_metadata.encode('ascii') 3975 ).decode('ascii') 3976 return { 3977 "error": None, 3978 "result": { 3979 "Metadata": md, 3980 "ModificationTime": 1511223407565078000, 3981 "FileSize": 152874, 3982 "IsDir": False, 3983 "InodeNumber": 9566555, 3984 "NumWrites": 1, 3985 }} 3986 3987 self.fake_rpc.register_handler( 3988 "Server.RpcHead", mock_RpcHead) 3989 3990 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3991 environ={"REQUEST_METHOD": "HEAD"}, 3992 headers={"If-Match": obj_etag}) 3993 # matches 3994 status, _, _ = self.call_pfs(req) 3995 self.assertEqual(status, '200 OK') 3996 3997 # doesn't match 3998 req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", 3999 environ={"REQUEST_METHOD": "HEAD"}, 4000 headers={"If-Match": obj_etag + "XYZZY"}) 4001 status, _, _ = self.call_pfs(req) 4002 self.assertEqual(status, '412 Precondition Failed') 4003 4004 4005 class TestObjectHeadDir(BaseMiddlewareTest): 4006 def test_dir(self): 4007 def mock_RpcHead(head_object_req): 4008 self.assertEqual(head_object_req['VirtPath'], 4009 '/v1/AUTH_test/c/a-dir') 4010 4011 return { 4012 "error": None, 4013 "result": { 4014 "Metadata": "", 4015 "ModificationTime": 1479173168018879490, 4016 "FileSize": 0, 4017 "IsDir": True, 4018 "InodeNumber": 1254, 4019 "NumWrites": 896, 4020 }} 4021 4022 self.fake_rpc.register_handler( 4023 "Server.RpcHead", mock_RpcHead) 4024 req = swob.Request.blank("/v1/AUTH_test/c/a-dir", 4025 environ={"REQUEST_METHOD": "HEAD"}) 4026 status, headers, body = self.call_pfs(req) 4027 self.assertEqual(status, '200 OK') 4028 self.assertEqual(headers["Content-Length"], "0") 4029 self.assertEqual(headers["Content-Type"], "application/directory") 4030 self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") 4031 self.assertEqual(headers["Last-Modified"], 4032 "Tue, 15 Nov 2016 01:26:09 GMT") 4033 4034 4035 class TestObjectCoalesce(BaseMiddlewareTest): 4036 def setUp(self): 4037 super(TestObjectCoalesce, self).setUp() 4038 4039 def mock_RpcHead(head_container_req): 4040 return { 4041 "error": None, 4042 "result": { 4043 "Metadata": "", 4044 "ModificationTime": 1485814697697650000, 4045 "FileSize": 0, 4046 "IsDir": True, 4047 "InodeNumber": 1828, 4048 "NumWrites": 893, 4049 }} 4050 4051 # this handler gets overwritten by the first test that 4052 # registers a "Server.RpcHead" 4053 self.fake_rpc.register_handler( 4054 "Server.RpcHead", mock_RpcHead) 4055 4056 def test_success(self): 4057 4058 # a map from an object's "virtual path" to its metadata 4059 self.obj_metadata = {} 4060 4061 def mock_RpcHead(head_req): 4062 '''Return the object informattion for the new coalesced object. This 4063 assumes that COALESCE operation has already created it. 4064 ''' 4065 4066 resp = { 4067 "error": None, 4068 "result": { 4069 "Metadata": "", 4070 "ModificationTime": 1488323796002909000, 4071 "FileSize": 80 * 1024 * 1024, 4072 "IsDir": False, 4073 "InodeNumber": 283253, 4074 "NumWrites": 5, 4075 } 4076 } 4077 virt_path = head_req['VirtPath'] 4078 if self.obj_metadata[virt_path] is not None: 4079 resp['result']['Metadata'] = self.obj_metadata[virt_path] 4080 return resp 4081 4082 self.fake_rpc.register_handler( 4083 "Server.RpcHead", mock_RpcHead) 4084 4085 def mock_RpcCoalesce(coalesce_req): 4086 4087 # if there's metadata for the new object, save it to return later 4088 if coalesce_req['NewMetaData'] != "": 4089 virt_path = coalesce_req['VirtPath'] 4090 self.obj_metadata[virt_path] = coalesce_req['NewMetaData'] 4091 4092 numWrites = len(coalesce_req['ElementAccountRelativePaths']) 4093 return { 4094 "error": None, 4095 "result": { 4096 "ModificationTime": 1488323796002909000, 4097 "InodeNumber": 283253, 4098 "NumWrites": numWrites, 4099 }} 4100 4101 self.fake_rpc.register_handler( 4102 "Server.RpcCoalesce", mock_RpcCoalesce) 4103 4104 # have the coalesce request suppply the headers that would 4105 # come from s3api for a "complete multi-part upload" request 4106 request_headers = { 4107 'X-Object-Sysmeta-S3Api-Acl': 4108 '{"Owner":"fc",' + 4109 '"Grant":[{"Grantee":"fc","Permission":"FULL_CONTROL"}]}', 4110 'X-Object-Sysmeta-S3Api-Etag': 4111 'cb45770d6cf51effdfb2ea35322459c3-205', 4112 'X-Object-Sysmeta-Slo-Etag': '363d958f0f4c8501a50408a728ba5599', 4113 'X-Object-Sysmeta-Slo-Size': '1073741824', 4114 'X-Object-Sysmeta-Container-Update-Override-Etag': 4115 '10340ab593ac8c32290a278e36d1f8df; ' + 4116 's3_etag=cb45770d6cf51effdfb2ea35322459c3-205; ' + 4117 'slo_etag=363d958f0f4c8501a50408a728ba5599', 4118 'X-Static-Large-Object': 'True', 4119 'Content-Type': 4120 'application/x-www-form-urlencoded; ' + 4121 'charset=utf-8;swift_bytes=1073741824', 4122 'Etag': '10340ab593ac8c32290a278e36d1f8df', 4123 'Date': 'Mon, 01 Apr 2019 22:53:31 GMT', 4124 'Host': 'sm-p1.swiftstack.org', 4125 'Accept': 'application/json', 4126 'Content-Length': '18356', 4127 'X-Auth-Token': None, 4128 } 4129 request_body = json.dumps({ 4130 "elements": [ 4131 "c1/seg1a", 4132 "c1/seg1b", 4133 "c2/seg space 2a", 4134 "c2/seg space 2b", 4135 "c3/seg3", 4136 ]}).encode('ascii') 4137 req = swob.Request.blank( 4138 "/v1/AUTH_test/con/obj", 4139 headers=request_headers, 4140 environ={"REQUEST_METHOD": "COALESCE", 4141 "wsgi.input": BytesIO(request_body)}) 4142 4143 status, headers, body = self.call_pfs(req) 4144 self.assertEqual(status, '201 Created') 4145 self.assertEqual(headers["Etag"], '10340ab593ac8c32290a278e36d1f8df') 4146 4147 # The first call is a call to RpcIsAccountBimodal, the last is the one 4148 # we care about: RpcCoalesce. There *could* be some intervening calls 4149 # to RpcHead to authorize read/write access to the segments, but since 4150 # there's no authorize callback installed, we skip it. 4151 self.assertEqual([method for method, args in self.fake_rpc.calls], [ 4152 "Server.RpcIsAccountBimodal", 4153 "Server.RpcCoalesce", 4154 ]) 4155 method, args = self.fake_rpc.calls[-1] 4156 self.assertEqual(method, "Server.RpcCoalesce") 4157 self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") 4158 self.assertEqual(args[0]["ElementAccountRelativePaths"], [ 4159 "c1/seg1a", 4160 "c1/seg1b", 4161 "c2/seg space 2a", 4162 "c2/seg space 2b", 4163 "c3/seg3", 4164 ]) 4165 4166 # verify that the metadata was munged correctly 4167 # (SLO headers stripped out, etc.) 4168 req = swob.Request.blank( 4169 "/v1/AUTH_test/con/obj", 4170 environ={"REQUEST_METHOD": "HEAD"}) 4171 status, headers, body = self.call_pfs(req) 4172 self.assertEqual(status, '200 OK') 4173 self.assertEqual(body, b'') 4174 self.assertEqual(headers["Etag"], 4175 '10340ab593ac8c32290a278e36d1f8df') 4176 self.assertIn('X-Object-Sysmeta-S3Api-Acl', headers) 4177 self.assertIn('X-Object-Sysmeta-S3Api-Etag', headers) 4178 self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', 4179 headers) 4180 self.assertNotIn('X-Static-Large-Object', headers) 4181 self.assertNotIn('X-Object-Sysmeta-Slo-Etag', headers) 4182 self.assertNotIn('X-Object-Sysmeta-Slo-Size', headers) 4183 4184 def test_not_authed(self): 4185 def mock_RpcHead(get_container_req): 4186 path = get_container_req['VirtPath'] 4187 return { 4188 "error": None, 4189 "result": { 4190 "Metadata": base64.b64encode(json.dumps({ 4191 "X-Container-Read": path + '\x00read-acl', 4192 "X-Container-Write": path + '\x00write-acl', 4193 }).encode('ascii')).decode('ascii'), 4194 "ModificationTime": 1479240451156825194, 4195 "FileSize": 0, 4196 "IsDir": True, 4197 "InodeNumber": 1255, 4198 "NumWrites": 897, 4199 }} 4200 4201 self.fake_rpc.register_handler( 4202 "Server.RpcHead", mock_RpcHead) 4203 4204 acls = [] 4205 4206 def auth_cb(req): 4207 acls.append(req.acl) 4208 # write access for the target, plus 3 distince element containers 4209 # each needing both read and write checks -- fail the last 4210 if len(acls) >= 7: 4211 return swob.HTTPForbidden(request=req) 4212 4213 request_body = json.dumps({ 4214 "elements": [ 4215 "c1/seg1a", 4216 "c1/seg1b", 4217 "c2/seg space 2a", 4218 "c2/seg space 2b", 4219 "c3/seg3", 4220 ]}).encode('ascii') 4221 req = swob.Request.blank( 4222 "/v1/AUTH_test/con/obj", 4223 environ={"REQUEST_METHOD": "COALESCE", 4224 "wsgi.input": BytesIO(request_body), 4225 "swift.authorize": auth_cb}) 4226 status, headers, body = self.call_pfs(req) 4227 # The first call is a call to RpcIsAccountBimodal, then a bunch of 4228 # RpcHead calls as we authorize the four containers involved. Since 4229 # we fail the authorization, we never make it to RpcCoalesce. 4230 self.assertEqual([method for method, args in self.fake_rpc.calls], [ 4231 "Server.RpcIsAccountBimodal", 4232 "Server.RpcHead", 4233 "Server.RpcHead", 4234 "Server.RpcHead", 4235 "Server.RpcHead", 4236 ]) 4237 container_paths = [ 4238 "/v1/AUTH_test/con", 4239 "/v1/AUTH_test/c1", 4240 "/v1/AUTH_test/c2", 4241 "/v1/AUTH_test/c3", 4242 ] 4243 self.assertEqual(container_paths, [ 4244 args[0]['VirtPath'] for method, args in self.fake_rpc.calls[1:]]) 4245 self.assertEqual(status, '403 Forbidden') 4246 self.assertEqual(acls, [ 4247 "/v1/AUTH_test/con\x00write-acl", 4248 "/v1/AUTH_test/c1\x00read-acl", 4249 "/v1/AUTH_test/c1\x00write-acl", 4250 "/v1/AUTH_test/c2\x00read-acl", 4251 "/v1/AUTH_test/c2\x00write-acl", 4252 "/v1/AUTH_test/c3\x00read-acl", 4253 "/v1/AUTH_test/c3\x00write-acl", 4254 ]) 4255 4256 def test_malformed_json(self): 4257 request_body = b"{{{[[[(((" 4258 req = swob.Request.blank( 4259 "/v1/AUTH_test/con/obj", 4260 environ={"REQUEST_METHOD": "COALESCE", 4261 "wsgi.input": BytesIO(request_body)}) 4262 status, headers, body = self.call_pfs(req) 4263 self.assertEqual(status, '400 Bad Request') 4264 4265 def test_incorrect_json(self): 4266 request_body = b"{}" 4267 req = swob.Request.blank( 4268 "/v1/AUTH_test/con/obj", 4269 environ={"REQUEST_METHOD": "COALESCE", 4270 "wsgi.input": BytesIO(request_body)}) 4271 status, headers, body = self.call_pfs(req) 4272 self.assertEqual(status, '400 Bad Request') 4273 4274 def test_incorrect_json_wrong_type(self): 4275 request_body = b"[]" 4276 req = swob.Request.blank( 4277 "/v1/AUTH_test/con/obj", 4278 environ={"REQUEST_METHOD": "COALESCE", 4279 "wsgi.input": BytesIO(request_body)}) 4280 status, headers, body = self.call_pfs(req) 4281 self.assertEqual(status, '400 Bad Request') 4282 4283 def test_incorrect_json_wrong_elements_type(self): 4284 request_body = json.dumps({"elements": {1: 2}}).encode('ascii') 4285 req = swob.Request.blank( 4286 "/v1/AUTH_test/con/obj", 4287 environ={"REQUEST_METHOD": "COALESCE", 4288 "wsgi.input": BytesIO(request_body)}) 4289 status, headers, body = self.call_pfs(req) 4290 self.assertEqual(status, '400 Bad Request') 4291 4292 def test_incorrect_json_wrong_element_type(self): 4293 request_body = json.dumps({"elements": [1, "two", {}]}).encode('ascii') 4294 req = swob.Request.blank( 4295 "/v1/AUTH_test/con/obj", 4296 environ={"REQUEST_METHOD": "COALESCE", 4297 "wsgi.input": BytesIO(request_body)}) 4298 status, headers, body = self.call_pfs(req) 4299 self.assertEqual(status, '400 Bad Request') 4300 4301 def test_incorrect_json_subtle(self): 4302 request_body = b'["elements"]' 4303 req = swob.Request.blank( 4304 "/v1/AUTH_test/con/obj", 4305 environ={"REQUEST_METHOD": "COALESCE", 4306 "wsgi.input": BytesIO(request_body)}) 4307 status, headers, body = self.call_pfs(req) 4308 self.assertEqual(status, '400 Bad Request') 4309 4310 def test_too_big(self): 4311 request_body = b'{' * (self.pfs.max_coalesce_request_size + 1) 4312 req = swob.Request.blank( 4313 "/v1/AUTH_test/con/obj", 4314 environ={"REQUEST_METHOD": "COALESCE", 4315 "wsgi.input": BytesIO(request_body)}) 4316 status, headers, body = self.call_pfs(req) 4317 self.assertEqual(status, '413 Request Entity Too Large') 4318 4319 def test_too_many_elements(self): 4320 request_body = json.dumps({ 4321 "elements": ["/c/o"] * (self.pfs.max_coalesce + 1), 4322 }).encode('ascii') 4323 req = swob.Request.blank( 4324 "/v1/AUTH_test/con/obj", 4325 environ={"REQUEST_METHOD": "COALESCE", 4326 "wsgi.input": BytesIO(request_body)}) 4327 status, headers, body = self.call_pfs(req) 4328 self.assertEqual(status, '413 Request Entity Too Large') 4329 4330 def test_not_found(self): 4331 # Note: this test covers all sorts of thing-not-found errors, as the 4332 # Server.RpcCoalesce remote procedure does not distinguish between 4333 # them. In particular, this covers the case when an element is not 4334 # found as well as the case when the destination container is not 4335 # found. 4336 def mock_RpcCoalesce(coalese_req): 4337 return { 4338 "error": "errno: 2", 4339 "result": None, 4340 } 4341 4342 self.fake_rpc.register_handler( 4343 "Server.RpcCoalesce", mock_RpcCoalesce) 4344 4345 request_body = json.dumps({ 4346 "elements": [ 4347 "some/stuff", 4348 ]}).encode('ascii') 4349 req = swob.Request.blank( 4350 "/v1/AUTH_test/con/obj", 4351 environ={"REQUEST_METHOD": "COALESCE", 4352 "wsgi.input": BytesIO(request_body)}) 4353 status, headers, body = self.call_pfs(req) 4354 self.assertEqual(status, '404 Not Found') 4355 4356 def test_coalesce_directory(self): 4357 # This happens when one of the named elements is a directory. 4358 def mock_RpcCoalesce(coalese_req): 4359 return { 4360 "error": "errno: 21", 4361 "result": None, 4362 } 4363 4364 self.fake_rpc.register_handler( 4365 "Server.RpcCoalesce", mock_RpcCoalesce) 4366 4367 request_body = json.dumps({ 4368 "elements": [ 4369 "some/stuff", 4370 ]}).encode('ascii') 4371 req = swob.Request.blank( 4372 "/v1/AUTH_test/con/obj", 4373 environ={"REQUEST_METHOD": "COALESCE", 4374 "wsgi.input": BytesIO(request_body)}) 4375 status, headers, body = self.call_pfs(req) 4376 self.assertEqual(status, '409 Conflict') 4377 4378 def test_coalesce_file_instead_of_dir(self): 4379 # This happens when one of the named elements is not a regular file. 4380 # Yes, you get IsDirError for not-a-file, even if it's a symlink. 4381 def mock_RpcCoalesce(coalese_req): 4382 return { 4383 "error": "errno: 20", 4384 "result": None, 4385 } 4386 4387 self.fake_rpc.register_handler( 4388 "Server.RpcCoalesce", mock_RpcCoalesce) 4389 4390 request_body = json.dumps({ 4391 "elements": [ 4392 "some/stuff", 4393 ]}).encode('ascii') 4394 req = swob.Request.blank( 4395 "/v1/AUTH_test/con/obj", 4396 environ={"REQUEST_METHOD": "COALESCE", 4397 "wsgi.input": BytesIO(request_body)}) 4398 status, headers, body = self.call_pfs(req) 4399 self.assertEqual(status, '409 Conflict') 4400 4401 def test_element_has_multiple_links(self): 4402 # If a file has multiple links to it (hard links, not symlinks), 4403 # then we can't coalesce it. 4404 def mock_RpcCoalesce(coalese_req): 4405 return { 4406 "error": "errno: 31", 4407 "result": None, 4408 } 4409 4410 self.fake_rpc.register_handler( 4411 "Server.RpcCoalesce", mock_RpcCoalesce) 4412 4413 request_body = json.dumps({ 4414 "elements": [ 4415 "some/stuff", 4416 ]}).encode('ascii') 4417 req = swob.Request.blank( 4418 "/v1/AUTH_test/con/obj", 4419 environ={"REQUEST_METHOD": "COALESCE", 4420 "wsgi.input": BytesIO(request_body)}) 4421 status, headers, body = self.call_pfs(req) 4422 self.assertEqual(status, '409 Conflict') 4423 4424 def test_other_error(self): 4425 # If a file has multiple links to it (hard links, not symlinks), 4426 # then we can't coalesce it. 4427 def mock_RpcCoalesce(coalese_req): 4428 return { 4429 "error": "errno: 1159268", 4430 "result": None, 4431 } 4432 4433 self.fake_rpc.register_handler( 4434 "Server.RpcCoalesce", mock_RpcCoalesce) 4435 4436 request_body = json.dumps({ 4437 "elements": [ 4438 "thing/one", 4439 "thing/two", 4440 ]}).encode('ascii') 4441 req = swob.Request.blank( 4442 "/v1/AUTH_test/con/obj", 4443 environ={"REQUEST_METHOD": "COALESCE", 4444 "wsgi.input": BytesIO(request_body)}) 4445 status, headers, body = self.call_pfs(req) 4446 self.assertEqual(status, '500 Internal Error') 4447 4448 4449 class TestAuth(BaseMiddlewareTest): 4450 def setUp(self): 4451 super(TestAuth, self).setUp() 4452 4453 def mock_RpcHead(get_container_req): 4454 return { 4455 "error": None, 4456 "result": { 4457 "Metadata": base64.b64encode(json.dumps({ 4458 "X-Container-Read": "the-x-con-read", 4459 "X-Container-Write": "the-x-con-write", 4460 }).encode('ascii')).decode('ascii'), 4461 "ModificationTime": 1479240451156825194, 4462 "FileSize": 0, 4463 "IsDir": True, 4464 "InodeNumber": 1255, 4465 "NumWrites": 897, 4466 }} 4467 4468 self.fake_rpc.register_handler( 4469 "Server.RpcHead", mock_RpcHead) 4470 4471 def test_auth_callback_args(self): 4472 want_x_container_read = (('GET', '/v1/AUTH_test/con/obj'), 4473 ('HEAD', '/v1/AUTH_test/con/obj'), 4474 ('GET', '/v1/AUTH_test/con'), 4475 ('HEAD', '/v1/AUTH_test/con')) 4476 4477 want_x_container_write = (('PUT', '/v1/AUTH_test/con/obj'), 4478 ('POST', '/v1/AUTH_test/con/obj'), 4479 ('DELETE', '/v1/AUTH_test/con/obj')) 4480 got_acls = [] 4481 4482 def capture_acl_and_deny(request): 4483 got_acls.append(request.acl) 4484 return swob.HTTPForbidden(request=request) 4485 4486 for method, path in want_x_container_read: 4487 req = swob.Request.blank( 4488 path, environ={'REQUEST_METHOD': method, 4489 'swift.authorize': capture_acl_and_deny}) 4490 status, _, _ = self.call_pfs(req) 4491 self.assertEqual(status, '403 Forbidden') 4492 self.assertEqual( 4493 got_acls, ["the-x-con-read"] * len(want_x_container_read)) 4494 4495 del got_acls[:] 4496 for method, path in want_x_container_write: 4497 req = swob.Request.blank( 4498 path, environ={'REQUEST_METHOD': method, 4499 'swift.authorize': capture_acl_and_deny}) 4500 status, _, _ = self.call_pfs(req) 4501 self.assertEqual(status, '403 Forbidden') 4502 self.assertEqual( 4503 got_acls, ["the-x-con-write"] * len(want_x_container_write)) 4504 4505 def test_auth_override(self): 4506 def auth_nope(request): 4507 return swob.HTTPForbidden(request=request) 4508 4509 req = swob.Request.blank( 4510 "/v1/AUTH_test/con", 4511 environ={'REQUEST_METHOD': 'HEAD', 4512 'swift.authorize': auth_nope, 4513 'swift.authorize_override': True}) 4514 status, _, _ = self.call_pfs(req) 4515 self.assertEqual(status, '403 Forbidden') 4516 4517 def test_auth_bypass(self): 4518 def auth_nope(request): 4519 return swob.HTTPForbidden(request=request) 4520 4521 # pfs is re-entrant when getting ACLs from what may be other accounts. 4522 req = swob.Request.blank( 4523 "/v1/AUTH_test/con", 4524 environ={'REQUEST_METHOD': 'HEAD', 4525 'swift.authorize': auth_nope, 4526 'swift.authorize_override': True, 4527 'swift.source': 'PFS'}) 4528 status, _, _ = self.call_pfs(req) 4529 self.assertEqual(status, '204 No Content') 4530 4531 def test_auth_allowed(self): 4532 def auth_its_fine(request): 4533 return None 4534 4535 req = swob.Request.blank( 4536 "/v1/AUTH_test/con", 4537 environ={'REQUEST_METHOD': 'HEAD', 4538 'swift.authorize': auth_its_fine}) 4539 status, _, _ = self.call_pfs(req) 4540 self.assertEqual(status, '204 No Content') 4541 4542 4543 class TestEtagHandling(unittest.TestCase): 4544 '''Test that mung_etags()/unmung_etags() are inverse functions (more 4545 or less), that the unmung_etags() correctly unmungs the current 4546 disk layout for the header, and that best_possible_etag() returns 4547 the correct etag value. 4548 ''' 4549 4550 # Both the header names and the way the values are calculated and 4551 # the way they are formatted are part of the "disk layout". If we 4552 # change them, we have to consider handling old data. 4553 # 4554 # Names and values duplicated here so this test will break if they 4555 # are changed. 4556 HEADER = "X-Object-Sysmeta-ProxyFS-Initial-MD5" 4557 4558 ETAG_HEADERS = { 4559 "ORIGINAL_MD5": { 4560 "name": "X-Object-Sysmeta-ProxyFS-Initial-MD5", 4561 "value": "5484c2634aa61c69fc02ef5400a61c94", 4562 "num_writes": 7, 4563 "munged_value": "7:5484c2634aa61c69fc02ef5400a61c94" 4564 }, 4565 "S3API_ETAG": { 4566 "name": "X-Object-Sysmeta-S3Api-Etag", 4567 "value": "cb0dc66d591395cdf93555dafd4145ad", 4568 "num_writes": 3, 4569 "munged_value": "3:cb0dc66d591395cdf93555dafd4145ad" 4570 }, 4571 "LISTING_ETAG_OVERRIDE": { 4572 "name": "X-Object-Sysmeta-Container-Update-Override-Etag", 4573 "value": "dbca19e0c46aa9b10e8de2f5856abc86", 4574 "num_writes": 9, 4575 "munged_value": "9:dbca19e0c46aa9b10e8de2f5856abc86" 4576 }, 4577 } 4578 4579 EMPTY_OBJECT_ETAG = "d41d8cd98f00b204e9800998ecf8427e" 4580 4581 def test_mung_etags(self): 4582 '''Verify that mung_etags() mungs each of the etag header values to 4583 the correct on-disk version with num_writes and unmung_etags() 4584 recovers the original value. 4585 ''' 4586 for header in self.ETAG_HEADERS.keys(): 4587 4588 name = self.ETAG_HEADERS[header]["name"] 4589 value = self.ETAG_HEADERS[header]["value"] 4590 num_writes = self.ETAG_HEADERS[header]["num_writes"] 4591 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4592 4593 obj_metadata = {name: value} 4594 mware.mung_etags(obj_metadata, None, num_writes) 4595 4596 # The handling of ORIGINAL_MD5_HEADER is a bit strange. 4597 # If an etag value is passed to mung_etags() as the 2nd 4598 # argument then the header is created and assigned the 4599 # munged version of that etag value. But if its None than 4600 # an ORIGINAL_MD5_HEADER, if present, is passed through 4601 # unmolested (but will be dropped by the unmung code if 4602 # the format is wrong or num_writes does not match, which 4603 # is likely). 4604 if header == "ORIGINAL_MD5": 4605 self.assertEqual(obj_metadata[name], value) 4606 else: 4607 self.assertEqual(obj_metadata[name], munged_value) 4608 4609 # same test but with an etag value passed to mung_etags() 4610 # instead of None 4611 md5_etag_value = self.ETAG_HEADERS["ORIGINAL_MD5"]["value"] 4612 4613 obj_metadata = {name: value} 4614 mware.mung_etags(obj_metadata, md5_etag_value, num_writes) 4615 self.assertEqual(obj_metadata[name], munged_value) 4616 4617 # and verify that it gets unmunged to the original value 4618 mware.unmung_etags(obj_metadata, num_writes) 4619 self.assertEqual(obj_metadata[name], value) 4620 4621 def test_unmung_etags(self): 4622 '''Verify that unmung_etags() recovers the correct value for 4623 each ETag header and discards header values where num_writes 4624 is incorrect or the value is corrupt. 4625 ''' 4626 # build metadata with all of the etag headers 4627 orig_obj_metadata = {} 4628 for header in self.ETAG_HEADERS.keys(): 4629 4630 name = self.ETAG_HEADERS[header]["name"] 4631 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4632 orig_obj_metadata[name] = munged_value 4633 4634 # unmung_etags() should strip out headers with stale 4635 # num_writes so only the one for header remains 4636 for header in self.ETAG_HEADERS.keys(): 4637 4638 name = self.ETAG_HEADERS[header]["name"] 4639 value = self.ETAG_HEADERS[header]["value"] 4640 num_writes = self.ETAG_HEADERS[header]["num_writes"] 4641 munged_value = self.ETAG_HEADERS[header]["munged_value"] 4642 4643 obj_metadata = orig_obj_metadata.copy() 4644 mware.unmung_etags(obj_metadata, num_writes) 4645 4646 # header should be the only one left 4647 for hdr2 in self.ETAG_HEADERS.keys(): 4648 4649 if hdr2 == header: 4650 self.assertEqual(obj_metadata[name], value) 4651 else: 4652 self.assertTrue( 4653 self.ETAG_HEADERS[hdr2]["name"] not in obj_metadata) 4654 4655 # verify mung_etags() takes it back to the munged value 4656 if header != "ORIGINAL_MD5": 4657 mware.mung_etags(obj_metadata, None, num_writes) 4658 self.assertEqual(obj_metadata[name], munged_value) 4659 4660 # Try to unmung on-disk headers with corrupt values. Instead 4661 # of a panic it should simply elide the headers (and possibly 4662 # log the corrupt value, which it doesn't currently do). 4663 for bad_value in ("counterfact-preformative", 4664 "magpie:interfollicular"): 4665 4666 obj_metadata = {} 4667 for header in self.ETAG_HEADERS.keys(): 4668 4669 name = self.ETAG_HEADERS[header]["name"] 4670 obj_metadata[name] = bad_value 4671 4672 mware.unmung_etags(obj_metadata, 0) 4673 self.assertEqual(obj_metadata, {}) 4674 4675 def test_best_possible_etag(self): 4676 '''Test that best_possible_etag() returns the best ETag. 4677 ''' 4678 4679 # Start with metadata that consists of all possible ETag 4680 # headers and verify that best_possible_headers() chooses the 4681 # right ETag based on the circumstance. 4682 obj_metadata = {} 4683 for header in self.ETAG_HEADERS.keys(): 4684 4685 name = self.ETAG_HEADERS[header]["name"] 4686 value = self.ETAG_HEADERS[header]["value"] 4687 obj_metadata[name] = value 4688 4689 # directories always return the same etag 4690 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, 4691 is_dir=True) 4692 self.assertEqual(etag, self.EMPTY_OBJECT_ETAG) 4693 4694 # a container listing should return the Container Listing Override ETag 4695 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, 4696 container_listing=True) 4697 self.assertEqual(etag, 4698 self.ETAG_HEADERS["LISTING_ETAG_OVERRIDE"]["value"]) 4699 4700 # if its not a directory and its not a container list, then 4701 # the function should return the original MD5 ETag (the 4702 # S3API_ETAG is returned as a header, but not as an ETag). 4703 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) 4704 self.assertEqual(etag, 4705 self.ETAG_HEADERS["ORIGINAL_MD5"]["value"]) 4706 4707 # if none of the headers provide the correct ETag then 4708 # best_possible_etag() will construct a unique one (which also 4709 # should not be changed without handling "old data", i.e. 4710 # its part of the "disk layout"). 4711 del obj_metadata[self.ETAG_HEADERS["ORIGINAL_MD5"]["name"]] 4712 4713 etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) 4714 self.assertEqual(etag, 4715 '"pfsv2/AUTH_test/0000002A/00000007-32"') 4716 4717 4718 class TestProxyfsMethod(BaseMiddlewareTest): 4719 def test_non_swift_owner(self): 4720 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS') 4721 status, _, _ = self.call_pfs(req) 4722 self.assertEqual(status, '405 Method Not Allowed') 4723 4724 def test_bad_content_type(self): 4725 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', 4726 environ={'swift_owner': True}, body=b'') 4727 status, _, body = self.call_pfs(req) 4728 self.assertEqual(status, '415 Unsupported Media Type') 4729 self.assertEqual( 4730 body, b'RPC body must have Content-Type application/json') 4731 4732 req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', 4733 headers={'Content-Type': 'text/plain'}, 4734 environ={'swift_owner': True}, body=b'') 4735 status, _, body = self.call_pfs(req) 4736 self.assertEqual(status, '415 Unsupported Media Type') 4737 self.assertEqual(body, (b'RPC body must have Content-Type ' 4738 b'application/json, not text/plain')) 4739 4740 def test_exactly_one_payload(self): 4741 def expect_one_payload(body): 4742 req = swob.Request.blank( 4743 "/v1/AUTH_test", method='PROXYFS', 4744 headers={'Content-Type': 'application/json'}, 4745 environ={'swift_owner': True}, body=body) 4746 status, _, body = self.call_pfs(req) 4747 self.assertEqual(status, '400 Bad Request') 4748 self.assertEqual(body, b'Expected exactly one JSON payload') 4749 4750 expect_one_payload(b'') 4751 expect_one_payload(b'\n') 4752 one_payload = json.dumps({ 4753 'jsonrpc': '2.0', 4754 'method': 'Server.RpcPing', 4755 'params': [{}], 4756 }) 4757 expect_one_payload(one_payload + '\n' + one_payload) 4758 4759 def test_bad_body(self): 4760 def expect_parse_error(req_body): 4761 req = swob.Request.blank( 4762 "/v1/AUTH_test", method='PROXYFS', 4763 headers={'Content-Type': 'application/json'}, 4764 environ={'swift_owner': True}, body=req_body) 4765 status, _, body = self.call_pfs(req) 4766 self.assertEqual(status, '400 Bad Request') 4767 exp_start = (b'Could not parse/validate JSON payload #0 ' 4768 b'%s:' % req_body) 4769 self.assertEqual(body[:len(exp_start)], exp_start) 4770 4771 expect_parse_error(b'[') 4772 expect_parse_error(b'{') 4773 expect_parse_error(b'[]') 4774 expect_parse_error(b'{}') 4775 4776 expect_parse_error(json.dumps({ 4777 'jsonrpc': '1.0', 4778 'method': 'Server.PingReq', 4779 'params': [{}], 4780 }).encode('ascii')) 4781 4782 expect_parse_error(json.dumps({ 4783 'jsonrpc': '2.0', 4784 'method': 'asdf', 4785 'params': [{}], 4786 }).encode('ascii')) 4787 4788 expect_parse_error(json.dumps({ 4789 'jsonrpc': '2.0', 4790 'method': 'Server.PingReq', 4791 'params': [{}, {}], 4792 }).encode('ascii')) 4793 expect_parse_error(json.dumps({ 4794 'jsonrpc': '2.0', 4795 'method': 'Server.PingReq', 4796 'params': [], 4797 }).encode('ascii')) 4798 expect_parse_error(json.dumps({ 4799 'jsonrpc': '2.0', 4800 'method': 'Server.PingReq', 4801 'params': [[]], 4802 }).encode('ascii')) 4803 expect_parse_error(json.dumps({ 4804 'jsonrpc': '2.0', 4805 'method': 'Server.PingReq', 4806 'params': {}, 4807 }).encode('ascii')) 4808 4809 def test_success(self): 4810 def mock_RpcGetAccount(params): 4811 self.assertEqual(params, { 4812 "AccountName": "AUTH_test", 4813 }) 4814 return { 4815 "error": None, 4816 "result": { 4817 "ModificationTime": 1498766381451119000, 4818 "AccountEntries": [{ 4819 "Basename": "chickens", 4820 "ModificationTime": 1510958440808682000, 4821 }, { 4822 "Basename": "cows", 4823 "ModificationTime": 1510958450657045000, 4824 }, { 4825 "Basename": "goats", 4826 "ModificationTime": 1510958452544251000, 4827 }, { 4828 "Basename": "pigs", 4829 "ModificationTime": 1510958459200130000, 4830 }], 4831 }} 4832 4833 self.fake_rpc.register_handler( 4834 "Server.RpcGetAccount", mock_RpcGetAccount) 4835 4836 req = swob.Request.blank( 4837 "/v1/AUTH_test", method='PROXYFS', 4838 headers={'Content-Type': 'application/json'}, 4839 environ={'swift_owner': True}, body=json.dumps({ 4840 'jsonrpc': '2.0', 4841 'method': 'Server.RpcGetAccount', 4842 'params': [{}], 4843 })) 4844 status, headers, body = self.call_pfs(req) 4845 self.assertEqual(status, '200 OK', body) 4846 self.assertEqual(headers.get('content-type'), 'application/json') 4847 resp = json.loads(body) 4848 self.assertIsNotNone(resp.pop('id', None)) 4849 self.assertEqual(resp, { 4850 "error": None, 4851 "result": { 4852 "ModificationTime": 1498766381451119000, 4853 "AccountEntries": [{ 4854 "Basename": "chickens", 4855 "ModificationTime": 1510958440808682000, 4856 }, { 4857 "Basename": "cows", 4858 "ModificationTime": 1510958450657045000, 4859 }, { 4860 "Basename": "goats", 4861 "ModificationTime": 1510958452544251000, 4862 }, { 4863 "Basename": "pigs", 4864 "ModificationTime": 1510958459200130000, 4865 }] 4866 } 4867 }) 4868 4869 def test_error(self): 4870 def mock_RpcGetAccount(params): 4871 self.assertEqual(params, { 4872 "AccountName": "AUTH_test", 4873 "some": "args", 4874 }) 4875 return {"error": "errno 2"} 4876 4877 self.fake_rpc.register_handler( 4878 "Server.RpcGetAccount", mock_RpcGetAccount) 4879 4880 req = swob.Request.blank( 4881 "/v1/AUTH_test", method='PROXYFS', 4882 headers={'Content-Type': 'application/json'}, 4883 environ={'swift_owner': True}, body=json.dumps({ 4884 'jsonrpc': '2.0', 4885 'method': 'Server.RpcGetAccount', 4886 'params': [{'some': 'args'}], 4887 })) 4888 status, headers, body = self.call_pfs(req) 4889 self.assertEqual(status, '200 OK', body) 4890 self.assertEqual(headers.get('content-type'), 'application/json') 4891 resp = json.loads(body) 4892 self.assertIsNotNone(resp.pop('id', None)) 4893 self.assertEqual(resp, {"error": "errno 2"})