github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/python/tests/unit/sdk/test_bucket.py (about) 1 import unittest 2 from unittest.mock import Mock, call, patch, MagicMock 3 4 from aistore.sdk.ais_source import AISSource 5 from aistore.sdk.bucket import Bucket, Header 6 from aistore.sdk.object import Object 7 from aistore.sdk.etl_const import DEFAULT_ETL_TIMEOUT 8 from aistore.sdk.object_iterator import ObjectIterator 9 from aistore.sdk import ListObjectFlag 10 11 from aistore.sdk.const import ( 12 ACT_COPY_BCK, 13 ACT_CREATE_BCK, 14 ACT_DESTROY_BCK, 15 ACT_ETL_BCK, 16 ACT_EVICT_REMOTE_BCK, 17 ACT_LIST, 18 ACT_MOVE_BCK, 19 ACT_SUMMARY_BCK, 20 PROVIDER_AMAZON, 21 PROVIDER_AIS, 22 QPARAM_BCK_TO, 23 QPARAM_NAMESPACE, 24 QPARAM_PROVIDER, 25 QPARAM_KEEP_REMOTE, 26 QPARAM_BSUMM_REMOTE, 27 QPARAM_FLT_PRESENCE, 28 QPARAM_UUID, 29 HTTP_METHOD_DELETE, 30 HTTP_METHOD_GET, 31 HTTP_METHOD_HEAD, 32 HTTP_METHOD_PUT, 33 HTTP_METHOD_POST, 34 URL_PATH_BUCKETS, 35 HEADER_ACCEPT, 36 HEADER_BUCKET_PROPS, 37 HEADER_XACTION_ID, 38 HEADER_BUCKET_SUMM, 39 MSGPACK_CONTENT_TYPE, 40 STATUS_ACCEPTED, 41 STATUS_BAD_REQUEST, 42 STATUS_OK, 43 ) 44 from aistore.sdk.dataset.dataset_config import DatasetConfig 45 from aistore.sdk.errors import ( 46 InvalidBckProvider, 47 ErrBckAlreadyExists, 48 ErrBckNotFound, 49 UnexpectedHTTPStatusCode, 50 ) 51 from aistore.sdk.request_client import RequestClient 52 from aistore.sdk.types import ( 53 ActionMsg, 54 BucketList, 55 BucketEntry, 56 BsummCtrlMsg, 57 Namespace, 58 TCBckMsg, 59 TransformBckMsg, 60 CopyBckMsg, 61 ) 62 from tests.const import ETL_NAME, PREFIX_NAME 63 64 BCK_NAME = "bucket_name" 65 66 67 # pylint: disable=too-many-public-methods,unused-variable 68 class TestBucket(unittest.TestCase): 69 def setUp(self) -> None: 70 self.mock_client = Mock(RequestClient) 71 self.amz_bck = Bucket( 72 name=BCK_NAME, client=self.mock_client, provider=PROVIDER_AMAZON 73 ) 74 self.amz_bck_params = self.amz_bck.qparam.copy() 75 self.ais_bck = Bucket(name=BCK_NAME, client=self.mock_client) 76 self.ais_bck_params = self.ais_bck.qparam.copy() 77 self.dataset_config = MagicMock(spec=DatasetConfig) 78 79 def test_default_props(self): 80 bucket = Bucket(name=BCK_NAME, client=self.mock_client) 81 self.assertEqual({QPARAM_PROVIDER: PROVIDER_AIS}, bucket.qparam) 82 self.assertEqual(PROVIDER_AIS, bucket.provider) 83 self.assertIsNone(bucket.namespace) 84 85 def test_properties(self): 86 self.assertEqual(self.mock_client, self.ais_bck.client) 87 expected_ns = Namespace(uuid="ns-id", name="ns-name") 88 client = RequestClient("test client name", skip_verify=False, ca_cert="") 89 bck = Bucket( 90 client=client, 91 name=BCK_NAME, 92 provider=PROVIDER_AMAZON, 93 namespace=expected_ns, 94 ) 95 self.assertEqual(client, bck.client) 96 self.assertEqual(PROVIDER_AMAZON, bck.provider) 97 self.assertEqual( 98 { 99 QPARAM_PROVIDER: PROVIDER_AMAZON, 100 QPARAM_NAMESPACE: expected_ns.get_path(), 101 }, 102 bck.qparam, 103 ) 104 self.assertEqual(BCK_NAME, bck.name) 105 self.assertEqual(expected_ns, bck.namespace) 106 107 def test_ais_source(self): 108 self.assertIsInstance(self.ais_bck, AISSource) 109 110 def test_create_invalid_provider(self): 111 self.assertRaises(InvalidBckProvider, self.amz_bck.create) 112 113 def _assert_bucket_created(self, bck): 114 self.mock_client.request.assert_called_with( 115 HTTP_METHOD_POST, 116 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 117 json=ActionMsg(action=ACT_CREATE_BCK).dict(), 118 params=self.ais_bck.qparam, 119 ) 120 self.assertIsInstance(bck, Bucket) 121 122 def test_create_success(self): 123 res = self.ais_bck.create() 124 self._assert_bucket_created(res) 125 126 def test_create_already_exists(self): 127 already_exists_err = ErrBckAlreadyExists(400, "message") 128 self.mock_client.request.side_effect = already_exists_err 129 with self.assertRaises(ErrBckAlreadyExists): 130 self.ais_bck.create() 131 132 res = self.ais_bck.create(exist_ok=True) 133 self._assert_bucket_created(res) 134 135 def test_rename_invalid_provider(self): 136 self.assertRaises(InvalidBckProvider, self.amz_bck.rename, "new_name") 137 138 def test_rename_success(self): 139 new_bck_name = "new_bucket" 140 expected_response = "rename_op_123" 141 self.ais_bck_params[QPARAM_BCK_TO] = f"{PROVIDER_AIS}/@#/{new_bck_name}/" 142 mock_response = Mock() 143 mock_response.text = expected_response 144 self.mock_client.request.return_value = mock_response 145 146 response = self.ais_bck.rename(new_bck_name) 147 148 self.assertEqual(expected_response, response) 149 self.mock_client.request.assert_called_with( 150 HTTP_METHOD_POST, 151 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 152 json=ActionMsg(action=ACT_MOVE_BCK).dict(), 153 params=self.ais_bck_params, 154 ) 155 self.assertEqual(self.ais_bck.name, new_bck_name) 156 157 def test_delete_invalid_provider(self): 158 self.assertRaises(InvalidBckProvider, self.amz_bck.delete) 159 160 def test_delete_success(self): 161 self.ais_bck.delete() 162 self.mock_client.request.assert_called_with( 163 HTTP_METHOD_DELETE, 164 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 165 json=ActionMsg(action=ACT_DESTROY_BCK).dict(), 166 params=self.ais_bck.qparam, 167 ) 168 169 def test_delete_missing(self): 170 self.mock_client.request.side_effect = ErrBckNotFound(400, "not found") 171 with self.assertRaises(ErrBckNotFound): 172 Bucket(client=self.mock_client, name="missing-bucket").delete() 173 self.ais_bck.delete(missing_ok=True) 174 self.mock_client.request.assert_called_with( 175 HTTP_METHOD_DELETE, 176 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 177 json=ActionMsg(action=ACT_DESTROY_BCK).dict(), 178 params=self.ais_bck.qparam, 179 ) 180 181 def test_evict_invalid_provider(self): 182 self.assertRaises(InvalidBckProvider, self.ais_bck.evict) 183 184 def test_evict_success(self): 185 for keep_md in [True, False]: 186 self.amz_bck_params[QPARAM_KEEP_REMOTE] = str(keep_md) 187 self.amz_bck.evict(keep_md=keep_md) 188 self.mock_client.request.assert_called_with( 189 HTTP_METHOD_DELETE, 190 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 191 json=ActionMsg(action=ACT_EVICT_REMOTE_BCK).dict(), 192 params=self.amz_bck_params, 193 ) 194 195 def test_head(self): 196 mock_header = Mock() 197 mock_header.headers = Header("value") 198 self.mock_client.request.return_value = mock_header 199 headers = self.ais_bck.head() 200 self.mock_client.request.assert_called_with( 201 HTTP_METHOD_HEAD, 202 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 203 params=self.ais_bck.qparam, 204 ) 205 self.assertEqual(headers, mock_header.headers) 206 207 def test_copy_default_params(self): 208 dest_bck = Bucket( 209 client=self.mock_client, 210 name="test-bck", 211 namespace=Namespace(uuid="namespace-id", name="ns-name"), 212 provider="any-provider", 213 ) 214 action_value = { 215 "prefix": "", 216 "prepend": "", 217 "dry_run": False, 218 "force": False, 219 "latest-ver": False, 220 "synchronize": False, 221 } 222 self._copy_exec_assert(dest_bck, action_value) 223 224 def test_copy(self): 225 prefix_filter = "existing-" 226 prepend_val = PREFIX_NAME 227 dry_run = True 228 force = True 229 latest = False 230 sync = False 231 action_value = { 232 "prefix": prefix_filter, 233 "prepend": prepend_val, 234 "dry_run": dry_run, 235 "force": force, 236 "latest-ver": latest, 237 "synchronize": sync, 238 } 239 240 self._copy_exec_assert( 241 self.ais_bck, 242 action_value, 243 prefix_filter=prefix_filter, 244 prepend=prepend_val, 245 dry_run=dry_run, 246 force=force, 247 ) 248 249 def _copy_exec_assert(self, to_bck, expected_act_value, **kwargs): 250 expected_response = "copy-action-id" 251 mock_response = Mock() 252 mock_response.text = expected_response 253 self.mock_client.request.return_value = mock_response 254 self.ais_bck_params[QPARAM_BCK_TO] = to_bck.get_path() 255 expected_action = ActionMsg( 256 action=ACT_COPY_BCK, value=expected_act_value 257 ).dict() 258 259 job_id = self.ais_bck.copy(to_bck=to_bck, **kwargs) 260 261 self.assertEqual(expected_response, job_id) 262 self.mock_client.request.assert_called_with( 263 HTTP_METHOD_POST, 264 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 265 json=expected_action, 266 params=self.ais_bck_params, 267 ) 268 269 def test_list_objects(self): 270 prefix = PREFIX_NAME 271 page_size = 0 272 uuid = "1234" 273 props = "name" 274 continuation_token = "token" 275 flags = [ListObjectFlag.CACHED, ListObjectFlag.DELETED] 276 flag_value = "5" 277 target_id = "target-node" 278 expected_act_value = { 279 "prefix": prefix, 280 "pagesize": page_size, 281 "uuid": uuid, 282 "props": props, 283 "continuation_token": continuation_token, 284 "flags": flag_value, 285 "target": target_id, 286 } 287 self._list_objects_exec_assert( 288 expected_act_value, 289 prefix=prefix, 290 page_size=page_size, 291 uuid=uuid, 292 props=props, 293 continuation_token=continuation_token, 294 flags=flags, 295 target=target_id, 296 ) 297 298 def test_list_objects_default_params(self): 299 expected_act_value = { 300 "prefix": "", 301 "pagesize": 0, 302 "uuid": "", 303 "props": "", 304 "continuation_token": "", 305 "flags": "0", 306 "target": "", 307 } 308 self._list_objects_exec_assert(expected_act_value) 309 310 def _list_objects_exec_assert(self, expected_act_value, **kwargs): 311 action = ActionMsg(action=ACT_LIST, value=expected_act_value).dict() 312 313 object_names = ["obj_name", "obj_name2"] 314 bucket_entries = [BucketEntry(n=name) for name in object_names] 315 mock_list = Mock(BucketList) 316 mock_list.entries = bucket_entries 317 self.mock_client.request_deserialize.return_value = mock_list 318 result = self.ais_bck.list_objects(**kwargs) 319 self.mock_client.request_deserialize.assert_called_with( 320 HTTP_METHOD_GET, 321 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 322 headers={HEADER_ACCEPT: MSGPACK_CONTENT_TYPE}, 323 res_model=BucketList, 324 json=action, 325 params=self.ais_bck_params, 326 ) 327 self.assertEqual(result, mock_list) 328 self.assertEqual(object_names, [entry.object.name for entry in result.entries]) 329 330 def test_list_objects_iter(self): 331 self.assertIsInstance( 332 self.ais_bck.list_objects_iter(PREFIX_NAME, "obj props", 123), 333 ObjectIterator, 334 ) 335 336 def test_list_all_objects(self): 337 list_1_id = "123" 338 list_1_cont = "cont" 339 prefix = PREFIX_NAME 340 page_size = 5 341 props = "name" 342 flags = [ListObjectFlag.CACHED, ListObjectFlag.DELETED] 343 flag_value = "5" 344 target_id = "target-node" 345 expected_act_value_1 = { 346 "prefix": prefix, 347 "pagesize": page_size, 348 "uuid": "", 349 "props": props, 350 "continuation_token": "", 351 "flags": flag_value, 352 "target": target_id, 353 } 354 expected_act_value_2 = { 355 "prefix": prefix, 356 "pagesize": page_size, 357 "uuid": list_1_id, 358 "props": props, 359 "continuation_token": list_1_cont, 360 "flags": flag_value, 361 "target": target_id, 362 } 363 self._list_all_objects_exec_assert( 364 list_1_id, 365 list_1_cont, 366 expected_act_value_1, 367 expected_act_value_2, 368 prefix=prefix, 369 page_size=page_size, 370 props=props, 371 flags=flags, 372 target=target_id, 373 ) 374 375 def test_list_all_objects_default_params(self): 376 list_1_id = "123" 377 list_1_cont = "cont" 378 expected_act_value_1 = { 379 "prefix": "", 380 "pagesize": 0, 381 "uuid": "", 382 "props": "", 383 "continuation_token": "", 384 "flags": "0", 385 "target": "", 386 } 387 expected_act_value_2 = { 388 "prefix": "", 389 "pagesize": 0, 390 "uuid": list_1_id, 391 "props": "", 392 "continuation_token": list_1_cont, 393 "flags": "0", 394 "target": "", 395 } 396 self._list_all_objects_exec_assert( 397 list_1_id, list_1_cont, expected_act_value_1, expected_act_value_2 398 ) 399 400 def _list_all_objects_exec_assert( 401 self, 402 list_1_id, 403 list_1_cont, 404 expected_act_value_1, 405 expected_act_value_2, 406 **kwargs, 407 ): 408 entry_1 = BucketEntry(n="entry1") 409 entry_2 = BucketEntry(n="entry2") 410 entry_3 = BucketEntry(n="entry3") 411 list_1 = BucketList( 412 UUID=list_1_id, ContinuationToken=list_1_cont, Flags=0, Entries=[entry_1] 413 ) 414 list_2 = BucketList( 415 UUID="456", ContinuationToken="", Flags=0, Entries=[entry_2, entry_3] 416 ) 417 418 # Test with empty list of entries 419 self.mock_client.request_deserialize.return_value = BucketList( 420 UUID="empty", ContinuationToken="", Flags=0 421 ) 422 423 self.assertEqual([], self.ais_bck.list_all_objects(**kwargs)) 424 425 # Test with non-empty lists 426 self.mock_client.request_deserialize.side_effect = [list_1, list_2] 427 self.assertEqual( 428 [entry_1, entry_2, entry_3], self.ais_bck.list_all_objects(**kwargs) 429 ) 430 431 expected_calls = [] 432 for expected_val in [expected_act_value_1, expected_act_value_2]: 433 expected_calls.append( 434 call( 435 HTTP_METHOD_GET, 436 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 437 headers={HEADER_ACCEPT: MSGPACK_CONTENT_TYPE}, 438 res_model=BucketList, 439 json=ActionMsg(action=ACT_LIST, value=expected_val).dict(), 440 params=self.ais_bck_params, 441 ) 442 ) 443 444 for expected in expected_calls: 445 self.assertIn(expected, self.mock_client.request_deserialize.call_args_list) 446 447 def test_transform(self): 448 prepend_val = PREFIX_NAME 449 prefix_filter = "required-prefix-" 450 ext = {"jpg": "txt"} 451 timeout = "4m" 452 force = True 453 dry_run = True 454 action_value = TCBckMsg( 455 ext=ext, 456 transform_msg=TransformBckMsg(etl_name=ETL_NAME, timeout=timeout), 457 copy_msg=CopyBckMsg( 458 prefix=prefix_filter, 459 prepend=prepend_val, 460 force=force, 461 dry_run=dry_run, 462 latest=False, 463 sync=False, 464 ), 465 ).as_dict() 466 467 self._transform_exec_assert( 468 ETL_NAME, 469 action_value, 470 prepend=prepend_val, 471 prefix_filter=prefix_filter, 472 ext=ext, 473 force=force, 474 dry_run=dry_run, 475 timeout=timeout, 476 ) 477 478 def test_transform_default_params(self): 479 action_value = { 480 "id": ETL_NAME, 481 "prefix": "", 482 "prepend": "", 483 "force": False, 484 "dry_run": False, 485 "request_timeout": DEFAULT_ETL_TIMEOUT, 486 "latest-ver": False, 487 "synchronize": False, 488 } 489 490 self._transform_exec_assert(ETL_NAME, action_value) 491 492 def _transform_exec_assert(self, etl_name, expected_act_value, **kwargs): 493 to_bck = Bucket(name="new-bucket") 494 self.ais_bck_params[QPARAM_BCK_TO] = to_bck.get_path() 495 expected_action = ActionMsg(action=ACT_ETL_BCK, value=expected_act_value).dict() 496 expected_response = "job-id" 497 mock_response = Mock() 498 mock_response.text = expected_response 499 self.mock_client.request.return_value = mock_response 500 501 result_id = self.ais_bck.transform(etl_name, to_bck, **kwargs) 502 503 self.mock_client.request.assert_called_with( 504 HTTP_METHOD_POST, 505 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 506 json=expected_action, 507 params=self.ais_bck_params, 508 ) 509 self.assertEqual(expected_response, result_id) 510 511 def test_object(self): 512 new_obj = self.ais_bck.object(obj_name="name") 513 self.assertEqual(self.ais_bck, new_obj.bucket) 514 515 @patch("aistore.sdk.object.read_file_bytes") 516 @patch("aistore.sdk.object.validate_file") 517 @patch("aistore.sdk.bucket.validate_directory") 518 @patch("pathlib.Path.glob") 519 def test_put_files( 520 self, mock_glob, mock_validate_dir, mock_validate_file, mock_read 521 ): 522 path = "directory" 523 file_1_name = "file_1_name" 524 file_2_name = "file_2_name" 525 path_1 = Mock() 526 path_1.is_file.return_value = True 527 path_1.relative_to.return_value = file_1_name 528 path_1.stat.return_value = Mock(st_size=123) 529 path_2 = Mock() 530 path_2.relative_to.return_value = file_2_name 531 path_2.is_file.return_value = True 532 path_2.stat.return_value = Mock(st_size=4567) 533 file_1_data = b"bytes in the first file" 534 file_2_data = b"bytes in the second file" 535 mock_glob.return_value = [path_1, path_2] 536 expected_obj_names = [file_1_name, file_2_name] 537 mock_read.side_effect = [file_1_data, file_2_data] 538 539 res = self.ais_bck.put_files(path) 540 541 mock_validate_dir.assert_called_with(path) 542 mock_validate_file.assert_has_calls([call(str(path_1)), call(str(path_2))]) 543 self.assertEqual(expected_obj_names, res) 544 expected_calls = [ 545 call( 546 HTTP_METHOD_PUT, 547 path=f"objects/{BCK_NAME}/{file_1_name}", 548 params=self.ais_bck_params, 549 data=file_1_data, 550 ), 551 call( 552 HTTP_METHOD_PUT, 553 path=f"objects/{BCK_NAME}/{file_2_name}", 554 params=self.ais_bck_params, 555 data=file_2_data, 556 ), 557 ] 558 self.mock_client.request.assert_has_calls(expected_calls) 559 560 def test_get_path(self): 561 namespace = Namespace(uuid="ns-id", name="ns-name") 562 bucket = Bucket(name=BCK_NAME, namespace=namespace, provider=PROVIDER_AMAZON) 563 expected_path = ( 564 f"{PROVIDER_AMAZON}/@{namespace.uuid}#{namespace.name}/{bucket.name}/" 565 ) 566 self.assertEqual(expected_path, bucket.get_path()) 567 self.assertEqual(f"{PROVIDER_AIS}/@#/{bucket.name}/", self.ais_bck.get_path()) 568 569 @patch("aistore.sdk.bucket.Bucket.object") 570 @patch("aistore.sdk.bucket.Bucket.list_objects_iter") 571 def test_list_urls(self, mock_list_obj, mock_object): 572 prefix = "my-prefix" 573 object_names = ["obj_name", "obj_name2"] 574 expected_obj_calls = [] 575 # Should create an object reference and get url for every object returned by listing 576 for name in object_names: 577 expected_obj_calls.append(call(name)) 578 expected_obj_calls.append(call().get_url(etl_name=ETL_NAME)) 579 mock_list_obj.return_value = [BucketEntry(n=name) for name in object_names] 580 list(self.ais_bck.list_urls(prefix=prefix, etl_name=ETL_NAME)) 581 mock_list_obj.assert_called_with(prefix=prefix, props="name") 582 mock_object.assert_has_calls(expected_obj_calls) 583 584 @patch("aistore.sdk.bucket.Bucket.list_objects_iter") 585 def test_list_all_objects_iter(self, mock_list_obj): 586 object_names = ["obj_name", "obj_name2"] 587 mock_list_obj.return_value = [BucketEntry(n=name) for name in object_names] 588 objects_iter = self.ais_bck.list_all_objects_iter() 589 for obj in objects_iter: 590 self.assertIsInstance(obj, Object) 591 592 def test_make_request_no_client(self): 593 bucket = Bucket(name="name") 594 with self.assertRaises(ValueError): 595 bucket.make_request("method", "action") 596 597 def test_make_request_default_params(self): 598 method = "method" 599 action = "action" 600 self.ais_bck.make_request(method, action) 601 self.mock_client.request.assert_called_with( 602 method, 603 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 604 json=ActionMsg(action=action, value=None).dict(), 605 params=self.ais_bck.qparam, 606 ) 607 608 def test_make_request(self): 609 method = "method" 610 action = "action" 611 value = {"request_key": "value"} 612 params = {"qparamkey": "qparamval"} 613 self.ais_bck.make_request(method, action, value, params) 614 self.mock_client.request.assert_called_with( 615 method, 616 path=f"{URL_PATH_BUCKETS}/{BCK_NAME}", 617 json=ActionMsg(action=action, value=value).dict(), 618 params=params, 619 ) 620 621 def test_summary(self): 622 # Mock responses for request calls 623 response1 = Mock() 624 response1.status_code = STATUS_ACCEPTED 625 response1.text = '"job_id"' 626 627 response2 = Mock() 628 response2.status_code = STATUS_OK 629 response2.content = ( 630 b'[{"name":"temporary","provider":"ais","namespace":{"uuid":"","name":""},' 631 b'"ObjCount":{"obj_count_present":"137160","obj_count_remote":"0"},' 632 b'"ObjSize":{"obj_min_size":1024,"obj_avg_size":1024,"obj_max_size":1024},' 633 b'"TotalSize":{"size_on_disk":"148832256","size_all_present_objs":"140451840",' 634 b'"size_all_remote_objs":"0","total_disks_size":"4955520307200"},' 635 b'"used_pct":0,"is_present":false}]\n' 636 ) 637 638 # Set the side_effect of the request method to return the two responses in sequence 639 self.mock_client.request.side_effect = [response1, response2] 640 641 # Call the summary method 642 result = self.ais_bck.summary() 643 644 # Ensure that request was called with the correct sequence of calls 645 bsumm_ctrl_msg = BsummCtrlMsg( 646 uuid="", prefix="", fast=True, cached=True, present=True 647 ).dict() 648 bsumm_ctrl_msg_with_uuid = BsummCtrlMsg( 649 uuid="job_id", prefix="", fast=True, cached=True, present=True 650 ).dict() 651 calls = [ 652 call( 653 HTTP_METHOD_GET, 654 path="buckets/bucket_name", 655 json={"action": ACT_SUMMARY_BCK, "name": "", "value": bsumm_ctrl_msg}, 656 params=self.ais_bck.qparam, 657 ), 658 call( 659 HTTP_METHOD_GET, 660 path="buckets/bucket_name", 661 json={ 662 "action": ACT_SUMMARY_BCK, 663 "name": "", 664 "value": bsumm_ctrl_msg_with_uuid, 665 }, 666 params=self.ais_bck.qparam, 667 ), 668 ] 669 self.mock_client.request.assert_has_calls(calls) 670 671 # Assert that the result has the expected structure 672 self.assertIsInstance(result, dict) 673 self.assertIn("name", result) 674 self.assertIn("provider", result) 675 self.assertIn("ObjCount", result) 676 self.assertIn("ObjSize", result) 677 self.assertIn("TotalSize", result) 678 679 def test_summary_error_handling(self): 680 # Mock responses for the first and second request call 681 first_response = Mock() 682 first_response.status_code = STATUS_ACCEPTED 683 684 second_response = Mock() 685 second_response.status_code = STATUS_BAD_REQUEST 686 second_response.text = '"job_id"' 687 688 # Set the side_effect of the request method to return the correct mock response 689 self.mock_client.request.side_effect = [first_response, second_response] 690 691 # Call the summary method and expect an UnexpectedHTTPStatusCode exception 692 with self.assertRaises(UnexpectedHTTPStatusCode): 693 self.ais_bck.summary() 694 695 # Verify that the request method was called twice 696 assert self.mock_client.request.call_count == 2 697 698 def test_info(self): 699 # Mock responses for request calls 700 response1 = Mock() 701 response1.status_code = STATUS_ACCEPTED 702 response1.headers = { 703 HEADER_BUCKET_PROPS: '{"some": "props"}', 704 HEADER_XACTION_ID: "some-id", 705 } 706 707 response2 = Mock() 708 response2.status_code = STATUS_OK 709 response2.headers = { 710 HEADER_BUCKET_SUMM: '{"some": "summary"}', 711 } 712 713 # Set the side_effect of the request method to return the two responses in sequence 714 self.mock_client.request.side_effect = [response1, response2] 715 716 # Call the info method 717 bucket_props, bucket_summ = self.ais_bck.info() 718 719 calls = [ 720 call( 721 HTTP_METHOD_HEAD, 722 path=f"{URL_PATH_BUCKETS}/{self.ais_bck.name}", 723 params={ 724 **self.ais_bck.qparam, 725 QPARAM_FLT_PRESENCE: 0, 726 QPARAM_BSUMM_REMOTE: True, 727 QPARAM_UUID: "some-id", 728 }, 729 ), 730 call( 731 HTTP_METHOD_HEAD, 732 path=f"{URL_PATH_BUCKETS}/{self.ais_bck.name}", 733 params={ 734 **self.ais_bck.qparam, 735 QPARAM_FLT_PRESENCE: 0, 736 QPARAM_BSUMM_REMOTE: True, 737 QPARAM_UUID: "some-id", 738 }, 739 ), 740 ] 741 self.mock_client.request.assert_has_calls(calls) 742 743 # Check the return values 744 self.assertEqual(bucket_props, '{"some": "props"}') 745 self.assertEqual(bucket_summ, {"some": "summary"}) 746 747 # Test with invalid flt_presence 748 with self.assertRaises(ValueError): 749 self.ais_bck.info(flt_presence=6) 750 751 def test_write_dataset_skip_missing_true_but_missing_attributes(self): 752 self.dataset_config.write_shards = MagicMock() 753 754 self.ais_bck.write_dataset( 755 self.dataset_config, log_dir="/fake/log", skip_missing=True 756 ) 757 758 self.dataset_config.write_shards.assert_called() 759 args, kwargs = self.dataset_config.write_shards.call_args 760 self.assertTrue(callable(kwargs["post"])) 761 762 def test_write_dataset_successful(self): 763 self.dataset_config.write_shards = MagicMock() 764 765 self.ais_bck.write_dataset(self.dataset_config, skip_missing=True) 766 self.dataset_config.write_shards.assert_called() 767 args, kwargs = self.dataset_config.write_shards.call_args 768 self.assertTrue(callable(kwargs["post"]))