github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/python/tests/integration/sdk/test_bucket_ops_stress.py (about)

     1  #
     2  # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
     3  #
     4  import unittest
     5  import random
     6  
     7  from tests.integration import REMOTE_SET
     8  from tests.integration.sdk.remote_enabled_test import RemoteEnabledTest
     9  from tests.const import STRESS_TEST_OBJECT_COUNT, TEST_TIMEOUT
    10  
    11  
    12  # pylint: disable=unused-variable,too-many-instance-attributes
    13  class TestBucketOpsStress(RemoteEnabledTest):
    14      def setUp(self) -> None:
    15          super().setUp()
    16          if REMOTE_SET:
    17              self.s3_client = self._get_boto3_client()
    18  
    19      @unittest.skipIf(
    20          not REMOTE_SET,
    21          "Remote bucket is not set",
    22      )
    23      def test_stress_copy_objects_sync_flag(self):
    24          obj_names = self._create_objects(
    25              num_obj=STRESS_TEST_OBJECT_COUNT, suffix="-suffix"
    26          )
    27          to_bck_name = "dst-bck-cp-sync"
    28          to_bck = self._create_bucket(to_bck_name)
    29  
    30          obj_group = self.bucket.objects(obj_names=obj_names)
    31          # self._evict_all_objects()
    32  
    33          # cache and verify
    34          job_id = obj_group.prefetch()
    35          self.client.job(job_id).wait(timeout=TEST_TIMEOUT * 2)
    36          self._verify_cached_objects(
    37              STRESS_TEST_OBJECT_COUNT, range(STRESS_TEST_OBJECT_COUNT)
    38          )
    39  
    40          # copy objs to dst bck
    41          copy_job = self.bucket.copy(prefix_filter=self.obj_prefix, to_bck=to_bck)
    42          # copy_job = self.bucket.objects(obj_names=self.obj_names).copy(to_bck=to_bck)
    43          self.client.job(job_id=copy_job).wait_for_idle(timeout=TEST_TIMEOUT)
    44          self.assertEqual(STRESS_TEST_OBJECT_COUNT, len(to_bck.list_all_objects()))
    45  
    46          # randomly delete 10% of the objects
    47          to_delete = random.sample(obj_names, int(STRESS_TEST_OBJECT_COUNT * 0.1))
    48  
    49          # out of band delete
    50          for obj_name in to_delete:
    51              self.s3_client.delete_object(Bucket=self.bucket.name, Key=obj_name)
    52  
    53          # test --sync flag
    54          copy_job = self.bucket.copy(
    55              prefix_filter=self.obj_prefix, to_bck=to_bck, sync=True
    56          )
    57          self.client.job(job_id=copy_job).wait_for_idle(timeout=TEST_TIMEOUT * 3)
    58          self.assertEqual(
    59              int(STRESS_TEST_OBJECT_COUNT * 0.9), len(to_bck.list_all_objects())
    60          )