github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/python/aistore/sdk/object_iterator.py (about)

     1  #
     2  # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
     3  #
     4  
     5  from typing import Optional, List, Callable
     6  
     7  from aistore.sdk.types import BucketEntry
     8  
     9  
    10  # pylint: disable=unused-variable
    11  class ObjectIterator:
    12      """
    13      Represents an iterable that will fetch all objects from a bucket, querying as needed with the specified function
    14  
    15      Args:
    16          list_objects (Callable): Function returning a BucketList from an AIS cluster
    17      """
    18  
    19      _fetched: Optional[List[BucketEntry]] = []
    20      _token: str = ""
    21      _uuid: str = ""
    22  
    23      def __init__(self, list_objects: Callable):
    24          self._list_objects = list_objects
    25  
    26      def __iter__(self):
    27          return self
    28  
    29      def __next__(self) -> BucketEntry:
    30          # Iterator is exhausted.
    31          if len(self._fetched) == 0 and self._token == "" and self._uuid != "":
    32              raise StopIteration
    33          # Read the next page of objects.
    34          if len(self._fetched) == 0:
    35              resp = self._list_objects(uuid=self._uuid, token=self._token)
    36              self._fetched = resp.entries
    37              self._uuid = resp.uuid
    38              self._token = resp.continuation_token
    39              # Empty page and token mean no more objects left.
    40              if len(self._fetched) == 0 and self._token == "":
    41                  raise StopIteration
    42          return self._fetched.pop(0)