github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/python/examples/ais-etl/etl_md5_hpush_streaming.py (about)

     1  """
     2  ETL to calculate md5 of an object with streaming.
     3  Communication Type: hpush://
     4  
     5  Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
     6  """
     7  
     8  import hashlib
     9  from aistore import Client
    10  from aistore.sdk import Bucket
    11  
    12  client = Client("http://192.168.49.2:8080")
    13  
    14  
    15  def transform(reader, writer):
    16      checksum = hashlib.md5()
    17      for b in reader:
    18          checksum.update(b)
    19      writer.write(checksum.hexdigest().encode())
    20  
    21  
    22  client.etl("etl-stream").init_code(
    23      transform=transform,
    24      chunk_size=32768,
    25  )
    26  
    27  
    28  job_id = client.bucket("from-bck").transform(
    29      etl_name="etl-stream", to_bck=Bucket("to-bck"), ext={"jpg": "txt"}
    30  )
    31  client.job(job_id).wait()