github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/docs/demos/etl-imagenet.cast (about)

     1  {"version":2,"width":100,"height":35,"timestamp":1607341712,"theme":{},"env":{"SHELL":"/bin/bash","TERM":"xterm-256color"}}
     2  [0.055,"o","$ "]
     3  [0.623,"o","kubectl get pods"]
     4  [1.653,"o","\r\n"]
     5  [6.321,"o","NAME                   READY   STATUS    RESTARTS   AGE\r\ndemo-ais-admin-98bh5   1/1     Running   0          27h\r\ndemo-ais-proxy-p7tvk   1/1     Running   0          3h19m\r\ndemo-ais-proxy-wn2xs   1/1     Running   0          27h\r\ndemo-ais-proxy-zlfh9   "]
     6  [6.322,"o","1/1     Running   0          3h19m\r\ndemo-ais-target-0      1/1     Running   1          3h20m\r\ndemo-ais-target-1      1/1     Running   0          3h16m\r\ndemo-ais-target-2      1/1     Running   0          27h\r\n"]
     7  [6.323,"o","$ "]
     8  [8.487,"o","# Login to admin container"]
     9  [9.746,"o","\r\n$ "]
    10  [11.215,"o","kubectl exec -it demo-ais-admin-98bh5 -- /bin/bash"]
    11  [11.854,"o","\r\n"]
    12  [14.233,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    13  [14.357,"o","\r\u001b[K\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    14  [15.641,"o","# Download ImageNet dataset from web"]
    15  [16.548,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    16  [18.446,"o","wget -nv -O imagenet.tar http://www.image-net.org/challenges/LSVRC/2012/dd31405981ef5f776aa17412e1f0c112/ILSVRC2012_img_train_t3.tar"]
    17  [20.87,"o","\r\n"]
    18  [23.375,"o","2020-12-03 12:06:54 URL:http://www.image-net.org/challenges/LSVRC/2012/dd31405981ef5f776aa17412e1f0c112/ILSVRC2012_img_train_t3.tar [762460160/762460160] -\u003e \"imagenet.tar\" [1]\r\n"]
    19  [23.406,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    20  [24.917,"o","# Untar the file to a directory"]
    21  [25.78,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    22  [27.619,"o","mkdir tars \u0026\u0026 tar -C tars -xvf imagenet.tar \u003e/dev/null"]
    23  [28.392,"o","\r\n"]
    24  [31.62,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    25  [32.291,"o","# See what is inside of the tars"]
    26  [33.173,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    27  [33.788,"o","tar -tvf tars/n02086910.tar | head -n 5"]
    28  [35.895,"o","\r\n"]
    29  [35.977,"o","-rw-r--r-- aditya86/users 28742 2012-07-03 23:19 n02086910_10147.JPEG\r\n-rw-r--r-- aditya86/users 42141 2012-07-03 23:19 n02086910_10204.JPEG\r\n"]
    30  [35.978,"o","-rw-r--r-- aditya86/users 13748 2012-07-03 23:19 n02086910_103.JPEG\r\n-rw-r--r-- aditya86/users 13750 2012-07-03 23:19 n02086910_1048.JPEG\r\n-rw-r--r-- aditya86/users 23563 2012-07-03 23:19 n02086910_1052.JPEG\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    31  [36.98,"o","# Create AIS bucket"]
    32  [37.917,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    33  [39.254,"o","ais bucket create ais://src"]
    34  [40.174,"o","\r\n"]
    35  [40.292,"o","\"ais://src\" bucket created"]
    36  [40.295,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    37  [41.255,"o","# Put tars to the bucket"]
    38  [42.046,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    39  [42.877,"o","ais object put tars ais://src -r"]
    40  [43.862,"o","\r\n"]
    41  [43.888,"o","Files to upload:\r\n"]
    42  [43.89,"o","EXTENSION"]
    43  [43.892,"o","\t COUNT"]
    44  [43.894,"o","\t SIZE"]
    45  [43.895,"o","\r\n.tar\t\t 120\t 727.07MiB\r\nTOTAL\t\t120\t727.07MiB\r\nProceed uploading to bucket \"ais://src\"? [Y/N]: "]
    46  [44.564,"o","y"]
    47  [44.877,"o","\r\n"]
    48  [45.603,"o","120 objects put into \"ais://src\" bucket"]
    49  [45.603,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    50  [45.91,"o","# See the content of the bucket"]
    51  [46.929,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    52  [47.874,"o","ais bucket ls ais://src | head -n 5"]
    53  [48.766,"o","\r\n"]
    54  [48.816,"o","NAME\t\t SIZE\t\t \r\nn02085620.tar\t 5.67MiB\t \r\nn02085782.tar\t 4.63MiB\t \r\nn02085936.tar\t 7.01MiB\t \r\nn02086079.tar\t 4.41MiB\t \r\n"]
    55  [48.817,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    56  [49.812,"o","# Create transformation function"]
    57  [50.858,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    58  [56.309,"o","cat code.py"]
    59  [57.12,"o","\r\n"]
    60  [57.137,"o","# Import necessary dependencies\r\nimport torch, tarfile, io\r\nfrom PIL import Image\r\nfrom torchvision import transforms\r\n\r\n# Define transformations\r\npreprocessing = transforms.Compose([\r\n    transforms.RandomResizedCrop(224),\r\n    transforms.RandomHorizontalFlip(),\r\n    transforms.ToTensor(),\r\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\r\n    transforms.ToPILImage(),\r\n    transforms.Lambda(lambda x: x.tobytes()),\r\n])\r\n\r\n# Define transform function\r\ndef transform(input_bytes: bytes) -\u003e bytes:\r\n    input_tar = tarfile.open(fileobj=io.BytesIO(input_bytes))\r\n\r\n    output_bytes = io.BytesIO()\r\n    output_tar = tarfile.open(fileobj=output_bytes, mode=\"w|\")\r\n\r\n    for member in input_tar:\r\n        image = Image.open(input_tar.extractfile(member))\r\n        processed = preprocessing(image)\r\n\r\n        member.size = len(processed)\r\n        output_tar.addfile(member, io.BytesIO(processed))\r\n\r\n    return output_bytes.getvalue()\r\n\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5"]
    61  [57.137,"o",":/# "]
    62  [63.829,"o","# Define dependencies"]
    63  [65.528,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    64  [71.496,"o","cat deps.txt"]
    65  [72.569,"o","\r\n"]
    66  [72.583,"o","torch==1.6.0\r\ntorchvision==0.7.0\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    67  [78.207,"o","# Start ETL"]
    68  [78.859,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    69  [80.183,"o","ais etl init code --from-file=code.py --deps-file=deps.txt --runtime=python3"]
    70  [82.062,"o","\r\n"]
    71  [83.557,"o","P7G4uj65at\r\n"]
    72  [83.582,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    73  [86.732,"o","# Transform the dataset and store result in ais://dst bucket"]
    74  [88.442,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    75  [90.295,"o","ais etl bucket"]
    76  [91.707,"o"," "]
    77  [93.495,"o","P7G4uj65at "]
    78  [94.705,"o","ais://src ais://dst"]
    79  [96.293,"o","\r\n"]
    80  [96.464,"o","A0CEfz_11"]
    81  [96.468,"o","\r\n"]
    82  [96.478,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
    83  [99.078,"o","ais job wait xaction"]
    84  [100.177,"o"," "]
    85  [101.177,"o","A0CEfz_11"]
    86  [103.367,"o","\r\n"]
    87  [104.861,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    88  [106.134,"o","# Show the content of the bucket"]
    89  [107.177,"o","\r\n"]
    90  [107.177,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    91  [111.904,"o","ais bucket ls ais://dst | head -n 6"]
    92  [115.596,"o","\r\n"]
    93  [115.742,"o","NAME\t\t SIZE\t\t \r\nn02085620.tar\t 21.88MiB\t \r\nn02085782.tar\t 26.64MiB\t \r\nn02085936.tar\t 36.29MiB\t \r\nn02086079.tar\t 21.46MiB\t \r\nn02086240.tar\t 30.82MiB\t \r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    94  [117.704,"o","# Save a transformed tar locally"]
    95  [118.145,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
    96  [120.605,"o","ais object get ais://dst/n02085620.tar output.tar"]
    97  [121.544,"o","\r\n"]
    98  [121.848,"o","\"n02085620.tar\" has the size 21.88MiB (22947840 B)"]
    99  [121.848,"o","\r\n"]
   100  [121.854,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# \u001b[K"]
   101  [127.488,"o","# See the content of the tar"]
   102  [128.289,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
   103  [129.267,"o","\r\n"]
   104  [129.584,"o","\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
   105  [131.341,"o","tar -tvf output.tar | head  -n 5"]
   106  [133.102,"o","\r\n"]
   107  [133.107,"o","-rw-r--r-- aditya86/users 150528 2012-07-03 23:19 n02085620_10074.JPEG"]
   108  [133.107,"o","\r\n-rw-r--r-- aditya86/users 150528 2012-07-03 23:19 n02085620_10131.JPEG"]
   109  [133.109,"o","\r\n-rw-r--r-- aditya86/users 150528 2012-07-03 23:19 n02085620_10621.JPEG"]
   110  [133.109,"o","\r\n-rw-r--r-- aditya86/users 150528 2012-07-03 23:19 n02085620_1073.JPEG"]
   111  [133.11,"o","\r\n-rw-r--r-- aditya86/users 150528 2012-07-03 23:19 n02085620_10976.JPEG"]
   112  [133.11,"o","\r\n\u001b]0;root@demo-ais-admin-98bh5: /\u0007root@demo-ais-admin-98bh5:/# "]
   113  [134.383,"o","# As expected all images were normalized"]
   114  [138.151,"o","\u001b]0;knopt@januszm-System-Product-Name: ~\u0007\u001b[01;32mknopt@januszm-System-Product-Name\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
   115  [140.7,"o","exit\r\n"]