github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/bench/tools/aisloader-composer/oci_s3_test.sh (about)

     1  #!/bin/bash
     2  
     3  # dont forget to check the number of aisloader workers and update path to lists below
     4  
     5  S3_1MB_BUCKET="s3://ais-vm"
     6  S3_10MB_BUCKET="s3://ais-jm"
     7  S3_1MB_OBJ_LIST="/Users/abhgaikwad/lists/ais-vm-objects.txt"
     8  S3_10MB_OBJ_LIST="/Users/abhgaikwad/lists/ais-jm-objects.txt"
     9  S3_ENDPOINT="https://s3.amazonaws.com"
    10  
    11  # Verify the accuracy of 'aws_creds' secret. If incorrect, update 'aistore/bench/tools/aisloader-composer/playbooks/cloud/vars/aws_config.yml' with the proper credentials and configuration, then execute the following commands.
    12  
    13  # source common.sh
    14  # PLAYBOOK=playbooks/cloud/ais_aws_config.yml
    15  # run_ansible_playbook "$PLAYBOOK"
    16  
    17  # run 5 minutes of DIRECT on S3
    18  AISLOADER_S3_ENDPOINT=$S3_ENDPOINT AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_1MB_BUCKET AISLOADER_OBJECTS=$S3_1MB_OBJ_LIST ./direct_get_bench.sh
    19  AISLOADER_S3_ENDPOINT=$S3_ENDPOINT AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_10MB_BUCKET AISLOADER_OBJECTS=$S3_10MB_OBJ_LIST ./direct_get_bench.sh
    20  
    21  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/direct_get/ais-vm/
    22  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/direct_get/ais-jm/
    23  
    24  # 5 min of COLD GET on AIStore
    25  ./clear_pagecache.sh
    26  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_1MB_BUCKET AISLOADER_OBJECTS=$S3_1MB_OBJ_LIST ./run_get_bench.sh
    27  ./clear_pagecache.sh
    28  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_10MB_BUCKET AISLOADER_OBJECTS=$S3_10MB_OBJ_LIST ./run_get_bench.sh
    29  
    30  # option for how long to run
    31  # AISLOADER_EPOCHS=1 OR AISLOADER_DURATION="5m"
    32  
    33  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-vm/
    34  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-jm/
    35  
    36  cp -r $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-vm $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-vm-old
    37  cp -r $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-jm $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-jm-old
    38  
    39  # cache all objects
    40  AISLOADER_EPOCHS=1 AISLOADER_BUCKET=$S3_1MB_BUCKET AISLOADER_OBJECTS=$S3_1MB_OBJ_LIST ./run_get_bench.sh
    41  AISLOADER_EPOCHS=1 AISLOADER_BUCKET=$S3_10MB_BUCKET AISLOADER_OBJECTS=$S3_10MB_OBJ_LIST ./run_get_bench.sh
    42  
    43  # 5 min of WARM GET on AIStore
    44  ./clear_pagecache.sh
    45  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_1MB_BUCKET AISLOADER_OBJECTS=$S3_1MB_OBJ_LIST ./run_get_bench.sh
    46  ./clear_pagecache.sh
    47  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$S3_10MB_BUCKET AISLOADER_OBJECTS=$S3_10MB_OBJ_LIST ./run_get_bench.sh
    48  
    49  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-vm/
    50  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-jm/