github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/bench/tools/aisloader-composer/oci_pbss_test.sh (about)

     1  #!/bin/bash
     2  
     3  # dont forget to check the number of aisloader workers and update path to lists below
     4  
     5  PBSS_1MB_BUCKET="s3://ais-throughput-test-replica"
     6  PBSS_10MB_BUCKET="s3://ais-ec-10MB"
     7  PBSS_1MB_OBJ_LIST="/PATH/TO/LIST/lists/ais-throughput-test-replica-objects.txt"
     8  PBSS_10MB_OBJ_LIST="/PATH/TO/LIST/lists/ais-ec-10MB-objects.txt"
     9  PBSS_ENDPOINT="https://pbss.s8k.io"
    10  
    11  # Verify the accuracy of 'aws_creds' secret. If incorrect, update 'aistore/bench/tools/aisloader-composer/playbooks/cloud/vars/aws_config.yml' with the proper credentials and configuration, then execute the following command.
    12  
    13  # source common.sh
    14  # PLAYBOOK=playbooks/cloud/ais_aws_config.yml
    15  # run_ansible_playbook "$PLAYBOOK"
    16  
    17  # run 5 minutes of DIRECT on PBSS
    18  AISLOADER_PBSS_ENDPOINT=$PBSS_ENDPOINT AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_1MB_BUCKET AISLOADER_OBJECTS=$PBSS_1MB_OBJ_LIST ./direct_get_bench.sh
    19  AISLOADER_PBSS_ENDPOINT=$PBSS_ENDPOINT AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_10MB_BUCKET AISLOADER_OBJECTS=$PBSS_10MB_OBJ_LIST ./direct_get_bench.sh
    20  
    21  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/direct_get/ais-throughput-test-replica/
    22  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/direct_get/ais-ec-10MB/
    23  
    24  # 5 min of COLD GET on AIStore
    25  ./clear_pagecache.sh
    26  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_1MB_BUCKET AISLOADER_OBJECTS=$PBSS_1MB_OBJ_LIST ./run_get_bench.sh
    27  ./clear_pagecache.sh
    28  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_10MB_BUCKET AISLOADER_OBJECTS=$PBSS_10MB_OBJ_LIST ./run_get_bench.sh
    29  
    30  # option for how long to run
    31  # AISLOADER_EPOCHS=1 OR AISLOADER_DURATION="5m"
    32  
    33  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-throughput-test-replica/
    34  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-ec-10MB/
    35  
    36  cp -r $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-throughput-test-replica $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-throughput-test-replica-old
    37  cp -r $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-ec-10MB $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-ec-10MB-old
    38  
    39  # cache all objects
    40  AISLOADER_EPOCHS=1 AISLOADER_BUCKET=$PBSS_1MB_BUCKET AISLOADER_OBJECTS=$PBSS_1MB_OBJ_LIST ./run_get_bench.sh
    41  AISLOADER_EPOCHS=1 AISLOADER_BUCKET=$PBSS_10MB_BUCKET AISLOADER_OBJECTS=$PBSS_10MB_OBJ_LIST ./run_get_bench.sh
    42  
    43  # 5 min of WARM GET on AIStore
    44  ./clear_pagecache.sh
    45  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_1MB_BUCKET AISLOADER_OBJECTS=$PBSS_1MB_OBJ_LIST ./run_get_bench.sh
    46  ./clear_pagecache.sh
    47  AISLOADER_DURATION="5m" AISLOADER_BUCKET=$PBSS_10MB_BUCKET AISLOADER_OBJECTS=$PBSS_10MB_OBJ_LIST ./run_get_bench.sh
    48  
    49  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-throughput-test-replica/
    50  python consolidate_results.py $GOPATH/src/github.com/NVIDIA/aistore/bench/tools/aisloader-composer/output/get/ais-ec-10MB/