github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/lightning_gcs/run.sh (about)

     1  #!/bin/bash
     2  #
     3  # Copyright 2020 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  set -eux
    17  DB="gcs_test"
    18  TABLE="tbl"
    19  
    20  check_cluster_version 4 0 0 'local backend' || exit 0
    21  
    22  GCS_HOST="localhost"
    23  GCS_PORT=21808
    24  GCS_STORAGE=$TEST_DIR/storage
    25  BUCKET="lightning-gcs"
    26  rm -rf "$TEST_DIR/$DB"
    27  # NOTE: if the bucket alredy exists, 
    28  #       the opration of the bucket in fake-gcs-server will fail.
    29  rm -rf $GCS_STORAGE
    30  mkdir -p "$TEST_DIR/$DB"
    31  
    32  # we need set public-host for download file, or it will return 404 when using client to read.
    33  bin/fake-gcs-server -scheme http -host $GCS_HOST -port $GCS_PORT -filesystem-root $GCS_STORAGE -public-host $GCS_HOST:$GCS_PORT &
    34  i=0
    35  while ! curl -o /dev/null -v -s "http://$GCS_HOST:$GCS_PORT/"; do
    36      i=$(($i+1))
    37      if [ $i -gt 7 ]; then
    38          echo 'Failed to start gcs-server'
    39          exit 1
    40      fi
    41      sleep 2
    42  done
    43  
    44  # start oauth server
    45  bin/oauth &
    46  
    47  stop_gcs() {
    48      killall -9 fake-gcs-server || true
    49      killall -9 oauth || true
    50  }
    51  trap stop_gcs EXIT
    52  
    53  
    54  # we need start a oauth server or gcs client will failed to handle request.
    55  KEY=$(cat <<- EOF
    56  {
    57    "type": "service_account",
    58    "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCT524vzG7uEVtX\nojcHbyQzVwlcaGkg1DWWLT+SufD08UYF0bsfcD0Etrtzo4ggwdxJQy5ygl3TNlcD\nKdelWbVyGfg9/sNB1RDlZYbQb0LVLHKjkVs7JyJsxrLk2e6NqD9ajwTEJUcLAQkj\nxlCcIi51beqrIRlvHjbtGwet/dNnRLSZf+i9SHvB2j64+RVYdnyf/IiLBvYyu7hF\nT6VjlljdbwC4TZ2jpfDL8nHRTiDiV+CX3/iH8MlMEOSM30AO5MPNVCZLlTA9W24a\nKi4NPBBlJLvG2mQELYdbhdM64iMvbPkDRtajJD6ogPB7wUoWbtSke5oOJNyV1HNt\nn91JH/dlAgMBAAECggEAQBwve2GSbfgxD0Xds4e9+dEO2jLZ6uSBS9TWOywFIa9Z\nqlkUUtbMZDgu/buTXJubeFg6EGGo+M4TnmfrNR2zFD/khj7hdS49kinVa5Dmt895\n66Osl3HprpvcXG2IxXd56q+Woc0Ew+TRiOPD+kGowLcB4ubIhw1iQpmWVRlyos6Q\nyvHssolrqOkRK9+1asixgow2Y15HtpXFN3XDIVj3gfdN1Zg80S66bTap1DS+dkJH\nSMgEZRilAjUGzbroqvZCiymlIJP5Jj5L5Wy8Qp/k1ixK10oaPgwvdmwXHX/DZ0vC\nT6XwpIaCYd3/XUWBHvrmQHFucWVPISZRi5WidggzuwKBgQDNHrxKaDrxcrV5Ncgu\npQrtQvTsIUCJGMo5m30X0Ac5CsIssOoQHdtEQW1ehJ8DtJRRb9rdWc4aelXsDUr+\no2m1zyZzM6S7IO2YhGDAo7Uu3fy1r33qYAt6uS/nHaJBpsKcyqqK+0wPDikdPLLx\nBBWZHF6WoswDEUVLQa/hHgpjPwKBgQC4l2/6xShNoobivzk8AE/Acq7PazA8gu4K\nY0UghTBlAst4RvBTURYZ2V3uw0S2FbfwL0/snHhNWZl5XjBX/H9oQmLri5qGOOpf\n9A11p5kd0x1mHDgTm/k7EgoskdXGB5NqXIB7l/3UI8Sk2N1PzHwyJJYfaB+EWTs8\n+LVy99VQWwKBgQCilRwVtiwSOSPSYWi8YCEbEpljmK+4eye/JZmviDpRYk+qcMf1\n4lRr85gm9OO9YiK1sf0+ufH9Vr5IDflFgG1HqFwHsAWANYdd/n9Z8eior1ehAurB\nHUO8EJEBlaGIfA+Bi7pF0w3kWQsJm5USKHSeGbh3ma4vOD8+eWBZBSCirQKBgQCe\n1uEq/sChnXtIXpgXg4Uc6xJ1tZy6VUgUdDulsjZklTUU+KYQa7QC5kKoFCtqK+It\nseiqiDIVDUa9Y0liTQotYwLQAT8kxJEZpF54oZFmUqX3mcy/QvYB2JIcrBkx4I7/\ndT2yHKX1CBpMZ7h41FMCquzrdaO5NTd+Td2FYrGSBQKBgEBnAerHh/NafYlVumlS\nVgouR9IketTegyEyntVyEvENx8OA5ZLMywCIKbPMFZgPR0RgDpyDxKauCU2E09e/\nboN76UOuOg11fknJh7vFbUbzM6BXvXVOTyX9ZtZBQcd5Y3tV+tYD1tHUgurGYWb+\nyHLBMOlXdpn0gZ4rwoIQgzD9\n-----END PRIVATE KEY-----\n",
    59    "client_email": "test@email.com",
    60    "token_uri": "http://localhost:5000/oauth/token"
    61  }
    62  EOF
    63  )
    64  
    65  # save CREDENTIALS to file
    66  echo $KEY > "tests/$TEST_NAME/config.json"
    67  
    68  # export test CREDENTIALS for gcs oauth
    69  export GOOGLE_APPLICATION_CREDENTIALS="tests/$TEST_NAME/config.json"
    70  
    71  # create gcs bucket
    72  curl -XPOST http://$GCS_HOST:$GCS_PORT/storage/v1/b -d "{\"name\":\"${BUCKET}\"}"
    73  
    74  DATA_PATH="$TEST_DIR/$DB"
    75  mkdir -p $DATA_PATH/$DB
    76  echo "CREATE DATABASE $DB;" > "$DATA_PATH/$DB-schema-create.sql"
    77  echo "CREATE TABLE $TABLE(i INT, s varchar(32));" > "$DATA_PATH/$DB.$TABLE-schema.sql"
    78  echo "INSERT INTO $TABLE (i, s) VALUES (1, \"1\"),(2, \"test2\"), (3, \"qqqtest\");" > "$DATA_PATH/$DB.$TABLE.sql"
    79  cat > "$DATA_PATH/$DB.$TABLE.0.csv" << _EOF_
    80  i,s
    81  100,"test100"
    82  101,"\""
    83  102,"😄😄😄😄😄"
    84  104,""
    85  _EOF_
    86  
    87  # upload files to gcs
    88  curl -XPOST --data-binary @$DATA_PATH/$DB-schema-create.sql "http://$GCS_HOST:$GCS_PORT/upload/storage/v1/b/$BUCKET/o?uploadType=media&name=$DB/$DB-schema-create.sql"
    89  curl -XPOST --data-binary @$DATA_PATH/$DB.$TABLE-schema.sql "http://$GCS_HOST:$GCS_PORT/upload/storage/v1/b/$BUCKET/o?uploadType=media&name=$DB/$DB.$TABLE-schema.sql"
    90  curl -XPOST --data-binary @$DATA_PATH/$DB.$TABLE.sql "http://$GCS_HOST:$GCS_PORT/upload/storage/v1/b/$BUCKET/o?uploadType=media&name=$DB/$DB.$TABLE.sql"
    91  curl -XPOST --data-binary @$DATA_PATH/$DB.$TABLE.0.csv "http://$GCS_HOST:$GCS_PORT/upload/storage/v1/b/$BUCKET/o?uploadType=media&name=$DB/$DB.$TABLE.0.csv"
    92  
    93  # Fill in the database
    94  # Start importing the tables.
    95  run_sql "DROP DATABASE IF EXISTS $DB;"
    96  run_sql "DROP TABLE IF EXISTS $DB.$TABLE;"
    97  
    98  SOURCE_DIR="gcs://$BUCKET/$DB?endpoint=http://$GCS_HOST:$GCS_PORT/storage/v1/"
    99  run_lightning -d $SOURCE_DIR --backend local 2> /dev/null
   100  run_sql "SELECT count(*), sum(i) FROM \`$DB\`.$TABLE"
   101  check_contains "count(*): 7"
   102  check_contains "sum(i): 413"