github.com/diadata-org/diadata@v1.4.593/testenv.sh (about)

     1  #!/usr/bin/env bash
     2  #
     3  # Description: This script is used to manage the DIA development environment. Helpful on start/stop the platform, and create/remove resources
     4  # Vesion: 0.1.0
     5  
     6  SOURCE="${BASH_SOURCE[0]}"
     7  while [ -h "$SOURCE" ]; do
     8      SCRIPT_FOLDER="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)"
     9      SOURCE="$(readlink "$SOURCE")"
    10      [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_FOLDER/$SOURCE"
    11  done
    12  SCRIPT_FOLDER="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)"
    13  unset SOURCE
    14  
    15  function usage() {
    16      echo "DIA Test Space"
    17      echo
    18      echo "Usage:"
    19      echo "  env [OPTIONS...] COMMAND [ARGS...]"
    20      echo
    21      echo "Options:"
    22      echo "  -h --help                 Print help"
    23      echo "  -v --verbose              Verbose mode on"
    24      echo "  -c --cpus <num>           Define the num of virtual CPUs (eg: 8)"
    25      echo "     --full                 Full mode on"
    26      echo "     --single               Single mode on"
    27      echo
    28      echo "Available commands:"
    29      echo "  start                     Start cluster"
    30      echo "  stop                      Stop the cluster"
    31      echo "  delete                    Delete all cluster resources"
    32      echo "  build                     Build DIA platform and prepare container images"
    33      echo "  install                   Install DIA platform"
    34      echo "  uninstall                 Un-install DIA platform"
    35      echo "  snapshot                  Pull and import snapshot of data"
    36      echo "  create <resource>         Create a new resource"
    37      echo "    scraper-cex [id]          Create a CEX scraper"
    38      echo "    scraper-dex [id]          Create a DEX scraper"
    39      echo "    scraper-foreign [id]      Create a foreign scraper"
    40      echo "    scraper-liquidity [id]    Create a liquidity scraper"
    41      echo "    cronjob                   Create a cronjob for snapshots"
    42      echo "    demos-scraper           Create a demo with all scrapers"
    43      echo "  remove                      Remove a resource"
    44      echo "    scraper-cex [id]          Remove a CEX scraper"
    45      echo "    scraper-dex [id]          Remove a DEX scraper"
    46      echo "    scraper-foreign [id]      Remove a foreign scraper"
    47      echo "    scraper-liquidity [id]    Remove a liquidity scraper"
    48      echo "    cronjob                   Remove a cronjob for snapshots"
    49      echo "    demos-scraper             Remove a demo with all scrapers"
    50      echo "  shell                     Connect to enviornment shell"
    51      echo "  logs                      Print logs"
    52      echo "  clean                     Clean unused files"
    53      echo "  info                      Show detailed information"
    54      echo "  ping                      Make ping tests"
    55      echo "  data                      List data"
    56      echo "    exchanges               List exchanges"
    57      echo "    exchange                Show exchange metadata"
    58      echo "    blockchain              Show blockchain metadata"
    59      echo "    scraper-cex             Show CEX scraper metadata"
    60      echo "    scraper-dex             Show DEX scraper metadata"
    61      echo "    scraper-liquidity       Show liquidity scraper metadata"
    62      echo "    scraper-foreign         Show foreign scraper metadata"
    63      echo "  data-reset                Reset data"
    64      echo
    65      echo "Report bugs to: <https://github.com/diadata-org/diadata/issues>"
    66  }
    67  
    68  function _image_exist(){
    69      minikube_docker_query="$(minikube -p "${minikube_profile}" ssh -- "docker images -q $1:latest" 2> /dev/null)"
    70      if [[ "${minikube_docker_query}" == "" ]]; then
    71          echo "Image $1 is not present"
    72          return 1
    73      else
    74          echo "Image $1 is present"
    75          return 0
    76      fi
    77  }
    78  
    79  function _build_ifnotexist(){
    80      if [ "$arg_single_mode" = true ]; then
    81          docker_query="$(docker images -q "$2:latest" 2> /dev/null)"
    82          if [[ "${docker_query}" == "" ]]; then
    83              echo "Image $2 is not present, building ..."
    84              docker buildx build -f "build/$1" -t "$2:latest" .
    85          else
    86              echo "Image $2 is present"
    87          fi
    88      else
    89          minikube_docker_query="$(minikube -p "${minikube_profile}" ssh -- "docker images -q $2:latest" 2> /dev/null)"
    90          if [[ "${minikube_docker_query}" == "" ]]; then
    91              echo "Image $2 is not present, building ..."
    92              if [[ "$minikube_driver" == "docker" ]]; then
    93                  eval "$(minikube -p "${minikube_profile}" docker-env)"
    94                  docker buildx build -f "build/$1" -t "$2:latest" .
    95                  eval "$(minikube -p "${minikube_profile}" docker-env --unset)"
    96              else
    97                  minikube -p "${minikube_profile}" image build -f "build/$1" -t "$2:latest" .
    98              fi
    99          else
   100              echo "Image $2 is present"
   101          fi
   102      fi
   103  }
   104  
   105  function _minikube_profile_isrunning() {
   106      minikube status --profile "$1" --format '{{.Host}}' >/dev/null 2>&1 || return 1
   107      minikube status --profile "$1" --format '{{.Kubelet}}' >/dev/null 2>&1 || return 1
   108      minikube status --profile "$1" --format '{{.APIServer}}' >/dev/null 2>&1 || return 1
   109      return 0
   110  }
   111  
   112  function _info() {
   113      echo "DIA Version: $version_detected"
   114      echo "Print Working Directory: $PWD"
   115      echo "Script Directory: $SCRIPT_FOLDER"
   116      if [[ "$OSTYPE" == "linux-gnu"* ]]; then
   117          echo "Operating System: GNU/Linux"
   118      elif [[ "$OSTYPE" == "darwin"* ]]; then
   119          echo "Operating System: Mac OSX"
   120      fi
   121      if [ "$arg_single_mode" = true ]; then
   122          echo "Docker version: v$(docker version --format '{{.Client.Version}}')"
   123      else
   124          echo "Minikube Version: $(minikube version --short)"
   125          if [[ "$minikube_driver" == "docker" ]]; then
   126              echo "Minikube Driver: ${minikube_driver} v$(docker version --format '{{.Client.Version}}')"
   127          else
   128              echo "Minikube Driver: ${minikube_driver}"
   129          fi
   130          echo "Minikube K8s: ${minikube_k8s_version}"
   131          echo "Minikube CPUs: ${minikube_hw_cpus}"
   132          echo "Minikube Memory: ${minikube_hw_ram}"
   133          echo "Minikube Disk: ${minikube_hw_disk}"
   134          echo "Minikube Profile: ${minikube_profile}"
   135      fi
   136  }
   137  
   138  function command_exists() {
   139    command -v "$1" >/dev/null 2>&1
   140  }
   141  
   142  function _status_host(){
   143      echo "Uptime: $(uptime -p)"
   144  }
   145  
   146  function main() {
   147      # Arguments parsing
   148      local command=("$@")
   149  
   150      local args=
   151      args=$(getopt -o hnsvc: --long cpus:,disk-space:,nodetached,full,single,verbose,help -n './testenv.sh' -- "$@")
   152      # shellcheck disable=SC2181
   153      if [ $? != 0 ]; then
   154          usage 1>&2
   155          return 1
   156      fi
   157      set -euo pipefail
   158      eval set -- "$args"
   159      local arg_full_mode=false
   160      local arg_single_mode=false
   161      local arg_verbose_mode=false
   162      local arg_cpus=""
   163      while true; do
   164          case "$1" in
   165              --verbose | -v)
   166                  arg_verbose_mode=true
   167                  shift ;;
   168              --single)
   169                  arg_single_mode=true
   170                  shift ;;
   171              --full | -f)
   172                  arg_full_mode=true
   173                  shift ;;
   174              --cpus | -c)
   175                  arg_cpus="$2"
   176                  if [ "$arg_cpus" = "" ]; then
   177                      echo "Invalid parameter" >&2
   178                      return 1
   179                  fi
   180                  shift 2 ;;
   181              --help | -h     ) usage; return 0 ;;
   182              --              ) shift; break    ;;
   183              *               )        break    ;;
   184          esac
   185      done
   186  
   187      # We should update the command variable after the shift operations!
   188      command=("$@")
   189      # Check dependencies
   190      if ! hash git 2> /dev/null; then echo "Git not found" >&2; return 1; fi
   191      if ! hash docker 2> /dev/null; then echo "Docker not found" >&2; return 1; fi
   192      if ! hash minikube 2> /dev/null; then echo "Minikube not found" >&2; return 1; fi
   193  
   194      # Safe checks
   195      if [[ "$OSTYPE" != "linux-gnu"* ]] && [[ "$OSTYPE" != "darwin"* ]]; then
   196          echo "Unknown operating system"
   197          exit 1
   198      fi
   199      if [ -z "${command[0]:-}" ]; then
   200          echo "No command specified" >&2
   201          exit 1
   202      fi
   203  
   204      # Variables
   205      local minikube_profile=dia
   206      local minikube_k8s_version=v1.25.7
   207      local minikube_hw_cpus=4
   208      local minikube_hw_ram=8g
   209      local minikube_hw_disk=50g
   210      local minikube_driver=docker
   211      local snapshot_docker_registry=https://registry.hub.docker.com/v2/
   212      local snapshot_docker_username=dia_contributor
   213      local snapshot_docker_password=dia_contributor_pw
   214      local snapshot_docker_email=dia_contributor@example.com
   215      local data_docker_registry=docker.io
   216      local data_docker_username=dia
   217      local data_docker_password=dia_pw
   218      local data_docker_email=dia@diadata.com
   219      # shellcheck disable=SC1091
   220      # TODO: this will break the script?
   221      if [ -e testenv.local ]; then
   222      source ./testspace/.testenv.local
   223      fi
   224      if [[ "$arg_cpus" != "" ]]; then
   225          minikube_hw_cpus="${arg_cpus}"
   226      fi
   227      local version_detected
   228      version_detected=$(git describe --tags --abbrev=0)
   229      declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc")
   230      declare -a demos_scraper_dex=("platypus" "orca", "curve")
   231      declare -a demos_scraper_liquidity=("platypus" "orca")
   232      declare -a demos_scraper_foreign=("yahoofinance")
   233  
   234      # Command
   235      case "${command[0]}" in
   236      start)
   237        echo "running start"
   238          if [ "${#command[@]}" -eq 1 ]; then
   239              if [ "$arg_verbose_mode" = true ]; then _info; fi
   240              if [ "$arg_single_mode" = true ]; then
   241                  echo "WIP"
   242              else
   243                  if ! _minikube_profile_isrunning "${minikube_profile}"; then
   244                      echo "Starting cluster ..."
   245                      minikube --profile "${minikube_profile}" start \
   246                          --kubernetes-version "${minikube_k8s_version}" \
   247                          --driver "${minikube_driver}" \
   248                          --cpus "${minikube_hw_cpus}" \
   249                          --memory "${minikube_hw_ram}" \
   250                          --disk-size "${minikube_hw_disk}" \
   251                          --mount-string="$(pwd):/mnt/diadata" \
   252                          --mount
   253                      echo "Minikube cluster started with success"
   254                      exit 0
   255                  else
   256                      echo "Cluster is already running"
   257                      exit 1
   258                  fi
   259              fi
   260          else
   261              echo "Unknown command" >&2
   262              exit 1
   263          fi
   264          ;;
   265      stop)
   266          if [ "${#command[@]}" -eq 1 ]; then
   267              echo "Stopping cluster ..."
   268              minikube -p "${minikube_profile}" stop
   269          else
   270              echo "Unknown command" >&2
   271              exit 1
   272          fi
   273          ;;
   274      delete)
   275          if [ "${#command[@]}" -eq 1 ]; then
   276              echo "Deleting cluster ..."
   277              minikube delete -p "${minikube_profile}"
   278          else
   279              echo "Unknown command" >&2
   280              exit 1
   281          fi
   282          ;;
   283      build)
   284          if [ "${#command[@]}" -eq 1 ]; then
   285              echo "Building images ..."
   286              _build_ifnotexist build/Dockerfile-DiadataBuild-114-Dev dia.build-114.dev
   287              _build_ifnotexist build/Dockerfile-DiadataBuild-117-Dev dia.build-117.dev
   288              _build_ifnotexist build/Dockerfile-DiadataBuild-119-Dev dia.build-119.dev
   289              _build_ifnotexist build/Dockerfile-DiadataBuild-120-Dev dia.build-120.dev
   290              _build_ifnotexist Dockerfile-filtersBlockService-Dev dia.filtersblockservice.dev
   291              _build_ifnotexist Dockerfile-tradesBlockService-Dev dia.tradesblockservice.dev
   292              _build_ifnotexist Dockerfile-pairDiscoveryService-Dev dia.pairdiscoveryservice.dev
   293              _build_ifnotexist Dockerfile-genericCollector-Dev dia.genericcollector.dev
   294              # FIXME: missing docker files!
   295              # _build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev
   296              # _build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev
   297              _build_ifnotexist Dockerfile-assetCollectionService-Dev dia.assetcollectionservice.dev
   298              if [ "$arg_full_mode" = true ]; then
   299                  _build_ifnotexist Dockerfile-blockchainservice-Dev dia.blockchainservice.dev
   300              fi
   301          else
   302              echo "Unknown command" >&2
   303              exit 1
   304          fi
   305          ;;
   306      install)
   307          if [ "${#command[@]}" -eq 1 ]; then
   308              _image_exist dia.build-114.dev || exit 1
   309              _image_exist dia.build-117.dev || exit 1
   310              _image_exist dia.build-119.dev || exit 1
   311              _image_exist dia.build-120.dev || exit 1
   312              _image_exist dia.filtersblockservice.dev || exit 1
   313              _image_exist dia.tradesblockservice.dev || exit 1
   314              _image_exist dia.pairdiscoveryservice.dev || exit 1
   315              _image_exist dia.genericcollector.dev || exit 1
   316              # FIXME: missing docker files!
   317              # _image_exist dia.genericforeignscraper.dev || exit 1
   318              # _image_exist dia.liquidityscraper.dev || exit 1
   319              _image_exist dia.assetcollectionservice.dev || exit 1
   320              if [ "$arg_full_mode" = true ]; then
   321                  _image_exist dia.blockchainservice.dev || exit 1
   322              fi
   323              echo "Installing services ..."
   324              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-filtersblockservice.yaml
   325              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-tradesblockservice.yaml
   326              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-kafka.yaml
   327              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-redis.yaml
   328              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-influx.yaml
   329              if [ "$arg_full_mode" = true ]; then
   330                  minikube -p "${minikube_profile}" kubectl -- create configmap postgres-schemma --from-file=deployments/config/pginit.sql
   331                  minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-postgres.yaml
   332                  minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-prepare.yaml
   333              else
   334                  minikube -p "${minikube_profile}" kubectl -- \
   335                      create secret docker-registry regcred-read \
   336                      --docker-server="${data_docker_registry}" \
   337                      --docker-username="${data_docker_username}" \
   338                      --docker-password="${data_docker_password}" \
   339                      --docker-email="${data_docker_email}"
   340                  minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-postgres-prepopulated.yaml
   341              fi
   342              echo "Installation ended successfully"
   343          else
   344              echo "Unknown command" >&2
   345              exit 1
   346          fi
   347          ;;
   348      uninstall)
   349          if [ "${#command[@]}" -eq 1 ]; then
   350              echo "Uninstalling services ..."
   351              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-filtersblockservice.yaml || true
   352              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-tradesblockservice.yaml || true
   353              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-kafka.yaml || true
   354              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-redis.yaml || true
   355              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-influx.yaml || true
   356              if [ "$arg_full_mode" = true ]; then
   357                  minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-postgres.yaml || true
   358                  minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-schemma || true
   359              else
   360                  minikube -p "${minikube_profile}" kubectl -- delete secret regcred-read || true
   361                  minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-postgres-prepopulated.yaml || true
   362              fi
   363              echo "Services uninstalled with success"
   364          else
   365              echo "Unknown command" >&2
   366              exit 1
   367          fi
   368          ;;
   369      snapshot)
   370          if [ "${#command[@]}" -gt 1 ]; then
   371          # Check if unzip, wget, and psql commands exist
   372          if ! command_exists unzip; then
   373            echo "Error: 'unzip' command not found. Please install unzip and try again."
   374            exit 1
   375          fi
   376  
   377          if ! command_exists wget; then
   378            echo "Error: 'wget' command not found. Please install wget and try again."
   379            exit 1
   380          fi
   381  
   382          if ! command_exists psql; then
   383            echo "Error: 'psql' command not found. Please install PostgreSQL client and try again."
   384            exit 1
   385          fi
   386  
   387          # Ask for variables with _EXTRACT and export them
   388          read -p "Enter Postgres Server: " PGHOST_EXTRACT
   389          read -p "Enter Postgres Port: " PGPORT_EXTRACT
   390          read -p "Enter Postgres User: " PGUSER_EXTRACT
   391          read -p "Enter Postgres Password: " PGPASSWORD_EXTRACT
   392          read -p "Enter Postgres Database: " PGDB_EXTRACT
   393  
   394          export PGHOST=${PGHOST_EXTRACT}
   395          export PGUSER=${PGUSER_EXTRACT}
   396          export PGDB=${PGDB_EXTRACT}
   397          export PGPASSWORD=${PGPASSWORD_EXTRACT}
   398          export PGPORT=${PGPORT_EXTRACT}
   399  
   400          # Download and unzip the file from rest.diadata.org
   401          wget -qO snapshot.zip http://rest.diadata.org/snapshot/latest
   402          unzip -o snapshot.zip
   403  
   404          # Run the psql commands
   405          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table blockchain cascade"
   406          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table asset cascade"
   407          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchange cascade"
   408          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchangepair cascade"
   409          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table pool cascade"
   410          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table poolasset cascade"
   411          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV"
   412          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV"
   413          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV"
   414          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV"
   415          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV"
   416          # FIXME: this will raise: ERROR:  invalid input syntax for type numeric: "2023-09-07 09:05:45.012443"
   417          # psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV"
   418          psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV"
   419  
   420  
   421          # Delete the output-*.sql and snapshot.zip files
   422          rm output_*.csv
   423          rm snapshot.zip
   424  
   425          echo "Data import and processing completed successfully."
   426        fi
   427        ;;
   428      create)
   429          if [ "${#command[@]}" -gt 1 ]; then
   430              case "${command[1]}" in
   431              scraper-cex)
   432                  if [ "${#command[@]}" -gt 2 ]; then
   433                      if [[ " ${demos_scraper_cex[*]} " == *" ${command[2]} "* ]]; then
   434                          minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-cex-${command[2]}.yaml"
   435                      else
   436                          echo "Unknown demo" >&2
   437                          exit 1
   438                      fi
   439                  else
   440                      minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-cex.yaml
   441                  fi
   442                  ;;
   443              scraper-dex)
   444                  if [ "${#command[@]}" -gt 2 ]; then
   445                      if [[ " ${demos_scraper_dex[*]} " == *" ${command[2]} "* ]]; then
   446                          minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-dex-${command[2]}.yaml"
   447                      else
   448                          echo "Unknown demo" >&2
   449                          exit 1
   450                      fi
   451                  else
   452                      minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-dex.yaml
   453                  fi
   454                  ;;
   455              scraper-foreign)
   456                  if [ "${#command[@]}" -gt 2 ]; then
   457                      if [[ " ${demos_scraper_foreign[*]} " == *" ${command[2]} "* ]]; then
   458                          minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-foreign-${command[2]}.yaml"
   459                      else
   460                          echo "Unknown demo" >&2
   461                          exit 1
   462                      fi
   463                  else
   464                      minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-foreign.yaml
   465                  fi
   466                  ;;
   467              scraper-liquidity)
   468                  if [ "${#command[@]}" -gt 2 ]; then
   469                      if [[ " ${demos_scraper_liquidity[*]} " == *" ${command[2]} "* ]]; then
   470                          minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-liquidity-${command[2]}.yaml"
   471                      else
   472                          echo "Unknown demo" >&2
   473                          exit 1
   474                      fi
   475                  else
   476                      minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-liquidity.yaml
   477                  fi
   478                  ;;
   479              cronjob)
   480                  if [ "${#command[@]}" -eq 2 ]; then
   481                      minikube -p "${minikube_profile}" kubectl -- create configmap postgres-entrypoint --from-file=deployments/config/postgres-docker-entrypoint.sh
   482                      minikube -p "${minikube_profile}" kubectl -- create configmap postgres-crondump --from-file=./scripts/dump.sh
   483                      minikube -p "${minikube_profile}" kubectl -- \
   484                          create secret docker-registry regcred \
   485                          --docker-server="${snapshot_docker_registry}" \
   486                          --docker-username="${snapshot_docker_username}" \
   487                          --docker-password="${snapshot_docker_password}" \
   488                          --docker-email="${snapshot_docker_email}"
   489                      minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/cronjob-snapshot.yaml"
   490                  else
   491                      echo "Unknown command" >&2
   492                      exit 1
   493                  fi
   494                  ;;
   495              demos-scraper)
   496                  echo "Creating demo ..."
   497                  if [ "$arg_verbose_mode" = true ]; then echo "Creating CEX scrapers ..."; fi
   498                  for i in "${demos_scraper_cex[@]}"
   499                  do
   500                      minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-cex-$i.yaml"
   501                  done
   502                  if [ "$arg_verbose_mode" = true ]; then echo "Creating DEX scrapers ..."; fi
   503                  for i in "${demos_scraper_dex[@]}"
   504                  do
   505                      minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-dex-$i.yaml"
   506                  done
   507                  if [ "$arg_verbose_mode" = true ]; then echo "Creating foreign scrapers ..."; fi
   508                  for i in "${demos_scraper_foreign[@]}"
   509                  do
   510                      minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-foreign-$i.yaml"
   511                  done
   512                  if [ "$arg_verbose_mode" = true ]; then echo "Creating liquidity scrapers ..."; fi
   513                  for i in "${demos_scraper_liquidity[@]}"
   514                  do
   515                      minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-liquidity-$i.yaml"
   516                  done
   517                  echo "Demo created with success"
   518                  ;;
   519              *)
   520                  echo "Unknown command" >&2
   521                  exit 1
   522                  ;;
   523              esac
   524          else
   525              echo "Unknown command" >&2
   526              exit 1
   527          fi
   528          ;;
   529      remove)
   530          if [ "${#command[@]}" -gt 1 ]; then
   531              case "${command[1]}" in
   532              scraper-cex)
   533                  if [ "${#command[@]}" -gt 2 ]; then
   534                      if [[ " ${demos_scraper_cex[*]} " == *" ${command[2]} "* ]]; then
   535                          minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-cex-${command[2]}.yaml" || true
   536                      else
   537                          echo "Unknown demo" >&2
   538                          exit 1
   539                      fi
   540                  else
   541                      minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-cex.yaml || true
   542                  fi
   543                  ;;
   544              scraper-dex)
   545                  if [ "${#command[@]}" -gt 2 ]; then
   546                      if [[ " ${demos_scraper_dex[*]} " == *" ${command[2]} "* ]]; then
   547                          minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-dex-${command[2]}.yaml" || true
   548                      else
   549                          echo "Unknown demo" >&2
   550                          exit 1
   551                      fi
   552                  else
   553                      minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-dex.yaml || true
   554                  fi
   555                  ;;
   556              scraper-liquidity)
   557                  if [ "${#command[@]}" -gt 2 ]; then
   558                      if [[ " ${demos_scraper_liquidity[*]} " == *" ${command[2]} "* ]]; then
   559                          minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-liquidity-${command[2]}.yaml" || true
   560                      else
   561                          echo "Unknown demo" >&2
   562                          exit 1
   563                      fi
   564                  else
   565                      minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-liquidity.yaml || true
   566                  fi
   567                  ;;
   568              scraper-foreign)
   569                  if [ "${#command[@]}" -gt 2 ]; then
   570                      if [[ " ${demos_scraper_foreign[*]} " == *" ${command[2]} "* ]]; then
   571                          minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-foreign-${command[2]}.yaml" || true
   572                      else
   573                          echo "Unknown demo" >&2
   574                          exit 1
   575                      fi
   576                  else
   577                      minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-foreign.yaml || true
   578                  fi
   579                  ;;
   580              cronjob)
   581                  if [ "${#command[@]}" -eq 2 ]; then
   582                      minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/cronjob-snapshot.yaml" || true
   583                      minikube -p "${minikube_profile}" kubectl -- delete secret regcred || true
   584                      minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-entrypoint || true
   585                      minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-crondump || true
   586                  else
   587                      echo "Unknown command" >&2
   588                      exit 1
   589                  fi
   590                  ;;
   591              demos-scraper)
   592                  echo "Removing demo ..."
   593                  if [ "$arg_verbose_mode" = true ]; then echo "Removing CEX scrapers ..."; fi
   594                  for i in "${demos_scraper_cex[@]}"
   595                  do
   596                      minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-cex-$i.yaml"
   597                  done
   598                  if [ "$arg_verbose_mode" = true ]; then echo "Removing DEX scrapers ..."; fi
   599                  for i in "${demos_scraper_dex[@]}"
   600                  do
   601                      minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-dex-$i.yaml"
   602                  done
   603                  if [ "$arg_verbose_mode" = true ]; then echo "Removing foreign scrapers ..."; fi
   604                  for i in "${demos_scraper_foreign[@]}"
   605                  do
   606                      minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-foreign-$i.yaml"
   607                  done
   608                  if [ "$arg_verbose_mode" = true ]; then echo "Removing liquidity scrapers ..."; fi
   609                  for i in "${demos_scraper_liquidity[@]}"
   610                  do
   611                      minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-liquidity-$i.yaml"
   612                  done
   613                  echo "Demo removed with success"
   614                  ;;
   615              *)
   616                  echo "Unknown command" >&2
   617                  exit 1
   618                  ;;
   619              esac
   620          else
   621              echo "Unknown command" >&2
   622              exit 1
   623          fi
   624          ;;
   625      shell)
   626          if [ "${#command[@]}" -eq 1 ]; then
   627              minikube -p "${minikube_profile}" ssh
   628          else
   629              echo "Unknown command" >&2
   630              exit 1
   631          fi
   632          ;;
   633      logs)
   634          if [ "${#command[@]}" -eq 1 ]; then
   635              minikube -p "${minikube_profile}" logs -f
   636          else
   637              echo "Unknown command" >&2
   638              exit 1
   639          fi
   640          ;;
   641      clean)
   642          if [ "${#command[@]}" -eq 1 ]; then
   643              if [[ "$minikube_driver" == "docker" ]]; then
   644                  eval "$(minikube -p "${minikube_profile}" docker-env)"
   645                  docker buildx prune -a -f
   646                  eval "$(minikube -p "${minikube_profile}" docker-env --unset)"
   647              fi
   648              minikube -p "${minikube_profile}" ssh -- docker image prune -a -f
   649              minikube -p "${minikube_profile}" ssh -- docker volume prune -f
   650              minikube -p "${minikube_profile}" ssh -- docker network prune -f
   651              minikube -p "${minikube_profile}" ssh -- docker system prune -a -f
   652          else
   653              echo "Unknown command" >&2
   654              exit 1
   655          fi
   656          ;;
   657      info)
   658          if [ "${#command[@]}" -eq 1 ]; then
   659              minikube profile list
   660              _info
   661              # TODO: uptime show the host uptime, not the minikube: https://stackoverflow.com/a/28353785/2042014
   662              _minikube_uptime=$(minikube -p "${minikube_profile}" ssh -- uptime -p)
   663              echo "Minikube Uptime: $_minikube_uptime"
   664              echo; echo "Images:"
   665              minikube -p "${minikube_profile}" image ls --format table
   666              echo; echo "Status:"
   667              minikube -p "${minikube_profile}" status
   668              echo; echo "Disk available:"
   669              minikube -p "${minikube_profile}" ssh -- df -h
   670              if [[ "$minikube_driver" == "docker" ]]; then
   671                  echo; echo "Disk available of Docker:"
   672                  minikube -p "${minikube_profile}" ssh -- docker system df
   673              fi
   674          else
   675              echo "Unknown command" >&2
   676              exit 1
   677          fi
   678          ;;
   679      ping)
   680          if [ "${#command[@]}" -eq 1 ]; then
   681              echo "Ping tests:"
   682              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-redis.yaml >/dev/null 2>&1 || true
   683              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-redis.yaml >/dev/null 2>&1
   684              minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-redis >/dev/null 2>&1 && echo "  Redis OK" || echo "  Redis FAILED" >&2
   685              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-influx.yaml >/dev/null 2>&1 || true
   686              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-influx.yaml >/dev/null 2>&1
   687              minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-influx >/dev/null 2>&1 && echo "  InfluxDB OK" || echo "  InfluxDB FAILED" >&2
   688              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-postgres.yaml >/dev/null 2>&1 || true
   689              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-postgres.yaml >/dev/null 2>&1
   690              minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-postgres >/dev/null 2>&1 && echo "  PostgreSQL OK" || echo "  PostgreSQL FAILED" >&2
   691              minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-kafka.yaml >/dev/null 2>&1 || true
   692              minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-kafka.yaml >/dev/null 2>&1
   693              minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-kafka >/dev/null 2>&1 && echo "  Kafka OK" || echo "  Kafka FAILED" >&2
   694          else
   695              echo "Unknown command" >&2
   696              exit 1
   697          fi
   698          ;;
   699      data-reset)
   700          # TODO: Also reset relational data and cache
   701          if [ "${#command[@]}" -eq 1 ]; then
   702              echo "Resetting timeseries data ..."
   703              minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'DROP MEASUREMENT foreignquotation'
   704              minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'DROP MEASUREMENT tradesTmp'
   705          else
   706              echo "Unknown command" >&2
   707              exit 1
   708          fi
   709          ;;
   710      data)
   711          # TODO: "SELECT ... | wc" is very slow, maybe investigate to use COUNT statement
   712          if [ "${#command[@]}" -gt 1 ]; then
   713              case "${command[1]}" in
   714              exchanges)
   715                  num_exchanges=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchange")
   716                  echo "Exchanges:"
   717                  minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT * FROM exchange"
   718                  echo "#Exchanges: $num_exchanges"
   719                  ;;
   720              exchange)
   721                  echo "Exchange:"
   722                  read -r inputExchangeName
   723                  minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -t -c "SELECT exchange_id FROM exchange WHERE name = '$inputExchangeName'" | if [[ $(wc -l) -ge 2 ]]; then echo "Found"; else echo "Not Found"; exit 1; fi
   724                  isbridge=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT bridge FROM exchange WHERE name = '$inputExchangeName'")
   725                  isactive=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT scraper_active FROM exchange WHERE name = '$inputExchangeName'")
   726                  iscentralized=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -AXqt -c "SELECT centralized FROM exchange WHERE name = '$inputExchangeName'" | sed 's/\r$//')
   727                  echo "Bridge: $isbridge"
   728                  echo "Active: $isactive"
   729                  echo "Centralized: $iscentralized"                
   730                  if [[ "${iscentralized}" == "t" ]]; then
   731                      num_exchangepairs=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangepair WHERE exchange = '$inputExchangeName'")
   732                      num_exchangepairassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT id_quotetoken FROM exchangepair WHERE exchange = '$inputExchangeName') OR asset_id IN (SELECT id_basetoken FROM exchangepair WHERE exchange = '$inputExchangeName')")
   733                      echo "PostgreSQL #exchangepair: $num_exchangepairs (#asset: $num_exchangepairassets)"
   734                      num_exchangesymbols=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangesymbol WHERE exchange = '$inputExchangeName'")
   735                      num_exchangesymbolsassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT asset_id FROM exchangesymbol WHERE exchange = '$inputExchangeName')")
   736                      echo "PostgreSQL #exchangesymbol: $num_exchangesymbols (#asset: $num_exchangesymbolsassets)"
   737                  else
   738                      blockchain=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT blockchain FROM exchange WHERE name = '$inputExchangeName'")
   739                      echo "Blockchain: $blockchain"
   740                      num_pools=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM pool WHERE exchange = '$inputExchangeName'")
   741                      last_liquidityupdate=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT time_stamp FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName') ORDER BY time_stamp DESC LIMIT 1")
   742                      num_poolassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName')")
   743                      num_poolassetsassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT asset_id FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName'))")
   744                      echo "PostgreSQL #pool: $num_pools (#poolasset: $num_poolassets updated at $last_liquidityupdate (#asset: $num_poolassetsassets))"
   745                  fi
   746                  num_trades=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM tradesTmp WHERE exchange = '$inputExchangeName'" | wc -l)
   747                  echo "InfluxDB #trades: $num_trades"
   748                  ;;
   749              blockchain)
   750                  echo "Blockchain:"
   751                  read -r inputBlockchainName
   752                  verificationmechanism=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT verificationmechanism FROM blockchain WHERE name = '$inputBlockchainName'")
   753                  genesisdate=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT genesisdate FROM blockchain WHERE name = '$inputBlockchainName'")
   754                  chainid=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT chain_id FROM blockchain WHERE name = '$inputBlockchainName'")
   755                  echo "Consensus: $verificationmechanism"
   756                  echo "Genesis Date: $genesisdate"
   757                  echo "Chain ID: $chainid"
   758                  num_blockchainassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE blockchain = '$inputBlockchainName'")
   759                  echo "PostgreSQL #asset: $num_blockchainassets"
   760                  ;;
   761              scraper-cex)
   762                  echo wip
   763                  ;;
   764              scraper-dex)
   765                  echo wip
   766                  ;;
   767              scraper-liquidity)
   768                  echo wip
   769                  ;;
   770              scraper-foreign)
   771                  echo "Exchange:"
   772                  read -r inputExchangeName
   773                  num_foreignquotation=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM foreignquotation WHERE source = '$inputExchangeName'" | wc -l)
   774                  echo "InfluxDB #foreignquotation: $num_foreignquotation"
   775                  minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM foreignquotation WHERE source = '$inputExchangeName' ORDER BY time DESC LIMIT 10"
   776                  ;;
   777              *)
   778                  echo "Unknown command" >&2
   779                  exit 1
   780                  ;;
   781              esac
   782          else
   783              # InfluxDB
   784              num_trades=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM trades' | wc -l)
   785              echo "InfluxDB #trades: $num_trades"
   786              num_tradestmp=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM tradesTmp' | wc -l)
   787              echo "InfluxDB #tradesTmp: $num_tradestmp"
   788              num_filters=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM filters' | wc -l)
   789              echo "InfluxDB #filters: $num_filters"
   790              num_quotations=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM assetQuotations' | wc -l)
   791              echo "InfluxDB #assetQuotations: $num_quotations"
   792              num_foreignquotation=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM foreignquotation' | wc -l)
   793              echo "InfluxDB #foreignquotation: $num_foreignquotation"
   794              # PostgreSQL
   795              num_exchanges=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchange")
   796              echo "PostgreSQL #exchange: $num_exchanges"
   797              num_pools=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM pool")
   798              echo "PostgreSQL #pool: $num_pools"
   799              num_poolassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM poolasset")
   800              echo "PostgreSQL #poolasset: $num_poolassets"
   801              num_assets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset")
   802              echo "PostgreSQL #asset: $num_assets"
   803              num_exchangepairs=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangepair")
   804              echo "PostgreSQL #exchangepair: $num_exchangepairs"
   805              num_exchangesymbols=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangesymbol")
   806              echo "PostgreSQL #exchangesymbol: $num_exchangesymbols"
   807          fi
   808          ;;
   809      *)
   810          echo "Unknown command ${command[0]}" >&2
   811          exit 1
   812          ;;
   813      esac
   814  }
   815  
   816  main "$@"