github.com/diadata-org/diadata@v1.4.593/testenv.sh (about) 1 #!/usr/bin/env bash 2 # 3 # Description: This script is used to manage the DIA development environment. Helpful on start/stop the platform, and create/remove resources 4 # Vesion: 0.1.0 5 6 SOURCE="${BASH_SOURCE[0]}" 7 while [ -h "$SOURCE" ]; do 8 SCRIPT_FOLDER="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" 9 SOURCE="$(readlink "$SOURCE")" 10 [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_FOLDER/$SOURCE" 11 done 12 SCRIPT_FOLDER="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" 13 unset SOURCE 14 15 function usage() { 16 echo "DIA Test Space" 17 echo 18 echo "Usage:" 19 echo " env [OPTIONS...] COMMAND [ARGS...]" 20 echo 21 echo "Options:" 22 echo " -h --help Print help" 23 echo " -v --verbose Verbose mode on" 24 echo " -c --cpus <num> Define the num of virtual CPUs (eg: 8)" 25 echo " --full Full mode on" 26 echo " --single Single mode on" 27 echo 28 echo "Available commands:" 29 echo " start Start cluster" 30 echo " stop Stop the cluster" 31 echo " delete Delete all cluster resources" 32 echo " build Build DIA platform and prepare container images" 33 echo " install Install DIA platform" 34 echo " uninstall Un-install DIA platform" 35 echo " snapshot Pull and import snapshot of data" 36 echo " create <resource> Create a new resource" 37 echo " scraper-cex [id] Create a CEX scraper" 38 echo " scraper-dex [id] Create a DEX scraper" 39 echo " scraper-foreign [id] Create a foreign scraper" 40 echo " scraper-liquidity [id] Create a liquidity scraper" 41 echo " cronjob Create a cronjob for snapshots" 42 echo " demos-scraper Create a demo with all scrapers" 43 echo " remove Remove a resource" 44 echo " scraper-cex [id] Remove a CEX scraper" 45 echo " scraper-dex [id] Remove a DEX scraper" 46 echo " scraper-foreign [id] Remove a foreign scraper" 47 echo " scraper-liquidity [id] Remove a liquidity scraper" 48 echo " cronjob Remove a cronjob for snapshots" 49 echo " demos-scraper Remove a demo with all scrapers" 50 echo " shell Connect to enviornment shell" 51 echo " logs Print logs" 52 echo " clean Clean unused files" 53 echo " info Show detailed information" 54 echo " ping Make ping tests" 55 echo " data List data" 56 echo " exchanges List exchanges" 57 echo " exchange Show exchange metadata" 58 echo " blockchain Show blockchain metadata" 59 echo " scraper-cex Show CEX scraper metadata" 60 echo " scraper-dex Show DEX scraper metadata" 61 echo " scraper-liquidity Show liquidity scraper metadata" 62 echo " scraper-foreign Show foreign scraper metadata" 63 echo " data-reset Reset data" 64 echo 65 echo "Report bugs to: <https://github.com/diadata-org/diadata/issues>" 66 } 67 68 function _image_exist(){ 69 minikube_docker_query="$(minikube -p "${minikube_profile}" ssh -- "docker images -q $1:latest" 2> /dev/null)" 70 if [[ "${minikube_docker_query}" == "" ]]; then 71 echo "Image $1 is not present" 72 return 1 73 else 74 echo "Image $1 is present" 75 return 0 76 fi 77 } 78 79 function _build_ifnotexist(){ 80 if [ "$arg_single_mode" = true ]; then 81 docker_query="$(docker images -q "$2:latest" 2> /dev/null)" 82 if [[ "${docker_query}" == "" ]]; then 83 echo "Image $2 is not present, building ..." 84 docker buildx build -f "build/$1" -t "$2:latest" . 85 else 86 echo "Image $2 is present" 87 fi 88 else 89 minikube_docker_query="$(minikube -p "${minikube_profile}" ssh -- "docker images -q $2:latest" 2> /dev/null)" 90 if [[ "${minikube_docker_query}" == "" ]]; then 91 echo "Image $2 is not present, building ..." 92 if [[ "$minikube_driver" == "docker" ]]; then 93 eval "$(minikube -p "${minikube_profile}" docker-env)" 94 docker buildx build -f "build/$1" -t "$2:latest" . 95 eval "$(minikube -p "${minikube_profile}" docker-env --unset)" 96 else 97 minikube -p "${minikube_profile}" image build -f "build/$1" -t "$2:latest" . 98 fi 99 else 100 echo "Image $2 is present" 101 fi 102 fi 103 } 104 105 function _minikube_profile_isrunning() { 106 minikube status --profile "$1" --format '{{.Host}}' >/dev/null 2>&1 || return 1 107 minikube status --profile "$1" --format '{{.Kubelet}}' >/dev/null 2>&1 || return 1 108 minikube status --profile "$1" --format '{{.APIServer}}' >/dev/null 2>&1 || return 1 109 return 0 110 } 111 112 function _info() { 113 echo "DIA Version: $version_detected" 114 echo "Print Working Directory: $PWD" 115 echo "Script Directory: $SCRIPT_FOLDER" 116 if [[ "$OSTYPE" == "linux-gnu"* ]]; then 117 echo "Operating System: GNU/Linux" 118 elif [[ "$OSTYPE" == "darwin"* ]]; then 119 echo "Operating System: Mac OSX" 120 fi 121 if [ "$arg_single_mode" = true ]; then 122 echo "Docker version: v$(docker version --format '{{.Client.Version}}')" 123 else 124 echo "Minikube Version: $(minikube version --short)" 125 if [[ "$minikube_driver" == "docker" ]]; then 126 echo "Minikube Driver: ${minikube_driver} v$(docker version --format '{{.Client.Version}}')" 127 else 128 echo "Minikube Driver: ${minikube_driver}" 129 fi 130 echo "Minikube K8s: ${minikube_k8s_version}" 131 echo "Minikube CPUs: ${minikube_hw_cpus}" 132 echo "Minikube Memory: ${minikube_hw_ram}" 133 echo "Minikube Disk: ${minikube_hw_disk}" 134 echo "Minikube Profile: ${minikube_profile}" 135 fi 136 } 137 138 function command_exists() { 139 command -v "$1" >/dev/null 2>&1 140 } 141 142 function _status_host(){ 143 echo "Uptime: $(uptime -p)" 144 } 145 146 function main() { 147 # Arguments parsing 148 local command=("$@") 149 150 local args= 151 args=$(getopt -o hnsvc: --long cpus:,disk-space:,nodetached,full,single,verbose,help -n './testenv.sh' -- "$@") 152 # shellcheck disable=SC2181 153 if [ $? != 0 ]; then 154 usage 1>&2 155 return 1 156 fi 157 set -euo pipefail 158 eval set -- "$args" 159 local arg_full_mode=false 160 local arg_single_mode=false 161 local arg_verbose_mode=false 162 local arg_cpus="" 163 while true; do 164 case "$1" in 165 --verbose | -v) 166 arg_verbose_mode=true 167 shift ;; 168 --single) 169 arg_single_mode=true 170 shift ;; 171 --full | -f) 172 arg_full_mode=true 173 shift ;; 174 --cpus | -c) 175 arg_cpus="$2" 176 if [ "$arg_cpus" = "" ]; then 177 echo "Invalid parameter" >&2 178 return 1 179 fi 180 shift 2 ;; 181 --help | -h ) usage; return 0 ;; 182 -- ) shift; break ;; 183 * ) break ;; 184 esac 185 done 186 187 # We should update the command variable after the shift operations! 188 command=("$@") 189 # Check dependencies 190 if ! hash git 2> /dev/null; then echo "Git not found" >&2; return 1; fi 191 if ! hash docker 2> /dev/null; then echo "Docker not found" >&2; return 1; fi 192 if ! hash minikube 2> /dev/null; then echo "Minikube not found" >&2; return 1; fi 193 194 # Safe checks 195 if [[ "$OSTYPE" != "linux-gnu"* ]] && [[ "$OSTYPE" != "darwin"* ]]; then 196 echo "Unknown operating system" 197 exit 1 198 fi 199 if [ -z "${command[0]:-}" ]; then 200 echo "No command specified" >&2 201 exit 1 202 fi 203 204 # Variables 205 local minikube_profile=dia 206 local minikube_k8s_version=v1.25.7 207 local minikube_hw_cpus=4 208 local minikube_hw_ram=8g 209 local minikube_hw_disk=50g 210 local minikube_driver=docker 211 local snapshot_docker_registry=https://registry.hub.docker.com/v2/ 212 local snapshot_docker_username=dia_contributor 213 local snapshot_docker_password=dia_contributor_pw 214 local snapshot_docker_email=dia_contributor@example.com 215 local data_docker_registry=docker.io 216 local data_docker_username=dia 217 local data_docker_password=dia_pw 218 local data_docker_email=dia@diadata.com 219 # shellcheck disable=SC1091 220 # TODO: this will break the script? 221 if [ -e testenv.local ]; then 222 source ./testspace/.testenv.local 223 fi 224 if [[ "$arg_cpus" != "" ]]; then 225 minikube_hw_cpus="${arg_cpus}" 226 fi 227 local version_detected 228 version_detected=$(git describe --tags --abbrev=0) 229 declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc") 230 declare -a demos_scraper_dex=("platypus" "orca", "curve") 231 declare -a demos_scraper_liquidity=("platypus" "orca") 232 declare -a demos_scraper_foreign=("yahoofinance") 233 234 # Command 235 case "${command[0]}" in 236 start) 237 echo "running start" 238 if [ "${#command[@]}" -eq 1 ]; then 239 if [ "$arg_verbose_mode" = true ]; then _info; fi 240 if [ "$arg_single_mode" = true ]; then 241 echo "WIP" 242 else 243 if ! _minikube_profile_isrunning "${minikube_profile}"; then 244 echo "Starting cluster ..." 245 minikube --profile "${minikube_profile}" start \ 246 --kubernetes-version "${minikube_k8s_version}" \ 247 --driver "${minikube_driver}" \ 248 --cpus "${minikube_hw_cpus}" \ 249 --memory "${minikube_hw_ram}" \ 250 --disk-size "${minikube_hw_disk}" \ 251 --mount-string="$(pwd):/mnt/diadata" \ 252 --mount 253 echo "Minikube cluster started with success" 254 exit 0 255 else 256 echo "Cluster is already running" 257 exit 1 258 fi 259 fi 260 else 261 echo "Unknown command" >&2 262 exit 1 263 fi 264 ;; 265 stop) 266 if [ "${#command[@]}" -eq 1 ]; then 267 echo "Stopping cluster ..." 268 minikube -p "${minikube_profile}" stop 269 else 270 echo "Unknown command" >&2 271 exit 1 272 fi 273 ;; 274 delete) 275 if [ "${#command[@]}" -eq 1 ]; then 276 echo "Deleting cluster ..." 277 minikube delete -p "${minikube_profile}" 278 else 279 echo "Unknown command" >&2 280 exit 1 281 fi 282 ;; 283 build) 284 if [ "${#command[@]}" -eq 1 ]; then 285 echo "Building images ..." 286 _build_ifnotexist build/Dockerfile-DiadataBuild-114-Dev dia.build-114.dev 287 _build_ifnotexist build/Dockerfile-DiadataBuild-117-Dev dia.build-117.dev 288 _build_ifnotexist build/Dockerfile-DiadataBuild-119-Dev dia.build-119.dev 289 _build_ifnotexist build/Dockerfile-DiadataBuild-120-Dev dia.build-120.dev 290 _build_ifnotexist Dockerfile-filtersBlockService-Dev dia.filtersblockservice.dev 291 _build_ifnotexist Dockerfile-tradesBlockService-Dev dia.tradesblockservice.dev 292 _build_ifnotexist Dockerfile-pairDiscoveryService-Dev dia.pairdiscoveryservice.dev 293 _build_ifnotexist Dockerfile-genericCollector-Dev dia.genericcollector.dev 294 # FIXME: missing docker files! 295 # _build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev 296 # _build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev 297 _build_ifnotexist Dockerfile-assetCollectionService-Dev dia.assetcollectionservice.dev 298 if [ "$arg_full_mode" = true ]; then 299 _build_ifnotexist Dockerfile-blockchainservice-Dev dia.blockchainservice.dev 300 fi 301 else 302 echo "Unknown command" >&2 303 exit 1 304 fi 305 ;; 306 install) 307 if [ "${#command[@]}" -eq 1 ]; then 308 _image_exist dia.build-114.dev || exit 1 309 _image_exist dia.build-117.dev || exit 1 310 _image_exist dia.build-119.dev || exit 1 311 _image_exist dia.build-120.dev || exit 1 312 _image_exist dia.filtersblockservice.dev || exit 1 313 _image_exist dia.tradesblockservice.dev || exit 1 314 _image_exist dia.pairdiscoveryservice.dev || exit 1 315 _image_exist dia.genericcollector.dev || exit 1 316 # FIXME: missing docker files! 317 # _image_exist dia.genericforeignscraper.dev || exit 1 318 # _image_exist dia.liquidityscraper.dev || exit 1 319 _image_exist dia.assetcollectionservice.dev || exit 1 320 if [ "$arg_full_mode" = true ]; then 321 _image_exist dia.blockchainservice.dev || exit 1 322 fi 323 echo "Installing services ..." 324 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-filtersblockservice.yaml 325 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-tradesblockservice.yaml 326 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-kafka.yaml 327 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-redis.yaml 328 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-influx.yaml 329 if [ "$arg_full_mode" = true ]; then 330 minikube -p "${minikube_profile}" kubectl -- create configmap postgres-schemma --from-file=deployments/config/pginit.sql 331 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-postgres.yaml 332 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-prepare.yaml 333 else 334 minikube -p "${minikube_profile}" kubectl -- \ 335 create secret docker-registry regcred-read \ 336 --docker-server="${data_docker_registry}" \ 337 --docker-username="${data_docker_username}" \ 338 --docker-password="${data_docker_password}" \ 339 --docker-email="${data_docker_email}" 340 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-postgres-prepopulated.yaml 341 fi 342 echo "Installation ended successfully" 343 else 344 echo "Unknown command" >&2 345 exit 1 346 fi 347 ;; 348 uninstall) 349 if [ "${#command[@]}" -eq 1 ]; then 350 echo "Uninstalling services ..." 351 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-filtersblockservice.yaml || true 352 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-tradesblockservice.yaml || true 353 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-kafka.yaml || true 354 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-redis.yaml || true 355 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-influx.yaml || true 356 if [ "$arg_full_mode" = true ]; then 357 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-postgres.yaml || true 358 minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-schemma || true 359 else 360 minikube -p "${minikube_profile}" kubectl -- delete secret regcred-read || true 361 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-postgres-prepopulated.yaml || true 362 fi 363 echo "Services uninstalled with success" 364 else 365 echo "Unknown command" >&2 366 exit 1 367 fi 368 ;; 369 snapshot) 370 if [ "${#command[@]}" -gt 1 ]; then 371 # Check if unzip, wget, and psql commands exist 372 if ! command_exists unzip; then 373 echo "Error: 'unzip' command not found. Please install unzip and try again." 374 exit 1 375 fi 376 377 if ! command_exists wget; then 378 echo "Error: 'wget' command not found. Please install wget and try again." 379 exit 1 380 fi 381 382 if ! command_exists psql; then 383 echo "Error: 'psql' command not found. Please install PostgreSQL client and try again." 384 exit 1 385 fi 386 387 # Ask for variables with _EXTRACT and export them 388 read -p "Enter Postgres Server: " PGHOST_EXTRACT 389 read -p "Enter Postgres Port: " PGPORT_EXTRACT 390 read -p "Enter Postgres User: " PGUSER_EXTRACT 391 read -p "Enter Postgres Password: " PGPASSWORD_EXTRACT 392 read -p "Enter Postgres Database: " PGDB_EXTRACT 393 394 export PGHOST=${PGHOST_EXTRACT} 395 export PGUSER=${PGUSER_EXTRACT} 396 export PGDB=${PGDB_EXTRACT} 397 export PGPASSWORD=${PGPASSWORD_EXTRACT} 398 export PGPORT=${PGPORT_EXTRACT} 399 400 # Download and unzip the file from rest.diadata.org 401 wget -qO snapshot.zip http://rest.diadata.org/snapshot/latest 402 unzip -o snapshot.zip 403 404 # Run the psql commands 405 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table blockchain cascade" 406 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table asset cascade" 407 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchange cascade" 408 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchangepair cascade" 409 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table pool cascade" 410 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table poolasset cascade" 411 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV" 412 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV" 413 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV" 414 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV" 415 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV" 416 # FIXME: this will raise: ERROR: invalid input syntax for type numeric: "2023-09-07 09:05:45.012443" 417 # psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV" 418 psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV" 419 420 421 # Delete the output-*.sql and snapshot.zip files 422 rm output_*.csv 423 rm snapshot.zip 424 425 echo "Data import and processing completed successfully." 426 fi 427 ;; 428 create) 429 if [ "${#command[@]}" -gt 1 ]; then 430 case "${command[1]}" in 431 scraper-cex) 432 if [ "${#command[@]}" -gt 2 ]; then 433 if [[ " ${demos_scraper_cex[*]} " == *" ${command[2]} "* ]]; then 434 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-cex-${command[2]}.yaml" 435 else 436 echo "Unknown demo" >&2 437 exit 1 438 fi 439 else 440 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-cex.yaml 441 fi 442 ;; 443 scraper-dex) 444 if [ "${#command[@]}" -gt 2 ]; then 445 if [[ " ${demos_scraper_dex[*]} " == *" ${command[2]} "* ]]; then 446 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-dex-${command[2]}.yaml" 447 else 448 echo "Unknown demo" >&2 449 exit 1 450 fi 451 else 452 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-dex.yaml 453 fi 454 ;; 455 scraper-foreign) 456 if [ "${#command[@]}" -gt 2 ]; then 457 if [[ " ${demos_scraper_foreign[*]} " == *" ${command[2]} "* ]]; then 458 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-foreign-${command[2]}.yaml" 459 else 460 echo "Unknown demo" >&2 461 exit 1 462 fi 463 else 464 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-foreign.yaml 465 fi 466 ;; 467 scraper-liquidity) 468 if [ "${#command[@]}" -gt 2 ]; then 469 if [[ " ${demos_scraper_liquidity[*]} " == *" ${command[2]} "* ]]; then 470 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-liquidity-${command[2]}.yaml" 471 else 472 echo "Unknown demo" >&2 473 exit 1 474 fi 475 else 476 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/scraper-liquidity.yaml 477 fi 478 ;; 479 cronjob) 480 if [ "${#command[@]}" -eq 2 ]; then 481 minikube -p "${minikube_profile}" kubectl -- create configmap postgres-entrypoint --from-file=deployments/config/postgres-docker-entrypoint.sh 482 minikube -p "${minikube_profile}" kubectl -- create configmap postgres-crondump --from-file=./scripts/dump.sh 483 minikube -p "${minikube_profile}" kubectl -- \ 484 create secret docker-registry regcred \ 485 --docker-server="${snapshot_docker_registry}" \ 486 --docker-username="${snapshot_docker_username}" \ 487 --docker-password="${snapshot_docker_password}" \ 488 --docker-email="${snapshot_docker_email}" 489 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/cronjob-snapshot.yaml" 490 else 491 echo "Unknown command" >&2 492 exit 1 493 fi 494 ;; 495 demos-scraper) 496 echo "Creating demo ..." 497 if [ "$arg_verbose_mode" = true ]; then echo "Creating CEX scrapers ..."; fi 498 for i in "${demos_scraper_cex[@]}" 499 do 500 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-cex-$i.yaml" 501 done 502 if [ "$arg_verbose_mode" = true ]; then echo "Creating DEX scrapers ..."; fi 503 for i in "${demos_scraper_dex[@]}" 504 do 505 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-dex-$i.yaml" 506 done 507 if [ "$arg_verbose_mode" = true ]; then echo "Creating foreign scrapers ..."; fi 508 for i in "${demos_scraper_foreign[@]}" 509 do 510 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-foreign-$i.yaml" 511 done 512 if [ "$arg_verbose_mode" = true ]; then echo "Creating liquidity scrapers ..."; fi 513 for i in "${demos_scraper_liquidity[@]}" 514 do 515 minikube -p "${minikube_profile}" kubectl -- create -f "deployments/k8s-yaml/scraper-liquidity-$i.yaml" 516 done 517 echo "Demo created with success" 518 ;; 519 *) 520 echo "Unknown command" >&2 521 exit 1 522 ;; 523 esac 524 else 525 echo "Unknown command" >&2 526 exit 1 527 fi 528 ;; 529 remove) 530 if [ "${#command[@]}" -gt 1 ]; then 531 case "${command[1]}" in 532 scraper-cex) 533 if [ "${#command[@]}" -gt 2 ]; then 534 if [[ " ${demos_scraper_cex[*]} " == *" ${command[2]} "* ]]; then 535 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-cex-${command[2]}.yaml" || true 536 else 537 echo "Unknown demo" >&2 538 exit 1 539 fi 540 else 541 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-cex.yaml || true 542 fi 543 ;; 544 scraper-dex) 545 if [ "${#command[@]}" -gt 2 ]; then 546 if [[ " ${demos_scraper_dex[*]} " == *" ${command[2]} "* ]]; then 547 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-dex-${command[2]}.yaml" || true 548 else 549 echo "Unknown demo" >&2 550 exit 1 551 fi 552 else 553 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-dex.yaml || true 554 fi 555 ;; 556 scraper-liquidity) 557 if [ "${#command[@]}" -gt 2 ]; then 558 if [[ " ${demos_scraper_liquidity[*]} " == *" ${command[2]} "* ]]; then 559 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-liquidity-${command[2]}.yaml" || true 560 else 561 echo "Unknown demo" >&2 562 exit 1 563 fi 564 else 565 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-liquidity.yaml || true 566 fi 567 ;; 568 scraper-foreign) 569 if [ "${#command[@]}" -gt 2 ]; then 570 if [[ " ${demos_scraper_foreign[*]} " == *" ${command[2]} "* ]]; then 571 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-foreign-${command[2]}.yaml" || true 572 else 573 echo "Unknown demo" >&2 574 exit 1 575 fi 576 else 577 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/scraper-foreign.yaml || true 578 fi 579 ;; 580 cronjob) 581 if [ "${#command[@]}" -eq 2 ]; then 582 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/cronjob-snapshot.yaml" || true 583 minikube -p "${minikube_profile}" kubectl -- delete secret regcred || true 584 minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-entrypoint || true 585 minikube -p "${minikube_profile}" kubectl -- delete configmap postgres-crondump || true 586 else 587 echo "Unknown command" >&2 588 exit 1 589 fi 590 ;; 591 demos-scraper) 592 echo "Removing demo ..." 593 if [ "$arg_verbose_mode" = true ]; then echo "Removing CEX scrapers ..."; fi 594 for i in "${demos_scraper_cex[@]}" 595 do 596 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-cex-$i.yaml" 597 done 598 if [ "$arg_verbose_mode" = true ]; then echo "Removing DEX scrapers ..."; fi 599 for i in "${demos_scraper_dex[@]}" 600 do 601 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-dex-$i.yaml" 602 done 603 if [ "$arg_verbose_mode" = true ]; then echo "Removing foreign scrapers ..."; fi 604 for i in "${demos_scraper_foreign[@]}" 605 do 606 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-foreign-$i.yaml" 607 done 608 if [ "$arg_verbose_mode" = true ]; then echo "Removing liquidity scrapers ..."; fi 609 for i in "${demos_scraper_liquidity[@]}" 610 do 611 minikube -p "${minikube_profile}" kubectl -- delete -f "deployments/k8s-yaml/scraper-liquidity-$i.yaml" 612 done 613 echo "Demo removed with success" 614 ;; 615 *) 616 echo "Unknown command" >&2 617 exit 1 618 ;; 619 esac 620 else 621 echo "Unknown command" >&2 622 exit 1 623 fi 624 ;; 625 shell) 626 if [ "${#command[@]}" -eq 1 ]; then 627 minikube -p "${minikube_profile}" ssh 628 else 629 echo "Unknown command" >&2 630 exit 1 631 fi 632 ;; 633 logs) 634 if [ "${#command[@]}" -eq 1 ]; then 635 minikube -p "${minikube_profile}" logs -f 636 else 637 echo "Unknown command" >&2 638 exit 1 639 fi 640 ;; 641 clean) 642 if [ "${#command[@]}" -eq 1 ]; then 643 if [[ "$minikube_driver" == "docker" ]]; then 644 eval "$(minikube -p "${minikube_profile}" docker-env)" 645 docker buildx prune -a -f 646 eval "$(minikube -p "${minikube_profile}" docker-env --unset)" 647 fi 648 minikube -p "${minikube_profile}" ssh -- docker image prune -a -f 649 minikube -p "${minikube_profile}" ssh -- docker volume prune -f 650 minikube -p "${minikube_profile}" ssh -- docker network prune -f 651 minikube -p "${minikube_profile}" ssh -- docker system prune -a -f 652 else 653 echo "Unknown command" >&2 654 exit 1 655 fi 656 ;; 657 info) 658 if [ "${#command[@]}" -eq 1 ]; then 659 minikube profile list 660 _info 661 # TODO: uptime show the host uptime, not the minikube: https://stackoverflow.com/a/28353785/2042014 662 _minikube_uptime=$(minikube -p "${minikube_profile}" ssh -- uptime -p) 663 echo "Minikube Uptime: $_minikube_uptime" 664 echo; echo "Images:" 665 minikube -p "${minikube_profile}" image ls --format table 666 echo; echo "Status:" 667 minikube -p "${minikube_profile}" status 668 echo; echo "Disk available:" 669 minikube -p "${minikube_profile}" ssh -- df -h 670 if [[ "$minikube_driver" == "docker" ]]; then 671 echo; echo "Disk available of Docker:" 672 minikube -p "${minikube_profile}" ssh -- docker system df 673 fi 674 else 675 echo "Unknown command" >&2 676 exit 1 677 fi 678 ;; 679 ping) 680 if [ "${#command[@]}" -eq 1 ]; then 681 echo "Ping tests:" 682 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-redis.yaml >/dev/null 2>&1 || true 683 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-redis.yaml >/dev/null 2>&1 684 minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-redis >/dev/null 2>&1 && echo " Redis OK" || echo " Redis FAILED" >&2 685 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-influx.yaml >/dev/null 2>&1 || true 686 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-influx.yaml >/dev/null 2>&1 687 minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-influx >/dev/null 2>&1 && echo " InfluxDB OK" || echo " InfluxDB FAILED" >&2 688 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-postgres.yaml >/dev/null 2>&1 || true 689 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-postgres.yaml >/dev/null 2>&1 690 minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-postgres >/dev/null 2>&1 && echo " PostgreSQL OK" || echo " PostgreSQL FAILED" >&2 691 minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/job-ping-kafka.yaml >/dev/null 2>&1 || true 692 minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/job-ping-kafka.yaml >/dev/null 2>&1 693 minikube -p "${minikube_profile}" kubectl -- wait --timeout=30s --for=condition=complete job/ping-kafka >/dev/null 2>&1 && echo " Kafka OK" || echo " Kafka FAILED" >&2 694 else 695 echo "Unknown command" >&2 696 exit 1 697 fi 698 ;; 699 data-reset) 700 # TODO: Also reset relational data and cache 701 if [ "${#command[@]}" -eq 1 ]; then 702 echo "Resetting timeseries data ..." 703 minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'DROP MEASUREMENT foreignquotation' 704 minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'DROP MEASUREMENT tradesTmp' 705 else 706 echo "Unknown command" >&2 707 exit 1 708 fi 709 ;; 710 data) 711 # TODO: "SELECT ... | wc" is very slow, maybe investigate to use COUNT statement 712 if [ "${#command[@]}" -gt 1 ]; then 713 case "${command[1]}" in 714 exchanges) 715 num_exchanges=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchange") 716 echo "Exchanges:" 717 minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT * FROM exchange" 718 echo "#Exchanges: $num_exchanges" 719 ;; 720 exchange) 721 echo "Exchange:" 722 read -r inputExchangeName 723 minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -t -c "SELECT exchange_id FROM exchange WHERE name = '$inputExchangeName'" | if [[ $(wc -l) -ge 2 ]]; then echo "Found"; else echo "Not Found"; exit 1; fi 724 isbridge=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT bridge FROM exchange WHERE name = '$inputExchangeName'") 725 isactive=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT scraper_active FROM exchange WHERE name = '$inputExchangeName'") 726 iscentralized=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -AXqt -c "SELECT centralized FROM exchange WHERE name = '$inputExchangeName'" | sed 's/\r$//') 727 echo "Bridge: $isbridge" 728 echo "Active: $isactive" 729 echo "Centralized: $iscentralized" 730 if [[ "${iscentralized}" == "t" ]]; then 731 num_exchangepairs=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangepair WHERE exchange = '$inputExchangeName'") 732 num_exchangepairassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT id_quotetoken FROM exchangepair WHERE exchange = '$inputExchangeName') OR asset_id IN (SELECT id_basetoken FROM exchangepair WHERE exchange = '$inputExchangeName')") 733 echo "PostgreSQL #exchangepair: $num_exchangepairs (#asset: $num_exchangepairassets)" 734 num_exchangesymbols=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangesymbol WHERE exchange = '$inputExchangeName'") 735 num_exchangesymbolsassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT asset_id FROM exchangesymbol WHERE exchange = '$inputExchangeName')") 736 echo "PostgreSQL #exchangesymbol: $num_exchangesymbols (#asset: $num_exchangesymbolsassets)" 737 else 738 blockchain=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT blockchain FROM exchange WHERE name = '$inputExchangeName'") 739 echo "Blockchain: $blockchain" 740 num_pools=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM pool WHERE exchange = '$inputExchangeName'") 741 last_liquidityupdate=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT time_stamp FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName') ORDER BY time_stamp DESC LIMIT 1") 742 num_poolassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName')") 743 num_poolassetsassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE asset_id IN (SELECT asset_id FROM poolasset WHERE pool_id IN (SELECT pool_id FROM pool WHERE exchange = '$inputExchangeName'))") 744 echo "PostgreSQL #pool: $num_pools (#poolasset: $num_poolassets updated at $last_liquidityupdate (#asset: $num_poolassetsassets))" 745 fi 746 num_trades=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM tradesTmp WHERE exchange = '$inputExchangeName'" | wc -l) 747 echo "InfluxDB #trades: $num_trades" 748 ;; 749 blockchain) 750 echo "Blockchain:" 751 read -r inputBlockchainName 752 verificationmechanism=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT verificationmechanism FROM blockchain WHERE name = '$inputBlockchainName'") 753 genesisdate=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT genesisdate FROM blockchain WHERE name = '$inputBlockchainName'") 754 chainid=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT chain_id FROM blockchain WHERE name = '$inputBlockchainName'") 755 echo "Consensus: $verificationmechanism" 756 echo "Genesis Date: $genesisdate" 757 echo "Chain ID: $chainid" 758 num_blockchainassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset WHERE blockchain = '$inputBlockchainName'") 759 echo "PostgreSQL #asset: $num_blockchainassets" 760 ;; 761 scraper-cex) 762 echo wip 763 ;; 764 scraper-dex) 765 echo wip 766 ;; 767 scraper-liquidity) 768 echo wip 769 ;; 770 scraper-foreign) 771 echo "Exchange:" 772 read -r inputExchangeName 773 num_foreignquotation=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM foreignquotation WHERE source = '$inputExchangeName'" | wc -l) 774 echo "InfluxDB #foreignquotation: $num_foreignquotation" 775 minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute "SELECT * FROM foreignquotation WHERE source = '$inputExchangeName' ORDER BY time DESC LIMIT 10" 776 ;; 777 *) 778 echo "Unknown command" >&2 779 exit 1 780 ;; 781 esac 782 else 783 # InfluxDB 784 num_trades=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM trades' | wc -l) 785 echo "InfluxDB #trades: $num_trades" 786 num_tradestmp=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM tradesTmp' | wc -l) 787 echo "InfluxDB #tradesTmp: $num_tradestmp" 788 num_filters=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM filters' | wc -l) 789 echo "InfluxDB #filters: $num_filters" 790 num_quotations=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM assetQuotations' | wc -l) 791 echo "InfluxDB #assetQuotations: $num_quotations" 792 num_foreignquotation=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-influx -- influx -host 'localhost' -port '8086' -database 'dia' -execute 'SELECT * FROM foreignquotation' | wc -l) 793 echo "InfluxDB #foreignquotation: $num_foreignquotation" 794 # PostgreSQL 795 num_exchanges=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchange") 796 echo "PostgreSQL #exchange: $num_exchanges" 797 num_pools=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM pool") 798 echo "PostgreSQL #pool: $num_pools" 799 num_poolassets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM poolasset") 800 echo "PostgreSQL #poolasset: $num_poolassets" 801 num_assets=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM asset") 802 echo "PostgreSQL #asset: $num_assets" 803 num_exchangepairs=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangepair") 804 echo "PostgreSQL #exchangepair: $num_exchangepairs" 805 num_exchangesymbols=$(minikube -p "${minikube_profile}" kubectl -- exec deployment/data-postgres -- psql -U postgres -tA -c "SELECT COUNT(*) FROM exchangesymbol") 806 echo "PostgreSQL #exchangesymbol: $num_exchangesymbols" 807 fi 808 ;; 809 *) 810 echo "Unknown command ${command[0]}" >&2 811 exit 1 812 ;; 813 esac 814 } 815 816 main "$@"