github.com/unigraph-dev/dgraph@v1.1.1-0.20200923154953-8b52b426f765/systest/loader-benchmark/loader-benchmark.sh (about)

     1  #!/bin/bash -e
     2  
     3  readonly ME=${0##*/}
     4  readonly SRCDIR=$(dirname $0)
     5  
     6  BENCHMARKS_REPO="https://github.com/dgraph-io/benchmarks"
     7  BENCHMARK_SIZE=${BENCHMARK_SIZE:=big}
     8  SCHEMA_URL="$BENCHMARKS_REPO/blob/master/data/21million.schema?raw=true"
     9  DGRAPH_LOADER=${DGRAPH_LOADER:=bulk}
    10  
    11  function Info {
    12      echo -e "INFO: $*"
    13  }
    14  
    15  function DockerCompose {
    16      docker-compose -p dgraph "$@"
    17  }
    18  
    19  if [[ $BENCHMARK_SIZE != small && $BENCHMARK_SIZE != big ]]; then
    20      echo >&2 "$ME: loader must be 'small' or 'big'  -- $BENCHMARK_SIZE"
    21      exit 1
    22  fi
    23  
    24  if [[ $BENCHMARK_SIZE == small ]]; then
    25      DATA_URL="$BENCHMARKS_REPO/blob/master/data/1million.rdf.gz?raw=true"
    26  else
    27      DATA_URL="$BENCHMARKS_REPO/blob/master/data/21million.rdf.gz?raw=true"
    28  fi
    29  
    30  if [[ $DGRAPH_LOADER != bulk && $DGRAPH_LOADER != live ]]; then
    31      echo >&2 "$ME: loader must be 'bulk' or 'live' -- $DGRAPH_LOADER"
    32      exit 1
    33  fi
    34  
    35  Info "entering directory $SRCDIR"
    36  cd $SRCDIR
    37  
    38  Info "removing old data"
    39  DockerCompose down -v
    40  
    41  Info "bringing up zero container"
    42  DockerCompose up -d zero1
    43  
    44  Info "waiting for zero to become leader"
    45  DockerCompose logs -f zero1 | grep -q -m1 "I've become the leader"
    46  
    47  if [[ $DGRAPH_LOADER == bulk ]]; then
    48      Info "bulk loading 21million data set"
    49      DockerCompose run --rm dg1 \
    50          bash -s <<EOF
    51              /gobin/dgraph bulk --schema=<(curl -LSs $SCHEMA_URL) --files=<(curl -LSs $DATA_URL) \
    52                                 --format=rdf --zero=zero1:5180 --out=/data/dg1/bulk
    53              mv /data/dg1/bulk/0/p /data/dg1
    54  EOF
    55  fi
    56  
    57  Info "bringing up alpha container"
    58  DockerCompose up -d dg1
    59  
    60  Info "waiting for alpha to be ready"
    61  DockerCompose logs -f dg1 | grep -q -m1 "Server is ready"
    62  
    63  if [[ $DGRAPH_LOADER == live ]]; then
    64      Info "live loading 21million data set"
    65      dgraph live --schema=<(curl -LSs $SCHEMA_URL) --files=<(curl -LSs $DATA_URL) \
    66                  --format=rdf --zero=:5180 --alpha=:9180 --logtostderr
    67  fi