github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/_utils/start_tidb_cluster_impl

#!/bin/bash

# --workdir: work directory
# --tidb-config: path to tidb config file
# --multiple-upstream-pd: whether to deploy multiple pd severs in upstream

set -e

OUT_DIR=
tidb_config=
pd_config=
tikv_config=
multiple_upstream_pd=
random_file_name=

# Random generate the sockets config.
# Make sure we dont use the same sock.
randomGenSocketsConf() {
	random_str=$(date '+%s%N')
	if [ "$(uname)" == "Darwin" ]; then
		random_str=$(cat /dev/random | LC_ALL=C tr -dc "a-zA-Z0-9" | head -c 10)
	fi
	random_file_name="$OUT_DIR/tidb-config-$random_str.toml"

	cat "$OUT_DIR/tidb-config.toml" >"$random_file_name"
	echo "socket = \"/tmp/tidb-$random_str.sock\"" >>"$random_file_name"
}

while [[ ${1} ]]; do
	case "${1}" in
	--workdir)
		OUT_DIR=${2}
		shift
		;;
	--tidb-config)
		tidb_config=${2}
		shift
		;;
	--pd-config)
		pd_config=${2}
		shift
		;;
	--tikv-config)
		tikv_config=${2}
		shift
		;;
	--multiple-upstream-pd)
		multiple_upstream_pd=${2}
		shift
		;;
	*)
		echo "Unknown parameter: ${1}" >&2
		exit 1
		;;
	esac

	if ! shift; then
		echo 'Missing parameter argument.' >&2
		exit 1
	fi
done

CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source $CUR/../_utils/test_prepare

stop_tidb_cluster

cd $OUT_DIR && echo "start tidb cluster in $OUT_DIR"

# pd server config file
if [[ "$pd_config" != "" ]]; then
	cat $pd_config >$OUT_DIR/pd-config.toml
else
	cat - >"$OUT_DIR/pd-config.toml" <<EOF
[replication]
# Set it to 1 to make sure we have enough replicas to run placement-rules.
max-replicas = 1
enable-placement-rules = true
EOF
fi

echo "Starting Upstream PD..."
pd-server --version
if [[ "$multiple_upstream_pd" == "true" ]]; then
	pd_count=3
	initial_cluster="pd1=http://${UP_PD_HOST_1}:${UP_PD_PEER_PORT_1},pd2=http://${UP_PD_HOST_2}:${UP_PD_PEER_PORT_2},pd3=http://${UP_PD_HOST_3}:${UP_PD_PEER_PORT_3}"
else
	pd_count=1
	initial_cluster="pd1=http://${UP_PD_HOST_1}:${UP_PD_PEER_PORT_1}"
fi
for idx in $(seq 1 $pd_count); do
	host="UP_PD_HOST_$idx"
	port="UP_PD_PORT_$idx"
	peer_port="UP_PD_PEER_PORT_$idx"
	pd-server \
		--advertise-client-urls http://${!host}:${!port} \
		--client-urls http://0.0.0.0:${!port} \
		--advertise-peer-urls http://${!host}:${!peer_port} \
		--peer-urls http://0.0.0.0:${!peer_port} \
		--config "$OUT_DIR/pd-config.toml" \
		--log-file "$OUT_DIR/pd$idx.log" \
		--data-dir "$OUT_DIR/pd$idx" \
		--name="pd$idx" \
		--initial-cluster=${initial_cluster} &
done

echo "Starting Downstream PD..."
pd-server --version
pd-server \
	--advertise-client-urls http://${DOWN_PD_HOST}:${DOWN_PD_PORT} \
	--client-urls http://0.0.0.0:${DOWN_PD_PORT} \
	--advertise-peer-urls http://${DOWN_PD_HOST}:${DOWN_PD_PEER_PORT} \
	--peer-urls http://0.0.0.0:${DOWN_PD_PEER_PORT} \
	--config "$OUT_DIR/pd-config.toml" \
	--log-file "$OUT_DIR/down_pd.log" \
	--data-dir "$OUT_DIR/down_pd" &

# wait until upstream PD is online...
echo "Verifying upstream PD is started..."
for idx in $(seq 1 $pd_count); do
	host="UP_PD_HOST_$idx"
	port="UP_PD_PORT_$idx"

	i=0
	while ! curl -o /dev/null -sf http://${!host}:${!port}/pd/api/v1/version; do
		i=$((i + 1))
		if [ "$i" -gt 60 ]; then
			echo 'Failed to start upstream PD'
			exit 1
		fi
		sleep 1
	done

	i=0
	while [ -z "$(curl http://${!host}:${!port}/pd/api/v1/health 2>/dev/null | grep 'health' | grep 'true')" ]; do
		i=$((i + 1))
		if [ "$i" -gt 60 ]; then
			echo 'Failed to start upstream PD'
			exit 1
		fi
		sleep 1
	done
done

# wait until downstream PD is online...
echo "Verifying downstream PD is started..."
i=0
while ! curl -o /dev/null -sf http://${DOWN_PD_HOST}:${DOWN_PD_PORT}/pd/api/v1/version; do
	i=$((i + 1))
	if [ "$i" -gt 60 ]; then
		echo 'Failed to start downstream PD'
		exit 1
	fi
	sleep 1
done

i=0
while [ -z "$(curl http://${DOWN_PD_HOST}:${DOWN_PD_PORT}/pd/api/v1/health 2>/dev/null | grep 'health' | grep 'true')" ]; do
	i=$((i + 1))
	if [ "$i" -gt 60 ]; then
		echo 'Failed to start downstream PD'
		exit 1
	fi
	sleep 1
done

# Tries to limit the max number of open files under the system limit
if [[ "$tikv_config" != "" ]]; then
	cat $tikv_config >$OUT_DIR/tikv-config.toml
else
	cat - >"$OUT_DIR/tikv-config.toml" <<EOF
[storage]
# Disable creating a large temp file.
reserve-space = "0MB"
[rocksdb]
max-open-files = 4096
[raftdb]
max-open-files = 4096
[raftstore]
# true (default value) for high reliability, this can prevent data loss when power failure.
sync-log = false
[cdc]
hibernate-regions-compatible = true
EOF
fi

# tidb server config file
if [[ "$tidb_config" != "" ]]; then
	cat $tidb_config >$OUT_DIR/tidb-config.toml
else
	cat - >"$OUT_DIR/tidb-config.toml" <<EOF
split-table = true
new_collations_enabled_on_first_bootstrap = true
EOF
fi

echo "Starting Upstream TiKV..."
tikv-server --version
for idx in $(seq 1 3); do
	host="UP_TIKV_HOST_$idx"
	port="UP_TIKV_PORT_$idx"
	status_port="UP_TIKV_STATUS_PORT_$idx"
	tikv-server \
		--pd ${UP_PD_HOST_1}:${UP_PD_PORT_1} \
		-A ${!host}:${!port} \
		--status-addr ${!host}:${!status_port} \
		--log-file "$OUT_DIR/tikv$idx.log" \
		--log-level debug \
		-C "$OUT_DIR/tikv-config.toml" \
		-s "$OUT_DIR/tikv$idx" &
done

echo "Starting Downstream TiKV..."
tikv-server --version
tikv-server \
	--pd ${DOWN_PD_HOST}:${DOWN_PD_PORT} \
	-A ${DOWN_TIKV_HOST}:${DOWN_TIKV_PORT} \
	--status-addr ${DOWN_TIKV_HOST}:${DOWN_TIKV_STATUS_PORT} \
	--log-file "$OUT_DIR/tikv_down.log" \
	--log-level debug \
	-C "$OUT_DIR/tikv-config.toml" \
	-s "$OUT_DIR/tikv_down" &

sleep 2

echo "Starting Upstream TiDB..."
tidb-server -V
randomGenSocketsConf
tidb-server \
	-P ${UP_TIDB_PORT} \
	-config "$random_file_name" \
	--store tikv \
	--path ${UP_PD_HOST_1}:${UP_PD_PORT_1} \
	--status=${UP_TIDB_STATUS} \
	--log-file "$OUT_DIR/tidb.log" &

randomGenSocketsConf
tidb-server \
	-P ${UP_TIDB_OTHER_PORT} \
	-config "$random_file_name" \
	--store tikv \
	--path ${UP_PD_HOST_1}:${UP_PD_PORT_1} \
	--status=${UP_TIDB_OTHER_STATUS} \
	--log-file "$OUT_DIR/tidb_other.log" &

echo "Starting Downstream TiDB..."
tidb-server -V
randomGenSocketsConf
tidb-server \
	-P ${DOWN_TIDB_PORT} \
	-config "$random_file_name" \
	--store tikv \
	--path ${DOWN_PD_HOST}:${DOWN_PD_PORT} \
	--status=${DOWN_TIDB_STATUS} \
	--log-file "$OUT_DIR/tidb_down.log" &

echo "Verifying Upstream TiDB is started..."
i=0
while ! mysql -uroot -h${UP_TIDB_HOST} -P${UP_TIDB_PORT} --default-character-set utf8mb4 -e 'select * from mysql.tidb;'; do
	i=$((i + 1))
	if [ "$i" -gt 60 ]; then
		echo 'Failed to start upstream TiDB'
		exit 2
	fi
	sleep 2
done

i=0
while ! mysql -uroot -h${UP_TIDB_HOST} -P${UP_TIDB_OTHER_PORT} --default-character-set utf8mb4 -e 'select * from mysql.tidb;'; do
	i=$((i + 1))
	if [ "$i" -gt 60 ]; then
		echo 'Failed to start upstream TiDB'
		exit 2
	fi
	sleep 2
done

echo "Verifying Downstream TiDB is started..."
i=0
while ! mysql -uroot -h${DOWN_TIDB_HOST} -P${DOWN_TIDB_PORT} --default-character-set utf8mb4 -e 'select * from mysql.tidb;'; do
	i=$((i + 1))
	if [ "$i" -gt 60 ]; then
		echo 'Failed to start downstream TiDB'
		exit 1
	fi
	sleep 2
done

run_sql "update mysql.tidb set variable_value='60m' where variable_name='tikv_gc_life_time';" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
run_sql "update mysql.tidb set variable_value='60m' where variable_name='tikv_gc_life_time';" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
run_sql "CREATE user 'normal'@'%' identified by '123456';" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
run_sql "GRANT select,insert,update,delete,index,create,drop,alter,create view,references ON *.* TO 'normal'@'%';" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
run_sql "FLUSH privileges" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}

cat - >"$OUT_DIR/tiflash-config.toml" <<EOF
tmp_path = "${OUT_DIR}/tiflash/tmp"
display_name = "TiFlash"
users_config = "${OUT_DIR}/tiflash/users.toml"
path = "${OUT_DIR}/tiflash/db"
mark_cache_size = 5368709120
listen_host = "127.0.0.1"
tcp_port = 5000
http_port = 4500
interserver_http_port = 5500

[flash]
tidb_status_addr = "127.0.0.1:8500"
service_addr = "127.0.0.1:9500"

[flash.proxy]
addr = "127.0.0.1:9000"
advertise-addr = "127.0.0.1:9000"
data-dir = "${OUT_DIR}/tiflash/db/proxy"
config = "${OUT_DIR}/tiflash-proxy.toml"
log-file = "${OUT_DIR}/tiflash/log/proxy.log"

[logger]
level = "trace"
log = "${OUT_DIR}/tiflash/log/server.log"
errorlog = "${OUT_DIR}/tiflash/log/error.log"
size = "4000M"
count = 10

[application]
runAsDaemon = true

[raft]
pd_addr = "${UP_PD_HOST_1}:${UP_PD_PORT_1}"
EOF

cat - >"$OUT_DIR/tiflash-proxy.toml" <<EOF
log-level = "info"

[server]
engine-addr = "127.0.0.1:9500"
status-addr = "127.0.0.1:17000"

[raftstore]
sync-log = true
capacity = "100GB"
hibernate-regions = false

[rocksdb]
wal-dir = ""
max-open-files = 1000

[rocksdb.defaultcf]
block-cache-size = "1GB"

[rocksdb.lockcf]
block-cache-size = "1GB"

[rocksdb.writecf]
block-cache-size = "1GB"

[raftdb]
max-open-files = 1000

[raftdb.defaultcf]
block-cache-size = "1GB"
EOF

echo "Starting Upstream TiFlash..."
mkdir -p ${OUT_DIR}/tiflash/ && cp $CUR/tiflash-users.toml ${OUT_DIR}/tiflash/users.toml
tiflash version
tiflash server --config-file "$OUT_DIR/tiflash-config.toml" &

echo "Verifying Upstream TiFlash is started..."
# Make sure TiFlash is started.
while ! curl -o /dev/null -sf http://127.0.0.1:17000/metrics 1>/dev/null 2>&1; do
	i=$((i + 1))
	if [ "$i" -gt 10 ]; then
		cat ${OUT_DIR}/tiflash/log/proxy.log
		cat ${OUT_DIR}/tiflash/log/server.log
		cat ${OUT_DIR}/tiflash/log/error.log
		echo 'Failed to start TiFlash'
		exit 1
	fi
	sleep 2
done