github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/Makefile (about)

     1  ### Makefile for tiflow
     2  .PHONY: build test check clean fmt cdc kafka_consumer storage_consumer coverage \
     3  	integration_test_build integration_test integration_test_mysql integration_test_kafka bank \
     4  	kafka_docker_integration_test kafka_docker_integration_test_with_build \
     5  	clean_integration_test_containers \
     6  	mysql_docker_integration_test mysql_docker_integration_test_with_build \
     7  	build_mysql_integration_test_images clean_integration_test_images \
     8  	dm dm-master dm-worker dmctl dm-syncer dm_coverage \
     9  	engine tiflow tiflow-demo tiflow-chaos-case engine_image help \
    10  	format-makefiles check-makefiles oauth2_server prepare_test_binaries
    11  
    12  
    13  .DEFAULT_GOAL := default
    14  
    15  # Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/
    16  help: ## Display this help and any documented user-facing targets. Other undocumented targets may be present in the Makefile.
    17  help:
    18  	@awk 'BEGIN {FS = ": ##"; printf "Usage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z0-9_\.\-\/%]+: ##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
    19  
    20  PROJECT=tiflow
    21  P=3
    22  
    23  FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1  }  }'
    24  
    25  CURDIR := $(shell pwd)
    26  path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH)))
    27  export PATH := $(CURDIR)/bin:$(CURDIR)/tools/bin:$(path_to_add):$(PATH)
    28  
    29  # DBUS_SESSION_BUS_ADDRESS pulsar client use dbus to detect the connection status,
    30  # but it will not exit when the connection is closed.
    31  # I try to use leak_helper to detect goroutine leak,but it does not work.
    32  # https://github.com/benthosdev/benthos/issues/1184 suggest to use environment variable to disable dbus.
    33  export DBUS_SESSION_BUS_ADDRESS := /dev/null
    34  
    35  SHELL := /usr/bin/env bash
    36  
    37  TEST_DIR := /tmp/tidb_cdc_test
    38  DM_TEST_DIR := /tmp/dm_test
    39  ENGINE_TEST_DIR := /tmp/engine_test
    40  
    41  GO       := GO111MODULE=on go
    42  ifeq (${CDC_ENABLE_VENDOR}, 1)
    43  GOVENDORFLAG := -mod=vendor
    44  endif
    45  
    46  # Since TiDB add a new dependency on github.com/cloudfoundry/gosigar,
    47  # We need to add CGO_ENABLED=1 to make it work when build TiCDC in Darwin OS.
    48  # These logic is to check if the OS is Darwin, if so, add CGO_ENABLED=1.
    49  # ref: https://github.com/cloudfoundry/gosigar/issues/58#issuecomment-1150925711
    50  # ref: https://github.com/pingcap/tidb/pull/39526#issuecomment-1407952955
    51  OS := "$(shell go env GOOS)"
    52  SED_IN_PLACE ?= $(shell which sed)
    53  ifeq (${OS}, "linux")
    54  	CGO := 0
    55  	SED_IN_PLACE += -i
    56  else ifeq (${OS}, "darwin")
    57  	CGO := 1
    58  	SED_IN_PLACE += -i ''
    59  endif
    60  
    61  BUILD_FLAG =
    62  GOEXPERIMENT=
    63  ifeq ("${ENABLE_FIPS}", "1")
    64  	BUILD_FLAG = -tags boringcrypto
    65  	GOEXPERIMENT = GOEXPERIMENT=boringcrypto
    66  	CGO = 1
    67  endif
    68  GOBUILD  := $(GOEXPERIMENT) CGO_ENABLED=$(CGO) $(GO) build $(BUILD_FLAG) -trimpath $(GOVENDORFLAG)
    69  GOBUILDNOVENDOR  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -trimpath
    70  GOTEST   := CGO_ENABLED=1 $(GO) test -p $(P) --race --tags=intest
    71  GOTESTNORACE := CGO_ENABLED=1 $(GO) test -p $(P)
    72  
    73  CDC_PKG := github.com/pingcap/tiflow
    74  DM_PKG := github.com/pingcap/tiflow/dm
    75  ENGINE_PKG := github.com/pingcap/tiflow/engine
    76  PACKAGE_LIST := go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'
    77  PACKAGE_LIST_WITHOUT_DM_ENGINE := $(PACKAGE_LIST) | grep -vE 'github.com/pingcap/tiflow/cmd|github.com/pingcap/tiflow/dm|github.com/pingcap/tiflow/engine'
    78  DM_PACKAGE_LIST := go list github.com/pingcap/tiflow/dm/... | grep -vE 'pb|pbmock'
    79  PACKAGES := $$($(PACKAGE_LIST))
    80  PACKAGES_TICDC := $$($(PACKAGE_LIST_WITHOUT_DM_ENGINE))
    81  DM_PACKAGES := $$($(DM_PACKAGE_LIST))
    82  # NOTE: ignore engine/framework because of a race in testify. See #9619
    83  # ENGINE_PACKAGE_LIST := go list github.com/pingcap/tiflow/engine/... | grep -vE 'pb|proto|engine/test/e2e'
    84  ENGINE_PACKAGE_LIST := go list github.com/pingcap/tiflow/engine/... | grep -vE "pb|proto|engine/test/e2e|engine/framework$$"
    85  ENGINE_PACKAGES := $$($(ENGINE_PACKAGE_LIST))
    86  FILES := $$(find . -name '*.go' -type f | grep -vE 'vendor|_gen|proto|pb\.go|pb\.gw\.go|_mock.go')
    87  TEST_FILES := $$(find . -name '*_test.go' -type f | grep -vE 'vendor|kv_gen|integration|testing_utils')
    88  TEST_FILES_WITHOUT_DM := $$(find . -name '*_test.go' -type f | grep -vE 'vendor|kv_gen|integration|testing_utils|^\./dm')
    89  FAILPOINT_DIR := $$(for p in $(PACKAGES); do echo $${p\#"github.com/pingcap/$(PROJECT)/"}|grep -v "github.com/pingcap/$(PROJECT)"; done)
    90  FAILPOINT := tools/bin/failpoint-ctl
    91  
    92  FAILPOINT_ENABLE  := $$(echo $(FAILPOINT_DIR) | xargs $(FAILPOINT) enable >/dev/null)
    93  FAILPOINT_DISABLE := $$(echo $(FAILPOINT_DIR) | xargs $(FAILPOINT) disable >/dev/null)
    94  
    95  TICDC_DOCKER_DEPLOYMENTS_DIR := deployments/ticdc/docker-compose/
    96  
    97  # MAKE_FILES is a list of make files to lint.
    98  # We purposefully avoid MAKEFILES as a variable name as it influences
    99  # the files included in recursive invocations of make
   100  MAKE_FILES = $(shell find . \( -name 'Makefile' -o -name '*.mk' \) -print)
   101  
   102  RELEASE_VERSION =
   103  ifeq ($(RELEASE_VERSION),)
   104  	RELEASE_VERSION := v8.2.0-master
   105  	release_version_regex := ^v[0-9]\..*$$
   106  	release_branch_regex := "^release-[0-9]\.[0-9].*$$|^HEAD$$|^.*/*tags/v[0-9]\.[0-9]\..*$$"
   107  	ifneq ($(shell git rev-parse --abbrev-ref HEAD | grep -E $(release_branch_regex)),)
   108  		# If we are in release branch, try to use tag version.
   109  		ifneq ($(shell git describe --tags --dirty | grep -E $(release_version_regex)),)
   110  			RELEASE_VERSION := $(shell git describe --tags --dirty)
   111  		endif
   112  	else ifneq ($(shell git status --porcelain),)
   113  		# Add -dirty if the working tree is dirty for non release branch.
   114  		RELEASE_VERSION := $(RELEASE_VERSION)-dirty
   115  	endif
   116  endif
   117  
   118  BUILDTS := $(shell date -u '+%Y-%m-%d %H:%M:%S')
   119  GITHASH := $(shell git rev-parse HEAD)
   120  GITBRANCH := $(shell git rev-parse --abbrev-ref HEAD)
   121  GOVERSION := $(shell go version)
   122  
   123  # Version LDFLAGS.
   124  LDFLAGS += -X "$(CDC_PKG)/pkg/version.ReleaseVersion=$(RELEASE_VERSION)"
   125  LDFLAGS += -X "$(CDC_PKG)/pkg/version.BuildTS=$(BUILDTS)"
   126  LDFLAGS += -X "$(CDC_PKG)/pkg/version.GitHash=$(GITHASH)"
   127  LDFLAGS += -X "$(CDC_PKG)/pkg/version.GitBranch=$(GITBRANCH)"
   128  LDFLAGS += -X "$(CDC_PKG)/pkg/version.GoVersion=$(GOVERSION)"
   129  LDFLAGS += -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=$(RELEASE_VERSION)"
   130  
   131  include tools/Makefile
   132  include Makefile.engine
   133  
   134  default: build buildsucc
   135  
   136  buildsucc:
   137  	@echo Build TiDB CDC successfully!
   138  
   139  all: dev install
   140  
   141  dev: check test
   142  
   143  test: unit_test dm_unit_test engine_unit_test
   144  
   145  build: cdc dm engine
   146  
   147  check-makefiles: ## Check the makefiles format. Please run this target after the changes are committed.
   148  check-makefiles: format-makefiles
   149  	@git diff --exit-code -- $(MAKE_FILES) || (echo "Please format Makefiles by running 'make format-makefiles'" && false)
   150  
   151  format-makefiles: ## Format all Makefiles.
   152  format-makefiles: $(MAKE_FILES)
   153  	$(SED_IN_PLACE) -e 's/^\(\t*\)  /\1\t/g' -e 's/^\(\t*\) /\1/' -- $?
   154  
   155  bank:
   156  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/bank ./tests/bank/bank.go ./tests/bank/case.go
   157  
   158  build-cdc-with-failpoint: check_failpoint_ctl
   159  build-cdc-with-failpoint: ## Build cdc with failpoint enabled.
   160  	$(FAILPOINT_ENABLE)
   161  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go
   162  	$(FAILPOINT_DISABLE)
   163  
   164  cdc:
   165  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc
   166  
   167  kafka_consumer:
   168  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go
   169  
   170  storage_consumer:
   171  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go
   172  
   173  pulsar_consumer:
   174  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go
   175  
   176  oauth2_server:
   177  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/oauth2-server ./cmd/oauth2-server/main.go
   178  
   179  kafka_consumer_image:
   180  	@which docker || (echo "docker not found in ${PATH}"; exit 1)
   181  	DOCKER_BUILDKIT=1 docker build -f ./deployments/ticdc/docker/kafka-consumer.Dockerfile . -t ticdc:kafka-consumer  --platform linux/amd64
   182  
   183  storage_consumer_image:
   184  	@which docker || (echo "docker not found in ${PATH}"; exit 1)
   185  	DOCKER_BUILDKIT=1 docker build -f ./deployments/ticdc/docker/storage-consumer.Dockerfile . -t ticdc:storage-consumer  --platform linux/amd64
   186  
   187  pulsar_consumer_image:
   188  	@which docker || (echo "docker not found in ${PATH}"; exit 1)
   189  	DOCKER_BUILDKIT=1 docker build -f ./deployments/ticdc/docker/pulsar-consumer.Dockerfile . -t ticdc:pulsar-consumer  --platform linux/amd64
   190  
   191  filter_helper:
   192  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_filter_helper ./cmd/filter-helper/main.go
   193  
   194  install:
   195  	go install ./...
   196  
   197  unit_test: check_failpoint_ctl generate_mock go-generate generate-protobuf
   198  	mkdir -p "$(TEST_DIR)"
   199  	$(FAILPOINT_ENABLE)
   200  	@export log_level=error;\
   201  	$(GOTEST) -cover -covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_TICDC) \
   202  	|| { $(FAILPOINT_DISABLE); exit 1; }
   203  	$(FAILPOINT_DISABLE)
   204  
   205  unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov tools/bin/gocov-xml
   206  	mkdir -p "$(TEST_DIR)"
   207  	$(FAILPOINT_ENABLE)
   208  	@export log_level=error;\
   209  	CGO_ENABLED=1 tools/bin/gotestsum --junitfile cdc-junit-report.xml -- -v -timeout 5m -p $(P) --race --tags=intest \
   210  	-covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_TICDC) \
   211  	|| { $(FAILPOINT_DISABLE); exit 1; }
   212  	tools/bin/gocov convert "$(TEST_DIR)/cov.unit.out" | tools/bin/gocov-xml > cdc-coverage.xml
   213  	$(FAILPOINT_DISABLE)
   214  
   215  leak_test: check_failpoint_ctl
   216  	$(FAILPOINT_ENABLE)
   217  	@export log_level=error;\
   218  	$(GOTEST) -count=1 --tags leak $(PACKAGES_TICDC) || { $(FAILPOINT_DISABLE); exit 1; }
   219  	$(FAILPOINT_DISABLE)
   220  
   221  check_third_party_binary:
   222  	@which bin/tidb-server
   223  	@which bin/tikv-server
   224  	@which bin/pd-server
   225  	@which bin/tiflash
   226  	@which bin/pd-ctl
   227  	@which bin/sync_diff_inspector
   228  	@which bin/go-ycsb
   229  	@which bin/etcdctl
   230  	@which bin/jq
   231  	@which bin/minio
   232  	@which bin/bin/schema-registry-start
   233  
   234  integration_test_build: check_failpoint_ctl storage_consumer kafka_consumer pulsar_consumer oauth2_server
   235  	$(FAILPOINT_ENABLE)
   236  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   237  		-coverpkg=github.com/pingcap/tiflow/... \
   238  		-o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \
   239  	|| { $(FAILPOINT_DISABLE); exit 1; }
   240  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go \
   241  	|| { $(FAILPOINT_DISABLE); exit 1; }
   242  	$(FAILPOINT_DISABLE)
   243  
   244  integration_test: integration_test_mysql
   245  
   246  integration_test_mysql:
   247  	tests/integration_tests/run.sh mysql "$(CASE)" "$(START_AT)"
   248  
   249  mysql_docker_integration_test: ## Run TiCDC MySQL all integration tests in Docker.
   250  mysql_docker_integration_test: clean_integration_test_containers
   251  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-mysql-integration.yml up
   252  
   253  mysql_docker_integration_test_with_build: ## Build images and run TiCDC MySQL all integration tests in Docker. Please use only after modifying the TiCDC non-test code.
   254  mysql_docker_integration_test_with_build: clean_integration_test_containers
   255  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-mysql-integration.yml up --build
   256  
   257  build_mysql_integration_test_images: ## Build MySQL integration test images without cache.
   258  build_mysql_integration_test_images: clean_integration_test_containers
   259  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-mysql-integration.yml build --no-cache
   260  
   261  integration_test_kafka: check_third_party_binary
   262  	tests/integration_tests/run.sh kafka "$(CASE)" "$(START_AT)"
   263  
   264  integration_test_storage:
   265  	tests/integration_tests/run.sh storage "$(CASE)" "$(START_AT)"
   266  
   267  integration_test_pulsar:
   268  	tests/integration_tests/run.sh pulsar "$(CASE)" "$(START_AT)"
   269  
   270  kafka_docker_integration_test: ## Run TiCDC Kafka all integration tests in Docker.
   271  kafka_docker_integration_test: clean_integration_test_containers
   272  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-kafka-integration.yml up
   273  
   274  kafka_docker_integration_test_with_build: ## Build images and run TiCDC Kafka all integration tests in Docker. Please use only after modifying the TiCDC non-test code.
   275  kafka_docker_integration_test_with_build: clean_integration_test_containers
   276  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-kafka-integration.yml up --build
   277  
   278  build_kafka_integration_test_images: ## Build Kafka integration test images without cache.
   279  build_kafka_integration_test_images: clean_integration_test_containers
   280  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-kafka-integration.yml build --no-cache
   281  
   282  clean_integration_test_containers: ## Clean MySQL and Kafka integration test containers.
   283  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-mysql-integration.yml down -v
   284  	docker-compose -f $(TICDC_DOCKER_DEPLOYMENTS_DIR)/docker-compose-kafka-integration.yml down -v
   285  
   286  fmt: tools/bin/gofumports tools/bin/shfmt tools/bin/gci
   287  	@echo "run gci (format imports)"
   288  	tools/bin/gci write $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
   289  	@echo "run gofumports"
   290  	tools/bin/gofumports -l -w $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
   291  	@echo "run shfmt"
   292  	tools/bin/shfmt -d -w .
   293  	@echo "check log style"
   294  	scripts/check-log-style.sh
   295  	@make check-diff-line-width
   296  
   297  errdoc: tools/bin/errdoc-gen
   298  	@echo "generator errors.toml"
   299  	# check-errdoc will skip DM directory.
   300  	./tools/check/check-errdoc.sh
   301  
   302  # terror_check is only used for DM errors.
   303  # TODO: unified the error framework of CDC and DM.
   304  terror_check:
   305  	@echo "check terror conflict"
   306  	@cd dm && _utils/terror_gen/check.sh
   307  
   308  check-copyright:
   309  	@echo "check-copyright"
   310  	@./scripts/check-copyright.sh
   311  
   312  check-merge-conflicts:
   313  	@echo "check-merge-conflicts"
   314  	@./scripts/check-merge-conflicts.sh
   315  
   316  check-ticdc-dashboard:
   317  	@echo "check-ticdc-dashboard"
   318  	@./scripts/check-ticdc-dashboard.sh
   319  
   320  check-diff-line-width:
   321  ifneq ($(shell echo $(RELEASE_VERSION) | grep master),)
   322  	@echo "check-file-width"
   323  	@./scripts/check-diff-line-width.sh
   324  endif
   325  
   326  go-generate: ## Run go generate on all packages.
   327  go-generate: tools/bin/msgp tools/bin/stringer tools/bin/mockery
   328  	@echo "go generate"
   329  	@go generate ./...
   330  
   331  generate-protobuf: ## Generate code from protobuf files.
   332  generate-protobuf: tools/bin/protoc tools/bin/protoc-gen-gogofaster \
   333  	tools/bin/protoc-gen-go tools/bin/protoc-gen-go-grpc \
   334  	tools/bin/protoc-gen-grpc-gateway tools/bin/protoc-gen-grpc-gateway-v2 \
   335  	tools/bin/protoc-gen-openapiv2
   336  	@echo "generate-protobuf"
   337  	./scripts/generate-protobuf.sh
   338  
   339  vet:
   340  	@echo "vet"
   341  	$(GO) vet $(PACKAGES) 2>&1 | $(FAIL_ON_STDOUT)
   342  
   343  tidy:
   344  	@echo "go mod tidy"
   345  	./tools/check/check-tidy.sh
   346  
   347  # TODO: Unified cdc and dm config.
   348  check-static: tools/bin/golangci-lint
   349  	tools/bin/golangci-lint run --timeout 10m0s --skip-dirs "^dm/","^tests/"
   350  	cd dm && ../tools/bin/golangci-lint run --timeout 10m0s
   351  
   352  check: check-copyright generate_mock go-generate fmt check-static tidy terror_check errdoc \
   353  	check-merge-conflicts check-ticdc-dashboard check-diff-line-width check-makefiles \
   354  	check_cdc_integration_test check_dm_integration_test check_engine_integration_test
   355  	@git --no-pager diff --exit-code || (echo "Please add changed files!" && false)
   356  
   357  fast_check: check-copyright fmt check-static tidy terror_check errdoc \
   358  	check-merge-conflicts check-ticdc-dashboard check-diff-line-width swagger-spec check-makefiles \
   359  	check_cdc_integration_test check_dm_integration_test check_engine_integration_test
   360  	@git --no-pager diff --exit-code || (echo "Please add changed files!" && false)
   361  
   362  integration_test_coverage: tools/bin/gocovmerge tools/bin/goveralls
   363  	tools/bin/gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/entry/schema_test_helper.go|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
   364  ifeq ("$(JenkinsCI)", "1")
   365  	GO111MODULE=off go get github.com/mattn/goveralls
   366  	tools/bin/goveralls -parallel -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
   367  else
   368  	go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html"
   369  endif
   370  
   371  swagger-spec: tools/bin/swag
   372  	tools/bin/swag init --exclude dm,engine --parseVendor -generalInfo cdc/api/v1/api.go --output docs/swagger
   373  
   374  unit_test_coverage:
   375  	grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out"
   376  	go tool cover -html "$(TEST_DIR)/unit_cov.out" -o "$(TEST_DIR)/unit_cov.html"
   377  	go tool cover -func="$(TEST_DIR)/unit_cov.out"
   378  
   379  data-flow-diagram: docs/data-flow.dot
   380  	dot -Tsvg docs/data-flow.dot > docs/data-flow.svg
   381  
   382  generate_mock: ## Generate mock code.
   383  generate_mock: tools/bin/mockgen
   384  	scripts/generate-mock.sh
   385  
   386  clean:
   387  	go clean -i ./...
   388  	rm -rf *.out
   389  	rm -rf bin
   390  	rm -rf tools/bin
   391  	rm -rf tools/include
   392  
   393  dm: dm-master dm-worker dmctl dm-syncer
   394  
   395  dm-master:
   396  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dm-master ./cmd/dm-master
   397  
   398  dm-master-with-webui:
   399  	@echo "build webui first"
   400  	cd dm/ui && yarn --ignore-scripts && yarn build
   401  	$(GOBUILD) -ldflags '$(LDFLAGS)' -tags dm_webui -o bin/dm-master ./cmd/dm-master
   402  
   403  dm-worker:
   404  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dm-worker ./cmd/dm-worker
   405  
   406  dmctl:
   407  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dmctl ./cmd/dm-ctl
   408  
   409  dm-syncer:
   410  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dm-syncer ./cmd/dm-syncer
   411  
   412  dm-chaos-case:
   413  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/dm-chaos-case ./dm/chaos/cases
   414  
   415  dm_debug-tools:
   416  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/binlog-event-blackhole ./dm/debug-tools/binlog-event-blackhole
   417  
   418  dm_generate_openapi: tools/bin/oapi-codegen
   419  	@echo "generate_openapi"
   420  	cd dm && ../tools/bin/oapi-codegen --config=openapi/spec/server-gen-cfg.yaml openapi/spec/dm.yaml
   421  	cd dm && ../tools/bin/oapi-codegen --config=openapi/spec/types-gen-cfg.yaml openapi/spec/dm.yaml
   422  	cd dm && ../tools/bin/oapi-codegen --config=openapi/spec/client-gen-cfg.yaml openapi/spec/dm.yaml
   423  
   424  define run_dm_unit_test
   425  	@echo "running unit test for packages:" $(1)
   426  	mkdir -p $(DM_TEST_DIR)
   427  	$(FAILPOINT_ENABLE)
   428  	@export log_level=error; \
   429  	$(GOTEST) -timeout 10m -covermode=atomic -coverprofile="$(DM_TEST_DIR)/cov.unit_test.out" $(1) \
   430  	|| { $(FAILPOINT_DISABLE); exit 1; }
   431  	$(FAILPOINT_DISABLE)
   432  endef
   433  
   434  dm_unit_test: check_failpoint_ctl
   435  	$(call run_dm_unit_test,$(DM_PACKAGES))
   436  
   437  # run unit test for the specified pkg only, like `make dm_unit_test_pkg PKG=github.com/pingcap/tiflow/dm/master`
   438  dm_unit_test_pkg: check_failpoint_ctl
   439  	$(call run_dm_unit_test,$(PKG))
   440  
   441  dm_unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov tools/bin/gocov-xml
   442  	mkdir -p $(DM_TEST_DIR)
   443  	$(FAILPOINT_ENABLE)
   444  	@export log_level=error; \
   445  	CGO_ENABLED=1 tools/bin/gotestsum --junitfile dm-junit-report.xml -- -v -timeout 10m -p $(P) --race \
   446  	-covermode=atomic -coverprofile="$(DM_TEST_DIR)/cov.unit_test.out" $(DM_PACKAGES) \
   447  	|| { $(FAILPOINT_DISABLE); exit 1; }
   448  	tools/bin/gocov convert "$(DM_TEST_DIR)/cov.unit_test.out" | tools/bin/gocov-xml > dm-coverage.xml
   449  	$(FAILPOINT_DISABLE)
   450  
   451  dm_integration_test_build: check_failpoint_ctl
   452  	$(FAILPOINT_ENABLE)
   453  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   454  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   455  		-o bin/dm-worker.test github.com/pingcap/tiflow/cmd/dm-worker \
   456  		|| { $(FAILPOINT_DISABLE); exit 1; }
   457  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   458  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   459  		-o bin/dm-master.test github.com/pingcap/tiflow/cmd/dm-master \
   460  		|| { $(FAILPOINT_DISABLE); exit 1; }
   461  	$(GOTESTNORACE) -ldflags '$(LDFLAGS)' -c -cover -covermode=count \
   462  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   463  		-o bin/dmctl.test github.com/pingcap/tiflow/cmd/dm-ctl \
   464  		|| { $(FAILPOINT_DISABLE); exit 1; }
   465  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   466  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   467  		-o bin/dm-syncer.test github.com/pingcap/tiflow/cmd/dm-syncer \
   468  		|| { $(FAILPOINT_DISABLE); exit 1; }
   469  	$(FAILPOINT_DISABLE)
   470  	./dm/tests/prepare_tools.sh
   471  
   472  dm_integration_test_build_worker: check_failpoint_ctl
   473  	$(FAILPOINT_ENABLE)
   474  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   475  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   476  		-o bin/dm-worker.test github.com/pingcap/tiflow/cmd/dm-worker \
   477  		|| { $(FAILPOINT_DISABLE); exit 1; }
   478  	$(FAILPOINT_DISABLE)
   479  	./dm/tests/prepare_tools.sh
   480  
   481  dm_integration_test_build_master: check_failpoint_ctl
   482  	$(FAILPOINT_ENABLE)
   483  	$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
   484  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   485  		-o bin/dm-master.test github.com/pingcap/tiflow/cmd/dm-master \
   486  		|| { $(FAILPOINT_DISABLE); exit 1; }
   487  	$(FAILPOINT_DISABLE)
   488  	./dm/tests/prepare_tools.sh
   489  
   490  dm_integration_test_build_ctl: check_failpoint_ctl
   491  	$(FAILPOINT_ENABLE)
   492  	$(GOTESTNORACE) -ldflags '$(LDFLAGS)' -c -cover -covermode=count \
   493  		-coverpkg=github.com/pingcap/tiflow/dm/... \
   494  		-o bin/dmctl.test github.com/pingcap/tiflow/cmd/dm-ctl \
   495  		|| { $(FAILPOINT_DISABLE); exit 1; }
   496  	$(FAILPOINT_DISABLE)
   497  	./dm/tests/prepare_tools.sh
   498  
   499  install_test_python_dep:
   500  	@echo "install python requirments for test"
   501  	pip install --user -q -r ./dm/tests/requirements.txt
   502  
   503  check_third_party_binary_for_dm:
   504  	@which bin/tidb-server
   505  	@which bin/sync_diff_inspector
   506  	@which mysql
   507  	@which bin/minio
   508  
   509  dm_integration_test: check_third_party_binary_for_dm install_test_python_dep
   510  	@which bin/dm-master.test
   511  	@which bin/dm-worker.test
   512  	@which bin/dm-syncer.test
   513  	cd dm && ln -sf ../bin .
   514  	cd dm && ./tests/run.sh $(CASE)
   515  
   516  dm_integration_test_in_group: check_third_party_binary_for_dm install_test_python_dep
   517  	@which bin/dm-master.test
   518  	@which bin/dm-worker.test
   519  	@which bin/dm-syncer.test
   520  	cd dm && ln -sf ../bin .
   521  	cd dm && ./tests/run_group.sh $(GROUP)
   522  
   523  dm_compatibility_test: check_third_party_binary_for_dm
   524  	@which bin/dm-master.test.current
   525  	@which bin/dm-worker.test.current
   526  	@which bin/dm-master.test.previous
   527  	@which bin/dm-worker.test.previous
   528  	cd dm && ln -sf ../bin .
   529  	cd dm && ./tests/compatibility_run.sh ${CASE}
   530  
   531  dm_coverage: tools/bin/gocovmerge tools/bin/goveralls
   532  	# unify cover mode in coverage files, more details refer to dm/tests/_utils/run_dm_ctl
   533  	find "$(DM_TEST_DIR)" -type f -name "cov.*.dmctl.*.out" -exec $(SED_IN_PLACE) "s/mode: count/mode: atomic/g" {} \;
   534  	tools/bin/gocovmerge "$(DM_TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*.pb.gw.go|.*.__failpoint_binding__.go|.*debug-tools.*|.*chaos.*" > "$(DM_TEST_DIR)/all_cov.out"
   535  	tools/bin/gocovmerge "$(DM_TEST_DIR)"/cov.unit_test*.out | grep -vE ".*.pb.go|.*.pb.gw.go|.*.__failpoint_binding__.go|.*debug-tools.*|.*chaos.*" > $(DM_TEST_DIR)/unit_test.out
   536  	go tool cover -html "$(DM_TEST_DIR)/all_cov.out" -o "$(DM_TEST_DIR)/all_cov.html"
   537  	go tool cover -html "$(DM_TEST_DIR)/unit_test.out" -o "$(DM_TEST_DIR)/unit_test_cov.html"
   538  
   539  
   540  check_failpoint_ctl: tools/bin/failpoint-ctl
   541  
   542  failpoint-enable: check_failpoint_ctl
   543  	$(FAILPOINT_ENABLE)
   544  
   545  failpoint-disable: check_failpoint_ctl
   546  	$(FAILPOINT_DISABLE)
   547  
   548  engine: tiflow
   549  
   550  tiflow:
   551  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiflow ./cmd/tiflow/main.go
   552  
   553  tiflow-demo:
   554  	@echo "this demo is deprecated, will be removed in next version"
   555  
   556  tiflow-chaos-case:
   557  	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiflow-chaos-case ./engine/chaos/cases
   558  
   559  engine_unit_test: check_failpoint_ctl
   560  	$(call run_engine_unit_test,$(ENGINE_PACKAGES))
   561  
   562  engine_integration_test: check_third_party_binary_for_engine
   563  	mkdir -p /tmp/tiflow_engine_test || true
   564  	./engine/test/integration_tests/run.sh "$(CASE)" "$(START_AT)" 2>&1 | tee /tmp/tiflow_engine_test/engine_it.log
   565  	./engine/test/utils/check_log.sh
   566  
   567  check_third_party_binary_for_engine:
   568  	@which bash || (echo "bash not found in ${PATH}"; exit 1)
   569  	@which docker || (echo "docker not found in ${PATH}"; exit 1)
   570  	@which go || (echo "go not found in ${PATH}"; exit 1)
   571  	@which mysql || (echo "mysql not found in ${PATH}"; exit 1)
   572  	@which jq || (echo "jq not found in ${PATH}"; exit 1)
   573  	@which mc || (echo "mc not found in ${PATH}, you can use 'make bin/mc' and move bin/mc to ${PATH}"; exit 1)
   574  	@which bin/sync_diff_inspector || (echo "run 'make bin/sync_diff_inspector' to download it if you need")
   575  
   576  check_engine_integration_test:
   577  	./engine/test/utils/check_case.sh
   578  	./engine/test/integration_tests/run_group.sh "check others"
   579  
   580  check_dm_integration_test:
   581  	./dm/tests/run_group.sh "check others"
   582  
   583  check_cdc_integration_test:
   584  	./tests/integration_tests/run_group.sh check "others"
   585  
   586  bin/mc:
   587  	./scripts/download-mc.sh
   588  
   589  bin/sync_diff_inspector:
   590  	./scripts/download-sync-diff.sh
   591  
   592  define run_engine_unit_test
   593  	@echo "running unit test for packages:" $(1)
   594  	mkdir -p $(ENGINE_TEST_DIR)
   595  	$(FAILPOINT_ENABLE)
   596  	@export log_level=error; \
   597  	$(GOTEST) -timeout 5m -covermode=atomic -coverprofile="$(ENGINE_TEST_DIR)/cov.unit_test.out" $(1) \
   598  	|| { $(FAILPOINT_DISABLE); exit 1; }
   599  	$(FAILPOINT_DISABLE)
   600  endef
   601  
   602  engine_unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov tools/bin/gocov-xml
   603  	mkdir -p $(ENGINE_TEST_DIR)
   604  	$(FAILPOINT_ENABLE)
   605  	@export log_level=error; \
   606  	CGO_ENABLED=1 tools/bin/gotestsum --junitfile engine-junit-report.xml -- -v -timeout 5m -p $(P) --race \
   607  	-covermode=atomic -coverprofile="$(ENGINE_TEST_DIR)/cov.unit_test.out" $(ENGINE_PACKAGES) \
   608  	|| { $(FAILPOINT_DISABLE); exit 1; }
   609  	tools/bin/gocov convert "$(ENGINE_TEST_DIR)/cov.unit_test.out" | tools/bin/gocov-xml > engine-coverage.xml
   610  	$(FAILPOINT_DISABLE)
   611  
   612  prepare_test_binaries:
   613  	./scripts/download-integration-test-binaries.sh "$(branch)" "$(community)" "$(ver)" "$(os)" "$(arch)"
   614  	touch prepare_test_binaries