github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/Makefile (about) 1 SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) 2 3 # Grab necessary submodules, in case the repo was cloned without --recursive 4 $(SELF_DIR)/.ci/common.mk: 5 git submodule update --init --recursive 6 7 include $(SELF_DIR)/.ci/common.mk 8 9 SHELL=/bin/bash -o pipefail 10 GOPATH=$(shell eval $$(go env | grep GOPATH) && echo $$GOPATH) 11 12 auto_gen := scripts/auto-gen.sh 13 process_coverfile := scripts/process-cover.sh 14 gopath_prefix := $(GOPATH)/src 15 gopath_bin_path := $(GOPATH)/bin 16 m3_package := github.com/m3db/m3 17 m3_package_path := $(gopath_prefix)/$(m3_package) 18 mockgen_package := github.com/golang/mock/mockgen 19 tools_bin_path := $(abspath ./_tools/bin) 20 combined_bin_paths := $(tools_bin_path):$(gopath_bin_path) 21 mocks_output_dir := generated/mocks 22 mocks_rules_dir := generated/mocks 23 proto_output_dir := generated/proto 24 proto_rules_dir := generated/proto 25 assets_output_dir := generated/assets 26 assets_rules_dir := generated/assets 27 thrift_output_dir := generated/thrift/rpc 28 thrift_rules_dir := generated/thrift 29 vendor_prefix := vendor 30 cache_policy ?= recently_read 31 aggregator_client ?= m3msg 32 genny_target ?= genny-all 33 34 BUILD := $(abspath ./bin) 35 VENDOR := $(m3_package_path)/$(vendor_prefix) 36 GO_BUILD_LDFLAGS_CMD := $(abspath ./scripts/go-build-ldflags.sh) 37 GO_BUILD_LDFLAGS := $(shell $(GO_BUILD_LDFLAGS_CMD) LDFLAG) 38 GO_BUILD_COMMON_ENV := CGO_ENABLED=0 39 LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 $(GO_BUILD_COMMON_ENV) 40 # GO_RELEASER_DOCKER_IMAGE is latest goreleaser for go 1.18 41 GO_RELEASER_DOCKER_IMAGE := goreleaser/goreleaser:v1.8.3 42 GO_RELEASER_RELEASE_ARGS ?= --rm-dist 43 GO_RELEASER_WORKING_DIR := /go/src/github.com/m3db/m3 44 GOLANGCI_LINT_VERSION := v1.45.2 45 46 export NPROC := 2 # Maximum package concurrency for unit tests. 47 48 SERVICES := \ 49 m3dbnode \ 50 m3coordinator \ 51 m3aggregator \ 52 m3query \ 53 m3em_agent \ 54 m3comparator \ 55 r2ctl \ 56 57 SUBDIRS := \ 58 x \ 59 cluster \ 60 msg \ 61 metrics \ 62 cmd \ 63 dbnode \ 64 query \ 65 m3em \ 66 m3ninx \ 67 aggregator \ 68 ctl \ 69 70 TOOLS := \ 71 annotation_checker \ 72 read_ids \ 73 read_index_ids \ 74 read_data_files \ 75 read_index_files \ 76 read_index_segments \ 77 read_commitlog \ 78 split_shards \ 79 split_index_shards \ 80 query_index_segments \ 81 clone_fileset \ 82 dtest \ 83 verify_data_files \ 84 verify_ids \ 85 verify_index_files \ 86 carbon_load \ 87 m3ctl \ 88 89 GOINSTALL_BUILD_TOOLS := \ 90 github.com/fossas/fossa-cli/cmd/fossa@latest \ 91 github.com/golang/mock/mockgen@latest \ 92 github.com/google/go-jsonnet/cmd/jsonnet@latest \ 93 github.com/m3db/build-tools/utilities/genclean@latest \ 94 github.com/m3db/tools/update-license@latest \ 95 github.com/mauricelam/genny@v0.0.0-20180903214747-eb2c5232c885 \ 96 github.com/mjibson/esc@latest \ 97 github.com/pointlander/peg@latest \ 98 github.com/rakyll/statik@latest \ 99 github.com/instrumenta/kubeval@latest \ 100 github.com/wjdp/htmltest@latest \ 101 github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) \ 102 103 .PHONY: setup 104 setup: 105 mkdir -p $(BUILD) 106 107 .PHONY: install-vendor-m3 108 install-vendor-m3: 109 [ -d $(VENDOR) ] || GOSUMDB=off go mod vendor 110 111 .PHONY: docker-dev-prep 112 docker-dev-prep: 113 mkdir -p ./bin/config 114 115 # Hacky way to find all configs and put into ./bin/config/ 116 find ./src | fgrep config | fgrep ".yml" | xargs -I{} cp {} ./bin/config/ 117 find ./src | fgrep config | fgrep ".yaml" | xargs -I{} cp {} ./bin/config/ 118 119 define SERVICE_RULES 120 121 .PHONY: $(SERVICE) 122 $(SERVICE): setup 123 ifeq ($(SERVICE),m3ctl) 124 @echo "Building $(SERVICE) dependencies" 125 make build-ui-ctl-statik-gen 126 endif 127 @echo Building $(SERVICE) 128 [ -d $(VENDOR) ] || make install-vendor-m3 129 $(GO_BUILD_COMMON_ENV) go build -ldflags '$(GO_BUILD_LDFLAGS)' -o $(BUILD)/$(SERVICE) ./src/cmd/services/$(SERVICE)/main/. 130 131 .PHONY: $(SERVICE)-linux-amd64 132 $(SERVICE)-linux-amd64: 133 $(LINUX_AMD64_ENV) make $(SERVICE) 134 135 .PHONY: $(SERVICE)-docker-dev 136 $(SERVICE)-docker-dev: clean-build $(SERVICE)-linux-amd64 137 make docker-dev-prep 138 139 # Build development docker image 140 docker build -t $(SERVICE):dev -t quay.io/m3dbtest/$(SERVICE):dev-$(USER) -f ./docker/$(SERVICE)/development.Dockerfile ./bin 141 142 .PHONY: $(SERVICE)-docker-dev-push 143 $(SERVICE)-docker-dev-push: $(SERVICE)-docker-dev 144 docker push quay.io/m3dbtest/$(SERVICE):dev-$(USER) 145 @echo "Pushed quay.io/m3dbtest/$(SERVICE):dev-$(USER)" 146 147 endef 148 149 $(foreach SERVICE,$(SERVICES),$(eval $(SERVICE_RULES))) 150 151 define TOOL_RULES 152 153 .PHONY: $(TOOL) 154 $(TOOL): setup 155 @echo Building $(TOOL) 156 go build -o $(BUILD)/$(TOOL) ./src/cmd/tools/$(TOOL)/main/. 157 158 .PHONY: $(TOOL)-linux-amd64 159 $(TOOL)-linux-amd64: 160 $(LINUX_AMD64_ENV) make $(TOOL) 161 162 endef 163 164 $(foreach TOOL,$(TOOLS),$(eval $(TOOL_RULES))) 165 166 .PHONY: services services-linux-amd64 167 services: $(SERVICES) 168 services-linux-amd64: 169 $(LINUX_AMD64_ENV) make services 170 171 .PHONY: tools tools-linux-amd64 172 tools: $(TOOLS) 173 tools-linux-amd64: 174 $(LINUX_AMD64_ENV) make tools 175 176 .PHONY: all 177 all: test-ci-unit test-ci-integration services tools 178 @echo Made all successfully 179 180 .SILENT: install-tools 181 .PHONY: install-tools 182 install-tools: 183 echo "Installing build tools" 184 for tool in $(GOINSTALL_BUILD_TOOLS); do \ 185 GOBIN=$(tools_bin_path) go install $$tool; \ 186 done 187 188 .PHONY: check-for-goreleaser-github-token 189 check-for-goreleaser-github-token: 190 ifndef GITHUB_TOKEN 191 echo "Usage: make GITHUB_TOKEN=\"<TOKEN>\" release" 192 exit 1 193 endif 194 195 .PHONY: release 196 release: check-for-goreleaser-github-token 197 @echo Releasing new version 198 $(GO_BUILD_LDFLAGS_CMD) ECHO > $(BUILD)/release-vars.env 199 docker run -e "GITHUB_TOKEN=$(GITHUB_TOKEN)" --env-file $(BUILD)/release-vars.env -v $(PWD):$(GO_RELEASER_WORKING_DIR) -w $(GO_RELEASER_WORKING_DIR) $(GO_RELEASER_DOCKER_IMAGE) release $(GO_RELEASER_RELEASE_ARGS) 200 201 .PHONY: release-snapshot 202 release-snapshot: check-for-goreleaser-github-token 203 @echo Creating snapshot release 204 make release GO_RELEASER_RELEASE_ARGS="--snapshot --rm-dist" 205 206 # NB(schallert): if updating this target, be sure to update the commands used in 207 # the .buildkite/docs_push.sh. We can't share the make targets because our 208 # Makefile assumes its running under bash and the container is alpine (ash 209 # shell). 210 211 .PHONY: docs-build 212 docs-build: 213 @HUGO_DOCKER=true ./scripts/site-build.sh 214 215 .PHONY: docs-test 216 docs-test: setup install-tools docs-build 217 cp site/.htmltest.yml $(BUILD)/.htmltest.yml 218 ifneq ($(DOCSTEST_AUTH_USER),) 219 ifneq ($(DOCSTEST_AUTH_TOKEN),) 220 @echo 'HTTPHeaders: {"Authorization":"Basic $(shell echo -n "$$DOCSTEST_AUTH_USER:$$DOCSTEST_AUTH_TOKEN" | base64 | xargs echo -n)"}' >> $(BUILD)/.htmltest.yml 221 endif 222 endif 223 $(tools_bin_path)/htmltest -c $(BUILD)/.htmltest.yml 224 225 .PHONY: docker-integration-test 226 docker-integration-test: 227 @echo "--- Running Docker integration test" 228 ./scripts/docker-integration-tests/run.sh 229 230 .PHONY: docker-compatibility-test 231 docker-compatibility-test: 232 @echo "--- Running Prometheus compatibility test" 233 ./scripts/comparator/run.sh 234 235 .PHONY: prom-compat 236 prom-compat: 237 @echo "--- Running local Prometheus compatibility test" 238 CI="false" make docker-compatibility-test 239 240 .PHONY: site-build 241 site-build: 242 @echo "Building site" 243 @./scripts/site-build.sh 244 245 # Generate configs in config/ 246 .PHONY: config-gen 247 config-gen: install-tools 248 @echo "--- Generating configs" 249 $(tools_bin_path)/jsonnet -S $(m3_package_path)/config/m3db/local-etcd/m3dbnode_cmd.jsonnet > $(m3_package_path)/config/m3db/local-etcd/generated.yaml 250 $(tools_bin_path)/jsonnet -S $(m3_package_path)/config/m3db/clustered-etcd/m3dbnode_cmd.jsonnet > $(m3_package_path)/config/m3db/clustered-etcd/generated.yaml 251 252 SUBDIR_TARGETS := \ 253 mock-gen \ 254 thrift-gen \ 255 proto-gen \ 256 asset-gen \ 257 genny-gen \ 258 license-gen \ 259 all-gen \ 260 all-gen 261 262 .PHONY: test-ci-unit 263 test-ci-unit: test-base 264 $(process_coverfile) $(coverfile) 265 266 .PHONY: test-ci-big-unit 267 test-ci-big-unit: test-big-base 268 $(process_coverfile) $(coverfile) 269 270 .PHONY: test-ci-integration 271 test-ci-integration: 272 INTEGRATION_TIMEOUT=10m TEST_SERIES_CACHE_POLICY=$(cache_policy) TEST_AGGREGATOR_CLIENT_TYPE=$(aggregator_client) make test-base-ci-integration 273 $(process_coverfile) $(coverfile) 274 275 .PHONY: test-ci-cluster-integration 276 test-ci-cluster-integration: 277 @echo "--- test-ci-cluster-integration" 278 go test -count=1 -p 1 -tags=cluster_integration ./src/integration/... 279 280 .PHONY: test-ci-test-harness 281 test-ci-test-harness: 282 @echo "--- test-ci-test-harness" 283 go test -count=1 -p 1 -tags=test_harness ./src/integration/resources/... 284 285 define SUBDIR_RULES 286 287 .PHONY: mock-gen-$(SUBDIR) 288 mock-gen-$(SUBDIR): install-tools 289 @echo "--- Generating mocks $(SUBDIR)" 290 @[ ! -d src/$(SUBDIR)/$(mocks_rules_dir) ] || \ 291 PATH="$(combined_bin_paths):$(PATH)" PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(mocks_output_dir) src/$(SUBDIR)/$(mocks_rules_dir) 292 293 .PHONY: thrift-gen-$(SUBDIR) 294 thrift-gen-$(SUBDIR): install-tools 295 @echo "--- Generating thrift files $(SUBDIR)" 296 @[ ! -d src/$(SUBDIR)/$(thrift_rules_dir) ] || \ 297 PATH="$(combined_bin_paths):$(PATH)" PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(thrift_output_dir) src/$(SUBDIR)/$(thrift_rules_dir) 298 299 .PHONY: proto-gen-$(SUBDIR) 300 proto-gen-$(SUBDIR): install-tools 301 @echo "--- Generating protobuf files $(SUBDIR)" 302 [ ! -d src/$(SUBDIR)/$(proto_rules_dir) ] || \ 303 PATH="$(combined_bin_paths):$(PATH)" PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(proto_output_dir) src/$(SUBDIR)/$(proto_rules_dir) 304 305 .PHONY: asset-gen-$(SUBDIR) 306 asset-gen-$(SUBDIR): install-tools 307 @echo "--- Generating asset files $(SUBDIR)" 308 @[ ! -d src/$(SUBDIR)/$(assets_rules_dir) ] || \ 309 PATH="$(combined_bin_paths):$(PATH)" PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(assets_output_dir) src/$(SUBDIR)/$(assets_rules_dir) 310 311 .PHONY: genny-gen-$(SUBDIR) 312 genny-gen-$(SUBDIR): install-tools 313 @echo "--- Generating genny files $(SUBDIR)" 314 @[ ! -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk ] || \ 315 PATH="$(combined_bin_paths):$(PATH)" make -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk $(genny_target) 316 @PATH="$(combined_bin_paths):$(PATH)" bash -c "source ./scripts/auto-gen-helpers.sh && gen_cleanup_dir '*_gen.go' $(SELF_DIR)/src/$(SUBDIR)/ && gen_cleanup_dir '*_gen_test.go' $(SELF_DIR)/src/$(SUBDIR)/" 317 318 .PHONY: license-gen-$(SUBDIR) 319 license-gen-$(SUBDIR): install-tools 320 @echo "--- Updating license in files $(SUBDIR)" 321 @find $(SELF_DIR)/src/$(SUBDIR) -name '*.go' | PATH="$(combined_bin_paths):$(PATH)" xargs -I{} update-license {} 322 323 .PHONY: all-gen-$(SUBDIR) 324 # NB(prateek): order matters here, mock-gen needs to be after proto/thrift because we sometimes 325 # generate mocks for thrift/proto generated code. Similarly, license-gen needs to be last because 326 # we make header changes. 327 all-gen-$(SUBDIR): thrift-gen-$(SUBDIR) proto-gen-$(SUBDIR) asset-gen-$(SUBDIR) genny-gen-$(SUBDIR) mock-gen-$(SUBDIR) license-gen-$(SUBDIR) 328 329 .PHONY: test-$(SUBDIR) 330 test-$(SUBDIR): 331 @echo testing $(SUBDIR) 332 SRC_ROOT=./src/$(SUBDIR) make test-base 333 gocov convert $(coverfile) | gocov report 334 335 .PHONY: test-xml-$(SUBDIR) 336 test-xml-$(SUBDIR): 337 @echo test-xml $(SUBDIR) 338 SRC_ROOT=./src/$(SUBDIR) make test-base-xml 339 340 .PHONY: test-html-$(SUBDIR) 341 test-html-$(SUBDIR): 342 @echo test-html $(SUBDIR) 343 SRC_ROOT=./src/$(SUBDIR) make test-base-html 344 345 .PHONY: test-integration-$(SUBDIR) 346 test-integration-$(SUBDIR): 347 @echo test-integration $(SUBDIR) 348 SRC_ROOT=./src/$(SUBDIR) make test-base-integration 349 350 # Usage: make test-single-integration name=<test_name> 351 .PHONY: test-single-integration-$(SUBDIR) 352 test-single-integration-$(SUBDIR): 353 SRC_ROOT=./src/$(SUBDIR) make test-base-single-integration name=$(name) 354 355 .PHONY: test-ci-unit-$(SUBDIR) 356 test-ci-unit-$(SUBDIR): 357 @echo "--- test-ci-unit $(SUBDIR)" 358 SRC_ROOT=./src/$(SUBDIR) make test-base 359 if [ -z "$(SKIP_CODECOV)" ]; then \ 360 @echo "--- uploading coverage report"; \ 361 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 362 fi 363 364 .PHONY: test-ci-big-unit-$(SUBDIR) 365 test-ci-big-unit-$(SUBDIR): 366 @echo "--- test-ci-big-unit $(SUBDIR)" 367 SRC_ROOT=./src/$(SUBDIR) make test-big-base 368 if [ -z "$(SKIP_CODECOV)" ]; then \ 369 @echo "--- uploading coverage report"; \ 370 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 371 fi 372 373 .PHONY: test-ci-integration-$(SUBDIR) 374 test-ci-integration-$(SUBDIR): 375 @echo "--- test-ci-integration $(SUBDIR)" 376 SRC_ROOT=./src/$(SUBDIR) PANIC_ON_INVARIANT_VIOLATED=true INTEGRATION_TIMEOUT=10m TEST_SERIES_CACHE_POLICY=$(cache_policy) TEST_AGGREGATOR_CLIENT_TYPE=$(aggregator_client) make test-base-ci-integration 377 if [ -z "$(SKIP_CODECOV)" ]; then \ 378 @echo "--- uploading coverage report"; \ 379 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 380 fi 381 382 .PHONY: lint-$(SUBDIR) 383 lint-$(SUBDIR): export GO_BUILD_TAGS = $(GO_BUILD_TAGS_LIST) 384 lint-$(SUBDIR): install-tools 385 @echo "--- :golang: Running linter on $(SUBDIR)" 386 ./scripts/run-ci-lint.sh $(tools_bin_path)/golangci-lint ./src/$(SUBDIR)/... 387 388 endef 389 390 # generate targets for each SUBDIR in SUBDIRS based on the rules specified above. 391 $(foreach SUBDIR,$(SUBDIRS),$(eval $(SUBDIR_RULES))) 392 393 define SUBDIR_TARGET_RULE 394 .PHONY: $(SUBDIR_TARGET) 395 $(SUBDIR_TARGET): $(foreach SUBDIR,$(SUBDIRS),$(SUBDIR_TARGET)-$(SUBDIR)) 396 endef 397 398 # generate targets across SUBDIRS for each SUBDIR_TARGET. i.e. generate rules 399 # which allow `make all-gen` to invoke `make all-gen-dbnode all-gen-coordinator ...` 400 # NB: we skip lint explicity as it runs as a separate CI step. 401 $(foreach SUBDIR_TARGET, $(SUBDIR_TARGETS), $(eval $(SUBDIR_TARGET_RULE))) 402 403 # Builds the single kube bundle from individual manifest files. 404 .PHONY: kube-gen-all 405 kube-gen-all: install-tools 406 @echo "--- Generating kube bundle" 407 @./kube/scripts/build_bundle.sh 408 find kube -name '*.yaml' -print0 | PATH="$(combined_bin_paths):$(PATH)" xargs -0 kubeval -v=1.12.0 409 410 .PHONY: go-mod-tidy 411 go-mod-tidy: 412 @echo "--- :golang: tidying modules" 413 go mod tidy 414 415 .PHONY: all-gen 416 all-gen: \ 417 install-tools \ 418 $(foreach SUBDIR_TARGET, $(filter-out lint all-gen,$(SUBDIR_TARGETS)), $(SUBDIR_TARGET)) \ 419 kube-gen-all \ 420 go-mod-tidy 421 422 .PHONY: build-ui-ctl 423 build-ui-ctl: 424 ifeq ($(shell ls ./src/ctl/ui/build 2>/dev/null),) 425 # Need to use subshell output of set-node-version as cannot 426 # set side-effects of nvm to later commands 427 @echo "Building UI components, if npm install or build fails try: npm cache clean" 428 make node-yarn-run \ 429 node_version="6" \ 430 node_cmd="cd $(m3_package_path)/src/ctl/ui && yarn install && yarn build" 431 # If we've installed nvm locally, remove it due to some cleanup permissions 432 # issue we run into in CI. 433 rm -rf .nvm 434 else 435 @echo "Skip building UI components, already built, to rebuild first make clean" 436 endif 437 # Move public assets into public subdirectory so that it can 438 # be included in the single statik package built from ./ui/build 439 rm -rf ./src/ctl/ui/build/public 440 cp -r ./src/ctl/public ./src/ctl/ui/build/public 441 442 .PHONY: build-ui-ctl-statik-gen 443 build-ui-ctl-statik-gen: build-ui-ctl-statik license-gen-ctl 444 445 .PHONY: build-ui-ctl-statik 446 build-ui-ctl-statik: build-ui-ctl install-tools 447 mkdir -p ./src/ctl/generated/ui 448 $(tools_bin_path)/statik -m -f -src ./src/ctl/ui/build -dest ./src/ctl/generated/ui -p statik 449 450 .PHONY: node-yarn-run 451 node-yarn-run: 452 make node-run \ 453 node_version="$(node_version)" \ 454 node_cmd="(yarn --version 2>&1 >/dev/null || npm install -g yarn@^1.17.0) && $(node_cmd)" 455 456 .PHONY: node-run 457 node-run: 458 ifneq ($(shell command -v nvm 2>/dev/null),) 459 @echo "Using nvm to select node version $(node_version)" 460 nvm use $(node_version) && bash -c "$(node_cmd)" 461 else 462 mkdir .nvm 463 # Install nvm locally 464 NVM_DIR=$(SELF_DIR)/.nvm PROFILE=/dev/null scripts/install_nvm.sh 465 bash -c "source $(SELF_DIR)/.nvm/nvm.sh; nvm install 6" 466 bash -c "source $(SELF_DIR)/.nvm/nvm.sh && nvm use 6 && $(node_cmd)" 467 endif 468 469 # Tests that all currently generated types match their contents if they were regenerated 470 .PHONY: test-all-gen 471 test-all-gen: all-gen 472 @test "$(shell git --no-pager diff --exit-code --shortstat 2>/dev/null)" = "" || (git --no-pager diff --text --exit-code && echo "Check git status, there are dirty files" && exit 1) 473 @test "$(shell git status --exit-code --porcelain 2>/dev/null | grep "^??")" = "" || (git status --exit-code --porcelain && echo "Check git status, there are untracked files" && exit 1) 474 475 # Runs a fossa license report 476 .PHONY: fossa 477 fossa: install-tools 478 PATH="$(combined_bin_paths):$(PATH)" fossa analyze --verbose --no-ansi 479 480 # Waits for the result of a fossa test and exits success if pass or fail if fails 481 .PHONY: fossa-test 482 fossa-test: fossa 483 PATH="$(combined_bin_paths):$(PATH)" fossa test 484 485 .PHONY: clean-build 486 clean-build: 487 @rm -rf $(BUILD) 488 489 .PHONY: clean 490 clean: clean-build 491 @rm -f *.html *.xml *.out *.test 492 @rm -rf $(VENDOR) 493 @rm -rf ./src/ctl/ui/build 494 495 .DEFAULT_GOAL := all 496 497 lint: install-tools 498 @echo "--- :golang: Running linter on 'src'" 499 ./scripts/run-ci-lint.sh $(tools_bin_path)/golangci-lint ./src/...