github.com/m3db/m3@v1.5.0/Makefile (about) 1 SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) 2 3 # Grab necessary submodules, in case the repo was cloned without --recursive 4 $(SELF_DIR)/.ci/common.mk: 5 git submodule update --init --recursive 6 7 include $(SELF_DIR)/.ci/common.mk 8 9 SHELL=/bin/bash -o pipefail 10 GOPATH=$(shell eval $$(go env | grep GOPATH) && echo $$GOPATH) 11 12 auto_gen := scripts/auto-gen.sh 13 process_coverfile := scripts/process-cover.sh 14 gopath_prefix := $(GOPATH)/src 15 gopath_bin_path := $(GOPATH)/bin 16 m3_package := github.com/m3db/m3 17 m3_package_path := $(gopath_prefix)/$(m3_package) 18 mockgen_package := github.com/golang/mock/mockgen 19 tools_bin_path := $(abspath ./_tools/bin) 20 combined_bin_paths := $(tools_bin_path):$(gopath_bin_path) 21 mocks_output_dir := generated/mocks 22 mocks_rules_dir := generated/mocks 23 proto_output_dir := generated/proto 24 proto_rules_dir := generated/proto 25 assets_output_dir := generated/assets 26 assets_rules_dir := generated/assets 27 thrift_output_dir := generated/thrift/rpc 28 thrift_rules_dir := generated/thrift 29 vendor_prefix := vendor 30 cache_policy ?= recently_read 31 aggregator_client ?= m3msg 32 genny_target ?= genny-all 33 34 BUILD := $(abspath ./bin) 35 VENDOR := $(m3_package_path)/$(vendor_prefix) 36 GO_BUILD_LDFLAGS_CMD := $(abspath ./scripts/go-build-ldflags.sh) 37 GO_BUILD_LDFLAGS := $(shell $(GO_BUILD_LDFLAGS_CMD) LDFLAG) 38 GO_BUILD_COMMON_ENV := CGO_ENABLED=0 39 LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 $(GO_BUILD_COMMON_ENV) 40 # GO_RELEASER_DOCKER_IMAGE is latest goreleaser for go 1.16 41 GO_RELEASER_DOCKER_IMAGE := goreleaser/goreleaser:v0.173.2 42 GO_RELEASER_RELEASE_ARGS ?= --rm-dist 43 GO_RELEASER_WORKING_DIR := /go/src/github.com/m3db/m3 44 GOLANGCI_LINT_VERSION := v1.37.0 45 46 export NPROC := 2 # Maximum package concurrency for unit tests. 47 48 SERVICES := \ 49 m3dbnode \ 50 m3coordinator \ 51 m3aggregator \ 52 m3query \ 53 m3em_agent \ 54 m3comparator \ 55 r2ctl \ 56 57 SUBDIRS := \ 58 x \ 59 cluster \ 60 msg \ 61 metrics \ 62 cmd \ 63 dbnode \ 64 query \ 65 m3em \ 66 m3ninx \ 67 aggregator \ 68 ctl \ 69 70 TOOLS := \ 71 annotation_checker \ 72 read_ids \ 73 read_index_ids \ 74 read_data_files \ 75 read_index_files \ 76 read_index_segments \ 77 read_commitlog \ 78 split_shards \ 79 split_index_shards \ 80 query_index_segments \ 81 clone_fileset \ 82 dtest \ 83 verify_data_files \ 84 verify_index_files \ 85 carbon_load \ 86 m3ctl \ 87 88 GOINSTALL_BUILD_TOOLS := \ 89 github.com/fossas/fossa-cli/cmd/fossa@latest \ 90 github.com/golang/mock/mockgen@latest \ 91 github.com/google/go-jsonnet/cmd/jsonnet@latest \ 92 github.com/m3db/build-tools/utilities/genclean@latest \ 93 github.com/m3db/tools/update-license@latest \ 94 github.com/mauricelam/genny@v0.0.0-20180903214747-eb2c5232c885 \ 95 github.com/mjibson/esc@latest \ 96 github.com/pointlander/peg@latest \ 97 github.com/rakyll/statik@latest \ 98 github.com/instrumenta/kubeval@latest \ 99 github.com/wjdp/htmltest@latest \ 100 github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) \ 101 102 .PHONY: setup 103 setup: 104 mkdir -p $(BUILD) 105 106 .PHONY: install-vendor-m3 107 install-vendor-m3: 108 [ -d $(VENDOR) ] || GOSUMDB=off go mod vendor 109 110 .PHONY: docker-dev-prep 111 docker-dev-prep: 112 mkdir -p ./bin/config 113 114 # Hacky way to find all configs and put into ./bin/config/ 115 find ./src | fgrep config | fgrep ".yml" | xargs -I{} cp {} ./bin/config/ 116 find ./src | fgrep config | fgrep ".yaml" | xargs -I{} cp {} ./bin/config/ 117 118 define SERVICE_RULES 119 120 .PHONY: $(SERVICE) 121 $(SERVICE): setup 122 ifeq ($(SERVICE),m3ctl) 123 @echo "Building $(SERVICE) dependencies" 124 make build-ui-ctl-statik-gen 125 endif 126 @echo Building $(SERVICE) 127 [ -d $(VENDOR) ] || make install-vendor-m3 128 $(GO_BUILD_COMMON_ENV) go build -ldflags '$(GO_BUILD_LDFLAGS)' -o $(BUILD)/$(SERVICE) ./src/cmd/services/$(SERVICE)/main/. 129 130 .PHONY: $(SERVICE)-linux-amd64 131 $(SERVICE)-linux-amd64: 132 $(LINUX_AMD64_ENV) make $(SERVICE) 133 134 .PHONY: $(SERVICE)-docker-dev 135 $(SERVICE)-docker-dev: clean-build $(SERVICE)-linux-amd64 136 make docker-dev-prep 137 138 # Build development docker image 139 docker build -t $(SERVICE):dev -t quay.io/m3dbtest/$(SERVICE):dev-$(USER) -f ./docker/$(SERVICE)/development.Dockerfile ./bin 140 141 .PHONY: $(SERVICE)-docker-dev-push 142 $(SERVICE)-docker-dev-push: $(SERVICE)-docker-dev 143 docker push quay.io/m3dbtest/$(SERVICE):dev-$(USER) 144 @echo "Pushed quay.io/m3dbtest/$(SERVICE):dev-$(USER)" 145 146 endef 147 148 $(foreach SERVICE,$(SERVICES),$(eval $(SERVICE_RULES))) 149 150 define TOOL_RULES 151 152 .PHONY: $(TOOL) 153 $(TOOL): setup 154 @echo Building $(TOOL) 155 go build -o $(BUILD)/$(TOOL) ./src/cmd/tools/$(TOOL)/main/. 156 157 .PHONY: $(TOOL)-linux-amd64 158 $(TOOL)-linux-amd64: 159 $(LINUX_AMD64_ENV) make $(TOOL) 160 161 endef 162 163 $(foreach TOOL,$(TOOLS),$(eval $(TOOL_RULES))) 164 165 .PHONY: services services-linux-amd64 166 services: $(SERVICES) 167 services-linux-amd64: 168 $(LINUX_AMD64_ENV) make services 169 170 .PHONY: tools tools-linux-amd64 171 tools: $(TOOLS) 172 tools-linux-amd64: 173 $(LINUX_AMD64_ENV) make tools 174 175 .PHONY: all 176 all: test-ci-unit test-ci-integration services tools 177 @echo Made all successfully 178 179 .SILENT: install-tools 180 .PHONY: install-tools 181 install-tools: 182 echo "Installing build tools" 183 for tool in $(GOINSTALL_BUILD_TOOLS); do \ 184 GOBIN=$(tools_bin_path) go install $$tool; \ 185 done 186 187 .PHONY: check-for-goreleaser-github-token 188 check-for-goreleaser-github-token: 189 ifndef GITHUB_TOKEN 190 echo "Usage: make GITHUB_TOKEN=\"<TOKEN>\" release" 191 exit 1 192 endif 193 194 .PHONY: release 195 release: check-for-goreleaser-github-token 196 @echo Releasing new version 197 $(GO_BUILD_LDFLAGS_CMD) ECHO > $(BUILD)/release-vars.env 198 docker run -e "GITHUB_TOKEN=$(GITHUB_TOKEN)" --env-file $(BUILD)/release-vars.env -v $(PWD):$(GO_RELEASER_WORKING_DIR) -w $(GO_RELEASER_WORKING_DIR) $(GO_RELEASER_DOCKER_IMAGE) release $(GO_RELEASER_RELEASE_ARGS) 199 200 .PHONY: release-snapshot 201 release-snapshot: check-for-goreleaser-github-token 202 @echo Creating snapshot release 203 make release GO_RELEASER_RELEASE_ARGS="--snapshot --rm-dist" 204 205 # NB(schallert): if updating this target, be sure to update the commands used in 206 # the .buildkite/docs_push.sh. We can't share the make targets because our 207 # Makefile assumes its running under bash and the container is alpine (ash 208 # shell). 209 210 .PHONY: docs-build 211 docs-build: 212 @HUGO_DOCKER=true ./scripts/site-build.sh 213 214 .PHONY: docs-test 215 docs-test: setup install-tools docs-build 216 cp site/.htmltest.yml $(BUILD)/.htmltest.yml 217 ifneq ($(DOCSTEST_AUTH_USER),) 218 ifneq ($(DOCSTEST_AUTH_TOKEN),) 219 @echo 'HTTPHeaders: {"Authorization":"Basic $(shell echo -n "$$DOCSTEST_AUTH_USER:$$DOCSTEST_AUTH_TOKEN" | base64 | xargs echo -n)"}' >> $(BUILD)/.htmltest.yml 220 endif 221 endif 222 $(tools_bin_path)/htmltest -c $(BUILD)/.htmltest.yml 223 224 .PHONY: docker-integration-test 225 docker-integration-test: 226 @echo "--- Running Docker integration test" 227 ./scripts/docker-integration-tests/run.sh 228 229 .PHONY: docker-compatibility-test 230 docker-compatibility-test: 231 @echo "--- Running Prometheus compatibility test" 232 ./scripts/comparator/run.sh 233 234 .PHONY: prom-compat 235 prom-compat: 236 @echo "--- Running local Prometheus compatibility test" 237 CI="false" make docker-compatibility-test 238 239 .PHONY: site-build 240 site-build: 241 @echo "Building site" 242 @./scripts/site-build.sh 243 244 # Generate configs in config/ 245 .PHONY: config-gen 246 config-gen: install-tools 247 @echo "--- Generating configs" 248 $(tools_bin_path)/jsonnet -S $(m3_package_path)/config/m3db/local-etcd/m3dbnode_cmd.jsonnet > $(m3_package_path)/config/m3db/local-etcd/generated.yaml 249 $(tools_bin_path)/jsonnet -S $(m3_package_path)/config/m3db/clustered-etcd/m3dbnode_cmd.jsonnet > $(m3_package_path)/config/m3db/clustered-etcd/generated.yaml 250 251 SUBDIR_TARGETS := \ 252 mock-gen \ 253 thrift-gen \ 254 proto-gen \ 255 asset-gen \ 256 genny-gen \ 257 license-gen \ 258 all-gen \ 259 all-gen 260 261 .PHONY: test-ci-unit 262 test-ci-unit: test-base 263 $(process_coverfile) $(coverfile) 264 265 .PHONY: test-ci-big-unit 266 test-ci-big-unit: test-big-base 267 $(process_coverfile) $(coverfile) 268 269 .PHONY: test-ci-integration 270 test-ci-integration: 271 INTEGRATION_TIMEOUT=10m TEST_SERIES_CACHE_POLICY=$(cache_policy) TEST_AGGREGATOR_CLIENT_TYPE=$(aggregator_client) make test-base-ci-integration 272 $(process_coverfile) $(coverfile) 273 274 .PHONY: test-ci-cluster-integration 275 test-ci-cluster-integration: 276 @echo "--- test-ci-cluster-integration" 277 go test -count=1 -p 1 -tags=cluster_integration ./src/integration/... 278 279 .PHONY: test-ci-test-harness 280 test-ci-test-harness: 281 @echo "--- test-ci-test-harness" 282 go test -count=1 -p 1 -tags=test_harness ./src/integration/resources/... 283 284 define SUBDIR_RULES 285 286 .PHONY: mock-gen-$(SUBDIR) 287 mock-gen-$(SUBDIR): install-tools 288 @echo "--- Generating mocks $(SUBDIR)" 289 @[ ! -d src/$(SUBDIR)/$(mocks_rules_dir) ] || \ 290 PATH=$(combined_bin_paths):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(mocks_output_dir) src/$(SUBDIR)/$(mocks_rules_dir) 291 292 .PHONY: thrift-gen-$(SUBDIR) 293 thrift-gen-$(SUBDIR): install-tools 294 @echo "--- Generating thrift files $(SUBDIR)" 295 @[ ! -d src/$(SUBDIR)/$(thrift_rules_dir) ] || \ 296 PATH=$(combined_bin_paths):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(thrift_output_dir) src/$(SUBDIR)/$(thrift_rules_dir) 297 298 .PHONY: proto-gen-$(SUBDIR) 299 proto-gen-$(SUBDIR): install-tools 300 @echo "--- Generating protobuf files $(SUBDIR)" 301 @[ ! -d src/$(SUBDIR)/$(proto_rules_dir) ] || \ 302 PATH=$(combined_bin_paths):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(proto_output_dir) src/$(SUBDIR)/$(proto_rules_dir) 303 304 .PHONY: asset-gen-$(SUBDIR) 305 asset-gen-$(SUBDIR): install-tools 306 @echo "--- Generating asset files $(SUBDIR)" 307 @[ ! -d src/$(SUBDIR)/$(assets_rules_dir) ] || \ 308 PATH=$(combined_bin_paths):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(assets_output_dir) src/$(SUBDIR)/$(assets_rules_dir) 309 310 .PHONY: genny-gen-$(SUBDIR) 311 genny-gen-$(SUBDIR): install-tools 312 @echo "--- Generating genny files $(SUBDIR)" 313 @[ ! -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk ] || \ 314 PATH=$(combined_bin_paths):$(PATH) make -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk $(genny_target) 315 @PATH=$(combined_bin_paths):$(PATH) bash -c "source ./scripts/auto-gen-helpers.sh && gen_cleanup_dir '*_gen.go' $(SELF_DIR)/src/$(SUBDIR)/ && gen_cleanup_dir '*_gen_test.go' $(SELF_DIR)/src/$(SUBDIR)/" 316 317 .PHONY: license-gen-$(SUBDIR) 318 license-gen-$(SUBDIR): install-tools 319 @echo "--- Updating license in files $(SUBDIR)" 320 @find $(SELF_DIR)/src/$(SUBDIR) -name '*.go' | PATH=$(combined_bin_paths):$(PATH) xargs -I{} update-license {} 321 322 .PHONY: all-gen-$(SUBDIR) 323 # NB(prateek): order matters here, mock-gen needs to be after proto/thrift because we sometimes 324 # generate mocks for thrift/proto generated code. Similarly, license-gen needs to be last because 325 # we make header changes. 326 all-gen-$(SUBDIR): thrift-gen-$(SUBDIR) proto-gen-$(SUBDIR) asset-gen-$(SUBDIR) genny-gen-$(SUBDIR) mock-gen-$(SUBDIR) license-gen-$(SUBDIR) 327 328 .PHONY: test-$(SUBDIR) 329 test-$(SUBDIR): 330 @echo testing $(SUBDIR) 331 SRC_ROOT=./src/$(SUBDIR) make test-base 332 gocov convert $(coverfile) | gocov report 333 334 .PHONY: test-xml-$(SUBDIR) 335 test-xml-$(SUBDIR): 336 @echo test-xml $(SUBDIR) 337 SRC_ROOT=./src/$(SUBDIR) make test-base-xml 338 339 .PHONY: test-html-$(SUBDIR) 340 test-html-$(SUBDIR): 341 @echo test-html $(SUBDIR) 342 SRC_ROOT=./src/$(SUBDIR) make test-base-html 343 344 .PHONY: test-integration-$(SUBDIR) 345 test-integration-$(SUBDIR): 346 @echo test-integration $(SUBDIR) 347 SRC_ROOT=./src/$(SUBDIR) make test-base-integration 348 349 # Usage: make test-single-integration name=<test_name> 350 .PHONY: test-single-integration-$(SUBDIR) 351 test-single-integration-$(SUBDIR): 352 SRC_ROOT=./src/$(SUBDIR) make test-base-single-integration name=$(name) 353 354 .PHONY: test-ci-unit-$(SUBDIR) 355 test-ci-unit-$(SUBDIR): 356 @echo "--- test-ci-unit $(SUBDIR)" 357 SRC_ROOT=./src/$(SUBDIR) make test-base 358 if [ -z "$(SKIP_CODECOV)" ]; then \ 359 @echo "--- uploading coverage report"; \ 360 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 361 fi 362 363 .PHONY: test-ci-big-unit-$(SUBDIR) 364 test-ci-big-unit-$(SUBDIR): 365 @echo "--- test-ci-big-unit $(SUBDIR)" 366 SRC_ROOT=./src/$(SUBDIR) make test-big-base 367 if [ -z "$(SKIP_CODECOV)" ]; then \ 368 @echo "--- uploading coverage report"; \ 369 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 370 fi 371 372 .PHONY: test-ci-integration-$(SUBDIR) 373 test-ci-integration-$(SUBDIR): 374 @echo "--- test-ci-integration $(SUBDIR)" 375 SRC_ROOT=./src/$(SUBDIR) PANIC_ON_INVARIANT_VIOLATED=true INTEGRATION_TIMEOUT=10m TEST_SERIES_CACHE_POLICY=$(cache_policy) TEST_AGGREGATOR_CLIENT_TYPE=$(aggregator_client) make test-base-ci-integration 376 if [ -z "$(SKIP_CODECOV)" ]; then \ 377 @echo "--- uploading coverage report"; \ 378 $(codecov_push) -f $(coverfile) -F $(SUBDIR); \ 379 fi 380 381 .PHONY: lint-$(SUBDIR) 382 lint-$(SUBDIR): export GO_BUILD_TAGS = $(GO_BUILD_TAGS_LIST) 383 lint-$(SUBDIR): install-tools 384 @echo "--- :golang: Running linter on $(SUBDIR)" 385 ./scripts/run-ci-lint.sh $(tools_bin_path)/golangci-lint ./src/$(SUBDIR)/... 386 387 endef 388 389 # generate targets for each SUBDIR in SUBDIRS based on the rules specified above. 390 $(foreach SUBDIR,$(SUBDIRS),$(eval $(SUBDIR_RULES))) 391 392 define SUBDIR_TARGET_RULE 393 .PHONY: $(SUBDIR_TARGET) 394 $(SUBDIR_TARGET): $(foreach SUBDIR,$(SUBDIRS),$(SUBDIR_TARGET)-$(SUBDIR)) 395 endef 396 397 # generate targets across SUBDIRS for each SUBDIR_TARGET. i.e. generate rules 398 # which allow `make all-gen` to invoke `make all-gen-dbnode all-gen-coordinator ...` 399 # NB: we skip lint explicity as it runs as a separate CI step. 400 $(foreach SUBDIR_TARGET, $(SUBDIR_TARGETS), $(eval $(SUBDIR_TARGET_RULE))) 401 402 # Builds the single kube bundle from individual manifest files. 403 .PHONY: kube-gen-all 404 kube-gen-all: install-tools 405 @echo "--- Generating kube bundle" 406 @./kube/scripts/build_bundle.sh 407 find kube -name '*.yaml' -print0 | PATH=$(combined_bin_paths):$(PATH) xargs -0 kubeval -v=1.12.0 408 409 .PHONY: go-mod-tidy 410 go-mod-tidy: 411 @echo "--- :golang: tidying modules" 412 go mod tidy 413 414 .PHONY: all-gen 415 all-gen: \ 416 install-tools \ 417 $(foreach SUBDIR_TARGET, $(filter-out lint all-gen,$(SUBDIR_TARGETS)), $(SUBDIR_TARGET)) \ 418 kube-gen-all \ 419 go-mod-tidy 420 421 .PHONY: build-ui-ctl 422 build-ui-ctl: 423 ifeq ($(shell ls ./src/ctl/ui/build 2>/dev/null),) 424 # Need to use subshell output of set-node-version as cannot 425 # set side-effects of nvm to later commands 426 @echo "Building UI components, if npm install or build fails try: npm cache clean" 427 make node-yarn-run \ 428 node_version="6" \ 429 node_cmd="cd $(m3_package_path)/src/ctl/ui && yarn install && yarn build" 430 # If we've installed nvm locally, remove it due to some cleanup permissions 431 # issue we run into in CI. 432 rm -rf .nvm 433 else 434 @echo "Skip building UI components, already built, to rebuild first make clean" 435 endif 436 # Move public assets into public subdirectory so that it can 437 # be included in the single statik package built from ./ui/build 438 rm -rf ./src/ctl/ui/build/public 439 cp -r ./src/ctl/public ./src/ctl/ui/build/public 440 441 .PHONY: build-ui-ctl-statik-gen 442 build-ui-ctl-statik-gen: build-ui-ctl-statik license-gen-ctl 443 444 .PHONY: build-ui-ctl-statik 445 build-ui-ctl-statik: build-ui-ctl install-tools 446 mkdir -p ./src/ctl/generated/ui 447 $(tools_bin_path)/statik -m -f -src ./src/ctl/ui/build -dest ./src/ctl/generated/ui -p statik 448 449 .PHONY: node-yarn-run 450 node-yarn-run: 451 make node-run \ 452 node_version="$(node_version)" \ 453 node_cmd="(yarn --version 2>&1 >/dev/null || npm install -g yarn@^1.17.0) && $(node_cmd)" 454 455 .PHONY: node-run 456 node-run: 457 ifneq ($(shell command -v nvm 2>/dev/null),) 458 @echo "Using nvm to select node version $(node_version)" 459 nvm use $(node_version) && bash -c "$(node_cmd)" 460 else 461 mkdir .nvm 462 # Install nvm locally 463 NVM_DIR=$(SELF_DIR)/.nvm PROFILE=/dev/null scripts/install_nvm.sh 464 bash -c "source $(SELF_DIR)/.nvm/nvm.sh; nvm install 6" 465 bash -c "source $(SELF_DIR)/.nvm/nvm.sh && nvm use 6 && $(node_cmd)" 466 endif 467 468 # Tests that all currently generated types match their contents if they were regenerated 469 .PHONY: test-all-gen 470 test-all-gen: all-gen 471 @test "$(shell git --no-pager diff --exit-code --shortstat 2>/dev/null)" = "" || (git --no-pager diff --text --exit-code && echo "Check git status, there are dirty files" && exit 1) 472 @test "$(shell git status --exit-code --porcelain 2>/dev/null | grep "^??")" = "" || (git status --exit-code --porcelain && echo "Check git status, there are untracked files" && exit 1) 473 474 # Runs a fossa license report 475 .PHONY: fossa 476 fossa: install-tools 477 PATH=$(combined_bin_paths):$(PATH) fossa analyze --verbose --no-ansi 478 479 # Waits for the result of a fossa test and exits success if pass or fail if fails 480 .PHONY: fossa-test 481 fossa-test: fossa 482 PATH=$(combined_bin_paths):$(PATH) fossa test 483 484 .PHONY: clean-build 485 clean-build: 486 @rm -rf $(BUILD) 487 488 .PHONY: clean 489 clean: clean-build 490 @rm -f *.html *.xml *.out *.test 491 @rm -rf $(VENDOR) 492 @rm -rf ./src/ctl/ui/build 493 494 .DEFAULT_GOAL := all 495 496 lint: install-tools 497 @echo "--- :golang: Running linter on 'src'" 498 ./scripts/run-ci-lint.sh $(tools_bin_path)/golangci-lint ./src/...