github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/Makefile (about) 1 # Local settings (optional). See Makefile.local.example for an example. 2 # WARNING: do not commit to a repository! 3 -include Makefile.local 4 5 .PHONY: all test cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image 6 .DEFAULT_GOAL := all 7 8 # Version number 9 VERSION=$(shell cat "./VERSION" 2> /dev/null) 10 GOPROXY_VALUE=$(shell go env GOPROXY) 11 12 # Boiler plate for building Docker containers. 13 # All this must go at top of file I'm afraid. 14 IMAGE_PREFIX ?= quay.io/cortexproject/ 15 16 # For a tag push GITHUB_REF will look like refs/tags/<tag_name>, 17 # If finding refs/tags/ does not equal emptystring then use 18 # the tag we are at as the image tag. 19 ifneq (,$(findstring refs/tags/, $(GITHUB_REF))) 20 GIT_TAG := $(shell git tag --points-at HEAD) 21 endif 22 IMAGE_TAG ?= $(if $(GIT_TAG),$(GIT_TAG),$(shell ./tools/image-tag)) 23 GIT_REVISION := $(shell git rev-parse --short HEAD) 24 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) 25 UPTODATE := .uptodate 26 27 .PHONY: image-tag 28 image-tag: 29 @echo $(IMAGE_TAG) 30 31 # Support gsed on OSX (installed via brew), falling back to sed. On Linux 32 # systems gsed won't be installed, so will use sed as expected. 33 SED ?= $(shell which gsed 2>/dev/null || which sed) 34 35 # Building Docker images is now automated. The convention is every directory 36 # with a Dockerfile in it builds an image calls quay.io/cortexproject/<dirname>. 37 # Dependencies (i.e. things that go in the image) still need to be explicitly 38 # declared. 39 %/$(UPTODATE): %/Dockerfile 40 @echo 41 $(SUDO) docker build --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) $(@D)/ 42 @echo 43 @echo Please use push-multiarch-build-image to build and push build image for all supported architectures. 44 touch $@ 45 46 # This target fetches current build image, and tags it with "latest" tag. It can be used instead of building the image locally. 47 fetch-build-image: 48 docker pull $(BUILD_IMAGE):$(LATEST_BUILD_IMAGE_TAG) 49 docker tag $(BUILD_IMAGE):$(LATEST_BUILD_IMAGE_TAG) $(BUILD_IMAGE):latest 50 touch build-image/.uptodate 51 52 push-multiarch-build-image: 53 @echo 54 # Build image for each platform separately... it tends to generate fewer errors. 55 $(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/ 56 $(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/ 57 # This command will run the same build as above, but it will reuse existing platform-specific images, 58 # put them together and push to registry. 59 $(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) build-image/ 60 61 # We don't want find to scan inside a bunch of directories, to accelerate the 62 # 'make: Entering directory '/go/src/github.com/cortexproject/cortex' phase. 63 DONT_FIND := -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o -name packaging -prune -o 64 65 # Get a list of directories containing Dockerfiles 66 DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile' -print) 67 UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) 68 DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) 69 IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) 70 images: 71 $(info $(IMAGE_NAMES)) 72 @echo > /dev/null 73 74 # Generating proto code is automated. 75 PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print) 76 PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) 77 78 # Building binaries is now automated. The convention is to build a binary 79 # for every directory with main.go in it, in the ./cmd directory. 80 MAIN_GO := $(shell find . $(DONT_FIND) -type f -name 'main.go' -print) 81 EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)) 82 GO_FILES := $(shell find . $(DONT_FIND) -name cmd -prune -o -name '*.pb.go' -prune -o -type f -name '*.go' -print) 83 define dep_exe 84 $(1): $(dir $(1))/main.go $(GO_FILES) protos 85 $(dir $(1))$(UPTODATE): $(1) 86 endef 87 $(foreach exe, $(EXES), $(eval $(call dep_exe, $(exe)))) 88 89 # Manually declared dependencies And what goes into each exe 90 pkg/cortexpb/cortex.pb.go: pkg/cortexpb/cortex.proto 91 pkg/ingester/client/ingester.pb.go: pkg/ingester/client/ingester.proto 92 pkg/distributor/distributorpb/distributor.pb.go: pkg/distributor/distributorpb/distributor.proto 93 pkg/ingester/wal.pb.go: pkg/ingester/wal.proto 94 pkg/ring/ring.pb.go: pkg/ring/ring.proto 95 pkg/frontend/v1/frontendv1pb/frontend.pb.go: pkg/frontend/v1/frontendv1pb/frontend.proto 96 pkg/frontend/v2/frontendv2pb/frontend.pb.go: pkg/frontend/v2/frontendv2pb/frontend.proto 97 pkg/querier/queryrange/queryrange.pb.go: pkg/querier/queryrange/queryrange.proto 98 pkg/querier/stats/stats.pb.go: pkg/querier/stats/stats.proto 99 pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto 100 pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto 101 pkg/ruler/rulespb/rules.pb.go: pkg/ruler/rulespb/rules.proto 102 pkg/ruler/ruler.pb.go: pkg/ruler/ruler.proto 103 pkg/ring/kv/memberlist/kv.pb.go: pkg/ring/kv/memberlist/kv.proto 104 pkg/scheduler/schedulerpb/scheduler.pb.go: pkg/scheduler/schedulerpb/scheduler.proto 105 pkg/storegateway/storegatewaypb/gateway.pb.go: pkg/storegateway/storegatewaypb/gateway.proto 106 pkg/chunk/grpc/grpc.pb.go: pkg/chunk/grpc/grpc.proto 107 tools/blocksconvert/scheduler.pb.go: tools/blocksconvert/scheduler.proto 108 pkg/alertmanager/alertmanagerpb/alertmanager.pb.go: pkg/alertmanager/alertmanagerpb/alertmanager.proto 109 pkg/alertmanager/alertspb/alerts.pb.go: pkg/alertmanager/alertspb/alerts.proto 110 111 all: $(UPTODATE_FILES) 112 test: protos 113 mod-check: protos 114 configs-integration-test: protos 115 lint: protos 116 build-image/$(UPTODATE): build-image/* 117 118 # All the boiler plate for building golang follows: 119 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E") 120 BUILD_IN_CONTAINER := true 121 BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image 122 LATEST_BUILD_IMAGE_TAG ?= 20210713_update-go-1.16.6-178ab0c4f 123 124 # TTY is parameterized to allow Google Cloud Builder to run builds, 125 # as it currently disallows TTY devices. This value needs to be overridden 126 # in any custom cloudbuild.yaml files 127 TTY := --tty 128 GO_FLAGS := -ldflags "-X main.Branch=$(GIT_BRANCH) -X main.Revision=$(GIT_REVISION) -X main.Version=$(VERSION) -extldflags \"-static\" -s -w" -tags netgo 129 130 ifeq ($(BUILD_IN_CONTAINER),true) 131 132 GOVOLUMES= -v $(shell pwd)/.cache:/go/cache:delegated,z \ 133 -v $(shell pwd)/.pkg:/go/pkg:delegated,z \ 134 -v $(shell pwd):/go/src/github.com/cortexproject/cortex:delegated,z 135 136 exes $(EXES) protos $(PROTO_GOS) lint test cover shell mod-check check-protos web-build web-pre web-deploy doc: build-image/$(UPTODATE) 137 @mkdir -p $(shell pwd)/.pkg 138 @mkdir -p $(shell pwd)/.cache 139 @echo 140 @echo ">>>> Entering build container: $@" 141 @$(SUDO) time docker run --rm $(TTY) -i $(GOVOLUMES) $(BUILD_IMAGE) $@; 142 143 configs-integration-test: build-image/$(UPTODATE) 144 @mkdir -p $(shell pwd)/.pkg 145 @mkdir -p $(shell pwd)/.cache 146 @DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=configs_test' postgres:9.6.16)"; \ 147 echo ; \ 148 echo ">>>> Entering build container: $@"; \ 149 $(SUDO) docker run --rm $(TTY) -i $(GOVOLUMES) \ 150 -v $(shell pwd)/cmd/cortex/migrations:/migrations:z \ 151 --workdir /go/src/github.com/cortexproject/cortex \ 152 --link "$$DB_CONTAINER":configs-db.cortex.local \ 153 -e DB_ADDR=configs-db.cortex.local \ 154 $(BUILD_IMAGE) $@; \ 155 status=$$?; \ 156 docker rm -f "$$DB_CONTAINER"; \ 157 exit $$status 158 159 else 160 161 exes: $(EXES) 162 163 $(EXES): 164 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 165 166 protos: $(PROTO_GOS) 167 168 %.pb.go: 169 @# The store-gateway RPC is based on Thanos which uses relative references to other protos, so we need 170 @# to configure all such relative paths. 171 protoc -I $(GOPATH)/src:./vendor/github.com/thanos-io/thanos/pkg:./vendor/github.com/gogo/protobuf:./vendor:./$(@D) --gogoslick_out=plugins=grpc,Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,:./$(@D) ./$(patsubst %.pb.go,%.proto,$@) 172 173 lint: 174 misspell -error docs 175 176 # Configured via .golangci.yml. 177 golangci-lint run 178 179 # Ensure no blocklisted package is imported. 180 GOFLAGS="-tags=requires_docker" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\ 181 golang.org/x/net/context=context,\ 182 sync/atomic=go.uber.org/atomic,\ 183 github.com/prometheus/client_golang/prometheus.{MultiError}=github.com/prometheus/prometheus/tsdb/errors.{NewMulti},\ 184 github.com/weaveworks/common/user.{ExtractOrgID}=github.com/cortexproject/cortex/pkg/tenant.{TenantID,TenantIDs},\ 185 github.com/weaveworks/common/user.{ExtractOrgIDFromHTTPRequest}=github.com/cortexproject/cortex/pkg/tenant.{ExtractTenantIDFromHTTPRequest}" ./pkg/... ./cmd/... ./tools/... ./integration/... 186 187 # Ensure clean pkg structure. 188 faillint -paths "\ 189 github.com/cortexproject/cortex/pkg/scheduler,\ 190 github.com/cortexproject/cortex/pkg/frontend,\ 191 github.com/cortexproject/cortex/pkg/frontend/transport,\ 192 github.com/cortexproject/cortex/pkg/frontend/v1,\ 193 github.com/cortexproject/cortex/pkg/frontend/v2" \ 194 ./pkg/querier/... 195 faillint -paths "github.com/cortexproject/cortex/pkg/querier/..." ./pkg/scheduler/... 196 faillint -paths "github.com/cortexproject/cortex/pkg/storage/tsdb/..." ./pkg/storage/bucket/... 197 faillint -paths "github.com/cortexproject/cortex/pkg/..." ./pkg/alertmanager/alertspb/... 198 faillint -paths "github.com/cortexproject/cortex/pkg/..." ./pkg/ruler/rulespb/... 199 200 # Ensure the query path is supporting multiple tenants 201 faillint -paths "\ 202 github.com/cortexproject/cortex/pkg/tenant.{TenantID}=github.com/cortexproject/cortex/pkg/tenant.{TenantIDs}" \ 203 ./pkg/scheduler/... \ 204 ./pkg/frontend/... \ 205 ./pkg/querier/tenantfederation/... \ 206 ./pkg/querier/queryrange/... 207 208 # Ensure packages that no longer use a global logger don't reintroduce it 209 faillint -paths "github.com/cortexproject/cortex/pkg/util/log.{Logger}" \ 210 ./pkg/alertmanager/alertstore/... \ 211 ./pkg/ingester/... \ 212 ./pkg/flusher/... \ 213 ./pkg/querier/... \ 214 ./pkg/ruler/... 215 216 test: 217 go test -tags netgo -timeout 30m -race -count 1 ./... 218 219 cover: 220 $(eval COVERDIR := $(shell mktemp -d coverage.XXXXXXXXXX)) 221 $(eval COVERFILE := $(shell mktemp $(COVERDIR)/unit.XXXXXXXXXX)) 222 go test -tags netgo -timeout 30m -race -count 1 -coverprofile=$(COVERFILE) ./... 223 go tool cover -html=$(COVERFILE) -o cover.html 224 go tool cover -func=cover.html | tail -n1 225 226 shell: 227 bash 228 229 configs-integration-test: 230 /bin/bash -c "go test -v -tags 'netgo integration' -timeout 30s ./pkg/configs/... ./pkg/ruler/..." 231 232 mod-check: 233 GO111MODULE=on go mod download 234 GO111MODULE=on go mod verify 235 GO111MODULE=on go mod tidy 236 GO111MODULE=on go mod vendor 237 @git diff --exit-code -- go.sum go.mod vendor/ 238 239 check-protos: clean-protos protos 240 @git diff --exit-code -- $(PROTO_GOS) 241 242 web-pre: 243 cd website && git submodule update --init --recursive 244 ./tools/website/web-pre.sh 245 246 web-build: web-pre 247 cd website && HUGO_ENV=production hugo --config config.toml --minify -v 248 249 web-deploy: 250 ./tools/website/web-deploy.sh 251 252 # Generates the config file documentation. 253 doc: clean-doc 254 go run ./tools/doc-generator ./docs/configuration/config-file-reference.template > ./docs/configuration/config-file-reference.md 255 go run ./tools/doc-generator ./docs/blocks-storage/compactor.template > ./docs/blocks-storage/compactor.md 256 go run ./tools/doc-generator ./docs/blocks-storage/store-gateway.template > ./docs/blocks-storage/store-gateway.md 257 go run ./tools/doc-generator ./docs/blocks-storage/querier.template > ./docs/blocks-storage/querier.md 258 go run ./tools/doc-generator ./docs/guides/encryption-at-rest.template > ./docs/guides/encryption-at-rest.md 259 embedmd -w docs/operations/requests-mirroring-to-secondary-cluster.md 260 embedmd -w docs/guides/overrides-exporter.md 261 262 endif 263 264 clean: 265 $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true 266 rm -rf -- $(UPTODATE_FILES) $(EXES) .cache dist 267 go clean ./... 268 269 clean-protos: 270 rm -rf $(PROTO_GOS) 271 272 save-images: 273 @mkdir -p docker-images 274 for image_name in $(IMAGE_NAMES); do \ 275 if ! echo $$image_name | grep build; then \ 276 docker save $$image_name:$(IMAGE_TAG) -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ 277 fi \ 278 done 279 280 load-images: 281 for image_name in $(IMAGE_NAMES); do \ 282 if ! echo $$image_name | grep build; then \ 283 docker load -i docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ 284 fi \ 285 done 286 287 clean-doc: 288 rm -f \ 289 ./docs/configuration/config-file-reference.md \ 290 ./docs/blocks-storage/compactor.md \ 291 ./docs/blocks-storage/store-gateway.md \ 292 ./docs/blocks-storage/querier.md \ 293 ./docs/guides/encryption-at-rest.md 294 295 check-doc: doc 296 @git diff --exit-code -- ./docs/configuration/config-file-reference.md ./docs/blocks-storage/*.md ./docs/configuration/*.md 297 298 clean-white-noise: 299 @find . -path ./.pkg -prune -o -path ./vendor -prune -o -path ./website -prune -or -type f -name "*.md" -print | \ 300 SED_BIN="$(SED)" xargs ./tools/cleanup-white-noise.sh 301 302 check-white-noise: clean-white-noise 303 @git diff --exit-code --quiet -- '*.md' || (echo "Please remove trailing whitespaces running 'make clean-white-noise'" && false) 304 305 web-serve: 306 cd website && hugo --config config.toml --minify -v server 307 308 # Generate binaries for a Cortex release 309 dist: dist/$(UPTODATE) 310 311 dist/$(UPTODATE): 312 rm -fr ./dist 313 mkdir -p ./dist 314 for os in linux darwin; do \ 315 for arch in amd64 arm64; do \ 316 echo "Building Cortex for $$os/$$arch"; \ 317 GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/cortex-$$os-$$arch ./cmd/cortex; \ 318 sha256sum ./dist/cortex-$$os-$$arch | cut -d ' ' -f 1 > ./dist/cortex-$$os-$$arch-sha-256; \ 319 echo "Building query-tee for $$os/$$arch"; \ 320 GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/query-tee-$$os-$$arch ./cmd/query-tee; \ 321 sha256sum ./dist/query-tee-$$os-$$arch | cut -d ' ' -f 1 > ./dist/query-tee-$$os-$$arch-sha-256; \ 322 done; \ 323 done; \ 324 touch $@ 325 326 # Generate packages for a Cortex release. 327 FPM_OPTS := fpm -s dir -v $(VERSION) -n cortex -f \ 328 --license "Apache 2.0" \ 329 --url "https://github.com/cortexproject/cortex" 330 331 PACKAGE_IN_CONTAINER := true 332 PACKAGE_IMAGE ?= $(IMAGE_PREFIX)fpm 333 ifeq ($(PACKAGE_IN_CONTAINER), true) 334 335 .PHONY: packages 336 packages: dist packaging/fpm/$(UPTODATE) 337 @mkdir -p $(shell pwd)/.pkg 338 @mkdir -p $(shell pwd)/.cache 339 @echo ">>>> Entering build container: $@" 340 @$(SUDO) time docker run --rm $(TTY) \ 341 -v $(shell pwd):/src/github.com/cortexproject/cortex:delegated,z \ 342 -i $(PACKAGE_IMAGE) $@; 343 344 else 345 346 packages: dist/$(UPTODATE)-packages 347 348 dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packaging/rpm/**) 349 for arch in amd64 arm64; do \ 350 rpm_arch=x86_64; \ 351 deb_arch=x86_64; \ 352 if [ "$$arch" = "arm64" ]; then \ 353 rpm_arch=aarch64; \ 354 deb_arch=arm64; \ 355 fi; \ 356 $(FPM_OPTS) -t deb \ 357 --architecture $$deb_arch \ 358 --after-install packaging/deb/control/postinst \ 359 --before-remove packaging/deb/control/prerm \ 360 --package dist/cortex-$(VERSION)_$$arch.deb \ 361 dist/cortex-linux-$$arch=/usr/local/bin/cortex \ 362 docs/chunks-storage/single-process-config.yaml=/etc/cortex/single-process-config.yaml \ 363 packaging/deb/default/cortex=/etc/default/cortex \ 364 packaging/deb/systemd/cortex.service=/etc/systemd/system/cortex.service; \ 365 $(FPM_OPTS) -t rpm \ 366 --architecture $$rpm_arch \ 367 --after-install packaging/rpm/control/post \ 368 --before-remove packaging/rpm/control/preun \ 369 --package dist/cortex-$(VERSION)_$$arch.rpm \ 370 dist/cortex-linux-$$arch=/usr/local/bin/cortex \ 371 docs/chunks-storage/single-process-config.yaml=/etc/cortex/single-process-config.yaml \ 372 packaging/rpm/sysconfig/cortex=/etc/sysconfig/cortex \ 373 packaging/rpm/systemd/cortex.service=/etc/systemd/system/cortex.service; \ 374 done 375 for pkg in dist/*.deb dist/*.rpm; do \ 376 sha256sum $$pkg | cut -d ' ' -f 1 > $${pkg}-sha-256; \ 377 done; \ 378 touch $@ 379 380 endif 381 382 # Build both arm64 and amd64 images, so that we can test deb/rpm packages for both architectures. 383 packaging/rpm/centos-systemd/$(UPTODATE): packaging/rpm/centos-systemd/Dockerfile 384 $(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/ 385 $(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/ 386 touch $@ 387 388 packaging/deb/debian-systemd/$(UPTODATE): packaging/deb/debian-systemd/Dockerfile 389 $(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/ 390 $(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/ 391 touch $@ 392 393 .PHONY: test-packages 394 test-packages: packages packaging/rpm/centos-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE) 395 ./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION)