github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/Makefile (about) 1 .DEFAULT_GOAL := all 2 .PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources format 3 .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push 4 .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test 5 .PHONY: fluentd-image, fluentd-push, fluentd-test 6 .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image 7 .PHONY: bigtable-backup, push-bigtable-backup 8 .PHONY: benchmark-store, drone, check-drone-drift, check-mod 9 .PHONY: migrate migrate-image lint-markdown ragel 10 .PHONY: validate-example-configs generate-example-config-doc check-example-config-doc 11 .PHONY: clean clean-protos 12 13 SHELL = /usr/bin/env bash -o pipefail 14 15 GOTEST ?= go test 16 17 ############# 18 # Variables # 19 ############# 20 21 DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) 22 23 # Certain aspects of the build are done in containers for consistency (e.g. yacc/protobuf generation) 24 # If you have the correct tools installed and you want to speed up development you can run 25 # make BUILD_IN_CONTAINER=false target 26 # or you can override this with an environment variable 27 BUILD_IN_CONTAINER ?= true 28 29 # ensure you run `make drone` after changing this 30 BUILD_IMAGE_VERSION := 0.22.0 31 32 # Docker image info 33 IMAGE_PREFIX ?= grafana 34 35 IMAGE_TAG := $(shell ./tools/image-tag) 36 37 # Version info for binaries 38 GIT_REVISION := $(shell git rev-parse --short HEAD) 39 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) 40 41 # We don't want find to scan inside a bunch of directories, to accelerate the 42 # 'make: Entering directory '/src/loki' phase. 43 DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o 44 45 # Build flags 46 VPREFIX := github.com/grafana/loki/pkg/util/build 47 GO_LDFLAGS := -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") 48 GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w $(GO_LDFLAGS)" -tags netgo 49 DYN_GO_FLAGS := -ldflags "-s -w $(GO_LDFLAGS)" -tags netgo 50 # Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented 51 # the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. 52 # Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. 53 DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo 54 DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo 55 # Docker mount flag, ignored on native docker host. see (https://docs.docker.com/docker-for-mac/osxfs-caching/#delegated) 56 MOUNT_FLAGS := :delegated 57 58 # Protobuf files 59 PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print) 60 PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) 61 62 # Yacc Files 63 YACC_DEFS := $(shell find . $(DONT_FIND) -type f -name *.y -print) 64 YACC_GOS := $(patsubst %.y,%.y.go,$(YACC_DEFS)) 65 66 # Ragel Files 67 RAGEL_DEFS := $(shell find . $(DONT_FIND) -type f -name *.rl -print) 68 RAGEL_GOS := $(patsubst %.rl,%.rl.go,$(RAGEL_DEFS)) 69 70 # Promtail UI files 71 PROMTAIL_GENERATED_FILE := clients/pkg/promtail/server/ui/assets_vfsdata.go 72 PROMTAIL_UI_FILES := $(shell find ./clients/pkg/promtail/server/ui -type f -name assets_vfsdata.go -prune -o -print) 73 74 ########## 75 # Docker # 76 ########## 77 78 # RM is parameterized to allow CircleCI to run builds, as it 79 # currently disallows `docker run --rm`. This value is overridden 80 # in circle.yml 81 RM := --rm 82 # TTY is parameterized to allow Google Cloud Builder to run builds, 83 # as it currently disallows TTY devices. This value needs to be overridden 84 # in any custom cloudbuild.yaml files 85 TTY := --tty 86 87 DOCKER_BUILDKIT=1 88 OCI_PLATFORMS=--platform=linux/amd64 --platform=linux/arm64 --platform=linux/arm/7 89 BUILD_IMAGE = BUILD_IMAGE=$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) 90 ifeq ($(CI), true) 91 BUILD_OCI=img build --no-console $(OCI_PLATFORMS) --build-arg $(BUILD_IMAGE) 92 PUSH_OCI=img push 93 TAG_OCI=img tag 94 else 95 BUILD_OCI=docker build --build-arg $(BUILD_IMAGE) 96 PUSH_OCI=docker push 97 TAG_OCI=docker tag 98 endif 99 100 binfmt: 101 $(SUDO) docker run --privileged linuxkit/binfmt:v0.6 102 103 ################ 104 # Main Targets # 105 ################ 106 all: promtail logcli loki loki-canary 107 108 # This is really a check for the CI to make sure generated files are built and checked in manually 109 check-generated-files: yacc ragel fmt-proto protos clients/pkg/promtail/server/ui/assets_vfsdata.go 110 @if ! (git diff --exit-code $(YACC_GOS) $(RAGEL_GOS) $(PROTO_DEFS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \ 111 echo "\nChanges found in generated files"; \ 112 echo "Run 'make check-generated-files' and commit the changes to fix this error."; \ 113 echo "If you are actively developing these files you can ignore this error"; \ 114 echo "(Don't forget to check in the generated files when finished)\n"; \ 115 exit 1; \ 116 fi 117 118 ########## 119 # Logcli # 120 ########## 121 .PHONY: cmd/logcli/logcli 122 logcli: cmd/logcli/logcli 123 124 logcli-image: 125 $(SUDO) docker build -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile . 126 127 cmd/logcli/logcli: 128 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./cmd/logcli 129 130 ######## 131 # Loki # 132 ######## 133 .PHONY: cmd/loki/loki cmd/loki/loki-debug 134 loki: cmd/loki/loki 135 loki-debug: cmd/loki/loki-debug 136 137 cmd/loki/loki: 138 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 139 140 cmd/loki/loki-debug: 141 CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D) 142 143 ############### 144 # Loki-Canary # 145 ############### 146 .PHONY: cmd/loki-canary/loki-canary 147 loki-canary: cmd/loki-canary/loki-canary 148 149 cmd/loki-canary/loki-canary: 150 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 151 152 ################# 153 # Loki-QueryTee # 154 ################# 155 .PHONY: cmd/querytee/querytee 156 loki-querytee: cmd/querytee/querytee 157 158 cmd/querytee/querytee: 159 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 160 161 ############ 162 # Promtail # 163 ############ 164 165 PROMTAIL_CGO := 0 166 PROMTAIL_GO_FLAGS := $(GO_FLAGS) 167 PROMTAIL_DEBUG_GO_FLAGS := $(DEBUG_GO_FLAGS) 168 169 # Validate GOHOSTOS=linux && GOOS=linux to use CGO. 170 ifeq ($(shell go env GOHOSTOS),linux) 171 ifeq ($(shell go env GOOS),linux) 172 ifneq ($(CGO_ENABLED), 0) 173 PROMTAIL_CGO = 1 174 endif 175 PROMTAIL_GO_FLAGS = $(DYN_GO_FLAGS) 176 PROMTAIL_DEBUG_GO_FLAGS = $(DYN_DEBUG_GO_FLAGS) 177 endif 178 endif 179 .PHONY: clients/cmd/promtail/promtail clients/cmd/promtail/promtail-debug 180 promtail: clients/cmd/promtail/promtail 181 promtail-debug: clients/cmd/promtail/promtail-debug 182 183 promtail-clean-assets: 184 rm -rf clients/pkg/promtail/server/ui/assets_vfsdata.go 185 186 # Rule to generate promtail static assets file 187 $(PROMTAIL_GENERATED_FILE): $(PROMTAIL_UI_FILES) 188 @echo ">> writing assets" 189 GOOS=$(shell go env GOHOSTOS) go generate -x -v ./clients/pkg/promtail/server/ui 190 191 clients/cmd/promtail/promtail: 192 CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_GO_FLAGS) -o $@ ./$(@D) 193 194 clients/cmd/promtail/promtail-debug: 195 CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) -o $@ ./$(@D) 196 197 ######### 198 # Mixin # 199 ######### 200 201 MIXIN_PATH := production/loki-mixin 202 MIXIN_OUT_PATH := production/loki-mixin-compiled 203 MIXIN_OUT_PATH_SSD := production/loki-mixin-compiled-ssd 204 205 loki-mixin: 206 ifeq ($(BUILD_IN_CONTAINER),true) 207 $(SUDO) docker run $(RM) $(TTY) -i \ 208 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 209 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 210 else 211 @rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH) 212 @cd $(MIXIN_PATH) && jb install 213 @mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin.libsonnet 214 215 @rm -rf $(MIXIN_OUT_PATH_SSD) && mkdir $(MIXIN_OUT_PATH_SSD) 216 @cd $(MIXIN_PATH) && jb install 217 @mixtool generate all --output-alerts $(MIXIN_OUT_PATH_SSD)/alerts.yaml --output-rules $(MIXIN_OUT_PATH_SSD)/rules.yaml --directory $(MIXIN_OUT_PATH_SSD)/dashboards ${MIXIN_PATH}/mixin-ssd.libsonnet 218 endif 219 220 loki-mixin-check: loki-mixin 221 @echo "Checking diff" 222 @git diff --exit-code -- $(MIXIN_OUT_PATH) || (echo "Please build mixin by running 'make loki-mixin'" && false) 223 @git diff --exit-code -- $(MIXIN_OUT_PATH_SSD) || (echo "Please build mixin by running 'make loki-mixin'" && false) 224 225 ############### 226 # Migrate # 227 ############### 228 .PHONY: cmd/migrate/migrate 229 migrate: cmd/migrate/migrate 230 231 cmd/migrate/migrate: 232 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 233 234 ############# 235 # Releasing # 236 ############# 237 GOX = gox $(GO_FLAGS) -output="dist/{{.Dir}}-{{.OS}}-{{.Arch}}" 238 CGO_GOX = gox $(DYN_GO_FLAGS) -cgo -output="dist/{{.Dir}}-{{.OS}}-{{.Arch}}" 239 dist: clean 240 CGO_ENABLED=0 $(GOX) -osarch="linux/amd64 linux/arm64 linux/arm darwin/amd64 darwin/arm64 windows/amd64 freebsd/amd64" ./cmd/loki 241 CGO_ENABLED=0 $(GOX) -osarch="linux/amd64 linux/arm64 linux/arm darwin/amd64 darwin/arm64 windows/amd64 freebsd/amd64" ./cmd/logcli 242 CGO_ENABLED=0 $(GOX) -osarch="linux/amd64 linux/arm64 linux/arm darwin/amd64 darwin/arm64 windows/amd64 freebsd/amd64" ./cmd/loki-canary 243 CGO_ENABLED=0 $(GOX) -osarch="linux/arm64 linux/arm darwin/amd64 darwin/arm64 windows/amd64 windows/386 freebsd/amd64" ./clients/cmd/promtail 244 CGO_ENABLED=1 $(CGO_GOX) -osarch="linux/amd64" ./clients/cmd/promtail 245 for i in dist/*; do zip -j -m $$i.zip $$i; done 246 pushd dist && sha256sum * > SHA256SUMS && popd 247 248 packages: dist 249 @tools/packaging/nfpm.sh 250 251 publish: packages 252 ./tools/release 253 254 ######## 255 # Lint # 256 ######## 257 258 # To run this efficiently on your workstation, run this from the root dir: 259 # docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.22.0 lint 260 lint: 261 GO111MODULE=on golangci-lint run -v 262 faillint -paths "sync/atomic=go.uber.org/atomic" ./... 263 264 ######## 265 # Test # 266 ######## 267 268 test: all 269 $(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./... | tee test_results.txt 270 271 compare-coverage: 272 ./tools/diff_coverage.sh $(old) $(new) $(packages) 273 274 ######### 275 # Clean # 276 ######### 277 278 clean-protos: 279 rm -rf $(PROTO_GOS) 280 281 clean: 282 rm -rf clients/cmd/promtail/promtail 283 rm -rf cmd/loki/loki 284 rm -rf cmd/logcli/logcli 285 rm -rf cmd/loki-canary/loki-canary 286 rm -rf cmd/querytee/querytee 287 rm -rf .cache 288 rm -rf clients/cmd/docker-driver/rootfs 289 rm -rf dist/ 290 rm -rf clients/cmd/fluent-bit/out_grafana_loki.h 291 rm -rf clients/cmd/fluent-bit/out_grafana_loki.so 292 rm -rf cmd/migrate/migrate 293 go clean ./... 294 295 ######### 296 # YACCs # 297 ######### 298 299 yacc: $(YACC_GOS) 300 301 %.y.go: %.y 302 ifeq ($(BUILD_IN_CONTAINER),true) 303 # I wish we could make this a multiline variable however you can't pass more than simple arguments to them 304 @mkdir -p $(shell pwd)/.pkg 305 @mkdir -p $(shell pwd)/.cache 306 $(SUDO) docker run $(RM) $(TTY) -i \ 307 -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ 308 -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ 309 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 310 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 311 else 312 goyacc -p $(basename $(notdir $<)) -o $@ $< 313 sed -i.back '/^\/\/line/ d' $@ 314 rm ${@}.back 315 endif 316 317 ######### 318 # Ragels # 319 ######### 320 321 ragel: $(RAGEL_GOS) 322 323 %.rl.go: %.rl 324 ifeq ($(BUILD_IN_CONTAINER),true) 325 @mkdir -p $(shell pwd)/.pkg 326 @mkdir -p $(shell pwd)/.cache 327 $(SUDO) docker run $(RM) $(TTY) -i \ 328 -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ 329 -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ 330 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 331 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 332 else 333 ragel -Z $< -o $@ 334 endif 335 336 ############# 337 # Protobufs # 338 ############# 339 340 protos: clean-protos $(PROTO_GOS) 341 342 %.pb.go: 343 ifeq ($(BUILD_IN_CONTAINER),true) 344 @mkdir -p $(shell pwd)/.pkg 345 @mkdir -p $(shell pwd)/.cache 346 $(SUDO) docker run $(RM) $(TTY) -i \ 347 -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ 348 -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ 349 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 350 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 351 else 352 @# The store-gateway RPC is based on Thanos which uses relative references to other protos, so we need 353 @# to configure all such relative paths. `gogo/protobuf` is used by it. 354 case "$@" in \ 355 vendor*) \ 356 protoc -I ./vendor:./$(@D) --gogoslick_out=plugins=grpc:./vendor ./$(patsubst %.pb.go,%.proto,$@); \ 357 ;; \ 358 *) \ 359 protoc -I .:./vendor/github.com/gogo/protobuf:./vendor/github.com/thanos-io/thanos/pkg:./vendor:./$(@D) --gogoslick_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,plugins=grpc,paths=source_relative:./ ./$(patsubst %.pb.go,%.proto,$@); \ 360 ;; \ 361 esac 362 endif 363 364 365 ################# 366 # Docker Driver # 367 ################# 368 369 # optionally set the tag or the arch suffix (-arm64) 370 LOKI_DOCKER_DRIVER ?= "grafana/loki-docker-driver" 371 PLUGIN_TAG ?= $(IMAGE_TAG) 372 PLUGIN_ARCH ?= 373 374 # build-rootfs 375 # builds the plugin rootfs 376 define build-rootfs 377 rm -rf clients/cmd/docker-driver/rootfs || true 378 mkdir clients/cmd/docker-driver/rootfs 379 docker build -t rootfsimage -f clients/cmd/docker-driver/Dockerfile . 380 381 ID=$$(docker create rootfsimage true) && \ 382 (docker export $$ID | tar -x -C clients/cmd/docker-driver/rootfs) && \ 383 docker rm -vf $$ID 384 385 docker rmi rootfsimage -f 386 endef 387 388 docker-driver: docker-driver-clean 389 $(build-rootfs) 390 docker plugin create $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) clients/cmd/docker-driver 391 392 $(build-rootfs) 393 docker plugin create $(LOKI_DOCKER_DRIVER):main$(PLUGIN_ARCH) clients/cmd/docker-driver 394 395 clients/cmd/docker-driver/docker-driver: 396 CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) 397 398 docker-driver-push: docker-driver 399 docker plugin push $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) 400 docker plugin push $(LOKI_DOCKER_DRIVER):main$(PLUGIN_ARCH) 401 402 docker-driver-enable: 403 docker plugin enable $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) 404 405 docker-driver-clean: 406 -docker plugin disable $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) 407 -docker plugin rm $(LOKI_DOCKER_DRIVER):$(PLUGIN_TAG)$(PLUGIN_ARCH) 408 -docker plugin rm $(LOKI_DOCKER_DRIVER):main$(PLUGIN_ARCH) 409 rm -rf clients/cmd/docker-driver/rootfs 410 411 ##################### 412 # fluent-bit plugin # 413 ##################### 414 fluent-bit-plugin: 415 go build $(DYN_GO_FLAGS) -buildmode=c-shared -o clients/cmd/fluent-bit/out_grafana_loki.so ./clients/cmd/fluent-bit/ 416 417 fluent-bit-image: 418 $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluent-bit/Dockerfile . 419 420 fluent-bit-push: 421 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) 422 423 fluent-bit-test: LOKI_URL ?= http://localhost:3100/loki/api/ 424 fluent-bit-test: 425 docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \ 426 $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) 427 428 429 ################## 430 # fluentd plugin # 431 ################## 432 fluentd-plugin: 433 gem install bundler --version 2.3.4 434 bundle config silence_root_warning true 435 bundle config set --local path clients/cmd/fluentd/vendor/bundle 436 bundle install --gemfile=clients/cmd/fluentd/Gemfile 437 438 fluentd-image: 439 $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile . 440 441 fluentd-push: 442 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) 443 444 fluentd-test: LOKI_URL ?= http://loki:3100 445 fluentd-test: 446 LOKI_URL="$(LOKI_URL)" docker-compose -f clients/cmd/fluentd/docker/docker-compose.yml up --build 447 448 ################## 449 # logstash plugin # 450 ################## 451 logstash-image: 452 $(SUDO) docker build -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f clients/cmd/logstash/Dockerfile ./ 453 454 # Send 10 lines to the local Loki instance. 455 logstash-push-test-logs: LOKI_URL ?= http://host.docker.internal:3100/loki/api/v1/push 456 logstash-push-test-logs: 457 $(SUDO) docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/clients/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \ 458 $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f loki.conf 459 460 logstash-push: 461 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) 462 463 # Enter an env already configure to build and test logstash output plugin. 464 logstash-env: 465 $(SUDO) docker run -v `pwd`/clients/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) 466 467 ######################## 468 # Bigtable Backup Tool # 469 ######################## 470 471 BIGTABLE_BACKUP_TOOL_FOLDER = ./tools/bigtable-backup 472 BIGTABLE_BACKUP_TOOL_TAG ?= $(IMAGE_TAG) 473 474 bigtable-backup: 475 docker build -t $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(BIGTABLE_BACKUP_TOOL_FOLDER) 476 docker tag $(IMAGE_PREFIX)/$(shell basename $(BIGTABLE_BACKUP_TOOL_FOLDER)) $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) 477 478 push-bigtable-backup: bigtable-backup 479 docker push $(IMAGE_PREFIX)/loki-bigtable-backup:$(BIGTABLE_BACKUP_TOOL_TAG) 480 481 ########## 482 # Images # 483 ########## 484 485 images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image fluentd-image 486 487 # push(app, optional tag) 488 # pushes the app, optionally tagging it differently before 489 define push 490 $(SUDO) $(TAG_OCI) $(IMAGE_PREFIX)/$(1):$(IMAGE_TAG) $(IMAGE_PREFIX)/$(1):$(2) 491 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/$(1):$(2) 492 endef 493 494 # push-image(app) 495 # pushes the app, also as :main 496 define push-image 497 $(call push,$(1),$(IMAGE_TAG)) 498 $(call push,$(1),main) 499 endef 500 501 # promtail 502 promtail-image: 503 $(SUDO) docker build -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile . 504 promtail-image-cross: 505 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f clients/cmd/promtail/Dockerfile.cross . 506 507 promtail-debug-image: OCI_PLATFORMS= 508 promtail-debug-image: 509 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG)-debug -f clients/cmd/promtail/Dockerfile.debug . 510 511 promtail-push: promtail-image-cross 512 $(call push-image,promtail) 513 514 # loki 515 loki-image: 516 $(SUDO) docker build -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . 517 loki-image-cross: 518 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross . 519 520 loki-debug-image: OCI_PLATFORMS= 521 loki-debug-image: 522 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)-debug -f cmd/loki/Dockerfile.debug . 523 524 loki-push: loki-image-cross 525 $(call push-image,loki) 526 527 # loki-canary 528 loki-canary-image: 529 $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . 530 loki-canary-image-cross: 531 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . 532 loki-canary-push: loki-canary-image-cross 533 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) 534 535 # loki-querytee 536 loki-querytee-image: 537 $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile . 538 loki-querytee-image-cross: 539 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) -f cmd/querytee/Dockerfile.cross . 540 loki-querytee-push: loki-querytee-image-cross 541 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-query-tee:$(IMAGE_TAG) 542 543 # migrate-image 544 migrate-image: 545 $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile . 546 547 548 # build-image (only amd64) 549 build-image: OCI_PLATFORMS= 550 build-image: 551 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) ./loki-build-image 552 build-image-push: build-image 553 ifneq (,$(findstring WIP,$(IMAGE_TAG))) 554 @echo "Cannot push a WIP image, commit changes first"; \ 555 false; 556 endif 557 $(call push,loki-build-image,$(BUILD_IMAGE_VERSION)) 558 $(call push,loki-build-image,latest) 559 560 # loki-operator 561 loki-operator-image: 562 $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile operator/ 563 loki-operator-image-cross: 564 $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) -f operator/Dockerfile.cross operator/ 565 loki-operator-push: loki-operator-image-cross 566 $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-operator:$(IMAGE_TAG) 567 568 ######## 569 # Misc # 570 ######## 571 572 benchmark-store: 573 go run ./pkg/storage/hack/main.go 574 $(GOTEST) ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out -trace trace.out 575 576 # regenerate drone yaml 577 drone: 578 ifeq ($(BUILD_IN_CONTAINER),true) 579 @mkdir -p $(shell pwd)/.pkg 580 @mkdir -p $(shell pwd)/.cache 581 $(SUDO) docker run $(RM) $(TTY) -i \ 582 -e DRONE_SERVER -e DRONE_TOKEN \ 583 -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ 584 -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ 585 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 586 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 587 else 588 drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml 589 drone lint .drone/drone.yml --trusted 590 drone sign --save grafana/loki .drone/drone.yml || echo "You must set DRONE_SERVER and DRONE_TOKEN. These values can be found on your [drone account](http://drone.grafana.net/account) page." 591 endif 592 593 check-drone-drift: 594 ./tools/check-drone-drift.sh $(BUILD_IMAGE_VERSION) 595 596 597 # support go modules 598 check-mod: 599 ifeq ($(BUILD_IN_CONTAINER),true) 600 $(SUDO) docker run $(RM) $(TTY) -i \ 601 -v $(shell go env GOPATH)/pkg:/go/pkg$(MOUNT_FLAGS) \ 602 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 603 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 604 else 605 GO111MODULE=on GOPROXY=https://proxy.golang.org go mod download 606 GO111MODULE=on GOPROXY=https://proxy.golang.org go mod verify 607 GO111MODULE=on GOPROXY=https://proxy.golang.org go mod tidy 608 GO111MODULE=on GOPROXY=https://proxy.golang.org go mod vendor 609 endif 610 @git diff --exit-code -- go.sum go.mod vendor/ 611 612 613 lint-jsonnet: 614 @RESULT=0; \ 615 for f in $$(find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print); do \ 616 jsonnetfmt -- "$$f" | diff -u "$$f" -; \ 617 RESULT=$$(($$RESULT + $$?)); \ 618 done; \ 619 for d in $$(find . -name '*-mixin' -a -type d -print); do \ 620 if [ -e "$$d/jsonnetfile.json" ]; then \ 621 echo "Installing dependencies for $$d"; \ 622 pushd "$$d" >/dev/null && jb install && popd >/dev/null; \ 623 fi; \ 624 done; \ 625 for m in $$(find . -name 'mixin.libsonnet' -not -path '*/vendor/*' -print); do \ 626 echo "Linting $$m"; \ 627 mixtool lint -J $$(dirname "$$m")/vendor "$$m"; \ 628 if [ $$? -ne 0 ]; then \ 629 RESULT=1; \ 630 fi; \ 631 done; \ 632 exit $$RESULT 633 634 fmt-jsonnet: 635 @find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ 636 xargs -n 1 -- jsonnetfmt -i 637 638 fmt-proto: 639 ifeq ($(BUILD_IN_CONTAINER),true) 640 # I wish we could make this a multiline variable however you can't pass more than simple arguments to them 641 @mkdir -p $(shell pwd)/.pkg 642 @mkdir -p $(shell pwd)/.cache 643 $(SUDO) docker run $(RM) $(TTY) -i \ 644 -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ 645 -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ 646 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 647 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 648 else 649 echo '$(PROTO_DEFS)' | \ 650 xargs -n 1 -- buf format -w 651 endif 652 653 654 lint-scripts: 655 # Ignore https://github.com/koalaman/shellcheck/wiki/SC2312 656 @find . -name '*.sh' -not -path '*/vendor/*' -print0 | \ 657 xargs -0 -n1 shellcheck -e SC2312 -x -o all 658 659 660 # search for dead link in our documentation. 661 # To avoid being rate limited by Github you can use an env variable GITHUB_TOKEN to pass a github token API. 662 # see https://github.com/settings/tokens 663 lint-markdown: 664 ifeq ($(BUILD_IN_CONTAINER),true) 665 $(SUDO) docker run $(RM) $(TTY) -i \ 666 -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ 667 $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; 668 else 669 lychee --verbose --config .lychee.toml ./*.md ./docs/**/*.md ./production/**/*.md ./cmd/**/*.md ./clients/**/*.md ./tools/**/*.md 670 endif 671 672 673 # usage: FUZZ_TESTCASE_PATH=/tmp/testcase make test-fuzz 674 # this will run the fuzzing using /tmp/testcase and save benchmark locally. 675 test-fuzz: 676 $(GOTEST) -timeout 30s -tags dev,gofuzz -cpuprofile cpu.prof -memprofile mem.prof \ 677 -run ^Test_Fuzz$$ github.com/grafana/loki/pkg/logql/syntax -v -count=1 -timeout=0s 678 679 format: 680 find . $(DONT_FIND) -name '*.pb.go' -prune -o -name '*.y.go' -prune -o -name '*.rl.go' -prune -o \ 681 -type f -name '*.go' -exec gofmt -w -s {} \; 682 find . $(DONT_FIND) -name '*.pb.go' -prune -o -name '*.y.go' -prune -o -name '*.rl.go' -prune -o \ 683 -type f -name '*.go' -exec goimports -w -local github.com/grafana/loki {} \; 684 685 ################### 686 # Example Configs # 687 ################### 688 689 # Validate the example configurations that we provide in ./docs/sources/configuration/examples 690 validate-example-configs: loki 691 for f in ./docs/sources/configuration/examples/*.yaml; do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done 692 693 # Dynamically generate ./docs/sources/configuration/examples.md using the example configs that we provide. 694 # This target should be run if any of our example configs change. 695 generate-example-config-doc: 696 echo "Removing existing doc at loki/docs/configuration/examples.md and re-generating. . ." 697 # Title and Heading 698 echo -e "---\ntitle: Examples\n---\n # Loki Configuration Examples" > ./docs/sources/configuration/examples.md 699 # Append each configuration and its file name to examples.md 700 for f in ./docs/sources/configuration/examples/*.yaml; do echo -e "\n## $$(basename $$f)\n\n\`\`\`yaml\n$$(cat $$f)\n\`\`\`\n" >> ./docs/sources/configuration/examples.md; done 701 702 # Fail our CI build if changes are made to example configurations but our doc is not updated 703 check-example-config-doc: generate-example-config-doc 704 @if ! (git diff --exit-code ./docs/sources/configuration/examples.md); then \ 705 echo -e "\nChanges found in generated example configuration doc"; \ 706 echo "Run 'make generate-example-config-doc' and commit the changes to fix this error."; \ 707 echo "If you are actively developing these files you can ignore this error"; \ 708 echo -e "(Don't forget to check in the generated files when finished)\n"; \ 709 exit 1; \ 710 fi