sigs.k8s.io/kueue@v0.6.2/Makefile (about) 1 # Copyright 2022 The Kubernetes Authors. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 16 ifeq (,$(shell go env GOBIN)) 17 GOBIN=$(shell go env GOPATH)/bin 18 else 19 GOBIN=$(shell go env GOBIN) 20 endif 21 22 GO_CMD ?= go 23 GO_FMT ?= gofmt 24 GO_TEST_FLAGS ?= -race 25 # Use go.mod go version as a single source of truth of GO version. 26 GO_VERSION := $(shell awk '/^go /{print $$2}' go.mod|head -n1) 27 28 # Use go.mod go version as a single source of truth of Ginkgo version. 29 GINKGO_VERSION ?= $(shell $(GO_CMD) list -m -f '{{.Version}}' github.com/onsi/ginkgo/v2) 30 31 GIT_TAG ?= $(shell git describe --tags --dirty --always) 32 # Image URL to use all building/pushing image targets 33 PLATFORMS ?= linux/amd64,linux/arm64 34 DOCKER_BUILDX_CMD ?= docker buildx 35 IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build 36 IMAGE_BUILD_EXTRA_OPTS ?= 37 # TODO(#52): Add kueue to k8s gcr registry 38 STAGING_IMAGE_REGISTRY := gcr.io/k8s-staging-kueue 39 IMAGE_REGISTRY ?= $(STAGING_IMAGE_REGISTRY) 40 IMAGE_NAME := kueue 41 IMAGE_REPO ?= $(IMAGE_REGISTRY)/$(IMAGE_NAME) 42 IMAGE_TAG ?= $(IMAGE_REPO):$(GIT_TAG) 43 44 ifdef EXTRA_TAG 45 IMAGE_EXTRA_TAG ?= $(IMAGE_REPO):$(EXTRA_TAG) 46 endif 47 ifdef IMAGE_EXTRA_TAG 48 IMAGE_BUILD_EXTRA_OPTS += -t $(IMAGE_EXTRA_TAG) 49 endif 50 51 ARTIFACTS ?= $(PROJECT_DIR)/bin 52 # Use distroless as minimal base image to package the manager binary 53 # Refer to https://github.com/GoogleContainerTools/distroless for more details 54 BASE_IMAGE ?= gcr.io/distroless/static:nonroot 55 BUILDER_IMAGE ?= golang:$(GO_VERSION) 56 CGO_ENABLED ?= 0 57 58 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. 59 ENVTEST_K8S_VERSION ?= 1.28 60 61 INTEGRATION_TARGET ?= ./test/integration/... 62 63 E2E_TARGET ?= ./test/e2e/... 64 65 E2E_KIND_VERSION ?= kindest/node:v1.28.0 66 67 # E2E_K8S_VERSIONS sets the list of k8s versions included in test-e2e-all 68 E2E_K8S_VERSIONS ?= 1.26.12 1.27.9 1.28.5 1.29.0 69 70 # For local testing, we should allow user to use different kind cluster name 71 # Default will delete default kind cluster 72 KIND_CLUSTER_NAME ?= kind 73 74 # Number of processes to use during integration tests to run specs within a 75 # suite in parallel. Suites still run sequentially. User may set this value to 1 76 # to run without parallelism. 77 INTEGRATION_NPROCS ?= 4 78 79 # Setting SHELL to bash allows bash commands to be executed by recipes. 80 # This is a requirement for 'setup-envtest.sh' in the test target. 81 # Options are set to exit when a recipe line exits non-zero or a piped command fails. 82 SHELL = /usr/bin/env bash -o pipefail 83 .SHELLFLAGS = -ec 84 85 version_pkg = sigs.k8s.io/kueue/pkg/version 86 LD_FLAGS += -X '$(version_pkg).GitVersion=$(GIT_TAG)' 87 LD_FLAGS += -X '$(version_pkg).GitCommit=$(shell git rev-parse HEAD)' 88 89 # Update these variables when preparing a new release or a release branch. 90 # Then run `make prepare-release-branch` 91 RELEASE_VERSION=v0.6.2 92 RELEASE_BRANCH=release-0.6 93 94 .PHONY: all 95 all: generate fmt vet build 96 97 ##@ General 98 99 # The help target prints out all targets with their descriptions organized 100 # beneath their categories. The categories are represented by '##@' and the 101 # target descriptions by '##'. The awk commands is responsible for reading the 102 # entire set of makefiles included in this invocation, looking for lines of the 103 # file as xyz: ## something, and then pretty-format the target and help. Then, 104 # if there's a line with ##@ something, that gets pretty-printed as a category. 105 # More info on the usage of ANSI control characters for terminal formatting: 106 # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 107 # More info on the awk command: 108 # http://linuxcommand.org/lc3_adv_awk.php 109 110 .PHONY: help 111 help: ## Display this help. 112 @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 113 114 ##@ Development 115 116 .PHONY: manifests 117 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. 118 $(CONTROLLER_GEN) \ 119 crd:generateEmbeddedObjectMeta=true output:crd:artifacts:config=config/components/crd/bases\ 120 paths="./apis/..." 121 $(CONTROLLER_GEN) \ 122 rbac:roleName=manager-role output:rbac:artifacts:config=config/components/rbac\ 123 webhook output:webhook:artifacts:config=config/components/webhook\ 124 paths="./pkg/controller/...;./pkg/webhooks/...;./pkg/util/cert/...;./pkg/visibility/..." 125 126 .PHONY: update-helm 127 update-helm: manifests yq 128 ./hack/update-helm.sh 129 130 .PHONY: generate 131 generate: gomod-download controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations and client-go libraries. 132 $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./apis/..." 133 ./hack/update-codegen.sh $(GO_CMD) 134 135 .PHONY: fmt 136 fmt: ## Run go fmt against code. 137 $(GO_CMD) fmt ./... 138 139 .PHONY: fmt-verify 140 fmt-verify: 141 @out=`$(GO_FMT) -w -l -d $$(find . -name '*.go')`; \ 142 if [ -n "$$out" ]; then \ 143 echo "$$out"; \ 144 exit 1; \ 145 fi 146 147 .PHONY: gomod-verify 148 gomod-verify: 149 $(GO_CMD) mod tidy 150 git --no-pager diff --exit-code go.mod go.sum 151 152 .PHONY: gomod-download 153 gomod-download: 154 $(GO_CMD) mod download 155 156 .PHONY: toc-update 157 toc-update: 158 ./hack/update-toc.sh 159 160 .PHONY: toc-verify 161 toc-verify: 162 ./hack/verify-toc.sh 163 164 .PHONY: vet 165 vet: ## Run go vet against code. 166 $(GO_CMD) vet ./... 167 168 .PHONY: test 169 test: gotestsum ## Run tests. 170 $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out 171 172 .PHONY: test-integration 173 test-integration: gomod-download envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd cluster-autoscaler-crd ## Run tests. 174 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ 175 $(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET) 176 177 CREATE_KIND_CLUSTER ?= true 178 .PHONY: test-e2e 179 test-e2e: kustomize ginkgo yq gomod-download jobset-operator-crd run-test-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%) run-test-multikueue-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%) 180 181 182 E2E_TARGETS := $(addprefix run-test-e2e-,${E2E_K8S_VERSIONS}) 183 MULTIKUEUE-E2E_TARGETS := $(addprefix run-test-multikueue-e2e-,${E2E_K8S_VERSIONS}) 184 .PHONY: test-e2e-all 185 test-e2e-all: ginkgo $(E2E_TARGETS) $(MULTIKUEUE-E2E_TARGETS) 186 187 FORCE: 188 189 run-test-e2e-%: K8S_VERSION = $(@:run-test-e2e-%=%) 190 run-test-e2e-%: FORCE 191 @echo Running e2e for k8s ${K8S_VERSION} 192 E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" ./hack/e2e-test.sh 193 194 JOBSET_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" sigs.k8s.io/jobset) 195 run-test-multikueue-e2e-%: K8S_VERSION = $(@:run-test-multikueue-e2e-%=%) 196 run-test-multikueue-e2e-%: FORCE 197 @echo Running multikueue e2e for k8s ${K8S_VERSION} 198 E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) ./hack/multikueue-e2e-test.sh 199 200 .PHONY: ci-lint 201 ci-lint: golangci-lint 202 $(GOLANGCI_LINT) run --timeout 15m0s 203 204 .PHONY: verify 205 verify: gomod-verify vet ci-lint fmt-verify toc-verify manifests generate update-helm generate-apiref prepare-release-branch 206 git --no-pager diff --exit-code config/components apis charts/kueue/templates client-go site/ 207 208 ##@ Build 209 210 .PHONY: build 211 build: 212 $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/manager cmd/kueue/main.go 213 214 .PHONY: run 215 run: manifests generate fmt vet ## Run a controller from your host. 216 $(GO_CMD) run ./main.go 217 218 # Build the multiplatform container image locally. 219 .PHONY: image-local-build 220 image-local-build: 221 BUILDER=$(shell $(DOCKER_BUILDX_CMD) create --use) 222 $(MAKE) image-build PUSH=$(PUSH) 223 $(DOCKER_BUILDX_CMD) rm $$BUILDER 224 225 # Build the multiplatform container image locally and push to repo. 226 .PHONY: image-local-push 227 image-local-push: PUSH=--push 228 image-local-push: image-local-build 229 230 .PHONY: image-build 231 image-build: 232 $(IMAGE_BUILD_CMD) -t $(IMAGE_TAG) \ 233 --platform=$(PLATFORMS) \ 234 --build-arg BASE_IMAGE=$(BASE_IMAGE) \ 235 --build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \ 236 --build-arg CGO_ENABLED=$(CGO_ENABLED) \ 237 $(PUSH) \ 238 $(IMAGE_BUILD_EXTRA_OPTS) ./ 239 240 .PHONY: image-push 241 image-push: PUSH=--push 242 image-push: image-build 243 244 # Build an amd64 image that can be used for Kind E2E tests. 245 .PHONY: kind-image-build 246 kind-image-build: PLATFORMS=linux/amd64 247 kind-image-build: IMAGE_BUILD_EXTRA_OPTS=--load 248 kind-image-build: kind image-build 249 250 ##@ Deployment 251 252 ifndef ignore-not-found 253 ignore-not-found = false 254 endif 255 256 clean-manifests = (cd config/components/manager && $(KUSTOMIZE) edit set image controller=gcr.io/k8s-staging-kueue/kueue:$(RELEASE_BRANCH)) 257 258 .PHONY: install 259 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. 260 $(KUSTOMIZE) build config/components/crd | kubectl apply --server-side -f - 261 262 .PHONY: uninstall 263 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. 264 $(KUSTOMIZE) build config/components/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - 265 266 .PHONY: deploy 267 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 268 cd config/components/manager && $(KUSTOMIZE) edit set image controller=${IMAGE_TAG} 269 kubectl apply --server-side -k config/default 270 @$(call clean-manifests) 271 272 .PHONY: prometheus 273 prometheus: 274 kubectl apply --server-side -k config/prometheus 275 276 .PHONY: undeploy 277 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. 278 $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - 279 280 ##@ Release 281 .PHONY: artifacts 282 artifacts: kustomize yq helm 283 cd config/components/manager && $(KUSTOMIZE) edit set image controller=${IMAGE_TAG} 284 if [ -d artifacts ]; then rm -rf artifacts; fi 285 mkdir -p artifacts 286 $(KUSTOMIZE) build config/default -o artifacts/manifests.yaml 287 $(KUSTOMIZE) build config/dev -o artifacts/manifests-dev.yaml 288 $(KUSTOMIZE) build config/alpha-enabled -o artifacts/manifests-alpha-enabled.yaml 289 $(KUSTOMIZE) build config/prometheus -o artifacts/prometheus.yaml 290 $(KUSTOMIZE) build config/visibility -o artifacts/visibility-api.yaml 291 @$(call clean-manifests) 292 # Update the image tag and policy 293 $(YQ) e '.controllerManager.manager.image.repository = "$(IMAGE_REPO)" | .controllerManager.manager.image.tag = "$(GIT_TAG)" | .controllerManager.manager.image.pullPolicy = "IfNotPresent"' -i charts/kueue/values.yaml 294 # create the package. TODO: consider signing it 295 $(HELM) package --version $(GIT_TAG) --app-version $(GIT_TAG) charts/kueue -d artifacts/ 296 mv artifacts/kueue-$(GIT_TAG).tgz artifacts/kueue-chart-$(GIT_TAG).tgz 297 # Revert the image changes 298 $(YQ) e '.controllerManager.manager.image.repository = "$(STAGING_IMAGE_REGISTRY)/$(IMAGE_NAME)" | .controllerManager.manager.image.tag = "main" | .controllerManager.manager.image.pullPolicy = "Always"' -i charts/kueue/values.yaml 299 300 .PHONY: prepare-release-branch 301 prepare-release-branch: yq kustomize 302 sed -r 's/v[0-9]+\.[0-9]+\.[0-9]+/$(RELEASE_VERSION)/g' -i README.md -i site/config.toml 303 $(YQ) e '.appVersion = "$(RELEASE_VERSION)"' -i charts/kueue/Chart.yaml 304 @$(call clean-manifests) 305 306 ##@ Tools 307 308 # Build an image that can be used with kubectl debug 309 # Developers don't need to build this image, as it will be available as gcr.io/k8s-staging-kueue/debug 310 .PHONY: debug-image-push 311 debug-image-push: 312 $(IMAGE_BUILD_CMD) -t $(STAGING_IMAGE_REGISTRY)/debug:$(GIT_TAG) \ 313 --platform=$(PLATFORMS) \ 314 --push ./hack/debugpod 315 316 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 317 GOLANGCI_LINT = $(PROJECT_DIR)/bin/golangci-lint 318 .PHONY: golangci-lint 319 golangci-lint: ## Download golangci-lint locally if necessary. 320 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 321 322 CONTROLLER_GEN = $(PROJECT_DIR)/bin/controller-gen 323 .PHONY: controller-gen 324 controller-gen: ## Download controller-gen locally if necessary. 325 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/controller-tools/cmd/controller-gen 326 327 KUSTOMIZE = $(PROJECT_DIR)/bin/kustomize 328 .PHONY: kustomize 329 kustomize: ## Download kustomize locally if necessary. 330 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/kustomize/kustomize/v4@v4.5.7 331 332 ENVTEST = $(PROJECT_DIR)/bin/setup-envtest 333 .PHONY: envtest 334 envtest: ## Download envtest-setup locally if necessary. 335 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240320141353-395cfc7486e6 336 337 GINKGO = $(PROJECT_DIR)/bin/ginkgo 338 .PHONY: ginkgo 339 ginkgo: ## Download ginkgo locally if necessary. 340 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) 341 342 GOTESTSUM = $(PROJECT_DIR)/bin/gotestsum 343 .PHONY: gotestsum 344 gotestsum: ## Download gotestsum locally if necessary. 345 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install gotest.tools/gotestsum@v1.8.2 346 347 KIND = $(PROJECT_DIR)/bin/kind 348 .PHONY: kind 349 kind: 350 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/kind@v0.20.0 351 352 YQ = $(PROJECT_DIR)/bin/yq 353 .PHONY: yq 354 yq: ## Download yq locally if necessary. 355 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install github.com/mikefarah/yq/v4@v4.34.1 356 357 HELM = $(PROJECT_DIR)/bin/helm 358 .PHONY: helm 359 helm: ## Download helm locally if necessary. 360 @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install helm.sh/helm/v3/cmd/helm@v3.12.1 361 362 GENREF = $(PROJECT_DIR)/bin/genref 363 .PHONY: genref 364 genref: ## Download genref locally if necessary. 365 @GOBIN=$(PROJECT_DIR)/bin $(GO_CMD) install github.com/kubernetes-sigs/reference-docs/genref@v0.28.0 366 367 MPIROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" github.com/kubeflow/mpi-operator) 368 .PHONY: mpi-operator-crd 369 mpi-operator-crd: 370 mkdir -p $(PROJECT_DIR)/dep-crds/mpi-operator/ 371 cp -f $(MPIROOT)/manifests/base/* $(PROJECT_DIR)/dep-crds/mpi-operator/ 372 373 KFTRAININGROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" github.com/kubeflow/training-operator) 374 .PHONY: kf-training-operator-crd 375 kf-training-operator-crd: 376 mkdir -p $(PROJECT_DIR)/dep-crds/training-operator/ 377 cp -f $(KFTRAININGROOT)/manifests/base/crds/* $(PROJECT_DIR)/dep-crds/training-operator/ 378 379 RAYROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator) 380 .PHONY: ray-operator-crd 381 ray-operator-crd: 382 mkdir -p $(PROJECT_DIR)/dep-crds/ray-operator/ 383 cp -f $(RAYROOT)/config/crd/bases/* $(PROJECT_DIR)/dep-crds/ray-operator/ 384 385 JOBSETROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" sigs.k8s.io/jobset) 386 .PHONY: jobset-operator-crd 387 jobset-operator-crd: 388 mkdir -p $(PROJECT_DIR)/dep-crds/jobset-operator/ 389 cp -f $(JOBSETROOT)/config/components/crd/bases/* $(PROJECT_DIR)/dep-crds/jobset-operator/ 390 391 392 CAROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" k8s.io/autoscaler/cluster-autoscaler/apis) 393 .PHONY: cluster-autoscaler-crd 394 cluster-autoscaler-crd: 395 mkdir -p $(PROJECT_DIR)/dep-crds/cluster-autoscaler/ 396 cp -f $(CAROOT)/config/crd/* $(PROJECT_DIR)/dep-crds/cluster-autoscaler/ 397 398 .PHONY: generate-apiref 399 generate-apiref: genref 400 cd $(PROJECT_DIR)/site/genref/ && $(GENREF) -o $(PROJECT_DIR)/site/content/en/docs/reference