github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/Makefile (about) 1 GOCMD=$(or $(shell which go), $(error "Missing dependency - no go in PATH")) 2 DOCKER=$(or $(shell which docker), $(error "Missing dependency - no docker in PATH")) 3 GOBINPATH=$(shell $(GOCMD) env GOPATH)/bin 4 NPM=$(or $(shell which npm), $(error "Missing dependency - no npm in PATH")) 5 6 UID_GID := $(shell id -u):$(shell id -g) 7 8 CLIENT_JARS_BUCKET="s3://treeverse-clients-us-east/" 9 10 # https://openapi-generator.tech 11 OPENAPI_LEGACY_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v5.3.0 12 OPENAPI_LEGACY_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_LEGACY_GENERATOR_IMAGE) 13 OPENAPI_GENERATOR_IMAGE=treeverse/openapi-generator-cli:v7.0.0.1 14 OPENAPI_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE) 15 OPENAPI_RUST_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v7.5.0 16 OPENAPI_RUST_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_RUST_GENERATOR_IMAGE) 17 PY_OPENAPI_GENERATOR=$(DOCKER) run -e PYTHON_POST_PROCESS_FILE="/mnt/clients/python/scripts/pydantic.sh" --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE) 18 19 GOLANGCI_LINT_VERSION=v1.58.1 20 BUF_CLI_VERSION=v1.28.1 21 22 ifndef PACKAGE_VERSION 23 PACKAGE_VERSION=0.1.0-SNAPSHOT 24 endif 25 26 PYTHON_IMAGE=python:3 27 28 export PATH:= $(PATH):$(GOBINPATH) 29 30 GOBUILD=$(GOCMD) build 31 GORUN=$(GOCMD) run 32 GOCLEAN=$(GOCMD) clean 33 GOTOOL=$(GOCMD) tool 34 GOGENERATE=$(GOCMD) generate 35 GOTEST=$(GOCMD) test 36 GOTESTRACE=$(GOTEST) -race 37 GOGET=$(GOCMD) get 38 GOFMT=$(GOCMD)fmt 39 40 GOTEST_PARALLELISM=4 41 42 LAKEFS_BINARY_NAME=lakefs 43 LAKECTL_BINARY_NAME=lakectl 44 45 UI_DIR=webui 46 UI_BUILD_DIR=$(UI_DIR)/dist 47 48 DOCKER_IMAGE=lakefs 49 DOCKER_TAG=dev 50 VERSION=dev 51 export VERSION 52 53 # This cannot detect whether untracked files have yet to be added. 54 # That is sort-of a git feature, but can be a limitation here. 55 DIRTY=$(shell git diff-index --quiet HEAD -- || echo '.with.local.changes') 56 GIT_REF=$(shell git rev-parse --short HEAD --) 57 REVISION=$(GIT_REF)$(DIRTY) 58 export REVISION 59 60 .PHONY: all clean esti lint test gen help 61 all: build 62 63 clean: 64 @rm -rf \ 65 $(LAKECTL_BINARY_NAME) \ 66 $(LAKEFS_BINARY_NAME) \ 67 $(UI_BUILD_DIR) \ 68 $(UI_DIR)/node_modules \ 69 pkg/api/apigen/lakefs.gen.go \ 70 pkg/auth/client.gen.go 71 72 check-licenses: check-licenses-go-mod check-licenses-npm 73 74 check-licenses-go-mod: 75 $(GOCMD) install github.com/google/go-licenses@latest 76 $(GOBINPATH)/go-licenses check ./cmd/$(LAKEFS_BINARY_NAME) 77 $(GOBINPATH)/go-licenses check ./cmd/$(LAKECTL_BINARY_NAME) 78 79 check-licenses-npm: 80 $(GOCMD) install github.com/senseyeio/diligent/cmd/diligent@latest 81 # The -i arg is a workaround to ignore NPM scoped packages until https://github.com/senseyeio/diligent/issues/77 is fixed 82 $(GOBINPATH)/diligent check -w permissive -i ^@[^/]+?/[^/]+ $(UI_DIR) 83 84 docs/assets/js/swagger.yml: api/swagger.yml 85 @cp api/swagger.yml docs/assets/js/swagger.yml 86 87 docs: docs/assets/js/swagger.yml 88 89 docs-serve: ### Serve local docs 90 cd docs; bundle exec jekyll serve --livereload 91 92 docs-serve-docker: ### Serve local docs from Docker 93 docker run --rm \ 94 --name lakefs_docs \ 95 -e TZ="Etc/UTC" \ 96 --publish 4000:4000 --publish 35729:35729 \ 97 --volume="$$PWD/docs:/srv/jekyll:Z" \ 98 --volume="$$PWD/docs/.jekyll-bundle-cache:/usr/local/bundle:Z" \ 99 --interactive --tty \ 100 jekyll/jekyll:4.2.2 \ 101 jekyll serve --livereload 102 103 gen-docs: ## Generate CLI docs automatically 104 $(GOCMD) run cmd/lakectl/main.go docs > docs/reference/cli.md 105 106 gen-metastore: ## Run Metastore Code generation 107 @thrift -r --gen go --gen go:package_prefix=github.com/treeverse/lakefs/pkg/metastore/hive/gen-go/ -o pkg/metastore/hive pkg/metastore/hive/hive_metastore.thrift 108 109 tools: ## Install tools 110 $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) 111 $(GOCMD) install github.com/bufbuild/buf/cmd/buf@$(BUF_CLI_VERSION) 112 113 client-python: sdk-python-legacy sdk-python 114 115 sdk-python-legacy: api/swagger.yml ## Generate SDK for Python client - openapi generator version 5.3.0 116 # remove the build folder as it also holds lakefs_client folder which keeps because we skip it during find 117 rm -rf clients/python-legacy/build; cd clients/python-legacy && \ 118 find . -depth -name lakefs_client -prune -o ! \( -name Gemfile -or -name Gemfile.lock -or -name _config.yml -or -name .openapi-generator-ignore -or -name templates -or -name setup.mustache -or -name client.mustache -or -name python-codegen-config.yaml \) -delete 119 $(OPENAPI_LEGACY_GENERATOR) generate \ 120 -i /mnt/$< \ 121 -g python \ 122 -t /mnt/clients/python-legacy/templates \ 123 --package-name lakefs_client \ 124 --http-user-agent "lakefs-python-sdk/$(PACKAGE_VERSION)-legacy" \ 125 --git-user-id treeverse --git-repo-id lakeFS \ 126 --additional-properties=infoName=Treeverse,infoEmail=services@treeverse.io,packageName=lakefs_client,packageVersion=$(PACKAGE_VERSION),projectName=lakefs-client,packageUrl=https://github.com/treeverse/lakeFS/tree/master/clients/python-legacy \ 127 -c /mnt/clients/python-legacy/python-codegen-config.yaml \ 128 -o /mnt/clients/python-legacy \ 129 --ignore-file-override /mnt/clients/python/.openapi-generator-ignore 130 131 sdk-python: api/swagger.yml ## Generate SDK for Python client - openapi generator version 7.0.0 132 # remove the build folder as it also holds lakefs_sdk folder which keeps because we skip it during find 133 rm -rf clients/python/build; cd clients/python && \ 134 find . -depth -name lakefs_sdk -prune -o ! \( -name Gemfile -or -name Gemfile.lock -or -name _config.yml -or -name .openapi-generator-ignore -or -name templates -or -name setup.mustache -or -name client.mustache -or -name requirements.mustache -or -name scripts -or -name pydantic.sh -or -name python-codegen-config.yaml \) -delete 135 $(PY_OPENAPI_GENERATOR) generate \ 136 --enable-post-process-file \ 137 -i /mnt/$< \ 138 -g python \ 139 -t /mnt/clients/python/templates \ 140 --package-name lakefs_sdk \ 141 --http-user-agent "lakefs-python-sdk/$(PACKAGE_VERSION)" \ 142 --git-user-id treeverse --git-repo-id lakeFS \ 143 --additional-properties=infoName=Treeverse,infoEmail=services@treeverse.io,packageVersion=$(PACKAGE_VERSION),projectName=lakefs-sdk,packageUrl=https://github.com/treeverse/lakeFS/tree/master/clients/python \ 144 -c /mnt/clients/python/python-codegen-config.yaml \ 145 -o /mnt/clients/python \ 146 --ignore-file-override /mnt/clients/python/.openapi-generator-ignore 147 148 sdk-rust: api/swagger.yml ## Generate SDK for Rust client - openapi generator version 7.1.0 149 rm -rf clients/rust 150 mkdir -p clients/rust 151 $(OPENAPI_RUST_GENERATOR) generate \ 152 -i /mnt/api/swagger.yml \ 153 -g rust \ 154 --additional-properties=infoName=Treeverse,infoEmail=services@treeverse.io,packageName=lakefs_sdk,packageVersion=$(PACKAGE_VERSION),packageUrl=https://github.com/treeverse/lakeFS/tree/master/clients/rust \ 155 -o /mnt/clients/rust 156 157 client-java-legacy: api/swagger.yml api/java-gen-ignore ## Generate legacy SDK for Java (and Scala) client 158 rm -rf clients/java-legacy 159 $(OPENAPI_LEGACY_GENERATOR) generate \ 160 -i /mnt/$< \ 161 --ignore-file-override /mnt/api/java-gen-ignore \ 162 -g java \ 163 --invoker-package io.lakefs.clients.api \ 164 --http-user-agent "lakefs-java-sdk/$(PACKAGE_VERSION)-legacy" \ 165 --additional-properties hideGenerationTimestamp=true,artifactVersion=$(PACKAGE_VERSION),parentArtifactId=lakefs-parent,parentGroupId=io.lakefs,parentVersion=0,groupId=io.lakefs,artifactId='api-client',artifactDescription='lakeFS OpenAPI Java client legacy SDK',artifactUrl=https://lakefs.io,apiPackage=io.lakefs.clients.api,modelPackage=io.lakefs.clients.api.model,mainPackage=io.lakefs.clients.api,developerEmail=services@treeverse.io,developerName='Treeverse lakeFS dev',developerOrganization='lakefs.io',developerOrganizationUrl='https://lakefs.io',licenseName=apache2,licenseUrl=http://www.apache.org/licenses/,scmConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmDeveloperConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmUrl=https://github.com/treeverse/lakeFS \ 166 -o /mnt/clients/java-legacy 167 168 client-java: api/swagger.yml api/java-gen-ignore ## Generate SDK for Java (and Scala) client 169 rm -rf clients/java 170 mkdir -p clients/java 171 cp api/java-gen-ignore clients/java/.openapi-generator-ignore 172 $(OPENAPI_GENERATOR) generate \ 173 -i /mnt/api/swagger.yml \ 174 -g java \ 175 --invoker-package io.lakefs.clients.sdk \ 176 --http-user-agent "lakefs-java-sdk/$(PACKAGE_VERSION)-v1" \ 177 --additional-properties disallowAdditionalPropertiesIfNotPresent=false,useSingleRequestParameter=true,hideGenerationTimestamp=true,artifactVersion=$(PACKAGE_VERSION),parentArtifactId=lakefs-parent,parentGroupId=io.lakefs,parentVersion=0,groupId=io.lakefs,artifactId='sdk',artifactDescription='lakeFS OpenAPI Java client',artifactUrl=https://lakefs.io,apiPackage=io.lakefs.clients.sdk,modelPackage=io.lakefs.clients.sdk.model,mainPackage=io.lakefs.clients.sdk,developerEmail=services@treeverse.io,developerName='Treeverse lakeFS dev',developerOrganization='lakefs.io',developerOrganizationUrl='https://lakefs.io',licenseName=apache2,licenseUrl=http://www.apache.org/licenses/,scmConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmDeveloperConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmUrl=https://github.com/treeverse/lakeFS \ 178 -o /mnt/clients/java 179 180 .PHONY: clients client-python sdk-python-legacy sdk-python client-java client-java-legacy 181 clients: client-python client-java client-java-legacy sdk-rust 182 183 package-python: package-python-client package-python-sdk 184 185 package-python-client: client-python 186 $(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python-legacy $(PYTHON_IMAGE) /bin/bash -c \ 187 "python -m pip install build --user && python -m build --sdist --wheel --outdir dist/" 188 189 package-python-sdk: sdk-python 190 $(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python $(PYTHON_IMAGE) /bin/bash -c \ 191 "python -m pip install build --user && python -m build --sdist --wheel --outdir dist/" 192 193 package-python-wrapper: 194 $(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python-wrapper $(PYTHON_IMAGE) /bin/bash -c \ 195 "python -m pip install build --user && python -m build --sdist --wheel --outdir dist/" 196 197 package: package-python 198 199 .PHONY: gen-api 200 gen-api: docs/assets/js/swagger.yml ## Run the swagger code generator 201 $(GOGENERATE) ./pkg/api/apigen ./pkg/auth ./pkg/authentication 202 203 .PHONY: gen-code 204 gen-code: gen-api ## Run the generator for inline commands 205 $(GOGENERATE) \ 206 ./pkg/actions \ 207 ./pkg/auth/ \ 208 ./pkg/authentication \ 209 ./pkg/graveler \ 210 ./pkg/graveler/committed \ 211 ./pkg/graveler/sstable \ 212 ./pkg/kv \ 213 ./pkg/permissions \ 214 ./pkg/pyramid 215 216 LD_FLAGS := "-X github.com/treeverse/lakefs/pkg/version.Version=$(VERSION)-$(REVISION)" 217 build: gen docs ## Download dependencies and build the default binary 218 $(GOBUILD) -o $(LAKEFS_BINARY_NAME) -ldflags $(LD_FLAGS) -v ./cmd/$(LAKEFS_BINARY_NAME) 219 $(GOBUILD) -o $(LAKECTL_BINARY_NAME) -ldflags $(LD_FLAGS) -v ./cmd/$(LAKECTL_BINARY_NAME) 220 221 lint: ## Lint code 222 $(GOCMD) run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run $(GOLANGCI_LINT_FLAGS) 223 npx eslint@8.57.0 $(UI_DIR)/src --ext .js,.jsx,.ts,.tsx 224 225 esti: ## run esti (system testing) 226 $(GOTEST) -v ./esti --args --system-tests 227 228 test: test-go test-hadoopfs ## Run tests for the project 229 230 test-go: gen-api # Run parallelism > num_cores: most of our slow tests are *not* CPU-bound. 231 $(GOTEST) -count=1 -coverprofile=cover.out -race -cover -failfast -parallel="$(GOTEST_PARALLELISM)" ./... 232 233 test-hadoopfs: 234 cd clients/hadoopfs && mvn test 235 236 run-test: ## Run tests without generating anything (faster if already generated) 237 $(GOTEST) -count=1 -coverprofile=cover.out -race -short -cover -failfast ./... 238 239 fast-test: ## Run tests without race detector (faster) 240 $(GOTEST) -count=1 -coverprofile=cover.out -short -cover -failfast ./... 241 242 test-html: test ## Run tests with HTML for the project 243 $(GOTOOL) cover -html=cover.out 244 245 system-tests: # Run system tests locally 246 ./esti/scripts/runner.sh -r all 247 248 build-docker: build ## Build Docker image file (Docker required) 249 $(DOCKER) buildx build --target lakefs -t treeverse/$(DOCKER_IMAGE):$(DOCKER_TAG) . 250 251 gofmt: ## gofmt code formating 252 @echo Running go formating with the following command: 253 $(GOFMT) -e -s -w . 254 255 validate-fmt: ## Validate go format 256 @echo checking gofmt... 257 @res=$$($(GOFMT) -d -e -s $$(find . -type d \( -path ./pkg/metastore/hive/gen-go \) -prune -prune -o \( -path ./pkg/api/gen \) -prune -o \( -path ./pkg/permissions/*.gen.go \) -prune -o -name '*.go' -print)); \ 258 if [ -n "$${res}" ]; then \ 259 echo checking gofmt fail... ; \ 260 echo "$${res}"; \ 261 exit 1; \ 262 else \ 263 echo Your code formatting is according to gofmt standards; \ 264 fi 265 266 .PHONY: validate-proto 267 validate-proto: gen-proto ## build proto and check if diff found 268 git diff --quiet -- pkg/actions/actions.pb.go || (echo "Modification verification failed! pkg/actions/actions.pb.go"; false) 269 git diff --quiet -- pkg/auth/model/model.pb.go || (echo "Modification verification failed! pkg/auth/model/model.pb.go"; false) 270 git diff --quiet -- pkg/catalog/catalog.pb.go || (echo "Modification verification failed! pkg/catalog/catalog.pb.go"; false) 271 git diff --quiet -- pkg/gateway/multipart/multipart.pb.go || (echo "Modification verification failed! pkg/gateway/multipart/multipart.pb.go"; false) 272 git diff --quiet -- pkg/graveler/graveler.pb.go || (echo "Modification verification failed! pkg/graveler/graveler.pb.go"; false) 273 git diff --quiet -- pkg/graveler/committed/committed.pb.go || (echo "Modification verification failed! pkg/graveler/committed/committed.pb.go"; false) 274 git diff --quiet -- pkg/graveler/settings/test_settings.pb.go || (echo "Modification verification failed! pkg/graveler/settings/test_settings.pb.go"; false) 275 git diff --quiet -- pkg/kv/secondary_index.pb.go || (echo "Modification verification failed! pkg/kv/secondary_index.pb.go"; false) 276 git diff --quiet -- pkg/kv/kvtest/test_model.pb.go || (echo "Modification verification failed! pkg/kv/kvtest/test_model.pb.go"; false) 277 278 .PHONY: validate-mockgen 279 validate-mockgen: gen-code 280 git diff --quiet -- pkg/actions/mock/mock_actions.go || (echo "Modification verification failed! pkg/actions/mock/mock_actions.go"; false) 281 git diff --quiet -- pkg/auth/mock/mock_auth_client.go || (echo "Modification verification failed! pkg/auth/mock/mock_auth_client.go"; false) 282 git diff --quiet -- pkg/authentication/api/mock_authentication_client.go || (echo "Modification verification failed! pkg/authentication/api/mock_authentication_client.go"; false) 283 git diff --quiet -- pkg/graveler/committed/mock/batch_write_closer.go || (echo "Modification verification failed! pkg/graveler/committed/mock/batch_write_closer.go"; false) 284 git diff --quiet -- pkg/graveler/committed/mock/meta_range.go || (echo "Modification verification failed! pkg/graveler/committed/mock/meta_range.go"; false) 285 git diff --quiet -- pkg/graveler/committed/mock/range_manager.go || (echo "Modification verification failed! pkg/graveler/committed/mock/range_manager.go"; false) 286 git diff --quiet -- pkg/graveler/mock/graveler.go || (echo "Modification verification failed! pkg/graveler/mock/graveler.go"; false) 287 git diff --quiet -- pkg/kv/mock/store.go || (echo "Modification verification failed! pkg/kv/mock/store.go"; false) 288 git diff --quiet -- pkg/pyramid/mock/pyramid.go || (echo "Modification verification failed! pkg/pyramid/mock/pyramid.go"; false) 289 290 .PHONY: validate-permissions-gen 291 validate-permissions-gen: gen-code 292 git diff --quiet -- pkg/permissions/actions.gen.go || (echo "Modification verification failed! pkg/permissions/actions.gen.go"; false) 293 294 validate-reference: 295 git diff --quiet -- docs/reference/cli.md || (echo "Modification verification failed! docs/reference/cli.md"; false) 296 297 validate-client-python: validate-python-sdk-legacy validate-python-sdk 298 299 validate-python-sdk-legacy: 300 git diff --quiet -- clients/python-legacy || (echo "Modification verification failed! python client"; false) 301 302 validate-python-sdk: 303 git diff --quiet -- clients/python || (echo "Modification verification failed! python client"; false) 304 305 validate-client-java: 306 git diff --quiet -- clients/java || (echo "Modification verification failed! java client"; false) 307 308 validate-client-rust: 309 git diff --quiet -- clients/rust || (echo "Modification verification failed! rust client"; false) 310 311 validate-python-wrapper: 312 sphinx-apidoc -o clients/python-wrapper/docs clients/python-wrapper/lakefs sphinx-apidoc --full -A 'Treeverse' -eq 313 git diff --quiet -- clients/python-wrapper || (echo 'Modification verification failed! python wrapper client'; false) 314 315 # Run all validation/linting steps 316 checks-validator: lint validate-fmt validate-proto validate-client-python validate-client-java validate-client-rust validate-reference validate-mockgen validate-permissions-gen 317 318 python-wrapper-lint: 319 $(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python-wrapper $(PYTHON_IMAGE) /bin/bash -c "./pylint.sh" 320 321 python-wrapper-gen-docs: 322 sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/ 323 sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/$$(python clients/python-wrapper/setup.py --version) 324 325 $(UI_DIR)/node_modules: 326 cd $(UI_DIR) && $(NPM) install 327 328 gen-ui: $(UI_DIR)/node_modules ## Build UI web app 329 cd $(UI_DIR) && $(NPM) run build 330 331 gen-proto: ## Build Protocol Buffers (proto) files using Buf CLI 332 go run github.com/bufbuild/buf/cmd/buf@$(BUF_CLI_VERSION) generate 333 334 publish-scala: ## sbt publish spark client jars to nexus and s3 bucket 335 cd clients/spark && sbt assembly && sbt s3Upload && sbt publishSigned 336 aws s3 cp --recursive --acl public-read $(CLIENT_JARS_BUCKET) $(CLIENT_JARS_BUCKET) --metadata-directive REPLACE 337 338 publish-lakefsfs-test: ## sbt publish spark lakefsfs test jars to s3 bucket 339 cd test/lakefsfs && sbt assembly && sbt s3Upload 340 aws s3 cp --recursive --acl public-read $(CLIENT_JARS_BUCKET) $(CLIENT_JARS_BUCKET) --metadata-directive REPLACE 341 342 help: ## Show Help menu 343 @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 344 345 # helpers 346 gen: gen-ui gen-api clients gen-docs