github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/Makefile (about)

     1  export GO = go
     2  export GOOS ?= $(shell $(GO) env GOOS)
     3  export GOARCH ?= $(shell $(GO) env GOARCH)
     4  
     5  ifeq ($(GOARCH),armv6)
     6  export GOARCH = arm
     7  export GOARM = 6
     8  endif
     9  
    10  ifeq ($(GOARCH),armv7)
    11  export GOARCH = arm
    12  export GOARM = 7
    13  endif
    14  
    15  # Env Variables
    16  export GO111MODULE = on
    17  export CGO_ENABLED = 0
    18  export PRECOMMIT = poetry run pre-commit
    19  
    20  BUILD_DIR = bacalhau
    21  BINARY_NAME = bacalhau
    22  
    23  ifeq ($(GOOS),windows)
    24  BINARY_NAME := ${BINARY_NAME}.exe
    25  CC = gcc.exe
    26  endif
    27  
    28  BINARY_PATH = bin/${GOOS}_${GOARCH}${GOARM}/${BINARY_NAME}
    29  
    30  TAG ?= $(eval TAG := $(shell git describe --tags --always))$(TAG)
    31  COMMIT ?= $(eval COMMIT := $(shell git rev-parse HEAD))$(COMMIT)
    32  REPO ?= $(shell echo $$(cd ../${BUILD_DIR} && git config --get remote.origin.url) | sed 's/git@\(.*\):\(.*\).git$$/https:\/\/\1\/\2/')
    33  BRANCH ?= $(shell cd ../${BUILD_DIR} && git branch | grep '^*' | awk '{print $$2}')
    34  BUILDDATE ?= $(eval BUILDDATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ'))$(BUILDDATE)
    35  PACKAGE := $(shell echo "bacalhau_$(TAG)_${GOOS}_$(GOARCH)${GOARM}")
    36  PRECOMMIT_HOOKS_INSTALLED ?= $(shell grep -R "pre-commit.com" .git/hooks)
    37  TEST_BUILD_TAGS ?= unit,integration
    38  TEST_PARALLEL_PACKAGES ?= 1
    39  
    40  PRIVATE_KEY_FILE := /tmp/private.pem
    41  PUBLIC_KEY_FILE := /tmp/public.pem
    42  
    43  define BUILD_FLAGS
    44  -X github.com/filecoin-project/bacalhau/pkg/version.GITVERSION=$(TAG)
    45  endef
    46  
    47  all: build
    48  
    49  # Run init repo after cloning it
    50  .PHONY: init
    51  init:
    52  	@ops/repo_init.sh 1>/dev/null
    53  	@echo "Build environment initialized."
    54  
    55  # Run install pre-commit
    56  .PHONY: install-pre-commit
    57  install-pre-commit:
    58  	@ops/install_pre_commit.sh 1>/dev/null
    59  	@echo "Pre-commit installed."
    60  
    61  ## Run all pre-commit hooks
    62  ################################################################################
    63  # Target: precommit
    64  ################################################################################
    65  .PHONY: precommit
    66  precommit: buildenvcorrect
    67  	${PRECOMMIT} run --all
    68  	cd python && make pre-commit
    69  
    70  .PHONY: buildenvcorrect
    71  buildenvcorrect:
    72  	@echo "Checking build environment..."
    73  # Checking GO
    74  # @echo "Checking for go..."
    75  # @which go
    76  # @echo "Checking for go version..."
    77  # @go version
    78  # @echo "Checking for go env..."
    79  # @go env
    80  # @echo "Checking for go env GOOS..."
    81  # @go env GOOS
    82  # @echo "Checking for go env GOARCH..."
    83  # @go env GOARCH
    84  # @echo "Checking for go env GO111MODULE..."
    85  # @go env GO111MODULE
    86  # @echo "Checking for go env GOPATH..."
    87  # @go env GOPATH
    88  # @echo "Checking for go env GOCACHE..."
    89  # @go env GOCACHE
    90  # ===============
    91  # Ensure that "pre-commit.com" is in .git/hooks/pre-commit to run all pre-commit hooks
    92  # before each commit.
    93  # Error if it's empty or not found.
    94  ifeq ($(PRECOMMIT_HOOKS_INSTALLED),)
    95  	@echo "Pre-commit is not installed in .git/hooks/pre-commit. Please run 'make install-pre-commit' to install it."
    96  	@exit 1
    97  endif
    98  	@echo "Build environment correct."
    99  
   100  ################################################################################
   101  # Target: swagger-docs
   102  ################################################################################
   103  .PHONY: swagger-docs
   104  swagger-docs:
   105  	@echo "Building swagger docs..."
   106  	swag fmt -g "pkg/publicapi/server.go" && \
   107  	swag init --outputTypes "go,json" --parseDependency --parseInternal --parseDepth 1 --markdownFiles docs/swagger -g "pkg/publicapi/server.go"
   108  	@echo "Swagger docs built."
   109  
   110  ################################################################################
   111  # Target: clients
   112  ################################################################################
   113  # Generate Bacalhau API clients but only if the swagger.json has actually been
   114  # updated because the clients include random numbers and timestamps and hence
   115  # will generate a lot of noisy diffs if regenerated all of the time
   116  .PHONY: clients
   117  clients:
   118  	(test -n "$(shell git ls-files --modified docs/swagger.json)" && \
   119  		cd clients && ${MAKE} -j all) || true
   120  
   121  ################################################################################
   122  # Target: build
   123  ################################################################################
   124  .PHONY: build
   125  build: buildenvcorrect build-bacalhau
   126  
   127  .PHONY: build-ci
   128  build-ci: build-bacalhau
   129  
   130  .PHONY: build-dev
   131  build-dev: build-ci
   132  	sudo cp ${BINARY_PATH} /usr/local/bin
   133  
   134  ################################################################################
   135  # Target: build-bacalhau
   136  ################################################################################
   137  .PHONY: build-bacalhau
   138  build-bacalhau: ${BINARY_PATH}
   139  
   140  CMD_FILES := $(shell bash -c 'comm -23 <(git ls-files cmd) <(git ls-files cmd --deleted)')
   141  PKG_FILES := $(shell bash -c 'comm -23 <(git ls-files pkg) <(git ls-files pkg --deleted)')
   142  
   143  ${BINARY_PATH}: ${CMD_FILES} ${PKG_FILES} main.go
   144  	${GO} build -ldflags "${BUILD_FLAGS}" -trimpath -o ${BINARY_PATH} .
   145  
   146  ################################################################################
   147  # Target: build-docker-images
   148  ################################################################################
   149  HTTP_GATEWAY_IMAGE ?= "ghcr.io/bacalhau-project/http-gateway"
   150  HTTP_GATEWAY_TAG ?= ${TAG}
   151  .PHONY: build-http-gateway-image
   152  build-http-gateway-image:
   153  	docker buildx build \
   154  		--platform linux/amd64,linux/arm64 \
   155  		-t ${HTTP_GATEWAY_IMAGE}:${HTTP_GATEWAY_TAG} \
   156  		pkg/executor/docker/gateway
   157  
   158  BACALHAU_IMAGE ?= ghcr.io/bacalhau-project/bacalhau
   159  BACALHAU_TAG ?= ${TAG}
   160  .PHONY: build-bacalhau-image
   161  build-bacalhau-image:
   162  	docker build --progress=plain \
   163  		--tag ${BACALHAU_IMAGE}:latest \
   164  		--file docker/bacalhau-image/Dockerfile \
   165  		.
   166  
   167  .PHONY: push-bacalhau-image
   168  push-bacalhau-image:
   169  	docker buildx build --push --progress=plain \
   170  		--platform linux/amd64,linux/arm64 \
   171  		--tag ${BACALHAU_IMAGE}:${BACALHAU_TAG} \
   172  		--tag ${BACALHAU_IMAGE}:latest \
   173  		--label org.opencontainers.artifact.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
   174  		--label org.opencontainers.image.version=${BACALHAU_TAG} \
   175  		--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest \
   176  		--file docker/bacalhau-image/Dockerfile \
   177  		.
   178  
   179  .PHONY: build-docker-images
   180  build-docker-images: build-http-gateway-image
   181  
   182  .PHONY: push-docker-images
   183  push-docker-images: build-http-gateway-image
   184  
   185  # Release tarballs suitable for upload to GitHub release pages
   186  ################################################################################
   187  # Target: build-bacalhau-tgz
   188  ################################################################################
   189  .PHONY: build-bacalhau-tgz
   190  build-bacalhau-tgz: dist/${PACKAGE}.tar.gz dist/${PACKAGE}.tar.gz.signature.sha256
   191  
   192  dist/:
   193  	mkdir -p $@
   194  
   195  dist/${PACKAGE}.tar.gz: ${BINARY_PATH} | dist/
   196  	tar cvzf $@ -C $(dir $(BINARY_PATH)) $(notdir ${BINARY_PATH})
   197  
   198  dist/${PACKAGE}.tar.gz.signature.sha256: dist/${PACKAGE}.tar.gz | dist/
   199  	openssl dgst -sha256 -sign $(PRIVATE_KEY_FILE) -passin pass:"$(PRIVATE_KEY_PASSPHRASE)" $^ | openssl base64 -out $@
   200  
   201  
   202  ################################################################################
   203  # Target: images
   204  ################################################################################
   205  IMAGE_REGEX := 'Image ?(:|=)\s*"[^"]+"'
   206  FILES_WITH_IMAGES := $(shell grep -Erl ${IMAGE_REGEX} pkg cmd)
   207  
   208  docker/.images: ${FILES_WITH_IMAGES}
   209  	grep -Eroh ${IMAGE_REGEX} $^ | cut -d'"' -f2 | sort | uniq > $@
   210  
   211  docker/.pulled: docker/.images
   212  	- cat $^ | xargs -n1 docker pull
   213  	touch $@
   214  
   215  .PHONY: images
   216  images: docker/.pulled
   217  
   218  ################################################################################
   219  # Target: clean
   220  ################################################################################
   221  .PHONY: clean
   222  clean:
   223  	${GO} clean
   224  	${RM} -r bin/*
   225  	${RM} dist/bacalhau_*
   226  	${RM} docker/.images
   227  	${RM} docker/.pulled
   228  
   229  
   230  ################################################################################
   231  # Target: schema
   232  ################################################################################
   233  SCHEMA_DIR ?= schema.bacalhau.org/jsonschema
   234  
   235  .PHONY: schema
   236  schema: ${SCHEMA_DIR}/$(shell git describe --tags --abbrev=0).json
   237  
   238  ${SCHEMA_DIR}/%.json:
   239  	./scripts/build-schema-file.sh $$(basename -s .json $@) > $@
   240  
   241  ################################################################################
   242  # Target: all_schemas
   243  ################################################################################
   244  EARLIEST_TAG := v0.3.12
   245  ALL_TAGS := $(shell git tag -l --contains $$(git rev-parse ${EARLIEST_TAG}) | grep -E 'v\d+\.\d+.\d+')
   246  ALL_SCHEMAS := $(patsubst %,${SCHEMA_DIR}/%.json,${ALL_TAGS})
   247  
   248  .PHONY: all_schemas
   249  all_schemas: ${ALL_SCHEMAS}
   250  
   251  
   252  ################################################################################
   253  # Target: test
   254  ################################################################################
   255  .PHONY: test
   256  test:
   257  # unittests parallelize well (default go test behavior is to parallelize)
   258  	go test ./... -v --tags=unit
   259  
   260  .PHONY: test-python
   261  test-python:
   262  	cd python && make test
   263  
   264  .PHONY: integration-test
   265  integration-test:
   266  # integration tests parallelize less well (hence -p 1)
   267  	go test ./... -v --tags=integration -p 1
   268  
   269  .PHONY: grc-test
   270  grc-test:
   271  	grc go test ./... -v
   272  .PHONY: grc-test-short
   273  grc-test-short:
   274  	grc go test ./... -test.short -v
   275  
   276  .PHONY: test-debug
   277  test-debug:
   278  	LOG_LEVEL=debug go test ./... -v
   279  
   280  .PHONY: grc-test-debug
   281  grc-test-debug:
   282  	LOG_LEVEL=debug grc go test ./... -v
   283  
   284  .PHONY: test-one
   285  test-one:
   286  	go test -v -count 1 -timeout 3000s -run ^$(TEST)$$ github.com/filecoin-project/bacalhau/cmd/bacalhau/
   287  
   288  .PHONY: test-devstack
   289  test-devstack:
   290  	go test -v -count 1 -timeout 3000s -run '^Test\w+Suite$$' github.com/filecoin-project/bacalhau/pkg/test/devstack/
   291  
   292  .PHONY: test-commands
   293  test-commands:
   294  	go test -v -count 1 -timeout 3000s -run '^Test\w+Suite$$' github.com/filecoin-project/bacalhau/cmd/bacalhau/
   295  
   296  # .PHONY: test-pythonwasm
   297  # test-pythonwasm:
   298  # # TestSimplestPythonWasmDashC
   299  # 	LOG_LEVEL=debug go test -v -count 1 -timeout 3000s -run ^TestSimplePythonWasm$$ github.com/filecoin-project/bacalhau/pkg/test/devstack/
   300  # #	LOG_LEVEL=debug go test -v -count 1 -timeout 3000s -run ^TestSimplestPythonWasmDashC$$ github.com/filecoin-project/bacalhau/pkg/test/devstack/
   301  
   302  ################################################################################
   303  # Target: devstack
   304  ################################################################################
   305  .PHONY: devstack
   306  devstack:
   307  	go run . devstack
   308  
   309  .PHONY: devstack-one
   310  devstack-one:
   311  	IGNORE_PID_AND_PORT_FILES=true PREDICTABLE_API_PORT=1 go run . devstack --requester-nodes 0 --compute-nodes 0 --hybrid-nodes 1
   312  
   313  .PHONY: devstack-100
   314  devstack-100:
   315  	go run . devstack --compute-nodes 100
   316  
   317  .PHONY: devstack-250
   318  devstack-250:
   319  	go run . devstack --compute-nodes 250
   320  
   321  .PHONY: devstack-20
   322  devstack-20:
   323  	go run . devstack --compute-nodes 20
   324  
   325  .PHONY: devstack-noop
   326  devstack-noop:
   327  	go run . devstack --noop
   328  
   329  .PHONY: devstack-noop-100
   330  devstack-noop-100:
   331  	go run . devstack --noop --compute-nodes 100
   332  
   333  .PHONY: devstack-race
   334  devstack-race:
   335  	go run -race . devstack
   336  
   337  .PHONY: devstack-badactor
   338  devstack-badactor:
   339  	go run . devstack --bad-compute-actors 1
   340  
   341  ################################################################################
   342  # Target: lint
   343  ################################################################################
   344  .PHONY: lint
   345  lint:
   346  	golangci-lint run --timeout 10m
   347  
   348  .PHONY: lint-fix
   349  lint-fix:
   350  	golangci-lint run --timeout 10m --fix
   351  
   352  ################################################################################
   353  # Target: modtidy
   354  ################################################################################
   355  .PHONY: modtidy
   356  modtidy:
   357  	go mod tidy
   358  
   359  ################################################################################
   360  # Target: check-diff
   361  ################################################################################
   362  .PHONY: check-diff
   363  check-diff:
   364  	git diff --exit-code ./go.mod # check no changes
   365  	git diff --exit-code ./go.sum # check no changes
   366  
   367  # Run the unittests and output results for recording
   368  ################################################################################
   369  # Target: test-test-and-report
   370  ################################################################################
   371  COMMA = ,
   372  COVER_FILE := coverage/${PACKAGE}_$(subst ${COMMA},_,${TEST_BUILD_TAGS}).coverage
   373  
   374  .PHONY: test-and-report
   375  test-and-report: unittests.xml ${COVER_FILE}
   376  
   377  ${COVER_FILE} unittests.xml ${TEST_OUTPUT_FILE_PREFIX}_unit.json: ${BINARY_PATH} $(dir ${COVER_FILE})
   378  	gotestsum \
   379  		--jsonfile ${TEST_OUTPUT_FILE_PREFIX}_unit.json \
   380  		--junitfile unittests.xml \
   381  		--format testname \
   382  		-- \
   383  			-p ${TEST_PARALLEL_PACKAGES} \
   384  			./pkg/... ./cmd/... \
   385  			-coverpkg=./... -coverprofile=${COVER_FILE} \
   386  			--tags=${TEST_BUILD_TAGS}
   387  
   388  ################################################################################
   389  # Target: coverage-report
   390  ################################################################################
   391  .PHONY:
   392  coverage-report: coverage/coverage.html
   393  
   394  coverage/coverage.out: $(wildcard coverage/*.coverage)
   395  	gocovmerge $^ > $@
   396  
   397  coverage/coverage.html: coverage/coverage.out coverage/
   398  	go tool cover -html=$< -o $@
   399  
   400  coverage/:
   401  	mkdir -p $@
   402  
   403  .PHONY: generate
   404  generate:
   405  	${GO} generate -gcflags '-N -l' -ldflags "-X main.VERSION=$(TAG)" ./...
   406  	echo "[OK] Files added to pipeline template directory!"
   407  
   408  .PHONY: security
   409  security:
   410  	gosec -exclude=G204,G304 -exclude-dir=test ./...
   411  	echo "[OK] Go security check was completed!"
   412  
   413  release: build-bacalhau
   414  	cp bin/bacalhau .