github.com/uber/kraken@v0.1.4/Makefile (about)

     1  SHELL = /bin/bash -o pipefail
     2  GO = go
     3  
     4  # Flags to pass to go build
     5  BUILD_FLAGS = -gcflags '-N -l'
     6  BUILD_QUIET ?= -q
     7  
     8  # Where to find your project
     9  PROJECT_ROOT = github.com/uber/kraken
    10  PACKAGE_VERSION ?= $(shell git describe --always --tags)
    11  
    12  ALL_SRC = $(shell find . -name "*.go" | grep -v \
    13  	-e ".*/\..*" \
    14  	-e ".*/_.*" \
    15  	-e ".*/mocks.*" \
    16  	-e ".*/*.pb.go")
    17  
    18  ALL_PKGS = $(shell go list $(sort $(dir $(ALL_SRC))))
    19  
    20  # ==== BASIC ====
    21  
    22  BUILD_LINUX = GOOS=linux GOARCH=amd64 $(GO) build -i -o $@ $(BUILD_FLAGS) $(BUILD_GC_FLAGS) $(BUILD_VERSION_FLAGS) ./$(dir $@)
    23  
    24  # Cross compiling cgo for sqlite3 is not well supported in Mac OSX.
    25  # This workaround builds the binary inside a linux container.
    26  CROSS_COMPILER = docker run --rm -it -v $(shell pwd):/go/src/github.com/uber/kraken -w /go/src/github.com/uber/kraken golang:1.14 go build -o ./$@ ./$(dir $@)
    27  
    28  LINUX_BINS = \
    29  	agent/agent \
    30  	build-index/build-index \
    31  	origin/origin \
    32  	proxy/proxy \
    33  	tools/bin/testfs/testfs \
    34  	tracker/tracker
    35  
    36  REGISTRY ?= gcr.io/uber-container-tools
    37  
    38  agent/agent:: $(wildcard agent/*.go)
    39  	$(CROSS_COMPILER)
    40  
    41  build-index/build-index:: $(wildcard build-index/*.go)
    42  	$(CROSS_COMPILER)
    43  
    44  origin/origin:: $(wildcard origin/*.go)
    45  	$(CROSS_COMPILER)
    46  
    47  proxy/proxy:: $(wildcard proxy/*.go)
    48  	$(CROSS_COMPILER)
    49  
    50  tools/bin/testfs/testfs:: $(wildcard tools/bin/testfs/*.go)
    51  	$(CROSS_COMPILER)
    52  
    53  tracker/tracker:: $(wildcard tracker/*.go)
    54  	$(CROSS_COMPILER)
    55  
    56  $(LINUX_BINS)::
    57  
    58  define tag_image
    59  	docker tag $(1):$(PACKAGE_VERSION) $(1):dev
    60  	docker tag $(1):$(PACKAGE_VERSION) $(REGISTRY)/$(1):$(PACKAGE_VERSION)
    61  endef
    62  
    63  .PHONY: images
    64  images: $(LINUX_BINS)
    65  	docker build $(BUILD_QUIET) -t kraken-agent:$(PACKAGE_VERSION) -f docker/agent/Dockerfile ./
    66  	docker build $(BUILD_QUIET) -t kraken-build-index:$(PACKAGE_VERSION) -f docker/build-index/Dockerfile ./
    67  	docker build $(BUILD_QUIET) -t kraken-origin:$(PACKAGE_VERSION) -f docker/origin/Dockerfile ./
    68  	docker build $(BUILD_QUIET) -t kraken-proxy:$(PACKAGE_VERSION) -f docker/proxy/Dockerfile ./
    69  	docker build $(BUILD_QUIET) -t kraken-testfs:$(PACKAGE_VERSION) -f docker/testfs/Dockerfile ./
    70  	docker build $(BUILD_QUIET) -t kraken-tracker:$(PACKAGE_VERSION) -f docker/tracker/Dockerfile ./
    71  	docker build $(BUILD_QUIET) -t kraken-herd:$(PACKAGE_VERSION) -f docker/herd/Dockerfile ./
    72  	$(call tag_image,kraken-agent)
    73  	$(call tag_image,kraken-build-index)
    74  	$(call tag_image,kraken-origin)
    75  	$(call tag_image,kraken-proxy)
    76  	$(call tag_image,kraken-proxy)
    77  	$(call tag_image,kraken-testfs)
    78  	$(call tag_image,kraken-tracker)
    79  	$(call tag_image,kraken-herd)
    80  
    81  .PHONY: publish
    82  publish: images
    83  	docker push $(REGISTRY)/kraken-agent:$(PACKAGE_VERSION)
    84  	docker push $(REGISTRY)/kraken-build-index:$(PACKAGE_VERSION)
    85  	docker push $(REGISTRY)/kraken-origin:$(PACKAGE_VERSION)
    86  	docker push $(REGISTRY)/kraken-proxy:$(PACKAGE_VERSION)
    87  	docker push $(REGISTRY)/kraken-testfs:$(PACKAGE_VERSION)
    88  	docker push $(REGISTRY)/kraken-tracker:$(PACKAGE_VERSION)
    89  	docker push $(REGISTRY)/kraken-herd:$(PACKAGE_VERSION)
    90  
    91  clean::
    92  	@rm -f $(LINUX_BINS)
    93  
    94  .PHONY: bins
    95  bins: $(LINUX_BINS)
    96  
    97  # ==== TEST ====
    98  .PHONY: unit-test
    99  unit-test:
   100  	-rm coverage.txt
   101  	$(GO) test -timeout=30s -race -coverprofile=coverage.txt $(ALL_PKGS) --tags "unit"
   102  
   103  .PHONY: docker_stop
   104  docker_stop:
   105  	-docker ps -a --format '{{.Names}}' | grep kraken | while read n; do docker rm -f $$n; done
   106  
   107  venv: requirements-tests.txt
   108  	virtualenv --python=$(shell which python2) --setuptools venv
   109  	source venv/bin/activate
   110  	venv/bin/pip install -r requirements-tests.txt
   111  
   112  .PHONY: integration
   113  FILE?=
   114  NAME?=test_
   115  USERNAME:=$(shell id -u -n)
   116  USERID:=$(shell id -u)
   117  integration: venv $(LINUX_BINS) docker_stop tools/bin/puller/puller
   118  	docker build $(BUILD_QUIET) -t kraken-agent:$(PACKAGE_VERSION) -f docker/agent/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   119  	docker build $(BUILD_QUIET) -t kraken-build-index:$(PACKAGE_VERSION) -f docker/build-index/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   120  	docker build $(BUILD_QUIET) -t kraken-origin:$(PACKAGE_VERSION) -f docker/origin/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   121  	docker build $(BUILD_QUIET) -t kraken-proxy:$(PACKAGE_VERSION) -f docker/proxy/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   122  	docker build $(BUILD_QUIET) -t kraken-testfs:$(PACKAGE_VERSION) -f docker/testfs/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   123  	docker build $(BUILD_QUIET) -t kraken-tracker:$(PACKAGE_VERSION) -f docker/tracker/Dockerfile --build-arg USERID=$(USERID) --build-arg USERNAME=$(USERNAME) ./
   124  	PACKAGE_VERSION=$(PACKAGE_VERSION) venv/bin/py.test --timeout=120 -v -k $(NAME) test/python/$(FILE)
   125  
   126  .PHONY: runtest
   127  NAME?=test_
   128  runtest: venv docker_stop
   129  	source venv/bin/activate
   130  	venv/bin/py.test --timeout=120 -v -k $(NAME) test/python
   131  
   132  .PHONY: devcluster
   133  devcluster: $(LINUX_BINS) docker_stop images
   134  	./examples/devcluster/herd_start_container.sh
   135  	./examples/devcluster/agent_one_start_container.sh
   136  	./examples/devcluster/agent_two_start_container.sh
   137  
   138  # ==== TOOLS ====
   139  
   140  TOOLS = \
   141  	tools/bin/puller/puller \
   142  	tools/bin/reload/reload \
   143  	tools/bin/visualization/visualization
   144  
   145  tools/bin/puller/puller:: $(wildcard tools/bin/puller/puller/*.go)
   146  	$(CROSS_COMPILER)
   147  
   148  tools/bin/reload/reload:: $(wildcard tools/bin/reload/reload/*.go)
   149  	$(CROSS_COMPILER)
   150  
   151  tools/bin/visualization/visualization:: $(wildcard tools/bin/visualization/visualization/*.go)
   152  	$(CROSS_COMPILER)
   153  
   154  .PHONY: tools
   155  tools: $(TOOLS)
   156  
   157  # Creates a release summary containing the build revisions of each component
   158  # for the specified version.
   159  releases/%:
   160  	./scripts/release.sh $(subst releases/,,$@)
   161  
   162  # ==== CODE GENERATION ====
   163  
   164  # In order for kraken to be imported by other projects, we need to check in all
   165  # the generated code, otherwise dependency management tools would report errors
   166  # caused by missing dependencies of kraken itself.
   167  
   168  # protoc must be installed on the system to make this work.
   169  # Install it by by following instructions on:
   170  # https://github.com/protocolbuffers/protobuf.
   171  PROTOC_BIN = protoc
   172  
   173  PROTO = $(GEN_DIR)/proto/p2p/p2p.pb.go
   174  
   175  GEN_DIR = gen/go
   176  
   177  .PHONY: protoc
   178  protoc:
   179  	mkdir -p $(GEN_DIR)
   180  	go get -u github.com/golang/protobuf/protoc-gen-go
   181  	$(PROTOC_BIN) --plugin=$(shell go env GOPATH)/bin/protoc-gen-go --go_out=$(GEN_DIR) $(subst .pb.go,.proto,$(subst $(GEN_DIR)/,,$(PROTO)))
   182  
   183  # mockgen must be installed on the system to make this work.
   184  # Install it by running:
   185  # `go get github.com/golang/mock/mockgen`.
   186  mockgen = $(shell go env GOPATH)/bin/mockgen
   187  
   188  define lowercase
   189  $(shell tr '[:upper:]' '[:lower:]' <<< $(1))
   190  endef
   191  
   192  define add_mock
   193  	mkdir -p mocks/$(1)
   194  	$(mockgen) \
   195  		-destination=mocks/$(1)/$(call lowercase,$(2)).go \
   196  		-package mock$(notdir $(1)) \
   197  		$(PROJECT_ROOT)/$(1) $(2)
   198  endef
   199  
   200  .PHONY: mocks
   201  mocks:
   202  	rm -rf mocks
   203  	mkdir -p $(shell go env GOPATH)/bin
   204  
   205  	$(call add_mock,agent/agentclient,Client)
   206  
   207  	$(call add_mock,lib/backend/s3backend,S3)
   208  	# mockgen doesn't play nice when importing vendor code. Must strip the vendor prefix
   209  	# from the imports.
   210  	sed -i '' s,github.com/uber/kraken/vendor/,, mocks/lib/backend/s3backend/s3.go
   211  
   212  	$(call add_mock,lib/backend/gcsbackend,GCS)
   213  	sed -i '' s,github.com/uber/kraken/vendor/,, mocks/lib/backend/gcsbackend/gcs.go
   214  
   215  	$(call add_mock,lib/hashring,Ring)
   216  	$(call add_mock,lib/hashring,Watcher)
   217  
   218  	$(call add_mock,lib/backend/hdfsbackend/webhdfs,Client)
   219  
   220  	$(call add_mock,lib/hostlist,List)
   221  
   222  	$(call add_mock,lib/healthcheck,Checker)
   223  	$(call add_mock,lib/healthcheck,Filter)
   224  	$(call add_mock,lib/healthcheck,PassiveFilter)
   225  
   226  	$(call add_mock,tracker/originstore,Store)
   227  
   228  	$(call add_mock,build-index/tagstore,Store)
   229  	$(call add_mock,build-index/tagstore,FileStore)
   230  
   231  	$(call add_mock,build-index/tagtype,DependencyResolver)
   232  
   233  	$(call add_mock,build-index/tagclient,Provider)
   234  	$(call add_mock,build-index/tagclient,Client)
   235  
   236  	$(call add_mock,tracker/announceclient,Client)
   237  
   238  	$(call add_mock,utils/dedup,TaskRunner)
   239  	$(call add_mock,utils/dedup,IntervalTask)
   240  
   241  	$(call add_mock,lib/backend,Client)
   242  
   243  	$(call add_mock,tracker/peerstore,Store)
   244  
   245  	$(call add_mock,lib/store,FileReadWriter)
   246  
   247  	$(call add_mock,lib/torrent/scheduler,ReloadableScheduler)
   248  	$(call add_mock,lib/torrent/scheduler,Scheduler)
   249  
   250  	$(call add_mock,origin/blobclient,Client)
   251  	$(call add_mock,origin/blobclient,Provider)
   252  	$(call add_mock,origin/blobclient,ClusterClient)
   253  	$(call add_mock,origin/blobclient,ClusterProvider)
   254  	$(call add_mock,origin/blobclient,ClientResolver)
   255  
   256  	$(call add_mock,lib/dockerdaemon,DockerClient)
   257  	$(call add_mock,lib/dockerregistry/transfer,ImageTransferer)
   258  
   259  	$(call add_mock,tracker/metainfoclient,Client)
   260  
   261  	$(call add_mock,lib/persistedretry,Store)
   262  	$(call add_mock,lib/persistedretry,Task)
   263  	$(call add_mock,lib/persistedretry,Executor)
   264  	$(call add_mock,lib/persistedretry,Manager)
   265  
   266  	$(call add_mock,lib/persistedretry/tagreplication,RemoteValidator)
   267  
   268  	$(call add_mock,utils/httputil,RoundTripper)
   269  
   270  # ==== MISC ====
   271  
   272  kubecluster:
   273  	cd ./examples/k8s && bash deploy.sh
   274  
   275  .PHONY: docs
   276  docs:
   277  	@./scripts/mkdocs.sh -q serve