github.com/pachyderm/pachyderm@v1.13.4/examples/spouts/go-kafka-spout/Makefile (about)

     1  .PHONY: docker-image kafka
     2  PROJNAME := pachkafka
     3  
     4  # customize these to your kafka deployment and needs
     5  KAFKA_HOST := kafka.kafka
     6  KAFKA_TOPIC := test_topic
     7  KAFKA_GROUP_ID := test_group
     8  KAFKA_PORT := 9092
     9  KAFKA_TIMEOUT := 5
    10  NAMED_PIPE := /pfs/out
    11  VERBOSE_LOGGING := true
    12  
    13  PIPELINES_DIR := pipelines
    14  SOURCE_DIR := source
    15  ADDITIONAL_MANIFESTS_DIR := additional_manifests
    16  PROJ_DIRS := $(SOURCE_DIR) $(PIPELINES_DIR)
    17  JQFLAGS :=
    18  
    19  # You should set these to appropriate values.
    20  CONTAINER_VERSION := $(shell pachctl version --client-only)
    21  DOCKER_ACCOUNT := pachyderm
    22  CONTAINER_NAME := kafka_spout
    23  CONTAINER_TAG := $(DOCKER_ACCOUNT)/$(CONTAINER_NAME):$(CONTAINER_VERSION)
    24  
    25  
    26  PIPELINE_TRANSFORM := .transform.image="$(CONTAINER_TAG)"|.transform.env.KAFKA_HOST="$(KAFKA_HOST)"|.transform.env.KAFKA_TOPIC="$(KAFKA_TOPIC)"|.transform.env.KAFKA_GROUP_ID="$(KAFKA_GROUP_ID)"|.transform.env.KAFKA_PORT="$(KAFKA_PORT)"|.transform.env.KAFKA_TIMEOUT="$(KAFKA_TIMEOUT)"|.transform.env.NAMED_PIPE="$(NAMED_PIPE)"|.transform.env.VERBOSE_LOGGING="$(VERBOSE_LOGGING)"
    27  PIPELINE_SOURCE_FILES = $(shell find $(PROJ_DIRS) -type f -name \*.go)
    28  PIPELINE_DEF_FILES = $(shell find $(PROJ_DIRS) -type f -name \*.pipeline)
    29  KAFKA_FILES = $(sort $(shell find $(ADDITIONAL_MANIFESTS_DIR) -type f -name [0-9][0-9][0-9]\*.yaml))
    30  PIPELINE_JSON_FILES = $(patsubst %.pipeline,%.json,$(PIPELINE_DEF_FILES))
    31  DAG := kafka_spout 
    32  #this needs to be recursively expanded because it's used in a foreach loop
    33  PACHCTL_FILE = -f $(PIPELINES_DIR)/$(stage).json
    34  
    35  
    36  pachctl-pipeline = @$(foreach stage,$(2), pachctl $(1) pipeline $(PACHCTL_FILE) ; )
    37  pachctl-delete-pipeline = @$(foreach stage,$(1), pachctl delete pipeline $(stage) ; )
    38  kubectl-apply = @$(foreach manifest,$(1), kubectl apply -f $(manifest)  ; )
    39  kubectl-delete = @$(foreach manifest,$(1), kubectl delete -f $(manifest)  ; )
    40  reverse = $(if $(1),$(call reverse,$(wordlist 2,$(words $(1)),$(1)))) $(firstword $(1))
    41  
    42  %.json: %.pipeline Makefile 
    43  	@jq $(JQFLAGS) '$(PIPELINE_TRANSFORM)'  $< > $@
    44  
    45  kafka: $(KAFKA_FILES)
    46  	@$(call kubectl-apply,$(KAFKA_FILES))
    47  
    48  docker-image: Dockerfile $(PIPELINE_JSON_FILES) $(PIPELINE_SOURCE_FILES)
    49  	@docker build -t $(CONTAINER_TAG) .
    50  	@docker push $(CONTAINER_TAG)
    51  
    52  create-dag: kafka docker-image $(PIPELINE_JSON_FILES) $(PIPELINE_SOURCE_FILES)
    53  	@$(call pachctl-pipeline,create,$(DAG))
    54  
    55  update-dag: kafka docker-image $(PIPELINE_JSON_FILES) $(PIPELINE_SOURCE_FILES)
    56  	@$(call pachctl-pipeline,update,$(DAG))
    57  
    58  clean:
    59  	-@$(call kubectl-delete,$(KAFKA_FILES))
    60  	-@$(call pachctl-delete-pipeline,$(call reverse,$(DAG)))
    61  	-@rm -f $(PIPELINE_JSON_FILES)
    62  
    63