github.com/pachyderm/pachyderm@v1.13.4/examples/deferred_processing/automated_deferred_processing/Makefile (about)

     1  .PHONY: create-branch-mover update-branch-mover delete-branch-mover \
     2  	create-branch-mover-no-auth update-branch-mover-no-auth delete-branch-mover-no-auth \
     3  	delete-secret clean
     4  # The executable paths below are set to generic values.
     5  # Modify them for your system by setting environment variables, using a command like this
     6  # to both fetch the version of pachyderm you want to use and execute it.
     7  # "../etc/fetch_release_pachctl.py 1.10.0 ; env PACHCTL=${GOPATH}/bin/pachctl make -e opencv"
     8  
     9  include ../../../etc/govars.mk
    10  
    11  SHELL := /bin/bash
    12  DOCKER := docker
    13  
    14  DEFERRED_PROCESSING_PIPELINES = ../deferred_processing_plus_transactions/edges_dp.json \
    15  				../deferred_processing_plus_transactions/montage_dp.json 
    16  
    17  DEFERRED_PROCESSING_1_FILES = ../deferred_processing_plus_transactions/images.txt \
    18  				../deferred_processing_plus_transactions/images2.txt
    19  
    20  DEFERRED_PROCESSING_2_FILES = ../deferred_processing_plus_transactions/images3.txt
    21  
    22  
    23  # The pipeline uses a timed auth token for authentication.
    24  # The auth token uses a golang duration, which allows h for hours, m for minutes, s for seconds.
    25  # This default is 26 weeks.
    26  # The token will only be valid for this duration if it is *shorter* than the lifetime of the session
    27  # for the user who's logged into the cluster from where this makefile is run.
    28  # See pachctl auth use-auth-token --help for more details.
    29  # If you'd like to experiment with other TTLs, you can customize the value like this:
    30  # env AUTH_TOKEN_TTL=1h make -e create-branch-mover
    31  AUTH_TOKEN_TTL := 624h
    32  
    33  create-deferred-processing-cluster: $(DEFERRED_PROCESSING_PIPELINES) $(DEFERRED_PROCESSING_1_FILES) $(DEFERRED_PROCESSING_2_FILES)
    34  	${PACHCTL} create repo images_dp_1
    35  	${PACHCTL} create repo images_dp_2
    36  	cat $(DEFERRED_PROCESSING_PIPELINES) | ${PACHCTL} create pipeline -f -
    37  	for file in $(DEFERRED_PROCESSING_1_FILES) ; \
    38  	do \
    39  	${PACHCTL} put file images_dp_1@master -i $$file; \
    40  	done
    41  	for file in $(DEFERRED_PROCESSING_2_FILES) ; \
    42  	do \
    43  	${PACHCTL} put file images_dp_2@master -i $$file; \
    44  	done
    45  
    46  
    47  delete-deferred-processing-cluster:
    48  	-@${PACHCTL} delete pipeline montage_dp
    49  	-@${PACHCTL} delete pipeline edges_dp
    50  	-@${PACHCTL} delete repo images_dp_1
    51  	-@${PACHCTL} delete repo images_dp_2
    52  
    53  pachyderm-user-secret.secret: pachyderm-user-secret.clear
    54  	jq ".data.auth_token=\"$$(${PACHCTL} auth get-auth-token --ttl "${AUTH_TOKEN_TTL}" | \
    55  		grep Token | awk '{print $$2}' | base64 -e | tr -d '\r\n')\"" $< > $@
    56  	-${PACHCTL} delete secret $$(jq .metadata.name $@ | tr -d '"')
    57  	${PACHCTL} create secret -f $@
    58  
    59  create-branch-mover: pachyderm-user-secret.secret branch-mover.json 
    60  	${PACHCTL} create pipeline -f branch-mover.json
    61  
    62  update-branch-mover: pachyderm-user-secret.secret branch-mover.json
    63  	${PACHCTL} update pipeline -f branch-mover.json
    64  
    65  delete-branch-mover:
    66  	-@${PACHCTL} delete pipeline branch-mover
    67  
    68  create-branch-mover-no-auth: branch-mover-no-auth.json
    69  	${PACHCTL} create pipeline -f $<
    70  
    71  update-branch-mover-no-auth: branch-mover-no-auth.json
    72  	${PACHCTL} update pipeline -f $<
    73  
    74  delete-branch-mover-no-auth:
    75  	-@${PACHCTL} delete pipeline branch-mover
    76  
    77  delete-secret:
    78  	-@${PACHCTL} delete secret pachyderm-user-secret > /dev/null 2>&1
    79  
    80  clean: delete-branch-mover delete-deferred-processing-cluster delete-secret
    81  	-@rm -f  pachyderm-user-secret.secret 
    82  
    83