github.com/pachyderm/pachyderm@v1.13.4/examples/Makefile (about) 1 # The executable paths below are set to generic values. 2 # Modify them for your system by setting environment variables, using a command like this 3 # to both fetch the version of pachyderm you want to use and execute it. 4 # "../etc/fetch_release_pachctl.py 1.10.0 ; env PACHCTL=${GOPATH}/bin/pachctl make -e opencv" 5 6 include ../etc/govars.mk 7 8 SHELL := /bin/bash 9 10 # kubectl is called by scripts, referenced here to make clear you can override default 11 export KUBECTL ?= kubectl 12 13 # This controls which version of iris is deployed. 14 # acceptable values are python, julia, rstats 15 # invoke another flavor like this: 16 # env IRIS_FLAVOR=julia make -e iris 17 IRIS_FLAVOR := python 18 19 GATK_GERMLINE_FILES = gatk/GATK_Germline/data/ref/ref.dict \ 20 gatk/GATK_Germline/data/ref/ref.fasta \ 21 gatk/GATK_Germline/data/ref/ref.fasta.fai \ 22 gatk/GATK_Germline/data/ref/refSDF/done \ 23 gatk/GATK_Germline/data/ref/refSDF/format.log \ 24 gatk/GATK_Germline/data/ref/refSDF/mainIndex \ 25 gatk/GATK_Germline/data/ref/refSDF/namedata0 \ 26 gatk/GATK_Germline/data/ref/refSDF/nameIndex0 \ 27 gatk/GATK_Germline/data/ref/refSDF/namepointer0 \ 28 gatk/GATK_Germline/data/ref/refSDF/progress \ 29 gatk/GATK_Germline/data/ref/refSDF/seqdata0 \ 30 gatk/GATK_Germline/data/ref/refSDF/seqpointer0 \ 31 gatk/GATK_Germline/data/ref/refSDF/sequenceIndex0 \ 32 gatk/GATK_Germline/data/ref/refSDF/summary.txt \ 33 gatk/GATK_Germline/data/bams/father.bai \ 34 gatk/GATK_Germline/data/bams/father.bam \ 35 gatk/GATK_Germline/data/bams/mother.bai \ 36 gatk/GATK_Germline/data/bams/mother.bam \ 37 gatk/GATK_Germline/data/bams/motherICE.bai \ 38 gatk/GATK_Germline/data/bams/motherICE.bam \ 39 gatk/GATK_Germline/data/bams/motherNEX.bai \ 40 gatk/GATK_Germline/data/bams/motherNEX.bam \ 41 gatk/GATK_Germline/data/bams/motherRnaseq.bai \ 42 gatk/GATK_Germline/data/bams/motherRnaseq.bam \ 43 gatk/GATK_Germline/data/bams/motherRnaseqPP.bai \ 44 gatk/GATK_Germline/data/bams/motherRnaseqPP.bam \ 45 gatk/GATK_Germline/data/bams/son.bai \ 46 gatk/GATK_Germline/data/bams/son.bam 47 48 iris-base: 49 $(PACHCTL) create repo training 50 $(PACHCTL) create repo attributes 51 $(PACHCTL) create pipeline -f ml/iris/$(IRIS_FLAVOR)_train.json 52 $(PACHCTL) create pipeline -f ml/iris/$(IRIS_FLAVOR)_infer.json 53 54 iris: iris-base 55 $(PACHCTL) start transaction 56 $(PACHCTL) start commit training@master 57 $(PACHCTL) start commit attributes@master 58 $(PACHCTL) finish transaction 59 $(PACHCTL) put file training@master:iris.csv -f ml/iris/data/iris.csv 60 $(PACHCTL) finish commit training@master 61 $(PACHCTL) put file attributes@master:1.csv -f ml/iris/data/test/1.csv 62 $(PACHCTL) finish commit attributes@master 63 $(PACHCTL) put file attributes@master:2.csv -f ml/iris/data/test/2.csv 64 65 opencv-base: 66 $(PACHCTL) create repo images 67 $(PACHCTL) create pipeline -f opencv/edges.json 68 $(PACHCTL) create pipeline -f opencv/montage.json 69 70 opencv: opencv-base 71 $(PACHCTL) put file images@master -i opencv/images.txt 72 $(PACHCTL) put file images@master -i opencv/images2.txt 73 74 opencv-delete: 75 $(PACHCTL) delete pipeline montage 76 $(PACHCTL) delete pipeline edges 77 $(PACHCTL) delete repo images 78 79 ml/object-detection/frozen_inference_graph.pb: 80 wget -p http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_11_06_2017.tar.gz --directory-prefix /tmp 81 tar -C ml/object-detection --strip-components 1 -xvf /tmp/download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_11_06_2017.tar.gz ssd_mobilenet_v1_coco_11_06_2017/frozen_inference_graph.pb 82 83 # This target is not intended to be invoked directly, it's intended for other targets to be built on. 84 # OBJECT_IMAGES is set to no operation by default, so this pipeline can use the images repo of 85 # opencv when used for testing. If invoked from the intended object-detection target, below, 86 # a separate images repo will be created for object-detection. 87 OBJECT_MODEL_PIPELINE = $(PACHCTL) create pipeline -f ml/object-detection/model.json 88 OBJECT_DETECT_PIPELINE = $(PACHCTL) create pipeline -f ml/object-detection/detect.json 89 object-detection-base: 90 @$(OBJECT_IMAGES) 91 $(PACHCTL) create repo training 92 @$(OBJECT_MODEL_PIPELINE) 93 @$(OBJECT_DETECT_PIPELINE) 94 95 object-detection-data: ml/object-detection/frozen_inference_graph.pb 96 $(PACHCTL) put file training@master:frozen_inference_graph.pb -f ml/object-detection/frozen_inference_graph.pb 97 $(PACHCTL) put file images@master:dogs.jpg -f ml/object-detection/images/dogs.jpg 98 99 object-detection: OBJECT_IMAGES = $(PACHCTL) create repo images 100 object-detection: object-detection-base object-detection-data 101 102 object-detection-delete: object-detection-delete 103 -$(PACHCTL) delete pipeline detect 104 -$(PACHCTL) delete pipeline model 105 -$(PACHCTL) delete repo training 106 -$(PACHCTL) delete repo images 107 108 housing-prices-base: 109 $(PACHCTL) create repo housing_data 110 $(PACHCTL) create pipeline -f ml/housing-prices/regression.json 111 112 housing-prices-1: 113 $(PACHCTL) put file housing_data@master:housing-simplified.csv -f ml/housing-prices/data/housing-simplified-1.csv 114 115 housing-prices-2: 116 $(PACHCTL) put file housing_data@master:housing-simplified.csv -f ml/housing-prices/data/housing-simplified-2.csv --overwrite 117 118 housing-prices-delete: 119 $(PACHCTL) delete pipeline regression 120 $(PACHCTL) delete repo housing_data 121 122 hyperparameter-common: 123 $(PACHCTL) create repo raw_data 124 $(PACHCTL) create repo parameters 125 126 hyperparameter-base: hyperparameter-common 127 $(PACHCTL) create pipeline -f ml/hyperparameter/split.json 128 $(PACHCTL) create pipeline -f ml/hyperparameter/model.json 129 $(PACHCTL) create pipeline -f ml/hyperparameter/test.json 130 $(PACHCTL) create pipeline -f ml/hyperparameter/select.json 131 132 hyperparameter-data: 133 $(PACHCTL) start transaction 134 $(PACHCTL) start commit raw_data@master 135 $(PACHCTL) start commit parameters@master 136 $(PACHCTL) finish transaction 137 $(PACHCTL) put file raw_data@master:iris.csv -f ml/hyperparameter/data/noisy_iris.csv 138 $(PACHCTL) put file parameters@master:c_parameters.txt -f ml/hyperparameter/data/parameters/c_parameters.txt --split line --target-file-datums 1 139 $(PACHCTL) put file parameters@master:gamma_parameters.txt -f ml/hyperparameter/data/parameters/gamma_parameters.txt --split line --target-file-datums 1 140 $(PACHCTL) finish commit raw_data@master 141 $(PACHCTL) finish commit parameters@master 142 143 hyperparameter: hyperparameter-common hyperparameter-base hyperparameter-data 144 145 hyperparameter-delete: 146 -$(PACHCTL) delete pipeline select 147 -$(PACHCTL) delete pipeline test 148 -$(PACHCTL) delete pipeline split 149 -$(PACHCTL) delete pipeline model 150 -$(PACHCTL) delete repo parameters 151 -$(PACHCTL) delete repo raw_data 152 153 $(GATK_GERMLINE_FILES): 154 mkdir -p gatk/GATK_Germline 155 wget -p https://s3-us-west-1.amazonaws.com/pachyderm.io/Examples_Data_Repo/GATK_Germline.zip --directory-prefix=/tmp 156 unzip -o /tmp/s3-us-west-1.amazonaws.com/pachyderm.io/Examples_Data_Repo/GATK_Germline.zip data/ref/ref.dict data/ref/ref.fasta data/ref/ref.fasta.fai data/ref/refSDF/* data/bams/* -d gatk/GATK_Germline 157 158 gatk-base: 159 $(PACHCTL) create repo reference 160 $(PACHCTL) create repo samples 161 $(PACHCTL) create pipeline -f gatk/likelihoods.json 162 $(PACHCTL) create pipeline -f gatk/joint-call.json 163 164 gatk-data: $(GATK_GERMLINE_FILES) 165 $(PACHCTL) start transaction 166 $(PACHCTL) start commit reference@master 167 $(PACHCTL) start commit samples@master 168 $(PACHCTL) finish transaction 169 $(PACHCTL) put file reference@master:ref.dict -f gatk/GATK_Germline/data/ref/ref.dict 170 $(PACHCTL) put file reference@master:ref.fasta -f gatk/GATK_Germline/data/ref/ref.fasta 171 $(PACHCTL) put file reference@master:ref.fasta.fai -f gatk/GATK_Germline/data/ref/ref.fasta.fai 172 $(PACHCTL) put file reference@master:refSDF -r -f gatk/GATK_Germline/data/ref/refSDF 173 $(PACHCTL) put file samples@master:mother/mother.bam -f gatk/GATK_Germline/data/bams/mother.bam 174 $(PACHCTL) put file samples@master:mother/mother.bai -f gatk/GATK_Germline/data/bams/mother.bai 175 $(PACHCTL) put file samples@master:father/father.bam -f gatk/GATK_Germline/data/bams/father.bam 176 $(PACHCTL) put file samples@master:father/father.bai -f gatk/GATK_Germline/data/bams/father.bai 177 $(PACHCTL) finish commit reference@master 178 $(PACHCTL) finish commit samples@master 179 180 gatk: gatk-base gatk-data 181 182 gatk-delete: 183 -$(PACHCTL) delete pipeline joint_call 184 -$(PACHCTL) delete pipeline likelihoods 185 -$(PACHCTL) delete repo reference 186 -$(PACHCTL) delete repo samples 187 188 delete: 189 yes | $(PACHCTL) delete all 190 191 clean: delete 192 rm -fr gatk/GATK_Germline 193 rm -f ml/object-detection/frozen_inference_graph.pb 194 195 minikube:echo hi 196 minikube start 197 @WHEEL="-\|/"; \ 198 until minikube ip 2>/dev/null; do \ 199 WHEEL=$${WHEEL:1}$${WHEEL:0:1}; \ 200 echo -en "\e[G\e[K$${WHEEL:0:1} waiting for minikube to start..."; \ 201 sleep 1; \ 202 done 203 $(PACHCTL) deploy local 204 @until "$$( 205 $(KUBECTL) get po \ 206 -l suite=pachyderm,app=dash \ 207 -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}')" = True; \ 208 do \ 209 WHEEL=$${WHEEL:1}$${WHEEL:0:1}; \ 210 $(ECHO) -en "\e[G\e[K$${WHEEL:0:1} waiting for pachyderm to start..."; \ 211 sleep 1; \ 212 done 213 pachctl port-forward &