github.com/pachyderm/pachyderm@v1.13.4/examples/Makefile.testing (about) 1 # This makefile is for creating clusters filled with repos and pipelines for end-to-end and functional testing. 2 # It builds on targets in the main Makefile. You would invoke it by running the 3 # "cd <source-location>/examples ; make -s -f Makefile.testing dirty-testing-cluster" commands 4 # That target, "dirty-testing-cluster" is the first workhorse testing target that's been developed. 5 # It will create a cluster that has failed jobs, failed commmits, and deleted commits. 6 # Invoking it with "-s" will produce terse, informative output on the steps it's taking to create the 7 # cluster as it's working. You might want to run a "watch -n 2 kubectl get pods" and a 8 # "watch -n 2 pachctl list job" in other windows so that you can understand what's happening as it runs. 9 # At the end, it will print a message telling you the cluster is ready for your testing, and you 10 # can use it to test extract/restore or for whatever other testing purposes you want. 11 # There are some top-level functions defined, like "pipeline-status" and "wait-for-jobs", that make it easier 12 # to write targets that methodically create a cluster that's useful for testing. 13 # The executable paths below are set to generic values. 14 # Modify them for your system by setting environment variables, using a command like this 15 # to both fetch the version of pachyderm you want to test and execute it. 16 # "../etc/fetch_release_pachctl.py 1.10.0 ; env PACHCTL=${GOPATH}/bin/pachctl make -e -f Makefile.testing dirty-testing-cluster)" 17 18 19 include ../etc/govars.mk 20 include Makefile 21 22 SHELL := /bin/bash 23 ECHO ?= echo 24 25 STATS := .enable_stats=true 26 DETECTMODEL := .pipeline.name="object-detect"|.input.cross[1].pfs.repo="object-model"|.input.cross[1].pfs.name="model" 27 DETECTIMAGES := .input.cross[0].pfs.repo="object-images"|.input.cross[0].pfs.name="images" 28 HYPERPSPLIT := .input.cross[0].pfs.repo="hp-split"|.input.cross[0].pfs.name="split" 29 HYPERPMODEL := .input.cross[1].pfs.repo="hp-model"|.input.cross[1].pfs.name="model" 30 HYPERPTEST := .input.cross[0].pfs.repo="hp-test"|.input.cross[0].pfs.name="test" 31 IRISMODEL := .input.pfs.repo="iris-training"|.input.pfs.name="training"|.pipeline.name="iris-model" 32 IRISINFERENCENAME := .pipeline.name="iris-inference" 33 IRISINFERENCEINPUT0 := .input.cross[0].pfs.repo="iris-attributes"|.input.cross[0].pfs.name="attributes" 34 IRISINFERENCEINPUT1 := .input.cross[1].pfs.repo="iris-model"|.input.cross[1].pfs.name="model" 35 36 iris-testing-base: 37 $(PACHCTL) create repo iris-training 38 $(PACHCTL) create repo iris-attributes 39 jq '$(IRISMODEL)' ml/iris/$(IRIS_FLAVOR)_train.json | $(PACHCTL) create pipeline -f - 40 jq '$(IRISINFERENCENAME)|$(IRISINFERENCEINPUT0)|$(IRISINFERENCEINPUT1)' ml/iris/$(IRIS_FLAVOR)_infer.json | \ 41 $(PACHCTL) create pipeline -f - 42 43 opencv-testing: 44 $(PACHCTL) create repo images 45 jq '$(STATS)' opencv/edges.json | $(PACHCTL) create pipeline -f - >/dev/null 46 jq '$(STATS)' opencv/montage.json | $(PACHCTL) create pipeline -f - >/dev/null 47 $(PACHCTL) start transaction 48 $(PACHCTL) start commit images@master 49 $(PACHCTL) finish transaction 50 $(PACHCTL) put file images@master -i opencv/images.txt 51 $(PACHCTL) put file images@master -i opencv/images2.txt 52 $(PACHCTL) put file images@master -i opencv/images3.txt 53 $(PACHCTL) put file images@master -i opencv/images4.txt 54 $(PACHCTL) delete file images@master:/w7RVTsv.jpg 55 $(PACHCTL) finish commit images@master 56 57 opencv-testing-delete: 58 -$(PACHCTL) delete pipeline montage 59 -$(PACHCTL) delete pipeline edges 60 61 62 object-detection-testing-base: OBJECT_IMAGES = $(ECHO) 'The target you selected has object-detection use the same "images" repo as opencv. Hit control-C and then type "make delete" if that is not desired.' 63 object-detection-testing-base: OBJECT_MODEL_PIPELINE = jq '$(STATS)|.pipeline.name="object-model"' ml/object-detection/model.json | $(PACHCTL) create pipeline -f - >/dev/null 64 object-detection-testing-base: OBJECT_DETECT_PIPELINE = jq '$(STATS)|$(DETECTMODEL)' ml/object-detection/detect.json | $(PACHCTL) create pipeline -f - >/dev/null 65 object-detection-testing-base: object-detection-base 66 67 object-detection-testing: object-detection-testing-base 68 $(PACHCTL) put file images@master:airplane.jpg -f ml/object-detection/images/airplane.jpg 69 70 object-detection-testing-delete: 71 -$(PACHCTL) delete pipeline object-detect 72 -$(PACHCTL) delete pipeline object-model 73 -$(PACHCTL) delete repo training 74 75 opencv-object-detection-testing: opencv-testing object-detection-testing 76 opencv-object-detection-testing-delete: object-detection-testing-delete opencv-delete 77 78 hyperparameter-testing-base: hyperparameter-common 79 jq '$(STATS)|.pipeline.name="hp-split"' ml/hyperparameter/split.json | $(PACHCTL) create pipeline -f - >/dev/null 80 jq '$(STATS)|.pipeline.name="hp-model"' ml/hyperparameter/model.json | $(PACHCTL) create pipeline -f - >/dev/null 81 jq '$(STATS)|.pipeline.name="hp-test"|$(HYPERPSPLIT)|$(HYPERPMODEL)' ml/hyperparameter/test.json | $(PACHCTL) create pipeline -f - >/dev/null 82 jq '$(STATS)|.pipeline.name="hp-select"|$(HYPERPTEST)|$(HYPERPMODEL)' ml/hyperparameter/select.json | $(PACHCTL) create pipeline -f - >/dev/null 83 84 hyperparameter-testing: hyperparameter-testing-base hyperparameter-data 85 86 hyperparameter-testing-delete: 87 -$(PACHCTL) delete pipeline hp-select 88 -$(PACHCTL) delete pipeline hp-test 89 -$(PACHCTL) delete pipeline hp-split 90 -$(PACHCTL) delete pipeline hp-model 91 -$(PACHCTL) delete repo parameters 92 -$(PACHCTL) delete repo raw_data 93 94 gatk-testing: gatk 95 $(PACHCTL) start commit samples@master 96 $(PACHCTL) put file samples@master:son/son.bam -f gatk/GATK_Germline/data/bams/son.bam 97 $(PACHCTL) put file samples@master:son/son.bai -f gatk/GATK_Germline/data/bams/son.bai 98 $(PACHCTL) finish commit samples@master 99 100 gatk-testing-delete: gatk-delete 101 102 testing-delete: gatk-testing-delete hyperparameter-testing-delete object-detection-testing-delete opencv-testing-delete 103 -$(PACHCTL) delete repo images 104 105 testing: opencv-testing object-detection-testing hyperparameter-testing gatk-testing 106 107 dirty-testing-cluster: opencv-base ml/object-detection/frozen_inference_graph.pb object-detection-testing-base hyperparameter-testing-base $(GATK_GERMLINE_FILES) gatk-base iris-testing-base 108 until [[ `../etc/examples/pipeline-status.sh edges` = True && \ 109 `../etc/examples/pipeline-status.sh montage` = True && \ 110 `../etc/examples/pipeline-status.sh object-model` = True && \ 111 `../etc/examples/pipeline-status.sh object-detect` = True && \ 112 `../etc/examples/pipeline-status.sh hp-split` = True && \ 113 `../etc/examples/pipeline-status.sh hp-model` = True && \ 114 `../etc/examples/pipeline-status.sh hp-test` = True && \ 115 `../etc/examples/pipeline-status.sh hp-select` = True && \ 116 `../etc/examples/pipeline-status.sh joint_call` = True && \ 117 `../etc/examples/pipeline-status.sh likelihoods` = True ]]; \ 118 do \ 119 $(ECHO) -en "\e[G\e[KWaiting for pipelines to start..."; \ 120 sleep 1; \ 121 done 122 @$(ECHO) -e "\e[G\e[KPipelines started."; 123 @$(ECHO) "Committing data to images@master and training@master in a single transaction..."; 124 $(PACHCTL) start transaction > /dev/null 125 $(PACHCTL) start commit images@master > /dev/null 2>/dev/null 126 $(PACHCTL) start commit training@master > /dev/null 2>/dev/null 127 $(PACHCTL) finish transaction > /dev/null 128 $(PACHCTL) put file training@master:frozen_inference_graph.pb -f ml/object-detection/frozen_inference_graph.pb 129 $(PACHCTL) put file images@master:airplane.jpg -f ml/object-detection/images/airplane.jpg 130 $(PACHCTL) put file images@master -i opencv/images.txt 131 $(PACHCTL) put file images@master -i opencv/images2.txt 132 $(PACHCTL) finish commit images@master > /dev/null 133 $(PACHCTL) finish commit training@master > /dev/null 134 ../etc/examples/wait-for-jobs.sh succeeded-only "First opencv and object-detection jobs done." 135 $(ECHO) -e "Adding stats to edges and montage..."; 136 jq "$(STATS)" opencv/edges.json | $(PACHCTL) update pipeline -f - >/dev/null 137 jq "$(STATS)" opencv/montage.json | $(PACHCTL) update pipeline -f - >/dev/null 138 @$(ECHO) -e "Adding images to images@master that will produce failed jobs..."; 139 $(PACHCTL) put file images@master -i opencv/images3.txt 140 ../etc/examples/wait-for-jobs.sh succeeded-or-failed "Done inserting images into images@master that produce object-detection job failures." 141 @$(ECHO) -e "Modifying object-detection to use new object-images repo and adding images..."; 142 $(PACHCTL) create repo object-images 143 jq '$(STATS)|$(DETECTMODEL)|$(DETECTIMAGES)' ml/object-detection/detect.json | $(PACHCTL) update pipeline -f - >/dev/null 144 $(PACHCTL) put file images@master -i opencv/images4.txt 145 $(PACHCTL) put file object-images@master:dogs.jpg -f ml/object-detection/images/dogs.jpg 146 $(PACHCTL) put file object-images@master:airplane.jpg -f ml/object-detection/images/airplane.jpg 147 ../etc/examples/wait-for-jobs.sh succeeded-only "Done adding new images in object-images@master and images@master." 148 @$(ECHO) -e "Adding data to gatk..."; 149 $(PACHCTL) start transaction > /dev/null 150 $(PACHCTL) start commit reference@master > /dev/null 2>/dev/null 151 $(PACHCTL) start commit samples@master >/dev/null 2>/dev/null 152 $(PACHCTL) finish transaction > /dev/null 153 $(PACHCTL) put file reference@master:ref.dict -f gatk/GATK_Germline/data/ref/ref.dict 154 $(PACHCTL) put file reference@master:ref.fasta -f gatk/GATK_Germline/data/ref/ref.fasta 155 $(PACHCTL) put file reference@master:ref.fasta.fai -f gatk/GATK_Germline/data/ref/ref.fasta.fai 156 $(PACHCTL) put file reference@master:refSDF -r -f gatk/GATK_Germline/data/ref/refSDF 157 $(PACHCTL) put file samples@master:mother/mother.bam -f gatk/GATK_Germline/data/bams/mother.bam 158 $(PACHCTL) put file samples@master:mother/mother.bai -f gatk/GATK_Germline/data/bams/mother.bai 159 $(PACHCTL) finish commit reference@master > /dev/null 160 $(PACHCTL) finish commit samples@master > /dev/null 161 ../etc/examples/wait-for-jobs.sh succeeded-only "Done adding initial gatk data in references and master." 162 if GATK_FATHER_COMMIT=`$(PACHCTL) start commit samples@master` && \ 163 [ -n $${GATK_FATHER_COMMIT} ] ; \ 164 then \ 165 $(ECHO) -e "Adding father to gatk, samples@$${GATK_FATHER_COMMIT}, we will delete his data later..." ; \ 166 $(PACHCTL) put file samples@master:father/father.bam -f gatk/GATK_Germline/data/bams/father.bam ; \ 167 $(PACHCTL) put file samples@master:father/father.bai -f gatk/GATK_Germline/data/bams/father.bai ; \ 168 $(PACHCTL) finish commit samples@master > /dev/null ; \ 169 GATK_SON_COMMIT=`$(PACHCTL) start commit samples@master` ; \ 170 $(ECHO) -e "Adding son, samples@$${GATK_SON_COMMIT}..." ; \ 171 $(PACHCTL) put file samples@master:son/son.bam -f gatk/GATK_Germline/data/bams/son.bam ; \ 172 $(PACHCTL) put file samples@master:son/son.bai -f gatk/GATK_Germline/data/bams/son.bai ; \ 173 $(PACHCTL) finish commit samples@master > /dev/null ; \ 174 $(ECHO) -e "Deleting father commit, samples@$${GATK_FATHER_COMMIT}..." ; \ 175 $(PACHCTL) delete commit samples@$${GATK_FATHER_COMMIT} ; \ 176 else \ 177 $(ECHO) -e "Error: could not create commit in samples@master." ; \ 178 exit 1 ; \ 179 fi 180 $(ECHO) -e "Adding data to iris." 181 $(PACHCTL) start transaction 182 $(PACHCTL) start commit iris-training@master 183 $(PACHCTL) start commit iris-attributes@master 184 $(PACHCTL) finish transaction 185 $(PACHCTL) put file iris-training@master:iris.csv -f ml/iris/data/iris.csv 186 $(PACHCTL) finish commit iris-training@master 187 ../etc/examples/wait-for-jobs.sh succeeded-only "Done training iris model." 188 $(PACHCTL) put file iris-attributes@master:1.csv -f ml/iris/data/test/1.csv 189 $(PACHCTL) finish commit iris-attributes@master 190 $(PACHCTL) put file iris-attributes@master:2.csv -f ml/iris/data/test/2.csv 191 ../etc/examples/wait-for-jobs.sh succeeded-only "Done adding iris data." 192 $(ECHO) -e "All jobs finished. Inspect output above to determine if cluster ready for testing." 193 194 195 196 197