github.com/mhlo/force@v0.22.28-0.20150915022417-6d05ecfb0b47/datapipe.go (about) 1 package main 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 ) 8 9 var cmdDataPipe = &Command{ 10 Usage: "datapipe <command> [<args>]", 11 Short: "Manage DataPipes", 12 Long: ` 13 Manage DataPipes 14 15 Usage: 16 17 force datapipe create -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion] 18 19 force datapipe update -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion] 20 21 force datapipe delete -n <name> 22 23 force datapipe list -f <"csv" or "json"> 24 25 force datapipe query -q <query string> 26 27 force datapipe createjob -n <pipeline name> 28 29 Commands: 30 create creates a new dataPipe 31 update update a dataPipe 32 delete delete a datapipe 33 list list all datapipes 34 query query for a specific datapipe(s) 35 createjob creates a new job for a specific datapipe 36 listjobs list the status of submitted jobs 37 queryjob returns data about a datapipeline job (not implemented) 38 retrieve (not implemented) 39 40 Examples: 41 42 force datapipe create -n=MyPipe -l="My Pipe" -t=Pig -v=34.0 \ 43 -c="A = load 'force://soql/Select Id, Name From Contact' using \ 44 gridforce.hadoop.pig.loadstore.func.ForceStorage();" 45 46 Defaults 47 -l Defaults to the name 48 -t Pig (only option available currently) 49 -c Pig script template 50 -v Current API version *Number only 51 52 `, 53 } 54 55 var defaultContent = ` 56 -- Sample script for a data pipeline 57 A = load 'ffx://REPLACE_ME' using gridforce.hadoop.pig.loadstore.func.ForceStorage(); 58 Store A into 'ffx://REPLACE_ME_TOO' using gridforce.hadoop.pig.loadstore.func.ForceStorage(); 59 ` 60 61 var ( 62 dpname string 63 masterlabel string 64 scriptcontent string 65 apiversion string 66 scripttype string 67 query string 68 format string 69 ) 70 71 func init() { 72 cmdDataPipe.Flag.StringVar(&dpname, "name", "", "set datapipeline name") 73 cmdDataPipe.Flag.StringVar(&dpname, "n", "", "set datapipeline name") 74 cmdDataPipe.Flag.StringVar(&masterlabel, "masterlabel", "", "set master label") 75 cmdDataPipe.Flag.StringVar(&masterlabel, "l", "", "set master label") 76 cmdDataPipe.Flag.StringVar(&scriptcontent, "scriptcontent", defaultContent, "set script content") 77 cmdDataPipe.Flag.StringVar(&scriptcontent, "c", defaultContent, "set script content") 78 cmdDataPipe.Flag.StringVar(&apiversion, "apiversion", apiVersionNumber, "set api version") 79 cmdDataPipe.Flag.StringVar(&apiversion, "v", apiVersionNumber, "set api version") 80 cmdDataPipe.Flag.StringVar(&scripttype, "scripttype", "Pig", "set script type") 81 cmdDataPipe.Flag.StringVar(&scripttype, "t", "Pig", "set script type") 82 cmdDataPipe.Flag.StringVar(&query, "q", "", "SOQL query string on DataPipeline object") 83 cmdDataPipe.Flag.StringVar(&query, "query", "", "SOQL query string on DataPipeline object") 84 cmdDataPipe.Flag.StringVar(&format, "f", "json", "format for listing datapipelines (csv or json)") 85 cmdDataPipe.Flag.StringVar(&format, "format", "json", "format for listing datapipelines (csv or json)") 86 cmdDataPipe.Run = runDataPipe 87 } 88 89 func runDataPipe(cmd *Command, args []string) { 90 if len(args) == 0 { 91 cmd.printUsage() 92 } else { 93 if err := cmd.Flag.Parse(args[1:]); err != nil { 94 os.Exit(2) 95 } 96 switch args[0] { 97 case "create": 98 runDataPipelineCreate() 99 case "update": 100 runDataPipelineUpdate() 101 case "delete": 102 runDataPipelineDelete() 103 case "list": 104 runDataPipelineList() 105 case "query": 106 runDataPipelineQuery() 107 case "createjob": 108 runDataPipelineJob() 109 case "listjobs": 110 runDataPipelineListJobs() 111 default: 112 ErrorAndExit("no such command: %s", args[0]) 113 } 114 } 115 } 116 117 func runDataPipelineJob() { 118 if len(dpname) == 0 { 119 ErrorAndExit("You need to provide the name of a pipeline to create a job for.") 120 } 121 force, _ := ActiveForce() 122 id := GetDataPipelineId(dpname) 123 _, err, _ := force.CreateDataPipelineJob(id) 124 if err != nil { 125 ErrorAndExit(err.Error()) 126 } 127 fmt.Printf("Successfully created DataPipeline job for %s\n", dpname) 128 } 129 130 func runDataPipelineListJobs() { 131 //query = "SELECT Id, DataPipeline.DeveloperName, Status, FailureState, LastModifiedDate, CreatedDate, CreatedById, DataPipelineId, JobErrorMessage FROM DataPipelineJob" 132 query = "SELECT DataPipeline.DeveloperName, Status, FailureState FROM DataPipelineJob" 133 force, _ := ActiveForce() 134 result, err := force.QueryDataPipelineJob(query) 135 if err != nil { 136 ErrorAndExit(err.Error()) 137 } 138 139 DisplayForceRecordsf(result.Records, "csv") 140 } 141 142 func runDataPipelineQuery() { 143 if len(query) == 0 { 144 ErrorAndExit("You have to supply a SOQL query using the -q flag.") 145 } 146 force, _ := ActiveForce() 147 result, err := force.QueryDataPipeline(query) 148 if err != nil { 149 ErrorAndExit(err.Error()) 150 } 151 152 fmt.Println("Result: \n", result) 153 } 154 155 func runDataPipelineCreate() { 156 if len(dpname) == 0 { 157 ErrorAndExit("You must specify a name for the datapipeline using the -n flag.") 158 } 159 if len(masterlabel) == 0 { 160 masterlabel = dpname 161 } 162 163 force, _ := ActiveForce() 164 _, err, _ := force.CreateDataPipeline(dpname, masterlabel, apiversion, scriptcontent, scripttype) 165 if err != nil { 166 ErrorAndExit(err.Error()) 167 } 168 fmt.Printf("DataPipeline %s successfully created.\n", dpname) 169 } 170 171 func runDataPipelineUpdate() { 172 if len(dpname) == 0 { 173 ErrorAndExit("You must specify a name for the datapipeline using the -n flag.") 174 } 175 if len(masterlabel) == 0 && len(scriptcontent) == 0 { 176 ErrorAndExit("You can change the master label or the script content.") 177 } 178 179 force, _ := ActiveForce() 180 181 result, err := force.GetDataPipeline(dpname) 182 if err != nil { 183 ErrorAndExit(err.Error()) 184 } 185 186 if len(result.Records) == 0 { 187 ErrorAndExit("No data pipeline found named " + dpname) 188 } 189 for _, record := range result.Records { 190 var id string 191 id = record["Id"].(string) 192 if len(masterlabel) == 0 { 193 masterlabel = record["MasterLabel"].(string) 194 } 195 if len(scriptcontent) == 0 { 196 scriptcontent = record["ScriptContent"].(string) 197 } 198 if _, err := os.Stat(scriptcontent); err == nil { 199 fmt.Printf("file exists; processing...") 200 scriptcontent, err = readScriptFile(scriptcontent) 201 if err != nil { 202 ErrorAndExit(err.Error()) 203 } 204 } 205 err = force.UpdateDataPipeline(id, masterlabel, scriptcontent) 206 if err != nil { 207 ErrorAndExit(err.Error()) 208 } 209 fmt.Printf("%s successfully updated.\n", dpname) 210 } 211 } 212 213 func readScriptFile(path string) (content string, err error) { 214 data, err := ioutil.ReadFile(path) 215 content = string(data) 216 return 217 } 218 219 func GetDataPipelineId(name string) (id string) { 220 force, _ := ActiveForce() 221 222 result, err := force.GetDataPipeline(dpname) 223 if err != nil { 224 ErrorAndExit(err.Error()) 225 } 226 227 if len(result.Records) == 0 { 228 ErrorAndExit("No data pipeline found named " + dpname) 229 } 230 231 record := result.Records[0] 232 id = record["Id"].(string) 233 return 234 } 235 236 func runDataPipelineDelete() { 237 force, _ := ActiveForce() 238 239 id := GetDataPipelineId(dpname) 240 err := force.DeleteDataPipeline(id) 241 if err != nil { 242 ErrorAndExit(err.Error()) 243 } 244 fmt.Printf("%s successfully deleted.\n", dpname) 245 } 246 247 func runDataPipelineList() { 248 force, _ := ActiveForce() 249 query = "SELECT Id, MasterLabel, DeveloperName, ScriptType FROM DataPipeline" 250 result, err := force.QueryDataPipeline(query) 251 if err != nil { 252 ErrorAndExit(err.Error()) 253 } 254 255 DisplayForceRecordsf(result.Records, format) 256 }