github.com/git-lfs/git-lfs@v2.5.2+incompatible/tq/custom.go (about) 1 package tq 2 3 import ( 4 "bufio" 5 "bytes" 6 "encoding/json" 7 "fmt" 8 "io" 9 "path/filepath" 10 "regexp" 11 "strings" 12 "time" 13 14 "github.com/git-lfs/git-lfs/errors" 15 "github.com/git-lfs/git-lfs/fs" 16 "github.com/git-lfs/git-lfs/tools" 17 18 "github.com/git-lfs/git-lfs/subprocess" 19 "github.com/rubyist/tracerx" 20 ) 21 22 // Adapter for custom transfer via external process 23 type customAdapter struct { 24 *adapterBase 25 path string 26 args string 27 concurrent bool 28 originalConcurrency int 29 standalone bool 30 } 31 32 // Struct to capture stderr and write to trace 33 type traceWriter struct { 34 buf bytes.Buffer 35 processName string 36 } 37 38 func (t *traceWriter) Write(b []byte) (int, error) { 39 n, err := t.buf.Write(b) 40 t.Flush() 41 return n, err 42 } 43 func (t *traceWriter) Flush() { 44 var err error 45 for err == nil { 46 var s string 47 s, err = t.buf.ReadString('\n') 48 if len(s) > 0 { 49 tracerx.Printf("xfer[%v]: %v", t.processName, strings.TrimSpace(s)) 50 } 51 } 52 } 53 54 type customAdapterWorkerContext struct { 55 workerNum int 56 cmd *subprocess.Cmd 57 stdout io.ReadCloser 58 bufferedOut *bufio.Reader 59 stdin io.WriteCloser 60 errTracer *traceWriter 61 } 62 63 type customAdapterInitRequest struct { 64 Event string `json:"event"` 65 Operation string `json:"operation"` 66 Remote string `json:"remote"` 67 Concurrent bool `json:"concurrent"` 68 ConcurrentTransfers int `json:"concurrenttransfers"` 69 } 70 71 func NewCustomAdapterInitRequest( 72 op string, remote string, concurrent bool, concurrentTransfers int, 73 ) *customAdapterInitRequest { 74 return &customAdapterInitRequest{"init", op, remote, concurrent, concurrentTransfers} 75 } 76 77 type customAdapterTransferRequest struct { 78 // common between upload/download 79 Event string `json:"event"` 80 Oid string `json:"oid"` 81 Size int64 `json:"size"` 82 Path string `json:"path,omitempty"` 83 Action *Action `json:"action"` 84 } 85 86 func NewCustomAdapterUploadRequest(oid string, size int64, path string, action *Action) *customAdapterTransferRequest { 87 return &customAdapterTransferRequest{"upload", oid, size, path, action} 88 } 89 func NewCustomAdapterDownloadRequest(oid string, size int64, action *Action) *customAdapterTransferRequest { 90 return &customAdapterTransferRequest{"download", oid, size, "", action} 91 } 92 93 type customAdapterTerminateRequest struct { 94 Event string `json:"event"` 95 } 96 97 func NewCustomAdapterTerminateRequest() *customAdapterTerminateRequest { 98 return &customAdapterTerminateRequest{"terminate"} 99 } 100 101 // A common struct that allows all types of response to be identified 102 type customAdapterResponseMessage struct { 103 Event string `json:"event"` 104 Error *ObjectError `json:"error"` 105 Oid string `json:"oid"` 106 Path string `json:"path,omitempty"` // always blank for upload 107 BytesSoFar int64 `json:"bytesSoFar"` 108 BytesSinceLast int `json:"bytesSinceLast"` 109 } 110 111 func (a *customAdapter) Begin(cfg AdapterConfig, cb ProgressCallback) error { 112 a.originalConcurrency = cfg.ConcurrentTransfers() 113 if a.concurrent { 114 // Use common workers impl, but downgrade workers to number of processes 115 return a.adapterBase.Begin(cfg, cb) 116 } 117 118 // If config says not to launch multiple processes, downgrade incoming value 119 return a.adapterBase.Begin(&customAdapterConfig{AdapterConfig: cfg}, cb) 120 } 121 122 func (a *customAdapter) ClearTempStorage() error { 123 // no action requred 124 return nil 125 } 126 127 func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) { 128 // Start a process per worker 129 // If concurrent = false we have already dialled back workers to 1 130 a.Trace("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum) 131 cmd := subprocess.ExecCommand(a.path, a.args) 132 outp, err := cmd.StdoutPipe() 133 if err != nil { 134 return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err) 135 } 136 inp, err := cmd.StdinPipe() 137 if err != nil { 138 return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err) 139 } 140 // Capture stderr to trace 141 tracer := &traceWriter{} 142 tracer.processName = filepath.Base(a.path) 143 cmd.Stderr = tracer 144 err = cmd.Start() 145 if err != nil { 146 return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err) 147 } 148 // Set up buffered reader/writer since we operate on lines 149 ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer} 150 151 // send initiate message 152 initReq := NewCustomAdapterInitRequest( 153 a.getOperationName(), a.remote, a.concurrent, a.originalConcurrency, 154 ) 155 resp, err := a.exchangeMessage(ctx, initReq) 156 if err != nil { 157 a.abortWorkerProcess(ctx) 158 return nil, err 159 } 160 if resp.Error != nil { 161 a.abortWorkerProcess(ctx) 162 return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error) 163 } 164 165 a.Trace("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum) 166 167 // Save this process context and use in future callbacks 168 return ctx, nil 169 } 170 171 func (a *customAdapter) getOperationName() string { 172 if a.direction == Download { 173 return "download" 174 } 175 return "upload" 176 } 177 178 // sendMessage sends a JSON message to the custom adapter process 179 func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error { 180 b, err := json.Marshal(req) 181 if err != nil { 182 return err 183 } 184 a.Trace("xfer: Custom adapter worker %d sending message: %v", ctx.workerNum, string(b)) 185 // Line oriented JSON 186 b = append(b, '\n') 187 _, err = ctx.stdin.Write(b) 188 return err 189 } 190 191 func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext) (*customAdapterResponseMessage, error) { 192 line, err := ctx.bufferedOut.ReadString('\n') 193 if err != nil { 194 return nil, err 195 } 196 a.Trace("xfer: Custom adapter worker %d received response: %v", ctx.workerNum, strings.TrimSpace(line)) 197 resp := &customAdapterResponseMessage{} 198 err = json.Unmarshal([]byte(line), resp) 199 return resp, err 200 } 201 202 // exchangeMessage sends a message to a process and reads a response if resp != nil 203 // Only fatal errors to communicate return an error, errors may be embedded in reply 204 func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req interface{}) (*customAdapterResponseMessage, error) { 205 err := a.sendMessage(ctx, req) 206 if err != nil { 207 return nil, err 208 } 209 return a.readResponse(ctx) 210 } 211 212 // shutdownWorkerProcess terminates gracefully a custom adapter process 213 // returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess) 214 func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error { 215 defer ctx.errTracer.Flush() 216 217 a.Trace("xfer: Shutting down adapter worker %d", ctx.workerNum) 218 219 finishChan := make(chan error, 1) 220 go func() { 221 termReq := NewCustomAdapterTerminateRequest() 222 err := a.sendMessage(ctx, termReq) 223 if err != nil { 224 finishChan <- err 225 } 226 ctx.stdin.Close() 227 ctx.stdout.Close() 228 finishChan <- ctx.cmd.Wait() 229 }() 230 select { 231 case err := <-finishChan: 232 return err 233 case <-time.After(30 * time.Second): 234 return fmt.Errorf("Timeout while shutting down worker process %d", ctx.workerNum) 235 } 236 } 237 238 // abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error 239 func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) { 240 a.Trace("xfer: Aborting worker process: %d", ctx.workerNum) 241 ctx.stdin.Close() 242 ctx.stdout.Close() 243 ctx.cmd.Process.Kill() 244 } 245 func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) { 246 customCtx, ok := ctx.(*customAdapterWorkerContext) 247 if !ok { 248 tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name) 249 return 250 } 251 252 err := a.shutdownWorkerProcess(customCtx) 253 if err != nil { 254 tracerx.Printf("xfer: error finishing up custom transfer process %q worker %d, aborting: %v", a.path, customCtx.workerNum, err) 255 a.abortWorkerProcess(customCtx) 256 } 257 } 258 259 func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error { 260 if ctx == nil { 261 return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name) 262 } 263 264 customCtx, ok := ctx.(*customAdapterWorkerContext) 265 if !ok { 266 return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name) 267 } 268 var authCalled bool 269 270 rel, err := t.Rel(a.getOperationName()) 271 if err != nil { 272 return err 273 } 274 if rel == nil && !a.standalone { 275 return errors.Errorf("Object %s not found on the server.", t.Oid) 276 } 277 var req *customAdapterTransferRequest 278 if a.direction == Upload { 279 req = NewCustomAdapterUploadRequest(t.Oid, t.Size, t.Path, rel) 280 } else { 281 req = NewCustomAdapterDownloadRequest(t.Oid, t.Size, rel) 282 } 283 if err = a.sendMessage(customCtx, req); err != nil { 284 return err 285 } 286 287 // 1..N replies (including progress & one of download / upload) 288 var complete bool 289 for !complete { 290 resp, err := a.readResponse(customCtx) 291 if err != nil { 292 return err 293 } 294 var wasAuthOk bool 295 switch resp.Event { 296 case "progress": 297 // Progress 298 if resp.Oid != t.Oid { 299 return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Oid) 300 } 301 if cb != nil { 302 cb(t.Name, t.Size, resp.BytesSoFar, resp.BytesSinceLast) 303 } 304 wasAuthOk = resp.BytesSoFar > 0 305 case "complete": 306 // Download/Upload complete 307 if resp.Oid != t.Oid { 308 return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Oid) 309 } 310 if resp.Error != nil { 311 return fmt.Errorf("Error transferring %q: %v", t.Oid, resp.Error) 312 } 313 if a.direction == Download { 314 // So we don't have to blindly trust external providers, check SHA 315 if err = tools.VerifyFileHash(t.Oid, resp.Path); err != nil { 316 return fmt.Errorf("Downloaded file failed checks: %v", err) 317 } 318 // Move file to final location 319 if err = tools.RenameFileCopyPermissions(resp.Path, t.Path); err != nil { 320 return fmt.Errorf("Failed to copy downloaded file: %v", err) 321 } 322 } else if a.direction == Upload { 323 if err = verifyUpload(a.apiClient, a.remote, t); err != nil { 324 return err 325 } 326 } 327 wasAuthOk = true 328 complete = true 329 default: 330 return fmt.Errorf("Invalid message %q from custom adapter %q", resp.Event, a.name) 331 } 332 // Fall through from both progress and completion messages 333 // Call auth on first progress or success to free up other workers 334 if wasAuthOk && authOkFunc != nil && !authCalled { 335 authOkFunc() 336 authCalled = true 337 } 338 } 339 340 return nil 341 } 342 343 func newCustomAdapter(f *fs.Filesystem, name string, dir Direction, path, args string, concurrent, standalone bool) *customAdapter { 344 c := &customAdapter{newAdapterBase(f, name, dir, nil), path, args, concurrent, 3, standalone} 345 // self implements impl 346 c.transferImpl = c 347 return c 348 } 349 350 // Initialise custom adapters based on current config 351 func configureCustomAdapters(git Env, m *Manifest) { 352 pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`) 353 for k, _ := range git.All() { 354 match := pathRegex.FindStringSubmatch(k) 355 if match == nil { 356 continue 357 } 358 359 name := match[1] 360 path, _ := git.Get(k) 361 // retrieve other values 362 args, _ := git.Get(fmt.Sprintf("lfs.customtransfer.%s.args", name)) 363 concurrent := git.Bool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true) 364 direction, _ := git.Get(fmt.Sprintf("lfs.customtransfer.%s.direction", name)) 365 if len(direction) == 0 { 366 direction = "both" 367 } else { 368 direction = strings.ToLower(direction) 369 } 370 371 // Separate closure for each since we need to capture vars above 372 newfunc := func(name string, dir Direction) Adapter { 373 standalone := m.standaloneTransferAgent != "" 374 return newCustomAdapter(m.fs, name, dir, path, args, concurrent, standalone) 375 } 376 377 if direction == "download" || direction == "both" { 378 m.RegisterNewAdapterFunc(name, Download, newfunc) 379 } 380 if direction == "upload" || direction == "both" { 381 m.RegisterNewAdapterFunc(name, Upload, newfunc) 382 } 383 } 384 } 385 386 type customAdapterConfig struct { 387 AdapterConfig 388 } 389 390 func (c *customAdapterConfig) ConcurrentTransfers() int { 391 return 1 392 }