src.elv.sh@v0.21.0-dev.0.20240515223629-06979efb9a2a/pkg/eval/compile_effect.go (about) 1 package eval 2 3 import ( 4 "context" 5 "fmt" 6 "os" 7 "sync" 8 "sync/atomic" 9 10 "src.elv.sh/pkg/diag" 11 "src.elv.sh/pkg/eval/errs" 12 "src.elv.sh/pkg/eval/vals" 13 "src.elv.sh/pkg/eval/vars" 14 "src.elv.sh/pkg/fsutil" 15 "src.elv.sh/pkg/parse" 16 "src.elv.sh/pkg/parse/cmpd" 17 ) 18 19 // An operation with some side effects. 20 type effectOp interface{ exec(*Frame) Exception } 21 22 func (cp *compiler) chunkOp(n *parse.Chunk) effectOp { 23 return chunkOp{n.Range(), cp.pipelineOps(n.Pipelines)} 24 } 25 26 type chunkOp struct { 27 diag.Ranging 28 subops []effectOp 29 } 30 31 func (op chunkOp) exec(fm *Frame) Exception { 32 for _, subop := range op.subops { 33 exc := subop.exec(fm) 34 if exc != nil { 35 return exc 36 } 37 } 38 // Check for interrupts after the chunk. 39 // We also check for interrupts before each pipeline, so there is no 40 // need to check it before the chunk or after each pipeline. 41 if fm.Canceled() { 42 return fm.errorp(op, ErrInterrupted) 43 } 44 return nil 45 } 46 47 func (cp *compiler) pipelineOp(n *parse.Pipeline) effectOp { 48 formOps := cp.formOps(n.Forms) 49 50 return &pipelineOp{n.Range(), n.Background, parse.SourceText(n), formOps} 51 } 52 53 func (cp *compiler) pipelineOps(ns []*parse.Pipeline) []effectOp { 54 ops := make([]effectOp, len(ns)) 55 for i, n := range ns { 56 ops[i] = cp.pipelineOp(n) 57 } 58 return ops 59 } 60 61 type pipelineOp struct { 62 diag.Ranging 63 bg bool 64 source string 65 subops []effectOp 66 } 67 68 const pipelineChanBufferSize = 32 69 70 func (op *pipelineOp) exec(fm *Frame) Exception { 71 if fm.Canceled() { 72 return fm.errorp(op, ErrInterrupted) 73 } 74 75 if op.bg { 76 fm = fm.Fork("background job" + op.source) 77 fm.ctx = context.Background() 78 fm.background = true 79 fm.Evaler.addNumBgJobs(1) 80 } 81 82 nforms := len(op.subops) 83 84 var wg sync.WaitGroup 85 wg.Add(nforms) 86 excs := make([]Exception, nforms) 87 88 var nextIn *Port 89 90 // For each form, create a dedicated evalCtx and run asynchronously 91 for i, formOp := range op.subops { 92 newFm := fm.Fork("[form op]") 93 inputIsPipe := i > 0 94 outputIsPipe := i < nforms-1 95 if inputIsPipe { 96 newFm.ports[0] = nextIn 97 } 98 if outputIsPipe { 99 // Each internal port pair consists of a (byte) pipe pair and a 100 // channel. 101 // os.Pipe sets O_CLOEXEC, which is what we want. 102 reader, writer, e := os.Pipe() 103 if e != nil { 104 return fm.errorpf(op, "failed to create pipe: %s", e) 105 } 106 ch := make(chan any, pipelineChanBufferSize) 107 sendStop := make(chan struct{}) 108 sendError := new(error) 109 readerGone := new(atomic.Bool) 110 newFm.ports[1] = &Port{ 111 File: writer, Chan: ch, 112 closeFile: true, closeChan: true, 113 sendStop: sendStop, sendError: sendError, readerGone: readerGone} 114 nextIn = &Port{ 115 File: reader, Chan: ch, 116 closeFile: true, closeChan: false, 117 // Store in input port for ease of retrieval later 118 sendStop: sendStop, sendError: sendError, readerGone: readerGone} 119 } 120 f := func(formOp effectOp, pexc *Exception) { 121 exc := formOp.exec(newFm) 122 if exc != nil && !(outputIsPipe && isReaderGone(exc)) { 123 *pexc = exc 124 } 125 if inputIsPipe { 126 input := newFm.ports[0] 127 *input.sendError = errs.ReaderGone{} 128 close(input.sendStop) 129 input.readerGone.Store(true) 130 } 131 newFm.Close() 132 wg.Done() 133 } 134 if i == nforms-1 && !op.bg { 135 f(formOp, &excs[i]) 136 } else { 137 go f(formOp, &excs[i]) 138 } 139 } 140 141 if op.bg { 142 // Background job, wait for form termination asynchronously. 143 go func() { 144 wg.Wait() 145 fm.Evaler.addNumBgJobs(-1) 146 if notify := fm.Evaler.BgJobNotify; notify != nil { 147 msg := "job " + op.source + " finished" 148 err := MakePipelineError(excs) 149 if err != nil { 150 msg += ", errors = " + err.Error() 151 } 152 if fm.Evaler.getNotifyBgJobSuccess() || err != nil { 153 notify(msg) 154 } 155 } 156 }() 157 return nil 158 } 159 wg.Wait() 160 return fm.errorp(op, MakePipelineError(excs)) 161 } 162 163 func isReaderGone(exc Exception) bool { 164 _, ok := exc.Reason().(errs.ReaderGone) 165 return ok 166 } 167 168 func (cp *compiler) formOp(n *parse.Form) effectOp { 169 var tempLValues []lvalue 170 var assignmentOps []effectOp 171 if len(n.Assignments) > 0 { 172 if n.Head == nil { 173 cp.errorpf(n, `using the syntax of temporary assignment for non-temporary assignment is no longer supported; use "var" or "set" instead`) 174 return nopOp{} 175 } else { 176 as := n.Assignments 177 cp.deprecate(diag.MixedRanging(as[0], as[len(as)-1]), 178 `the legacy temporary assignment syntax is deprecated; use "tmp" instead`, 18) 179 } 180 assignmentOps = cp.assignmentOps(n.Assignments) 181 for _, a := range n.Assignments { 182 lvalues := cp.parseIndexingLValue(a.Left, setLValue|newLValue) 183 tempLValues = append(tempLValues, lvalues.lvalues...) 184 } 185 logger.Println("temporary assignment of", len(n.Assignments), "pairs") 186 } 187 188 redirOps := cp.redirOps(n.Redirs) 189 body := cp.formBody(n) 190 191 return &formOp{n.Range(), tempLValues, assignmentOps, redirOps, body} 192 } 193 194 func (cp *compiler) formBody(n *parse.Form) formBody { 195 if n.Head == nil { 196 // Compiling an incomplete form node, return an empty body. 197 return formBody{} 198 } 199 200 // Determine if this form is a special command. 201 if head, ok := cmpd.StringLiteral(n.Head); ok { 202 special, _ := resolveCmdHeadInternally(cp, head, n.Head) 203 if special != nil { 204 specialOp := special(cp, n) 205 return formBody{specialOp: specialOp} 206 } 207 } 208 209 var headOp valuesOp 210 if head, ok := cmpd.StringLiteral(n.Head); ok { 211 // Head is a literal string: resolve to function or external (special 212 // commands are already handled above). 213 if _, fnRef := resolveCmdHeadInternally(cp, head, n.Head); fnRef != nil { 214 headOp = variableOp{n.Head.Range(), false, head + FnSuffix, fnRef} 215 } else { 216 cp.autofixUnresolvedVar(head + FnSuffix) 217 if cp.currentPragma().unknownCommandIsExternal || fsutil.DontSearch(head) { 218 headOp = literalValues(n.Head, NewExternalCmd(head)) 219 } else { 220 cp.errorpf(n.Head, "unknown command disallowed by current pragma") 221 } 222 } 223 } else { 224 // Head is not a literal string: evaluate as a normal expression. 225 headOp = cp.compoundOp(n.Head) 226 } 227 228 argOps := cp.compoundOps(n.Args) 229 optsOp := cp.mapPairs(n.Opts) 230 return formBody{ordinaryCmd: ordinaryCmd{headOp, argOps, optsOp}} 231 } 232 233 func (cp *compiler) formOps(ns []*parse.Form) []effectOp { 234 ops := make([]effectOp, len(ns)) 235 for i, n := range ns { 236 ops[i] = cp.formOp(n) 237 } 238 return ops 239 } 240 241 type formOp struct { 242 diag.Ranging 243 tempLValues []lvalue 244 tempAssignOps []effectOp 245 redirOps []effectOp 246 body formBody 247 } 248 249 type formBody struct { 250 // Exactly one field will be populated. 251 specialOp effectOp 252 assignOp effectOp 253 ordinaryCmd ordinaryCmd 254 } 255 256 type ordinaryCmd struct { 257 headOp valuesOp 258 argOps []valuesOp 259 optsOp *mapPairsOp 260 } 261 262 func (op *formOp) exec(fm *Frame) (errRet Exception) { 263 // fm here is always a sub-frame created in compiler.pipeline, so it can 264 // be safely modified. 265 266 // Temporary assignment. 267 if len(op.tempLValues) > 0 { 268 // There is a temporary assignment. 269 // Save variables. 270 var saveVars []vars.Var 271 var saveVals []any 272 for _, lv := range op.tempLValues { 273 variable, err := derefLValue(fm, lv) 274 if err != nil { 275 return fm.errorp(op, err) 276 } 277 saveVars = append(saveVars, variable) 278 } 279 for i, v := range saveVars { 280 // TODO(xiaq): If the variable to save is a elemVariable, save 281 // the outermost variable instead. 282 if u := vars.HeadOfElement(v); u != nil { 283 v = u 284 saveVars[i] = v 285 } 286 val := v.Get() 287 saveVals = append(saveVals, val) 288 logger.Printf("saved %s = %s", v, val) 289 } 290 // Do assignment. 291 for _, subop := range op.tempAssignOps { 292 exc := subop.exec(fm) 293 if exc != nil { 294 return exc 295 } 296 } 297 // Defer variable restoration. Will be executed even if an error 298 // occurs when evaling other part of the form. 299 defer func() { 300 for i, v := range saveVars { 301 val := saveVals[i] 302 if val == nil { 303 // TODO(xiaq): Old value is nonexistent. We should delete 304 // the variable. However, since the compiler now doesn't 305 // delete it, we don't delete it in the evaler either. 306 val = "" 307 } 308 err := v.Set(val) 309 if err != nil { 310 errRet = fm.errorp(op, err) 311 } 312 logger.Printf("restored %s = %s", v, val) 313 } 314 }() 315 } 316 317 // Redirections. 318 for _, redirOp := range op.redirOps { 319 exc := redirOp.exec(fm) 320 if exc != nil { 321 return exc 322 } 323 } 324 325 if op.body.specialOp != nil { 326 return op.body.specialOp.exec(fm) 327 } 328 if op.body.assignOp != nil { 329 return op.body.assignOp.exec(fm) 330 } 331 332 // Ordinary command: evaluate head, arguments and options. 333 cmd := op.body.ordinaryCmd 334 335 // Special case: evaluating an incomplete form node. Return directly. 336 if cmd.headOp == nil { 337 return nil 338 } 339 340 headFn, err := evalForCommand(fm, cmd.headOp, "command") 341 if err != nil { 342 return fm.errorp(cmd.headOp, err) 343 } 344 345 var args []any 346 for _, argOp := range cmd.argOps { 347 moreArgs, exc := argOp.exec(fm) 348 if exc != nil { 349 return exc 350 } 351 args = append(args, moreArgs...) 352 } 353 354 // TODO(xiaq): This conversion should be avoided. 355 convertedOpts := make(map[string]any) 356 exc := cmd.optsOp.exec(fm, func(k, v any) Exception { 357 if ks, ok := k.(string); ok { 358 convertedOpts[ks] = v 359 return nil 360 } 361 // TODO(xiaq): Point to the particular key. 362 return fm.errorp(op, errs.BadValue{ 363 What: "option key", Valid: "string", Actual: vals.Kind(k)}) 364 }) 365 if exc != nil { 366 return exc 367 } 368 369 fm.traceback = fm.addTraceback(op) 370 err = headFn.Call(fm, args, convertedOpts) 371 if exc, ok := err.(Exception); ok { 372 return exc 373 } 374 return &exception{err, fm.traceback} 375 } 376 377 func evalForCommand(fm *Frame, op valuesOp, what string) (Callable, error) { 378 value, err := evalForValue(fm, op, what) 379 if err != nil { 380 return nil, err 381 } 382 switch value := value.(type) { 383 case Callable: 384 return value, nil 385 case string: 386 if fsutil.DontSearch(value) { 387 return NewExternalCmd(value), nil 388 } 389 } 390 return nil, fm.errorp(op, errs.BadValue{ 391 What: what, 392 Valid: "callable or string containing slash", 393 Actual: vals.ReprPlain(value)}) 394 } 395 396 func allTrue(vs []any) bool { 397 for _, v := range vs { 398 if !vals.Bool(v) { 399 return false 400 } 401 } 402 return true 403 } 404 405 func (cp *compiler) assignmentOp(n *parse.Assignment) effectOp { 406 lhs := cp.parseIndexingLValue(n.Left, setLValue|newLValue) 407 rhs := cp.compoundOp(n.Right) 408 return &assignOp{n.Range(), lhs, rhs, false} 409 } 410 411 func (cp *compiler) assignmentOps(ns []*parse.Assignment) []effectOp { 412 ops := make([]effectOp, len(ns)) 413 for i, n := range ns { 414 ops[i] = cp.assignmentOp(n) 415 } 416 return ops 417 } 418 419 const defaultFileRedirPerm = 0644 420 421 // redir compiles a Redir into a op. 422 func (cp *compiler) redirOp(n *parse.Redir) effectOp { 423 var dstOp valuesOp 424 if n.Left != nil { 425 dstOp = cp.compoundOp(n.Left) 426 } 427 flag := makeFlag(n.Mode) 428 if flag == -1 { 429 // TODO: Record and get redirection sign position 430 cp.errorpf(n, "bad redirection sign") 431 } 432 return &redirOp{n.Range(), dstOp, cp.compoundOp(n.Right), n.RightIsFd, n.Mode, flag} 433 } 434 435 func (cp *compiler) redirOps(ns []*parse.Redir) []effectOp { 436 ops := make([]effectOp, len(ns)) 437 for i, n := range ns { 438 ops[i] = cp.redirOp(n) 439 } 440 return ops 441 } 442 443 func makeFlag(m parse.RedirMode) int { 444 switch m { 445 case parse.Read: 446 return os.O_RDONLY 447 case parse.Write: 448 return os.O_WRONLY | os.O_CREATE | os.O_TRUNC 449 case parse.ReadWrite: 450 return os.O_RDWR | os.O_CREATE 451 case parse.Append: 452 return os.O_WRONLY | os.O_CREATE | os.O_APPEND 453 default: 454 return -1 455 } 456 } 457 458 type redirOp struct { 459 diag.Ranging 460 dstOp valuesOp 461 srcOp valuesOp 462 srcIsFd bool 463 mode parse.RedirMode 464 flag int 465 } 466 467 type InvalidFD struct{ FD int } 468 469 func (err InvalidFD) Error() string { return fmt.Sprintf("invalid fd: %d", err.FD) } 470 471 func (op *redirOp) exec(fm *Frame) Exception { 472 var dst int 473 if op.dstOp == nil { 474 // No explicit FD destination specified; use default destinations 475 switch op.mode { 476 case parse.Read: 477 dst = 0 478 case parse.Write, parse.ReadWrite, parse.Append: 479 dst = 1 480 default: 481 return fm.errorpf(op, "bad RedirMode; parser bug") 482 } 483 } else { 484 // An explicit FD destination specified, evaluate it. 485 var err error 486 dst, err = evalForFd(fm, op.dstOp, false, "redirection destination") 487 if err != nil { 488 return fm.errorp(op, err) 489 } 490 } 491 492 growPorts(&fm.ports, dst+1) 493 fm.ports[dst].close() 494 495 if op.srcIsFd { 496 src, err := evalForFd(fm, op.srcOp, true, "redirection source") 497 if err != nil { 498 return fm.errorp(op, err) 499 } 500 switch { 501 case src == -1: 502 // close 503 fm.ports[dst] = &Port{ 504 // Ensure that writing to value output throws an exception 505 sendStop: closedSendStop, sendError: &ErrPortDoesNotSupportValueOutput} 506 case src >= len(fm.ports) || fm.ports[src] == nil: 507 return fm.errorp(op, InvalidFD{FD: src}) 508 default: 509 fm.ports[dst] = fm.ports[src].fork() 510 } 511 return nil 512 } 513 src, err := evalForValue(fm, op.srcOp, "redirection source") 514 if err != nil { 515 return fm.errorp(op, err) 516 } 517 switch src := src.(type) { 518 case string: 519 f, err := os.OpenFile(src, op.flag, defaultFileRedirPerm) 520 if err != nil { 521 return fm.errorpf(op, "failed to open file %s: %s", vals.ReprPlain(src), err) 522 } 523 fm.ports[dst] = fileRedirPort(op.mode, f, true) 524 case vals.File: 525 fm.ports[dst] = fileRedirPort(op.mode, src, false) 526 case vals.Map, vals.StructMap: 527 var srcFile *os.File 528 switch op.mode { 529 case parse.Read: 530 v, err := vals.Index(src, "r") 531 f, ok := v.(*os.File) 532 if err != nil || !ok { 533 return fm.errorp(op.srcOp, errs.BadValue{ 534 What: "map for input redirection", 535 Valid: "map with file in the 'r' field", 536 Actual: vals.ReprPlain(src)}) 537 } 538 srcFile = f 539 case parse.Write: 540 v, err := vals.Index(src, "w") 541 f, ok := v.(*os.File) 542 if err != nil || !ok { 543 return fm.errorp(op.srcOp, errs.BadValue{ 544 What: "map for output redirection", 545 Valid: "map with file in the 'w' field", 546 Actual: vals.ReprPlain(src)}) 547 } 548 srcFile = f 549 default: 550 return fm.errorpf(op, "can only use < or > with maps") 551 } 552 fm.ports[dst] = fileRedirPort(op.mode, srcFile, false) 553 default: 554 return fm.errorp(op.srcOp, errs.BadValue{ 555 What: "redirection source", 556 Valid: "string, file or map", Actual: vals.Kind(src)}) 557 } 558 return nil 559 } 560 561 // Creates a port that only have a file component, populating the 562 // channel-related fields with suitable values depending on the redirection 563 // mode. 564 func fileRedirPort(mode parse.RedirMode, f *os.File, closeFile bool) *Port { 565 if mode == parse.Read { 566 return &Port{ 567 File: f, closeFile: closeFile, 568 // ClosedChan produces no values when reading. 569 Chan: ClosedChan, 570 } 571 } 572 return &Port{ 573 File: f, closeFile: closeFile, 574 // Throws errValueOutputIsClosed when writing. 575 Chan: nil, sendStop: closedSendStop, sendError: &ErrPortDoesNotSupportValueOutput, 576 } 577 } 578 579 // Makes the size of *ports at least n, adding nil's if necessary. 580 func growPorts(ports *[]*Port, n int) { 581 if len(*ports) >= n { 582 return 583 } 584 oldPorts := *ports 585 *ports = make([]*Port, n) 586 copy(*ports, oldPorts) 587 } 588 589 func evalForFd(fm *Frame, op valuesOp, closeOK bool, what string) (int, error) { 590 value, err := evalForValue(fm, op, what) 591 if err != nil { 592 return -1, err 593 } 594 switch value { 595 case "stdin": 596 return 0, nil 597 case "stdout": 598 return 1, nil 599 case "stderr": 600 return 2, nil 601 } 602 var fd int 603 if vals.ScanToGo(value, &fd) == nil { 604 return fd, nil 605 } else if value == "-" && closeOK { 606 return -1, nil 607 } 608 valid := "fd name or number" 609 if closeOK { 610 valid = "fd name or number or '-'" 611 } 612 return -1, fm.errorp(op, errs.BadValue{ 613 What: what, Valid: valid, Actual: vals.ReprPlain(value)}) 614 } 615 616 type seqOp struct{ subops []effectOp } 617 618 func (op seqOp) exec(fm *Frame) Exception { 619 for _, subop := range op.subops { 620 exc := subop.exec(fm) 621 if exc != nil { 622 return exc 623 } 624 } 625 return nil 626 } 627 628 type nopOp struct{} 629 630 func (nopOp) exec(fm *Frame) Exception { return nil }