src.elv.sh@v0.21.0-dev.0.20240515223629-06979efb9a2a/pkg/eval/compile_effect.go (about)

     1  package eval
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"sync"
     8  	"sync/atomic"
     9  
    10  	"src.elv.sh/pkg/diag"
    11  	"src.elv.sh/pkg/eval/errs"
    12  	"src.elv.sh/pkg/eval/vals"
    13  	"src.elv.sh/pkg/eval/vars"
    14  	"src.elv.sh/pkg/fsutil"
    15  	"src.elv.sh/pkg/parse"
    16  	"src.elv.sh/pkg/parse/cmpd"
    17  )
    18  
    19  // An operation with some side effects.
    20  type effectOp interface{ exec(*Frame) Exception }
    21  
    22  func (cp *compiler) chunkOp(n *parse.Chunk) effectOp {
    23  	return chunkOp{n.Range(), cp.pipelineOps(n.Pipelines)}
    24  }
    25  
    26  type chunkOp struct {
    27  	diag.Ranging
    28  	subops []effectOp
    29  }
    30  
    31  func (op chunkOp) exec(fm *Frame) Exception {
    32  	for _, subop := range op.subops {
    33  		exc := subop.exec(fm)
    34  		if exc != nil {
    35  			return exc
    36  		}
    37  	}
    38  	// Check for interrupts after the chunk.
    39  	// We also check for interrupts before each pipeline, so there is no
    40  	// need to check it before the chunk or after each pipeline.
    41  	if fm.Canceled() {
    42  		return fm.errorp(op, ErrInterrupted)
    43  	}
    44  	return nil
    45  }
    46  
    47  func (cp *compiler) pipelineOp(n *parse.Pipeline) effectOp {
    48  	formOps := cp.formOps(n.Forms)
    49  
    50  	return &pipelineOp{n.Range(), n.Background, parse.SourceText(n), formOps}
    51  }
    52  
    53  func (cp *compiler) pipelineOps(ns []*parse.Pipeline) []effectOp {
    54  	ops := make([]effectOp, len(ns))
    55  	for i, n := range ns {
    56  		ops[i] = cp.pipelineOp(n)
    57  	}
    58  	return ops
    59  }
    60  
    61  type pipelineOp struct {
    62  	diag.Ranging
    63  	bg     bool
    64  	source string
    65  	subops []effectOp
    66  }
    67  
    68  const pipelineChanBufferSize = 32
    69  
    70  func (op *pipelineOp) exec(fm *Frame) Exception {
    71  	if fm.Canceled() {
    72  		return fm.errorp(op, ErrInterrupted)
    73  	}
    74  
    75  	if op.bg {
    76  		fm = fm.Fork("background job" + op.source)
    77  		fm.ctx = context.Background()
    78  		fm.background = true
    79  		fm.Evaler.addNumBgJobs(1)
    80  	}
    81  
    82  	nforms := len(op.subops)
    83  
    84  	var wg sync.WaitGroup
    85  	wg.Add(nforms)
    86  	excs := make([]Exception, nforms)
    87  
    88  	var nextIn *Port
    89  
    90  	// For each form, create a dedicated evalCtx and run asynchronously
    91  	for i, formOp := range op.subops {
    92  		newFm := fm.Fork("[form op]")
    93  		inputIsPipe := i > 0
    94  		outputIsPipe := i < nforms-1
    95  		if inputIsPipe {
    96  			newFm.ports[0] = nextIn
    97  		}
    98  		if outputIsPipe {
    99  			// Each internal port pair consists of a (byte) pipe pair and a
   100  			// channel.
   101  			// os.Pipe sets O_CLOEXEC, which is what we want.
   102  			reader, writer, e := os.Pipe()
   103  			if e != nil {
   104  				return fm.errorpf(op, "failed to create pipe: %s", e)
   105  			}
   106  			ch := make(chan any, pipelineChanBufferSize)
   107  			sendStop := make(chan struct{})
   108  			sendError := new(error)
   109  			readerGone := new(atomic.Bool)
   110  			newFm.ports[1] = &Port{
   111  				File: writer, Chan: ch,
   112  				closeFile: true, closeChan: true,
   113  				sendStop: sendStop, sendError: sendError, readerGone: readerGone}
   114  			nextIn = &Port{
   115  				File: reader, Chan: ch,
   116  				closeFile: true, closeChan: false,
   117  				// Store in input port for ease of retrieval later
   118  				sendStop: sendStop, sendError: sendError, readerGone: readerGone}
   119  		}
   120  		f := func(formOp effectOp, pexc *Exception) {
   121  			exc := formOp.exec(newFm)
   122  			if exc != nil && !(outputIsPipe && isReaderGone(exc)) {
   123  				*pexc = exc
   124  			}
   125  			if inputIsPipe {
   126  				input := newFm.ports[0]
   127  				*input.sendError = errs.ReaderGone{}
   128  				close(input.sendStop)
   129  				input.readerGone.Store(true)
   130  			}
   131  			newFm.Close()
   132  			wg.Done()
   133  		}
   134  		if i == nforms-1 && !op.bg {
   135  			f(formOp, &excs[i])
   136  		} else {
   137  			go f(formOp, &excs[i])
   138  		}
   139  	}
   140  
   141  	if op.bg {
   142  		// Background job, wait for form termination asynchronously.
   143  		go func() {
   144  			wg.Wait()
   145  			fm.Evaler.addNumBgJobs(-1)
   146  			if notify := fm.Evaler.BgJobNotify; notify != nil {
   147  				msg := "job " + op.source + " finished"
   148  				err := MakePipelineError(excs)
   149  				if err != nil {
   150  					msg += ", errors = " + err.Error()
   151  				}
   152  				if fm.Evaler.getNotifyBgJobSuccess() || err != nil {
   153  					notify(msg)
   154  				}
   155  			}
   156  		}()
   157  		return nil
   158  	}
   159  	wg.Wait()
   160  	return fm.errorp(op, MakePipelineError(excs))
   161  }
   162  
   163  func isReaderGone(exc Exception) bool {
   164  	_, ok := exc.Reason().(errs.ReaderGone)
   165  	return ok
   166  }
   167  
   168  func (cp *compiler) formOp(n *parse.Form) effectOp {
   169  	var tempLValues []lvalue
   170  	var assignmentOps []effectOp
   171  	if len(n.Assignments) > 0 {
   172  		if n.Head == nil {
   173  			cp.errorpf(n, `using the syntax of temporary assignment for non-temporary assignment is no longer supported; use "var" or "set" instead`)
   174  			return nopOp{}
   175  		} else {
   176  			as := n.Assignments
   177  			cp.deprecate(diag.MixedRanging(as[0], as[len(as)-1]),
   178  				`the legacy temporary assignment syntax is deprecated; use "tmp" instead`, 18)
   179  		}
   180  		assignmentOps = cp.assignmentOps(n.Assignments)
   181  		for _, a := range n.Assignments {
   182  			lvalues := cp.parseIndexingLValue(a.Left, setLValue|newLValue)
   183  			tempLValues = append(tempLValues, lvalues.lvalues...)
   184  		}
   185  		logger.Println("temporary assignment of", len(n.Assignments), "pairs")
   186  	}
   187  
   188  	redirOps := cp.redirOps(n.Redirs)
   189  	body := cp.formBody(n)
   190  
   191  	return &formOp{n.Range(), tempLValues, assignmentOps, redirOps, body}
   192  }
   193  
   194  func (cp *compiler) formBody(n *parse.Form) formBody {
   195  	if n.Head == nil {
   196  		// Compiling an incomplete form node, return an empty body.
   197  		return formBody{}
   198  	}
   199  
   200  	// Determine if this form is a special command.
   201  	if head, ok := cmpd.StringLiteral(n.Head); ok {
   202  		special, _ := resolveCmdHeadInternally(cp, head, n.Head)
   203  		if special != nil {
   204  			specialOp := special(cp, n)
   205  			return formBody{specialOp: specialOp}
   206  		}
   207  	}
   208  
   209  	var headOp valuesOp
   210  	if head, ok := cmpd.StringLiteral(n.Head); ok {
   211  		// Head is a literal string: resolve to function or external (special
   212  		// commands are already handled above).
   213  		if _, fnRef := resolveCmdHeadInternally(cp, head, n.Head); fnRef != nil {
   214  			headOp = variableOp{n.Head.Range(), false, head + FnSuffix, fnRef}
   215  		} else {
   216  			cp.autofixUnresolvedVar(head + FnSuffix)
   217  			if cp.currentPragma().unknownCommandIsExternal || fsutil.DontSearch(head) {
   218  				headOp = literalValues(n.Head, NewExternalCmd(head))
   219  			} else {
   220  				cp.errorpf(n.Head, "unknown command disallowed by current pragma")
   221  			}
   222  		}
   223  	} else {
   224  		// Head is not a literal string: evaluate as a normal expression.
   225  		headOp = cp.compoundOp(n.Head)
   226  	}
   227  
   228  	argOps := cp.compoundOps(n.Args)
   229  	optsOp := cp.mapPairs(n.Opts)
   230  	return formBody{ordinaryCmd: ordinaryCmd{headOp, argOps, optsOp}}
   231  }
   232  
   233  func (cp *compiler) formOps(ns []*parse.Form) []effectOp {
   234  	ops := make([]effectOp, len(ns))
   235  	for i, n := range ns {
   236  		ops[i] = cp.formOp(n)
   237  	}
   238  	return ops
   239  }
   240  
   241  type formOp struct {
   242  	diag.Ranging
   243  	tempLValues   []lvalue
   244  	tempAssignOps []effectOp
   245  	redirOps      []effectOp
   246  	body          formBody
   247  }
   248  
   249  type formBody struct {
   250  	// Exactly one field will be populated.
   251  	specialOp   effectOp
   252  	assignOp    effectOp
   253  	ordinaryCmd ordinaryCmd
   254  }
   255  
   256  type ordinaryCmd struct {
   257  	headOp valuesOp
   258  	argOps []valuesOp
   259  	optsOp *mapPairsOp
   260  }
   261  
   262  func (op *formOp) exec(fm *Frame) (errRet Exception) {
   263  	// fm here is always a sub-frame created in compiler.pipeline, so it can
   264  	// be safely modified.
   265  
   266  	// Temporary assignment.
   267  	if len(op.tempLValues) > 0 {
   268  		// There is a temporary assignment.
   269  		// Save variables.
   270  		var saveVars []vars.Var
   271  		var saveVals []any
   272  		for _, lv := range op.tempLValues {
   273  			variable, err := derefLValue(fm, lv)
   274  			if err != nil {
   275  				return fm.errorp(op, err)
   276  			}
   277  			saveVars = append(saveVars, variable)
   278  		}
   279  		for i, v := range saveVars {
   280  			// TODO(xiaq): If the variable to save is a elemVariable, save
   281  			// the outermost variable instead.
   282  			if u := vars.HeadOfElement(v); u != nil {
   283  				v = u
   284  				saveVars[i] = v
   285  			}
   286  			val := v.Get()
   287  			saveVals = append(saveVals, val)
   288  			logger.Printf("saved %s = %s", v, val)
   289  		}
   290  		// Do assignment.
   291  		for _, subop := range op.tempAssignOps {
   292  			exc := subop.exec(fm)
   293  			if exc != nil {
   294  				return exc
   295  			}
   296  		}
   297  		// Defer variable restoration. Will be executed even if an error
   298  		// occurs when evaling other part of the form.
   299  		defer func() {
   300  			for i, v := range saveVars {
   301  				val := saveVals[i]
   302  				if val == nil {
   303  					// TODO(xiaq): Old value is nonexistent. We should delete
   304  					// the variable. However, since the compiler now doesn't
   305  					// delete it, we don't delete it in the evaler either.
   306  					val = ""
   307  				}
   308  				err := v.Set(val)
   309  				if err != nil {
   310  					errRet = fm.errorp(op, err)
   311  				}
   312  				logger.Printf("restored %s = %s", v, val)
   313  			}
   314  		}()
   315  	}
   316  
   317  	// Redirections.
   318  	for _, redirOp := range op.redirOps {
   319  		exc := redirOp.exec(fm)
   320  		if exc != nil {
   321  			return exc
   322  		}
   323  	}
   324  
   325  	if op.body.specialOp != nil {
   326  		return op.body.specialOp.exec(fm)
   327  	}
   328  	if op.body.assignOp != nil {
   329  		return op.body.assignOp.exec(fm)
   330  	}
   331  
   332  	// Ordinary command: evaluate head, arguments and options.
   333  	cmd := op.body.ordinaryCmd
   334  
   335  	// Special case: evaluating an incomplete form node. Return directly.
   336  	if cmd.headOp == nil {
   337  		return nil
   338  	}
   339  
   340  	headFn, err := evalForCommand(fm, cmd.headOp, "command")
   341  	if err != nil {
   342  		return fm.errorp(cmd.headOp, err)
   343  	}
   344  
   345  	var args []any
   346  	for _, argOp := range cmd.argOps {
   347  		moreArgs, exc := argOp.exec(fm)
   348  		if exc != nil {
   349  			return exc
   350  		}
   351  		args = append(args, moreArgs...)
   352  	}
   353  
   354  	// TODO(xiaq): This conversion should be avoided.
   355  	convertedOpts := make(map[string]any)
   356  	exc := cmd.optsOp.exec(fm, func(k, v any) Exception {
   357  		if ks, ok := k.(string); ok {
   358  			convertedOpts[ks] = v
   359  			return nil
   360  		}
   361  		// TODO(xiaq): Point to the particular key.
   362  		return fm.errorp(op, errs.BadValue{
   363  			What: "option key", Valid: "string", Actual: vals.Kind(k)})
   364  	})
   365  	if exc != nil {
   366  		return exc
   367  	}
   368  
   369  	fm.traceback = fm.addTraceback(op)
   370  	err = headFn.Call(fm, args, convertedOpts)
   371  	if exc, ok := err.(Exception); ok {
   372  		return exc
   373  	}
   374  	return &exception{err, fm.traceback}
   375  }
   376  
   377  func evalForCommand(fm *Frame, op valuesOp, what string) (Callable, error) {
   378  	value, err := evalForValue(fm, op, what)
   379  	if err != nil {
   380  		return nil, err
   381  	}
   382  	switch value := value.(type) {
   383  	case Callable:
   384  		return value, nil
   385  	case string:
   386  		if fsutil.DontSearch(value) {
   387  			return NewExternalCmd(value), nil
   388  		}
   389  	}
   390  	return nil, fm.errorp(op, errs.BadValue{
   391  		What:   what,
   392  		Valid:  "callable or string containing slash",
   393  		Actual: vals.ReprPlain(value)})
   394  }
   395  
   396  func allTrue(vs []any) bool {
   397  	for _, v := range vs {
   398  		if !vals.Bool(v) {
   399  			return false
   400  		}
   401  	}
   402  	return true
   403  }
   404  
   405  func (cp *compiler) assignmentOp(n *parse.Assignment) effectOp {
   406  	lhs := cp.parseIndexingLValue(n.Left, setLValue|newLValue)
   407  	rhs := cp.compoundOp(n.Right)
   408  	return &assignOp{n.Range(), lhs, rhs, false}
   409  }
   410  
   411  func (cp *compiler) assignmentOps(ns []*parse.Assignment) []effectOp {
   412  	ops := make([]effectOp, len(ns))
   413  	for i, n := range ns {
   414  		ops[i] = cp.assignmentOp(n)
   415  	}
   416  	return ops
   417  }
   418  
   419  const defaultFileRedirPerm = 0644
   420  
   421  // redir compiles a Redir into a op.
   422  func (cp *compiler) redirOp(n *parse.Redir) effectOp {
   423  	var dstOp valuesOp
   424  	if n.Left != nil {
   425  		dstOp = cp.compoundOp(n.Left)
   426  	}
   427  	flag := makeFlag(n.Mode)
   428  	if flag == -1 {
   429  		// TODO: Record and get redirection sign position
   430  		cp.errorpf(n, "bad redirection sign")
   431  	}
   432  	return &redirOp{n.Range(), dstOp, cp.compoundOp(n.Right), n.RightIsFd, n.Mode, flag}
   433  }
   434  
   435  func (cp *compiler) redirOps(ns []*parse.Redir) []effectOp {
   436  	ops := make([]effectOp, len(ns))
   437  	for i, n := range ns {
   438  		ops[i] = cp.redirOp(n)
   439  	}
   440  	return ops
   441  }
   442  
   443  func makeFlag(m parse.RedirMode) int {
   444  	switch m {
   445  	case parse.Read:
   446  		return os.O_RDONLY
   447  	case parse.Write:
   448  		return os.O_WRONLY | os.O_CREATE | os.O_TRUNC
   449  	case parse.ReadWrite:
   450  		return os.O_RDWR | os.O_CREATE
   451  	case parse.Append:
   452  		return os.O_WRONLY | os.O_CREATE | os.O_APPEND
   453  	default:
   454  		return -1
   455  	}
   456  }
   457  
   458  type redirOp struct {
   459  	diag.Ranging
   460  	dstOp   valuesOp
   461  	srcOp   valuesOp
   462  	srcIsFd bool
   463  	mode    parse.RedirMode
   464  	flag    int
   465  }
   466  
   467  type InvalidFD struct{ FD int }
   468  
   469  func (err InvalidFD) Error() string { return fmt.Sprintf("invalid fd: %d", err.FD) }
   470  
   471  func (op *redirOp) exec(fm *Frame) Exception {
   472  	var dst int
   473  	if op.dstOp == nil {
   474  		// No explicit FD destination specified; use default destinations
   475  		switch op.mode {
   476  		case parse.Read:
   477  			dst = 0
   478  		case parse.Write, parse.ReadWrite, parse.Append:
   479  			dst = 1
   480  		default:
   481  			return fm.errorpf(op, "bad RedirMode; parser bug")
   482  		}
   483  	} else {
   484  		// An explicit FD destination specified, evaluate it.
   485  		var err error
   486  		dst, err = evalForFd(fm, op.dstOp, false, "redirection destination")
   487  		if err != nil {
   488  			return fm.errorp(op, err)
   489  		}
   490  	}
   491  
   492  	growPorts(&fm.ports, dst+1)
   493  	fm.ports[dst].close()
   494  
   495  	if op.srcIsFd {
   496  		src, err := evalForFd(fm, op.srcOp, true, "redirection source")
   497  		if err != nil {
   498  			return fm.errorp(op, err)
   499  		}
   500  		switch {
   501  		case src == -1:
   502  			// close
   503  			fm.ports[dst] = &Port{
   504  				// Ensure that writing to value output throws an exception
   505  				sendStop: closedSendStop, sendError: &ErrPortDoesNotSupportValueOutput}
   506  		case src >= len(fm.ports) || fm.ports[src] == nil:
   507  			return fm.errorp(op, InvalidFD{FD: src})
   508  		default:
   509  			fm.ports[dst] = fm.ports[src].fork()
   510  		}
   511  		return nil
   512  	}
   513  	src, err := evalForValue(fm, op.srcOp, "redirection source")
   514  	if err != nil {
   515  		return fm.errorp(op, err)
   516  	}
   517  	switch src := src.(type) {
   518  	case string:
   519  		f, err := os.OpenFile(src, op.flag, defaultFileRedirPerm)
   520  		if err != nil {
   521  			return fm.errorpf(op, "failed to open file %s: %s", vals.ReprPlain(src), err)
   522  		}
   523  		fm.ports[dst] = fileRedirPort(op.mode, f, true)
   524  	case vals.File:
   525  		fm.ports[dst] = fileRedirPort(op.mode, src, false)
   526  	case vals.Map, vals.StructMap:
   527  		var srcFile *os.File
   528  		switch op.mode {
   529  		case parse.Read:
   530  			v, err := vals.Index(src, "r")
   531  			f, ok := v.(*os.File)
   532  			if err != nil || !ok {
   533  				return fm.errorp(op.srcOp, errs.BadValue{
   534  					What:   "map for input redirection",
   535  					Valid:  "map with file in the 'r' field",
   536  					Actual: vals.ReprPlain(src)})
   537  			}
   538  			srcFile = f
   539  		case parse.Write:
   540  			v, err := vals.Index(src, "w")
   541  			f, ok := v.(*os.File)
   542  			if err != nil || !ok {
   543  				return fm.errorp(op.srcOp, errs.BadValue{
   544  					What:   "map for output redirection",
   545  					Valid:  "map with file in the 'w' field",
   546  					Actual: vals.ReprPlain(src)})
   547  			}
   548  			srcFile = f
   549  		default:
   550  			return fm.errorpf(op, "can only use < or > with maps")
   551  		}
   552  		fm.ports[dst] = fileRedirPort(op.mode, srcFile, false)
   553  	default:
   554  		return fm.errorp(op.srcOp, errs.BadValue{
   555  			What:  "redirection source",
   556  			Valid: "string, file or map", Actual: vals.Kind(src)})
   557  	}
   558  	return nil
   559  }
   560  
   561  // Creates a port that only have a file component, populating the
   562  // channel-related fields with suitable values depending on the redirection
   563  // mode.
   564  func fileRedirPort(mode parse.RedirMode, f *os.File, closeFile bool) *Port {
   565  	if mode == parse.Read {
   566  		return &Port{
   567  			File: f, closeFile: closeFile,
   568  			// ClosedChan produces no values when reading.
   569  			Chan: ClosedChan,
   570  		}
   571  	}
   572  	return &Port{
   573  		File: f, closeFile: closeFile,
   574  		// Throws errValueOutputIsClosed when writing.
   575  		Chan: nil, sendStop: closedSendStop, sendError: &ErrPortDoesNotSupportValueOutput,
   576  	}
   577  }
   578  
   579  // Makes the size of *ports at least n, adding nil's if necessary.
   580  func growPorts(ports *[]*Port, n int) {
   581  	if len(*ports) >= n {
   582  		return
   583  	}
   584  	oldPorts := *ports
   585  	*ports = make([]*Port, n)
   586  	copy(*ports, oldPorts)
   587  }
   588  
   589  func evalForFd(fm *Frame, op valuesOp, closeOK bool, what string) (int, error) {
   590  	value, err := evalForValue(fm, op, what)
   591  	if err != nil {
   592  		return -1, err
   593  	}
   594  	switch value {
   595  	case "stdin":
   596  		return 0, nil
   597  	case "stdout":
   598  		return 1, nil
   599  	case "stderr":
   600  		return 2, nil
   601  	}
   602  	var fd int
   603  	if vals.ScanToGo(value, &fd) == nil {
   604  		return fd, nil
   605  	} else if value == "-" && closeOK {
   606  		return -1, nil
   607  	}
   608  	valid := "fd name or number"
   609  	if closeOK {
   610  		valid = "fd name or number or '-'"
   611  	}
   612  	return -1, fm.errorp(op, errs.BadValue{
   613  		What: what, Valid: valid, Actual: vals.ReprPlain(value)})
   614  }
   615  
   616  type seqOp struct{ subops []effectOp }
   617  
   618  func (op seqOp) exec(fm *Frame) Exception {
   619  	for _, subop := range op.subops {
   620  		exc := subop.exec(fm)
   621  		if exc != nil {
   622  			return exc
   623  		}
   624  	}
   625  	return nil
   626  }
   627  
   628  type nopOp struct{}
   629  
   630  func (nopOp) exec(fm *Frame) Exception { return nil }