github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/tools/flow/run.go (about)

     1  // Copyright 2020 CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package flow
    16  
    17  // This file contains logic for running tasks.
    18  //
    19  // This implementation anticipates that workflows can also be used for defining
    20  // servers, not just batch scripts. In the future, tasks may be long running and
    21  // provide streams of results.
    22  //
    23  // The implementation starts a goroutine for each user-defined task, instead of
    24  // having a fixed pool of workers. The main reason for this is that tasks are
    25  // inherently heterogeneous and may be blocking on top of that. Also, in the
    26  // future tasks may be long running, as discussed above.
    27  
    28  import (
    29  	"github.com/joomcode/cue/cue/errors"
    30  	"github.com/joomcode/cue/internal/core/adt"
    31  	"github.com/joomcode/cue/internal/core/eval"
    32  	"github.com/joomcode/cue/internal/value"
    33  )
    34  
    35  func (c *Controller) runLoop() {
    36  	_, root := value.ToInternal(c.inst)
    37  
    38  	// Copy the initial conjuncts.
    39  	n := len(root.Conjuncts)
    40  	c.conjuncts = make([]adt.Conjunct, n, n+len(c.tasks))
    41  	copy(c.conjuncts, root.Conjuncts)
    42  
    43  	c.markReady(nil)
    44  
    45  	for c.errs == nil {
    46  		// Dispatch all unblocked tasks to workers. Only update
    47  		// the configuration when all have been dispatched.
    48  
    49  		waiting := false
    50  		running := false
    51  
    52  		// Mark tasks as Ready.
    53  		for _, t := range c.tasks {
    54  			switch t.state {
    55  			case Waiting:
    56  				waiting = true
    57  
    58  			case Ready:
    59  				running = true
    60  
    61  				t.state = Running
    62  				c.updateTaskValue(t)
    63  
    64  				t.ctxt = eval.NewContext(value.ToInternal(t.v))
    65  
    66  				go func(t *Task) {
    67  					if err := t.r.Run(t, nil); err != nil {
    68  						t.err = errors.Promote(err, "task failed")
    69  					}
    70  
    71  					t.c.taskCh <- t
    72  				}(t)
    73  
    74  			case Running:
    75  				running = true
    76  
    77  			case Terminated:
    78  			}
    79  		}
    80  
    81  		if !running {
    82  			if waiting {
    83  				// Should not happen ever, as cycle detection should have caught
    84  				// this. But keep this around as a defensive measure.
    85  				c.addErr(errors.New("deadlock"), "run loop")
    86  			}
    87  			break
    88  		}
    89  
    90  		select {
    91  		case <-c.context.Done():
    92  			return
    93  
    94  		case t := <-c.taskCh:
    95  			t.state = Terminated
    96  
    97  			switch t.err {
    98  			case nil:
    99  				c.updateTaskResults(t)
   100  
   101  			case ErrAbort:
   102  				// TODO: do something cleverer.
   103  				fallthrough
   104  
   105  			default:
   106  				c.addErr(t.err, "task failure")
   107  				return
   108  			}
   109  
   110  			// Recompute the configuration, if necessary.
   111  			if c.updateValue() {
   112  				// initTasks was already called in New to catch initialization
   113  				// errors earlier.
   114  				c.initTasks()
   115  			}
   116  
   117  			c.updateTaskValue(t)
   118  
   119  			c.markReady(t)
   120  		}
   121  	}
   122  }
   123  
   124  func (c *Controller) markReady(t *Task) {
   125  	for _, x := range c.tasks {
   126  		if x.state == Waiting && x.isReady() {
   127  			x.state = Ready
   128  		}
   129  	}
   130  
   131  	if c.cfg.UpdateFunc != nil {
   132  		if err := c.cfg.UpdateFunc(c, t); err != nil {
   133  			c.addErr(err, "task completed")
   134  			c.cancel()
   135  			return
   136  		}
   137  	}
   138  }
   139  
   140  // updateValue recomputes the workflow configuration if it is out of date. It
   141  // reports whether the values were updated.
   142  func (c *Controller) updateValue() bool {
   143  
   144  	if c.valueSeqNum == c.conjunctSeq {
   145  		return false
   146  	}
   147  
   148  	// TODO: incrementally update results. Currently, the entire tree is
   149  	// recomputed on every update. This should not be necessary with the right
   150  	// notification structure in place.
   151  
   152  	v := &adt.Vertex{Conjuncts: c.conjuncts}
   153  	v.Finalize(c.opCtx)
   154  
   155  	c.inst = value.Make(c.opCtx, v)
   156  	c.valueSeqNum = c.conjunctSeq
   157  	return true
   158  }
   159  
   160  // updateTaskValue updates the value of the task in the configuration if it is
   161  // out of date.
   162  func (c *Controller) updateTaskValue(t *Task) {
   163  	required := t.conjunctSeq
   164  	for _, dep := range t.depTasks {
   165  		if dep.conjunctSeq > required {
   166  			required = dep.conjunctSeq
   167  		}
   168  	}
   169  
   170  	if t.valueSeq == required {
   171  		return
   172  	}
   173  
   174  	if c.valueSeqNum < required {
   175  		c.updateValue()
   176  	}
   177  
   178  	t.v = c.inst.LookupPath(t.path)
   179  	t.valueSeq = required
   180  }
   181  
   182  // updateTaskResults updates the result status of the task and adds any result
   183  // values to the overall configuration.
   184  func (c *Controller) updateTaskResults(t *Task) bool {
   185  	if t.update == nil {
   186  		return false
   187  	}
   188  
   189  	expr := t.update
   190  	for i := len(t.labels) - 1; i >= 0; i-- {
   191  		expr = &adt.StructLit{
   192  			Decls: []adt.Decl{
   193  				&adt.Field{
   194  					Label: t.labels[i],
   195  					Value: expr,
   196  				},
   197  			},
   198  		}
   199  	}
   200  
   201  	t.update = nil
   202  
   203  	// TODO: replace rather than add conjunct if this task already added a
   204  	// conjunct before. This will allow for serving applications.
   205  	c.conjuncts = append(c.conjuncts, adt.MakeRootConjunct(c.env, expr))
   206  	c.conjunctSeq++
   207  	t.conjunctSeq = c.conjunctSeq
   208  
   209  	return true
   210  }