github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/workflow_branch_map.go (about) 1 package processor 2 3 import ( 4 "context" 5 "fmt" 6 "sort" 7 "sync" 8 "time" 9 10 "github.com/Jeffail/benthos/v3/internal/interop" 11 "github.com/Jeffail/benthos/v3/lib/log" 12 "github.com/Jeffail/benthos/v3/lib/metrics" 13 "github.com/Jeffail/benthos/v3/lib/types" 14 "github.com/quipo/dependencysolver" 15 ) 16 17 type workflowBranch interface { 18 lock() (*Branch, func()) 19 } 20 21 //------------------------------------------------------------------------------ 22 23 type workflowBranchMap struct { 24 static bool 25 dag [][]string 26 staticBranches map[string]*Branch 27 28 dynamicBranches map[string]workflowBranch 29 } 30 31 func lockAll(dynBranches map[string]workflowBranch) (branches map[string]*Branch, unlockFn func(), err error) { 32 unlocks := make([]func(), 0, len(dynBranches)) 33 unlockFn = func() { 34 for _, u := range unlocks { 35 if u != nil { 36 u() 37 } 38 } 39 } 40 41 branches = make(map[string]*Branch, len(dynBranches)) 42 for k, v := range dynBranches { 43 var branchUnlock func() 44 branches[k], branchUnlock = v.lock() 45 unlocks = append(unlocks, branchUnlock) 46 if branches[k] == nil { 47 err = fmt.Errorf("missing branch resource: %v", k) 48 unlockFn() 49 return 50 } 51 } 52 return 53 } 54 55 // Locks all branches contained in the branch map and returns the latest DAG, a 56 // map of resources, and a func to unlock the resources that were locked. If 57 // any error occurs in locked each branch (the resource is missing, or the DAG 58 // is malformed) then an error is returned instead. 59 func (w *workflowBranchMap) Lock() (dag [][]string, branches map[string]*Branch, unlockFn func(), err error) { 60 if w.static { 61 return w.dag, w.staticBranches, func() {}, nil 62 } 63 64 if branches, unlockFn, err = lockAll(w.dynamicBranches); err != nil { 65 return 66 } 67 if len(w.dag) > 0 { 68 dag = w.dag 69 return 70 } 71 72 if dag, err = resolveDynamicBranchDAG(branches); err != nil { 73 unlockFn() 74 err = fmt.Errorf("failed to resolve DAG: %w", err) 75 } 76 return 77 } 78 79 func (w *workflowBranchMap) CloseAsync() { 80 for _, b := range w.staticBranches { 81 b.CloseAsync() 82 } 83 } 84 85 func (w *workflowBranchMap) WaitForClose(timeout time.Duration) error { 86 stopBy := time.Now().Add(timeout) 87 for _, c := range w.staticBranches { 88 if err := c.WaitForClose(time.Until(stopBy)); err != nil { 89 return err 90 } 91 } 92 return nil 93 } 94 95 //------------------------------------------------------------------------------ 96 97 func newWorkflowBranchMap(conf WorkflowConfig, mgr types.Manager, log log.Modular, stats metrics.Type) (*workflowBranchMap, error) { 98 dynamicBranches, staticBranches := map[string]workflowBranch{}, map[string]*Branch{} 99 for k, v := range conf.Branches { 100 if len(processDAGStageName.FindString(k)) != len(k) { 101 return nil, fmt.Errorf("workflow branch name '%v' contains invalid characters", k) 102 } 103 104 bMgr, bLog, bStats := interop.LabelChild(k, mgr, log, stats) 105 child, err := newBranch(v, bMgr, bLog, bStats) 106 if err != nil { 107 return nil, fmt.Errorf("failed to create branch '%v': %v", k, err) 108 } 109 110 dynamicBranches[k] = &normalBranch{child} 111 staticBranches[k] = child 112 } 113 114 for _, k := range conf.BranchResources { 115 if _, exists := dynamicBranches[k]; exists { 116 return nil, fmt.Errorf("branch resource name '%v' collides with an explicit branch", k) 117 } 118 if err := interop.ProbeProcessor(context.Background(), mgr, k); err != nil { 119 return nil, err 120 } 121 dynamicBranches[k] = &resourcedBranch{ 122 name: k, 123 mgr: mgr, 124 } 125 } 126 127 // When order is specified we infer that names missing from our explicit 128 // branches are resources. 129 for _, tier := range conf.Order { 130 for _, k := range tier { 131 if _, exists := dynamicBranches[k]; !exists { 132 if err := interop.ProbeProcessor(context.Background(), mgr, k); err != nil { 133 return nil, err 134 } 135 dynamicBranches[k] = &resourcedBranch{ 136 name: k, 137 mgr: mgr, 138 } 139 } 140 } 141 } 142 143 static := len(dynamicBranches) == len(staticBranches) 144 145 var dag [][]string 146 if len(conf.Order) > 0 { 147 dag = conf.Order 148 if err := verifyStaticBranchDAG(dag, dynamicBranches); err != nil { 149 return nil, err 150 } 151 } else if static { 152 var err error 153 if dag, err = resolveDynamicBranchDAG(staticBranches); err != nil { 154 return nil, err 155 } 156 log.Infof("Automatically resolved workflow DAG: %v\n", dag) 157 } 158 159 return &workflowBranchMap{ 160 static: static, 161 dag: dag, 162 staticBranches: staticBranches, 163 dynamicBranches: dynamicBranches, 164 }, nil 165 } 166 167 //------------------------------------------------------------------------------ 168 169 type resourcedBranch struct { 170 name string 171 mgr types.Manager 172 } 173 174 func (r *resourcedBranch) lock() (branch *Branch, unlockFn func()) { 175 var openOnce, releaseOnce sync.Once 176 open, release := make(chan struct{}), make(chan struct{}) 177 unlockFn = func() { 178 releaseOnce.Do(func() { 179 close(release) 180 }) 181 } 182 183 go func() { 184 _ = interop.AccessProcessor(context.Background(), r.mgr, r.name, func(p types.Processor) { 185 branch, _ = p.(*Branch) 186 openOnce.Do(func() { 187 close(open) 188 }) 189 <-release 190 }) 191 openOnce.Do(func() { 192 close(open) 193 }) 194 }() 195 196 <-open 197 return 198 } 199 200 //------------------------------------------------------------------------------ 201 202 type normalBranch struct { 203 *Branch 204 } 205 206 func (r *normalBranch) lock() (branch *Branch, unlockFn func()) { 207 return r.Branch, nil 208 } 209 210 //------------------------------------------------------------------------------ 211 212 func depHasPrefix(wanted, provided []string) bool { 213 if len(wanted) < len(provided) { 214 return false 215 } 216 for i, s := range provided { 217 if wanted[i] != s { 218 return false 219 } 220 } 221 return true 222 } 223 224 func getBranchDeps(id string, wanted [][]string, branches map[string]*Branch) []string { 225 dependencies := []string{} 226 227 for k, b := range branches { 228 if k == id { 229 continue 230 } 231 for _, tp := range b.targetsProvided() { 232 for _, tn := range wanted { 233 if depHasPrefix(tn, tp) { 234 dependencies = append(dependencies, k) 235 break 236 } 237 } 238 } 239 } 240 241 return dependencies 242 } 243 244 func verifyStaticBranchDAG(order [][]string, branches map[string]workflowBranch) error { 245 remaining := map[string]struct{}{} 246 seen := map[string]struct{}{} 247 for id := range branches { 248 remaining[id] = struct{}{} 249 } 250 for i, tier := range order { 251 if len(tier) == 0 { 252 return fmt.Errorf("explicit order tier '%v' was empty", i) 253 } 254 for _, t := range tier { 255 if _, exists := seen[t]; exists { 256 return fmt.Errorf("branch specified in order listed multiple times: %v", t) 257 } 258 seen[t] = struct{}{} 259 delete(remaining, t) 260 } 261 } 262 if len(remaining) > 0 { 263 names := make([]string, 0, len(remaining)) 264 for k := range remaining { 265 names = append(names, k) 266 } 267 return fmt.Errorf("the following branches were missing from order: %v", names) 268 } 269 return nil 270 } 271 272 func resolveDynamicBranchDAG(branches map[string]*Branch) ([][]string, error) { 273 if len(branches) == 0 { 274 return [][]string{}, nil 275 } 276 remaining := map[string]struct{}{} 277 278 var entries []dependencysolver.Entry 279 for id, b := range branches { 280 wanted := b.targetsUsed() 281 282 remaining[id] = struct{}{} 283 entries = append(entries, dependencysolver.Entry{ 284 ID: id, Deps: getBranchDeps(id, wanted, branches), 285 }) 286 } 287 288 layers := dependencysolver.LayeredTopologicalSort(entries) 289 for _, l := range layers { 290 for _, id := range l { 291 delete(remaining, id) 292 } 293 } 294 295 if len(remaining) > 0 { 296 var tProcs []string 297 for k := range remaining { 298 tProcs = append(tProcs, k) 299 } 300 sort.Strings(tProcs) 301 return nil, fmt.Errorf("failed to automatically resolve DAG, circular dependencies detected for branches: %v", tProcs) 302 } 303 304 return layers, nil 305 }