github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/workflow_deprecated.go (about) 1 package processor 2 3 import ( 4 "fmt" 5 "sort" 6 "sync" 7 "time" 8 9 "github.com/Jeffail/benthos/v3/internal/interop" 10 "github.com/Jeffail/benthos/v3/internal/tracing" 11 "github.com/Jeffail/benthos/v3/lib/log" 12 "github.com/Jeffail/benthos/v3/lib/metrics" 13 "github.com/Jeffail/benthos/v3/lib/types" 14 "github.com/Jeffail/gabs/v2" 15 ) 16 17 //------------------------------------------------------------------------------ 18 19 type workflowDeprecated struct { 20 log log.Modular 21 stats metrics.Type 22 23 children map[string]*ProcessMap 24 dag [][]string 25 allStages map[string]struct{} 26 metaPath []string 27 28 mCount metrics.StatCounter 29 mSent metrics.StatCounter 30 mSentParts metrics.StatCounter 31 mSkippedNoStages metrics.StatCounter 32 mErr metrics.StatCounter 33 mErrJSON metrics.StatCounter 34 mErrMeta metrics.StatCounter 35 mErrOverlay metrics.StatCounter 36 mErrStages map[string]metrics.StatCounter 37 mSuccStages map[string]metrics.StatCounter 38 } 39 40 func newWorkflowDeprecated( 41 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 42 ) (Type, error) { 43 w := &workflowDeprecated{ 44 log: log, 45 stats: stats, 46 mErrStages: map[string]metrics.StatCounter{}, 47 mSuccStages: map[string]metrics.StatCounter{}, 48 metaPath: nil, 49 allStages: map[string]struct{}{}, 50 } 51 if len(conf.Workflow.MetaPath) > 0 { 52 w.metaPath = gabs.DotPathToSlice(conf.Workflow.MetaPath) 53 } 54 55 explicitDeps := map[string][]string{} 56 w.children = map[string]*ProcessMap{} 57 58 for k, v := range conf.Workflow.Stages { 59 if len(processDAGStageName.FindString(k)) != len(k) { 60 return nil, fmt.Errorf("workflow stage name '%v' contains invalid characters", k) 61 } 62 63 bMgr, bLog, bStats := interop.LabelChild(k, mgr, log, stats) 64 child, err := NewProcessMap(v.ProcessMapConfig, bMgr, bLog, bStats) 65 if err != nil { 66 return nil, fmt.Errorf("failed to create child process_map '%v': %v", k, err) 67 } 68 69 w.children[k] = child 70 explicitDeps[k] = v.Dependencies 71 w.allStages[k] = struct{}{} 72 } 73 74 var err error 75 if w.dag, err = resolveProcessMapDAG(explicitDeps, w.children); err != nil { 76 return nil, err 77 } 78 79 w.mCount = stats.GetCounter("count") 80 w.mSent = stats.GetCounter("sent") 81 w.mSentParts = stats.GetCounter("parts.sent") 82 w.mSkippedNoStages = stats.GetCounter("skipped.no_stages") 83 w.mErr = stats.GetCounter("error") 84 w.mErrJSON = stats.GetCounter("error.json_parse") 85 w.mErrMeta = stats.GetCounter("error.meta_set") 86 w.mErrOverlay = stats.GetCounter("error.overlay") 87 88 w.log.Infof("Resolved workflow DAG: %v\n", w.dag) 89 return w, nil 90 } 91 92 //------------------------------------------------------------------------------ 93 94 func (w *workflowDeprecated) incrStageErr(id string) { 95 if ctr, exists := w.mErrStages[id]; exists { 96 ctr.Incr(1) 97 return 98 } 99 100 ctr := w.stats.GetCounter(fmt.Sprintf("%v.error", id)) 101 ctr.Incr(1) 102 w.mErrStages[id] = ctr 103 } 104 105 func (w *workflowDeprecated) incrStageSucc(id string) { 106 if ctr, exists := w.mSuccStages[id]; exists { 107 ctr.Incr(1) 108 return 109 } 110 111 ctr := w.stats.GetCounter(fmt.Sprintf("%v.success", id)) 112 ctr.Incr(1) 113 w.mSuccStages[id] = ctr 114 } 115 116 type deprecatedResultTracker struct { 117 succeeded map[string]struct{} 118 skipped map[string]struct{} 119 failed map[string]struct{} 120 sync.Mutex 121 } 122 123 func deprecatedTrackerFromTree(tree [][]string) *deprecatedResultTracker { 124 r := &deprecatedResultTracker{ 125 succeeded: map[string]struct{}{}, 126 skipped: map[string]struct{}{}, 127 failed: map[string]struct{}{}, 128 } 129 for _, layer := range tree { 130 for _, k := range layer { 131 r.succeeded[k] = struct{}{} 132 } 133 } 134 return r 135 } 136 137 func (r *deprecatedResultTracker) Skipped(k string) { 138 r.Lock() 139 delete(r.succeeded, k) 140 141 r.skipped[k] = struct{}{} 142 r.Unlock() 143 } 144 145 func (r *deprecatedResultTracker) Failed(k string) { 146 r.Lock() 147 delete(r.succeeded, k) 148 delete(r.skipped, k) 149 150 r.failed[k] = struct{}{} 151 r.Unlock() 152 } 153 154 func (r *deprecatedResultTracker) ToSlices() (succeeded, skipped, failed []string) { 155 r.Lock() 156 157 succeeded = make([]string, 0, len(r.succeeded)) 158 skipped = make([]string, 0, len(r.skipped)) 159 failed = make([]string, 0, len(r.failed)) 160 161 for k := range r.succeeded { 162 succeeded = append(succeeded, k) 163 } 164 sort.Strings(succeeded) 165 for k := range r.skipped { 166 skipped = append(skipped, k) 167 } 168 sort.Strings(skipped) 169 for k := range r.failed { 170 failed = append(failed, k) 171 } 172 sort.Strings(failed) 173 174 r.Unlock() 175 return 176 } 177 178 // Returns a map of enrichment IDs that should be skipped for this payload. 179 func (w *workflowDeprecated) skipFromMeta(root interface{}) map[string]struct{} { 180 skipList := map[string]struct{}{} 181 if len(w.metaPath) == 0 { 182 return skipList 183 } 184 185 gObj := gabs.Wrap(root) 186 187 // If a whitelist is provided for this flow then skip stages that aren't 188 // within it. 189 if apply, ok := gObj.S(append(w.metaPath, "apply")...).Data().([]interface{}); ok { 190 if len(apply) > 0 { 191 for k := range w.allStages { 192 skipList[k] = struct{}{} 193 } 194 for _, id := range apply { 195 if idStr, isString := id.(string); isString { 196 delete(skipList, idStr) 197 } 198 } 199 } 200 } 201 202 // Skip stages that already succeeded in a previous run of this workflow. 203 if succeeded, ok := gObj.S(append(w.metaPath, "succeeded")...).Data().([]interface{}); ok { 204 for _, id := range succeeded { 205 if idStr, isString := id.(string); isString { 206 if _, exists := w.allStages[idStr]; exists { 207 skipList[idStr] = struct{}{} 208 } 209 } 210 } 211 } 212 213 // Skip stages that were already skipped in a previous run of this workflow. 214 if skipped, ok := gObj.S(append(w.metaPath, "skipped")...).Data().([]interface{}); ok { 215 for _, id := range skipped { 216 if idStr, isString := id.(string); isString { 217 if _, exists := w.allStages[idStr]; exists { 218 skipList[idStr] = struct{}{} 219 } 220 } 221 } 222 } 223 224 return skipList 225 } 226 227 // ProcessMessage applies workflow stages to each part of a message type. 228 func (w *workflowDeprecated) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 229 w.mCount.Incr(1) 230 231 skipOnMeta := make([]map[string]struct{}, msg.Len()) 232 payload := msg.DeepCopy() 233 payload.Iter(func(i int, p types.Part) error { 234 p.Get() 235 p.Metadata() 236 if jObj, err := p.JSON(); err == nil { 237 skipOnMeta[i] = w.skipFromMeta(jObj) 238 } else { 239 skipOnMeta[i] = map[string]struct{}{} 240 } 241 return nil 242 }) 243 244 propMsg, _ := tracing.WithChildSpans("workflow", payload) 245 246 records := make([]*deprecatedResultTracker, payload.Len()) 247 for i := range records { 248 records[i] = deprecatedTrackerFromTree(w.dag) 249 } 250 251 for _, layer := range w.dag { 252 results := make([]types.Message, len(layer)) 253 errors := make([]error, len(layer)) 254 255 wg := sync.WaitGroup{} 256 wg.Add(len(layer)) 257 for i, eid := range layer { 258 go func(id string, index int) { 259 msgCopy := propMsg.Copy() 260 msgCopy.Iter(func(partIndex int, p types.Part) error { 261 if _, exists := skipOnMeta[partIndex][id]; exists { 262 p.Set(nil) 263 } 264 return nil 265 }) 266 267 var resSpans []*tracing.Span 268 results[index], resSpans = tracing.WithChildSpans(id, msgCopy) 269 errors[index] = w.children[id].CreateResult(results[index]) 270 for _, s := range resSpans { 271 s.Finish() 272 } 273 results[index].Iter(func(j int, p types.Part) error { 274 if p.IsEmpty() { 275 records[j].Skipped(id) 276 } 277 if HasFailed(p) { 278 records[j].Failed(id) 279 p.Set(nil) 280 } 281 return nil 282 }) 283 wg.Done() 284 }(eid, i) 285 } 286 wg.Wait() 287 288 for i, id := range layer { 289 var failed []int 290 err := errors[i] 291 if err == nil { 292 if failed, err = w.children[id].OverlayResult(payload, results[i]); err != nil { 293 w.mErrOverlay.Incr(1) 294 } 295 } 296 if err != nil { 297 w.incrStageErr(id) 298 w.mErr.Incr(1) 299 w.log.Errorf("Failed to perform enrichment '%v': %v\n", id, err) 300 for j := range records { 301 records[j].Failed(id) 302 } 303 continue 304 } 305 for _, j := range failed { 306 records[j].Failed(id) 307 } 308 w.incrStageSucc(id) 309 } 310 } 311 312 // Finally, set the meta records of each document. 313 if len(w.metaPath) > 0 { 314 payload.Iter(func(i int, p types.Part) error { 315 pJSON, err := p.JSON() 316 if err != nil { 317 w.mErr.Incr(1) 318 w.mErrMeta.Incr(1) 319 w.log.Errorf("Failed to parse message for meta update: %v\n", err) 320 return nil 321 } 322 323 gObj := gabs.Wrap(pJSON) 324 if oldRecord := gObj.S(w.metaPath...).Data(); oldRecord != nil { 325 gObj.Delete(w.metaPath...) 326 gObj.Set(oldRecord, append(w.metaPath, "previous")...) 327 } 328 329 succStrs, skipStrs, failStrs := records[i].ToSlices() 330 succeeded := make([]interface{}, len(succStrs)) 331 skipped := make([]interface{}, len(skipStrs)) 332 failed := make([]interface{}, len(failStrs)) 333 334 for j, v := range succStrs { 335 succeeded[j] = v 336 } 337 for j, v := range skipStrs { 338 skipped[j] = v 339 } 340 for j, v := range failStrs { 341 failed[j] = v 342 } 343 344 gObj.Set(succeeded, append(w.metaPath, "succeeded")...) 345 gObj.Set(skipped, append(w.metaPath, "skipped")...) 346 gObj.Set(failed, append(w.metaPath, "failed")...) 347 348 p.SetJSON(gObj.Data()) 349 return nil 350 }) 351 } 352 353 tracing.FinishSpans(propMsg) 354 355 w.mSentParts.Incr(int64(payload.Len())) 356 w.mSent.Incr(1) 357 msgs := [1]types.Message{payload} 358 return msgs[:], nil 359 } 360 361 // CloseAsync shuts down the processor and stops processing requests. 362 func (w *workflowDeprecated) CloseAsync() { 363 for _, c := range w.children { 364 c.CloseAsync() 365 } 366 } 367 368 // WaitForClose blocks until the processor has closed down. 369 func (w *workflowDeprecated) WaitForClose(timeout time.Duration) error { 370 stopBy := time.Now().Add(timeout) 371 for _, c := range w.children { 372 if err := c.WaitForClose(time.Until(stopBy)); err != nil { 373 return err 374 } 375 } 376 return nil 377 } 378 379 //------------------------------------------------------------------------------