github.com/iron-io/functions@v0.0.0-20180820112432-d59d7d1c40b2/api/runner/worker.go (about) 1 package runner 2 3 import ( 4 "bufio" 5 "context" 6 "fmt" 7 "io" 8 "sync" 9 "time" 10 11 "github.com/Sirupsen/logrus" 12 "github.com/iron-io/functions/api/runner/protocol" 13 "github.com/iron-io/functions/api/runner/task" 14 "github.com/iron-io/runner/drivers" 15 ) 16 17 // hot functions - theory of operation 18 // 19 // A function is converted into a hot function if its `Format` is either 20 // a streamable format/protocol. At the very first task request a hot 21 // container shall be started and run it. Each hot function has an internal 22 // clock that actually halts the container if it goes idle long enough. In the 23 // absence of workload, it just stops the whole clockwork. 24 // 25 // Internally, the hot function uses a modified Config whose Stdin and Stdout 26 // are bound to an internal pipe. This internal pipe is fed with incoming tasks 27 // Stdin and feeds incoming tasks with Stdout. 28 // 29 // Each execution is the alternation of feeding hot functions stdin with tasks 30 // stdin, and reading the answer back from containers stdout. For all `Format`s 31 // we send embedded into the message metadata to help the container to know when 32 // to stop reading from its stdin and Functions expect the container to do the 33 // same. Refer to api/runner/protocol.go for details of these communications. 34 // 35 // hot functions implementation relies in two moving parts (drawn below): 36 // htfnmgr and htfn. Refer to their respective comments for 37 // details. 38 // │ 39 // Incoming 40 // Task 41 // │ 42 // ▼ 43 // ┌───────────────┐ 44 // │ Task Request │ 45 // │ Main Loop │ 46 // └───────────────┘ 47 // │ 48 // ┌──────▼────────┐ 49 // ┌┴──────────────┐│ 50 // │ Per Function ││ non-streamable f() 51 // ┌───────│ Container │├──────┐───────────────┐ 52 // │ │ Manager ├┘ │ │ 53 // │ └───────────────┘ │ │ 54 // │ │ │ │ 55 // ▼ ▼ ▼ ▼ 56 // ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ 57 // │ Hot │ │ Hot │ │ Hot │ │ Cold │ 58 // │ Function │ │ Function │ │ Function │ │ Function │ 59 // └───────────┘ └───────────┘ └───────────┘ └───────────┘ 60 // Timeout 61 // Terminate 62 // (internal clock) 63 64 // RunTask helps sending a task.Request into the common concurrency stream. 65 // Refer to StartWorkers() to understand what this is about. 66 func RunTask(tasks chan task.Request, ctx context.Context, cfg *task.Config) (drivers.RunResult, error) { 67 tresp := make(chan task.Response) 68 treq := task.Request{Ctx: ctx, Config: cfg, Response: tresp} 69 tasks <- treq 70 resp := <-treq.Response 71 return resp.Result, resp.Err 72 } 73 74 // StartWorkers operates the common concurrency stream, ie, it will process all 75 // IronFunctions tasks, either sync or async. In the process, it also dispatches 76 // the workload to either regular or hot functions. 77 func StartWorkers(ctx context.Context, rnr *Runner, tasks <-chan task.Request) { 78 var wg sync.WaitGroup 79 defer wg.Wait() 80 var hcmgr htfnmgr 81 82 for { 83 select { 84 case <-ctx.Done(): 85 return 86 case task := <-tasks: 87 p := hcmgr.getPipe(ctx, rnr, task.Config) 88 if p == nil { 89 wg.Add(1) 90 go runTaskReq(rnr, &wg, task) 91 continue 92 } 93 94 rnr.Start() 95 select { 96 case <-ctx.Done(): 97 return 98 case p <- task: 99 rnr.Complete() 100 } 101 } 102 } 103 } 104 105 // htfnmgr is the intermediate between the common concurrency stream and 106 // hot functions. All hot functions share a single task.Request stream per 107 // function (chn), but each function may have more than one hot function (hc). 108 type htfnmgr struct { 109 chn map[string]chan task.Request 110 hc map[string]*htfnsvr 111 } 112 113 func (h *htfnmgr) getPipe(ctx context.Context, rnr *Runner, cfg *task.Config) chan task.Request { 114 isStream, err := protocol.IsStreamable(cfg.Format) 115 if err != nil { 116 logrus.WithError(err).Info("could not detect container IO protocol") 117 return nil 118 } else if !isStream { 119 return nil 120 } 121 122 if h.chn == nil { 123 h.chn = make(map[string]chan task.Request) 124 h.hc = make(map[string]*htfnsvr) 125 } 126 127 // TODO(ccirello): re-implement this without memory allocation (fmt.Sprint) 128 fn := fmt.Sprint(cfg.AppName, ",", cfg.Path, cfg.Image, cfg.Timeout, cfg.Memory, cfg.Format, cfg.MaxConcurrency) 129 tasks, ok := h.chn[fn] 130 if !ok { 131 h.chn[fn] = make(chan task.Request) 132 tasks = h.chn[fn] 133 svr := newhtfnsvr(ctx, cfg, rnr, tasks) 134 if err := svr.launch(ctx); err != nil { 135 logrus.WithError(err).Error("cannot start hot function supervisor") 136 return nil 137 } 138 h.hc[fn] = svr 139 } 140 141 return tasks 142 } 143 144 // htfnsvr is part of htfnmgr, abstracted apart for simplicity, its only 145 // purpose is to test for hot functions saturation and try starting as many as 146 // needed. In case of absence of workload, it will stop trying to start new hot 147 // containers. 148 type htfnsvr struct { 149 cfg *task.Config 150 rnr *Runner 151 tasksin <-chan task.Request 152 tasksout chan task.Request 153 maxc chan struct{} 154 } 155 156 func newhtfnsvr(ctx context.Context, cfg *task.Config, rnr *Runner, tasks <-chan task.Request) *htfnsvr { 157 svr := &htfnsvr{ 158 cfg: cfg, 159 rnr: rnr, 160 tasksin: tasks, 161 tasksout: make(chan task.Request, 1), 162 maxc: make(chan struct{}, cfg.MaxConcurrency), 163 } 164 165 // This pipe will take all incoming tasks and just forward them to the 166 // started hot functions. The catch here is that it feeds a buffered 167 // channel from an unbuffered one. And this buffered channel is 168 // then used to determine the presence of running hot functions. 169 // If no hot function is available, tasksout will fill up to its 170 // capacity and pipe() will start them. 171 go svr.pipe(ctx) 172 return svr 173 } 174 175 func (svr *htfnsvr) pipe(ctx context.Context) { 176 for { 177 select { 178 case t := <-svr.tasksin: 179 svr.tasksout <- t 180 if len(svr.tasksout) > 0 { 181 if err := svr.launch(ctx); err != nil { 182 logrus.WithError(err).Error("cannot start more hot functions") 183 } 184 } 185 case <-ctx.Done(): 186 return 187 } 188 } 189 } 190 191 func (svr *htfnsvr) launch(ctx context.Context) error { 192 select { 193 case svr.maxc <- struct{}{}: 194 hc, err := newhtfn( 195 svr.cfg, 196 protocol.Protocol(svr.cfg.Format), 197 svr.tasksout, 198 svr.rnr, 199 ) 200 if err != nil { 201 return err 202 } 203 go func() { 204 hc.serve(ctx) 205 <-svr.maxc 206 }() 207 default: 208 } 209 210 return nil 211 } 212 213 // htfn actually interfaces an incoming task from the common concurrency 214 // stream into a long lived container. If idle long enough, it will stop. It 215 // uses route configuration to determine which protocol to use. 216 type htfn struct { 217 cfg *task.Config 218 proto protocol.ContainerIO 219 tasks <-chan task.Request 220 221 // Side of the pipe that takes information from outer world 222 // and injects into the container. 223 in io.Writer 224 out io.Reader 225 226 // Receiving side of the container. 227 containerIn io.Reader 228 containerOut io.Writer 229 230 rnr *Runner 231 } 232 233 func newhtfn(cfg *task.Config, proto protocol.Protocol, tasks <-chan task.Request, rnr *Runner) (*htfn, error) { 234 stdinr, stdinw := io.Pipe() 235 stdoutr, stdoutw := io.Pipe() 236 237 p, err := protocol.New(proto, stdinw, stdoutr) 238 if err != nil { 239 return nil, err 240 } 241 242 hc := &htfn{ 243 cfg: cfg, 244 proto: p, 245 tasks: tasks, 246 247 in: stdinw, 248 out: stdoutr, 249 250 containerIn: stdinr, 251 containerOut: stdoutw, 252 253 rnr: rnr, 254 } 255 256 return hc, nil 257 } 258 259 func (hc *htfn) serve(ctx context.Context) { 260 lctx, cancel := context.WithCancel(ctx) 261 var wg sync.WaitGroup 262 cfg := *hc.cfg 263 logger := logrus.WithFields(logrus.Fields{ 264 "app": cfg.AppName, 265 "route": cfg.Path, 266 "image": cfg.Image, 267 "memory": cfg.Memory, 268 "format": cfg.Format, 269 "max_concurrency": cfg.MaxConcurrency, 270 "idle_timeout": cfg.IdleTimeout, 271 }) 272 273 wg.Add(1) 274 go func() { 275 defer wg.Done() 276 for { 277 inactivity := time.After(cfg.IdleTimeout) 278 279 select { 280 case <-lctx.Done(): 281 return 282 283 case <-inactivity: 284 logger.Info("Canceling inactive hot function") 285 cancel() 286 287 case t := <-hc.tasks: 288 if err := hc.proto.Dispatch(lctx, t); err != nil { 289 logrus.WithField("ctx", lctx).Info("task failed") 290 t.Response <- task.Response{ 291 &runResult{StatusValue: "error", error: err}, 292 err, 293 } 294 continue 295 } 296 297 t.Response <- task.Response{ 298 &runResult{StatusValue: "success"}, 299 nil, 300 } 301 } 302 } 303 }() 304 305 cfg.Env["FN_FORMAT"] = cfg.Format 306 cfg.Timeout = 0 // add a timeout to simulate ab.end. failure. 307 cfg.Stdin = hc.containerIn 308 cfg.Stdout = hc.containerOut 309 310 // Why can we not attach stderr to the task like we do for stdin and 311 // stdout? 312 // 313 // Stdin/Stdout are completely known to the scope of the task. You must 314 // have a task stdin to feed containers stdin, and also the other way 315 // around when reading from stdout. So both are directly related to the 316 // life cycle of the request. 317 // 318 // Stderr, on the other hand, can be written by anything any time: 319 // failure between requests, failures inside requests and messages send 320 // right after stdout has been finished being transmitted. Thus, with 321 // hot functions, there is not a 1:1 relation between stderr and tasks. 322 // 323 // Still, we do pass - at protocol level - a Task-ID header, from which 324 // the application running inside the hot function can use to identify 325 // its own stderr output. 326 errr, errw := io.Pipe() 327 cfg.Stderr = errw 328 wg.Add(1) 329 go func() { 330 defer wg.Done() 331 scanner := bufio.NewScanner(errr) 332 for scanner.Scan() { 333 logger.Info(scanner.Text()) 334 } 335 }() 336 337 result, err := hc.rnr.Run(lctx, &cfg) 338 if err != nil { 339 logrus.WithError(err).Error("hot function failure detected") 340 } 341 errw.Close() 342 wg.Wait() 343 logrus.WithField("result", result).Info("hot function terminated") 344 } 345 346 func runTaskReq(rnr *Runner, wg *sync.WaitGroup, t task.Request) { 347 defer wg.Done() 348 rnr.Start() 349 defer rnr.Complete() 350 result, err := rnr.Run(t.Ctx, t.Config) 351 select { 352 case t.Response <- task.Response{result, err}: 353 close(t.Response) 354 default: 355 } 356 } 357 358 type runResult struct { 359 error 360 StatusValue string 361 } 362 363 func (r *runResult) Error() string { 364 if r.error == nil { 365 return "" 366 } 367 return r.error.Error() 368 } 369 370 func (r *runResult) Status() string { return r.StatusValue }