github.com/whiteboxio/flow@v0.0.3-0.20190918184116-508d75d68a2c/pkg/corev1alpha1/pipeline/pipeline.go (about)

     1  package pipeline
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  
     7  	core "github.com/awesome-flow/flow/pkg/corev1alpha1"
     8  	"github.com/awesome-flow/flow/pkg/types"
     9  	"github.com/awesome-flow/flow/pkg/util/data"
    10  )
    11  
    12  type Pipeline struct {
    13  	ctx       *core.Context
    14  	actors    map[string]core.Actor
    15  	topology  *data.Topology
    16  	factories map[string]ActorFactory
    17  }
    18  
    19  var _ core.Runner = (*Pipeline)(nil)
    20  
    21  func NewPipeline(ctx *core.Context) (*Pipeline, error) {
    22  	return NewPipelineWithFactories(
    23  		ctx,
    24  		map[string]ActorFactory{
    25  			"core":   NewCoreActorFactory(),
    26  			"plugin": NewPluginActorFactory(),
    27  		},
    28  	)
    29  }
    30  
    31  func NewPipelineWithFactories(ctx *core.Context, factories map[string]ActorFactory) (*Pipeline, error) {
    32  	actors, err := buildActors(ctx, factories)
    33  	if err != nil {
    34  		return nil, err
    35  	}
    36  
    37  	topology, err := buildTopology(ctx, actors)
    38  	if err != nil {
    39  		return nil, err
    40  	}
    41  
    42  	p := &Pipeline{
    43  		ctx:       ctx,
    44  		actors:    actors,
    45  		topology:  topology,
    46  		factories: factories,
    47  	}
    48  
    49  	return p, nil
    50  }
    51  
    52  func (p *Pipeline) Start() error {
    53  	actors, err := p.topology.Sort()
    54  	if err != nil {
    55  		return err
    56  	}
    57  	for _, actor := range actors {
    58  		p.ctx.Logger().Trace("starting %s", actor.(core.Actor).Name())
    59  		if err := actor.(core.Actor).Start(); err != nil {
    60  			return err
    61  		}
    62  	}
    63  
    64  	return nil
    65  }
    66  
    67  func (p *Pipeline) Stop() error {
    68  	actors, err := p.topology.Sort()
    69  	if err != nil {
    70  		return err
    71  	}
    72  	l := len(actors)
    73  	for i := 0; i < l/2; i++ {
    74  		actors[i], actors[l-i-1] = actors[l-i-1], actors[i]
    75  	}
    76  	for _, actor := range actors {
    77  		p.ctx.Logger().Trace("stopping %s", actor.(core.Actor).Name())
    78  		if err := actor.(core.Actor).Stop(); err != nil {
    79  			return err
    80  		}
    81  	}
    82  
    83  	return nil
    84  }
    85  
    86  func (p *Pipeline) Context() *core.Context {
    87  	return p.ctx
    88  }
    89  
    90  func buildActors(ctx *core.Context, factories map[string]ActorFactory) (map[string]core.Actor, error) {
    91  	actblocks, ok := ctx.Config().Get(types.NewKey("actors"))
    92  	if !ok {
    93  		return nil, fmt.Errorf("`actors` config is missing")
    94  	}
    95  	actors := make(map[string]core.Actor)
    96  
    97  	for name, actorcfg := range actblocks.(map[string]types.CfgBlockActor) {
    98  		module := actorcfg.Module
    99  
   100  		factkey := strings.Split(module, ".")[0]
   101  		if len(factkey) == 0 {
   102  			factkey = module
   103  		}
   104  
   105  		if _, ok := factories[factkey]; !ok {
   106  			return nil, fmt.Errorf("failed to find an actor factory for key %s", factkey)
   107  		}
   108  
   109  		actor, err := factories[factkey].Build(name, ctx, &actorcfg)
   110  		if err != nil {
   111  			return nil, err
   112  		}
   113  
   114  		actors[name] = actor
   115  	}
   116  
   117  	return actors, nil
   118  }
   119  
   120  func buildTopology(ctx *core.Context, actors map[string]core.Actor) (*data.Topology, error) {
   121  	topology := data.NewTopology()
   122  	for _, actor := range actors {
   123  		topology.AddNode(actor)
   124  	}
   125  
   126  	pipeline, ok := ctx.Config().Get(types.NewKey("pipeline"))
   127  	if !ok {
   128  		return nil, fmt.Errorf("pipeline config is missing")
   129  	}
   130  
   131  	nthreads, _ := ctx.Config().Get(types.NewKey("system.maxprocs"))
   132  
   133  	for name, cfg := range pipeline.(map[string]types.CfgBlockPipeline) {
   134  		actor, ok := actors[name]
   135  		if !ok {
   136  			return nil, fmt.Errorf("unknown actor in the pipeline config: %s", name)
   137  		}
   138  		if len(cfg.Connect) != 0 {
   139  			for _, connect := range cfg.Connect {
   140  				peer, ok := actors[connect]
   141  				if !ok {
   142  					return nil, fmt.Errorf("unknown peer in the pipeline config: %s", cfg.Connect)
   143  				}
   144  				if err := actor.Connect(nthreads.(int), peer); err != nil {
   145  					return nil, err
   146  				}
   147  				if err := topology.Connect(actor, peer); err != nil {
   148  					return nil, err
   149  				}
   150  			}
   151  		}
   152  	}
   153  
   154  	return topology, nil
   155  }