github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/client/pluginmanager/csimanager/manager.go (about)

     1  package csimanager
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sync"
     7  	"time"
     8  
     9  	"github.com/hashicorp/go-hclog"
    10  	"github.com/hashicorp/nomad/client/dynamicplugins"
    11  	"github.com/hashicorp/nomad/client/pluginmanager"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  )
    14  
    15  // defaultPluginResyncPeriod is the time interval used to do a full resync
    16  // against the dynamicplugins, to account for missed updates.
    17  const defaultPluginResyncPeriod = 30 * time.Second
    18  
    19  // UpdateNodeCSIInfoFunc is the callback used to update the node from
    20  // fingerprinting
    21  type UpdateNodeCSIInfoFunc func(string, *structs.CSIInfo)
    22  type TriggerNodeEvent func(*structs.NodeEvent)
    23  
    24  type Config struct {
    25  	Logger                hclog.Logger
    26  	DynamicRegistry       dynamicplugins.Registry
    27  	UpdateNodeCSIInfoFunc UpdateNodeCSIInfoFunc
    28  	PluginResyncPeriod    time.Duration
    29  	TriggerNodeEvent      TriggerNodeEvent
    30  }
    31  
    32  // New returns a new PluginManager that will handle managing CSI plugins from
    33  // the dynamicRegistry from the provided Config.
    34  func New(config *Config) Manager {
    35  	// Use a dedicated internal context for managing plugin shutdown.
    36  	ctx, cancelFn := context.WithCancel(context.Background())
    37  	if config.PluginResyncPeriod == 0 {
    38  		config.PluginResyncPeriod = defaultPluginResyncPeriod
    39  	}
    40  
    41  	return &csiManager{
    42  		logger:    config.Logger,
    43  		eventer:   config.TriggerNodeEvent,
    44  		registry:  config.DynamicRegistry,
    45  		instances: make(map[string]map[string]*instanceManager),
    46  
    47  		updateNodeCSIInfoFunc: config.UpdateNodeCSIInfoFunc,
    48  		pluginResyncPeriod:    config.PluginResyncPeriod,
    49  
    50  		shutdownCtx:         ctx,
    51  		shutdownCtxCancelFn: cancelFn,
    52  		shutdownCh:          make(chan struct{}),
    53  	}
    54  }
    55  
    56  type csiManager struct {
    57  	// instances should only be accessed from the run() goroutine and the shutdown
    58  	// fn. It is a map of PluginType : [PluginName : instanceManager]
    59  	instances map[string]map[string]*instanceManager
    60  
    61  	registry           dynamicplugins.Registry
    62  	logger             hclog.Logger
    63  	eventer            TriggerNodeEvent
    64  	pluginResyncPeriod time.Duration
    65  
    66  	updateNodeCSIInfoFunc UpdateNodeCSIInfoFunc
    67  
    68  	shutdownCtx         context.Context
    69  	shutdownCtxCancelFn context.CancelFunc
    70  	shutdownCh          chan struct{}
    71  }
    72  
    73  func (c *csiManager) PluginManager() pluginmanager.PluginManager {
    74  	return c
    75  }
    76  
    77  func (c *csiManager) MounterForPlugin(ctx context.Context, pluginID string) (VolumeMounter, error) {
    78  	nodePlugins, hasAnyNodePlugins := c.instances["csi-node"]
    79  	if !hasAnyNodePlugins {
    80  		return nil, fmt.Errorf("no storage node plugins found")
    81  	}
    82  
    83  	mgr, hasPlugin := nodePlugins[pluginID]
    84  	if !hasPlugin {
    85  		return nil, fmt.Errorf("plugin %s for type csi-node not found", pluginID)
    86  	}
    87  
    88  	return mgr.VolumeMounter(ctx)
    89  }
    90  
    91  // Run starts a plugin manager and should return early
    92  func (c *csiManager) Run() {
    93  	go c.runLoop()
    94  }
    95  
    96  func (c *csiManager) runLoop() {
    97  	timer := time.NewTimer(0) // ensure we sync immediately in first pass
    98  	controllerUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-controller")
    99  	nodeUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-node")
   100  	for {
   101  		select {
   102  		case <-timer.C:
   103  			c.resyncPluginsFromRegistry("csi-controller")
   104  			c.resyncPluginsFromRegistry("csi-node")
   105  			timer.Reset(c.pluginResyncPeriod)
   106  		case event := <-controllerUpdates:
   107  			c.handlePluginEvent(event)
   108  		case event := <-nodeUpdates:
   109  			c.handlePluginEvent(event)
   110  		case <-c.shutdownCtx.Done():
   111  			close(c.shutdownCh)
   112  			return
   113  		}
   114  	}
   115  }
   116  
   117  // resyncPluginsFromRegistry does a full sync of the running instance
   118  // managers against those in the registry. we primarily will use update
   119  // events from the registry.
   120  func (c *csiManager) resyncPluginsFromRegistry(ptype string) {
   121  	plugins := c.registry.ListPlugins(ptype)
   122  	seen := make(map[string]struct{}, len(plugins))
   123  
   124  	// For every plugin in the registry, ensure that we have an existing plugin
   125  	// running. Also build the map of valid plugin names.
   126  	// Note: monolith plugins that run as both controllers and nodes get a
   127  	// separate instance manager for both modes.
   128  	for _, plugin := range plugins {
   129  		seen[plugin.Name] = struct{}{}
   130  		c.ensureInstance(plugin)
   131  	}
   132  
   133  	// For every instance manager, if we did not find it during the plugin
   134  	// iterator, shut it down and remove it from the table.
   135  	instances := c.instancesForType(ptype)
   136  	for name, mgr := range instances {
   137  		if _, ok := seen[name]; !ok {
   138  			c.ensureNoInstance(mgr.info)
   139  		}
   140  	}
   141  }
   142  
   143  // handlePluginEvent syncs a single event against the plugin registry
   144  func (c *csiManager) handlePluginEvent(event *dynamicplugins.PluginUpdateEvent) {
   145  	if event == nil {
   146  		return
   147  	}
   148  	c.logger.Trace("dynamic plugin event",
   149  		"event", event.EventType,
   150  		"plugin_id", event.Info.Name,
   151  		"plugin_alloc_id", event.Info.AllocID)
   152  
   153  	switch event.EventType {
   154  	case dynamicplugins.EventTypeRegistered:
   155  		c.ensureInstance(event.Info)
   156  	case dynamicplugins.EventTypeDeregistered:
   157  		c.ensureNoInstance(event.Info)
   158  	default:
   159  		c.logger.Error("received unknown dynamic plugin event type",
   160  			"type", event.EventType)
   161  	}
   162  }
   163  
   164  // Ensure we have an instance manager for the plugin and add it to
   165  // the CSI manager's tracking table for that plugin type.
   166  func (c *csiManager) ensureInstance(plugin *dynamicplugins.PluginInfo) {
   167  	name := plugin.Name
   168  	ptype := plugin.Type
   169  	instances := c.instancesForType(ptype)
   170  	if _, ok := instances[name]; !ok {
   171  		c.logger.Debug("detected new CSI plugin", "name", name, "type", ptype)
   172  		mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin)
   173  		instances[name] = mgr
   174  		mgr.run()
   175  	}
   176  }
   177  
   178  // Shut down the instance manager for a plugin and remove it from
   179  // the CSI manager's tracking table for that plugin type.
   180  func (c *csiManager) ensureNoInstance(plugin *dynamicplugins.PluginInfo) {
   181  	name := plugin.Name
   182  	ptype := plugin.Type
   183  	instances := c.instancesForType(ptype)
   184  	if mgr, ok := instances[name]; ok {
   185  		c.logger.Debug("shutting down CSI plugin", "name", name, "type", ptype)
   186  		mgr.shutdown()
   187  		delete(instances, name)
   188  	}
   189  }
   190  
   191  // Get the instance managers table for a specific plugin type,
   192  // ensuring it's been initialized if it doesn't exist.
   193  func (c *csiManager) instancesForType(ptype string) map[string]*instanceManager {
   194  	pluginMap, ok := c.instances[ptype]
   195  	if !ok {
   196  		pluginMap = make(map[string]*instanceManager)
   197  		c.instances[ptype] = pluginMap
   198  	}
   199  	return pluginMap
   200  }
   201  
   202  // Shutdown should gracefully shutdown all plugins managed by the manager.
   203  // It must block until shutdown is complete
   204  func (c *csiManager) Shutdown() {
   205  	// Shut down the run loop
   206  	c.shutdownCtxCancelFn()
   207  
   208  	// Wait for plugin manager shutdown to complete so that we
   209  	// don't try to shutdown instance managers while runLoop is
   210  	// doing a resync
   211  	<-c.shutdownCh
   212  
   213  	// Shutdown all the instance managers in parallel
   214  	var wg sync.WaitGroup
   215  	for _, pluginMap := range c.instances {
   216  		for _, mgr := range pluginMap {
   217  			wg.Add(1)
   218  			go func(mgr *instanceManager) {
   219  				mgr.shutdown()
   220  				wg.Done()
   221  			}(mgr)
   222  		}
   223  	}
   224  	wg.Wait()
   225  }
   226  
   227  // PluginType is the type of plugin which the manager manages
   228  func (c *csiManager) PluginType() string {
   229  	return "csi"
   230  }