github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/client/pluginmanager/csimanager/manager.go (about)

     1  package csimanager
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sync"
     7  	"time"
     8  
     9  	"github.com/hashicorp/go-hclog"
    10  	"github.com/hashicorp/nomad/client/dynamicplugins"
    11  	"github.com/hashicorp/nomad/client/pluginmanager"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  )
    14  
    15  // defaultPluginResyncPeriod is the time interval used to do a full resync
    16  // against the dynamicplugins, to account for missed updates.
    17  const defaultPluginResyncPeriod = 30 * time.Second
    18  
    19  // UpdateNodeCSIInfoFunc is the callback used to update the node from
    20  // fingerprinting
    21  type UpdateNodeCSIInfoFunc func(string, *structs.CSIInfo)
    22  type TriggerNodeEvent func(*structs.NodeEvent)
    23  
    24  type Config struct {
    25  	Logger                hclog.Logger
    26  	DynamicRegistry       dynamicplugins.Registry
    27  	UpdateNodeCSIInfoFunc UpdateNodeCSIInfoFunc
    28  	PluginResyncPeriod    time.Duration
    29  	TriggerNodeEvent      TriggerNodeEvent
    30  }
    31  
    32  // New returns a new PluginManager that will handle managing CSI plugins from
    33  // the dynamicRegistry from the provided Config.
    34  func New(config *Config) Manager {
    35  	// Use a dedicated internal context for managing plugin shutdown.
    36  	ctx, cancelFn := context.WithCancel(context.Background())
    37  	if config.PluginResyncPeriod == 0 {
    38  		config.PluginResyncPeriod = defaultPluginResyncPeriod
    39  	}
    40  
    41  	return &csiManager{
    42  		logger:    config.Logger,
    43  		eventer:   config.TriggerNodeEvent,
    44  		registry:  config.DynamicRegistry,
    45  		instances: make(map[string]map[string]*instanceManager),
    46  
    47  		updateNodeCSIInfoFunc: config.UpdateNodeCSIInfoFunc,
    48  		pluginResyncPeriod:    config.PluginResyncPeriod,
    49  
    50  		shutdownCtx:         ctx,
    51  		shutdownCtxCancelFn: cancelFn,
    52  		shutdownCh:          make(chan struct{}),
    53  	}
    54  }
    55  
    56  type csiManager struct {
    57  	// instances should only be accessed after locking with instancesLock.
    58  	// It is a map of PluginType : [PluginName : *instanceManager]
    59  	instances     map[string]map[string]*instanceManager
    60  	instancesLock sync.RWMutex
    61  
    62  	registry           dynamicplugins.Registry
    63  	logger             hclog.Logger
    64  	eventer            TriggerNodeEvent
    65  	pluginResyncPeriod time.Duration
    66  
    67  	updateNodeCSIInfoFunc UpdateNodeCSIInfoFunc
    68  
    69  	shutdownCtx         context.Context
    70  	shutdownCtxCancelFn context.CancelFunc
    71  	shutdownCh          chan struct{}
    72  }
    73  
    74  func (c *csiManager) PluginManager() pluginmanager.PluginManager {
    75  	return c
    76  }
    77  
    78  func (c *csiManager) MounterForPlugin(ctx context.Context, pluginID string) (VolumeMounter, error) {
    79  	c.instancesLock.RLock()
    80  	defer c.instancesLock.RUnlock()
    81  	nodePlugins, hasAnyNodePlugins := c.instances["csi-node"]
    82  	if !hasAnyNodePlugins {
    83  		return nil, fmt.Errorf("no storage node plugins found")
    84  	}
    85  
    86  	mgr, hasPlugin := nodePlugins[pluginID]
    87  	if !hasPlugin {
    88  		return nil, fmt.Errorf("plugin %s for type csi-node not found", pluginID)
    89  	}
    90  
    91  	return mgr.VolumeMounter(ctx)
    92  }
    93  
    94  // Run starts a plugin manager and should return early
    95  func (c *csiManager) Run() {
    96  	go c.runLoop()
    97  }
    98  
    99  func (c *csiManager) runLoop() {
   100  	timer := time.NewTimer(0) // ensure we sync immediately in first pass
   101  	controllerUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-controller")
   102  	nodeUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-node")
   103  	for {
   104  		select {
   105  		case <-timer.C:
   106  			c.resyncPluginsFromRegistry("csi-controller")
   107  			c.resyncPluginsFromRegistry("csi-node")
   108  			timer.Reset(c.pluginResyncPeriod)
   109  		case event := <-controllerUpdates:
   110  			c.handlePluginEvent(event)
   111  		case event := <-nodeUpdates:
   112  			c.handlePluginEvent(event)
   113  		case <-c.shutdownCtx.Done():
   114  			close(c.shutdownCh)
   115  			return
   116  		}
   117  	}
   118  }
   119  
   120  // resyncPluginsFromRegistry does a full sync of the running instance
   121  // managers against those in the registry. we primarily will use update
   122  // events from the registry.
   123  func (c *csiManager) resyncPluginsFromRegistry(ptype string) {
   124  
   125  	c.instancesLock.Lock()
   126  	defer c.instancesLock.Unlock()
   127  
   128  	plugins := c.registry.ListPlugins(ptype)
   129  	seen := make(map[string]struct{}, len(plugins))
   130  
   131  	// For every plugin in the registry, ensure that we have an existing plugin
   132  	// running. Also build the map of valid plugin names.
   133  	// Note: monolith plugins that run as both controllers and nodes get a
   134  	// separate instance manager for both modes.
   135  	for _, plugin := range plugins {
   136  		seen[plugin.Name] = struct{}{}
   137  		c.ensureInstance(plugin)
   138  	}
   139  
   140  	// For every instance manager, if we did not find it during the plugin
   141  	// iterator, shut it down and remove it from the table.
   142  	instances := c.instancesForType(ptype)
   143  	for name, mgr := range instances {
   144  		if _, ok := seen[name]; !ok {
   145  			c.ensureNoInstance(mgr.info)
   146  		}
   147  	}
   148  }
   149  
   150  // handlePluginEvent syncs a single event against the plugin registry
   151  func (c *csiManager) handlePluginEvent(event *dynamicplugins.PluginUpdateEvent) {
   152  	if event == nil || event.Info == nil {
   153  		return
   154  	}
   155  	c.logger.Trace("dynamic plugin event",
   156  		"event", event.EventType,
   157  		"plugin_id", event.Info.Name,
   158  		"plugin_alloc_id", event.Info.AllocID)
   159  
   160  	c.instancesLock.Lock()
   161  	defer c.instancesLock.Unlock()
   162  
   163  	switch event.EventType {
   164  	case dynamicplugins.EventTypeRegistered:
   165  		c.ensureInstance(event.Info)
   166  	case dynamicplugins.EventTypeDeregistered:
   167  		c.ensureNoInstance(event.Info)
   168  	default:
   169  		c.logger.Error("received unknown dynamic plugin event type",
   170  			"type", event.EventType)
   171  	}
   172  }
   173  
   174  // Ensure we have an instance manager for the plugin and add it to
   175  // the CSI manager's tracking table for that plugin type.
   176  // Assumes that c.instances has been locked.
   177  func (c *csiManager) ensureInstance(plugin *dynamicplugins.PluginInfo) {
   178  	name := plugin.Name
   179  	ptype := plugin.Type
   180  	instances := c.instancesForType(ptype)
   181  	mgr, ok := instances[name]
   182  	if !ok {
   183  		c.logger.Debug("detected new CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID)
   184  		mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin)
   185  		instances[name] = mgr
   186  		mgr.run()
   187  	} else if mgr.allocID != plugin.AllocID {
   188  		mgr.shutdown()
   189  		c.logger.Debug("detected update for CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID)
   190  		mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin)
   191  		instances[name] = mgr
   192  		mgr.run()
   193  
   194  	}
   195  }
   196  
   197  // Shut down the instance manager for a plugin and remove it from
   198  // the CSI manager's tracking table for that plugin type.
   199  // Assumes that c.instances has been locked.
   200  func (c *csiManager) ensureNoInstance(plugin *dynamicplugins.PluginInfo) {
   201  	name := plugin.Name
   202  	ptype := plugin.Type
   203  	instances := c.instancesForType(ptype)
   204  	if mgr, ok := instances[name]; ok {
   205  		if mgr.allocID == plugin.AllocID {
   206  			c.logger.Debug("shutting down CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID)
   207  			mgr.shutdown()
   208  			delete(instances, name)
   209  		}
   210  	}
   211  }
   212  
   213  // Get the instance managers table for a specific plugin type,
   214  // ensuring it's been initialized if it doesn't exist.
   215  // Assumes that c.instances has been locked.
   216  func (c *csiManager) instancesForType(ptype string) map[string]*instanceManager {
   217  	pluginMap, ok := c.instances[ptype]
   218  	if !ok {
   219  		pluginMap = make(map[string]*instanceManager)
   220  		c.instances[ptype] = pluginMap
   221  	}
   222  	return pluginMap
   223  }
   224  
   225  // Shutdown should gracefully shutdown all plugins managed by the manager.
   226  // It must block until shutdown is complete
   227  func (c *csiManager) Shutdown() {
   228  	// Shut down the run loop
   229  	c.shutdownCtxCancelFn()
   230  
   231  	// Wait for plugin manager shutdown to complete so that we
   232  	// don't try to shutdown instance managers while runLoop is
   233  	// doing a resync
   234  	<-c.shutdownCh
   235  
   236  	// Shutdown all the instance managers in parallel
   237  	var wg sync.WaitGroup
   238  	for _, pluginMap := range c.instances {
   239  		for _, mgr := range pluginMap {
   240  			wg.Add(1)
   241  			go func(mgr *instanceManager) {
   242  				mgr.shutdown()
   243  				wg.Done()
   244  			}(mgr)
   245  		}
   246  	}
   247  	wg.Wait()
   248  }
   249  
   250  // PluginType is the type of plugin which the manager manages
   251  func (c *csiManager) PluginType() string {
   252  	return "csi"
   253  }