github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/client/pluginmanager/csimanager/manager.go (about) 1 package csimanager 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 "time" 8 9 "github.com/hashicorp/go-hclog" 10 "github.com/hashicorp/nomad/client/dynamicplugins" 11 "github.com/hashicorp/nomad/client/pluginmanager" 12 "github.com/hashicorp/nomad/nomad/structs" 13 ) 14 15 // defaultPluginResyncPeriod is the time interval used to do a full resync 16 // against the dynamicplugins, to account for missed updates. 17 const defaultPluginResyncPeriod = 30 * time.Second 18 19 // UpdateNodeCSIInfoFunc is the callback used to update the node from 20 // fingerprinting 21 type UpdateNodeCSIInfoFunc func(string, *structs.CSIInfo) 22 type TriggerNodeEvent func(*structs.NodeEvent) 23 24 type Config struct { 25 Logger hclog.Logger 26 DynamicRegistry dynamicplugins.Registry 27 UpdateNodeCSIInfoFunc UpdateNodeCSIInfoFunc 28 PluginResyncPeriod time.Duration 29 TriggerNodeEvent TriggerNodeEvent 30 } 31 32 // New returns a new PluginManager that will handle managing CSI plugins from 33 // the dynamicRegistry from the provided Config. 34 func New(config *Config) Manager { 35 // Use a dedicated internal context for managing plugin shutdown. 36 ctx, cancelFn := context.WithCancel(context.Background()) 37 if config.PluginResyncPeriod == 0 { 38 config.PluginResyncPeriod = defaultPluginResyncPeriod 39 } 40 41 return &csiManager{ 42 logger: config.Logger, 43 eventer: config.TriggerNodeEvent, 44 registry: config.DynamicRegistry, 45 instances: make(map[string]map[string]*instanceManager), 46 47 updateNodeCSIInfoFunc: config.UpdateNodeCSIInfoFunc, 48 pluginResyncPeriod: config.PluginResyncPeriod, 49 50 shutdownCtx: ctx, 51 shutdownCtxCancelFn: cancelFn, 52 shutdownCh: make(chan struct{}), 53 } 54 } 55 56 type csiManager struct { 57 // instances should only be accessed after locking with instancesLock. 58 // It is a map of PluginType : [PluginName : *instanceManager] 59 instances map[string]map[string]*instanceManager 60 instancesLock sync.RWMutex 61 62 registry dynamicplugins.Registry 63 logger hclog.Logger 64 eventer TriggerNodeEvent 65 pluginResyncPeriod time.Duration 66 67 updateNodeCSIInfoFunc UpdateNodeCSIInfoFunc 68 69 shutdownCtx context.Context 70 shutdownCtxCancelFn context.CancelFunc 71 shutdownCh chan struct{} 72 } 73 74 func (c *csiManager) PluginManager() pluginmanager.PluginManager { 75 return c 76 } 77 78 func (c *csiManager) MounterForPlugin(ctx context.Context, pluginID string) (VolumeMounter, error) { 79 c.instancesLock.RLock() 80 defer c.instancesLock.RUnlock() 81 nodePlugins, hasAnyNodePlugins := c.instances["csi-node"] 82 if !hasAnyNodePlugins { 83 return nil, fmt.Errorf("no storage node plugins found") 84 } 85 86 mgr, hasPlugin := nodePlugins[pluginID] 87 if !hasPlugin { 88 return nil, fmt.Errorf("plugin %s for type csi-node not found", pluginID) 89 } 90 91 return mgr.VolumeMounter(ctx) 92 } 93 94 // Run starts a plugin manager and should return early 95 func (c *csiManager) Run() { 96 go c.runLoop() 97 } 98 99 func (c *csiManager) runLoop() { 100 timer := time.NewTimer(0) // ensure we sync immediately in first pass 101 controllerUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-controller") 102 nodeUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-node") 103 for { 104 select { 105 case <-timer.C: 106 c.resyncPluginsFromRegistry("csi-controller") 107 c.resyncPluginsFromRegistry("csi-node") 108 timer.Reset(c.pluginResyncPeriod) 109 case event := <-controllerUpdates: 110 c.handlePluginEvent(event) 111 case event := <-nodeUpdates: 112 c.handlePluginEvent(event) 113 case <-c.shutdownCtx.Done(): 114 close(c.shutdownCh) 115 return 116 } 117 } 118 } 119 120 // resyncPluginsFromRegistry does a full sync of the running instance 121 // managers against those in the registry. we primarily will use update 122 // events from the registry. 123 func (c *csiManager) resyncPluginsFromRegistry(ptype string) { 124 125 c.instancesLock.Lock() 126 defer c.instancesLock.Unlock() 127 128 plugins := c.registry.ListPlugins(ptype) 129 seen := make(map[string]struct{}, len(plugins)) 130 131 // For every plugin in the registry, ensure that we have an existing plugin 132 // running. Also build the map of valid plugin names. 133 // Note: monolith plugins that run as both controllers and nodes get a 134 // separate instance manager for both modes. 135 for _, plugin := range plugins { 136 seen[plugin.Name] = struct{}{} 137 c.ensureInstance(plugin) 138 } 139 140 // For every instance manager, if we did not find it during the plugin 141 // iterator, shut it down and remove it from the table. 142 instances := c.instancesForType(ptype) 143 for name, mgr := range instances { 144 if _, ok := seen[name]; !ok { 145 c.ensureNoInstance(mgr.info) 146 } 147 } 148 } 149 150 // handlePluginEvent syncs a single event against the plugin registry 151 func (c *csiManager) handlePluginEvent(event *dynamicplugins.PluginUpdateEvent) { 152 if event == nil || event.Info == nil { 153 return 154 } 155 c.logger.Trace("dynamic plugin event", 156 "event", event.EventType, 157 "plugin_id", event.Info.Name, 158 "plugin_alloc_id", event.Info.AllocID) 159 160 c.instancesLock.Lock() 161 defer c.instancesLock.Unlock() 162 163 switch event.EventType { 164 case dynamicplugins.EventTypeRegistered: 165 c.ensureInstance(event.Info) 166 case dynamicplugins.EventTypeDeregistered: 167 c.ensureNoInstance(event.Info) 168 default: 169 c.logger.Error("received unknown dynamic plugin event type", 170 "type", event.EventType) 171 } 172 } 173 174 // Ensure we have an instance manager for the plugin and add it to 175 // the CSI manager's tracking table for that plugin type. 176 // Assumes that c.instances has been locked. 177 func (c *csiManager) ensureInstance(plugin *dynamicplugins.PluginInfo) { 178 name := plugin.Name 179 ptype := plugin.Type 180 instances := c.instancesForType(ptype) 181 mgr, ok := instances[name] 182 if !ok { 183 c.logger.Debug("detected new CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID) 184 mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin) 185 instances[name] = mgr 186 mgr.run() 187 } else if mgr.allocID != plugin.AllocID { 188 mgr.shutdown() 189 c.logger.Debug("detected update for CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID) 190 mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin) 191 instances[name] = mgr 192 mgr.run() 193 194 } 195 } 196 197 // Shut down the instance manager for a plugin and remove it from 198 // the CSI manager's tracking table for that plugin type. 199 // Assumes that c.instances has been locked. 200 func (c *csiManager) ensureNoInstance(plugin *dynamicplugins.PluginInfo) { 201 name := plugin.Name 202 ptype := plugin.Type 203 instances := c.instancesForType(ptype) 204 if mgr, ok := instances[name]; ok { 205 if mgr.allocID == plugin.AllocID { 206 c.logger.Debug("shutting down CSI plugin", "name", name, "type", ptype, "alloc", plugin.AllocID) 207 mgr.shutdown() 208 delete(instances, name) 209 } 210 } 211 } 212 213 // Get the instance managers table for a specific plugin type, 214 // ensuring it's been initialized if it doesn't exist. 215 // Assumes that c.instances has been locked. 216 func (c *csiManager) instancesForType(ptype string) map[string]*instanceManager { 217 pluginMap, ok := c.instances[ptype] 218 if !ok { 219 pluginMap = make(map[string]*instanceManager) 220 c.instances[ptype] = pluginMap 221 } 222 return pluginMap 223 } 224 225 // Shutdown should gracefully shutdown all plugins managed by the manager. 226 // It must block until shutdown is complete 227 func (c *csiManager) Shutdown() { 228 // Shut down the run loop 229 c.shutdownCtxCancelFn() 230 231 // Wait for plugin manager shutdown to complete so that we 232 // don't try to shutdown instance managers while runLoop is 233 // doing a resync 234 <-c.shutdownCh 235 236 // Shutdown all the instance managers in parallel 237 var wg sync.WaitGroup 238 for _, pluginMap := range c.instances { 239 for _, mgr := range pluginMap { 240 wg.Add(1) 241 go func(mgr *instanceManager) { 242 mgr.shutdown() 243 wg.Done() 244 }(mgr) 245 } 246 } 247 wg.Wait() 248 } 249 250 // PluginType is the type of plugin which the manager manages 251 func (c *csiManager) PluginType() string { 252 return "csi" 253 }