github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/client/pluginmanager/csimanager/manager.go (about) 1 package csimanager 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 "time" 8 9 "github.com/hashicorp/go-hclog" 10 "github.com/hashicorp/nomad/client/dynamicplugins" 11 "github.com/hashicorp/nomad/client/pluginmanager" 12 "github.com/hashicorp/nomad/nomad/structs" 13 ) 14 15 // defaultPluginResyncPeriod is the time interval used to do a full resync 16 // against the dynamicplugins, to account for missed updates. 17 const defaultPluginResyncPeriod = 30 * time.Second 18 19 // UpdateNodeCSIInfoFunc is the callback used to update the node from 20 // fingerprinting 21 type UpdateNodeCSIInfoFunc func(string, *structs.CSIInfo) 22 type TriggerNodeEvent func(*structs.NodeEvent) 23 24 type Config struct { 25 Logger hclog.Logger 26 DynamicRegistry dynamicplugins.Registry 27 UpdateNodeCSIInfoFunc UpdateNodeCSIInfoFunc 28 PluginResyncPeriod time.Duration 29 TriggerNodeEvent TriggerNodeEvent 30 } 31 32 // New returns a new PluginManager that will handle managing CSI plugins from 33 // the dynamicRegistry from the provided Config. 34 func New(config *Config) Manager { 35 // Use a dedicated internal context for managing plugin shutdown. 36 ctx, cancelFn := context.WithCancel(context.Background()) 37 if config.PluginResyncPeriod == 0 { 38 config.PluginResyncPeriod = defaultPluginResyncPeriod 39 } 40 41 return &csiManager{ 42 logger: config.Logger, 43 eventer: config.TriggerNodeEvent, 44 registry: config.DynamicRegistry, 45 instances: make(map[string]map[string]*instanceManager), 46 47 updateNodeCSIInfoFunc: config.UpdateNodeCSIInfoFunc, 48 pluginResyncPeriod: config.PluginResyncPeriod, 49 50 shutdownCtx: ctx, 51 shutdownCtxCancelFn: cancelFn, 52 shutdownCh: make(chan struct{}), 53 } 54 } 55 56 type csiManager struct { 57 // instances should only be accessed from the run() goroutine and the shutdown 58 // fn. It is a map of PluginType : [PluginName : instanceManager] 59 instances map[string]map[string]*instanceManager 60 61 registry dynamicplugins.Registry 62 logger hclog.Logger 63 eventer TriggerNodeEvent 64 pluginResyncPeriod time.Duration 65 66 updateNodeCSIInfoFunc UpdateNodeCSIInfoFunc 67 68 shutdownCtx context.Context 69 shutdownCtxCancelFn context.CancelFunc 70 shutdownCh chan struct{} 71 } 72 73 func (c *csiManager) PluginManager() pluginmanager.PluginManager { 74 return c 75 } 76 77 func (c *csiManager) MounterForPlugin(ctx context.Context, pluginID string) (VolumeMounter, error) { 78 nodePlugins, hasAnyNodePlugins := c.instances["csi-node"] 79 if !hasAnyNodePlugins { 80 return nil, fmt.Errorf("no storage node plugins found") 81 } 82 83 mgr, hasPlugin := nodePlugins[pluginID] 84 if !hasPlugin { 85 return nil, fmt.Errorf("plugin %s for type csi-node not found", pluginID) 86 } 87 88 return mgr.VolumeMounter(ctx) 89 } 90 91 // Run starts a plugin manager and should return early 92 func (c *csiManager) Run() { 93 go c.runLoop() 94 } 95 96 func (c *csiManager) runLoop() { 97 timer := time.NewTimer(0) // ensure we sync immediately in first pass 98 controllerUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-controller") 99 nodeUpdates := c.registry.PluginsUpdatedCh(c.shutdownCtx, "csi-node") 100 for { 101 select { 102 case <-timer.C: 103 c.resyncPluginsFromRegistry("csi-controller") 104 c.resyncPluginsFromRegistry("csi-node") 105 timer.Reset(c.pluginResyncPeriod) 106 case event := <-controllerUpdates: 107 c.handlePluginEvent(event) 108 case event := <-nodeUpdates: 109 c.handlePluginEvent(event) 110 case <-c.shutdownCtx.Done(): 111 close(c.shutdownCh) 112 return 113 } 114 } 115 } 116 117 // resyncPluginsFromRegistry does a full sync of the running instance 118 // managers against those in the registry. we primarily will use update 119 // events from the registry. 120 func (c *csiManager) resyncPluginsFromRegistry(ptype string) { 121 plugins := c.registry.ListPlugins(ptype) 122 seen := make(map[string]struct{}, len(plugins)) 123 124 // For every plugin in the registry, ensure that we have an existing plugin 125 // running. Also build the map of valid plugin names. 126 // Note: monolith plugins that run as both controllers and nodes get a 127 // separate instance manager for both modes. 128 for _, plugin := range plugins { 129 seen[plugin.Name] = struct{}{} 130 c.ensureInstance(plugin) 131 } 132 133 // For every instance manager, if we did not find it during the plugin 134 // iterator, shut it down and remove it from the table. 135 instances := c.instancesForType(ptype) 136 for name, mgr := range instances { 137 if _, ok := seen[name]; !ok { 138 c.ensureNoInstance(mgr.info) 139 } 140 } 141 } 142 143 // handlePluginEvent syncs a single event against the plugin registry 144 func (c *csiManager) handlePluginEvent(event *dynamicplugins.PluginUpdateEvent) { 145 if event == nil { 146 return 147 } 148 c.logger.Trace("dynamic plugin event", 149 "event", event.EventType, 150 "plugin_id", event.Info.Name, 151 "plugin_alloc_id", event.Info.AllocID) 152 153 switch event.EventType { 154 case dynamicplugins.EventTypeRegistered: 155 c.ensureInstance(event.Info) 156 case dynamicplugins.EventTypeDeregistered: 157 c.ensureNoInstance(event.Info) 158 default: 159 c.logger.Error("received unknown dynamic plugin event type", 160 "type", event.EventType) 161 } 162 } 163 164 // Ensure we have an instance manager for the plugin and add it to 165 // the CSI manager's tracking table for that plugin type. 166 func (c *csiManager) ensureInstance(plugin *dynamicplugins.PluginInfo) { 167 name := plugin.Name 168 ptype := plugin.Type 169 instances := c.instancesForType(ptype) 170 if _, ok := instances[name]; !ok { 171 c.logger.Debug("detected new CSI plugin", "name", name, "type", ptype) 172 mgr := newInstanceManager(c.logger, c.eventer, c.updateNodeCSIInfoFunc, plugin) 173 instances[name] = mgr 174 mgr.run() 175 } 176 } 177 178 // Shut down the instance manager for a plugin and remove it from 179 // the CSI manager's tracking table for that plugin type. 180 func (c *csiManager) ensureNoInstance(plugin *dynamicplugins.PluginInfo) { 181 name := plugin.Name 182 ptype := plugin.Type 183 instances := c.instancesForType(ptype) 184 if mgr, ok := instances[name]; ok { 185 c.logger.Debug("shutting down CSI plugin", "name", name, "type", ptype) 186 mgr.shutdown() 187 delete(instances, name) 188 } 189 } 190 191 // Get the instance managers table for a specific plugin type, 192 // ensuring it's been initialized if it doesn't exist. 193 func (c *csiManager) instancesForType(ptype string) map[string]*instanceManager { 194 pluginMap, ok := c.instances[ptype] 195 if !ok { 196 pluginMap = make(map[string]*instanceManager) 197 c.instances[ptype] = pluginMap 198 } 199 return pluginMap 200 } 201 202 // Shutdown should gracefully shutdown all plugins managed by the manager. 203 // It must block until shutdown is complete 204 func (c *csiManager) Shutdown() { 205 // Shut down the run loop 206 c.shutdownCtxCancelFn() 207 208 // Wait for plugin manager shutdown to complete so that we 209 // don't try to shutdown instance managers while runLoop is 210 // doing a resync 211 <-c.shutdownCh 212 213 // Shutdown all the instance managers in parallel 214 var wg sync.WaitGroup 215 for _, pluginMap := range c.instances { 216 for _, mgr := range pluginMap { 217 wg.Add(1) 218 go func(mgr *instanceManager) { 219 mgr.shutdown() 220 wg.Done() 221 }(mgr) 222 } 223 } 224 wg.Wait() 225 } 226 227 // PluginType is the type of plugin which the manager manages 228 func (c *csiManager) PluginType() string { 229 return "csi" 230 }