github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/daemon/cluster/noderunner.go (about) 1 package cluster // import "github.com/docker/docker/daemon/cluster" 2 3 import ( 4 "context" 5 "fmt" 6 "path/filepath" 7 "runtime" 8 "strings" 9 "sync" 10 "time" 11 12 types "github.com/docker/docker/api/types/swarm" 13 "github.com/docker/docker/daemon/cluster/executor/container" 14 lncluster "github.com/docker/libnetwork/cluster" 15 swarmapi "github.com/docker/swarmkit/api" 16 "github.com/docker/swarmkit/manager/allocator/cnmallocator" 17 swarmnode "github.com/docker/swarmkit/node" 18 "github.com/pkg/errors" 19 "github.com/sirupsen/logrus" 20 "google.golang.org/grpc" 21 "google.golang.org/grpc/codes" 22 "google.golang.org/grpc/status" 23 ) 24 25 // nodeRunner implements a manager for continuously running swarmkit node, restarting them with backoff delays if needed. 26 type nodeRunner struct { 27 nodeState 28 mu sync.RWMutex 29 done chan struct{} // closed when swarmNode exits 30 ready chan struct{} // closed when swarmNode becomes active 31 reconnectDelay time.Duration 32 config nodeStartConfig 33 34 repeatedRun bool 35 cancelReconnect func() 36 stopping bool 37 cluster *Cluster // only for accessing config helpers, never call any methods. TODO: change to config struct 38 } 39 40 // nodeStartConfig holds configuration needed to start a new node. Exported 41 // fields of this structure are saved to disk in json. Unexported fields 42 // contain data that shouldn't be persisted between daemon reloads. 43 type nodeStartConfig struct { 44 // LocalAddr is this machine's local IP or hostname, if specified. 45 LocalAddr string 46 // RemoteAddr is the address that was given to "swarm join". It is used 47 // to find LocalAddr if necessary. 48 RemoteAddr string 49 // ListenAddr is the address we bind to, including a port. 50 ListenAddr string 51 // AdvertiseAddr is the address other nodes should connect to, 52 // including a port. 53 AdvertiseAddr string 54 // DataPathAddr is the address that has to be used for the data path 55 DataPathAddr string 56 // DefaultAddressPool contains list of subnets 57 DefaultAddressPool []string 58 // SubnetSize contains subnet size of DefaultAddressPool 59 SubnetSize uint32 60 // JoinInProgress is set to true if a join operation has started, but 61 // not completed yet. 62 JoinInProgress bool 63 64 joinAddr string 65 forceNewCluster bool 66 joinToken string 67 lockKey []byte 68 autolock bool 69 availability types.NodeAvailability 70 } 71 72 func (n *nodeRunner) Ready() chan error { 73 c := make(chan error, 1) 74 n.mu.RLock() 75 ready, done := n.ready, n.done 76 n.mu.RUnlock() 77 go func() { 78 select { 79 case <-ready: 80 case <-done: 81 } 82 select { 83 case <-ready: 84 default: 85 n.mu.RLock() 86 c <- n.err 87 n.mu.RUnlock() 88 } 89 close(c) 90 }() 91 return c 92 } 93 94 func (n *nodeRunner) Start(conf nodeStartConfig) error { 95 n.mu.Lock() 96 defer n.mu.Unlock() 97 98 n.reconnectDelay = initialReconnectDelay 99 100 return n.start(conf) 101 } 102 103 func (n *nodeRunner) start(conf nodeStartConfig) error { 104 var control string 105 if runtime.GOOS == "windows" { 106 control = `\\.\pipe\` + controlSocket 107 } else { 108 control = filepath.Join(n.cluster.runtimeRoot, controlSocket) 109 } 110 111 joinAddr := conf.joinAddr 112 if joinAddr == "" && conf.JoinInProgress { 113 // We must have been restarted while trying to join a cluster. 114 // Continue trying to join instead of forming our own cluster. 115 joinAddr = conf.RemoteAddr 116 } 117 118 // Hostname is not set here. Instead, it is obtained from 119 // the node description that is reported periodically 120 swarmnodeConfig := swarmnode.Config{ 121 ForceNewCluster: conf.forceNewCluster, 122 ListenControlAPI: control, 123 ListenRemoteAPI: conf.ListenAddr, 124 AdvertiseRemoteAPI: conf.AdvertiseAddr, 125 NetworkConfig: &cnmallocator.NetworkConfig{ 126 DefaultAddrPool: conf.DefaultAddressPool, 127 SubnetSize: conf.SubnetSize, 128 }, 129 JoinAddr: joinAddr, 130 StateDir: n.cluster.root, 131 JoinToken: conf.joinToken, 132 Executor: container.NewExecutor( 133 n.cluster.config.Backend, 134 n.cluster.config.PluginBackend, 135 n.cluster.config.ImageBackend, 136 n.cluster.config.VolumeBackend, 137 ), 138 HeartbeatTick: n.cluster.config.RaftHeartbeatTick, 139 // Recommended value in etcd/raft is 10 x (HeartbeatTick). 140 // Lower values were seen to have caused instability because of 141 // frequent leader elections when running on flakey networks. 142 ElectionTick: n.cluster.config.RaftElectionTick, 143 UnlockKey: conf.lockKey, 144 AutoLockManagers: conf.autolock, 145 PluginGetter: n.cluster.config.Backend.PluginGetter(), 146 } 147 if conf.availability != "" { 148 avail, ok := swarmapi.NodeSpec_Availability_value[strings.ToUpper(string(conf.availability))] 149 if !ok { 150 return fmt.Errorf("invalid Availability: %q", conf.availability) 151 } 152 swarmnodeConfig.Availability = swarmapi.NodeSpec_Availability(avail) 153 } 154 node, err := swarmnode.New(&swarmnodeConfig) 155 if err != nil { 156 return err 157 } 158 if err := node.Start(context.Background()); err != nil { 159 return err 160 } 161 162 n.done = make(chan struct{}) 163 n.ready = make(chan struct{}) 164 n.swarmNode = node 165 if conf.joinAddr != "" { 166 conf.JoinInProgress = true 167 } 168 n.config = conf 169 savePersistentState(n.cluster.root, conf) 170 171 ctx, cancel := context.WithCancel(context.Background()) 172 173 go func() { 174 n.handleNodeExit(node) 175 cancel() 176 }() 177 178 go n.handleReadyEvent(ctx, node, n.ready) 179 go n.handleControlSocketChange(ctx, node) 180 181 return nil 182 } 183 184 func (n *nodeRunner) handleControlSocketChange(ctx context.Context, node *swarmnode.Node) { 185 for conn := range node.ListenControlSocket(ctx) { 186 n.mu.Lock() 187 if n.grpcConn != conn { 188 if conn == nil { 189 n.controlClient = nil 190 n.logsClient = nil 191 } else { 192 n.controlClient = swarmapi.NewControlClient(conn) 193 n.logsClient = swarmapi.NewLogsClient(conn) 194 // push store changes to daemon 195 go n.watchClusterEvents(ctx, conn) 196 } 197 } 198 n.grpcConn = conn 199 n.mu.Unlock() 200 n.cluster.SendClusterEvent(lncluster.EventSocketChange) 201 } 202 } 203 204 func (n *nodeRunner) watchClusterEvents(ctx context.Context, conn *grpc.ClientConn) { 205 client := swarmapi.NewWatchClient(conn) 206 watch, err := client.Watch(ctx, &swarmapi.WatchRequest{ 207 Entries: []*swarmapi.WatchRequest_WatchEntry{ 208 { 209 Kind: "node", 210 Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove, 211 }, 212 { 213 Kind: "service", 214 Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove, 215 }, 216 { 217 Kind: "network", 218 Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove, 219 }, 220 { 221 Kind: "secret", 222 Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove, 223 }, 224 { 225 Kind: "config", 226 Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove, 227 }, 228 }, 229 IncludeOldObject: true, 230 }) 231 if err != nil { 232 logrus.WithError(err).Error("failed to watch cluster store") 233 return 234 } 235 for { 236 msg, err := watch.Recv() 237 if err != nil { 238 // store watch is broken 239 errStatus, ok := status.FromError(err) 240 if !ok || errStatus.Code() != codes.Canceled { 241 logrus.WithError(err).Error("failed to receive changes from store watch API") 242 } 243 return 244 } 245 select { 246 case <-ctx.Done(): 247 return 248 case n.cluster.watchStream <- msg: 249 } 250 } 251 } 252 253 func (n *nodeRunner) handleReadyEvent(ctx context.Context, node *swarmnode.Node, ready chan struct{}) { 254 select { 255 case <-node.Ready(): 256 n.mu.Lock() 257 n.err = nil 258 if n.config.JoinInProgress { 259 n.config.JoinInProgress = false 260 savePersistentState(n.cluster.root, n.config) 261 } 262 n.mu.Unlock() 263 close(ready) 264 case <-ctx.Done(): 265 } 266 n.cluster.SendClusterEvent(lncluster.EventNodeReady) 267 } 268 269 func (n *nodeRunner) handleNodeExit(node *swarmnode.Node) { 270 err := detectLockedError(node.Err(context.Background())) 271 if err != nil { 272 logrus.Errorf("cluster exited with error: %v", err) 273 } 274 n.mu.Lock() 275 n.swarmNode = nil 276 n.err = err 277 close(n.done) 278 select { 279 case <-n.ready: 280 n.enableReconnectWatcher() 281 default: 282 if n.repeatedRun { 283 n.enableReconnectWatcher() 284 } 285 } 286 n.repeatedRun = true 287 n.mu.Unlock() 288 } 289 290 // Stop stops the current swarm node if it is running. 291 func (n *nodeRunner) Stop() error { 292 n.mu.Lock() 293 if n.cancelReconnect != nil { // between restarts 294 n.cancelReconnect() 295 n.cancelReconnect = nil 296 } 297 if n.swarmNode == nil { 298 n.mu.Unlock() 299 return nil 300 } 301 n.stopping = true 302 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) 303 defer cancel() 304 n.mu.Unlock() 305 if err := n.swarmNode.Stop(ctx); err != nil && !strings.Contains(err.Error(), "context canceled") { 306 return err 307 } 308 n.cluster.SendClusterEvent(lncluster.EventNodeLeave) 309 <-n.done 310 return nil 311 } 312 313 func (n *nodeRunner) State() nodeState { 314 if n == nil { 315 return nodeState{status: types.LocalNodeStateInactive} 316 } 317 n.mu.RLock() 318 defer n.mu.RUnlock() 319 320 ns := n.nodeState 321 322 if ns.err != nil || n.cancelReconnect != nil { 323 if errors.Cause(ns.err) == errSwarmLocked { 324 ns.status = types.LocalNodeStateLocked 325 } else { 326 ns.status = types.LocalNodeStateError 327 } 328 } else { 329 select { 330 case <-n.ready: 331 ns.status = types.LocalNodeStateActive 332 default: 333 ns.status = types.LocalNodeStatePending 334 } 335 } 336 337 return ns 338 } 339 340 func (n *nodeRunner) enableReconnectWatcher() { 341 if n.stopping { 342 return 343 } 344 n.reconnectDelay *= 2 345 if n.reconnectDelay > maxReconnectDelay { 346 n.reconnectDelay = maxReconnectDelay 347 } 348 logrus.Warnf("Restarting swarm in %.2f seconds", n.reconnectDelay.Seconds()) 349 delayCtx, cancel := context.WithTimeout(context.Background(), n.reconnectDelay) 350 n.cancelReconnect = cancel 351 352 go func() { 353 <-delayCtx.Done() 354 if delayCtx.Err() != context.DeadlineExceeded { 355 return 356 } 357 n.mu.Lock() 358 defer n.mu.Unlock() 359 if n.stopping { 360 return 361 } 362 363 if err := n.start(n.config); err != nil { 364 n.err = err 365 } 366 }() 367 } 368 369 // nodeState represents information about the current state of the cluster and 370 // provides access to the grpc clients. 371 type nodeState struct { 372 swarmNode *swarmnode.Node 373 grpcConn *grpc.ClientConn 374 controlClient swarmapi.ControlClient 375 logsClient swarmapi.LogsClient 376 status types.LocalNodeState 377 actualLocalAddr string 378 err error 379 } 380 381 // IsActiveManager returns true if node is a manager ready to accept control requests. It is safe to access the client properties if this returns true. 382 func (ns nodeState) IsActiveManager() bool { 383 return ns.controlClient != nil 384 } 385 386 // IsManager returns true if node is a manager. 387 func (ns nodeState) IsManager() bool { 388 return ns.swarmNode != nil && ns.swarmNode.Manager() != nil 389 } 390 391 // NodeID returns node's ID or empty string if node is inactive. 392 func (ns nodeState) NodeID() string { 393 if ns.swarmNode != nil { 394 return ns.swarmNode.NodeID() 395 } 396 return "" 397 }