github.com/fabiokung/docker@v0.11.2-0.20170222101415-4534dcd49497/daemon/cluster/cluster.go (about) 1 package cluster 2 3 // 4 // ## Swarmkit integration 5 // 6 // Cluster - static configurable object for accessing everything swarm related. 7 // Contains methods for connecting and controlling the cluster. Exists always, 8 // even if swarm mode is not enabled. 9 // 10 // NodeRunner - Manager for starting the swarmkit node. Is present only and 11 // always if swarm mode is enabled. Implements backoff restart loop in case of 12 // errors. 13 // 14 // NodeState - Information about the current node status including access to 15 // gRPC clients if a manager is active. 16 // 17 // ### Locking 18 // 19 // `cluster.controlMutex` - taken for the whole lifecycle of the processes that 20 // can reconfigure cluster(init/join/leave etc). Protects that one 21 // reconfiguration action has fully completed before another can start. 22 // 23 // `cluster.mu` - taken when the actual changes in cluster configurations 24 // happen. Different from `controlMutex` because in some cases we need to 25 // access current cluster state even if the long-running reconfiguration is 26 // going on. For example network stack may ask for the current cluster state in 27 // the middle of the shutdown. Any time current cluster state is asked you 28 // should take the read lock of `cluster.mu`. If you are writing an API 29 // responder that returns synchronously, hold `cluster.mu.RLock()` for the 30 // duration of the whole handler function. That ensures that node will not be 31 // shut down until the handler has finished. 32 // 33 // NodeRunner implements its internal locks that should not be used outside of 34 // the struct. Instead, you should just call `nodeRunner.State()` method to get 35 // the current state of the cluster(still need `cluster.mu.RLock()` to access 36 // `cluster.nr` reference itself). Most of the changes in NodeRunner happen 37 // because of an external event(network problem, unexpected swarmkit error) and 38 // Docker shouldn't take any locks that delay these changes from happening. 39 // 40 41 import ( 42 "crypto/x509" 43 "fmt" 44 "net" 45 "os" 46 "path/filepath" 47 "sync" 48 "time" 49 50 "github.com/Sirupsen/logrus" 51 "github.com/docker/docker/api/types/network" 52 types "github.com/docker/docker/api/types/swarm" 53 executorpkg "github.com/docker/docker/daemon/cluster/executor" 54 "github.com/docker/docker/pkg/signal" 55 swarmapi "github.com/docker/swarmkit/api" 56 swarmnode "github.com/docker/swarmkit/node" 57 "github.com/pkg/errors" 58 "golang.org/x/net/context" 59 ) 60 61 const swarmDirName = "swarm" 62 const controlSocket = "control.sock" 63 const swarmConnectTimeout = 20 * time.Second 64 const swarmRequestTimeout = 20 * time.Second 65 const stateFile = "docker-state.json" 66 const defaultAddr = "0.0.0.0:2377" 67 68 const ( 69 initialReconnectDelay = 100 * time.Millisecond 70 maxReconnectDelay = 30 * time.Second 71 contextPrefix = "com.docker.swarm" 72 ) 73 74 // errNoSwarm is returned on leaving a cluster that was never initialized 75 var errNoSwarm = errors.New("This node is not part of a swarm") 76 77 // errSwarmExists is returned on initialize or join request for a cluster that has already been activated 78 var errSwarmExists = errors.New("This node is already part of a swarm. Use \"docker swarm leave\" to leave this swarm and join another one.") 79 80 // errSwarmJoinTimeoutReached is returned when cluster join could not complete before timeout was reached. 81 var errSwarmJoinTimeoutReached = errors.New("Timeout was reached before node was joined. The attempt to join the swarm will continue in the background. Use the \"docker info\" command to see the current swarm status of your node.") 82 83 // errSwarmLocked is returned if the swarm is encrypted and needs a key to unlock it. 84 var errSwarmLocked = errors.New("Swarm is encrypted and needs to be unlocked before it can be used. Please use \"docker swarm unlock\" to unlock it.") 85 86 // errSwarmCertificatesExpired is returned if docker was not started for the whole validity period and they had no chance to renew automatically. 87 var errSwarmCertificatesExpired = errors.New("Swarm certificates have expired. To replace them, leave the swarm and join again.") 88 89 // NetworkSubnetsProvider exposes functions for retrieving the subnets 90 // of networks managed by Docker, so they can be filtered. 91 type NetworkSubnetsProvider interface { 92 V4Subnets() []net.IPNet 93 V6Subnets() []net.IPNet 94 } 95 96 // Config provides values for Cluster. 97 type Config struct { 98 Root string 99 Name string 100 Backend executorpkg.Backend 101 NetworkSubnetsProvider NetworkSubnetsProvider 102 103 // DefaultAdvertiseAddr is the default host/IP or network interface to use 104 // if no AdvertiseAddr value is specified. 105 DefaultAdvertiseAddr string 106 107 // path to store runtime state, such as the swarm control socket 108 RuntimeRoot string 109 } 110 111 // Cluster provides capabilities to participate in a cluster as a worker or a 112 // manager. 113 type Cluster struct { 114 mu sync.RWMutex 115 controlMutex sync.RWMutex // protect init/join/leave user operations 116 nr *nodeRunner 117 root string 118 runtimeRoot string 119 config Config 120 configEvent chan struct{} // todo: make this array and goroutine safe 121 attachers map[string]*attacher 122 } 123 124 // attacher manages the in-memory attachment state of a container 125 // attachment to a global scope network managed by swarm manager. It 126 // helps in identifying the attachment ID via the taskID and the 127 // corresponding attachment configuration obtained from the manager. 128 type attacher struct { 129 taskID string 130 config *network.NetworkingConfig 131 attachWaitCh chan *network.NetworkingConfig 132 attachCompleteCh chan struct{} 133 detachWaitCh chan struct{} 134 } 135 136 // New creates a new Cluster instance using provided config. 137 func New(config Config) (*Cluster, error) { 138 root := filepath.Join(config.Root, swarmDirName) 139 if err := os.MkdirAll(root, 0700); err != nil { 140 return nil, err 141 } 142 if config.RuntimeRoot == "" { 143 config.RuntimeRoot = root 144 } 145 if err := os.MkdirAll(config.RuntimeRoot, 0700); err != nil { 146 return nil, err 147 } 148 c := &Cluster{ 149 root: root, 150 config: config, 151 configEvent: make(chan struct{}, 10), 152 runtimeRoot: config.RuntimeRoot, 153 attachers: make(map[string]*attacher), 154 } 155 156 nodeConfig, err := loadPersistentState(root) 157 if err != nil { 158 if os.IsNotExist(err) { 159 return c, nil 160 } 161 return nil, err 162 } 163 164 nr, err := c.newNodeRunner(*nodeConfig) 165 if err != nil { 166 return nil, err 167 } 168 c.nr = nr 169 170 select { 171 case <-time.After(swarmConnectTimeout): 172 logrus.Error("swarm component could not be started before timeout was reached") 173 case err := <-nr.Ready(): 174 if err != nil { 175 if errors.Cause(err) == errSwarmLocked { 176 return c, nil 177 } 178 if err, ok := errors.Cause(c.nr.err).(x509.CertificateInvalidError); ok && err.Reason == x509.Expired { 179 return c, nil 180 } 181 return nil, errors.Wrap(err, "swarm component could not be started") 182 } 183 } 184 return c, nil 185 } 186 187 func (c *Cluster) newNodeRunner(conf nodeStartConfig) (*nodeRunner, error) { 188 if err := c.config.Backend.IsSwarmCompatible(); err != nil { 189 return nil, err 190 } 191 192 actualLocalAddr := conf.LocalAddr 193 if actualLocalAddr == "" { 194 // If localAddr was not specified, resolve it automatically 195 // based on the route to joinAddr. localAddr can only be left 196 // empty on "join". 197 listenHost, _, err := net.SplitHostPort(conf.ListenAddr) 198 if err != nil { 199 return nil, fmt.Errorf("could not parse listen address: %v", err) 200 } 201 202 listenAddrIP := net.ParseIP(listenHost) 203 if listenAddrIP == nil || !listenAddrIP.IsUnspecified() { 204 actualLocalAddr = listenHost 205 } else { 206 if conf.RemoteAddr == "" { 207 // Should never happen except using swarms created by 208 // old versions that didn't save remoteAddr. 209 conf.RemoteAddr = "8.8.8.8:53" 210 } 211 conn, err := net.Dial("udp", conf.RemoteAddr) 212 if err != nil { 213 return nil, fmt.Errorf("could not find local IP address: %v", err) 214 } 215 localHostPort := conn.LocalAddr().String() 216 actualLocalAddr, _, _ = net.SplitHostPort(localHostPort) 217 conn.Close() 218 } 219 } 220 221 nr := &nodeRunner{cluster: c} 222 nr.actualLocalAddr = actualLocalAddr 223 224 if err := nr.Start(conf); err != nil { 225 return nil, err 226 } 227 228 c.config.Backend.DaemonJoinsCluster(c) 229 230 return nr, nil 231 } 232 233 func (c *Cluster) getRequestContext() (context.Context, func()) { // TODO: not needed when requests don't block on qourum lost 234 return context.WithTimeout(context.Background(), swarmRequestTimeout) 235 } 236 237 // IsManager returns true if Cluster is participating as a manager. 238 func (c *Cluster) IsManager() bool { 239 c.mu.RLock() 240 defer c.mu.RUnlock() 241 return c.currentNodeState().IsActiveManager() 242 } 243 244 // IsAgent returns true if Cluster is participating as a worker/agent. 245 func (c *Cluster) IsAgent() bool { 246 c.mu.RLock() 247 defer c.mu.RUnlock() 248 return c.currentNodeState().status == types.LocalNodeStateActive 249 } 250 251 // GetLocalAddress returns the local address. 252 func (c *Cluster) GetLocalAddress() string { 253 c.mu.RLock() 254 defer c.mu.RUnlock() 255 return c.currentNodeState().actualLocalAddr 256 } 257 258 // GetListenAddress returns the listen address. 259 func (c *Cluster) GetListenAddress() string { 260 c.mu.RLock() 261 defer c.mu.RUnlock() 262 if c.nr != nil { 263 return c.nr.config.ListenAddr 264 } 265 return "" 266 } 267 268 // GetAdvertiseAddress returns the remotely reachable address of this node. 269 func (c *Cluster) GetAdvertiseAddress() string { 270 c.mu.RLock() 271 defer c.mu.RUnlock() 272 if c.nr != nil && c.nr.config.AdvertiseAddr != "" { 273 advertiseHost, _, _ := net.SplitHostPort(c.nr.config.AdvertiseAddr) 274 return advertiseHost 275 } 276 return c.currentNodeState().actualLocalAddr 277 } 278 279 // GetRemoteAddress returns a known advertise address of a remote manager if 280 // available. 281 // todo: change to array/connect with info 282 func (c *Cluster) GetRemoteAddress() string { 283 c.mu.RLock() 284 defer c.mu.RUnlock() 285 return c.getRemoteAddress() 286 } 287 288 func (c *Cluster) getRemoteAddress() string { 289 state := c.currentNodeState() 290 if state.swarmNode == nil { 291 return "" 292 } 293 nodeID := state.swarmNode.NodeID() 294 for _, r := range state.swarmNode.Remotes() { 295 if r.NodeID != nodeID { 296 return r.Addr 297 } 298 } 299 return "" 300 } 301 302 // ListenClusterEvents returns a channel that receives messages on cluster 303 // participation changes. 304 // todo: make cancelable and accessible to multiple callers 305 func (c *Cluster) ListenClusterEvents() <-chan struct{} { 306 return c.configEvent 307 } 308 309 // currentNodeState should not be called without a read lock 310 func (c *Cluster) currentNodeState() nodeState { 311 return c.nr.State() 312 } 313 314 // errNoManager returns error describing why manager commands can't be used. 315 // Call with read lock. 316 func (c *Cluster) errNoManager(st nodeState) error { 317 if st.swarmNode == nil { 318 if errors.Cause(st.err) == errSwarmLocked { 319 return errSwarmLocked 320 } 321 if st.err == errSwarmCertificatesExpired { 322 return errSwarmCertificatesExpired 323 } 324 return errors.New("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again.") 325 } 326 if st.swarmNode.Manager() != nil { 327 return errors.New("This node is not a swarm manager. Manager is being prepared or has trouble connecting to the cluster.") 328 } 329 return errors.New("This node is not a swarm manager. Worker nodes can't be used to view or modify cluster state. Please run this command on a manager node or promote the current node to a manager.") 330 } 331 332 // Cleanup stops active swarm node. This is run before daemon shutdown. 333 func (c *Cluster) Cleanup() { 334 c.controlMutex.Lock() 335 defer c.controlMutex.Unlock() 336 337 c.mu.Lock() 338 node := c.nr 339 if node == nil { 340 c.mu.Unlock() 341 return 342 } 343 defer c.mu.Unlock() 344 state := c.currentNodeState() 345 if state.IsActiveManager() { 346 active, reachable, unreachable, err := managerStats(state.controlClient, state.NodeID()) 347 if err == nil { 348 singlenode := active && isLastManager(reachable, unreachable) 349 if active && !singlenode && removingManagerCausesLossOfQuorum(reachable, unreachable) { 350 logrus.Errorf("Leaving cluster with %v managers left out of %v. Raft quorum will be lost.", reachable-1, reachable+unreachable) 351 } 352 } 353 } 354 if err := node.Stop(); err != nil { 355 logrus.Errorf("failed to shut down cluster node: %v", err) 356 signal.DumpStacks("") 357 } 358 c.nr = nil 359 } 360 361 func managerStats(client swarmapi.ControlClient, currentNodeID string) (current bool, reachable int, unreachable int, err error) { 362 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 363 defer cancel() 364 nodes, err := client.ListNodes(ctx, &swarmapi.ListNodesRequest{}) 365 if err != nil { 366 return false, 0, 0, err 367 } 368 for _, n := range nodes.Nodes { 369 if n.ManagerStatus != nil { 370 if n.ManagerStatus.Reachability == swarmapi.RaftMemberStatus_REACHABLE { 371 reachable++ 372 if n.ID == currentNodeID { 373 current = true 374 } 375 } 376 if n.ManagerStatus.Reachability == swarmapi.RaftMemberStatus_UNREACHABLE { 377 unreachable++ 378 } 379 } 380 } 381 return 382 } 383 384 func detectLockedError(err error) error { 385 if err == swarmnode.ErrInvalidUnlockKey { 386 return errors.WithStack(errSwarmLocked) 387 } 388 return err 389 }