github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/daemon/cluster/noderunner.go (about)

     1  package cluster // import "github.com/docker/docker/daemon/cluster"
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"path/filepath"
     7  	"runtime"
     8  	"strings"
     9  	"sync"
    10  	"time"
    11  
    12  	types "github.com/docker/docker/api/types/swarm"
    13  	"github.com/docker/docker/daemon/cluster/executor/container"
    14  	lncluster "github.com/docker/libnetwork/cluster"
    15  	swarmapi "github.com/docker/swarmkit/api"
    16  	"github.com/docker/swarmkit/manager/allocator/cnmallocator"
    17  	swarmnode "github.com/docker/swarmkit/node"
    18  	"github.com/pkg/errors"
    19  	"github.com/sirupsen/logrus"
    20  	"google.golang.org/grpc"
    21  	"google.golang.org/grpc/codes"
    22  	"google.golang.org/grpc/status"
    23  )
    24  
    25  // nodeRunner implements a manager for continuously running swarmkit node, restarting them with backoff delays if needed.
    26  type nodeRunner struct {
    27  	nodeState
    28  	mu             sync.RWMutex
    29  	done           chan struct{} // closed when swarmNode exits
    30  	ready          chan struct{} // closed when swarmNode becomes active
    31  	reconnectDelay time.Duration
    32  	config         nodeStartConfig
    33  
    34  	repeatedRun     bool
    35  	cancelReconnect func()
    36  	stopping        bool
    37  	cluster         *Cluster // only for accessing config helpers, never call any methods. TODO: change to config struct
    38  }
    39  
    40  // nodeStartConfig holds configuration needed to start a new node. Exported
    41  // fields of this structure are saved to disk in json. Unexported fields
    42  // contain data that shouldn't be persisted between daemon reloads.
    43  type nodeStartConfig struct {
    44  	// LocalAddr is this machine's local IP or hostname, if specified.
    45  	LocalAddr string
    46  	// RemoteAddr is the address that was given to "swarm join". It is used
    47  	// to find LocalAddr if necessary.
    48  	RemoteAddr string
    49  	// ListenAddr is the address we bind to, including a port.
    50  	ListenAddr string
    51  	// AdvertiseAddr is the address other nodes should connect to,
    52  	// including a port.
    53  	AdvertiseAddr string
    54  	// DataPathAddr is the address that has to be used for the data path
    55  	DataPathAddr string
    56  	// DefaultAddressPool contains list of subnets
    57  	DefaultAddressPool []string
    58  	// SubnetSize contains subnet size of DefaultAddressPool
    59  	SubnetSize uint32
    60  	// JoinInProgress is set to true if a join operation has started, but
    61  	// not completed yet.
    62  	JoinInProgress bool
    63  
    64  	joinAddr        string
    65  	forceNewCluster bool
    66  	joinToken       string
    67  	lockKey         []byte
    68  	autolock        bool
    69  	availability    types.NodeAvailability
    70  }
    71  
    72  func (n *nodeRunner) Ready() chan error {
    73  	c := make(chan error, 1)
    74  	n.mu.RLock()
    75  	ready, done := n.ready, n.done
    76  	n.mu.RUnlock()
    77  	go func() {
    78  		select {
    79  		case <-ready:
    80  		case <-done:
    81  		}
    82  		select {
    83  		case <-ready:
    84  		default:
    85  			n.mu.RLock()
    86  			c <- n.err
    87  			n.mu.RUnlock()
    88  		}
    89  		close(c)
    90  	}()
    91  	return c
    92  }
    93  
    94  func (n *nodeRunner) Start(conf nodeStartConfig) error {
    95  	n.mu.Lock()
    96  	defer n.mu.Unlock()
    97  
    98  	n.reconnectDelay = initialReconnectDelay
    99  
   100  	return n.start(conf)
   101  }
   102  
   103  func (n *nodeRunner) start(conf nodeStartConfig) error {
   104  	var control string
   105  	if runtime.GOOS == "windows" {
   106  		control = `\\.\pipe\` + controlSocket
   107  	} else {
   108  		control = filepath.Join(n.cluster.runtimeRoot, controlSocket)
   109  	}
   110  
   111  	joinAddr := conf.joinAddr
   112  	if joinAddr == "" && conf.JoinInProgress {
   113  		// We must have been restarted while trying to join a cluster.
   114  		// Continue trying to join instead of forming our own cluster.
   115  		joinAddr = conf.RemoteAddr
   116  	}
   117  
   118  	// Hostname is not set here. Instead, it is obtained from
   119  	// the node description that is reported periodically
   120  	swarmnodeConfig := swarmnode.Config{
   121  		ForceNewCluster:    conf.forceNewCluster,
   122  		ListenControlAPI:   control,
   123  		ListenRemoteAPI:    conf.ListenAddr,
   124  		AdvertiseRemoteAPI: conf.AdvertiseAddr,
   125  		NetworkConfig: &cnmallocator.NetworkConfig{
   126  			DefaultAddrPool: conf.DefaultAddressPool,
   127  			SubnetSize:      conf.SubnetSize,
   128  		},
   129  		JoinAddr:  joinAddr,
   130  		StateDir:  n.cluster.root,
   131  		JoinToken: conf.joinToken,
   132  		Executor: container.NewExecutor(
   133  			n.cluster.config.Backend,
   134  			n.cluster.config.PluginBackend,
   135  			n.cluster.config.ImageBackend,
   136  			n.cluster.config.VolumeBackend,
   137  		),
   138  		HeartbeatTick: n.cluster.config.RaftHeartbeatTick,
   139  		// Recommended value in etcd/raft is 10 x (HeartbeatTick).
   140  		// Lower values were seen to have caused instability because of
   141  		// frequent leader elections when running on flakey networks.
   142  		ElectionTick:     n.cluster.config.RaftElectionTick,
   143  		UnlockKey:        conf.lockKey,
   144  		AutoLockManagers: conf.autolock,
   145  		PluginGetter:     n.cluster.config.Backend.PluginGetter(),
   146  	}
   147  	if conf.availability != "" {
   148  		avail, ok := swarmapi.NodeSpec_Availability_value[strings.ToUpper(string(conf.availability))]
   149  		if !ok {
   150  			return fmt.Errorf("invalid Availability: %q", conf.availability)
   151  		}
   152  		swarmnodeConfig.Availability = swarmapi.NodeSpec_Availability(avail)
   153  	}
   154  	node, err := swarmnode.New(&swarmnodeConfig)
   155  	if err != nil {
   156  		return err
   157  	}
   158  	if err := node.Start(context.Background()); err != nil {
   159  		return err
   160  	}
   161  
   162  	n.done = make(chan struct{})
   163  	n.ready = make(chan struct{})
   164  	n.swarmNode = node
   165  	if conf.joinAddr != "" {
   166  		conf.JoinInProgress = true
   167  	}
   168  	n.config = conf
   169  	savePersistentState(n.cluster.root, conf)
   170  
   171  	ctx, cancel := context.WithCancel(context.Background())
   172  
   173  	go func() {
   174  		n.handleNodeExit(node)
   175  		cancel()
   176  	}()
   177  
   178  	go n.handleReadyEvent(ctx, node, n.ready)
   179  	go n.handleControlSocketChange(ctx, node)
   180  
   181  	return nil
   182  }
   183  
   184  func (n *nodeRunner) handleControlSocketChange(ctx context.Context, node *swarmnode.Node) {
   185  	for conn := range node.ListenControlSocket(ctx) {
   186  		n.mu.Lock()
   187  		if n.grpcConn != conn {
   188  			if conn == nil {
   189  				n.controlClient = nil
   190  				n.logsClient = nil
   191  			} else {
   192  				n.controlClient = swarmapi.NewControlClient(conn)
   193  				n.logsClient = swarmapi.NewLogsClient(conn)
   194  				// push store changes to daemon
   195  				go n.watchClusterEvents(ctx, conn)
   196  			}
   197  		}
   198  		n.grpcConn = conn
   199  		n.mu.Unlock()
   200  		n.cluster.SendClusterEvent(lncluster.EventSocketChange)
   201  	}
   202  }
   203  
   204  func (n *nodeRunner) watchClusterEvents(ctx context.Context, conn *grpc.ClientConn) {
   205  	client := swarmapi.NewWatchClient(conn)
   206  	watch, err := client.Watch(ctx, &swarmapi.WatchRequest{
   207  		Entries: []*swarmapi.WatchRequest_WatchEntry{
   208  			{
   209  				Kind:   "node",
   210  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   211  			},
   212  			{
   213  				Kind:   "service",
   214  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   215  			},
   216  			{
   217  				Kind:   "network",
   218  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   219  			},
   220  			{
   221  				Kind:   "secret",
   222  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   223  			},
   224  			{
   225  				Kind:   "config",
   226  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   227  			},
   228  		},
   229  		IncludeOldObject: true,
   230  	})
   231  	if err != nil {
   232  		logrus.WithError(err).Error("failed to watch cluster store")
   233  		return
   234  	}
   235  	for {
   236  		msg, err := watch.Recv()
   237  		if err != nil {
   238  			// store watch is broken
   239  			errStatus, ok := status.FromError(err)
   240  			if !ok || errStatus.Code() != codes.Canceled {
   241  				logrus.WithError(err).Error("failed to receive changes from store watch API")
   242  			}
   243  			return
   244  		}
   245  		select {
   246  		case <-ctx.Done():
   247  			return
   248  		case n.cluster.watchStream <- msg:
   249  		}
   250  	}
   251  }
   252  
   253  func (n *nodeRunner) handleReadyEvent(ctx context.Context, node *swarmnode.Node, ready chan struct{}) {
   254  	select {
   255  	case <-node.Ready():
   256  		n.mu.Lock()
   257  		n.err = nil
   258  		if n.config.JoinInProgress {
   259  			n.config.JoinInProgress = false
   260  			savePersistentState(n.cluster.root, n.config)
   261  		}
   262  		n.mu.Unlock()
   263  		close(ready)
   264  	case <-ctx.Done():
   265  	}
   266  	n.cluster.SendClusterEvent(lncluster.EventNodeReady)
   267  }
   268  
   269  func (n *nodeRunner) handleNodeExit(node *swarmnode.Node) {
   270  	err := detectLockedError(node.Err(context.Background()))
   271  	if err != nil {
   272  		logrus.Errorf("cluster exited with error: %v", err)
   273  	}
   274  	n.mu.Lock()
   275  	n.swarmNode = nil
   276  	n.err = err
   277  	close(n.done)
   278  	select {
   279  	case <-n.ready:
   280  		n.enableReconnectWatcher()
   281  	default:
   282  		if n.repeatedRun {
   283  			n.enableReconnectWatcher()
   284  		}
   285  	}
   286  	n.repeatedRun = true
   287  	n.mu.Unlock()
   288  }
   289  
   290  // Stop stops the current swarm node if it is running.
   291  func (n *nodeRunner) Stop() error {
   292  	n.mu.Lock()
   293  	if n.cancelReconnect != nil { // between restarts
   294  		n.cancelReconnect()
   295  		n.cancelReconnect = nil
   296  	}
   297  	if n.swarmNode == nil {
   298  		n.mu.Unlock()
   299  		return nil
   300  	}
   301  	n.stopping = true
   302  	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
   303  	defer cancel()
   304  	n.mu.Unlock()
   305  	if err := n.swarmNode.Stop(ctx); err != nil && !strings.Contains(err.Error(), "context canceled") {
   306  		return err
   307  	}
   308  	n.cluster.SendClusterEvent(lncluster.EventNodeLeave)
   309  	<-n.done
   310  	return nil
   311  }
   312  
   313  func (n *nodeRunner) State() nodeState {
   314  	if n == nil {
   315  		return nodeState{status: types.LocalNodeStateInactive}
   316  	}
   317  	n.mu.RLock()
   318  	defer n.mu.RUnlock()
   319  
   320  	ns := n.nodeState
   321  
   322  	if ns.err != nil || n.cancelReconnect != nil {
   323  		if errors.Cause(ns.err) == errSwarmLocked {
   324  			ns.status = types.LocalNodeStateLocked
   325  		} else {
   326  			ns.status = types.LocalNodeStateError
   327  		}
   328  	} else {
   329  		select {
   330  		case <-n.ready:
   331  			ns.status = types.LocalNodeStateActive
   332  		default:
   333  			ns.status = types.LocalNodeStatePending
   334  		}
   335  	}
   336  
   337  	return ns
   338  }
   339  
   340  func (n *nodeRunner) enableReconnectWatcher() {
   341  	if n.stopping {
   342  		return
   343  	}
   344  	n.reconnectDelay *= 2
   345  	if n.reconnectDelay > maxReconnectDelay {
   346  		n.reconnectDelay = maxReconnectDelay
   347  	}
   348  	logrus.Warnf("Restarting swarm in %.2f seconds", n.reconnectDelay.Seconds())
   349  	delayCtx, cancel := context.WithTimeout(context.Background(), n.reconnectDelay)
   350  	n.cancelReconnect = cancel
   351  
   352  	go func() {
   353  		<-delayCtx.Done()
   354  		if delayCtx.Err() != context.DeadlineExceeded {
   355  			return
   356  		}
   357  		n.mu.Lock()
   358  		defer n.mu.Unlock()
   359  		if n.stopping {
   360  			return
   361  		}
   362  
   363  		if err := n.start(n.config); err != nil {
   364  			n.err = err
   365  		}
   366  	}()
   367  }
   368  
   369  // nodeState represents information about the current state of the cluster and
   370  // provides access to the grpc clients.
   371  type nodeState struct {
   372  	swarmNode       *swarmnode.Node
   373  	grpcConn        *grpc.ClientConn
   374  	controlClient   swarmapi.ControlClient
   375  	logsClient      swarmapi.LogsClient
   376  	status          types.LocalNodeState
   377  	actualLocalAddr string
   378  	err             error
   379  }
   380  
   381  // IsActiveManager returns true if node is a manager ready to accept control requests. It is safe to access the client properties if this returns true.
   382  func (ns nodeState) IsActiveManager() bool {
   383  	return ns.controlClient != nil
   384  }
   385  
   386  // IsManager returns true if node is a manager.
   387  func (ns nodeState) IsManager() bool {
   388  	return ns.swarmNode != nil && ns.swarmNode.Manager() != nil
   389  }
   390  
   391  // NodeID returns node's ID or empty string if node is inactive.
   392  func (ns nodeState) NodeID() string {
   393  	if ns.swarmNode != nil {
   394  		return ns.swarmNode.NodeID()
   395  	}
   396  	return ""
   397  }