github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/cluster/noderunner.go (about)

     1  package cluster // import "github.com/demonoid81/moby/daemon/cluster"
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"path/filepath"
     7  	"strings"
     8  	"sync"
     9  	"time"
    10  
    11  	types "github.com/demonoid81/moby/api/types/swarm"
    12  	"github.com/demonoid81/moby/daemon/cluster/executor/container"
    13  	lncluster "github.com/demonoid81/libnetwork/cluster"
    14  	swarmapi "github.com/docker/swarmkit/api"
    15  	swarmallocator "github.com/docker/swarmkit/manager/allocator/cnmallocator"
    16  	swarmnode "github.com/docker/swarmkit/node"
    17  	"github.com/pkg/errors"
    18  	"github.com/sirupsen/logrus"
    19  	"google.golang.org/grpc"
    20  	"google.golang.org/grpc/codes"
    21  	"google.golang.org/grpc/status"
    22  )
    23  
    24  // nodeRunner implements a manager for continuously running swarmkit node, restarting them with backoff delays if needed.
    25  type nodeRunner struct {
    26  	nodeState
    27  	mu             sync.RWMutex
    28  	done           chan struct{} // closed when swarmNode exits
    29  	ready          chan struct{} // closed when swarmNode becomes active
    30  	reconnectDelay time.Duration
    31  	config         nodeStartConfig
    32  
    33  	repeatedRun     bool
    34  	cancelReconnect func()
    35  	stopping        bool
    36  	cluster         *Cluster // only for accessing config helpers, never call any methods. TODO: change to config struct
    37  }
    38  
    39  // nodeStartConfig holds configuration needed to start a new node. Exported
    40  // fields of this structure are saved to disk in json. Unexported fields
    41  // contain data that shouldn't be persisted between daemon reloads.
    42  type nodeStartConfig struct {
    43  	// LocalAddr is this machine's local IP or hostname, if specified.
    44  	LocalAddr string
    45  	// RemoteAddr is the address that was given to "swarm join". It is used
    46  	// to find LocalAddr if necessary.
    47  	RemoteAddr string
    48  	// ListenAddr is the address we bind to, including a port.
    49  	ListenAddr string
    50  	// AdvertiseAddr is the address other nodes should connect to,
    51  	// including a port.
    52  	AdvertiseAddr string
    53  	// DataPathAddr is the address that has to be used for the data path
    54  	DataPathAddr string
    55  	// DefaultAddressPool contains list of subnets
    56  	DefaultAddressPool []string
    57  	// SubnetSize contains subnet size of DefaultAddressPool
    58  	SubnetSize uint32
    59  	// DataPathPort contains Data path port (VXLAN UDP port) number that is used for data traffic.
    60  	DataPathPort uint32
    61  	// JoinInProgress is set to true if a join operation has started, but
    62  	// not completed yet.
    63  	JoinInProgress bool
    64  
    65  	joinAddr        string
    66  	forceNewCluster bool
    67  	joinToken       string
    68  	lockKey         []byte
    69  	autolock        bool
    70  	availability    types.NodeAvailability
    71  }
    72  
    73  func (n *nodeRunner) Ready() chan error {
    74  	c := make(chan error, 1)
    75  	n.mu.RLock()
    76  	ready, done := n.ready, n.done
    77  	n.mu.RUnlock()
    78  	go func() {
    79  		select {
    80  		case <-ready:
    81  		case <-done:
    82  		}
    83  		select {
    84  		case <-ready:
    85  		default:
    86  			n.mu.RLock()
    87  			c <- n.err
    88  			n.mu.RUnlock()
    89  		}
    90  		close(c)
    91  	}()
    92  	return c
    93  }
    94  
    95  func (n *nodeRunner) Start(conf nodeStartConfig) error {
    96  	n.mu.Lock()
    97  	defer n.mu.Unlock()
    98  
    99  	n.reconnectDelay = initialReconnectDelay
   100  
   101  	return n.start(conf)
   102  }
   103  
   104  func (n *nodeRunner) start(conf nodeStartConfig) error {
   105  	var control string
   106  	if isWindows {
   107  		control = `\\.\pipe\` + controlSocket
   108  	} else {
   109  		control = filepath.Join(n.cluster.runtimeRoot, controlSocket)
   110  	}
   111  
   112  	joinAddr := conf.joinAddr
   113  	if joinAddr == "" && conf.JoinInProgress {
   114  		// We must have been restarted while trying to join a cluster.
   115  		// Continue trying to join instead of forming our own cluster.
   116  		joinAddr = conf.RemoteAddr
   117  	}
   118  
   119  	// Hostname is not set here. Instead, it is obtained from
   120  	// the node description that is reported periodically
   121  	swarmnodeConfig := swarmnode.Config{
   122  		ForceNewCluster:    conf.forceNewCluster,
   123  		ListenControlAPI:   control,
   124  		ListenRemoteAPI:    conf.ListenAddr,
   125  		AdvertiseRemoteAPI: conf.AdvertiseAddr,
   126  		NetworkConfig: &swarmallocator.NetworkConfig{
   127  			DefaultAddrPool: conf.DefaultAddressPool,
   128  			SubnetSize:      conf.SubnetSize,
   129  			VXLANUDPPort:    conf.DataPathPort,
   130  		},
   131  		JoinAddr:  joinAddr,
   132  		StateDir:  n.cluster.root,
   133  		JoinToken: conf.joinToken,
   134  		Executor: container.NewExecutor(
   135  			n.cluster.config.Backend,
   136  			n.cluster.config.PluginBackend,
   137  			n.cluster.config.ImageBackend,
   138  			n.cluster.config.VolumeBackend,
   139  		),
   140  		HeartbeatTick: n.cluster.config.RaftHeartbeatTick,
   141  		// Recommended value in etcd/raft is 10 x (HeartbeatTick).
   142  		// Lower values were seen to have caused instability because of
   143  		// frequent leader elections when running on flakey networks.
   144  		ElectionTick:     n.cluster.config.RaftElectionTick,
   145  		UnlockKey:        conf.lockKey,
   146  		AutoLockManagers: conf.autolock,
   147  		PluginGetter:     n.cluster.config.Backend.PluginGetter(),
   148  	}
   149  	if conf.availability != "" {
   150  		avail, ok := swarmapi.NodeSpec_Availability_value[strings.ToUpper(string(conf.availability))]
   151  		if !ok {
   152  			return fmt.Errorf("invalid Availability: %q", conf.availability)
   153  		}
   154  		swarmnodeConfig.Availability = swarmapi.NodeSpec_Availability(avail)
   155  	}
   156  	node, err := swarmnode.New(&swarmnodeConfig)
   157  	if err != nil {
   158  		return err
   159  	}
   160  	if err := node.Start(context.Background()); err != nil {
   161  		return err
   162  	}
   163  
   164  	n.done = make(chan struct{})
   165  	n.ready = make(chan struct{})
   166  	n.swarmNode = node
   167  	if conf.joinAddr != "" {
   168  		conf.JoinInProgress = true
   169  	}
   170  	n.config = conf
   171  	savePersistentState(n.cluster.root, conf)
   172  
   173  	ctx, cancel := context.WithCancel(context.Background())
   174  
   175  	go func() {
   176  		n.handleNodeExit(node)
   177  		cancel()
   178  	}()
   179  
   180  	go n.handleReadyEvent(ctx, node, n.ready)
   181  	go n.handleControlSocketChange(ctx, node)
   182  
   183  	return nil
   184  }
   185  
   186  func (n *nodeRunner) handleControlSocketChange(ctx context.Context, node *swarmnode.Node) {
   187  	for conn := range node.ListenControlSocket(ctx) {
   188  		n.mu.Lock()
   189  		if n.grpcConn != conn {
   190  			if conn == nil {
   191  				n.controlClient = nil
   192  				n.logsClient = nil
   193  			} else {
   194  				n.controlClient = swarmapi.NewControlClient(conn)
   195  				n.logsClient = swarmapi.NewLogsClient(conn)
   196  				// push store changes to daemon
   197  				go n.watchClusterEvents(ctx, conn)
   198  			}
   199  		}
   200  		n.grpcConn = conn
   201  		n.mu.Unlock()
   202  		n.cluster.SendClusterEvent(lncluster.EventSocketChange)
   203  	}
   204  }
   205  
   206  func (n *nodeRunner) watchClusterEvents(ctx context.Context, conn *grpc.ClientConn) {
   207  	client := swarmapi.NewWatchClient(conn)
   208  	watch, err := client.Watch(ctx, &swarmapi.WatchRequest{
   209  		Entries: []*swarmapi.WatchRequest_WatchEntry{
   210  			{
   211  				Kind:   "node",
   212  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   213  			},
   214  			{
   215  				Kind:   "service",
   216  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   217  			},
   218  			{
   219  				Kind:   "network",
   220  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   221  			},
   222  			{
   223  				Kind:   "secret",
   224  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   225  			},
   226  			{
   227  				Kind:   "config",
   228  				Action: swarmapi.WatchActionKindCreate | swarmapi.WatchActionKindUpdate | swarmapi.WatchActionKindRemove,
   229  			},
   230  		},
   231  		IncludeOldObject: true,
   232  	})
   233  	if err != nil {
   234  		logrus.WithError(err).Error("failed to watch cluster store")
   235  		return
   236  	}
   237  	for {
   238  		msg, err := watch.Recv()
   239  		if err != nil {
   240  			// store watch is broken
   241  			errStatus, ok := status.FromError(err)
   242  			if !ok || errStatus.Code() != codes.Canceled {
   243  				logrus.WithError(err).Error("failed to receive changes from store watch API")
   244  			}
   245  			return
   246  		}
   247  		select {
   248  		case <-ctx.Done():
   249  			return
   250  		case n.cluster.watchStream <- msg:
   251  		}
   252  	}
   253  }
   254  
   255  func (n *nodeRunner) handleReadyEvent(ctx context.Context, node *swarmnode.Node, ready chan struct{}) {
   256  	select {
   257  	case <-node.Ready():
   258  		n.mu.Lock()
   259  		n.err = nil
   260  		if n.config.JoinInProgress {
   261  			n.config.JoinInProgress = false
   262  			savePersistentState(n.cluster.root, n.config)
   263  		}
   264  		n.mu.Unlock()
   265  		close(ready)
   266  	case <-ctx.Done():
   267  	}
   268  	n.cluster.SendClusterEvent(lncluster.EventNodeReady)
   269  }
   270  
   271  func (n *nodeRunner) handleNodeExit(node *swarmnode.Node) {
   272  	err := detectLockedError(node.Err(context.Background()))
   273  	if err != nil {
   274  		logrus.Errorf("cluster exited with error: %v", err)
   275  	}
   276  	n.mu.Lock()
   277  	n.swarmNode = nil
   278  	n.err = err
   279  	close(n.done)
   280  	select {
   281  	case <-n.ready:
   282  		n.enableReconnectWatcher()
   283  	default:
   284  		if n.repeatedRun {
   285  			n.enableReconnectWatcher()
   286  		}
   287  	}
   288  	n.repeatedRun = true
   289  	n.mu.Unlock()
   290  }
   291  
   292  // Stop stops the current swarm node if it is running.
   293  func (n *nodeRunner) Stop() error {
   294  	n.mu.Lock()
   295  	if n.cancelReconnect != nil { // between restarts
   296  		n.cancelReconnect()
   297  		n.cancelReconnect = nil
   298  	}
   299  	if n.swarmNode == nil {
   300  		// even though the swarm node is nil we still may need
   301  		// to send a node leave event to perform any cleanup required.
   302  		if n.cluster != nil {
   303  			n.cluster.SendClusterEvent(lncluster.EventNodeLeave)
   304  		}
   305  		n.mu.Unlock()
   306  		return nil
   307  	}
   308  	n.stopping = true
   309  	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
   310  	defer cancel()
   311  	n.mu.Unlock()
   312  	if err := n.swarmNode.Stop(ctx); err != nil && !strings.Contains(err.Error(), "context canceled") {
   313  		return err
   314  	}
   315  	n.cluster.SendClusterEvent(lncluster.EventNodeLeave)
   316  	<-n.done
   317  	return nil
   318  }
   319  
   320  func (n *nodeRunner) State() nodeState {
   321  	if n == nil {
   322  		return nodeState{status: types.LocalNodeStateInactive}
   323  	}
   324  	n.mu.RLock()
   325  	defer n.mu.RUnlock()
   326  
   327  	ns := n.nodeState
   328  
   329  	if ns.err != nil || n.cancelReconnect != nil {
   330  		if errors.Is(ns.err, errSwarmLocked) {
   331  			ns.status = types.LocalNodeStateLocked
   332  		} else {
   333  			ns.status = types.LocalNodeStateError
   334  		}
   335  	} else {
   336  		select {
   337  		case <-n.ready:
   338  			ns.status = types.LocalNodeStateActive
   339  		default:
   340  			ns.status = types.LocalNodeStatePending
   341  		}
   342  	}
   343  
   344  	return ns
   345  }
   346  
   347  func (n *nodeRunner) enableReconnectWatcher() {
   348  	if n.stopping {
   349  		return
   350  	}
   351  	n.reconnectDelay *= 2
   352  	if n.reconnectDelay > maxReconnectDelay {
   353  		n.reconnectDelay = maxReconnectDelay
   354  	}
   355  	logrus.Warnf("Restarting swarm in %.2f seconds", n.reconnectDelay.Seconds())
   356  	delayCtx, cancel := context.WithTimeout(context.Background(), n.reconnectDelay)
   357  	n.cancelReconnect = cancel
   358  
   359  	go func() {
   360  		<-delayCtx.Done()
   361  		if delayCtx.Err() != context.DeadlineExceeded {
   362  			return
   363  		}
   364  		n.mu.Lock()
   365  		defer n.mu.Unlock()
   366  		if n.stopping {
   367  			return
   368  		}
   369  
   370  		if err := n.start(n.config); err != nil {
   371  			n.err = err
   372  		}
   373  	}()
   374  }
   375  
   376  // nodeState represents information about the current state of the cluster and
   377  // provides access to the grpc clients.
   378  type nodeState struct {
   379  	swarmNode       *swarmnode.Node
   380  	grpcConn        *grpc.ClientConn
   381  	controlClient   swarmapi.ControlClient
   382  	logsClient      swarmapi.LogsClient
   383  	status          types.LocalNodeState
   384  	actualLocalAddr string
   385  	err             error
   386  }
   387  
   388  // IsActiveManager returns true if node is a manager ready to accept control requests. It is safe to access the client properties if this returns true.
   389  func (ns nodeState) IsActiveManager() bool {
   390  	return ns.controlClient != nil
   391  }
   392  
   393  // IsManager returns true if node is a manager.
   394  func (ns nodeState) IsManager() bool {
   395  	return ns.swarmNode != nil && ns.swarmNode.Manager() != nil
   396  }
   397  
   398  // NodeID returns node's ID or empty string if node is inactive.
   399  func (ns nodeState) NodeID() string {
   400  	if ns.swarmNode != nil {
   401  		return ns.swarmNode.NodeID()
   402  	}
   403  	return ""
   404  }