github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/m3em/cluster/cluster.go (about)

     1  // Copyright (c) 2017 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package cluster
    22  
    23  import (
    24  	"fmt"
    25  	"sync"
    26  
    27  	"github.com/m3db/m3/src/cluster/placement"
    28  	"github.com/m3db/m3/src/cluster/shard"
    29  	"github.com/m3db/m3/src/m3em/node"
    30  	xerrors "github.com/m3db/m3/src/x/errors"
    31  
    32  	"go.uber.org/zap"
    33  )
    34  
    35  var (
    36  	errInsufficientCapacity          = fmt.Errorf("insufficient node capacity in environment")
    37  	errNodeNotInUse                  = fmt.Errorf("unable to remove node, not in use")
    38  	errClusterNotUnitialized         = fmt.Errorf("unable to setup cluster, it is not unitialized")
    39  	errClusterUnableToAlterPlacement = fmt.Errorf("unable to alter cluster placement, it needs to be setup/running")
    40  	errUnableToStartUnsetupCluster   = fmt.Errorf("unable to start cluster, it has not been setup")
    41  	errClusterUnableToTeardown       = fmt.Errorf("unable to teardown cluster, it has not been setup")
    42  	errUnableToStopNotRunningCluster = fmt.Errorf("unable to stop cluster, it is running")
    43  )
    44  
    45  type idToNodeMap map[string]node.ServiceNode
    46  
    47  func (im idToNodeMap) values() []node.ServiceNode {
    48  	returnNodes := make([]node.ServiceNode, 0, len(im))
    49  	for _, node := range im {
    50  		returnNodes = append(returnNodes, node)
    51  	}
    52  	return returnNodes
    53  }
    54  
    55  type svcCluster struct {
    56  	sync.RWMutex
    57  
    58  	logger       *zap.Logger
    59  	opts         Options
    60  	knownNodes   node.ServiceNodes
    61  	usedNodes    idToNodeMap
    62  	spares       []node.ServiceNode
    63  	sparesByID   map[string]node.ServiceNode
    64  	placementSvc placement.Service
    65  	placement    placement.Placement
    66  	status       Status
    67  	lastErr      error
    68  }
    69  
    70  // New returns a new cluster backed by provided service nodes
    71  func New(
    72  	nodes node.ServiceNodes,
    73  	opts Options,
    74  ) (Cluster, error) {
    75  	if err := opts.Validate(); err != nil {
    76  		return nil, err
    77  	}
    78  
    79  	cluster := &svcCluster{
    80  		logger:       opts.InstrumentOptions().Logger(),
    81  		opts:         opts,
    82  		knownNodes:   nodes,
    83  		usedNodes:    make(idToNodeMap, len(nodes)),
    84  		spares:       make([]node.ServiceNode, 0, len(nodes)),
    85  		sparesByID:   make(map[string]node.ServiceNode, len(nodes)),
    86  		placementSvc: opts.PlacementService(),
    87  		status:       ClusterStatusUninitialized,
    88  	}
    89  	cluster.addSparesWithLock(nodes)
    90  
    91  	return cluster, nil
    92  }
    93  
    94  func (c *svcCluster) addSparesWithLock(spares []node.ServiceNode) {
    95  	for _, spare := range spares {
    96  		c.spares = append(c.spares, spare)
    97  		c.sparesByID[spare.ID()] = spare
    98  	}
    99  }
   100  
   101  func nodeSliceWithoutID(originalSlice node.ServiceNodes, removeID string) node.ServiceNodes {
   102  	newSlice := make(node.ServiceNodes, 0, len(originalSlice))
   103  	for _, elem := range originalSlice {
   104  		if elem.ID() != removeID {
   105  			newSlice = append(newSlice, elem)
   106  		}
   107  	}
   108  	return newSlice
   109  }
   110  
   111  func (c *svcCluster) newExecutor(
   112  	nodes node.ServiceNodes,
   113  	fn node.ServiceNodeFn,
   114  ) node.ConcurrentExecutor {
   115  	return node.NewConcurrentExecutor(nodes, c.opts.NodeConcurrency(), c.opts.NodeOperationTimeout(), fn)
   116  }
   117  
   118  func (c *svcCluster) Placement() placement.Placement {
   119  	c.Lock()
   120  	defer c.Unlock()
   121  	return c.placement
   122  }
   123  
   124  func (c *svcCluster) initWithLock() error {
   125  	psvc := c.placementSvc
   126  	_, err := psvc.Placement()
   127  	if err != nil { // attempt to retrieve current placement
   128  		c.logger.Info("unable to retrieve existing placement, skipping delete attempt")
   129  	} else {
   130  		// delete existing placement
   131  		err = c.opts.PlacementServiceRetrier().Attempt(psvc.Delete)
   132  		if err != nil {
   133  			return fmt.Errorf("unable to delete existing placement during setup(): %+v", err)
   134  		}
   135  		c.logger.Info("successfully deleted existing placement")
   136  	}
   137  
   138  	var (
   139  		svcBuild        = c.opts.ServiceBuild()
   140  		svcConf         = c.opts.ServiceConfig()
   141  		sessionToken    = c.opts.SessionToken()
   142  		sessionOverride = c.opts.SessionOverride()
   143  		listener        = c.opts.NodeListener()
   144  	)
   145  
   146  	// setup all known service nodes with build, config
   147  	executor := c.newExecutor(c.knownNodes, func(node node.ServiceNode) error {
   148  		err := node.Setup(svcBuild, svcConf, sessionToken, sessionOverride)
   149  		if err != nil {
   150  			return err
   151  		}
   152  		if listener != nil {
   153  			// NB: no need to track returned listenerID here, it's cleaned up in node.Teardown()
   154  			node.RegisterListener(listener)
   155  		}
   156  		return nil
   157  	})
   158  	return executor.Run()
   159  }
   160  
   161  func (c *svcCluster) Setup(numNodes int) ([]node.ServiceNode, error) {
   162  	c.Lock()
   163  	defer c.Unlock()
   164  
   165  	if c.status != ClusterStatusUninitialized {
   166  		return nil, errClusterNotUnitialized
   167  	}
   168  
   169  	numSpares := len(c.spares)
   170  	if numSpares < numNodes {
   171  		return nil, errInsufficientCapacity
   172  	}
   173  
   174  	if err := c.initWithLock(); err != nil {
   175  		return nil, err
   176  	}
   177  
   178  	psvc := c.placementSvc
   179  	spares := c.sparesAsPlacementInstaceWithLock()[:numNodes]
   180  
   181  	// we don't need to use the retrier here as there are no other users of this placement yet
   182  	placement, err := psvc.BuildInitialPlacement(spares, c.opts.NumShards(), c.opts.Replication())
   183  	if err != nil {
   184  		return nil, err
   185  	}
   186  
   187  	// update ServiceNode with new shards from placement
   188  	var (
   189  		multiErr      xerrors.MultiError
   190  		usedInstances = placement.Instances()
   191  		setupNodes    = make([]node.ServiceNode, 0, len(usedInstances))
   192  	)
   193  	for _, instance := range usedInstances {
   194  		setupNode, err := c.markSpareUsedWithLock(instance)
   195  		if err != nil {
   196  			multiErr = multiErr.Add(err)
   197  			continue
   198  		}
   199  		setupNodes = append(setupNodes, setupNode)
   200  	}
   201  
   202  	multiErr = multiErr.
   203  		Add(c.setPlacementWithLock(placement))
   204  
   205  	return setupNodes, c.markStatusWithLock(ClusterStatusSetup, multiErr.FinalError())
   206  }
   207  
   208  func (c *svcCluster) markSpareUsedWithLock(spare placement.Instance) (node.ServiceNode, error) {
   209  	id := spare.ID()
   210  	spareNode, ok := c.sparesByID[id]
   211  	if !ok {
   212  		// should never happen
   213  		return nil, fmt.Errorf("unable to find spare node with id: %s", id)
   214  	}
   215  	delete(c.sparesByID, id)
   216  	c.spares = nodeSliceWithoutID(c.spares, id)
   217  	c.usedNodes[id] = spareNode
   218  	return spareNode, nil
   219  }
   220  
   221  func (c *svcCluster) AddSpecifiedNode(newNode node.ServiceNode) error {
   222  	c.Lock()
   223  	defer c.Unlock()
   224  
   225  	if !c.isSpareNodeWithLock(newNode) {
   226  		return fmt.Errorf("provided node is not a known spare")
   227  	}
   228  
   229  	_, err := c.addNodeFromListWithLock([]placement.Instance{newNode.(placement.Instance)})
   230  	return err
   231  }
   232  
   233  func (c *svcCluster) isSpareNodeWithLock(n node.ServiceNode) bool {
   234  	_, ok := c.sparesByID[n.ID()]
   235  	return ok
   236  }
   237  
   238  func (c *svcCluster) addNodeFromListWithLock(candidates []placement.Instance) (node.ServiceNode, error) {
   239  	if c.status != ClusterStatusRunning && c.status != ClusterStatusSetup {
   240  		return nil, errClusterUnableToAlterPlacement
   241  	}
   242  
   243  	var (
   244  		psvc          = c.placementSvc
   245  		newPlacement  placement.Placement
   246  		usedInstances []placement.Instance
   247  	)
   248  	if err := c.opts.PlacementServiceRetrier().Attempt(func() error {
   249  		var internalErr error
   250  		newPlacement, usedInstances, internalErr = psvc.AddInstances(candidates)
   251  		return internalErr
   252  	}); err != nil {
   253  		return nil, err
   254  	}
   255  
   256  	if len(usedInstances) != 1 {
   257  		return nil, fmt.Errorf("%d instances added to the placement, expecting 1", len(usedInstances))
   258  	}
   259  
   260  	setupNode, err := c.markSpareUsedWithLock(usedInstances[0])
   261  	if err != nil {
   262  		return nil, err
   263  	}
   264  
   265  	return setupNode, c.setPlacementWithLock(newPlacement)
   266  }
   267  
   268  func (c *svcCluster) AddNode() (node.ServiceNode, error) {
   269  	c.Lock()
   270  	defer c.Unlock()
   271  
   272  	numSpares := len(c.spares)
   273  	if numSpares < 1 {
   274  		return nil, errInsufficientCapacity
   275  	}
   276  
   277  	return c.addNodeFromListWithLock(c.sparesAsPlacementInstaceWithLock())
   278  }
   279  
   280  func (c *svcCluster) setPlacementWithLock(p placement.Placement) error {
   281  	for _, instance := range p.Instances() {
   282  		// nb(prateek): update usedNodes with the new shards.
   283  		instanceID := instance.ID()
   284  		usedNode, ok := c.usedNodes[instanceID]
   285  		if ok {
   286  			usedNode.SetShards(instance.Shards())
   287  		}
   288  	}
   289  
   290  	c.placement = p
   291  	return nil
   292  }
   293  
   294  func (c *svcCluster) sparesAsPlacementInstaceWithLock() []placement.Instance {
   295  	spares := make([]placement.Instance, 0, len(c.spares))
   296  	for _, spare := range c.spares {
   297  		spares = append(spares, spare.(placement.Instance))
   298  	}
   299  	return spares
   300  }
   301  
   302  func (c *svcCluster) RemoveNode(i node.ServiceNode) error {
   303  	c.Lock()
   304  	defer c.Unlock()
   305  
   306  	if c.status != ClusterStatusRunning && c.status != ClusterStatusSetup {
   307  		return errClusterUnableToAlterPlacement
   308  	}
   309  
   310  	usedNode, ok := c.usedNodes[i.ID()]
   311  	if !ok {
   312  		return errNodeNotInUse
   313  	}
   314  
   315  	var (
   316  		newPlacement placement.Placement
   317  		psvc         = c.placementSvc
   318  	)
   319  	if err := c.opts.PlacementServiceRetrier().Attempt(func() error {
   320  		var internalErr error
   321  		newPlacement, internalErr = psvc.RemoveInstances([]string{i.ID()})
   322  		return internalErr
   323  	}); err != nil {
   324  		return err
   325  	}
   326  
   327  	// update removed instance from used -> spare
   328  	// nb(prateek): this omits modeling "leaving" shards on the node being removed
   329  	usedNode.SetShards(shard.NewShards(nil))
   330  	delete(c.usedNodes, usedNode.ID())
   331  	c.addSparesWithLock([]node.ServiceNode{usedNode})
   332  
   333  	return c.setPlacementWithLock(newPlacement)
   334  }
   335  
   336  func (c *svcCluster) ReplaceNode(oldNode node.ServiceNode) ([]node.ServiceNode, error) {
   337  	c.Lock()
   338  	defer c.Unlock()
   339  
   340  	if c.status != ClusterStatusRunning && c.status != ClusterStatusSetup {
   341  		return nil, errClusterUnableToAlterPlacement
   342  	}
   343  
   344  	if _, ok := c.usedNodes[oldNode.ID()]; !ok {
   345  		return nil, errNodeNotInUse
   346  	}
   347  
   348  	var (
   349  		psvc            = c.placementSvc
   350  		spareCandidates = c.sparesAsPlacementInstaceWithLock()
   351  		newPlacement    placement.Placement
   352  		newInstances    []placement.Instance
   353  	)
   354  	if err := c.opts.PlacementServiceRetrier().Attempt(func() error {
   355  		var internalErr error
   356  		newPlacement, newInstances, internalErr = psvc.ReplaceInstances([]string{oldNode.ID()}, spareCandidates)
   357  		return internalErr
   358  	}); err != nil {
   359  		return nil, err
   360  	}
   361  
   362  	// mark old node no longer used
   363  	oldNode.SetShards(shard.NewShards(nil))
   364  	delete(c.usedNodes, oldNode.ID())
   365  	c.addSparesWithLock([]node.ServiceNode{oldNode})
   366  
   367  	var (
   368  		multiErr xerrors.MultiError
   369  		newNodes = make([]node.ServiceNode, 0, len(newInstances))
   370  	)
   371  	for _, instance := range newInstances {
   372  		newNode, err := c.markSpareUsedWithLock(instance)
   373  		if err != nil {
   374  			multiErr = multiErr.Add(err)
   375  			continue
   376  		}
   377  		newNodes = append(newNodes, newNode)
   378  	}
   379  
   380  	multiErr = multiErr.
   381  		Add(c.setPlacementWithLock(newPlacement))
   382  
   383  	return newNodes, multiErr.FinalError()
   384  }
   385  
   386  func (c *svcCluster) SpareNodes() []node.ServiceNode {
   387  	c.Lock()
   388  	defer c.Unlock()
   389  	return c.spares
   390  }
   391  
   392  func (c *svcCluster) ActiveNodes() []node.ServiceNode {
   393  	c.Lock()
   394  	defer c.Unlock()
   395  	return c.usedNodes.values()
   396  }
   397  
   398  func (c *svcCluster) KnownNodes() []node.ServiceNode {
   399  	c.Lock()
   400  	defer c.Unlock()
   401  	return c.knownNodes
   402  }
   403  
   404  func (c *svcCluster) markStatusWithLock(status Status, err error) error {
   405  	if err == nil {
   406  		c.status = status
   407  		return nil
   408  	}
   409  
   410  	c.status = ClusterStatusError
   411  	c.lastErr = err
   412  	return err
   413  }
   414  
   415  func (c *svcCluster) Teardown() error {
   416  	c.Lock()
   417  	defer c.Unlock()
   418  
   419  	if c.status == ClusterStatusUninitialized {
   420  		return errClusterUnableToTeardown
   421  	}
   422  
   423  	err := c.newExecutor(c.knownNodes, func(node node.ServiceNode) error {
   424  		return node.Teardown()
   425  	}).Run()
   426  
   427  	for id, usedNode := range c.usedNodes {
   428  		usedNode.SetShards(shard.NewShards(nil))
   429  		delete(c.usedNodes, id)
   430  	}
   431  	c.spares = make([]node.ServiceNode, 0, len(c.knownNodes))
   432  	c.sparesByID = make(map[string]node.ServiceNode, len(c.knownNodes))
   433  	c.addSparesWithLock(c.knownNodes)
   434  
   435  	return c.markStatusWithLock(ClusterStatusUninitialized, err)
   436  }
   437  
   438  func (c *svcCluster) Start() error {
   439  	c.Lock()
   440  	defer c.Unlock()
   441  
   442  	if c.status != ClusterStatusSetup {
   443  		return errUnableToStartUnsetupCluster
   444  	}
   445  
   446  	err := c.newExecutor(c.usedNodes.values(), func(node node.ServiceNode) error {
   447  		return node.Start()
   448  	}).Run()
   449  
   450  	return c.markStatusWithLock(ClusterStatusRunning, err)
   451  }
   452  
   453  func (c *svcCluster) Stop() error {
   454  	c.Lock()
   455  	defer c.Unlock()
   456  
   457  	if c.status != ClusterStatusRunning {
   458  		return errUnableToStopNotRunningCluster
   459  	}
   460  
   461  	err := c.newExecutor(c.usedNodes.values(), func(node node.ServiceNode) error {
   462  		return node.Stop()
   463  	}).Run()
   464  
   465  	return c.markStatusWithLock(ClusterStatusSetup, err)
   466  }
   467  
   468  func (c *svcCluster) Status() Status {
   469  	c.RLock()
   470  	defer c.RUnlock()
   471  	return c.status
   472  }