github.com/m3db/m3@v1.5.0/src/aggregator/tools/deploy/validator.go (about)

     1  // Copyright (c) 2017 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package deploy
    22  
    23  import (
    24  	"errors"
    25  	"fmt"
    26  	"sync"
    27  
    28  	"github.com/m3db/m3/src/aggregator/aggregator"
    29  	xsync "github.com/m3db/m3/src/x/sync"
    30  )
    31  
    32  // validator performs validation on a deployment target, returning nil if
    33  // validation succeeds, and an error otherwise.
    34  type validator func() error
    35  
    36  // validatorFactory is a factory of validators.
    37  type validatorFactory interface {
    38  	// ValidatorFor generates a validator based on the instance,
    39  	// the instance group, and the instance target type.
    40  	ValidatorFor(
    41  		instance instanceMetadata,
    42  		group *instanceGroup,
    43  		targetType targetType,
    44  	) validator
    45  }
    46  
    47  type factory struct {
    48  	client  AggregatorClient
    49  	workers xsync.WorkerPool
    50  }
    51  
    52  func newValidatorFactory(client AggregatorClient, workers xsync.WorkerPool) validatorFactory {
    53  	return factory{client: client, workers: workers}
    54  }
    55  
    56  func (f factory) ValidatorFor(
    57  	instance instanceMetadata,
    58  	group *instanceGroup,
    59  	targetType targetType,
    60  ) validator {
    61  	if targetType == leaderTarget {
    62  		return f.validatorForLeader(instance, group)
    63  	}
    64  	return f.validatorForFollower(instance)
    65  }
    66  
    67  func (f factory) validatorForFollower(follower instanceMetadata) validator {
    68  	return func() error {
    69  		status, err := f.client.Status(follower.APIEndpoint)
    70  		if err != nil {
    71  			return err
    72  		}
    73  		electionState := status.FlushStatus.ElectionState
    74  		if electionState != aggregator.FollowerState {
    75  			return fmt.Errorf("election state is %s not follower", electionState.String())
    76  		}
    77  		return nil
    78  	}
    79  }
    80  
    81  func (f factory) validatorForLeader(
    82  	leader instanceMetadata,
    83  	group *instanceGroup,
    84  ) validator {
    85  	return func() error {
    86  		status, err := f.client.Status(leader.APIEndpoint)
    87  		if err != nil {
    88  			return err
    89  		}
    90  		electionState := status.FlushStatus.ElectionState
    91  		if electionState != aggregator.LeaderState {
    92  			return fmt.Errorf("election state is %s not leader", electionState.String())
    93  		}
    94  		// We also need to verify the followers in the same group are ready to take
    95  		// over the leader role when the leader resigns.
    96  		var (
    97  			found     bool
    98  			wg        sync.WaitGroup
    99  			canLeadCh = make(chan bool, 1)
   100  		)
   101  		for _, instance := range group.All {
   102  			// Skip check if the instance is the leader.
   103  			if instance.PlacementInstanceID == group.LeaderID {
   104  				found = true
   105  				continue
   106  			}
   107  			instance := instance
   108  			wg.Add(1)
   109  			f.workers.Go(func() {
   110  				defer wg.Done()
   111  
   112  				status, err := f.client.Status(instance.APIEndpoint)
   113  				if err != nil {
   114  					return
   115  				}
   116  				isFollower := status.FlushStatus.ElectionState == aggregator.FollowerState
   117  				canLead := status.FlushStatus.CanLead
   118  				if isFollower && canLead {
   119  					select {
   120  					case canLeadCh <- true:
   121  					default:
   122  					}
   123  				}
   124  			})
   125  		}
   126  
   127  		// Asynchronously close the channel after all goroutines return.
   128  		f.workers.Go(func() {
   129  			wg.Wait()
   130  			close(canLeadCh)
   131  		})
   132  
   133  		if !found {
   134  			return fmt.Errorf("instance %s is not in the instance group", leader.PlacementInstanceID)
   135  		}
   136  		if canLead := <-canLeadCh; !canLead {
   137  			return errors.New("no follower instance is ready to lead")
   138  		}
   139  		return nil
   140  	}
   141  }