github.com/hernad/nomad@v1.6.112/nomad/volumewatcher/volumes_watcher.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package volumewatcher
     5  
     6  import (
     7  	"context"
     8  	"sync"
     9  	"time"
    10  
    11  	log "github.com/hashicorp/go-hclog"
    12  	memdb "github.com/hashicorp/go-memdb"
    13  	"github.com/hernad/nomad/nomad/state"
    14  	"github.com/hernad/nomad/nomad/structs"
    15  )
    16  
    17  // Watcher is used to watch volumes and their allocations created
    18  // by the scheduler and trigger the scheduler when allocation health
    19  // transitions.
    20  type Watcher struct {
    21  	enabled bool
    22  	logger  log.Logger
    23  
    24  	// rpc contains the set of Server methods that can be used by
    25  	// the volumes watcher for RPC
    26  	rpc CSIVolumeRPC
    27  
    28  	// the ACL needed to send RPCs
    29  	leaderAcl string
    30  
    31  	// state is the state that is watched for state changes.
    32  	state *state.StateStore
    33  
    34  	// watchers is the set of active watchers, one per volume
    35  	watchers map[string]*volumeWatcher
    36  
    37  	// ctx and exitFn are used to cancel the watcher
    38  	ctx    context.Context
    39  	exitFn context.CancelFunc
    40  
    41  	// quiescentTimeout is the time we wait until the volume has "settled"
    42  	// before stopping the child watcher goroutines
    43  	quiescentTimeout time.Duration
    44  
    45  	wlock sync.RWMutex
    46  }
    47  
    48  var defaultQuiescentTimeout = time.Minute * 5
    49  
    50  // NewVolumesWatcher returns a volumes watcher that is used to watch
    51  // volumes and trigger the scheduler as needed.
    52  func NewVolumesWatcher(logger log.Logger, rpc CSIVolumeRPC, leaderAcl string) *Watcher {
    53  
    54  	// the leader step-down calls SetEnabled(false) which is what
    55  	// cancels this context, rather than passing in its own shutdown
    56  	// context
    57  	ctx, exitFn := context.WithCancel(context.Background())
    58  
    59  	return &Watcher{
    60  		rpc:              rpc,
    61  		logger:           logger.Named("volumes_watcher"),
    62  		ctx:              ctx,
    63  		exitFn:           exitFn,
    64  		leaderAcl:        leaderAcl,
    65  		quiescentTimeout: defaultQuiescentTimeout,
    66  	}
    67  }
    68  
    69  // SetEnabled is used to control if the watcher is enabled. The
    70  // watcher should only be enabled on the active leader. When being
    71  // enabled the state and leader's ACL is passed in as it is no longer
    72  // valid once a leader election has taken place.
    73  func (w *Watcher) SetEnabled(enabled bool, state *state.StateStore, leaderAcl string) {
    74  	w.wlock.Lock()
    75  	defer w.wlock.Unlock()
    76  
    77  	wasEnabled := w.enabled
    78  	w.enabled = enabled
    79  	w.leaderAcl = leaderAcl
    80  
    81  	if state != nil {
    82  		w.state = state
    83  	}
    84  
    85  	// Flush the state to create the necessary objects
    86  	w.flush(enabled)
    87  
    88  	// If we are starting now, launch the watch daemon
    89  	if enabled && !wasEnabled {
    90  		go w.watchVolumes(w.ctx)
    91  	}
    92  }
    93  
    94  // flush is used to clear the state of the watcher
    95  func (w *Watcher) flush(enabled bool) {
    96  	// Stop all the watchers and clear it
    97  	for _, watcher := range w.watchers {
    98  		watcher.Stop()
    99  	}
   100  
   101  	// Kill everything associated with the watcher
   102  	if w.exitFn != nil {
   103  		w.exitFn()
   104  	}
   105  
   106  	w.watchers = make(map[string]*volumeWatcher, 32)
   107  	w.ctx, w.exitFn = context.WithCancel(context.Background())
   108  }
   109  
   110  // watchVolumes is the long lived go-routine that watches for volumes to
   111  // add and remove watchers on.
   112  func (w *Watcher) watchVolumes(ctx context.Context) {
   113  	vIndex := uint64(1)
   114  	for {
   115  		volumes, idx, err := w.getVolumes(ctx, vIndex)
   116  		if err != nil {
   117  			if err == context.Canceled {
   118  				return
   119  			}
   120  			w.logger.Error("failed to retrieve volumes", "error", err)
   121  		}
   122  
   123  		vIndex = idx // last-seen index
   124  		for _, v := range volumes {
   125  			if err := w.add(v); err != nil {
   126  				w.logger.Error("failed to track volume", "volume_id", v.ID, "error", err)
   127  			}
   128  
   129  		}
   130  	}
   131  }
   132  
   133  // getVolumes retrieves all volumes blocking at the given index.
   134  func (w *Watcher) getVolumes(ctx context.Context, minIndex uint64) ([]*structs.CSIVolume, uint64, error) {
   135  	resp, index, err := w.state.BlockingQuery(w.getVolumesImpl, minIndex, ctx)
   136  	if err != nil {
   137  		return nil, 0, err
   138  	}
   139  
   140  	return resp.([]*structs.CSIVolume), index, nil
   141  }
   142  
   143  // getVolumesImpl retrieves all volumes from the passed state store.
   144  func (w *Watcher) getVolumesImpl(ws memdb.WatchSet, state *state.StateStore) (interface{}, uint64, error) {
   145  
   146  	iter, err := state.CSIVolumes(ws)
   147  	if err != nil {
   148  		return nil, 0, err
   149  	}
   150  
   151  	var volumes []*structs.CSIVolume
   152  	for {
   153  		raw := iter.Next()
   154  		if raw == nil {
   155  			break
   156  		}
   157  		volume := raw.(*structs.CSIVolume)
   158  		volumes = append(volumes, volume)
   159  	}
   160  
   161  	// Use the last index that affected the volume table
   162  	index, err := state.Index("csi_volumes")
   163  	if err != nil {
   164  		return nil, 0, err
   165  	}
   166  
   167  	return volumes, index, nil
   168  }
   169  
   170  // add adds a volume to the watch list
   171  func (w *Watcher) add(v *structs.CSIVolume) error {
   172  	w.wlock.Lock()
   173  	defer w.wlock.Unlock()
   174  	_, err := w.addLocked(v)
   175  	return err
   176  }
   177  
   178  // addLocked adds a volume to the watch list and should only be called when
   179  // locked. Creating the volumeWatcher starts a go routine to .watch() it
   180  func (w *Watcher) addLocked(v *structs.CSIVolume) (*volumeWatcher, error) {
   181  	// Not enabled so no-op
   182  	if !w.enabled {
   183  		return nil, nil
   184  	}
   185  
   186  	// Already watched so trigger an update for the volume
   187  	if watcher, ok := w.watchers[v.ID+v.Namespace]; ok {
   188  		watcher.Notify(v)
   189  		return nil, nil
   190  	}
   191  
   192  	watcher := newVolumeWatcher(w, v)
   193  	w.watchers[v.ID+v.Namespace] = watcher
   194  
   195  	// Sending the first volume update here before we return ensures we've hit
   196  	// the run loop in the goroutine before freeing the lock. This prevents a
   197  	// race between shutting down the watcher and the blocking query.
   198  	//
   199  	// It also ensures that we don't drop events that happened during leadership
   200  	// transitions and didn't get completed by the prior leader
   201  	watcher.updateCh <- v
   202  	return watcher, nil
   203  }
   204  
   205  // removes a volume from the watch list
   206  func (w *Watcher) remove(volID string) {
   207  	w.wlock.Lock()
   208  	defer w.wlock.Unlock()
   209  	delete(w.watchers, volID)
   210  }