github.com/hernad/nomad@v1.6.112/nomad/volumewatcher/volumes_watcher.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package volumewatcher 5 6 import ( 7 "context" 8 "sync" 9 "time" 10 11 log "github.com/hashicorp/go-hclog" 12 memdb "github.com/hashicorp/go-memdb" 13 "github.com/hernad/nomad/nomad/state" 14 "github.com/hernad/nomad/nomad/structs" 15 ) 16 17 // Watcher is used to watch volumes and their allocations created 18 // by the scheduler and trigger the scheduler when allocation health 19 // transitions. 20 type Watcher struct { 21 enabled bool 22 logger log.Logger 23 24 // rpc contains the set of Server methods that can be used by 25 // the volumes watcher for RPC 26 rpc CSIVolumeRPC 27 28 // the ACL needed to send RPCs 29 leaderAcl string 30 31 // state is the state that is watched for state changes. 32 state *state.StateStore 33 34 // watchers is the set of active watchers, one per volume 35 watchers map[string]*volumeWatcher 36 37 // ctx and exitFn are used to cancel the watcher 38 ctx context.Context 39 exitFn context.CancelFunc 40 41 // quiescentTimeout is the time we wait until the volume has "settled" 42 // before stopping the child watcher goroutines 43 quiescentTimeout time.Duration 44 45 wlock sync.RWMutex 46 } 47 48 var defaultQuiescentTimeout = time.Minute * 5 49 50 // NewVolumesWatcher returns a volumes watcher that is used to watch 51 // volumes and trigger the scheduler as needed. 52 func NewVolumesWatcher(logger log.Logger, rpc CSIVolumeRPC, leaderAcl string) *Watcher { 53 54 // the leader step-down calls SetEnabled(false) which is what 55 // cancels this context, rather than passing in its own shutdown 56 // context 57 ctx, exitFn := context.WithCancel(context.Background()) 58 59 return &Watcher{ 60 rpc: rpc, 61 logger: logger.Named("volumes_watcher"), 62 ctx: ctx, 63 exitFn: exitFn, 64 leaderAcl: leaderAcl, 65 quiescentTimeout: defaultQuiescentTimeout, 66 } 67 } 68 69 // SetEnabled is used to control if the watcher is enabled. The 70 // watcher should only be enabled on the active leader. When being 71 // enabled the state and leader's ACL is passed in as it is no longer 72 // valid once a leader election has taken place. 73 func (w *Watcher) SetEnabled(enabled bool, state *state.StateStore, leaderAcl string) { 74 w.wlock.Lock() 75 defer w.wlock.Unlock() 76 77 wasEnabled := w.enabled 78 w.enabled = enabled 79 w.leaderAcl = leaderAcl 80 81 if state != nil { 82 w.state = state 83 } 84 85 // Flush the state to create the necessary objects 86 w.flush(enabled) 87 88 // If we are starting now, launch the watch daemon 89 if enabled && !wasEnabled { 90 go w.watchVolumes(w.ctx) 91 } 92 } 93 94 // flush is used to clear the state of the watcher 95 func (w *Watcher) flush(enabled bool) { 96 // Stop all the watchers and clear it 97 for _, watcher := range w.watchers { 98 watcher.Stop() 99 } 100 101 // Kill everything associated with the watcher 102 if w.exitFn != nil { 103 w.exitFn() 104 } 105 106 w.watchers = make(map[string]*volumeWatcher, 32) 107 w.ctx, w.exitFn = context.WithCancel(context.Background()) 108 } 109 110 // watchVolumes is the long lived go-routine that watches for volumes to 111 // add and remove watchers on. 112 func (w *Watcher) watchVolumes(ctx context.Context) { 113 vIndex := uint64(1) 114 for { 115 volumes, idx, err := w.getVolumes(ctx, vIndex) 116 if err != nil { 117 if err == context.Canceled { 118 return 119 } 120 w.logger.Error("failed to retrieve volumes", "error", err) 121 } 122 123 vIndex = idx // last-seen index 124 for _, v := range volumes { 125 if err := w.add(v); err != nil { 126 w.logger.Error("failed to track volume", "volume_id", v.ID, "error", err) 127 } 128 129 } 130 } 131 } 132 133 // getVolumes retrieves all volumes blocking at the given index. 134 func (w *Watcher) getVolumes(ctx context.Context, minIndex uint64) ([]*structs.CSIVolume, uint64, error) { 135 resp, index, err := w.state.BlockingQuery(w.getVolumesImpl, minIndex, ctx) 136 if err != nil { 137 return nil, 0, err 138 } 139 140 return resp.([]*structs.CSIVolume), index, nil 141 } 142 143 // getVolumesImpl retrieves all volumes from the passed state store. 144 func (w *Watcher) getVolumesImpl(ws memdb.WatchSet, state *state.StateStore) (interface{}, uint64, error) { 145 146 iter, err := state.CSIVolumes(ws) 147 if err != nil { 148 return nil, 0, err 149 } 150 151 var volumes []*structs.CSIVolume 152 for { 153 raw := iter.Next() 154 if raw == nil { 155 break 156 } 157 volume := raw.(*structs.CSIVolume) 158 volumes = append(volumes, volume) 159 } 160 161 // Use the last index that affected the volume table 162 index, err := state.Index("csi_volumes") 163 if err != nil { 164 return nil, 0, err 165 } 166 167 return volumes, index, nil 168 } 169 170 // add adds a volume to the watch list 171 func (w *Watcher) add(v *structs.CSIVolume) error { 172 w.wlock.Lock() 173 defer w.wlock.Unlock() 174 _, err := w.addLocked(v) 175 return err 176 } 177 178 // addLocked adds a volume to the watch list and should only be called when 179 // locked. Creating the volumeWatcher starts a go routine to .watch() it 180 func (w *Watcher) addLocked(v *structs.CSIVolume) (*volumeWatcher, error) { 181 // Not enabled so no-op 182 if !w.enabled { 183 return nil, nil 184 } 185 186 // Already watched so trigger an update for the volume 187 if watcher, ok := w.watchers[v.ID+v.Namespace]; ok { 188 watcher.Notify(v) 189 return nil, nil 190 } 191 192 watcher := newVolumeWatcher(w, v) 193 w.watchers[v.ID+v.Namespace] = watcher 194 195 // Sending the first volume update here before we return ensures we've hit 196 // the run loop in the goroutine before freeing the lock. This prevents a 197 // race between shutting down the watcher and the blocking query. 198 // 199 // It also ensures that we don't drop events that happened during leadership 200 // transitions and didn't get completed by the prior leader 201 watcher.updateCh <- v 202 return watcher, nil 203 } 204 205 // removes a volume from the watch list 206 func (w *Watcher) remove(volID string) { 207 w.wlock.Lock() 208 defer w.wlock.Unlock() 209 delete(w.watchers, volID) 210 }