github.com/hernad/nomad@v1.6.112/helper/raftutil/fsm.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package raftutil 5 6 import ( 7 "fmt" 8 "io" 9 "path/filepath" 10 "strings" 11 12 "github.com/hashicorp/go-hclog" 13 "github.com/hashicorp/go-memdb" 14 "github.com/hernad/nomad/nomad" 15 "github.com/hernad/nomad/nomad/state" 16 "github.com/hashicorp/raft" 17 raftboltdb "github.com/hashicorp/raft-boltdb/v2" 18 ) 19 20 var ErrNoMoreLogs = fmt.Errorf("no more logs") 21 22 type nomadFSM interface { 23 raft.FSM 24 State() *state.StateStore 25 Restore(io.ReadCloser) error 26 RestoreWithFilter(io.ReadCloser, *nomad.FSMFilter) error 27 } 28 29 type FSMHelper struct { 30 path string 31 32 logger hclog.Logger 33 34 // nomad state 35 store *raftboltdb.BoltStore 36 fsm nomadFSM 37 snaps *raft.FileSnapshotStore 38 39 // raft 40 logFirstIdx uint64 41 logLastIdx uint64 42 nextIdx uint64 43 } 44 45 func NewFSM(p string) (*FSMHelper, error) { 46 store, firstIdx, lastIdx, err := RaftStateInfo(filepath.Join(p, "raft.db")) 47 if err != nil { 48 return nil, fmt.Errorf("failed to open raft database %v: %v", p, err) 49 } 50 51 logger := hclog.L() 52 53 snaps, err := raft.NewFileSnapshotStoreWithLogger(p, 1000, logger) 54 if err != nil { 55 store.Close() 56 return nil, fmt.Errorf("failed to open snapshot dir: %v", err) 57 } 58 59 fsm, err := dummyFSM(logger) 60 if err != nil { 61 store.Close() 62 return nil, err 63 } 64 65 return &FSMHelper{ 66 path: p, 67 logger: logger, 68 store: store, 69 fsm: fsm, 70 snaps: snaps, 71 72 logFirstIdx: firstIdx, 73 logLastIdx: lastIdx, 74 nextIdx: uint64(1), 75 }, nil 76 } 77 78 func dummyFSM(logger hclog.Logger) (nomadFSM, error) { 79 // use dummy non-enabled FSM dependencies 80 periodicDispatch := nomad.NewPeriodicDispatch(logger, nil) 81 blockedEvals := nomad.NewBlockedEvals(nil, logger) 82 evalBroker, err := nomad.NewEvalBroker(1, 1, 1, 1) 83 if err != nil { 84 return nil, err 85 } 86 87 fsmConfig := &nomad.FSMConfig{ 88 EvalBroker: evalBroker, 89 Periodic: periodicDispatch, 90 Blocked: blockedEvals, 91 Logger: logger, 92 Region: "default", 93 } 94 95 return nomad.NewFSM(fsmConfig) 96 } 97 98 func (f *FSMHelper) Close() { 99 f.store.Close() 100 101 } 102 103 func (f *FSMHelper) ApplyNext() (index uint64, term uint64, err error) { 104 if f.nextIdx == 1 { 105 // check snapshots first 106 index, term, err := f.restoreFromSnapshot() 107 if err != nil { 108 return 0, 0, err 109 } 110 111 if index != 0 { 112 f.nextIdx = index + 1 113 return index, term, nil 114 } 115 } 116 117 if f.nextIdx < f.logFirstIdx { 118 return 0, 0, fmt.Errorf("missing logs [%v, %v]", f.nextIdx, f.logFirstIdx-1) 119 } 120 121 if f.nextIdx > f.logLastIdx { 122 return 0, 0, ErrNoMoreLogs 123 } 124 125 var e raft.Log 126 err = f.store.GetLog(f.nextIdx, &e) 127 if err != nil { 128 return 0, 0, fmt.Errorf("failed to read log entry at index %d: %v", f.nextIdx, err) 129 } 130 131 defer func() { 132 r := recover() 133 if r != nil && strings.HasPrefix(fmt.Sprint(r), "failed to apply request") { 134 // Enterprise specific log entries will fail to load in OSS repository with "failed to apply request." 135 // If not relevant to investigation, we can ignore them and simply worn. 136 f.logger.Warn("failed to apply log; loading Enterprise data-dir in OSS binary?", "index", e.Index) 137 138 f.nextIdx++ 139 } else if r != nil { 140 panic(r) 141 } 142 }() 143 144 if e.Type == raft.LogCommand { 145 f.fsm.Apply(&e) 146 } 147 148 f.nextIdx++ 149 return e.Index, e.Term, nil 150 } 151 152 // ApplyUntil applies all raft entries until (inclusive) the passed index. 153 func (f *FSMHelper) ApplyUntil(stopIdx uint64) (idx uint64, term uint64, err error) { 154 var lastIdx, lastTerm uint64 155 for { 156 idx, term, err := f.ApplyNext() 157 if err == ErrNoMoreLogs { 158 return lastIdx, lastTerm, nil 159 } else if err != nil { 160 return lastIdx, lastTerm, err 161 } else if idx >= stopIdx { 162 return lastIdx, lastTerm, nil 163 } 164 165 lastIdx, lastTerm = idx, term 166 } 167 } 168 169 func (f *FSMHelper) ApplyAll() (index uint64, term uint64, err error) { 170 var lastIdx, lastTerm uint64 171 for { 172 idx, term, err := f.ApplyNext() 173 if err == ErrNoMoreLogs { 174 return lastIdx, lastTerm, nil 175 } else if err != nil { 176 return lastIdx, lastTerm, err 177 } 178 179 lastIdx, lastTerm = idx, term 180 } 181 } 182 183 func (f *FSMHelper) State() *state.StateStore { 184 return f.fsm.State() 185 } 186 187 func (f *FSMHelper) StateAsMap() map[string][]interface{} { 188 return StateAsMap(f.fsm.State()) 189 } 190 191 // StateAsMap returns a json-able representation of the state 192 func StateAsMap(store *state.StateStore) map[string][]interface{} { 193 result := map[string][]interface{}{ 194 "ACLPolicies": toArray(store.ACLPolicies(nil)), 195 "ACLTokens": toArray(store.ACLTokens(nil, state.SortDefault)), 196 "Allocs": toArray(store.Allocs(nil, state.SortDefault)), 197 "CSIPlugins": toArray(store.CSIPlugins(nil)), 198 "CSIVolumes": toArray(store.CSIVolumes(nil)), 199 "Deployments": toArray(store.Deployments(nil, state.SortDefault)), 200 "Evals": toArray(store.Evals(nil, state.SortDefault)), 201 "Indexes": toArray(store.Indexes()), 202 "JobSummaries": toArray(store.JobSummaries(nil)), 203 "JobVersions": toArray(store.JobVersions(nil)), 204 "Jobs": toArray(store.Jobs(nil)), 205 "Nodes": toArray(store.Nodes(nil)), 206 "PeriodicLaunches": toArray(store.PeriodicLaunches(nil)), 207 "SITokenAccessors": toArray(store.SITokenAccessors(nil)), 208 "ScalingEvents": toArray(store.ScalingEvents(nil)), 209 "ScalingPolicies": toArray(store.ScalingPolicies(nil)), 210 "VaultAccessors": toArray(store.VaultAccessors(nil)), 211 } 212 213 insertEnterpriseState(result, store) 214 215 return result 216 217 } 218 219 func (f *FSMHelper) restoreFromSnapshot() (index uint64, term uint64, err error) { 220 snapshots, err := f.snaps.List() 221 if err != nil { 222 return 0, 0, err 223 } 224 f.logger.Debug("found snapshots", "count", len(snapshots)) 225 226 for _, snapshot := range snapshots { 227 _, source, err := f.snaps.Open(snapshot.ID) 228 if err != nil { 229 f.logger.Warn("failed to open a snapshot", "snapshot_id", snapshot.ID, "error", err) 230 continue 231 } 232 233 err = f.fsm.Restore(source) 234 source.Close() 235 if err != nil { 236 f.logger.Warn("failed to restore a snapshot", "snapshot_id", snapshot.ID, "error", err) 237 continue 238 } 239 240 return snapshot.Index, snapshot.Term, nil 241 } 242 243 return 0, 0, nil 244 } 245 246 func toArray(iter memdb.ResultIterator, err error) []interface{} { 247 if err != nil { 248 return []interface{}{err} 249 } 250 251 r := []interface{}{} 252 253 if iter != nil { 254 item := iter.Next() 255 for item != nil { 256 r = append(r, item) 257 item = iter.Next() 258 } 259 } 260 261 return r 262 }