github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/helper/raftutil/fsm.go (about) 1 package raftutil 2 3 import ( 4 "fmt" 5 "io" 6 "path/filepath" 7 "strings" 8 9 "github.com/hashicorp/go-hclog" 10 "github.com/hashicorp/go-memdb" 11 "github.com/hashicorp/nomad/nomad" 12 "github.com/hashicorp/nomad/nomad/state" 13 "github.com/hashicorp/raft" 14 raftboltdb "github.com/hashicorp/raft-boltdb/v2" 15 ) 16 17 var ErrNoMoreLogs = fmt.Errorf("no more logs") 18 19 type nomadFSM interface { 20 raft.FSM 21 State() *state.StateStore 22 Restore(io.ReadCloser) error 23 RestoreWithFilter(io.ReadCloser, *nomad.FSMFilter) error 24 } 25 26 type FSMHelper struct { 27 path string 28 29 logger hclog.Logger 30 31 // nomad state 32 store *raftboltdb.BoltStore 33 fsm nomadFSM 34 snaps *raft.FileSnapshotStore 35 36 // raft 37 logFirstIdx uint64 38 logLastIdx uint64 39 nextIdx uint64 40 } 41 42 func NewFSM(p string) (*FSMHelper, error) { 43 store, firstIdx, lastIdx, err := RaftStateInfo(filepath.Join(p, "raft.db")) 44 if err != nil { 45 return nil, fmt.Errorf("failed to open raft database %v: %v", p, err) 46 } 47 48 logger := hclog.L() 49 50 snaps, err := raft.NewFileSnapshotStoreWithLogger(p, 1000, logger) 51 if err != nil { 52 store.Close() 53 return nil, fmt.Errorf("failed to open snapshot dir: %v", err) 54 } 55 56 fsm, err := dummyFSM(logger) 57 if err != nil { 58 store.Close() 59 return nil, err 60 } 61 62 return &FSMHelper{ 63 path: p, 64 logger: logger, 65 store: store, 66 fsm: fsm, 67 snaps: snaps, 68 69 logFirstIdx: firstIdx, 70 logLastIdx: lastIdx, 71 nextIdx: uint64(1), 72 }, nil 73 } 74 75 func dummyFSM(logger hclog.Logger) (nomadFSM, error) { 76 // use dummy non-enabled FSM dependencies 77 periodicDispatch := nomad.NewPeriodicDispatch(logger, nil) 78 blockedEvals := nomad.NewBlockedEvals(nil, logger) 79 evalBroker, err := nomad.NewEvalBroker(1, 1, 1, 1) 80 if err != nil { 81 return nil, err 82 } 83 84 fsmConfig := &nomad.FSMConfig{ 85 EvalBroker: evalBroker, 86 Periodic: periodicDispatch, 87 Blocked: blockedEvals, 88 Logger: logger, 89 Region: "default", 90 } 91 92 return nomad.NewFSM(fsmConfig) 93 } 94 95 func (f *FSMHelper) Close() { 96 f.store.Close() 97 98 } 99 100 func (f *FSMHelper) ApplyNext() (index uint64, term uint64, err error) { 101 if f.nextIdx == 1 { 102 // check snapshots first 103 index, term, err := f.restoreFromSnapshot() 104 if err != nil { 105 return 0, 0, err 106 } 107 108 if index != 0 { 109 f.nextIdx = index + 1 110 return index, term, nil 111 } 112 } 113 114 if f.nextIdx < f.logFirstIdx { 115 return 0, 0, fmt.Errorf("missing logs [%v, %v]", f.nextIdx, f.logFirstIdx-1) 116 } 117 118 if f.nextIdx > f.logLastIdx { 119 return 0, 0, ErrNoMoreLogs 120 } 121 122 var e raft.Log 123 err = f.store.GetLog(f.nextIdx, &e) 124 if err != nil { 125 return 0, 0, fmt.Errorf("failed to read log entry at index %d: %v", f.nextIdx, err) 126 } 127 128 defer func() { 129 r := recover() 130 if r != nil && strings.HasPrefix(fmt.Sprint(r), "failed to apply request") { 131 // Enterprise specific log entries will fail to load in OSS repository with "failed to apply request." 132 // If not relevant to investigation, we can ignore them and simply worn. 133 f.logger.Warn("failed to apply log; loading Enterprise data-dir in OSS binary?", "index", e.Index) 134 135 f.nextIdx++ 136 } else if r != nil { 137 panic(r) 138 } 139 }() 140 141 if e.Type == raft.LogCommand { 142 f.fsm.Apply(&e) 143 } 144 145 f.nextIdx++ 146 return e.Index, e.Term, nil 147 } 148 149 // ApplyUntil applies all raft entries until (inclusive) the passed index. 150 func (f *FSMHelper) ApplyUntil(stopIdx uint64) (idx uint64, term uint64, err error) { 151 var lastIdx, lastTerm uint64 152 for { 153 idx, term, err := f.ApplyNext() 154 if err == ErrNoMoreLogs { 155 return lastIdx, lastTerm, nil 156 } else if err != nil { 157 return lastIdx, lastTerm, err 158 } else if idx >= stopIdx { 159 return lastIdx, lastTerm, nil 160 } 161 162 lastIdx, lastTerm = idx, term 163 } 164 } 165 166 func (f *FSMHelper) ApplyAll() (index uint64, term uint64, err error) { 167 var lastIdx, lastTerm uint64 168 for { 169 idx, term, err := f.ApplyNext() 170 if err == ErrNoMoreLogs { 171 return lastIdx, lastTerm, nil 172 } else if err != nil { 173 return lastIdx, lastTerm, err 174 } 175 176 lastIdx, lastTerm = idx, term 177 } 178 } 179 180 func (f *FSMHelper) State() *state.StateStore { 181 return f.fsm.State() 182 } 183 184 func (f *FSMHelper) StateAsMap() map[string][]interface{} { 185 return StateAsMap(f.fsm.State()) 186 } 187 188 // StateAsMap returns a json-able representation of the state 189 func StateAsMap(store *state.StateStore) map[string][]interface{} { 190 result := map[string][]interface{}{ 191 "ACLPolicies": toArray(store.ACLPolicies(nil)), 192 "ACLTokens": toArray(store.ACLTokens(nil, state.SortDefault)), 193 "Allocs": toArray(store.Allocs(nil, state.SortDefault)), 194 "CSIPlugins": toArray(store.CSIPlugins(nil)), 195 "CSIVolumes": toArray(store.CSIVolumes(nil)), 196 "Deployments": toArray(store.Deployments(nil, state.SortDefault)), 197 "Evals": toArray(store.Evals(nil, state.SortDefault)), 198 "Indexes": toArray(store.Indexes()), 199 "JobSummaries": toArray(store.JobSummaries(nil)), 200 "JobVersions": toArray(store.JobVersions(nil)), 201 "Jobs": toArray(store.Jobs(nil)), 202 "Nodes": toArray(store.Nodes(nil)), 203 "PeriodicLaunches": toArray(store.PeriodicLaunches(nil)), 204 "SITokenAccessors": toArray(store.SITokenAccessors(nil)), 205 "ScalingEvents": toArray(store.ScalingEvents(nil)), 206 "ScalingPolicies": toArray(store.ScalingPolicies(nil)), 207 "VaultAccessors": toArray(store.VaultAccessors(nil)), 208 } 209 210 insertEnterpriseState(result, store) 211 212 return result 213 214 } 215 216 func (f *FSMHelper) restoreFromSnapshot() (index uint64, term uint64, err error) { 217 snapshots, err := f.snaps.List() 218 if err != nil { 219 return 0, 0, err 220 } 221 f.logger.Debug("found snapshots", "count", len(snapshots)) 222 223 for _, snapshot := range snapshots { 224 _, source, err := f.snaps.Open(snapshot.ID) 225 if err != nil { 226 f.logger.Warn("failed to open a snapshot", "snapshot_id", snapshot.ID, "error", err) 227 continue 228 } 229 230 err = f.fsm.Restore(source) 231 source.Close() 232 if err != nil { 233 f.logger.Warn("failed to restore a snapshot", "snapshot_id", snapshot.ID, "error", err) 234 continue 235 } 236 237 return snapshot.Index, snapshot.Term, nil 238 } 239 240 return 0, 0, nil 241 } 242 243 func toArray(iter memdb.ResultIterator, err error) []interface{} { 244 if err != nil { 245 return []interface{}{err} 246 } 247 248 r := []interface{}{} 249 250 if iter != nil { 251 item := iter.Next() 252 for item != nil { 253 r = append(r, item) 254 item = iter.Next() 255 } 256 } 257 258 return r 259 }