github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/helper/raftutil/fsm.go (about)

     1  package raftutil
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"path/filepath"
     7  	"strings"
     8  
     9  	"github.com/hashicorp/go-hclog"
    10  	"github.com/hashicorp/go-memdb"
    11  	"github.com/hashicorp/nomad/nomad"
    12  	"github.com/hashicorp/nomad/nomad/state"
    13  	"github.com/hashicorp/raft"
    14  	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
    15  )
    16  
    17  var ErrNoMoreLogs = fmt.Errorf("no more logs")
    18  
    19  type nomadFSM interface {
    20  	raft.FSM
    21  	State() *state.StateStore
    22  	Restore(io.ReadCloser) error
    23  	RestoreWithFilter(io.ReadCloser, *nomad.FSMFilter) error
    24  }
    25  
    26  type FSMHelper struct {
    27  	path string
    28  
    29  	logger hclog.Logger
    30  
    31  	// nomad state
    32  	store *raftboltdb.BoltStore
    33  	fsm   nomadFSM
    34  	snaps *raft.FileSnapshotStore
    35  
    36  	// raft
    37  	logFirstIdx uint64
    38  	logLastIdx  uint64
    39  	nextIdx     uint64
    40  }
    41  
    42  func NewFSM(p string) (*FSMHelper, error) {
    43  	store, firstIdx, lastIdx, err := RaftStateInfo(filepath.Join(p, "raft.db"))
    44  	if err != nil {
    45  		return nil, fmt.Errorf("failed to open raft database %v: %v", p, err)
    46  	}
    47  
    48  	logger := hclog.L()
    49  
    50  	snaps, err := raft.NewFileSnapshotStoreWithLogger(p, 1000, logger)
    51  	if err != nil {
    52  		store.Close()
    53  		return nil, fmt.Errorf("failed to open snapshot dir: %v", err)
    54  	}
    55  
    56  	fsm, err := dummyFSM(logger)
    57  	if err != nil {
    58  		store.Close()
    59  		return nil, err
    60  	}
    61  
    62  	return &FSMHelper{
    63  		path:   p,
    64  		logger: logger,
    65  		store:  store,
    66  		fsm:    fsm,
    67  		snaps:  snaps,
    68  
    69  		logFirstIdx: firstIdx,
    70  		logLastIdx:  lastIdx,
    71  		nextIdx:     uint64(1),
    72  	}, nil
    73  }
    74  
    75  func dummyFSM(logger hclog.Logger) (nomadFSM, error) {
    76  	// use dummy non-enabled FSM dependencies
    77  	periodicDispatch := nomad.NewPeriodicDispatch(logger, nil)
    78  	blockedEvals := nomad.NewBlockedEvals(nil, logger)
    79  	evalBroker, err := nomad.NewEvalBroker(1, 1, 1, 1)
    80  	if err != nil {
    81  		return nil, err
    82  	}
    83  
    84  	fsmConfig := &nomad.FSMConfig{
    85  		EvalBroker: evalBroker,
    86  		Periodic:   periodicDispatch,
    87  		Blocked:    blockedEvals,
    88  		Logger:     logger,
    89  		Region:     "default",
    90  	}
    91  
    92  	return nomad.NewFSM(fsmConfig)
    93  }
    94  
    95  func (f *FSMHelper) Close() {
    96  	f.store.Close()
    97  
    98  }
    99  
   100  func (f *FSMHelper) ApplyNext() (index uint64, term uint64, err error) {
   101  	if f.nextIdx == 1 {
   102  		// check snapshots first
   103  		index, term, err := f.restoreFromSnapshot()
   104  		if err != nil {
   105  			return 0, 0, err
   106  		}
   107  
   108  		if index != 0 {
   109  			f.nextIdx = index + 1
   110  			return index, term, nil
   111  		}
   112  	}
   113  
   114  	if f.nextIdx < f.logFirstIdx {
   115  		return 0, 0, fmt.Errorf("missing logs [%v, %v]", f.nextIdx, f.logFirstIdx-1)
   116  	}
   117  
   118  	if f.nextIdx > f.logLastIdx {
   119  		return 0, 0, ErrNoMoreLogs
   120  	}
   121  
   122  	var e raft.Log
   123  	err = f.store.GetLog(f.nextIdx, &e)
   124  	if err != nil {
   125  		return 0, 0, fmt.Errorf("failed to read log entry at index %d: %v", f.nextIdx, err)
   126  	}
   127  
   128  	defer func() {
   129  		r := recover()
   130  		if r != nil && strings.HasPrefix(fmt.Sprint(r), "failed to apply request") {
   131  			// Enterprise specific log entries will fail to load in OSS repository with "failed to apply request."
   132  			// If not relevant to investigation, we can ignore them and simply worn.
   133  			f.logger.Warn("failed to apply log; loading Enterprise data-dir in OSS binary?", "index", e.Index)
   134  
   135  			f.nextIdx++
   136  		} else if r != nil {
   137  			panic(r)
   138  		}
   139  	}()
   140  
   141  	if e.Type == raft.LogCommand {
   142  		f.fsm.Apply(&e)
   143  	}
   144  
   145  	f.nextIdx++
   146  	return e.Index, e.Term, nil
   147  }
   148  
   149  // ApplyUntil applies all raft entries until (inclusive) the passed index.
   150  func (f *FSMHelper) ApplyUntil(stopIdx uint64) (idx uint64, term uint64, err error) {
   151  	var lastIdx, lastTerm uint64
   152  	for {
   153  		idx, term, err := f.ApplyNext()
   154  		if err == ErrNoMoreLogs {
   155  			return lastIdx, lastTerm, nil
   156  		} else if err != nil {
   157  			return lastIdx, lastTerm, err
   158  		} else if idx >= stopIdx {
   159  			return lastIdx, lastTerm, nil
   160  		}
   161  
   162  		lastIdx, lastTerm = idx, term
   163  	}
   164  }
   165  
   166  func (f *FSMHelper) ApplyAll() (index uint64, term uint64, err error) {
   167  	var lastIdx, lastTerm uint64
   168  	for {
   169  		idx, term, err := f.ApplyNext()
   170  		if err == ErrNoMoreLogs {
   171  			return lastIdx, lastTerm, nil
   172  		} else if err != nil {
   173  			return lastIdx, lastTerm, err
   174  		}
   175  
   176  		lastIdx, lastTerm = idx, term
   177  	}
   178  }
   179  
   180  func (f *FSMHelper) State() *state.StateStore {
   181  	return f.fsm.State()
   182  }
   183  
   184  func (f *FSMHelper) StateAsMap() map[string][]interface{} {
   185  	return StateAsMap(f.fsm.State())
   186  }
   187  
   188  // StateAsMap returns a json-able representation of the state
   189  func StateAsMap(store *state.StateStore) map[string][]interface{} {
   190  	result := map[string][]interface{}{
   191  		"ACLPolicies":      toArray(store.ACLPolicies(nil)),
   192  		"ACLTokens":        toArray(store.ACLTokens(nil, state.SortDefault)),
   193  		"Allocs":           toArray(store.Allocs(nil, state.SortDefault)),
   194  		"CSIPlugins":       toArray(store.CSIPlugins(nil)),
   195  		"CSIVolumes":       toArray(store.CSIVolumes(nil)),
   196  		"Deployments":      toArray(store.Deployments(nil, state.SortDefault)),
   197  		"Evals":            toArray(store.Evals(nil, state.SortDefault)),
   198  		"Indexes":          toArray(store.Indexes()),
   199  		"JobSummaries":     toArray(store.JobSummaries(nil)),
   200  		"JobVersions":      toArray(store.JobVersions(nil)),
   201  		"Jobs":             toArray(store.Jobs(nil)),
   202  		"Nodes":            toArray(store.Nodes(nil)),
   203  		"PeriodicLaunches": toArray(store.PeriodicLaunches(nil)),
   204  		"SITokenAccessors": toArray(store.SITokenAccessors(nil)),
   205  		"ScalingEvents":    toArray(store.ScalingEvents(nil)),
   206  		"ScalingPolicies":  toArray(store.ScalingPolicies(nil)),
   207  		"VaultAccessors":   toArray(store.VaultAccessors(nil)),
   208  	}
   209  
   210  	insertEnterpriseState(result, store)
   211  
   212  	return result
   213  
   214  }
   215  
   216  func (f *FSMHelper) restoreFromSnapshot() (index uint64, term uint64, err error) {
   217  	snapshots, err := f.snaps.List()
   218  	if err != nil {
   219  		return 0, 0, err
   220  	}
   221  	f.logger.Debug("found snapshots", "count", len(snapshots))
   222  
   223  	for _, snapshot := range snapshots {
   224  		_, source, err := f.snaps.Open(snapshot.ID)
   225  		if err != nil {
   226  			f.logger.Warn("failed to open a snapshot", "snapshot_id", snapshot.ID, "error", err)
   227  			continue
   228  		}
   229  
   230  		err = f.fsm.Restore(source)
   231  		source.Close()
   232  		if err != nil {
   233  			f.logger.Warn("failed to restore a snapshot", "snapshot_id", snapshot.ID, "error", err)
   234  			continue
   235  		}
   236  
   237  		return snapshot.Index, snapshot.Term, nil
   238  	}
   239  
   240  	return 0, 0, nil
   241  }
   242  
   243  func toArray(iter memdb.ResultIterator, err error) []interface{} {
   244  	if err != nil {
   245  		return []interface{}{err}
   246  	}
   247  
   248  	r := []interface{}{}
   249  
   250  	if iter != nil {
   251  		item := iter.Next()
   252  		for item != nil {
   253  			r = append(r, item)
   254  			item = iter.Next()
   255  		}
   256  	}
   257  
   258  	return r
   259  }