github.com/pavelkrolevets/ton618@v1.9.26-0.20220108073458-82e0736ad23d/p2p/nodestate/nodestate.go (about)

     1  // Copyright 2020 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package nodestate
    18  
    19  import (
    20  	"errors"
    21  	"reflect"
    22  	"sync"
    23  	"time"
    24  	"unsafe"
    25  
    26  	"github.com/pavelkrolevets/ton618/common/mclock"
    27  	"github.com/pavelkrolevets/ton618/ethdb"
    28  	"github.com/pavelkrolevets/ton618/log"
    29  	"github.com/pavelkrolevets/ton618/metrics"
    30  	"github.com/pavelkrolevets/ton618/p2p/enode"
    31  	"github.com/pavelkrolevets/ton618/p2p/enr"
    32  	"github.com/pavelkrolevets/ton618/rlp"
    33  )
    34  
    35  var (
    36  	ErrInvalidField = errors.New("invalid field type")
    37  	ErrClosed       = errors.New("already closed")
    38  )
    39  
    40  type (
    41  	// NodeStateMachine implements a network node-related event subscription system.
    42  	// It can assign binary state flags and fields of arbitrary type to each node and allows
    43  	// subscriptions to flag/field changes which can also modify further flags and fields,
    44  	// potentially triggering further subscriptions. An operation includes an initial change
    45  	// and all resulting subsequent changes and always ends in a consistent global state.
    46  	// It is initiated by a "top level" SetState/SetField call that blocks (also blocking other
    47  	// top-level functions) until the operation is finished. Callbacks making further changes
    48  	// should use the non-blocking SetStateSub/SetFieldSub functions. The tree of events
    49  	// resulting from the initial changes is traversed in a breadth-first order, ensuring for
    50  	// each subscription callback that all other callbacks caused by the same change triggering
    51  	// the current callback are processed before anything is triggered by the changes made in the
    52  	// current callback. In practice this logic ensures that all subscriptions "see" events in
    53  	// the logical order, callbacks are never called concurrently and "back and forth" effects
    54  	// are also possible. The state machine design should ensure that infinite event cycles
    55  	// cannot happen.
    56  	// The caller can also add timeouts assigned to a certain node and a subset of state flags.
    57  	// If the timeout elapses, the flags are reset. If all relevant flags are reset then the timer
    58  	// is dropped. State flags with no timeout are persisted in the database if the flag
    59  	// descriptor enables saving. If a node has no state flags set at any moment then it is discarded.
    60  	// Note: in order to avoid mutex deadlocks the callbacks should never lock a mutex that
    61  	// might be locked when the top level SetState/SetField functions are called. If a function
    62  	// potentially performs state/field changes then it is recommended to mention this fact in the
    63  	// function description, along with whether it should run inside an operation callback.
    64  	NodeStateMachine struct {
    65  		started, closed     bool
    66  		lock                sync.Mutex
    67  		clock               mclock.Clock
    68  		db                  ethdb.KeyValueStore
    69  		dbNodeKey           []byte
    70  		nodes               map[enode.ID]*nodeInfo
    71  		offlineCallbackList []offlineCallback
    72  		opFlag              bool       // an operation has started
    73  		opWait              *sync.Cond // signaled when the operation ends
    74  		opPending           []func()   // pending callback list of the current operation
    75  
    76  		// Registered state flags or fields. Modifications are allowed
    77  		// only when the node state machine has not been started.
    78  		setup     *Setup
    79  		fields    []*fieldInfo
    80  		saveFlags bitMask
    81  
    82  		// Installed callbacks. Modifications are allowed only when the
    83  		// node state machine has not been started.
    84  		stateSubs []stateSub
    85  
    86  		// Testing hooks, only for testing purposes.
    87  		saveNodeHook func(*nodeInfo)
    88  	}
    89  
    90  	// Flags represents a set of flags from a certain setup
    91  	Flags struct {
    92  		mask  bitMask
    93  		setup *Setup
    94  	}
    95  
    96  	// Field represents a field from a certain setup
    97  	Field struct {
    98  		index int
    99  		setup *Setup
   100  	}
   101  
   102  	// flagDefinition describes a node state flag. Each registered instance is automatically
   103  	// mapped to a bit of the 64 bit node states.
   104  	// If persistent is true then the node is saved when state machine is shutdown.
   105  	flagDefinition struct {
   106  		name       string
   107  		persistent bool
   108  	}
   109  
   110  	// fieldDefinition describes an optional node field of the given type. The contents
   111  	// of the field are only retained for each node as long as at least one of the
   112  	// state flags is set.
   113  	fieldDefinition struct {
   114  		name   string
   115  		ftype  reflect.Type
   116  		encode func(interface{}) ([]byte, error)
   117  		decode func([]byte) (interface{}, error)
   118  	}
   119  
   120  	// stateSetup contains the list of flags and fields used by the application
   121  	Setup struct {
   122  		Version uint
   123  		flags   []flagDefinition
   124  		fields  []fieldDefinition
   125  	}
   126  
   127  	// bitMask describes a node state or state mask. It represents a subset
   128  	// of node flags with each bit assigned to a flag index (LSB represents flag 0).
   129  	bitMask uint64
   130  
   131  	// StateCallback is a subscription callback which is called when one of the
   132  	// state flags that is included in the subscription state mask is changed.
   133  	// Note: oldState and newState are also masked with the subscription mask so only
   134  	// the relevant bits are included.
   135  	StateCallback func(n *enode.Node, oldState, newState Flags)
   136  
   137  	// FieldCallback is a subscription callback which is called when the value of
   138  	// a specific field is changed.
   139  	FieldCallback func(n *enode.Node, state Flags, oldValue, newValue interface{})
   140  
   141  	// nodeInfo contains node state, fields and state timeouts
   142  	nodeInfo struct {
   143  		node       *enode.Node
   144  		state      bitMask
   145  		timeouts   []*nodeStateTimeout
   146  		fields     []interface{}
   147  		fieldCount int
   148  		db, dirty  bool
   149  	}
   150  
   151  	nodeInfoEnc struct {
   152  		Enr     enr.Record
   153  		Version uint
   154  		State   bitMask
   155  		Fields  [][]byte
   156  	}
   157  
   158  	stateSub struct {
   159  		mask     bitMask
   160  		callback StateCallback
   161  	}
   162  
   163  	nodeStateTimeout struct {
   164  		mask  bitMask
   165  		timer mclock.Timer
   166  	}
   167  
   168  	fieldInfo struct {
   169  		fieldDefinition
   170  		subs []FieldCallback
   171  	}
   172  
   173  	offlineCallback struct {
   174  		node   *nodeInfo
   175  		state  bitMask
   176  		fields []interface{}
   177  	}
   178  )
   179  
   180  // offlineState is a special state that is assumed to be set before a node is loaded from
   181  // the database and after it is shut down.
   182  const offlineState = bitMask(1)
   183  
   184  // NewFlag creates a new node state flag
   185  func (s *Setup) NewFlag(name string) Flags {
   186  	if s.flags == nil {
   187  		s.flags = []flagDefinition{{name: "offline"}}
   188  	}
   189  	f := Flags{mask: bitMask(1) << uint(len(s.flags)), setup: s}
   190  	s.flags = append(s.flags, flagDefinition{name: name})
   191  	return f
   192  }
   193  
   194  // NewPersistentFlag creates a new persistent node state flag
   195  func (s *Setup) NewPersistentFlag(name string) Flags {
   196  	if s.flags == nil {
   197  		s.flags = []flagDefinition{{name: "offline"}}
   198  	}
   199  	f := Flags{mask: bitMask(1) << uint(len(s.flags)), setup: s}
   200  	s.flags = append(s.flags, flagDefinition{name: name, persistent: true})
   201  	return f
   202  }
   203  
   204  // OfflineFlag returns the system-defined offline flag belonging to the given setup
   205  func (s *Setup) OfflineFlag() Flags {
   206  	return Flags{mask: offlineState, setup: s}
   207  }
   208  
   209  // NewField creates a new node state field
   210  func (s *Setup) NewField(name string, ftype reflect.Type) Field {
   211  	f := Field{index: len(s.fields), setup: s}
   212  	s.fields = append(s.fields, fieldDefinition{
   213  		name:  name,
   214  		ftype: ftype,
   215  	})
   216  	return f
   217  }
   218  
   219  // NewPersistentField creates a new persistent node field
   220  func (s *Setup) NewPersistentField(name string, ftype reflect.Type, encode func(interface{}) ([]byte, error), decode func([]byte) (interface{}, error)) Field {
   221  	f := Field{index: len(s.fields), setup: s}
   222  	s.fields = append(s.fields, fieldDefinition{
   223  		name:   name,
   224  		ftype:  ftype,
   225  		encode: encode,
   226  		decode: decode,
   227  	})
   228  	return f
   229  }
   230  
   231  // flagOp implements binary flag operations and also checks whether the operands belong to the same setup
   232  func flagOp(a, b Flags, trueIfA, trueIfB, trueIfBoth bool) Flags {
   233  	if a.setup == nil {
   234  		if a.mask != 0 {
   235  			panic("Node state flags have no setup reference")
   236  		}
   237  		a.setup = b.setup
   238  	}
   239  	if b.setup == nil {
   240  		if b.mask != 0 {
   241  			panic("Node state flags have no setup reference")
   242  		}
   243  		b.setup = a.setup
   244  	}
   245  	if a.setup != b.setup {
   246  		panic("Node state flags belong to a different setup")
   247  	}
   248  	res := Flags{setup: a.setup}
   249  	if trueIfA {
   250  		res.mask |= a.mask & ^b.mask
   251  	}
   252  	if trueIfB {
   253  		res.mask |= b.mask & ^a.mask
   254  	}
   255  	if trueIfBoth {
   256  		res.mask |= a.mask & b.mask
   257  	}
   258  	return res
   259  }
   260  
   261  // And returns the set of flags present in both a and b
   262  func (a Flags) And(b Flags) Flags { return flagOp(a, b, false, false, true) }
   263  
   264  // AndNot returns the set of flags present in a but not in b
   265  func (a Flags) AndNot(b Flags) Flags { return flagOp(a, b, true, false, false) }
   266  
   267  // Or returns the set of flags present in either a or b
   268  func (a Flags) Or(b Flags) Flags { return flagOp(a, b, true, true, true) }
   269  
   270  // Xor returns the set of flags present in either a or b but not both
   271  func (a Flags) Xor(b Flags) Flags { return flagOp(a, b, true, true, false) }
   272  
   273  // HasAll returns true if b is a subset of a
   274  func (a Flags) HasAll(b Flags) bool { return flagOp(a, b, false, true, false).mask == 0 }
   275  
   276  // HasNone returns true if a and b have no shared flags
   277  func (a Flags) HasNone(b Flags) bool { return flagOp(a, b, false, false, true).mask == 0 }
   278  
   279  // Equals returns true if a and b have the same flags set
   280  func (a Flags) Equals(b Flags) bool { return flagOp(a, b, true, true, false).mask == 0 }
   281  
   282  // IsEmpty returns true if a has no flags set
   283  func (a Flags) IsEmpty() bool { return a.mask == 0 }
   284  
   285  // MergeFlags merges multiple sets of state flags
   286  func MergeFlags(list ...Flags) Flags {
   287  	if len(list) == 0 {
   288  		return Flags{}
   289  	}
   290  	res := list[0]
   291  	for i := 1; i < len(list); i++ {
   292  		res = res.Or(list[i])
   293  	}
   294  	return res
   295  }
   296  
   297  // String returns a list of the names of the flags specified in the bit mask
   298  func (f Flags) String() string {
   299  	if f.mask == 0 {
   300  		return "[]"
   301  	}
   302  	s := "["
   303  	comma := false
   304  	for index, flag := range f.setup.flags {
   305  		if f.mask&(bitMask(1)<<uint(index)) != 0 {
   306  			if comma {
   307  				s = s + ", "
   308  			}
   309  			s = s + flag.name
   310  			comma = true
   311  		}
   312  	}
   313  	s = s + "]"
   314  	return s
   315  }
   316  
   317  // NewNodeStateMachine creates a new node state machine.
   318  // If db is not nil then the node states, fields and active timeouts are persisted.
   319  // Persistence can be enabled or disabled for each state flag and field.
   320  func NewNodeStateMachine(db ethdb.KeyValueStore, dbKey []byte, clock mclock.Clock, setup *Setup) *NodeStateMachine {
   321  	if setup.flags == nil {
   322  		panic("No state flags defined")
   323  	}
   324  	if len(setup.flags) > 8*int(unsafe.Sizeof(bitMask(0))) {
   325  		panic("Too many node state flags")
   326  	}
   327  	ns := &NodeStateMachine{
   328  		db:        db,
   329  		dbNodeKey: dbKey,
   330  		clock:     clock,
   331  		setup:     setup,
   332  		nodes:     make(map[enode.ID]*nodeInfo),
   333  		fields:    make([]*fieldInfo, len(setup.fields)),
   334  	}
   335  	ns.opWait = sync.NewCond(&ns.lock)
   336  	stateNameMap := make(map[string]int)
   337  	for index, flag := range setup.flags {
   338  		if _, ok := stateNameMap[flag.name]; ok {
   339  			panic("Node state flag name collision: " + flag.name)
   340  		}
   341  		stateNameMap[flag.name] = index
   342  		if flag.persistent {
   343  			ns.saveFlags |= bitMask(1) << uint(index)
   344  		}
   345  	}
   346  	fieldNameMap := make(map[string]int)
   347  	for index, field := range setup.fields {
   348  		if _, ok := fieldNameMap[field.name]; ok {
   349  			panic("Node field name collision: " + field.name)
   350  		}
   351  		ns.fields[index] = &fieldInfo{fieldDefinition: field}
   352  		fieldNameMap[field.name] = index
   353  	}
   354  	return ns
   355  }
   356  
   357  // stateMask checks whether the set of flags belongs to the same setup and returns its internal bit mask
   358  func (ns *NodeStateMachine) stateMask(flags Flags) bitMask {
   359  	if flags.setup != ns.setup && flags.mask != 0 {
   360  		panic("Node state flags belong to a different setup")
   361  	}
   362  	return flags.mask
   363  }
   364  
   365  // fieldIndex checks whether the field belongs to the same setup and returns its internal index
   366  func (ns *NodeStateMachine) fieldIndex(field Field) int {
   367  	if field.setup != ns.setup {
   368  		panic("Node field belongs to a different setup")
   369  	}
   370  	return field.index
   371  }
   372  
   373  // SubscribeState adds a node state subscription. The callback is called while the state
   374  // machine mutex is not held and it is allowed to make further state updates using the
   375  // non-blocking SetStateSub/SetFieldSub functions. All callbacks of an operation are running
   376  // from the thread/goroutine of the initial caller and parallel operations are not permitted.
   377  // Therefore the callback is never called concurrently. It is the responsibility of the
   378  // implemented state logic to avoid deadlocks and to reach a stable state in a finite amount
   379  // of steps.
   380  // State subscriptions should be installed before loading the node database or making the
   381  // first state update.
   382  func (ns *NodeStateMachine) SubscribeState(flags Flags, callback StateCallback) {
   383  	ns.lock.Lock()
   384  	defer ns.lock.Unlock()
   385  
   386  	if ns.started {
   387  		panic("state machine already started")
   388  	}
   389  	ns.stateSubs = append(ns.stateSubs, stateSub{ns.stateMask(flags), callback})
   390  }
   391  
   392  // SubscribeField adds a node field subscription. Same rules apply as for SubscribeState.
   393  func (ns *NodeStateMachine) SubscribeField(field Field, callback FieldCallback) {
   394  	ns.lock.Lock()
   395  	defer ns.lock.Unlock()
   396  
   397  	if ns.started {
   398  		panic("state machine already started")
   399  	}
   400  	f := ns.fields[ns.fieldIndex(field)]
   401  	f.subs = append(f.subs, callback)
   402  }
   403  
   404  // newNode creates a new nodeInfo
   405  func (ns *NodeStateMachine) newNode(n *enode.Node) *nodeInfo {
   406  	return &nodeInfo{node: n, fields: make([]interface{}, len(ns.fields))}
   407  }
   408  
   409  // checkStarted checks whether the state machine has already been started and panics otherwise.
   410  func (ns *NodeStateMachine) checkStarted() {
   411  	if !ns.started {
   412  		panic("state machine not started yet")
   413  	}
   414  }
   415  
   416  // Start starts the state machine, enabling state and field operations and disabling
   417  // further subscriptions.
   418  func (ns *NodeStateMachine) Start() {
   419  	ns.lock.Lock()
   420  	if ns.started {
   421  		panic("state machine already started")
   422  	}
   423  	ns.started = true
   424  	if ns.db != nil {
   425  		ns.loadFromDb()
   426  	}
   427  
   428  	ns.opStart()
   429  	ns.offlineCallbacks(true)
   430  	ns.opFinish()
   431  	ns.lock.Unlock()
   432  }
   433  
   434  // Stop stops the state machine and saves its state if a database was supplied
   435  func (ns *NodeStateMachine) Stop() {
   436  	ns.lock.Lock()
   437  	defer ns.lock.Unlock()
   438  
   439  	ns.checkStarted()
   440  	if !ns.opStart() {
   441  		panic("already closed")
   442  	}
   443  	for _, node := range ns.nodes {
   444  		fields := make([]interface{}, len(node.fields))
   445  		copy(fields, node.fields)
   446  		ns.offlineCallbackList = append(ns.offlineCallbackList, offlineCallback{node, node.state, fields})
   447  	}
   448  	if ns.db != nil {
   449  		ns.saveToDb()
   450  	}
   451  	ns.offlineCallbacks(false)
   452  	ns.closed = true
   453  	ns.opFinish()
   454  }
   455  
   456  // loadFromDb loads persisted node states from the database
   457  func (ns *NodeStateMachine) loadFromDb() {
   458  	it := ns.db.NewIterator(ns.dbNodeKey, nil)
   459  	for it.Next() {
   460  		var id enode.ID
   461  		if len(it.Key()) != len(ns.dbNodeKey)+len(id) {
   462  			log.Error("Node state db entry with invalid length", "found", len(it.Key()), "expected", len(ns.dbNodeKey)+len(id))
   463  			continue
   464  		}
   465  		copy(id[:], it.Key()[len(ns.dbNodeKey):])
   466  		ns.decodeNode(id, it.Value())
   467  	}
   468  }
   469  
   470  type dummyIdentity enode.ID
   471  
   472  func (id dummyIdentity) Verify(r *enr.Record, sig []byte) error { return nil }
   473  func (id dummyIdentity) NodeAddr(r *enr.Record) []byte          { return id[:] }
   474  
   475  // decodeNode decodes a node database entry and adds it to the node set if successful
   476  func (ns *NodeStateMachine) decodeNode(id enode.ID, data []byte) {
   477  	var enc nodeInfoEnc
   478  	if err := rlp.DecodeBytes(data, &enc); err != nil {
   479  		log.Error("Failed to decode node info", "id", id, "error", err)
   480  		return
   481  	}
   482  	n, _ := enode.New(dummyIdentity(id), &enc.Enr)
   483  	node := ns.newNode(n)
   484  	node.db = true
   485  
   486  	if enc.Version != ns.setup.Version {
   487  		log.Debug("Removing stored node with unknown version", "current", ns.setup.Version, "stored", enc.Version)
   488  		ns.deleteNode(id)
   489  		return
   490  	}
   491  	if len(enc.Fields) > len(ns.setup.fields) {
   492  		log.Error("Invalid node field count", "id", id, "stored", len(enc.Fields))
   493  		return
   494  	}
   495  	// Resolve persisted node fields
   496  	for i, encField := range enc.Fields {
   497  		if len(encField) == 0 {
   498  			continue
   499  		}
   500  		if decode := ns.fields[i].decode; decode != nil {
   501  			if field, err := decode(encField); err == nil {
   502  				node.fields[i] = field
   503  				node.fieldCount++
   504  			} else {
   505  				log.Error("Failed to decode node field", "id", id, "field name", ns.fields[i].name, "error", err)
   506  				return
   507  			}
   508  		} else {
   509  			log.Error("Cannot decode node field", "id", id, "field name", ns.fields[i].name)
   510  			return
   511  		}
   512  	}
   513  	// It's a compatible node record, add it to set.
   514  	ns.nodes[id] = node
   515  	node.state = enc.State
   516  	fields := make([]interface{}, len(node.fields))
   517  	copy(fields, node.fields)
   518  	ns.offlineCallbackList = append(ns.offlineCallbackList, offlineCallback{node, node.state, fields})
   519  	log.Debug("Loaded node state", "id", id, "state", Flags{mask: enc.State, setup: ns.setup})
   520  }
   521  
   522  // saveNode saves the given node info to the database
   523  func (ns *NodeStateMachine) saveNode(id enode.ID, node *nodeInfo) error {
   524  	if ns.db == nil {
   525  		return nil
   526  	}
   527  
   528  	storedState := node.state & ns.saveFlags
   529  	for _, t := range node.timeouts {
   530  		storedState &= ^t.mask
   531  	}
   532  	enc := nodeInfoEnc{
   533  		Enr:     *node.node.Record(),
   534  		Version: ns.setup.Version,
   535  		State:   storedState,
   536  		Fields:  make([][]byte, len(ns.fields)),
   537  	}
   538  	log.Debug("Saved node state", "id", id, "state", Flags{mask: enc.State, setup: ns.setup})
   539  	lastIndex := -1
   540  	for i, f := range node.fields {
   541  		if f == nil {
   542  			continue
   543  		}
   544  		encode := ns.fields[i].encode
   545  		if encode == nil {
   546  			continue
   547  		}
   548  		blob, err := encode(f)
   549  		if err != nil {
   550  			return err
   551  		}
   552  		enc.Fields[i] = blob
   553  		lastIndex = i
   554  	}
   555  	if storedState == 0 && lastIndex == -1 {
   556  		if node.db {
   557  			node.db = false
   558  			ns.deleteNode(id)
   559  		}
   560  		node.dirty = false
   561  		return nil
   562  	}
   563  	enc.Fields = enc.Fields[:lastIndex+1]
   564  	data, err := rlp.EncodeToBytes(&enc)
   565  	if err != nil {
   566  		return err
   567  	}
   568  	if err := ns.db.Put(append(ns.dbNodeKey, id[:]...), data); err != nil {
   569  		return err
   570  	}
   571  	node.dirty, node.db = false, true
   572  
   573  	if ns.saveNodeHook != nil {
   574  		ns.saveNodeHook(node)
   575  	}
   576  	return nil
   577  }
   578  
   579  // deleteNode removes a node info from the database
   580  func (ns *NodeStateMachine) deleteNode(id enode.ID) {
   581  	ns.db.Delete(append(ns.dbNodeKey, id[:]...))
   582  }
   583  
   584  // saveToDb saves the persistent flags and fields of all nodes that have been changed
   585  func (ns *NodeStateMachine) saveToDb() {
   586  	for id, node := range ns.nodes {
   587  		if node.dirty {
   588  			err := ns.saveNode(id, node)
   589  			if err != nil {
   590  				log.Error("Failed to save node", "id", id, "error", err)
   591  			}
   592  		}
   593  	}
   594  }
   595  
   596  // updateEnode updates the enode entry belonging to the given node if it already exists
   597  func (ns *NodeStateMachine) updateEnode(n *enode.Node) (enode.ID, *nodeInfo) {
   598  	id := n.ID()
   599  	node := ns.nodes[id]
   600  	if node != nil && n.Seq() > node.node.Seq() {
   601  		node.node = n
   602  	}
   603  	return id, node
   604  }
   605  
   606  // Persist saves the persistent state and fields of the given node immediately
   607  func (ns *NodeStateMachine) Persist(n *enode.Node) error {
   608  	ns.lock.Lock()
   609  	defer ns.lock.Unlock()
   610  
   611  	ns.checkStarted()
   612  	if id, node := ns.updateEnode(n); node != nil && node.dirty {
   613  		err := ns.saveNode(id, node)
   614  		if err != nil {
   615  			log.Error("Failed to save node", "id", id, "error", err)
   616  		}
   617  		return err
   618  	}
   619  	return nil
   620  }
   621  
   622  // SetState updates the given node state flags and blocks until the operation is finished.
   623  // If a flag with a timeout is set again, the operation removes or replaces the existing timeout.
   624  func (ns *NodeStateMachine) SetState(n *enode.Node, setFlags, resetFlags Flags, timeout time.Duration) error {
   625  	ns.lock.Lock()
   626  	defer ns.lock.Unlock()
   627  
   628  	if !ns.opStart() {
   629  		return ErrClosed
   630  	}
   631  	ns.setState(n, setFlags, resetFlags, timeout)
   632  	ns.opFinish()
   633  	return nil
   634  }
   635  
   636  // SetStateSub updates the given node state flags without blocking (should be called
   637  // from a subscription/operation callback).
   638  func (ns *NodeStateMachine) SetStateSub(n *enode.Node, setFlags, resetFlags Flags, timeout time.Duration) {
   639  	ns.lock.Lock()
   640  	defer ns.lock.Unlock()
   641  
   642  	ns.opCheck()
   643  	ns.setState(n, setFlags, resetFlags, timeout)
   644  }
   645  
   646  func (ns *NodeStateMachine) setState(n *enode.Node, setFlags, resetFlags Flags, timeout time.Duration) {
   647  	ns.checkStarted()
   648  	set, reset := ns.stateMask(setFlags), ns.stateMask(resetFlags)
   649  	id, node := ns.updateEnode(n)
   650  	if node == nil {
   651  		if set == 0 {
   652  			return
   653  		}
   654  		node = ns.newNode(n)
   655  		ns.nodes[id] = node
   656  	}
   657  	oldState := node.state
   658  	newState := (node.state & (^reset)) | set
   659  	changed := oldState ^ newState
   660  	node.state = newState
   661  
   662  	// Remove the timeout callbacks for all reset and set flags,
   663  	// even they are not existent(it's noop).
   664  	ns.removeTimeouts(node, set|reset)
   665  
   666  	// Register the timeout callback if required
   667  	if timeout != 0 && set != 0 {
   668  		ns.addTimeout(n, set, timeout)
   669  	}
   670  	if newState == oldState {
   671  		return
   672  	}
   673  	if newState == 0 && node.fieldCount == 0 {
   674  		delete(ns.nodes, id)
   675  		if node.db {
   676  			ns.deleteNode(id)
   677  		}
   678  	} else {
   679  		if changed&ns.saveFlags != 0 {
   680  			node.dirty = true
   681  		}
   682  	}
   683  	callback := func() {
   684  		for _, sub := range ns.stateSubs {
   685  			if changed&sub.mask != 0 {
   686  				sub.callback(n, Flags{mask: oldState & sub.mask, setup: ns.setup}, Flags{mask: newState & sub.mask, setup: ns.setup})
   687  			}
   688  		}
   689  	}
   690  	ns.opPending = append(ns.opPending, callback)
   691  }
   692  
   693  // opCheck checks whether an operation is active
   694  func (ns *NodeStateMachine) opCheck() {
   695  	if !ns.opFlag {
   696  		panic("Operation has not started")
   697  	}
   698  }
   699  
   700  // opStart waits until other operations are finished and starts a new one
   701  func (ns *NodeStateMachine) opStart() bool {
   702  	for ns.opFlag {
   703  		ns.opWait.Wait()
   704  	}
   705  	if ns.closed {
   706  		return false
   707  	}
   708  	ns.opFlag = true
   709  	return true
   710  }
   711  
   712  // opFinish finishes the current operation by running all pending callbacks.
   713  // Callbacks resulting from a state/field change performed in a previous callback are always
   714  // put at the end of the pending list and therefore processed after all callbacks resulting
   715  // from the previous state/field change.
   716  func (ns *NodeStateMachine) opFinish() {
   717  	for len(ns.opPending) != 0 {
   718  		list := ns.opPending
   719  		ns.lock.Unlock()
   720  		for _, cb := range list {
   721  			cb()
   722  		}
   723  		ns.lock.Lock()
   724  		ns.opPending = ns.opPending[len(list):]
   725  	}
   726  	ns.opPending = nil
   727  	ns.opFlag = false
   728  	ns.opWait.Broadcast()
   729  }
   730  
   731  // Operation calls the given function as an operation callback. This allows the caller
   732  // to start an operation with multiple initial changes. The same rules apply as for
   733  // subscription callbacks.
   734  func (ns *NodeStateMachine) Operation(fn func()) error {
   735  	ns.lock.Lock()
   736  	started := ns.opStart()
   737  	ns.lock.Unlock()
   738  	if !started {
   739  		return ErrClosed
   740  	}
   741  	fn()
   742  	ns.lock.Lock()
   743  	ns.opFinish()
   744  	ns.lock.Unlock()
   745  	return nil
   746  }
   747  
   748  // offlineCallbacks calls state update callbacks at startup or shutdown
   749  func (ns *NodeStateMachine) offlineCallbacks(start bool) {
   750  	for _, cb := range ns.offlineCallbackList {
   751  		cb := cb
   752  		callback := func() {
   753  			for _, sub := range ns.stateSubs {
   754  				offState := offlineState & sub.mask
   755  				onState := cb.state & sub.mask
   756  				if offState == onState {
   757  					continue
   758  				}
   759  				if start {
   760  					sub.callback(cb.node.node, Flags{mask: offState, setup: ns.setup}, Flags{mask: onState, setup: ns.setup})
   761  				} else {
   762  					sub.callback(cb.node.node, Flags{mask: onState, setup: ns.setup}, Flags{mask: offState, setup: ns.setup})
   763  				}
   764  			}
   765  			for i, f := range cb.fields {
   766  				if f == nil || ns.fields[i].subs == nil {
   767  					continue
   768  				}
   769  				for _, fsub := range ns.fields[i].subs {
   770  					if start {
   771  						fsub(cb.node.node, Flags{mask: offlineState, setup: ns.setup}, nil, f)
   772  					} else {
   773  						fsub(cb.node.node, Flags{mask: offlineState, setup: ns.setup}, f, nil)
   774  					}
   775  				}
   776  			}
   777  		}
   778  		ns.opPending = append(ns.opPending, callback)
   779  	}
   780  	ns.offlineCallbackList = nil
   781  }
   782  
   783  // AddTimeout adds a node state timeout associated to the given state flag(s).
   784  // After the specified time interval, the relevant states will be reset.
   785  func (ns *NodeStateMachine) AddTimeout(n *enode.Node, flags Flags, timeout time.Duration) error {
   786  	ns.lock.Lock()
   787  	defer ns.lock.Unlock()
   788  
   789  	ns.checkStarted()
   790  	if ns.closed {
   791  		return ErrClosed
   792  	}
   793  	ns.addTimeout(n, ns.stateMask(flags), timeout)
   794  	return nil
   795  }
   796  
   797  // addTimeout adds a node state timeout associated to the given state flag(s).
   798  func (ns *NodeStateMachine) addTimeout(n *enode.Node, mask bitMask, timeout time.Duration) {
   799  	_, node := ns.updateEnode(n)
   800  	if node == nil {
   801  		return
   802  	}
   803  	mask &= node.state
   804  	if mask == 0 {
   805  		return
   806  	}
   807  	ns.removeTimeouts(node, mask)
   808  	t := &nodeStateTimeout{mask: mask}
   809  	t.timer = ns.clock.AfterFunc(timeout, func() {
   810  		ns.SetState(n, Flags{}, Flags{mask: t.mask, setup: ns.setup}, 0)
   811  	})
   812  	node.timeouts = append(node.timeouts, t)
   813  	if mask&ns.saveFlags != 0 {
   814  		node.dirty = true
   815  	}
   816  }
   817  
   818  // removeTimeout removes node state timeouts associated to the given state flag(s).
   819  // If a timeout was associated to multiple flags which are not all included in the
   820  // specified remove mask then only the included flags are de-associated and the timer
   821  // stays active.
   822  func (ns *NodeStateMachine) removeTimeouts(node *nodeInfo, mask bitMask) {
   823  	for i := 0; i < len(node.timeouts); i++ {
   824  		t := node.timeouts[i]
   825  		match := t.mask & mask
   826  		if match == 0 {
   827  			continue
   828  		}
   829  		t.mask -= match
   830  		if t.mask != 0 {
   831  			continue
   832  		}
   833  		t.timer.Stop()
   834  		node.timeouts[i] = node.timeouts[len(node.timeouts)-1]
   835  		node.timeouts = node.timeouts[:len(node.timeouts)-1]
   836  		i--
   837  		if match&ns.saveFlags != 0 {
   838  			node.dirty = true
   839  		}
   840  	}
   841  }
   842  
   843  // GetField retrieves the given field of the given node. Note that when used in a
   844  // subscription callback the result can be out of sync with the state change represented
   845  // by the callback parameters so extra safety checks might be necessary.
   846  func (ns *NodeStateMachine) GetField(n *enode.Node, field Field) interface{} {
   847  	ns.lock.Lock()
   848  	defer ns.lock.Unlock()
   849  
   850  	ns.checkStarted()
   851  	if ns.closed {
   852  		return nil
   853  	}
   854  	if _, node := ns.updateEnode(n); node != nil {
   855  		return node.fields[ns.fieldIndex(field)]
   856  	}
   857  	return nil
   858  }
   859  
   860  // SetField sets the given field of the given node and blocks until the operation is finished
   861  func (ns *NodeStateMachine) SetField(n *enode.Node, field Field, value interface{}) error {
   862  	ns.lock.Lock()
   863  	defer ns.lock.Unlock()
   864  
   865  	if !ns.opStart() {
   866  		return ErrClosed
   867  	}
   868  	err := ns.setField(n, field, value)
   869  	ns.opFinish()
   870  	return err
   871  }
   872  
   873  // SetFieldSub sets the given field of the given node without blocking (should be called
   874  // from a subscription/operation callback).
   875  func (ns *NodeStateMachine) SetFieldSub(n *enode.Node, field Field, value interface{}) error {
   876  	ns.lock.Lock()
   877  	defer ns.lock.Unlock()
   878  
   879  	ns.opCheck()
   880  	return ns.setField(n, field, value)
   881  }
   882  
   883  func (ns *NodeStateMachine) setField(n *enode.Node, field Field, value interface{}) error {
   884  	ns.checkStarted()
   885  	id, node := ns.updateEnode(n)
   886  	if node == nil {
   887  		if value == nil {
   888  			return nil
   889  		}
   890  		node = ns.newNode(n)
   891  		ns.nodes[id] = node
   892  	}
   893  	fieldIndex := ns.fieldIndex(field)
   894  	f := ns.fields[fieldIndex]
   895  	if value != nil && reflect.TypeOf(value) != f.ftype {
   896  		log.Error("Invalid field type", "type", reflect.TypeOf(value), "required", f.ftype)
   897  		return ErrInvalidField
   898  	}
   899  	oldValue := node.fields[fieldIndex]
   900  	if value == oldValue {
   901  		return nil
   902  	}
   903  	if oldValue != nil {
   904  		node.fieldCount--
   905  	}
   906  	if value != nil {
   907  		node.fieldCount++
   908  	}
   909  	node.fields[fieldIndex] = value
   910  	if node.state == 0 && node.fieldCount == 0 {
   911  		delete(ns.nodes, id)
   912  		if node.db {
   913  			ns.deleteNode(id)
   914  		}
   915  	} else {
   916  		if f.encode != nil {
   917  			node.dirty = true
   918  		}
   919  	}
   920  	state := node.state
   921  	callback := func() {
   922  		for _, cb := range f.subs {
   923  			cb(n, Flags{mask: state, setup: ns.setup}, oldValue, value)
   924  		}
   925  	}
   926  	ns.opPending = append(ns.opPending, callback)
   927  	return nil
   928  }
   929  
   930  // ForEach calls the callback for each node having all of the required and none of the
   931  // disabled flags set.
   932  // Note that this callback is not an operation callback but ForEach can be called from an
   933  // Operation callback or Operation can also be called from a ForEach callback if necessary.
   934  func (ns *NodeStateMachine) ForEach(requireFlags, disableFlags Flags, cb func(n *enode.Node, state Flags)) {
   935  	ns.lock.Lock()
   936  	ns.checkStarted()
   937  	type callback struct {
   938  		node  *enode.Node
   939  		state bitMask
   940  	}
   941  	require, disable := ns.stateMask(requireFlags), ns.stateMask(disableFlags)
   942  	var callbacks []callback
   943  	for _, node := range ns.nodes {
   944  		if node.state&require == require && node.state&disable == 0 {
   945  			callbacks = append(callbacks, callback{node.node, node.state & (require | disable)})
   946  		}
   947  	}
   948  	ns.lock.Unlock()
   949  	for _, c := range callbacks {
   950  		cb(c.node, Flags{mask: c.state, setup: ns.setup})
   951  	}
   952  }
   953  
   954  // GetNode returns the enode currently associated with the given ID
   955  func (ns *NodeStateMachine) GetNode(id enode.ID) *enode.Node {
   956  	ns.lock.Lock()
   957  	defer ns.lock.Unlock()
   958  
   959  	ns.checkStarted()
   960  	if node := ns.nodes[id]; node != nil {
   961  		return node.node
   962  	}
   963  	return nil
   964  }
   965  
   966  // AddLogMetrics adds logging and/or metrics for nodes entering, exiting and currently
   967  // being in a given set specified by required and disabled state flags
   968  func (ns *NodeStateMachine) AddLogMetrics(requireFlags, disableFlags Flags, name string, inMeter, outMeter metrics.Meter, gauge metrics.Gauge) {
   969  	var count int64
   970  	ns.SubscribeState(requireFlags.Or(disableFlags), func(n *enode.Node, oldState, newState Flags) {
   971  		oldMatch := oldState.HasAll(requireFlags) && oldState.HasNone(disableFlags)
   972  		newMatch := newState.HasAll(requireFlags) && newState.HasNone(disableFlags)
   973  		if newMatch == oldMatch {
   974  			return
   975  		}
   976  
   977  		if newMatch {
   978  			count++
   979  			if name != "" {
   980  				log.Debug("Node entered", "set", name, "id", n.ID(), "count", count)
   981  			}
   982  			if inMeter != nil {
   983  				inMeter.Mark(1)
   984  			}
   985  		} else {
   986  			count--
   987  			if name != "" {
   988  				log.Debug("Node left", "set", name, "id", n.ID(), "count", count)
   989  			}
   990  			if outMeter != nil {
   991  				outMeter.Mark(1)
   992  			}
   993  		}
   994  		if gauge != nil {
   995  			gauge.Update(count)
   996  		}
   997  	})
   998  }