github.com/MetalBlockchain/metalgo@v1.11.9/utils/hashing/consistent/ring.go (about)

     1  // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
     2  // See the file LICENSE for licensing terms.
     3  
     4  package consistent
     5  
     6  import (
     7  	"errors"
     8  	"sync"
     9  
    10  	"github.com/google/btree"
    11  
    12  	"github.com/MetalBlockchain/metalgo/utils/hashing"
    13  )
    14  
    15  var (
    16  	_ Ring                     = (*hashRing)(nil)
    17  	_ btree.LessFunc[ringItem] = ringItem.Less
    18  
    19  	errEmptyRing = errors.New("ring doesn't have any members")
    20  )
    21  
    22  // Ring is an interface for a consistent hashing ring
    23  // (ref: https://en.wikipedia.org/wiki/Consistent_hashing).
    24  //
    25  // Consistent hashing is a method of distributing keys across an arbitrary set
    26  // of destinations.
    27  //
    28  // Consider a naive approach, which uses modulo to map a key to a node to serve
    29  // cached requests. Let N be the number of keys and M is the amount of possible
    30  // hashing destinations. Here, a key would be routed to a node based on
    31  // h(key) % M = node assigned.
    32  //
    33  // With this approach, we can route a key to a node in O(1) time, but this
    34  // results in cache misses as nodes are introduced and removed, which results in
    35  // the keys being reshuffled across nodes, as modulo's output changes with M.
    36  // This approach results in O(N) keys being shuffled, which is undesirable for a
    37  // caching use-case.
    38  //
    39  // Consistent hashing works by hashing all keys into a circle, which can be
    40  // visualized as a clock. Keys are routed to a node by hashing the key, and
    41  // searching for the first clockwise neighbor. This requires O(N) memory to
    42  // maintain the state of the ring and log(N) time to route a key using
    43  // binary-search, but results in O(N/M) amount of keys being shuffled when a
    44  // node is added/removed because the addition/removal of a node results in a
    45  // split/merge of its counter-clockwise neighbor's hash space.
    46  //
    47  // As an example, assume we have a ring that supports hashes from 1-12.
    48  //
    49  //	           12
    50  //	     11          1
    51  //
    52  //	 10                  2
    53  //
    54  //	9                      3
    55  //
    56  //	 8                   4
    57  //
    58  //	     7           5
    59  //	           6
    60  //
    61  // Add node 1 (n1). Let h(n1) = 12.
    62  // First, we compute the hash the node, and insert it into its corresponding
    63  // location on the ring.
    64  //
    65  //	           12 (n1)
    66  //	     11          1
    67  //
    68  //	 10                  2
    69  //
    70  //	9                      3
    71  //
    72  //	 8                   4
    73  //
    74  //	     7           5
    75  //	           6
    76  //
    77  // Now, to see which node a key (k1) should map to, we hash the key and search
    78  // for its closest clockwise neighbor.
    79  // Let h(k1) = 3. Here, we see that since n1 is the closest neighbor, as there
    80  // are no other nodes in the ring.
    81  //
    82  //	           12 (n1)
    83  //	     11          1
    84  //
    85  //	 10                  2
    86  //
    87  //	9                      3 (k1)
    88  //
    89  //	 8                   4
    90  //
    91  //	     7           5
    92  //	           6
    93  //
    94  // Now, let's insert another node (n2), such that h(n2) = 6.
    95  // Here we observe that k1 has shuffled to n2, as n2 is the closest clockwise
    96  // neighbor to k1.
    97  //
    98  //	           12 (n1)
    99  //	     11          1
   100  //
   101  //	 10                  2
   102  //
   103  //	9                      3 (k1)
   104  //
   105  //	 8                   4
   106  //
   107  //	     7           5
   108  //	           6 (n2)
   109  //
   110  // Other optimizations can be made to help reduce blast radius of failures and
   111  // the variance in keys (hot shards). One such optimization is introducing
   112  // virtual nodes, which is to replicate a single node multiple times by salting
   113  // it, (e.g n1-0, n1-1...).
   114  //
   115  // Without virtualization, failures of a node cascade as each node failing
   116  // results in the load of the failed node being shuffled into its clockwise
   117  // neighbor, which can result in a snowball effect across the network.
   118  type Ring interface {
   119  	RingReader
   120  	ringMutator
   121  }
   122  
   123  // RingReader is an interface to read values from Ring.
   124  type RingReader interface {
   125  	// Get gets the closest clockwise node for a key in the ring.
   126  	//
   127  	// Each ring member is responsible for the hashes which fall in the range
   128  	// between (myself, clockwise-neighbor].
   129  	// This behavior is desirable so that we can re-use the return value of Get
   130  	// to iterate around the ring (Ex. replication, retries, etc).
   131  	//
   132  	// Returns the node routed to and an error if we're unable to resolve a node
   133  	// to map to.
   134  	Get(Hashable) (Hashable, error)
   135  }
   136  
   137  // ringMutator defines an interface that mutates Ring.
   138  type ringMutator interface {
   139  	// Add adds a node to the ring.
   140  	Add(Hashable)
   141  
   142  	// Remove removes the node from the ring.
   143  	//
   144  	// Returns true if the node was removed, and false if it wasn't present to
   145  	// begin with.
   146  	Remove(Hashable) bool
   147  }
   148  
   149  // hashRing is an implementation of Ring
   150  type hashRing struct {
   151  	// Hashing algorithm to use when hashing keys.
   152  	hasher hashing.Hasher
   153  
   154  	// Replication factor for nodes; must be greater than zero.
   155  	virtualNodes int
   156  
   157  	lock sync.RWMutex
   158  	ring *btree.BTreeG[ringItem]
   159  }
   160  
   161  // RingConfig configures settings for a Ring.
   162  type RingConfig struct {
   163  	// Replication factor for nodes in the ring.
   164  	VirtualNodes int
   165  	// Hashing implementation to use.
   166  	Hasher hashing.Hasher
   167  	// Degree represents the degree of the b-tree
   168  	Degree int
   169  }
   170  
   171  // NewHashRing instantiates an instance of hashRing.
   172  func NewHashRing(config RingConfig) Ring {
   173  	return &hashRing{
   174  		hasher:       config.Hasher,
   175  		virtualNodes: config.VirtualNodes,
   176  		ring:         btree.NewG(config.Degree, ringItem.Less),
   177  	}
   178  }
   179  
   180  func (h *hashRing) Get(key Hashable) (Hashable, error) {
   181  	h.lock.RLock()
   182  	defer h.lock.RUnlock()
   183  
   184  	return h.get(key)
   185  }
   186  
   187  func (h *hashRing) get(key Hashable) (Hashable, error) {
   188  	// If we have no members in the ring, it's not possible to find where the
   189  	// key belongs.
   190  	if h.ring.Len() == 0 {
   191  		return nil, errEmptyRing
   192  	}
   193  
   194  	var (
   195  		// Compute this key's hash
   196  		hash   = h.hasher.Hash(key.ConsistentHashKey())
   197  		result Hashable
   198  	)
   199  	h.ring.AscendGreaterOrEqual(
   200  		ringItem{
   201  			hash:  hash,
   202  			value: key,
   203  		},
   204  		func(item ringItem) bool {
   205  			if hash < item.hash {
   206  				result = item.value
   207  				return false
   208  			}
   209  			return true
   210  		},
   211  	)
   212  
   213  	// If found nothing ascending the tree, we need to wrap around the ring to
   214  	// the left-most (min) node.
   215  	if result == nil {
   216  		min, _ := h.ring.Min()
   217  		result = min.value
   218  	}
   219  	return result, nil
   220  }
   221  
   222  func (h *hashRing) Add(key Hashable) {
   223  	h.lock.Lock()
   224  	defer h.lock.Unlock()
   225  
   226  	h.add(key)
   227  }
   228  
   229  func (h *hashRing) add(key Hashable) {
   230  	// Replicate the node in the ring.
   231  	hashKey := key.ConsistentHashKey()
   232  	for i := 0; i < h.virtualNodes; i++ {
   233  		virtualNode := getHashKey(hashKey, i)
   234  		virtualNodeHash := h.hasher.Hash(virtualNode)
   235  
   236  		// Insert it into the ring.
   237  		h.ring.ReplaceOrInsert(ringItem{
   238  			hash:  virtualNodeHash,
   239  			value: key,
   240  		})
   241  	}
   242  }
   243  
   244  func (h *hashRing) Remove(key Hashable) bool {
   245  	h.lock.Lock()
   246  	defer h.lock.Unlock()
   247  
   248  	return h.remove(key)
   249  }
   250  
   251  func (h *hashRing) remove(key Hashable) bool {
   252  	var (
   253  		hashKey = key.ConsistentHashKey()
   254  		removed = false
   255  	)
   256  
   257  	// We need to delete all virtual nodes created for a single node.
   258  	for i := 0; i < h.virtualNodes; i++ {
   259  		virtualNode := getHashKey(hashKey, i)
   260  		virtualNodeHash := h.hasher.Hash(virtualNode)
   261  		item := ringItem{
   262  			hash: virtualNodeHash,
   263  		}
   264  		_, removed = h.ring.Delete(item)
   265  	}
   266  	return removed
   267  }
   268  
   269  // getHashKey builds a key given a base key and a virtual node number.
   270  func getHashKey(key []byte, virtualNode int) []byte {
   271  	return append(key, byte(virtualNode))
   272  }
   273  
   274  // ringItem is a helper class to represent ring nodes in the b-tree.
   275  type ringItem struct {
   276  	hash  uint64
   277  	value Hashable
   278  }
   279  
   280  func (r ringItem) Less(than ringItem) bool {
   281  	return r.hash < than.hash
   282  }