github.com/MetalBlockchain/metalgo@v1.11.9/utils/hashing/consistent/ring.go (about) 1 // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. 2 // See the file LICENSE for licensing terms. 3 4 package consistent 5 6 import ( 7 "errors" 8 "sync" 9 10 "github.com/google/btree" 11 12 "github.com/MetalBlockchain/metalgo/utils/hashing" 13 ) 14 15 var ( 16 _ Ring = (*hashRing)(nil) 17 _ btree.LessFunc[ringItem] = ringItem.Less 18 19 errEmptyRing = errors.New("ring doesn't have any members") 20 ) 21 22 // Ring is an interface for a consistent hashing ring 23 // (ref: https://en.wikipedia.org/wiki/Consistent_hashing). 24 // 25 // Consistent hashing is a method of distributing keys across an arbitrary set 26 // of destinations. 27 // 28 // Consider a naive approach, which uses modulo to map a key to a node to serve 29 // cached requests. Let N be the number of keys and M is the amount of possible 30 // hashing destinations. Here, a key would be routed to a node based on 31 // h(key) % M = node assigned. 32 // 33 // With this approach, we can route a key to a node in O(1) time, but this 34 // results in cache misses as nodes are introduced and removed, which results in 35 // the keys being reshuffled across nodes, as modulo's output changes with M. 36 // This approach results in O(N) keys being shuffled, which is undesirable for a 37 // caching use-case. 38 // 39 // Consistent hashing works by hashing all keys into a circle, which can be 40 // visualized as a clock. Keys are routed to a node by hashing the key, and 41 // searching for the first clockwise neighbor. This requires O(N) memory to 42 // maintain the state of the ring and log(N) time to route a key using 43 // binary-search, but results in O(N/M) amount of keys being shuffled when a 44 // node is added/removed because the addition/removal of a node results in a 45 // split/merge of its counter-clockwise neighbor's hash space. 46 // 47 // As an example, assume we have a ring that supports hashes from 1-12. 48 // 49 // 12 50 // 11 1 51 // 52 // 10 2 53 // 54 // 9 3 55 // 56 // 8 4 57 // 58 // 7 5 59 // 6 60 // 61 // Add node 1 (n1). Let h(n1) = 12. 62 // First, we compute the hash the node, and insert it into its corresponding 63 // location on the ring. 64 // 65 // 12 (n1) 66 // 11 1 67 // 68 // 10 2 69 // 70 // 9 3 71 // 72 // 8 4 73 // 74 // 7 5 75 // 6 76 // 77 // Now, to see which node a key (k1) should map to, we hash the key and search 78 // for its closest clockwise neighbor. 79 // Let h(k1) = 3. Here, we see that since n1 is the closest neighbor, as there 80 // are no other nodes in the ring. 81 // 82 // 12 (n1) 83 // 11 1 84 // 85 // 10 2 86 // 87 // 9 3 (k1) 88 // 89 // 8 4 90 // 91 // 7 5 92 // 6 93 // 94 // Now, let's insert another node (n2), such that h(n2) = 6. 95 // Here we observe that k1 has shuffled to n2, as n2 is the closest clockwise 96 // neighbor to k1. 97 // 98 // 12 (n1) 99 // 11 1 100 // 101 // 10 2 102 // 103 // 9 3 (k1) 104 // 105 // 8 4 106 // 107 // 7 5 108 // 6 (n2) 109 // 110 // Other optimizations can be made to help reduce blast radius of failures and 111 // the variance in keys (hot shards). One such optimization is introducing 112 // virtual nodes, which is to replicate a single node multiple times by salting 113 // it, (e.g n1-0, n1-1...). 114 // 115 // Without virtualization, failures of a node cascade as each node failing 116 // results in the load of the failed node being shuffled into its clockwise 117 // neighbor, which can result in a snowball effect across the network. 118 type Ring interface { 119 RingReader 120 ringMutator 121 } 122 123 // RingReader is an interface to read values from Ring. 124 type RingReader interface { 125 // Get gets the closest clockwise node for a key in the ring. 126 // 127 // Each ring member is responsible for the hashes which fall in the range 128 // between (myself, clockwise-neighbor]. 129 // This behavior is desirable so that we can re-use the return value of Get 130 // to iterate around the ring (Ex. replication, retries, etc). 131 // 132 // Returns the node routed to and an error if we're unable to resolve a node 133 // to map to. 134 Get(Hashable) (Hashable, error) 135 } 136 137 // ringMutator defines an interface that mutates Ring. 138 type ringMutator interface { 139 // Add adds a node to the ring. 140 Add(Hashable) 141 142 // Remove removes the node from the ring. 143 // 144 // Returns true if the node was removed, and false if it wasn't present to 145 // begin with. 146 Remove(Hashable) bool 147 } 148 149 // hashRing is an implementation of Ring 150 type hashRing struct { 151 // Hashing algorithm to use when hashing keys. 152 hasher hashing.Hasher 153 154 // Replication factor for nodes; must be greater than zero. 155 virtualNodes int 156 157 lock sync.RWMutex 158 ring *btree.BTreeG[ringItem] 159 } 160 161 // RingConfig configures settings for a Ring. 162 type RingConfig struct { 163 // Replication factor for nodes in the ring. 164 VirtualNodes int 165 // Hashing implementation to use. 166 Hasher hashing.Hasher 167 // Degree represents the degree of the b-tree 168 Degree int 169 } 170 171 // NewHashRing instantiates an instance of hashRing. 172 func NewHashRing(config RingConfig) Ring { 173 return &hashRing{ 174 hasher: config.Hasher, 175 virtualNodes: config.VirtualNodes, 176 ring: btree.NewG(config.Degree, ringItem.Less), 177 } 178 } 179 180 func (h *hashRing) Get(key Hashable) (Hashable, error) { 181 h.lock.RLock() 182 defer h.lock.RUnlock() 183 184 return h.get(key) 185 } 186 187 func (h *hashRing) get(key Hashable) (Hashable, error) { 188 // If we have no members in the ring, it's not possible to find where the 189 // key belongs. 190 if h.ring.Len() == 0 { 191 return nil, errEmptyRing 192 } 193 194 var ( 195 // Compute this key's hash 196 hash = h.hasher.Hash(key.ConsistentHashKey()) 197 result Hashable 198 ) 199 h.ring.AscendGreaterOrEqual( 200 ringItem{ 201 hash: hash, 202 value: key, 203 }, 204 func(item ringItem) bool { 205 if hash < item.hash { 206 result = item.value 207 return false 208 } 209 return true 210 }, 211 ) 212 213 // If found nothing ascending the tree, we need to wrap around the ring to 214 // the left-most (min) node. 215 if result == nil { 216 min, _ := h.ring.Min() 217 result = min.value 218 } 219 return result, nil 220 } 221 222 func (h *hashRing) Add(key Hashable) { 223 h.lock.Lock() 224 defer h.lock.Unlock() 225 226 h.add(key) 227 } 228 229 func (h *hashRing) add(key Hashable) { 230 // Replicate the node in the ring. 231 hashKey := key.ConsistentHashKey() 232 for i := 0; i < h.virtualNodes; i++ { 233 virtualNode := getHashKey(hashKey, i) 234 virtualNodeHash := h.hasher.Hash(virtualNode) 235 236 // Insert it into the ring. 237 h.ring.ReplaceOrInsert(ringItem{ 238 hash: virtualNodeHash, 239 value: key, 240 }) 241 } 242 } 243 244 func (h *hashRing) Remove(key Hashable) bool { 245 h.lock.Lock() 246 defer h.lock.Unlock() 247 248 return h.remove(key) 249 } 250 251 func (h *hashRing) remove(key Hashable) bool { 252 var ( 253 hashKey = key.ConsistentHashKey() 254 removed = false 255 ) 256 257 // We need to delete all virtual nodes created for a single node. 258 for i := 0; i < h.virtualNodes; i++ { 259 virtualNode := getHashKey(hashKey, i) 260 virtualNodeHash := h.hasher.Hash(virtualNode) 261 item := ringItem{ 262 hash: virtualNodeHash, 263 } 264 _, removed = h.ring.Delete(item) 265 } 266 return removed 267 } 268 269 // getHashKey builds a key given a base key and a virtual node number. 270 func getHashKey(key []byte, virtualNode int) []byte { 271 return append(key, byte(virtualNode)) 272 } 273 274 // ringItem is a helper class to represent ring nodes in the b-tree. 275 type ringItem struct { 276 hash uint64 277 value Hashable 278 } 279 280 func (r ringItem) Less(than ringItem) bool { 281 return r.hash < than.hash 282 }