github.com/aristanetworks/gomap@v0.0.0-20240103001659-f6b0e31fb1a7/map.go (about)

     1  // Modifications copyright (c) Arista Networks, Inc. 2022
     2  // Underlying
     3  // Copyright 2014 The Go Authors. All rights reserved.
     4  // Use of this source code is governed by a BSD-style
     5  // license that can be found in the LICENSE file.
     6  
     7  // Pacakge gomap provides the Map type, which implements a hash table.
     8  // It's implementation is heavily inspired by Go's built-in map, with
     9  // the additional requirement that users provide a equal and hash
    10  // function.
    11  //
    12  // The following requirements are the user's responsibility to follow:
    13  //   - equal(a, b) => hash(a) == hash(b)
    14  //   - equal(a, a) must be true for all values of a. Be careful around NaN
    15  //     float values. Go's built-in `map` has special cases for handling
    16  //     this, but `Map` does not.
    17  //   - If a key in a `Map` contains references -- such as pointers, maps,
    18  //     or slices -- modifying the referefenced data in a way that effects
    19  //     the result of the equal or hash functions will result in undefined
    20  //     behavior.
    21  //   - For good performance hash functions should return uniformly
    22  //     distributed data across the entire 64-bits of the value.
    23  package gomap
    24  
    25  // This file contains a reimplementation of Go's map type using type
    26  // parameters. See
    27  // https://github.com/golang/go/blob/master/src/runtime/map.go
    28  //
    29  // A map is just a hash table. The data is arranged
    30  // into an array of buckets. Each bucket contains up to
    31  // 8 key/elem pairs. The low-order bits of the hash are
    32  // used to select a bucket. Each bucket contains a few
    33  // high-order bits of each hash to distinguish the entries
    34  // within a single bucket.
    35  //
    36  // If more than 8 keys hash to a bucket, we chain on
    37  // extra buckets.
    38  //
    39  // When the hashtable grows, we allocate a new array
    40  // of buckets twice as big. Buckets are incrementally
    41  // copied from the old bucket array to the new bucket array.
    42  //
    43  // Map iterators walk through the array of buckets and
    44  // return the keys in walk order (bucket #, then overflow
    45  // chain order, then bucket index).  To maintain iteration
    46  // semantics, we never move keys within their bucket (if
    47  // we did, keys might be returned 0 or 2 times).  When
    48  // growing the table, iterators remain iterating through the
    49  // old table and must check the new table if the bucket
    50  // they are iterating through has been moved ("evacuated")
    51  // to the new table.
    52  
    53  // Picking loadFactor: too large and we have lots of overflow
    54  // buckets, too small and we waste a lot of space. I wrote
    55  // a simple program to check some stats for different loads:
    56  // (64-bit, 8 byte keys and elems)
    57  //  loadFactor    %overflow  bytes/entry     hitprobe    missprobe
    58  //        4.00         2.13        20.77         3.00         4.00
    59  //        4.50         4.05        17.30         3.25         4.50
    60  //        5.00         6.85        14.77         3.50         5.00
    61  //        5.50        10.55        12.94         3.75         5.50
    62  //        6.00        15.27        11.67         4.00         6.00
    63  //        6.50        20.90        10.79         4.25         6.50
    64  //        7.00        27.14        10.15         4.50         7.00
    65  //        7.50        34.03         9.73         4.75         7.50
    66  //        8.00        41.10         9.40         5.00         8.00
    67  //
    68  // %overflow   = percentage of buckets which have an overflow bucket
    69  // bytes/entry = overhead bytes used per key/elem pair
    70  // hitprobe    = # of entries to check when looking up a present key
    71  // missprobe   = # of entries to check when looking up an absent key
    72  //
    73  // Keep in mind this data is for maximally loaded tables, i.e. just
    74  // before the table grows. Typical tables will be somewhat less loaded.
    75  
    76  import (
    77  	"fmt"
    78  	"hash/maphash"
    79  	"sync/atomic"
    80  )
    81  
    82  const (
    83  	// Maximum number of key/elem pairs a bucket can hold.
    84  	bucketCntBits = 3
    85  	bucketCnt     = 1 << bucketCntBits
    86  
    87  	// Maximum average load of a bucket that triggers growth is 6.5.
    88  	// Represent as loadFactorNum/loadFactorDen, to allow integer math.
    89  	loadFactorNum = 13
    90  	loadFactorDen = 2
    91  
    92  	// Possible tophash values. We reserve a few possibilities for special marks.
    93  	// Each bucket (including its overflow buckets, if any) will have either all or none of its
    94  	// entries in the evacuated* states (except during the evacuate() method, which only happens
    95  	// during map writes and thus no one else can observe the map during that time).
    96  
    97  	// this cell is empty, and there are no more non-empty cells at higher indexes or overflows.
    98  	emptyRest = 0
    99  	// this cell is empty
   100  	emptyOne = 1
   101  	// key/elem is valid.  Entry has been evacuated to first half of larger table.
   102  	evacuatedX = 2
   103  	// same as above, but evacuated to second half of larger table.
   104  	evacuatedY = 3
   105  	// cell is empty, bucket is evacuated.
   106  	evacuatedEmpty = 4
   107  	// minimum tophash for a normal filled cell.
   108  	minTopHash = 5
   109  
   110  	// flags
   111  	iterator     = 1 // there may be an iterator using buckets
   112  	oldIterator  = 2 // there may be an iterator using oldbuckets
   113  	hashWriting  = 4 // a goroutine is writing to the map
   114  	sameSizeGrow = 8 // the current map growth is to a new map of the same size
   115  
   116  	// sentinel bucket ID for iterator checks
   117  	noCheck = -1
   118  )
   119  
   120  // isEmpty reports whether the given tophash array entry represents an empty bucket entry.
   121  func isEmpty(x uint8) bool {
   122  	return x <= emptyOne
   123  }
   124  
   125  // Map implements a hashmap
   126  type Map[K, E any] struct {
   127  	count int // # live cells == size of map
   128  	// Only the first 8 bits are used. uint32 is used here to allow
   129  	// use of atomic.*Uint32 operations
   130  	flags     uint32
   131  	noverflow uint32 // number of overflow buckets; see incrnoverflow for details
   132  
   133  	// array of buckets. may be nil if count==0.
   134  	// Pre-allocated overflow buckets exist as indexes [len(buckets), cap(buckets)-1]
   135  	buckets []bucket[K, E]
   136  	// nextoverflow is an index into buckets[:cap(buckets)]. It is the
   137  	// next unused overflow bucket.
   138  	nextoverflow int
   139  
   140  	oldbuckets *[]bucket[K, E] // previous bucket array of half the size, non-nil only when growing
   141  
   142  	nevacuate int // progress counter for evacuation (buckets less than this have been evacuated)
   143  	seed      maphash.Seed
   144  
   145  	hash  func(maphash.Seed, K) uint64
   146  	equal func(K, K) bool
   147  }
   148  
   149  type bucket[K, E any] struct {
   150  	// tophash generally contains the top byte of the hash value
   151  	// for each key in this bucket. If tophash[0] < minTopHash,
   152  	// tophash[0] is a bucket evacuation state instead.
   153  	tophash [bucketCnt]uint8
   154  	// Followed by bucketCnt keys and then bucketCnt elems.
   155  	// NOTE: packing all the keys together and then all the elems together makes the
   156  	// code a bit more complicated than alternating key/elem/key/elem/... but it allows
   157  	// us to eliminate padding which would be needed for, e.g., map[int64]int8.
   158  	keys  [bucketCnt]K
   159  	elems [bucketCnt]E
   160  	// Followed by an overflow pointer.
   161  	overflow *bucket[K, E]
   162  }
   163  
   164  // Iterator is instantiated by a call Iter(). It allows iterating over
   165  // a Map.
   166  type Iterator[K, E any] struct {
   167  	key         K
   168  	elem        E
   169  	m           *Map[K, E]
   170  	buckets     []bucket[K, E]
   171  	bptr        *bucket[K, E]
   172  	startBucket int
   173  	offset      uint8
   174  	wrapped     bool
   175  	i           uint8
   176  	bucket      int
   177  	checkBucket int
   178  }
   179  
   180  // Key returns the key at the iterator's current position. This is
   181  // only valid after a call to Next() that returns true.
   182  func (it *Iterator[K, E]) Key() K {
   183  	return it.key
   184  }
   185  
   186  // Elem returns the element at the iterator's current position. This
   187  // is only valid after a call to Next() that returns true.
   188  func (it *Iterator[K, E]) Elem() E {
   189  	return it.elem
   190  }
   191  
   192  // tophash calculates the tophash value for hash.
   193  func tophash(hash uint64) uint8 {
   194  	top := uint8(hash >> 56)
   195  	if top < minTopHash {
   196  		top += minTopHash
   197  	}
   198  	return top
   199  }
   200  
   201  func evacuated[K, E any](b *bucket[K, E]) bool {
   202  	h := b.tophash[0]
   203  	return h > emptyOne && h < minTopHash
   204  }
   205  
   206  func (m *Map[K, E]) newoverflow(b *bucket[K, E]) *bucket[K, E] {
   207  	if m.nextoverflow < cap(m.buckets) {
   208  		// We have preallocated overflow buckets available.
   209  		// See makeBucketArray for more details.
   210  		b.overflow = &m.buckets[:cap(m.buckets)][m.nextoverflow]
   211  		m.nextoverflow++
   212  	} else {
   213  		b.overflow = &bucket[K, E]{}
   214  	}
   215  	m.noverflow++
   216  	return b.overflow
   217  }
   218  
   219  // KeyElem contains a Key and Elem.
   220  type KeyElem[K, E any] struct {
   221  	Key  K
   222  	Elem E
   223  }
   224  
   225  // New instantiates a new Map initialized with any KeyElems passed.
   226  // The equal func must return true for two values of K that are equal
   227  // and false otherwise. The hash func should return a uniformly
   228  // distributed hash value. If equal(a, b) then hash(a) == hash(b). The
   229  // hash function is passed a [hash/maphash.Seed], this is meant to be
   230  // used with functions and types in the [hash/maphash] package, though
   231  // can be ignored.
   232  func New[K, E any](
   233  	equal func(a, b K) bool,
   234  	hash func(maphash.Seed, K) uint64,
   235  	kes ...KeyElem[K, E]) *Map[K, E] {
   236  
   237  	if len(kes) == 0 {
   238  		return NewHint[K, E](0, equal, hash)
   239  	}
   240  	m := NewHint[K, E](len(kes), equal, hash)
   241  	for _, ke := range kes {
   242  		m.Set(ke.Key, ke.Elem)
   243  	}
   244  	return m
   245  }
   246  
   247  // NewHint instantiates a new Map with a hint as to how many elements
   248  // will be inserted. See [New] for discussion of the equal and hash
   249  // arguments.
   250  func NewHint[K, E any](
   251  	hint int,
   252  	equal func(a, b K) bool,
   253  	hash func(maphash.Seed, K) uint64) *Map[K, E] {
   254  
   255  	if hint <= 0 {
   256  		return &Map[K, E]{seed: maphash.MakeSeed(), hash: hash, equal: equal}
   257  	}
   258  	nbuckets := 1
   259  	for overLoadFactor(hint, nbuckets) {
   260  		nbuckets *= 2
   261  	}
   262  	buckets := makeBucketArray[K, E](nbuckets)
   263  
   264  	return &Map[K, E]{seed: maphash.MakeSeed(), buckets: buckets, nextoverflow: len(buckets),
   265  		hash: hash, equal: equal}
   266  }
   267  
   268  func makeBucketArray[K, E any](nbuckets int) []bucket[K, E] {
   269  	if nbuckets&(nbuckets-1) != 0 {
   270  		panic("nbuckets is not power of 2")
   271  	}
   272  	var newbuckets []bucket[K, E]
   273  	// Preallocate expected overflow buckets at the end of the buckets
   274  	// slice
   275  	additional := nbuckets >> 4
   276  	if additional == 0 {
   277  		newbuckets = make([]bucket[K, E], nbuckets)
   278  	} else {
   279  		// Using append here allows the go runtime to round up the
   280  		// capacity of newbuckets to fit the next size class, giving
   281  		// us some free buckets we don't need to allocate later.
   282  		newbuckets = append([]bucket[K, E](nil),
   283  			make([]bucket[K, E], nbuckets+additional)...)
   284  		newbuckets = newbuckets[:nbuckets]
   285  	}
   286  	return newbuckets
   287  }
   288  
   289  // Len returns the count of occupied elements in m.
   290  func (m *Map[K, E]) Len() int {
   291  	if m == nil {
   292  		return 0
   293  	}
   294  	return m.count
   295  }
   296  
   297  // String converts m to a string. Keys and Elements are stringified
   298  // using fmt.Sprint. Use [String] for better control over stringifying
   299  // m's contents.
   300  func (m *Map[K, E]) String() string {
   301  	return StringFunc(m,
   302  		func(key K) string { return fmt.Sprint(key) },
   303  		func(elem E) string { return fmt.Sprint(elem) },
   304  	)
   305  }
   306  
   307  // Get returns the element associated with key and true if that key is
   308  // in the Map, otherwise it returns the zero value of E and false.
   309  func (m *Map[K, E]) Get(key K) (E, bool) {
   310  	var zeroE E
   311  	_, e := m.mapaccessK(key)
   312  	if e == nil {
   313  		return zeroE, false
   314  	}
   315  	return *e, true
   316  }
   317  
   318  // returns both key and elem. Used by map iterator
   319  func (m *Map[K, E]) mapaccessK(key K) (*K, *E) {
   320  	if m == nil || m.count == 0 {
   321  		return nil, nil
   322  	}
   323  	// This check is disabled when the race detector is running
   324  	// becuase it flags this non-atomic read of m.flags, which
   325  	// can be concurrently updated by Map.Iter.
   326  	if !raceEnabled && m.flags&hashWriting != 0 {
   327  		panic("concurrent map read and map write")
   328  	}
   329  	hash := m.hash(m.seed, key)
   330  	mask := m.bucketMask()
   331  	b := &m.buckets[int(hash&mask)]
   332  	if c := m.oldbuckets; c != nil {
   333  		if !m.sameSizeGrow() {
   334  			// There used to be half as many buckets; mask down one more power of two.
   335  			mask >>= 1
   336  		}
   337  		oldb := &(*c)[int(hash&mask)]
   338  		if !evacuated(oldb) {
   339  			b = oldb
   340  		}
   341  	}
   342  	top := tophash(hash)
   343  bucketloop:
   344  	for ; b != nil; b = b.overflow {
   345  		for i := uintptr(0); i < bucketCnt; i++ {
   346  			if b.tophash[i] != top {
   347  				if b.tophash[i] == emptyRest {
   348  					break bucketloop
   349  				}
   350  				continue
   351  			}
   352  			if m.equal(key, b.keys[i]) {
   353  				return &b.keys[i], &b.elems[i]
   354  			}
   355  		}
   356  	}
   357  	return nil, nil
   358  }
   359  
   360  // Set associates key with elem in m.
   361  func (m *Map[K, E]) Set(key K, elem E) {
   362  	if m == nil {
   363  		// We have to panic here rather than initialize an empty map
   364  		// because we need the user to pass in hash and equal
   365  		// functions
   366  		panic("Set called on nil map")
   367  	}
   368  	if m.flags&hashWriting != 0 {
   369  		panic("concurrent map writes")
   370  	}
   371  	hash := m.hash(m.seed, key)
   372  	// Set hashWriting after calling t.hash, since t.hash may panic,
   373  	// in which case we have not actually done a write.
   374  	m.flags ^= hashWriting
   375  
   376  	if m.buckets == nil {
   377  		m.buckets = make([]bucket[K, E], 1)
   378  		m.nextoverflow = len(m.buckets)
   379  	}
   380  
   381  again:
   382  	mask := m.bucketMask()
   383  	bucket := hash & mask
   384  	if m.growing() {
   385  		m.growWork(int(bucket))
   386  	}
   387  	b := &m.buckets[hash&mask]
   388  	top := tophash(hash)
   389  
   390  	var inserti *uint8
   391  	var insertk *K
   392  	var inserte *E
   393  bucketloop:
   394  	for {
   395  		for i := uintptr(0); i < bucketCnt; i++ {
   396  			if b.tophash[i] != top {
   397  				if isEmpty(b.tophash[i]) && inserti == nil {
   398  					inserti = &b.tophash[i]
   399  					insertk = &b.keys[i]
   400  					inserte = &b.elems[i]
   401  				}
   402  				if b.tophash[i] == emptyRest {
   403  					break bucketloop
   404  				}
   405  				continue
   406  			}
   407  			k := b.keys[i]
   408  			if !m.equal(key, k) {
   409  				continue
   410  			}
   411  			// already have a mapping for key. Update it.
   412  			b.keys[i] = key
   413  			b.elems[i] = elem
   414  			goto done
   415  		}
   416  		ovf := b.overflow
   417  		if ovf == nil {
   418  			break
   419  		}
   420  		b = ovf
   421  	}
   422  
   423  	// Did not find mapping for key. Allocate new cell & add entry.
   424  
   425  	// If we hit the max load factor or we have too many overflow buckets,
   426  	// and we're not already in the middle of growing, start growing.
   427  	if !m.growing() && (overLoadFactor(m.count+1, len(m.buckets)) ||
   428  		tooManyOverflowBuckets(m.noverflow, len(m.buckets))) {
   429  		m.hashGrow()
   430  		goto again // Growing the table invalidates everything, so try again
   431  	}
   432  
   433  	if inserti == nil {
   434  		// The current bucket and all the overflow buckets connected
   435  		// to it are full, allocate a new one.
   436  		newb := m.newoverflow(b)
   437  		inserti = &newb.tophash[0]
   438  		insertk = &newb.keys[0]
   439  		inserte = &newb.elems[0]
   440  	}
   441  
   442  	// store new key/elem at insert position
   443  	*insertk = key
   444  	*inserte = elem
   445  	*inserti = top
   446  	m.count++
   447  
   448  done:
   449  	if m.flags&hashWriting == 0 {
   450  		panic("concurrent map writes")
   451  	}
   452  	m.flags &^= hashWriting
   453  }
   454  
   455  // Update calls fn with the elem associated with key, or the zero
   456  // value of E if key is not present, the value returned by fn will be
   457  // set in the map.
   458  //
   459  // Update is equivalent to:
   460  //
   461  //	elem, _ := m.Get(key)
   462  //	m.Set(fn(elem))
   463  func (m *Map[K, E]) Update(key K, fn func(elem E) E) {
   464  	if m == nil {
   465  		// We have to panic here rather than initialize an empty map
   466  		// because we need the user to pass in hash and equal
   467  		// functions
   468  		panic("Set called on nil map")
   469  	}
   470  	if m.flags&hashWriting != 0 {
   471  		panic("concurrent map writes")
   472  	}
   473  	hash := m.hash(m.seed, key)
   474  	// Set hashWriting after calling t.hash, since t.hash may panic,
   475  	// in which case we have not actually done a write.
   476  	m.flags ^= hashWriting
   477  
   478  	if m.buckets == nil {
   479  		m.buckets = make([]bucket[K, E], 1)
   480  		m.nextoverflow = len(m.buckets)
   481  	}
   482  
   483  	var zeroE E
   484  
   485  again:
   486  	mask := m.bucketMask()
   487  	bucket := hash & mask
   488  	if m.growing() {
   489  		m.growWork(int(bucket))
   490  	}
   491  	b := &m.buckets[hash&mask]
   492  	top := tophash(hash)
   493  
   494  	var inserti *uint8
   495  	var insertk *K
   496  	var inserte *E
   497  bucketloop:
   498  	for {
   499  		for i := uintptr(0); i < bucketCnt; i++ {
   500  			if b.tophash[i] != top {
   501  				if isEmpty(b.tophash[i]) && inserti == nil {
   502  					inserti = &b.tophash[i]
   503  					insertk = &b.keys[i]
   504  					inserte = &b.elems[i]
   505  				}
   506  				if b.tophash[i] == emptyRest {
   507  					break bucketloop
   508  				}
   509  				continue
   510  			}
   511  			k := b.keys[i]
   512  			if !m.equal(key, k) {
   513  				continue
   514  			}
   515  			// already have a mapping for key. Update it.
   516  			b.keys[i] = key
   517  			b.elems[i] = fn(b.elems[i])
   518  			goto done
   519  		}
   520  		ovf := b.overflow
   521  		if ovf == nil {
   522  			break
   523  		}
   524  		b = ovf
   525  	}
   526  
   527  	// Did not find mapping for key. Allocate new cell & add entry.
   528  
   529  	// If we hit the max load factor or we have too many overflow buckets,
   530  	// and we're not already in the middle of growing, start growing.
   531  	if !m.growing() && (overLoadFactor(m.count+1, len(m.buckets)) ||
   532  		tooManyOverflowBuckets(m.noverflow, len(m.buckets))) {
   533  		m.hashGrow()
   534  		goto again // Growing the table invalidates everything, so try again
   535  	}
   536  
   537  	if inserti == nil {
   538  		// The current bucket and all the overflow buckets connected
   539  		// to it are full, allocate a new one.
   540  		newb := m.newoverflow(b)
   541  		inserti = &newb.tophash[0]
   542  		insertk = &newb.keys[0]
   543  		inserte = &newb.elems[0]
   544  	}
   545  
   546  	// store new key/elem at insert position
   547  	*insertk = key
   548  	*inserte = fn(zeroE)
   549  	*inserti = top
   550  	m.count++
   551  
   552  done:
   553  	if m.flags&hashWriting == 0 {
   554  		panic("concurrent map writes")
   555  	}
   556  	m.flags &^= hashWriting
   557  }
   558  
   559  // Delete removes key and it's associated value from the map.
   560  func (m *Map[K, E]) Delete(key K) {
   561  	if m == nil || m.count == 0 {
   562  		return
   563  	}
   564  	if m.flags&hashWriting != 0 {
   565  		panic("concurrent map writes")
   566  	}
   567  
   568  	hash := m.hash(m.seed, key)
   569  
   570  	// Set hashWriting after calling t.hash, since t.hash may panic,
   571  	// in which case we have not actually done a write (delete).
   572  	m.flags ^= hashWriting
   573  
   574  	bucket := hash & m.bucketMask()
   575  	if m.growing() {
   576  		m.growWork(int(bucket))
   577  	}
   578  	b := &m.buckets[bucket]
   579  	bOrig := b
   580  	top := tophash(hash)
   581  search:
   582  	for ; b != nil; b = b.overflow {
   583  		for i := uintptr(0); i < bucketCnt; i++ {
   584  			if b.tophash[i] != top {
   585  				if b.tophash[i] == emptyRest {
   586  					break search
   587  				}
   588  				continue
   589  			}
   590  			k := b.keys[i]
   591  			if !m.equal(key, k) {
   592  				continue
   593  			}
   594  			var (
   595  				zeroK K
   596  				zeroE E
   597  			)
   598  			// Clear key and elem in case they have pointers
   599  			b.keys[i] = zeroK
   600  			b.elems[i] = zeroE
   601  			b.tophash[i] = emptyOne
   602  			// If the bucket now ends in a bunch of emptyOne states,
   603  			// change those to emptyRest states.
   604  			// It would be nice to make this a separate function, but
   605  			// for loops are not currently inlineable.
   606  			if i == bucketCnt-1 {
   607  				if b.overflow != nil && b.overflow.tophash[0] != emptyRest {
   608  					goto notLast
   609  				}
   610  			} else {
   611  				if b.tophash[i+1] != emptyRest {
   612  					goto notLast
   613  				}
   614  			}
   615  			for {
   616  				b.tophash[i] = emptyRest
   617  				if i == 0 {
   618  					if b == bOrig {
   619  						break // beginning of initial bucket, we're done.
   620  					}
   621  					// Find previous bucket, continue at its last entry.
   622  					c := b
   623  					for b = bOrig; b.overflow != c; b = b.overflow {
   624  					}
   625  					i = bucketCnt - 1
   626  				} else {
   627  					i--
   628  				}
   629  				if b.tophash[i] != emptyOne {
   630  					break
   631  				}
   632  			}
   633  		notLast:
   634  			m.count--
   635  			// Reset the hash seed to make it more difficult for attackers to
   636  			// repeatedly trigger hash collisions. See issue 25237.
   637  			if m.count == 0 {
   638  				m.seed = maphash.MakeSeed()
   639  			}
   640  			break search
   641  		}
   642  
   643  	}
   644  
   645  	if m.flags&hashWriting == 0 {
   646  		panic("concurrent map writes")
   647  	}
   648  	m.flags &^= hashWriting
   649  }
   650  
   651  // Iter instantiates an Iterator to explore the elements of the Map.
   652  // Ordering is undefined and is intentionally randomized.
   653  func (m *Map[K, E]) Iter() *Iterator[K, E] {
   654  	// Iter() is a small function to encourage the compiler to inline
   655  	// it into its caller and let `it` be kept on the stack.
   656  	var it Iterator[K, E]
   657  	m.iter(&it)
   658  	return &it
   659  }
   660  
   661  func (m *Map[K, E]) iter(it *Iterator[K, E]) {
   662  	if m == nil || m.count == 0 {
   663  		return
   664  	}
   665  	r := rand64()
   666  	it.m = m
   667  	it.buckets = m.buckets
   668  	it.startBucket = int(r & m.bucketMask())
   669  	it.bucket = it.startBucket
   670  	it.offset = uint8(r >> (64 - bucketCntBits))
   671  
   672  	// Remember we have an iterator.
   673  	// Can run concurrently with another m.Iter().
   674  	atomicOr(&m.flags, iterator|oldIterator)
   675  	return
   676  }
   677  
   678  func atomicOr(flags *uint32, or uint32) {
   679  	old := atomic.LoadUint32(flags)
   680  	for !atomic.CompareAndSwapUint32(flags, old, old|or) {
   681  		// force re-reading from memory
   682  		old = atomic.LoadUint32(flags)
   683  	}
   684  }
   685  
   686  // Next moves the iterator to the next element. Next returns false
   687  // when the iterator is complete.
   688  func (it *Iterator[K, E]) Next() bool {
   689  	m := it.m
   690  	if m == nil {
   691  		return false
   692  	}
   693  	// This check is disabled when the race detector is running
   694  	// becuase it flags this non-atomic read of m.flags, which
   695  	// can be concurrently updated by Map.Iter.
   696  	if !raceEnabled && m.flags&hashWriting != 0 {
   697  		panic("concurrent map iteration and map write")
   698  	}
   699  	bucket := it.bucket
   700  	b := it.bptr
   701  	i := it.i
   702  	checkBucket := it.checkBucket
   703  
   704  next:
   705  	if b == nil {
   706  		if bucket == it.startBucket && it.wrapped {
   707  			// end of iteration
   708  			var (
   709  				zeroK K
   710  				zeroE E
   711  			)
   712  			it.key = zeroK
   713  			it.elem = zeroE
   714  			return false
   715  		}
   716  		if m.growing() && len(it.buckets) == len(m.buckets) {
   717  			// Iterator was started in the middle of a grow, and the grow isn't done yet.
   718  			// If the bucket we're looking at hasn't been filled in yet (i.e. the old
   719  			// bucket hasn't been evacuated) then we need to iterate through the old
   720  			// bucket and only return the ones that will be migrated to this bucket.
   721  			oldbucket := uint64(bucket) & it.m.oldbucketmask()
   722  			b = &(*m.oldbuckets)[oldbucket]
   723  			if !evacuated(b) {
   724  				checkBucket = bucket
   725  			} else {
   726  				b = &it.buckets[bucket]
   727  				checkBucket = noCheck
   728  			}
   729  		} else {
   730  			b = &it.buckets[bucket]
   731  			checkBucket = noCheck
   732  		}
   733  		bucket++
   734  		if bucket == len(it.buckets) {
   735  			bucket = 0
   736  			it.wrapped = true
   737  		}
   738  		i = 0
   739  	}
   740  	for ; i < bucketCnt; i++ {
   741  		offi := (i + it.offset) & (bucketCnt - 1)
   742  		if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty {
   743  			// TODO: emptyRest is hard to use here, as we start iterating
   744  			// in the middle of a bucket. It's feasible, just tricky.
   745  			continue
   746  		}
   747  		k := b.keys[offi]
   748  		if checkBucket != noCheck && !m.sameSizeGrow() {
   749  			// Special case: iterator was started during a grow to a larger size
   750  			// and the grow is not done yet. We're working on a bucket whose
   751  			// oldbucket has not been evacuated yet. Or at least, it wasn't
   752  			// evacuated when we started the bucket. So we're iterating
   753  			// through the oldbucket, skipping any keys that will go
   754  			// to the other new bucket (each oldbucket expands to two
   755  			// buckets during a grow).
   756  			// If the item in the oldbucket is not destined for
   757  			// the current new bucket in the iteration, skip it.
   758  			hash := m.hash(m.seed, k)
   759  			if int(hash&m.bucketMask()) != checkBucket {
   760  				continue
   761  			}
   762  		}
   763  		if b.tophash[offi] != evacuatedX && b.tophash[offi] != evacuatedY {
   764  			// This is the golden data, we can return it.
   765  			it.key = k
   766  			it.elem = b.elems[offi]
   767  		} else {
   768  			// The hash table has grown since the iterator was started.
   769  			// The golden data for this key is now somewhere else.
   770  			// Check the current hash table for the data.
   771  			// This code handles the case where the key
   772  			// has been deleted, updated, or deleted and reinserted.
   773  			// NOTE: we need to regrab the key as it has potentially been
   774  			// updated to an equal() but not identical key (e.g. +0.0 vs -0.0).
   775  			rk, re := m.mapaccessK(k)
   776  			if rk == nil {
   777  				continue // key has been deleted
   778  			}
   779  			it.key = *rk
   780  			it.elem = *re
   781  		}
   782  		it.bucket = bucket
   783  		if it.bptr != b { // avoid unnecessary write barrier; see issue 14921
   784  			it.bptr = b
   785  		}
   786  		it.i = i + 1
   787  		it.checkBucket = checkBucket
   788  		return true
   789  	}
   790  	b = b.overflow
   791  	i = 0
   792  	goto next
   793  }
   794  
   795  // Clear deletes all keys from m.
   796  func (m *Map[K, E]) Clear() {
   797  	if m == nil || m.count == 0 {
   798  		return
   799  	}
   800  	if m.flags&hashWriting != 0 {
   801  		panic("concurrent map writes")
   802  	}
   803  	m.flags ^= hashWriting
   804  
   805  	m.flags &^= sameSizeGrow
   806  	m.oldbuckets = nil
   807  	m.nevacuate = 0
   808  	m.noverflow = 0
   809  	m.count = 0
   810  
   811  	m.seed = maphash.MakeSeed()
   812  
   813  	// zero out all buckets including used preallocated overflow buckets
   814  	buckets := m.buckets[:m.nextoverflow]
   815  	for i := range buckets {
   816  		buckets[i] = bucket[K, E]{}
   817  	}
   818  
   819  	if m.flags&hashWriting == 0 {
   820  		panic("concurrent map writes")
   821  	}
   822  	m.flags &^= hashWriting
   823  }
   824  
   825  func (m *Map[K, E]) hashGrow() {
   826  	// If we've hit the load factor, get bigger.
   827  	// Otherwise, there are too many overflow buckets,
   828  	// so keep the same number of buckets and "grow" laterally.
   829  	newsize := len(m.buckets) * 2
   830  	if !overLoadFactor(m.count+1, len(m.buckets)) {
   831  		newsize = len(m.buckets)
   832  		m.flags |= sameSizeGrow
   833  	}
   834  	oldbuckets := m.buckets
   835  	newbuckets := makeBucketArray[K, E](newsize)
   836  
   837  	flags := m.flags &^ (iterator | oldIterator)
   838  	if m.flags&iterator != 0 {
   839  		flags |= oldIterator
   840  	}
   841  	// commit the grow
   842  	m.flags = flags
   843  	m.oldbuckets = &oldbuckets
   844  	m.buckets = newbuckets
   845  	m.nextoverflow = len(m.buckets)
   846  	m.nevacuate = 0
   847  	m.noverflow = 0
   848  
   849  	// the actual copying of the hash table data is done incrementally
   850  	// by growWork() and evacuate().
   851  }
   852  
   853  // overLoadFactor reports whether count items placed in 1<<B buckets is over loadFactor.
   854  func overLoadFactor(count int, nbuckets int) bool {
   855  	return count > bucketCnt && uint64(count) > loadFactorNum*(uint64(nbuckets)/loadFactorDen)
   856  }
   857  
   858  // tooManyOverflowBuckets reports whether noverflow buckets is too many for a map with 1<<B buckets.
   859  // Note that most of these overflow buckets must be in sparse use;
   860  // if use was dense, then we'd have already triggered regular map growth.
   861  func tooManyOverflowBuckets(noverflow uint32, nbuckets int) bool {
   862  	// If the threshold is too low, we do extraneous work.
   863  	// If the threshold is too high, maps that grow and shrink can hold on to lots of unused memory.
   864  	// "too many" means (approximately) as many overflow buckets as regular buckets.
   865  	// See incrnoverflow for more details.
   866  	// The compiler doesn't see here that B < 16; mask B to generate shorter shift code.
   867  	return noverflow >= uint32(nbuckets)
   868  }
   869  
   870  // growing reports whether h is growing. The growth may be to the same size or bigger.
   871  func (m *Map[K, E]) growing() bool {
   872  	return m.oldbuckets != nil
   873  }
   874  
   875  // sameSizeGrow reports whether the current growth is to a map of the same size.
   876  func (m *Map[K, E]) sameSizeGrow() bool {
   877  	return m.flags&sameSizeGrow != 0
   878  }
   879  
   880  func (m *Map[K, E]) bucketMask() uint64 {
   881  	return uint64(len(m.buckets) - 1)
   882  }
   883  
   884  // oldbucketmask provides a mask that can be applied to calculate n % noldbuckets().
   885  func (m *Map[K, E]) oldbucketmask() uint64 {
   886  	return uint64(len(*m.oldbuckets) - 1)
   887  }
   888  
   889  func (m *Map[K, E]) growWork(bucket int) {
   890  	// make sure we evacuate the oldbucket corresponding
   891  	// to the bucket we're about to use
   892  	m.evacuate(int(uint64(bucket) & m.oldbucketmask()))
   893  
   894  	// evacuate one more oldbucket to make progress on growing
   895  	if m.growing() {
   896  		m.evacuate(m.nevacuate)
   897  	}
   898  }
   899  
   900  func (m *Map[K, E]) bucketEvacuated(bucket uint64) bool {
   901  	return evacuated(&(*m.oldbuckets)[bucket])
   902  }
   903  
   904  // evacDst is an evacuation destination.
   905  type evacDst[K, E any] struct {
   906  	b *bucket[K, E] // current destination bucket
   907  	i int           // key/elem index into b
   908  }
   909  
   910  func (m *Map[K, E]) evacuate(oldbucket int) {
   911  	b := &(*m.oldbuckets)[oldbucket]
   912  	newbit := len(*m.oldbuckets)
   913  	if !evacuated(b) {
   914  		// TODO: reuse overflow buckets instead of using new ones, if there
   915  		// is no iterator using the old buckets.  (If !oldIterator.)
   916  
   917  		// xy contains the x and y (low and high) evacuation destinations.
   918  		var xy [2]evacDst[K, E]
   919  		x := &xy[0]
   920  		x.b = &m.buckets[oldbucket]
   921  
   922  		if !m.sameSizeGrow() {
   923  			// Only calculate y pointers if we're growing bigger.
   924  			// Otherwise GC can see bad pointers.
   925  			y := &xy[1]
   926  			y.b = &m.buckets[oldbucket+newbit]
   927  		}
   928  
   929  		for ; b != nil; b = b.overflow {
   930  			for i := 0; i < bucketCnt; i++ {
   931  				top := b.tophash[i]
   932  				if isEmpty(top) {
   933  					b.tophash[i] = evacuatedEmpty
   934  					continue
   935  				}
   936  				if top < minTopHash {
   937  					panic("bad map state")
   938  				}
   939  				var useY uint8
   940  				if !m.sameSizeGrow() {
   941  					// Compute hash to make our evacuation decision (whether we need
   942  					// to send this key/elem to bucket x or bucket y).
   943  					hash := m.hash(m.seed, b.keys[i])
   944  					if hash&uint64(newbit) != 0 {
   945  						useY = 1
   946  					}
   947  				}
   948  
   949  				if evacuatedX+1 != evacuatedY || evacuatedX^1 != evacuatedY {
   950  					panic("bad evacuatedN")
   951  				}
   952  
   953  				b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY
   954  				dst := &xy[useY]                 // evacuation destination
   955  
   956  				if dst.i == bucketCnt {
   957  					dst.b = m.newoverflow(dst.b)
   958  					dst.i = 0
   959  				}
   960  				// mask dst.i as an optimization, to avoid a bounds check
   961  				dst.b.tophash[dst.i&(bucketCnt-1)] = top
   962  				dst.b.keys[dst.i&(bucketCnt-1)] = b.keys[i]
   963  				dst.b.elems[dst.i&(bucketCnt-1)] = b.elems[i]
   964  				dst.i++
   965  			}
   966  		}
   967  		// Unlink the overflow buckets & clear key/elem to help GC.
   968  		if m.flags&oldIterator == 0 {
   969  			b := &(*m.oldbuckets)[oldbucket]
   970  			// Preserve b.tophash because the evacuation
   971  			// state is maintained there.
   972  			b.keys = [bucketCnt]K{}
   973  			b.elems = [bucketCnt]E{}
   974  			b.overflow = nil
   975  		}
   976  	}
   977  
   978  	if oldbucket == m.nevacuate {
   979  		m.advanceEvacuationMark(newbit)
   980  	}
   981  }
   982  
   983  func (m *Map[K, E]) advanceEvacuationMark(newbit int) {
   984  	m.nevacuate++
   985  	// Experiments suggest that 1024 is overkill by at least an order of magnitude.
   986  	// Put it in there as a safeguard anyway, to ensure O(1) behavior.
   987  	stop := m.nevacuate + 1024
   988  	if stop > newbit {
   989  		stop = newbit
   990  	}
   991  	for m.nevacuate != stop && m.bucketEvacuated(uint64(m.nevacuate)) {
   992  		m.nevacuate++
   993  	}
   994  	if m.nevacuate == newbit { // newbit == # of oldbuckets
   995  		// Growing is all done. Free old main bucket array.
   996  		m.oldbuckets = nil
   997  		m.flags &^= sameSizeGrow
   998  	}
   999  }