github.com/bingoohuang/gg@v0.0.0-20240325092523-45da7dee9335/pkg/uid/ksuid.go (about)

     1  package uid
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/rand"
     6  	"database/sql/driver"
     7  	"encoding/binary"
     8  	"errors"
     9  	"fmt"
    10  	"io"
    11  	"math"
    12  	"math/bits"
    13  	mrand "math/rand"
    14  	"sync"
    15  	"time"
    16  )
    17  
    18  const (
    19  	// KSUID's epoch starts more recently so that the 32-bit number space gives a
    20  	// significantly higher useful lifetime of around 136 years from March 2017.
    21  	// This number (14e8) was picked to be easy to remember.
    22  	epochStamp int64 = 1400000000
    23  
    24  	// Timestamp is a uint32
    25  	timestampBytesNum = 4
    26  
    27  	// Payload is 16-bytes
    28  	payloadBytesNum = 16
    29  
    30  	// KSUIDs are 20 bytes when binary encoded
    31  	byteLength = timestampBytesNum + payloadBytesNum
    32  
    33  	// The length of a KSUID when string (base62) encoded
    34  	stringEncodedLength = 27
    35  
    36  	// A string-encoded minimum value for a KSUID
    37  	minStringEncoded = "000000000000000000000000000"
    38  
    39  	// A string-encoded maximum value for a KSUID
    40  	maxStringEncoded = "aWgEPTl1tmebfsQzFP4bxwgy80V"
    41  )
    42  
    43  // KSUID is 20 bytes:
    44  //
    45  //	00-03 byte: uint32 BE UTC timestamp with custom epoch
    46  //	04-19 byte: random "payload"
    47  type KSUID [byteLength]byte
    48  
    49  var (
    50  	rander     = rand.Reader
    51  	randMutex  = sync.Mutex{}
    52  	randBuffer = [payloadBytesNum]byte{}
    53  
    54  	errSize        = fmt.Errorf("valid KSUIDs are %v bytes", byteLength)
    55  	errStrSize     = fmt.Errorf("valid encoded KSUIDs are %v characters", stringEncodedLength)
    56  	errStrValue    = fmt.Errorf("valid encoded KSUIDs are bounded by %s and %s", minStringEncoded, maxStringEncoded)
    57  	errPayloadSize = fmt.Errorf("valid KSUID payloads are %v bytes", payloadBytesNum)
    58  
    59  	// Nil represents a completely empty (invalid) KSUID.
    60  	Nil KSUID
    61  	// Max represents the highest value a KSUID can have.
    62  	Max = KSUID{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}
    63  )
    64  
    65  // Append appends the string representation of i to b, returning a slice to a
    66  // potentially larger memory area.
    67  func (i KSUID) Append(b []byte) []byte { return fastAppendEncodeBase62(b, i[:]) }
    68  
    69  // Time is the timestamp portion of the ID as a Time object.
    70  func (i KSUID) Time() time.Time { return correctedUTCTimestampToTime(i.Timestamp()) }
    71  
    72  // Timestamp is the timestamp portion of the ID as a bare integer which is uncorrected
    73  // for KSUID's special epoch.
    74  func (i KSUID) Timestamp() uint32 { return binary.BigEndian.Uint32(i[:timestampBytesNum]) }
    75  
    76  // Payload is the 16-byte random payload without the timestamp.
    77  func (i KSUID) Payload() []byte { return i[timestampBytesNum:] }
    78  
    79  // String is string-encoded representation that can be passed through Parse().
    80  func (i KSUID) String() string { return string(i.Append(make([]byte, 0, stringEncodedLength))) }
    81  
    82  // Bytes raw byte representation of KSUID.
    83  func (i KSUID) Bytes() []byte { /* Safe because this is by-value*/ return i[:] }
    84  
    85  // IsNil returns true if this is a "nil" KSUID.
    86  func (i KSUID) IsNil() bool { return i == Nil }
    87  
    88  // Get satisfies the flag.Getter interface, making it possible to use KSUIDs as
    89  // part of the command line options of a program.
    90  func (i KSUID) Get() interface{} { return i }
    91  
    92  // Set satisfies the flag.Value interface, making it possible to use KSUIDs as
    93  // part of the command line options of a program.
    94  func (i *KSUID) Set(s string) error            { return i.UnmarshalText([]byte(s)) }
    95  func (i KSUID) MarshalText() ([]byte, error)   { return []byte(i.String()), nil }
    96  func (i KSUID) MarshalBinary() ([]byte, error) { return i.Bytes(), nil }
    97  
    98  func (i *KSUID) UnmarshalText(b []byte) error {
    99  	id, err := Parse(string(b))
   100  	if err != nil {
   101  		return err
   102  	}
   103  	*i = id
   104  	return nil
   105  }
   106  
   107  func (i *KSUID) UnmarshalBinary(b []byte) error {
   108  	id, err := FromBytes(b)
   109  	if err != nil {
   110  		return err
   111  	}
   112  	*i = id
   113  	return nil
   114  }
   115  
   116  // Value converts the KSUID into a SQL driver value which can be used to
   117  // directly use the KSUID as parameter to a SQL query.
   118  func (i KSUID) Value() (driver.Value, error) {
   119  	if i.IsNil() {
   120  		return nil, nil
   121  	}
   122  	return i.String(), nil
   123  }
   124  
   125  // Scan implements the sql.Scanner interface. It supports converting from
   126  // string, []byte, or nil into a KSUID value. Attempting to convert from
   127  // another type will return an error.
   128  func (i *KSUID) Scan(src interface{}) error {
   129  	switch v := src.(type) {
   130  	case nil:
   131  		return i.scan(nil)
   132  	case []byte:
   133  		return i.scan(v)
   134  	case string:
   135  		return i.scan([]byte(v))
   136  	default:
   137  		return fmt.Errorf("Scan: unable to scan type %T into KSUID", v)
   138  	}
   139  }
   140  
   141  func (i *KSUID) scan(b []byte) error {
   142  	switch len(b) {
   143  	case 0:
   144  		*i = Nil
   145  		return nil
   146  	case byteLength:
   147  		return i.UnmarshalBinary(b)
   148  	case stringEncodedLength:
   149  		return i.UnmarshalText(b)
   150  	default:
   151  		return errSize
   152  	}
   153  }
   154  
   155  // Parse decodes a string-encoded representation of a KSUID object
   156  func Parse(s string) (KSUID, error) {
   157  	if len(s) != stringEncodedLength {
   158  		return Nil, errStrSize
   159  	}
   160  
   161  	src := [stringEncodedLength]byte{}
   162  	dst := [byteLength]byte{}
   163  
   164  	copy(src[:], s[:])
   165  
   166  	if err := fastDecodeBase62(dst[:], src[:]); err != nil {
   167  		return Nil, errStrValue
   168  	}
   169  
   170  	return FromBytes(dst[:])
   171  }
   172  
   173  func timeToCorrectedUTCTimestamp(t time.Time) uint32  { return uint32(t.Unix() - epochStamp) }
   174  func correctedUTCTimestampToTime(ts uint32) time.Time { return time.Unix(int64(ts)+epochStamp, 0) }
   175  
   176  // New generates a new KSUID. In the strange case that random bytes
   177  // can't be read, it will panic.
   178  func New() KSUID {
   179  	ksuid, err := NewRandom()
   180  	if err != nil {
   181  		panic(fmt.Sprintf("Couldn't generate KSUID, inconceivable! error: %v", err))
   182  	}
   183  	return ksuid
   184  }
   185  
   186  // NewRandom generates a new KSUID
   187  func NewRandom() (ksuid KSUID, err error) { return NewRandomWithTime(time.Now()) }
   188  
   189  func NewRandomWithTime(t time.Time) (ksuid KSUID, err error) {
   190  	// Go's default random number generators are not safe for concurrent use by
   191  	// multiple goroutines, the use of the rander and randBuffer are explicitly
   192  	// synchronized here.
   193  	randMutex.Lock()
   194  
   195  	_, err = io.ReadAtLeast(rander, randBuffer[:], len(randBuffer))
   196  	copy(ksuid[timestampBytesNum:], randBuffer[:])
   197  
   198  	randMutex.Unlock()
   199  
   200  	if err != nil {
   201  		ksuid = Nil // don't leak random bytes on error
   202  		return
   203  	}
   204  
   205  	ts := timeToCorrectedUTCTimestamp(t)
   206  	binary.BigEndian.PutUint32(ksuid[:timestampBytesNum], ts)
   207  	return
   208  }
   209  
   210  // FromParts constructs a KSUID from constituent parts.
   211  func FromParts(t time.Time, payload []byte) (KSUID, error) {
   212  	if len(payload) != payloadBytesNum {
   213  		return Nil, errPayloadSize
   214  	}
   215  
   216  	var ksuid KSUID
   217  
   218  	ts := timeToCorrectedUTCTimestamp(t)
   219  	binary.BigEndian.PutUint32(ksuid[:timestampBytesNum], ts)
   220  
   221  	copy(ksuid[timestampBytesNum:], payload)
   222  
   223  	return ksuid, nil
   224  }
   225  
   226  // FromBytes constructs a KSUID from a 20-byte binary representation
   227  func FromBytes(b []byte) (KSUID, error) {
   228  	var ksuid KSUID
   229  
   230  	if len(b) != byteLength {
   231  		return Nil, errSize
   232  	}
   233  
   234  	copy(ksuid[:], b)
   235  	return ksuid, nil
   236  }
   237  
   238  // SetRand Sets the global source of random bytes for KSUID generation. This
   239  // should probably only be set once globally. While this is technically
   240  // thread-safe as in it won't cause corruption, there's no guarantee
   241  // on ordering.
   242  func SetRand(r io.Reader) {
   243  	if r == nil {
   244  		rander = rand.Reader
   245  		return
   246  	}
   247  	rander = r
   248  }
   249  
   250  // Compare implements comparison for KSUID type.
   251  func Compare(a, b KSUID) int {
   252  	return bytes.Compare(a[:], b[:])
   253  }
   254  
   255  // Sort sorts the given slice of KSUIDs.
   256  func Sort(ids []KSUID) { quickSort(ids, 0, len(ids)-1) }
   257  
   258  // IsSorted checks whether a slice of KSUIDs is sorted
   259  func IsSorted(ids []KSUID) bool {
   260  	if len(ids) != 0 {
   261  		min := ids[0]
   262  		for _, id := range ids[1:] {
   263  			if bytes.Compare(min[:], id[:]) > 0 {
   264  				return false
   265  			}
   266  			min = id
   267  		}
   268  	}
   269  	return true
   270  }
   271  
   272  func quickSort(a []KSUID, lo int, hi int) {
   273  	if lo < hi {
   274  		pivot := a[hi]
   275  		i := lo - 1
   276  
   277  		for j, n := lo, hi; j != n; j++ {
   278  			if bytes.Compare(a[j][:], pivot[:]) < 0 {
   279  				i++
   280  				a[i], a[j] = a[j], a[i]
   281  			}
   282  		}
   283  
   284  		i++
   285  		if bytes.Compare(a[hi][:], a[i][:]) < 0 {
   286  			a[i], a[hi] = a[hi], a[i]
   287  		}
   288  
   289  		quickSort(a, lo, i-1)
   290  		quickSort(a, i+1, hi)
   291  	}
   292  }
   293  
   294  // Next returns the next KSUID after id.
   295  func (i KSUID) Next() KSUID {
   296  	zero := makeUint128(0, 0)
   297  
   298  	t := i.Timestamp()
   299  	u := uint128Payload(i)
   300  	v := add128(u, makeUint128(0, 1))
   301  
   302  	if v == zero { // overflow
   303  		t++
   304  	}
   305  
   306  	return v.ksuid(t)
   307  }
   308  
   309  // Prev returns the previous KSUID before id.
   310  func (i KSUID) Prev() KSUID {
   311  	max := makeUint128(math.MaxUint64, math.MaxUint64)
   312  
   313  	t := i.Timestamp()
   314  	u := uint128Payload(i)
   315  	v := sub128(u, makeUint128(0, 1))
   316  
   317  	if v == max { // overflow
   318  		t--
   319  	}
   320  
   321  	return v.ksuid(t)
   322  }
   323  
   324  // uint128 represents an unsigned 128 bits little endian integer.
   325  type uint128 [2]uint64
   326  
   327  func uint128Payload(ksuid KSUID) uint128 {
   328  	return makeUint128FromPayload(ksuid[timestampBytesNum:])
   329  }
   330  
   331  func makeUint128(high uint64, low uint64) uint128 { return uint128{low, high} }
   332  
   333  func makeUint128FromPayload(payload []byte) uint128 {
   334  	return uint128{
   335  		binary.BigEndian.Uint64(payload[8:]), // low
   336  		binary.BigEndian.Uint64(payload[:8]), // high
   337  	}
   338  }
   339  
   340  func (v uint128) ksuid(timestamp uint32) (out KSUID) {
   341  	binary.BigEndian.PutUint32(out[:4], timestamp) // time
   342  	binary.BigEndian.PutUint64(out[4:12], v[1])    // high
   343  	binary.BigEndian.PutUint64(out[12:], v[0])     // low
   344  	return
   345  }
   346  
   347  func (v uint128) bytes() (out [16]byte) {
   348  	binary.BigEndian.PutUint64(out[:8], v[1])
   349  	binary.BigEndian.PutUint64(out[8:], v[0])
   350  	return
   351  }
   352  
   353  func (v uint128) String() string { return fmt.Sprintf("0x%016X%016X", v[0], v[1]) }
   354  
   355  func cmp128(x, y uint128) int {
   356  	if x[1] < y[1] {
   357  		return -1
   358  	}
   359  	if x[1] > y[1] {
   360  		return 1
   361  	}
   362  	if x[0] < y[0] {
   363  		return -1
   364  	}
   365  	if x[0] > y[0] {
   366  		return 1
   367  	}
   368  	return 0
   369  }
   370  
   371  func add128(x, y uint128) (z uint128) {
   372  	var c uint64
   373  	z[0], c = bits.Add64(x[0], y[0], 0)
   374  	z[1], _ = bits.Add64(x[1], y[1], c)
   375  	return
   376  }
   377  
   378  func sub128(x, y uint128) (z uint128) {
   379  	var b uint64
   380  	z[0], b = bits.Sub64(x[0], y[0], 0)
   381  	z[1], _ = bits.Sub64(x[1], y[1], b)
   382  	return
   383  }
   384  
   385  func incr128(x uint128) (z uint128) {
   386  	var c uint64
   387  	z[0], c = bits.Add64(x[0], 1, 0)
   388  	z[1] = x[1] + c
   389  	return
   390  }
   391  
   392  // Sequence is a KSUID generator which produces a sequence of ordered KSUIDs
   393  // from a seed.
   394  //
   395  // Up to 65536 KSUIDs can be generated by for a single seed.
   396  //
   397  // A typical usage of a Sequence looks like this:
   398  //
   399  //	seq := ksuid.Sequence{
   400  //		Seed: ksuid.New(),
   401  //	}
   402  //	id, err := seq.Next()
   403  //
   404  // Sequence values are not safe to use concurrently from multiple goroutines.
   405  type Sequence struct {
   406  	// The seed is used as base for the KSUID generator, all generated KSUIDs
   407  	// share the same leading 18 bytes of the seed.
   408  	Seed  KSUID
   409  	count uint32 // uint32 for overflow, only 2 bytes are used
   410  }
   411  
   412  // Next produces the next KSUID in the sequence, or returns an error if the
   413  // sequence has been exhausted.
   414  func (seq *Sequence) Next() (KSUID, error) {
   415  	id := seq.Seed // copy
   416  	count := seq.count
   417  	if count > math.MaxUint16 {
   418  		return Nil, errors.New("too many IDs were generated")
   419  	}
   420  	seq.count++
   421  	return withSequenceNumber(id, uint16(count)), nil
   422  }
   423  
   424  // Bounds returns the inclusive min and max bounds of the KSUIDs that may be
   425  // generated by the sequence. If all ids have been generated already then the
   426  // returned min value is equal to the max.
   427  func (seq *Sequence) Bounds() (min KSUID, max KSUID) {
   428  	count := seq.count
   429  	if count > math.MaxUint16 {
   430  		count = math.MaxUint16
   431  	}
   432  	return withSequenceNumber(seq.Seed, uint16(count)), withSequenceNumber(seq.Seed, math.MaxUint16)
   433  }
   434  
   435  func withSequenceNumber(id KSUID, n uint16) KSUID {
   436  	binary.BigEndian.PutUint16(id[len(id)-2:], n)
   437  	return id
   438  }
   439  
   440  const (
   441  	// lexographic ordering (based on Unicode table) is 0-9A-Za-z
   442  	base62Characters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
   443  	zeroString       = "000000000000000000000000000"
   444  	offsetUppercase  = 10
   445  	offsetLowercase  = 36
   446  )
   447  
   448  var errShortBuffer = errors.New("the output buffer is too small to hold to decoded value")
   449  
   450  // Converts a base 62 byte into the number value that it represents.
   451  func base62Value(digit byte) byte {
   452  	switch {
   453  	case digit >= '0' && digit <= '9':
   454  		return digit - '0'
   455  	case digit >= 'A' && digit <= 'Z':
   456  		return offsetUppercase + (digit - 'A')
   457  	default:
   458  		return offsetLowercase + (digit - 'a')
   459  	}
   460  }
   461  
   462  // This function encodes the base 62 representation of the src KSUID in binary
   463  // form into dst.
   464  //
   465  // In order to support a couple of optimizations the function assumes that src
   466  // is 20 bytes long and dst is 27 bytes long.
   467  //
   468  // Any unused bytes in dst will be set to the padding '0' byte.
   469  func fastEncodeBase62(dst []byte, src []byte) {
   470  	const srcBase = 4294967296
   471  	const dstBase = 62
   472  
   473  	// Split src into 5 4-byte words, this is where most of the efficiency comes
   474  	// from because this is a O(N^2) algorithm, and we make N = N / 4 by working
   475  	// on 32 bits at a time.
   476  	parts := [5]uint32{
   477  		binary.BigEndian.Uint32(src[0:4]),
   478  		binary.BigEndian.Uint32(src[4:8]),
   479  		binary.BigEndian.Uint32(src[8:12]),
   480  		binary.BigEndian.Uint32(src[12:16]),
   481  		binary.BigEndian.Uint32(src[16:20]),
   482  	}
   483  
   484  	n := len(dst)
   485  	bp := parts[:]
   486  	bq := [5]uint32{}
   487  
   488  	for len(bp) != 0 {
   489  		quotient := bq[:0]
   490  		remainder := uint64(0)
   491  
   492  		for _, c := range bp {
   493  			value := uint64(c) + uint64(remainder)*srcBase
   494  			digit := value / dstBase
   495  			remainder = value % dstBase
   496  
   497  			if len(quotient) != 0 || digit != 0 {
   498  				quotient = append(quotient, uint32(digit))
   499  			}
   500  		}
   501  
   502  		// Writes at the end of the destination buffer because we computed the
   503  		// lowest bits first.
   504  		n--
   505  		dst[n] = base62Characters[remainder]
   506  		bp = quotient
   507  	}
   508  
   509  	// Add padding at the head of the destination buffer for all bytes that were
   510  	// not set.
   511  	copy(dst[:n], zeroString)
   512  }
   513  
   514  // This function appends the base 62 representation of the KSUID in src to dst,
   515  // and returns the extended byte slice.
   516  // The result is left-padded with '0' bytes to always append 27 bytes to the
   517  // destination buffer.
   518  func fastAppendEncodeBase62(dst []byte, src []byte) []byte {
   519  	dst = reserve(dst, stringEncodedLength)
   520  	n := len(dst)
   521  	fastEncodeBase62(dst[n:n+stringEncodedLength], src)
   522  	return dst[:n+stringEncodedLength]
   523  }
   524  
   525  // This function decodes the base 62 representation of the src KSUID to the
   526  // binary form into dst.
   527  //
   528  // In order to support a couple of optimizations the function assumes that src
   529  // is 27 bytes long and dst is 20 bytes long.
   530  //
   531  // Any unused bytes in dst will be set to zero.
   532  func fastDecodeBase62(dst []byte, src []byte) error {
   533  	const srcBase = 62
   534  	const dstBase = 4294967296
   535  
   536  	// This line helps BCE (Bounds Check Elimination).
   537  	// It may be safely removed.
   538  	_ = src[26]
   539  
   540  	parts := [27]byte{
   541  		base62Value(src[0]),
   542  		base62Value(src[1]),
   543  		base62Value(src[2]),
   544  		base62Value(src[3]),
   545  		base62Value(src[4]),
   546  		base62Value(src[5]),
   547  		base62Value(src[6]),
   548  		base62Value(src[7]),
   549  		base62Value(src[8]),
   550  		base62Value(src[9]),
   551  
   552  		base62Value(src[10]),
   553  		base62Value(src[11]),
   554  		base62Value(src[12]),
   555  		base62Value(src[13]),
   556  		base62Value(src[14]),
   557  		base62Value(src[15]),
   558  		base62Value(src[16]),
   559  		base62Value(src[17]),
   560  		base62Value(src[18]),
   561  		base62Value(src[19]),
   562  
   563  		base62Value(src[20]),
   564  		base62Value(src[21]),
   565  		base62Value(src[22]),
   566  		base62Value(src[23]),
   567  		base62Value(src[24]),
   568  		base62Value(src[25]),
   569  		base62Value(src[26]),
   570  	}
   571  
   572  	n := len(dst)
   573  	bp := parts[:]
   574  	bq := [stringEncodedLength]byte{}
   575  
   576  	for len(bp) > 0 {
   577  		quotient := bq[:0]
   578  		remainder := uint64(0)
   579  
   580  		for _, c := range bp {
   581  			value := uint64(c) + uint64(remainder)*srcBase
   582  			digit := value / dstBase
   583  			remainder = value % dstBase
   584  
   585  			if len(quotient) != 0 || digit != 0 {
   586  				quotient = append(quotient, byte(digit))
   587  			}
   588  		}
   589  
   590  		if n < 4 {
   591  			return errShortBuffer
   592  		}
   593  
   594  		dst[n-4] = byte(remainder >> 24)
   595  		dst[n-3] = byte(remainder >> 16)
   596  		dst[n-2] = byte(remainder >> 8)
   597  		dst[n-1] = byte(remainder)
   598  		n -= 4
   599  		bp = quotient
   600  	}
   601  
   602  	var zero [20]byte
   603  	copy(dst[:n], zero[:])
   604  	return nil
   605  }
   606  
   607  // This function appends the base 62 decoded version of src into dst.
   608  func fastAppendDecodeBase62(dst []byte, src []byte) []byte {
   609  	dst = reserve(dst, byteLength)
   610  	n := len(dst)
   611  	fastDecodeBase62(dst[n:n+byteLength], src)
   612  	return dst[:n+byteLength]
   613  }
   614  
   615  // Ensures that at least nbytes are available in the remaining capacity of the
   616  // destination slice, if not, a new copy is made and returned by the function.
   617  func reserve(dst []byte, nbytes int) []byte {
   618  	c := cap(dst)
   619  	n := len(dst)
   620  
   621  	if avail := c - n; avail < nbytes {
   622  		c *= 2
   623  		if (c - n) < nbytes {
   624  			c = n + nbytes
   625  		}
   626  		b := make([]byte, n, c)
   627  		copy(b, dst)
   628  		dst = b
   629  	}
   630  
   631  	return dst
   632  }
   633  
   634  // CompressedSet is an immutable data type which stores a set of KSUIDs.
   635  type CompressedSet []byte
   636  
   637  // Iter returns an iterator that produces all KSUIDs in the set.
   638  func (set CompressedSet) Iter() CompressedSetIter {
   639  	return CompressedSetIter{
   640  		content: set,
   641  	}
   642  }
   643  
   644  // String satisfies the fmt.Stringer interface, returns a human-readable string
   645  // representation of the set.
   646  func (set CompressedSet) String() string {
   647  	b := bytes.Buffer{}
   648  	b.WriteByte('[')
   649  	set.writeTo(&b)
   650  	b.WriteByte(']')
   651  	return b.String()
   652  }
   653  
   654  // GoString satisfies the fmt.GoStringer interface, returns a Go representation of
   655  // the set.
   656  func (set CompressedSet) GoString() string {
   657  	b := bytes.Buffer{}
   658  	b.WriteString("ksuid.CompressedSet{")
   659  	set.writeTo(&b)
   660  	b.WriteByte('}')
   661  	return b.String()
   662  }
   663  
   664  func (set CompressedSet) writeTo(b *bytes.Buffer) {
   665  	a := [27]byte{}
   666  
   667  	for i, it := 0, set.Iter(); it.Next(); i++ {
   668  		if i != 0 {
   669  			b.WriteString(", ")
   670  		}
   671  		b.WriteByte('"')
   672  		it.KSUID.Append(a[:0])
   673  		b.Write(a[:])
   674  		b.WriteByte('"')
   675  	}
   676  }
   677  
   678  // Compress creates and returns a compressed set of KSUIDs from the list given
   679  // as arguments.
   680  func Compress(ids ...KSUID) CompressedSet {
   681  	c := 1 + byteLength + (len(ids) / 5)
   682  	b := make([]byte, 0, c)
   683  	return AppendCompressed(b, ids...)
   684  }
   685  
   686  // AppendCompressed uses the given byte slice as pre-allocated storage space to
   687  // build a KSUID set.
   688  //
   689  // Note that the set uses a compression technique to store the KSUIDs, so the
   690  // resulting length is not 20 x len(ids). The rule of thumb here is for the given
   691  // byte slice to reserve the amount of memory that the application would be OK
   692  // to waste.
   693  func AppendCompressed(set []byte, ids ...KSUID) CompressedSet {
   694  	if len(ids) != 0 {
   695  		if !IsSorted(ids) {
   696  			Sort(ids)
   697  		}
   698  		one := makeUint128(0, 1)
   699  
   700  		// The first KSUID is always written to the set, this is the starting
   701  		// point for all deltas.
   702  		set = append(set, byte(rawKSUID))
   703  		set = append(set, ids[0][:]...)
   704  
   705  		timestamp := ids[0].Timestamp()
   706  		lastKSUID := ids[0]
   707  		lastPayload := uint128Payload(ids[0])
   708  
   709  		for i := 1; i != len(ids); i++ {
   710  			id := ids[i]
   711  
   712  			if id == lastKSUID {
   713  				continue
   714  			}
   715  
   716  			t := id.Timestamp()
   717  			p := uint128Payload(id)
   718  
   719  			if t != timestamp {
   720  				d := t - timestamp
   721  				n := varintLength32(d)
   722  
   723  				set = append(set, timeDelta|byte(n))
   724  				set = appendVarint32(set, d, n)
   725  				set = append(set, id[timestampBytesNum:]...)
   726  
   727  				timestamp = t
   728  			} else {
   729  				d := sub128(p, lastPayload)
   730  
   731  				if d != one {
   732  					n := varintLength128(d)
   733  
   734  					set = append(set, payloadDelta|byte(n))
   735  					set = appendVarint128(set, d, n)
   736  				} else {
   737  					l, c := rangeLength(ids[i+1:], t, id, p)
   738  					m := uint64(l + 1)
   739  					n := varintLength64(m)
   740  
   741  					set = append(set, payloadRange|byte(n))
   742  					set = appendVarint64(set, m, n)
   743  
   744  					i += c
   745  					id = ids[i]
   746  					p = uint128Payload(id)
   747  				}
   748  			}
   749  
   750  			lastKSUID = id
   751  			lastPayload = p
   752  		}
   753  	}
   754  	return set
   755  }
   756  
   757  func rangeLength(ids []KSUID, timestamp uint32, lastKSUID KSUID, lastValue uint128) (length int, count int) {
   758  	one := makeUint128(0, 1)
   759  
   760  	for i := range ids {
   761  		id := ids[i]
   762  
   763  		if id == lastKSUID {
   764  			continue
   765  		}
   766  
   767  		if id.Timestamp() != timestamp {
   768  			count = i
   769  			return
   770  		}
   771  
   772  		v := uint128Payload(id)
   773  
   774  		if sub128(v, lastValue) != one {
   775  			count = i
   776  			return
   777  		}
   778  
   779  		lastKSUID = id
   780  		lastValue = v
   781  		length++
   782  	}
   783  
   784  	count = len(ids)
   785  	return
   786  }
   787  
   788  func appendVarint128(b []byte, v uint128, n int) []byte {
   789  	c := v.bytes()
   790  	return append(b, c[len(c)-n:]...)
   791  }
   792  
   793  func appendVarint64(b []byte, v uint64, n int) []byte {
   794  	c := [8]byte{}
   795  	binary.BigEndian.PutUint64(c[:], v)
   796  	return append(b, c[len(c)-n:]...)
   797  }
   798  
   799  func appendVarint32(b []byte, v uint32, n int) []byte {
   800  	c := [4]byte{}
   801  	binary.BigEndian.PutUint32(c[:], v)
   802  	return append(b, c[len(c)-n:]...)
   803  }
   804  
   805  func varint128(b []byte) uint128 {
   806  	a := [16]byte{}
   807  	copy(a[16-len(b):], b)
   808  	return makeUint128FromPayload(a[:])
   809  }
   810  
   811  func varint64(b []byte) uint64 {
   812  	a := [8]byte{}
   813  	copy(a[8-len(b):], b)
   814  	return binary.BigEndian.Uint64(a[:])
   815  }
   816  
   817  func varint32(b []byte) uint32 {
   818  	a := [4]byte{}
   819  	copy(a[4-len(b):], b)
   820  	return binary.BigEndian.Uint32(a[:])
   821  }
   822  
   823  func varintLength128(v uint128) int {
   824  	if v[1] != 0 {
   825  		return 8 + varintLength64(v[1])
   826  	}
   827  	return varintLength64(v[0])
   828  }
   829  
   830  func varintLength64(v uint64) int {
   831  	switch {
   832  	case (v & 0xFFFFFFFFFFFFFF00) == 0:
   833  		return 1
   834  	case (v & 0xFFFFFFFFFFFF0000) == 0:
   835  		return 2
   836  	case (v & 0xFFFFFFFFFF000000) == 0:
   837  		return 3
   838  	case (v & 0xFFFFFFFF00000000) == 0:
   839  		return 4
   840  	case (v & 0xFFFFFF0000000000) == 0:
   841  		return 5
   842  	case (v & 0xFFFF000000000000) == 0:
   843  		return 6
   844  	case (v & 0xFF00000000000000) == 0:
   845  		return 7
   846  	default:
   847  		return 8
   848  	}
   849  }
   850  
   851  func varintLength32(v uint32) int {
   852  	switch {
   853  	case (v & 0xFFFFFF00) == 0:
   854  		return 1
   855  	case (v & 0xFFFF0000) == 0:
   856  		return 2
   857  	case (v & 0xFF000000) == 0:
   858  		return 3
   859  	default:
   860  		return 4
   861  	}
   862  }
   863  
   864  const (
   865  	rawKSUID     = 0
   866  	timeDelta    = 1 << 6
   867  	payloadDelta = 1 << 7
   868  	payloadRange = (1 << 6) | (1 << 7)
   869  )
   870  
   871  // CompressedSetIter is an iterator type returned by Set.Iter to produce the
   872  // list of KSUIDs stored in a set.
   873  //
   874  // Here's is how the iterator type is commonly used:
   875  //
   876  //	for it := set.Iter(); it.Next(); {
   877  //		id := it.KSUID
   878  //		// ...
   879  //	}
   880  //
   881  // CompressedSetIter values are not safe to use concurrently from multiple
   882  // goroutines.
   883  type CompressedSetIter struct {
   884  	// KSUID is modified by calls to the Next method to hold the KSUID loaded
   885  	// by the iterator.
   886  	KSUID
   887  
   888  	content []byte
   889  	offset  int
   890  
   891  	seqLength uint64
   892  	timestamp uint32
   893  	lastValue uint128
   894  }
   895  
   896  // Next moves the iterator forward, returning true if there is a KSUID was found,
   897  // or false if the iterator as reached the end of the set it was created from.
   898  func (it *CompressedSetIter) Next() bool {
   899  	if it.seqLength != 0 {
   900  		value := incr128(it.lastValue)
   901  		it.KSUID = value.ksuid(it.timestamp)
   902  		it.seqLength--
   903  		it.lastValue = value
   904  		return true
   905  	}
   906  
   907  	if it.offset == len(it.content) {
   908  		return false
   909  	}
   910  
   911  	b := it.content[it.offset]
   912  	it.offset++
   913  
   914  	const mask = rawKSUID | timeDelta | payloadDelta | payloadRange
   915  	tag := int(b) & mask
   916  	cnt := int(b) & ^mask
   917  
   918  	switch tag {
   919  	case rawKSUID:
   920  		off0 := it.offset
   921  		off1 := off0 + byteLength
   922  
   923  		copy(it.KSUID[:], it.content[off0:off1])
   924  
   925  		it.offset = off1
   926  		it.timestamp = it.KSUID.Timestamp()
   927  		it.lastValue = uint128Payload(it.KSUID)
   928  
   929  	case timeDelta:
   930  		off0 := it.offset
   931  		off1 := off0 + cnt
   932  		off2 := off1 + payloadBytesNum
   933  
   934  		it.timestamp += varint32(it.content[off0:off1])
   935  
   936  		binary.BigEndian.PutUint32(it.KSUID[:timestampBytesNum], it.timestamp)
   937  		copy(it.KSUID[timestampBytesNum:], it.content[off1:off2])
   938  
   939  		it.offset = off2
   940  		it.lastValue = uint128Payload(it.KSUID)
   941  
   942  	case payloadDelta:
   943  		off0 := it.offset
   944  		off1 := off0 + cnt
   945  
   946  		delta := varint128(it.content[off0:off1])
   947  		value := add128(it.lastValue, delta)
   948  
   949  		it.KSUID = value.ksuid(it.timestamp)
   950  		it.offset = off1
   951  		it.lastValue = value
   952  
   953  	case payloadRange:
   954  		off0 := it.offset
   955  		off1 := off0 + cnt
   956  
   957  		value := incr128(it.lastValue)
   958  		it.KSUID = value.ksuid(it.timestamp)
   959  		it.seqLength = varint64(it.content[off0:off1])
   960  		it.offset = off1
   961  		it.seqLength--
   962  		it.lastValue = value
   963  
   964  	default:
   965  		panic("KSUID set iterator is reading malformed data")
   966  	}
   967  
   968  	return true
   969  }
   970  
   971  // FastRander is an io.Reader that uses math/rand and is optimized for
   972  // generating 16 bytes KSUID payloads. It is intended to be used as a
   973  // performance improvements for programs that have no need for
   974  // cryptographically secure KSUIDs and are generating a lot of them.
   975  var FastRander = newRBG()
   976  
   977  func newRBG() io.Reader {
   978  	r, err := newRandomBitsGenerator()
   979  	if err != nil {
   980  		panic(err)
   981  	}
   982  	return r
   983  }
   984  
   985  func newRandomBitsGenerator() (r io.Reader, err error) {
   986  	var seed int64
   987  
   988  	if seed, err = readCryptoRandomSeed(); err != nil {
   989  		return
   990  	}
   991  
   992  	r = &randSourceReader{source: mrand.NewSource(seed).(mrand.Source64)}
   993  	return
   994  }
   995  
   996  func readCryptoRandomSeed() (seed int64, err error) {
   997  	var b [8]byte
   998  
   999  	if _, err = io.ReadFull(rand.Reader, b[:]); err != nil {
  1000  		return
  1001  	}
  1002  
  1003  	seed = int64(binary.LittleEndian.Uint64(b[:]))
  1004  	return
  1005  }
  1006  
  1007  type randSourceReader struct {
  1008  	source mrand.Source64
  1009  }
  1010  
  1011  func (r *randSourceReader) Read(b []byte) (int, error) {
  1012  	// optimized for generating 16 bytes payloads
  1013  	binary.LittleEndian.PutUint64(b[:8], r.source.Uint64())
  1014  	binary.LittleEndian.PutUint64(b[8:], r.source.Uint64())
  1015  	return 16, nil
  1016  }