github.com/bingoohuang/gg@v0.0.0-20240325092523-45da7dee9335/pkg/uid/ksuid.go (about) 1 package uid 2 3 import ( 4 "bytes" 5 "crypto/rand" 6 "database/sql/driver" 7 "encoding/binary" 8 "errors" 9 "fmt" 10 "io" 11 "math" 12 "math/bits" 13 mrand "math/rand" 14 "sync" 15 "time" 16 ) 17 18 const ( 19 // KSUID's epoch starts more recently so that the 32-bit number space gives a 20 // significantly higher useful lifetime of around 136 years from March 2017. 21 // This number (14e8) was picked to be easy to remember. 22 epochStamp int64 = 1400000000 23 24 // Timestamp is a uint32 25 timestampBytesNum = 4 26 27 // Payload is 16-bytes 28 payloadBytesNum = 16 29 30 // KSUIDs are 20 bytes when binary encoded 31 byteLength = timestampBytesNum + payloadBytesNum 32 33 // The length of a KSUID when string (base62) encoded 34 stringEncodedLength = 27 35 36 // A string-encoded minimum value for a KSUID 37 minStringEncoded = "000000000000000000000000000" 38 39 // A string-encoded maximum value for a KSUID 40 maxStringEncoded = "aWgEPTl1tmebfsQzFP4bxwgy80V" 41 ) 42 43 // KSUID is 20 bytes: 44 // 45 // 00-03 byte: uint32 BE UTC timestamp with custom epoch 46 // 04-19 byte: random "payload" 47 type KSUID [byteLength]byte 48 49 var ( 50 rander = rand.Reader 51 randMutex = sync.Mutex{} 52 randBuffer = [payloadBytesNum]byte{} 53 54 errSize = fmt.Errorf("valid KSUIDs are %v bytes", byteLength) 55 errStrSize = fmt.Errorf("valid encoded KSUIDs are %v characters", stringEncodedLength) 56 errStrValue = fmt.Errorf("valid encoded KSUIDs are bounded by %s and %s", minStringEncoded, maxStringEncoded) 57 errPayloadSize = fmt.Errorf("valid KSUID payloads are %v bytes", payloadBytesNum) 58 59 // Nil represents a completely empty (invalid) KSUID. 60 Nil KSUID 61 // Max represents the highest value a KSUID can have. 62 Max = KSUID{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255} 63 ) 64 65 // Append appends the string representation of i to b, returning a slice to a 66 // potentially larger memory area. 67 func (i KSUID) Append(b []byte) []byte { return fastAppendEncodeBase62(b, i[:]) } 68 69 // Time is the timestamp portion of the ID as a Time object. 70 func (i KSUID) Time() time.Time { return correctedUTCTimestampToTime(i.Timestamp()) } 71 72 // Timestamp is the timestamp portion of the ID as a bare integer which is uncorrected 73 // for KSUID's special epoch. 74 func (i KSUID) Timestamp() uint32 { return binary.BigEndian.Uint32(i[:timestampBytesNum]) } 75 76 // Payload is the 16-byte random payload without the timestamp. 77 func (i KSUID) Payload() []byte { return i[timestampBytesNum:] } 78 79 // String is string-encoded representation that can be passed through Parse(). 80 func (i KSUID) String() string { return string(i.Append(make([]byte, 0, stringEncodedLength))) } 81 82 // Bytes raw byte representation of KSUID. 83 func (i KSUID) Bytes() []byte { /* Safe because this is by-value*/ return i[:] } 84 85 // IsNil returns true if this is a "nil" KSUID. 86 func (i KSUID) IsNil() bool { return i == Nil } 87 88 // Get satisfies the flag.Getter interface, making it possible to use KSUIDs as 89 // part of the command line options of a program. 90 func (i KSUID) Get() interface{} { return i } 91 92 // Set satisfies the flag.Value interface, making it possible to use KSUIDs as 93 // part of the command line options of a program. 94 func (i *KSUID) Set(s string) error { return i.UnmarshalText([]byte(s)) } 95 func (i KSUID) MarshalText() ([]byte, error) { return []byte(i.String()), nil } 96 func (i KSUID) MarshalBinary() ([]byte, error) { return i.Bytes(), nil } 97 98 func (i *KSUID) UnmarshalText(b []byte) error { 99 id, err := Parse(string(b)) 100 if err != nil { 101 return err 102 } 103 *i = id 104 return nil 105 } 106 107 func (i *KSUID) UnmarshalBinary(b []byte) error { 108 id, err := FromBytes(b) 109 if err != nil { 110 return err 111 } 112 *i = id 113 return nil 114 } 115 116 // Value converts the KSUID into a SQL driver value which can be used to 117 // directly use the KSUID as parameter to a SQL query. 118 func (i KSUID) Value() (driver.Value, error) { 119 if i.IsNil() { 120 return nil, nil 121 } 122 return i.String(), nil 123 } 124 125 // Scan implements the sql.Scanner interface. It supports converting from 126 // string, []byte, or nil into a KSUID value. Attempting to convert from 127 // another type will return an error. 128 func (i *KSUID) Scan(src interface{}) error { 129 switch v := src.(type) { 130 case nil: 131 return i.scan(nil) 132 case []byte: 133 return i.scan(v) 134 case string: 135 return i.scan([]byte(v)) 136 default: 137 return fmt.Errorf("Scan: unable to scan type %T into KSUID", v) 138 } 139 } 140 141 func (i *KSUID) scan(b []byte) error { 142 switch len(b) { 143 case 0: 144 *i = Nil 145 return nil 146 case byteLength: 147 return i.UnmarshalBinary(b) 148 case stringEncodedLength: 149 return i.UnmarshalText(b) 150 default: 151 return errSize 152 } 153 } 154 155 // Parse decodes a string-encoded representation of a KSUID object 156 func Parse(s string) (KSUID, error) { 157 if len(s) != stringEncodedLength { 158 return Nil, errStrSize 159 } 160 161 src := [stringEncodedLength]byte{} 162 dst := [byteLength]byte{} 163 164 copy(src[:], s[:]) 165 166 if err := fastDecodeBase62(dst[:], src[:]); err != nil { 167 return Nil, errStrValue 168 } 169 170 return FromBytes(dst[:]) 171 } 172 173 func timeToCorrectedUTCTimestamp(t time.Time) uint32 { return uint32(t.Unix() - epochStamp) } 174 func correctedUTCTimestampToTime(ts uint32) time.Time { return time.Unix(int64(ts)+epochStamp, 0) } 175 176 // New generates a new KSUID. In the strange case that random bytes 177 // can't be read, it will panic. 178 func New() KSUID { 179 ksuid, err := NewRandom() 180 if err != nil { 181 panic(fmt.Sprintf("Couldn't generate KSUID, inconceivable! error: %v", err)) 182 } 183 return ksuid 184 } 185 186 // NewRandom generates a new KSUID 187 func NewRandom() (ksuid KSUID, err error) { return NewRandomWithTime(time.Now()) } 188 189 func NewRandomWithTime(t time.Time) (ksuid KSUID, err error) { 190 // Go's default random number generators are not safe for concurrent use by 191 // multiple goroutines, the use of the rander and randBuffer are explicitly 192 // synchronized here. 193 randMutex.Lock() 194 195 _, err = io.ReadAtLeast(rander, randBuffer[:], len(randBuffer)) 196 copy(ksuid[timestampBytesNum:], randBuffer[:]) 197 198 randMutex.Unlock() 199 200 if err != nil { 201 ksuid = Nil // don't leak random bytes on error 202 return 203 } 204 205 ts := timeToCorrectedUTCTimestamp(t) 206 binary.BigEndian.PutUint32(ksuid[:timestampBytesNum], ts) 207 return 208 } 209 210 // FromParts constructs a KSUID from constituent parts. 211 func FromParts(t time.Time, payload []byte) (KSUID, error) { 212 if len(payload) != payloadBytesNum { 213 return Nil, errPayloadSize 214 } 215 216 var ksuid KSUID 217 218 ts := timeToCorrectedUTCTimestamp(t) 219 binary.BigEndian.PutUint32(ksuid[:timestampBytesNum], ts) 220 221 copy(ksuid[timestampBytesNum:], payload) 222 223 return ksuid, nil 224 } 225 226 // FromBytes constructs a KSUID from a 20-byte binary representation 227 func FromBytes(b []byte) (KSUID, error) { 228 var ksuid KSUID 229 230 if len(b) != byteLength { 231 return Nil, errSize 232 } 233 234 copy(ksuid[:], b) 235 return ksuid, nil 236 } 237 238 // SetRand Sets the global source of random bytes for KSUID generation. This 239 // should probably only be set once globally. While this is technically 240 // thread-safe as in it won't cause corruption, there's no guarantee 241 // on ordering. 242 func SetRand(r io.Reader) { 243 if r == nil { 244 rander = rand.Reader 245 return 246 } 247 rander = r 248 } 249 250 // Compare implements comparison for KSUID type. 251 func Compare(a, b KSUID) int { 252 return bytes.Compare(a[:], b[:]) 253 } 254 255 // Sort sorts the given slice of KSUIDs. 256 func Sort(ids []KSUID) { quickSort(ids, 0, len(ids)-1) } 257 258 // IsSorted checks whether a slice of KSUIDs is sorted 259 func IsSorted(ids []KSUID) bool { 260 if len(ids) != 0 { 261 min := ids[0] 262 for _, id := range ids[1:] { 263 if bytes.Compare(min[:], id[:]) > 0 { 264 return false 265 } 266 min = id 267 } 268 } 269 return true 270 } 271 272 func quickSort(a []KSUID, lo int, hi int) { 273 if lo < hi { 274 pivot := a[hi] 275 i := lo - 1 276 277 for j, n := lo, hi; j != n; j++ { 278 if bytes.Compare(a[j][:], pivot[:]) < 0 { 279 i++ 280 a[i], a[j] = a[j], a[i] 281 } 282 } 283 284 i++ 285 if bytes.Compare(a[hi][:], a[i][:]) < 0 { 286 a[i], a[hi] = a[hi], a[i] 287 } 288 289 quickSort(a, lo, i-1) 290 quickSort(a, i+1, hi) 291 } 292 } 293 294 // Next returns the next KSUID after id. 295 func (i KSUID) Next() KSUID { 296 zero := makeUint128(0, 0) 297 298 t := i.Timestamp() 299 u := uint128Payload(i) 300 v := add128(u, makeUint128(0, 1)) 301 302 if v == zero { // overflow 303 t++ 304 } 305 306 return v.ksuid(t) 307 } 308 309 // Prev returns the previous KSUID before id. 310 func (i KSUID) Prev() KSUID { 311 max := makeUint128(math.MaxUint64, math.MaxUint64) 312 313 t := i.Timestamp() 314 u := uint128Payload(i) 315 v := sub128(u, makeUint128(0, 1)) 316 317 if v == max { // overflow 318 t-- 319 } 320 321 return v.ksuid(t) 322 } 323 324 // uint128 represents an unsigned 128 bits little endian integer. 325 type uint128 [2]uint64 326 327 func uint128Payload(ksuid KSUID) uint128 { 328 return makeUint128FromPayload(ksuid[timestampBytesNum:]) 329 } 330 331 func makeUint128(high uint64, low uint64) uint128 { return uint128{low, high} } 332 333 func makeUint128FromPayload(payload []byte) uint128 { 334 return uint128{ 335 binary.BigEndian.Uint64(payload[8:]), // low 336 binary.BigEndian.Uint64(payload[:8]), // high 337 } 338 } 339 340 func (v uint128) ksuid(timestamp uint32) (out KSUID) { 341 binary.BigEndian.PutUint32(out[:4], timestamp) // time 342 binary.BigEndian.PutUint64(out[4:12], v[1]) // high 343 binary.BigEndian.PutUint64(out[12:], v[0]) // low 344 return 345 } 346 347 func (v uint128) bytes() (out [16]byte) { 348 binary.BigEndian.PutUint64(out[:8], v[1]) 349 binary.BigEndian.PutUint64(out[8:], v[0]) 350 return 351 } 352 353 func (v uint128) String() string { return fmt.Sprintf("0x%016X%016X", v[0], v[1]) } 354 355 func cmp128(x, y uint128) int { 356 if x[1] < y[1] { 357 return -1 358 } 359 if x[1] > y[1] { 360 return 1 361 } 362 if x[0] < y[0] { 363 return -1 364 } 365 if x[0] > y[0] { 366 return 1 367 } 368 return 0 369 } 370 371 func add128(x, y uint128) (z uint128) { 372 var c uint64 373 z[0], c = bits.Add64(x[0], y[0], 0) 374 z[1], _ = bits.Add64(x[1], y[1], c) 375 return 376 } 377 378 func sub128(x, y uint128) (z uint128) { 379 var b uint64 380 z[0], b = bits.Sub64(x[0], y[0], 0) 381 z[1], _ = bits.Sub64(x[1], y[1], b) 382 return 383 } 384 385 func incr128(x uint128) (z uint128) { 386 var c uint64 387 z[0], c = bits.Add64(x[0], 1, 0) 388 z[1] = x[1] + c 389 return 390 } 391 392 // Sequence is a KSUID generator which produces a sequence of ordered KSUIDs 393 // from a seed. 394 // 395 // Up to 65536 KSUIDs can be generated by for a single seed. 396 // 397 // A typical usage of a Sequence looks like this: 398 // 399 // seq := ksuid.Sequence{ 400 // Seed: ksuid.New(), 401 // } 402 // id, err := seq.Next() 403 // 404 // Sequence values are not safe to use concurrently from multiple goroutines. 405 type Sequence struct { 406 // The seed is used as base for the KSUID generator, all generated KSUIDs 407 // share the same leading 18 bytes of the seed. 408 Seed KSUID 409 count uint32 // uint32 for overflow, only 2 bytes are used 410 } 411 412 // Next produces the next KSUID in the sequence, or returns an error if the 413 // sequence has been exhausted. 414 func (seq *Sequence) Next() (KSUID, error) { 415 id := seq.Seed // copy 416 count := seq.count 417 if count > math.MaxUint16 { 418 return Nil, errors.New("too many IDs were generated") 419 } 420 seq.count++ 421 return withSequenceNumber(id, uint16(count)), nil 422 } 423 424 // Bounds returns the inclusive min and max bounds of the KSUIDs that may be 425 // generated by the sequence. If all ids have been generated already then the 426 // returned min value is equal to the max. 427 func (seq *Sequence) Bounds() (min KSUID, max KSUID) { 428 count := seq.count 429 if count > math.MaxUint16 { 430 count = math.MaxUint16 431 } 432 return withSequenceNumber(seq.Seed, uint16(count)), withSequenceNumber(seq.Seed, math.MaxUint16) 433 } 434 435 func withSequenceNumber(id KSUID, n uint16) KSUID { 436 binary.BigEndian.PutUint16(id[len(id)-2:], n) 437 return id 438 } 439 440 const ( 441 // lexographic ordering (based on Unicode table) is 0-9A-Za-z 442 base62Characters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 443 zeroString = "000000000000000000000000000" 444 offsetUppercase = 10 445 offsetLowercase = 36 446 ) 447 448 var errShortBuffer = errors.New("the output buffer is too small to hold to decoded value") 449 450 // Converts a base 62 byte into the number value that it represents. 451 func base62Value(digit byte) byte { 452 switch { 453 case digit >= '0' && digit <= '9': 454 return digit - '0' 455 case digit >= 'A' && digit <= 'Z': 456 return offsetUppercase + (digit - 'A') 457 default: 458 return offsetLowercase + (digit - 'a') 459 } 460 } 461 462 // This function encodes the base 62 representation of the src KSUID in binary 463 // form into dst. 464 // 465 // In order to support a couple of optimizations the function assumes that src 466 // is 20 bytes long and dst is 27 bytes long. 467 // 468 // Any unused bytes in dst will be set to the padding '0' byte. 469 func fastEncodeBase62(dst []byte, src []byte) { 470 const srcBase = 4294967296 471 const dstBase = 62 472 473 // Split src into 5 4-byte words, this is where most of the efficiency comes 474 // from because this is a O(N^2) algorithm, and we make N = N / 4 by working 475 // on 32 bits at a time. 476 parts := [5]uint32{ 477 binary.BigEndian.Uint32(src[0:4]), 478 binary.BigEndian.Uint32(src[4:8]), 479 binary.BigEndian.Uint32(src[8:12]), 480 binary.BigEndian.Uint32(src[12:16]), 481 binary.BigEndian.Uint32(src[16:20]), 482 } 483 484 n := len(dst) 485 bp := parts[:] 486 bq := [5]uint32{} 487 488 for len(bp) != 0 { 489 quotient := bq[:0] 490 remainder := uint64(0) 491 492 for _, c := range bp { 493 value := uint64(c) + uint64(remainder)*srcBase 494 digit := value / dstBase 495 remainder = value % dstBase 496 497 if len(quotient) != 0 || digit != 0 { 498 quotient = append(quotient, uint32(digit)) 499 } 500 } 501 502 // Writes at the end of the destination buffer because we computed the 503 // lowest bits first. 504 n-- 505 dst[n] = base62Characters[remainder] 506 bp = quotient 507 } 508 509 // Add padding at the head of the destination buffer for all bytes that were 510 // not set. 511 copy(dst[:n], zeroString) 512 } 513 514 // This function appends the base 62 representation of the KSUID in src to dst, 515 // and returns the extended byte slice. 516 // The result is left-padded with '0' bytes to always append 27 bytes to the 517 // destination buffer. 518 func fastAppendEncodeBase62(dst []byte, src []byte) []byte { 519 dst = reserve(dst, stringEncodedLength) 520 n := len(dst) 521 fastEncodeBase62(dst[n:n+stringEncodedLength], src) 522 return dst[:n+stringEncodedLength] 523 } 524 525 // This function decodes the base 62 representation of the src KSUID to the 526 // binary form into dst. 527 // 528 // In order to support a couple of optimizations the function assumes that src 529 // is 27 bytes long and dst is 20 bytes long. 530 // 531 // Any unused bytes in dst will be set to zero. 532 func fastDecodeBase62(dst []byte, src []byte) error { 533 const srcBase = 62 534 const dstBase = 4294967296 535 536 // This line helps BCE (Bounds Check Elimination). 537 // It may be safely removed. 538 _ = src[26] 539 540 parts := [27]byte{ 541 base62Value(src[0]), 542 base62Value(src[1]), 543 base62Value(src[2]), 544 base62Value(src[3]), 545 base62Value(src[4]), 546 base62Value(src[5]), 547 base62Value(src[6]), 548 base62Value(src[7]), 549 base62Value(src[8]), 550 base62Value(src[9]), 551 552 base62Value(src[10]), 553 base62Value(src[11]), 554 base62Value(src[12]), 555 base62Value(src[13]), 556 base62Value(src[14]), 557 base62Value(src[15]), 558 base62Value(src[16]), 559 base62Value(src[17]), 560 base62Value(src[18]), 561 base62Value(src[19]), 562 563 base62Value(src[20]), 564 base62Value(src[21]), 565 base62Value(src[22]), 566 base62Value(src[23]), 567 base62Value(src[24]), 568 base62Value(src[25]), 569 base62Value(src[26]), 570 } 571 572 n := len(dst) 573 bp := parts[:] 574 bq := [stringEncodedLength]byte{} 575 576 for len(bp) > 0 { 577 quotient := bq[:0] 578 remainder := uint64(0) 579 580 for _, c := range bp { 581 value := uint64(c) + uint64(remainder)*srcBase 582 digit := value / dstBase 583 remainder = value % dstBase 584 585 if len(quotient) != 0 || digit != 0 { 586 quotient = append(quotient, byte(digit)) 587 } 588 } 589 590 if n < 4 { 591 return errShortBuffer 592 } 593 594 dst[n-4] = byte(remainder >> 24) 595 dst[n-3] = byte(remainder >> 16) 596 dst[n-2] = byte(remainder >> 8) 597 dst[n-1] = byte(remainder) 598 n -= 4 599 bp = quotient 600 } 601 602 var zero [20]byte 603 copy(dst[:n], zero[:]) 604 return nil 605 } 606 607 // This function appends the base 62 decoded version of src into dst. 608 func fastAppendDecodeBase62(dst []byte, src []byte) []byte { 609 dst = reserve(dst, byteLength) 610 n := len(dst) 611 fastDecodeBase62(dst[n:n+byteLength], src) 612 return dst[:n+byteLength] 613 } 614 615 // Ensures that at least nbytes are available in the remaining capacity of the 616 // destination slice, if not, a new copy is made and returned by the function. 617 func reserve(dst []byte, nbytes int) []byte { 618 c := cap(dst) 619 n := len(dst) 620 621 if avail := c - n; avail < nbytes { 622 c *= 2 623 if (c - n) < nbytes { 624 c = n + nbytes 625 } 626 b := make([]byte, n, c) 627 copy(b, dst) 628 dst = b 629 } 630 631 return dst 632 } 633 634 // CompressedSet is an immutable data type which stores a set of KSUIDs. 635 type CompressedSet []byte 636 637 // Iter returns an iterator that produces all KSUIDs in the set. 638 func (set CompressedSet) Iter() CompressedSetIter { 639 return CompressedSetIter{ 640 content: set, 641 } 642 } 643 644 // String satisfies the fmt.Stringer interface, returns a human-readable string 645 // representation of the set. 646 func (set CompressedSet) String() string { 647 b := bytes.Buffer{} 648 b.WriteByte('[') 649 set.writeTo(&b) 650 b.WriteByte(']') 651 return b.String() 652 } 653 654 // GoString satisfies the fmt.GoStringer interface, returns a Go representation of 655 // the set. 656 func (set CompressedSet) GoString() string { 657 b := bytes.Buffer{} 658 b.WriteString("ksuid.CompressedSet{") 659 set.writeTo(&b) 660 b.WriteByte('}') 661 return b.String() 662 } 663 664 func (set CompressedSet) writeTo(b *bytes.Buffer) { 665 a := [27]byte{} 666 667 for i, it := 0, set.Iter(); it.Next(); i++ { 668 if i != 0 { 669 b.WriteString(", ") 670 } 671 b.WriteByte('"') 672 it.KSUID.Append(a[:0]) 673 b.Write(a[:]) 674 b.WriteByte('"') 675 } 676 } 677 678 // Compress creates and returns a compressed set of KSUIDs from the list given 679 // as arguments. 680 func Compress(ids ...KSUID) CompressedSet { 681 c := 1 + byteLength + (len(ids) / 5) 682 b := make([]byte, 0, c) 683 return AppendCompressed(b, ids...) 684 } 685 686 // AppendCompressed uses the given byte slice as pre-allocated storage space to 687 // build a KSUID set. 688 // 689 // Note that the set uses a compression technique to store the KSUIDs, so the 690 // resulting length is not 20 x len(ids). The rule of thumb here is for the given 691 // byte slice to reserve the amount of memory that the application would be OK 692 // to waste. 693 func AppendCompressed(set []byte, ids ...KSUID) CompressedSet { 694 if len(ids) != 0 { 695 if !IsSorted(ids) { 696 Sort(ids) 697 } 698 one := makeUint128(0, 1) 699 700 // The first KSUID is always written to the set, this is the starting 701 // point for all deltas. 702 set = append(set, byte(rawKSUID)) 703 set = append(set, ids[0][:]...) 704 705 timestamp := ids[0].Timestamp() 706 lastKSUID := ids[0] 707 lastPayload := uint128Payload(ids[0]) 708 709 for i := 1; i != len(ids); i++ { 710 id := ids[i] 711 712 if id == lastKSUID { 713 continue 714 } 715 716 t := id.Timestamp() 717 p := uint128Payload(id) 718 719 if t != timestamp { 720 d := t - timestamp 721 n := varintLength32(d) 722 723 set = append(set, timeDelta|byte(n)) 724 set = appendVarint32(set, d, n) 725 set = append(set, id[timestampBytesNum:]...) 726 727 timestamp = t 728 } else { 729 d := sub128(p, lastPayload) 730 731 if d != one { 732 n := varintLength128(d) 733 734 set = append(set, payloadDelta|byte(n)) 735 set = appendVarint128(set, d, n) 736 } else { 737 l, c := rangeLength(ids[i+1:], t, id, p) 738 m := uint64(l + 1) 739 n := varintLength64(m) 740 741 set = append(set, payloadRange|byte(n)) 742 set = appendVarint64(set, m, n) 743 744 i += c 745 id = ids[i] 746 p = uint128Payload(id) 747 } 748 } 749 750 lastKSUID = id 751 lastPayload = p 752 } 753 } 754 return set 755 } 756 757 func rangeLength(ids []KSUID, timestamp uint32, lastKSUID KSUID, lastValue uint128) (length int, count int) { 758 one := makeUint128(0, 1) 759 760 for i := range ids { 761 id := ids[i] 762 763 if id == lastKSUID { 764 continue 765 } 766 767 if id.Timestamp() != timestamp { 768 count = i 769 return 770 } 771 772 v := uint128Payload(id) 773 774 if sub128(v, lastValue) != one { 775 count = i 776 return 777 } 778 779 lastKSUID = id 780 lastValue = v 781 length++ 782 } 783 784 count = len(ids) 785 return 786 } 787 788 func appendVarint128(b []byte, v uint128, n int) []byte { 789 c := v.bytes() 790 return append(b, c[len(c)-n:]...) 791 } 792 793 func appendVarint64(b []byte, v uint64, n int) []byte { 794 c := [8]byte{} 795 binary.BigEndian.PutUint64(c[:], v) 796 return append(b, c[len(c)-n:]...) 797 } 798 799 func appendVarint32(b []byte, v uint32, n int) []byte { 800 c := [4]byte{} 801 binary.BigEndian.PutUint32(c[:], v) 802 return append(b, c[len(c)-n:]...) 803 } 804 805 func varint128(b []byte) uint128 { 806 a := [16]byte{} 807 copy(a[16-len(b):], b) 808 return makeUint128FromPayload(a[:]) 809 } 810 811 func varint64(b []byte) uint64 { 812 a := [8]byte{} 813 copy(a[8-len(b):], b) 814 return binary.BigEndian.Uint64(a[:]) 815 } 816 817 func varint32(b []byte) uint32 { 818 a := [4]byte{} 819 copy(a[4-len(b):], b) 820 return binary.BigEndian.Uint32(a[:]) 821 } 822 823 func varintLength128(v uint128) int { 824 if v[1] != 0 { 825 return 8 + varintLength64(v[1]) 826 } 827 return varintLength64(v[0]) 828 } 829 830 func varintLength64(v uint64) int { 831 switch { 832 case (v & 0xFFFFFFFFFFFFFF00) == 0: 833 return 1 834 case (v & 0xFFFFFFFFFFFF0000) == 0: 835 return 2 836 case (v & 0xFFFFFFFFFF000000) == 0: 837 return 3 838 case (v & 0xFFFFFFFF00000000) == 0: 839 return 4 840 case (v & 0xFFFFFF0000000000) == 0: 841 return 5 842 case (v & 0xFFFF000000000000) == 0: 843 return 6 844 case (v & 0xFF00000000000000) == 0: 845 return 7 846 default: 847 return 8 848 } 849 } 850 851 func varintLength32(v uint32) int { 852 switch { 853 case (v & 0xFFFFFF00) == 0: 854 return 1 855 case (v & 0xFFFF0000) == 0: 856 return 2 857 case (v & 0xFF000000) == 0: 858 return 3 859 default: 860 return 4 861 } 862 } 863 864 const ( 865 rawKSUID = 0 866 timeDelta = 1 << 6 867 payloadDelta = 1 << 7 868 payloadRange = (1 << 6) | (1 << 7) 869 ) 870 871 // CompressedSetIter is an iterator type returned by Set.Iter to produce the 872 // list of KSUIDs stored in a set. 873 // 874 // Here's is how the iterator type is commonly used: 875 // 876 // for it := set.Iter(); it.Next(); { 877 // id := it.KSUID 878 // // ... 879 // } 880 // 881 // CompressedSetIter values are not safe to use concurrently from multiple 882 // goroutines. 883 type CompressedSetIter struct { 884 // KSUID is modified by calls to the Next method to hold the KSUID loaded 885 // by the iterator. 886 KSUID 887 888 content []byte 889 offset int 890 891 seqLength uint64 892 timestamp uint32 893 lastValue uint128 894 } 895 896 // Next moves the iterator forward, returning true if there is a KSUID was found, 897 // or false if the iterator as reached the end of the set it was created from. 898 func (it *CompressedSetIter) Next() bool { 899 if it.seqLength != 0 { 900 value := incr128(it.lastValue) 901 it.KSUID = value.ksuid(it.timestamp) 902 it.seqLength-- 903 it.lastValue = value 904 return true 905 } 906 907 if it.offset == len(it.content) { 908 return false 909 } 910 911 b := it.content[it.offset] 912 it.offset++ 913 914 const mask = rawKSUID | timeDelta | payloadDelta | payloadRange 915 tag := int(b) & mask 916 cnt := int(b) & ^mask 917 918 switch tag { 919 case rawKSUID: 920 off0 := it.offset 921 off1 := off0 + byteLength 922 923 copy(it.KSUID[:], it.content[off0:off1]) 924 925 it.offset = off1 926 it.timestamp = it.KSUID.Timestamp() 927 it.lastValue = uint128Payload(it.KSUID) 928 929 case timeDelta: 930 off0 := it.offset 931 off1 := off0 + cnt 932 off2 := off1 + payloadBytesNum 933 934 it.timestamp += varint32(it.content[off0:off1]) 935 936 binary.BigEndian.PutUint32(it.KSUID[:timestampBytesNum], it.timestamp) 937 copy(it.KSUID[timestampBytesNum:], it.content[off1:off2]) 938 939 it.offset = off2 940 it.lastValue = uint128Payload(it.KSUID) 941 942 case payloadDelta: 943 off0 := it.offset 944 off1 := off0 + cnt 945 946 delta := varint128(it.content[off0:off1]) 947 value := add128(it.lastValue, delta) 948 949 it.KSUID = value.ksuid(it.timestamp) 950 it.offset = off1 951 it.lastValue = value 952 953 case payloadRange: 954 off0 := it.offset 955 off1 := off0 + cnt 956 957 value := incr128(it.lastValue) 958 it.KSUID = value.ksuid(it.timestamp) 959 it.seqLength = varint64(it.content[off0:off1]) 960 it.offset = off1 961 it.seqLength-- 962 it.lastValue = value 963 964 default: 965 panic("KSUID set iterator is reading malformed data") 966 } 967 968 return true 969 } 970 971 // FastRander is an io.Reader that uses math/rand and is optimized for 972 // generating 16 bytes KSUID payloads. It is intended to be used as a 973 // performance improvements for programs that have no need for 974 // cryptographically secure KSUIDs and are generating a lot of them. 975 var FastRander = newRBG() 976 977 func newRBG() io.Reader { 978 r, err := newRandomBitsGenerator() 979 if err != nil { 980 panic(err) 981 } 982 return r 983 } 984 985 func newRandomBitsGenerator() (r io.Reader, err error) { 986 var seed int64 987 988 if seed, err = readCryptoRandomSeed(); err != nil { 989 return 990 } 991 992 r = &randSourceReader{source: mrand.NewSource(seed).(mrand.Source64)} 993 return 994 } 995 996 func readCryptoRandomSeed() (seed int64, err error) { 997 var b [8]byte 998 999 if _, err = io.ReadFull(rand.Reader, b[:]); err != nil { 1000 return 1001 } 1002 1003 seed = int64(binary.LittleEndian.Uint64(b[:])) 1004 return 1005 } 1006 1007 type randSourceReader struct { 1008 source mrand.Source64 1009 } 1010 1011 func (r *randSourceReader) Read(b []byte) (int, error) { 1012 // optimized for generating 16 bytes payloads 1013 binary.LittleEndian.PutUint64(b[:8], r.source.Uint64()) 1014 binary.LittleEndian.PutUint64(b[8:], r.source.Uint64()) 1015 return 16, nil 1016 }