github.com/twelsh-aw/go/src@v0.0.0-20230516233729-a56fe86a7c81/bytes/bytes.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package bytes implements functions for the manipulation of byte slices. 6 // It is analogous to the facilities of the [strings] package. 7 package bytes 8 9 import ( 10 "internal/bytealg" 11 "unicode" 12 "unicode/utf8" 13 ) 14 15 // Equal reports whether a and b 16 // are the same length and contain the same bytes. 17 // A nil argument is equivalent to an empty slice. 18 func Equal(a, b []byte) bool { 19 // Neither cmd/compile nor gccgo allocates for these string conversions. 20 return string(a) == string(b) 21 } 22 23 // Compare returns an integer comparing two byte slices lexicographically. 24 // The result will be 0 if a == b, -1 if a < b, and +1 if a > b. 25 // A nil argument is equivalent to an empty slice. 26 func Compare(a, b []byte) int { 27 return bytealg.Compare(a, b) 28 } 29 30 // explode splits s into a slice of UTF-8 sequences, one per Unicode code point (still slices of bytes), 31 // up to a maximum of n byte slices. Invalid UTF-8 sequences are chopped into individual bytes. 32 func explode(s []byte, n int) [][]byte { 33 if n <= 0 || n > len(s) { 34 n = len(s) 35 } 36 a := make([][]byte, n) 37 var size int 38 na := 0 39 for len(s) > 0 { 40 if na+1 >= n { 41 a[na] = s 42 na++ 43 break 44 } 45 _, size = utf8.DecodeRune(s) 46 a[na] = s[0:size:size] 47 s = s[size:] 48 na++ 49 } 50 return a[0:na] 51 } 52 53 // Count counts the number of non-overlapping instances of sep in s. 54 // If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s. 55 func Count(s, sep []byte) int { 56 // special case 57 if len(sep) == 0 { 58 return utf8.RuneCount(s) + 1 59 } 60 if len(sep) == 1 { 61 return bytealg.Count(s, sep[0]) 62 } 63 n := 0 64 for { 65 i := Index(s, sep) 66 if i == -1 { 67 return n 68 } 69 n++ 70 s = s[i+len(sep):] 71 } 72 } 73 74 // Contains reports whether subslice is within b. 75 func Contains(b, subslice []byte) bool { 76 return Index(b, subslice) != -1 77 } 78 79 // ContainsAny reports whether any of the UTF-8-encoded code points in chars are within b. 80 func ContainsAny(b []byte, chars string) bool { 81 return IndexAny(b, chars) >= 0 82 } 83 84 // ContainsRune reports whether the rune is contained in the UTF-8-encoded byte slice b. 85 func ContainsRune(b []byte, r rune) bool { 86 return IndexRune(b, r) >= 0 87 } 88 89 // ContainsFunc reports whether any of the UTF-8-encoded code points r within b satisfy f(r). 90 func ContainsFunc(b []byte, f func(rune) bool) bool { 91 return IndexFunc(b, f) >= 0 92 } 93 94 // IndexByte returns the index of the first instance of c in b, or -1 if c is not present in b. 95 func IndexByte(b []byte, c byte) int { 96 return bytealg.IndexByte(b, c) 97 } 98 99 func indexBytePortable(s []byte, c byte) int { 100 for i, b := range s { 101 if b == c { 102 return i 103 } 104 } 105 return -1 106 } 107 108 // LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. 109 func LastIndex(s, sep []byte) int { 110 n := len(sep) 111 switch { 112 case n == 0: 113 return len(s) 114 case n == 1: 115 return LastIndexByte(s, sep[0]) 116 case n == len(s): 117 if Equal(s, sep) { 118 return 0 119 } 120 return -1 121 case n > len(s): 122 return -1 123 } 124 // Rabin-Karp search from the end of the string 125 hashss, pow := bytealg.HashStrRevBytes(sep) 126 last := len(s) - n 127 var h uint32 128 for i := len(s) - 1; i >= last; i-- { 129 h = h*bytealg.PrimeRK + uint32(s[i]) 130 } 131 if h == hashss && Equal(s[last:], sep) { 132 return last 133 } 134 for i := last - 1; i >= 0; i-- { 135 h *= bytealg.PrimeRK 136 h += uint32(s[i]) 137 h -= pow * uint32(s[i+n]) 138 if h == hashss && Equal(s[i:i+n], sep) { 139 return i 140 } 141 } 142 return -1 143 } 144 145 // LastIndexByte returns the index of the last instance of c in s, or -1 if c is not present in s. 146 func LastIndexByte(s []byte, c byte) int { 147 for i := len(s) - 1; i >= 0; i-- { 148 if s[i] == c { 149 return i 150 } 151 } 152 return -1 153 } 154 155 // IndexRune interprets s as a sequence of UTF-8-encoded code points. 156 // It returns the byte index of the first occurrence in s of the given rune. 157 // It returns -1 if rune is not present in s. 158 // If r is utf8.RuneError, it returns the first instance of any 159 // invalid UTF-8 byte sequence. 160 func IndexRune(s []byte, r rune) int { 161 switch { 162 case 0 <= r && r < utf8.RuneSelf: 163 return IndexByte(s, byte(r)) 164 case r == utf8.RuneError: 165 for i := 0; i < len(s); { 166 r1, n := utf8.DecodeRune(s[i:]) 167 if r1 == utf8.RuneError { 168 return i 169 } 170 i += n 171 } 172 return -1 173 case !utf8.ValidRune(r): 174 return -1 175 default: 176 var b [utf8.UTFMax]byte 177 n := utf8.EncodeRune(b[:], r) 178 return Index(s, b[:n]) 179 } 180 } 181 182 // IndexAny interprets s as a sequence of UTF-8-encoded Unicode code points. 183 // It returns the byte index of the first occurrence in s of any of the Unicode 184 // code points in chars. It returns -1 if chars is empty or if there is no code 185 // point in common. 186 func IndexAny(s []byte, chars string) int { 187 if chars == "" { 188 // Avoid scanning all of s. 189 return -1 190 } 191 if len(s) == 1 { 192 r := rune(s[0]) 193 if r >= utf8.RuneSelf { 194 // search utf8.RuneError. 195 for _, r = range chars { 196 if r == utf8.RuneError { 197 return 0 198 } 199 } 200 return -1 201 } 202 if bytealg.IndexByteString(chars, s[0]) >= 0 { 203 return 0 204 } 205 return -1 206 } 207 if len(chars) == 1 { 208 r := rune(chars[0]) 209 if r >= utf8.RuneSelf { 210 r = utf8.RuneError 211 } 212 return IndexRune(s, r) 213 } 214 if len(s) > 8 { 215 if as, isASCII := makeASCIISet(chars); isASCII { 216 for i, c := range s { 217 if as.contains(c) { 218 return i 219 } 220 } 221 return -1 222 } 223 } 224 var width int 225 for i := 0; i < len(s); i += width { 226 r := rune(s[i]) 227 if r < utf8.RuneSelf { 228 if bytealg.IndexByteString(chars, s[i]) >= 0 { 229 return i 230 } 231 width = 1 232 continue 233 } 234 r, width = utf8.DecodeRune(s[i:]) 235 if r != utf8.RuneError { 236 // r is 2 to 4 bytes 237 if len(chars) == width { 238 if chars == string(r) { 239 return i 240 } 241 continue 242 } 243 // Use bytealg.IndexString for performance if available. 244 if bytealg.MaxLen >= width { 245 if bytealg.IndexString(chars, string(r)) >= 0 { 246 return i 247 } 248 continue 249 } 250 } 251 for _, ch := range chars { 252 if r == ch { 253 return i 254 } 255 } 256 } 257 return -1 258 } 259 260 // LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code 261 // points. It returns the byte index of the last occurrence in s of any of 262 // the Unicode code points in chars. It returns -1 if chars is empty or if 263 // there is no code point in common. 264 func LastIndexAny(s []byte, chars string) int { 265 if chars == "" { 266 // Avoid scanning all of s. 267 return -1 268 } 269 if len(s) > 8 { 270 if as, isASCII := makeASCIISet(chars); isASCII { 271 for i := len(s) - 1; i >= 0; i-- { 272 if as.contains(s[i]) { 273 return i 274 } 275 } 276 return -1 277 } 278 } 279 if len(s) == 1 { 280 r := rune(s[0]) 281 if r >= utf8.RuneSelf { 282 for _, r = range chars { 283 if r == utf8.RuneError { 284 return 0 285 } 286 } 287 return -1 288 } 289 if bytealg.IndexByteString(chars, s[0]) >= 0 { 290 return 0 291 } 292 return -1 293 } 294 if len(chars) == 1 { 295 cr := rune(chars[0]) 296 if cr >= utf8.RuneSelf { 297 cr = utf8.RuneError 298 } 299 for i := len(s); i > 0; { 300 r, size := utf8.DecodeLastRune(s[:i]) 301 i -= size 302 if r == cr { 303 return i 304 } 305 } 306 return -1 307 } 308 for i := len(s); i > 0; { 309 r := rune(s[i-1]) 310 if r < utf8.RuneSelf { 311 if bytealg.IndexByteString(chars, s[i-1]) >= 0 { 312 return i - 1 313 } 314 i-- 315 continue 316 } 317 r, size := utf8.DecodeLastRune(s[:i]) 318 i -= size 319 if r != utf8.RuneError { 320 // r is 2 to 4 bytes 321 if len(chars) == size { 322 if chars == string(r) { 323 return i 324 } 325 continue 326 } 327 // Use bytealg.IndexString for performance if available. 328 if bytealg.MaxLen >= size { 329 if bytealg.IndexString(chars, string(r)) >= 0 { 330 return i 331 } 332 continue 333 } 334 } 335 for _, ch := range chars { 336 if r == ch { 337 return i 338 } 339 } 340 } 341 return -1 342 } 343 344 // Generic split: splits after each instance of sep, 345 // including sepSave bytes of sep in the subslices. 346 func genSplit(s, sep []byte, sepSave, n int) [][]byte { 347 if n == 0 { 348 return nil 349 } 350 if len(sep) == 0 { 351 return explode(s, n) 352 } 353 if n < 0 { 354 n = Count(s, sep) + 1 355 } 356 if n > len(s)+1 { 357 n = len(s) + 1 358 } 359 360 a := make([][]byte, n) 361 n-- 362 i := 0 363 for i < n { 364 m := Index(s, sep) 365 if m < 0 { 366 break 367 } 368 a[i] = s[: m+sepSave : m+sepSave] 369 s = s[m+len(sep):] 370 i++ 371 } 372 a[i] = s 373 return a[:i+1] 374 } 375 376 // SplitN slices s into subslices separated by sep and returns a slice of 377 // the subslices between those separators. 378 // If sep is empty, SplitN splits after each UTF-8 sequence. 379 // The count determines the number of subslices to return: 380 // 381 // n > 0: at most n subslices; the last subslice will be the unsplit remainder. 382 // n == 0: the result is nil (zero subslices) 383 // n < 0: all subslices 384 // 385 // To split around the first instance of a separator, see Cut. 386 func SplitN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } 387 388 // SplitAfterN slices s into subslices after each instance of sep and 389 // returns a slice of those subslices. 390 // If sep is empty, SplitAfterN splits after each UTF-8 sequence. 391 // The count determines the number of subslices to return: 392 // 393 // n > 0: at most n subslices; the last subslice will be the unsplit remainder. 394 // n == 0: the result is nil (zero subslices) 395 // n < 0: all subslices 396 func SplitAfterN(s, sep []byte, n int) [][]byte { 397 return genSplit(s, sep, len(sep), n) 398 } 399 400 // Split slices s into all subslices separated by sep and returns a slice of 401 // the subslices between those separators. 402 // If sep is empty, Split splits after each UTF-8 sequence. 403 // It is equivalent to SplitN with a count of -1. 404 // 405 // To split around the first instance of a separator, see Cut. 406 func Split(s, sep []byte) [][]byte { return genSplit(s, sep, 0, -1) } 407 408 // SplitAfter slices s into all subslices after each instance of sep and 409 // returns a slice of those subslices. 410 // If sep is empty, SplitAfter splits after each UTF-8 sequence. 411 // It is equivalent to SplitAfterN with a count of -1. 412 func SplitAfter(s, sep []byte) [][]byte { 413 return genSplit(s, sep, len(sep), -1) 414 } 415 416 var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} 417 418 // Fields interprets s as a sequence of UTF-8-encoded code points. 419 // It splits the slice s around each instance of one or more consecutive white space 420 // characters, as defined by unicode.IsSpace, returning a slice of subslices of s or an 421 // empty slice if s contains only white space. 422 func Fields(s []byte) [][]byte { 423 // First count the fields. 424 // This is an exact count if s is ASCII, otherwise it is an approximation. 425 n := 0 426 wasSpace := 1 427 // setBits is used to track which bits are set in the bytes of s. 428 setBits := uint8(0) 429 for i := 0; i < len(s); i++ { 430 r := s[i] 431 setBits |= r 432 isSpace := int(asciiSpace[r]) 433 n += wasSpace & ^isSpace 434 wasSpace = isSpace 435 } 436 437 if setBits >= utf8.RuneSelf { 438 // Some runes in the input slice are not ASCII. 439 return FieldsFunc(s, unicode.IsSpace) 440 } 441 442 // ASCII fast path 443 a := make([][]byte, n) 444 na := 0 445 fieldStart := 0 446 i := 0 447 // Skip spaces in the front of the input. 448 for i < len(s) && asciiSpace[s[i]] != 0 { 449 i++ 450 } 451 fieldStart = i 452 for i < len(s) { 453 if asciiSpace[s[i]] == 0 { 454 i++ 455 continue 456 } 457 a[na] = s[fieldStart:i:i] 458 na++ 459 i++ 460 // Skip spaces in between fields. 461 for i < len(s) && asciiSpace[s[i]] != 0 { 462 i++ 463 } 464 fieldStart = i 465 } 466 if fieldStart < len(s) { // Last field might end at EOF. 467 a[na] = s[fieldStart:len(s):len(s)] 468 } 469 return a 470 } 471 472 // FieldsFunc interprets s as a sequence of UTF-8-encoded code points. 473 // It splits the slice s at each run of code points c satisfying f(c) and 474 // returns a slice of subslices of s. If all code points in s satisfy f(c), or 475 // len(s) == 0, an empty slice is returned. 476 // 477 // FieldsFunc makes no guarantees about the order in which it calls f(c) 478 // and assumes that f always returns the same value for a given c. 479 func FieldsFunc(s []byte, f func(rune) bool) [][]byte { 480 // A span is used to record a slice of s of the form s[start:end]. 481 // The start index is inclusive and the end index is exclusive. 482 type span struct { 483 start int 484 end int 485 } 486 spans := make([]span, 0, 32) 487 488 // Find the field start and end indices. 489 // Doing this in a separate pass (rather than slicing the string s 490 // and collecting the result substrings right away) is significantly 491 // more efficient, possibly due to cache effects. 492 start := -1 // valid span start if >= 0 493 for i := 0; i < len(s); { 494 size := 1 495 r := rune(s[i]) 496 if r >= utf8.RuneSelf { 497 r, size = utf8.DecodeRune(s[i:]) 498 } 499 if f(r) { 500 if start >= 0 { 501 spans = append(spans, span{start, i}) 502 start = -1 503 } 504 } else { 505 if start < 0 { 506 start = i 507 } 508 } 509 i += size 510 } 511 512 // Last field might end at EOF. 513 if start >= 0 { 514 spans = append(spans, span{start, len(s)}) 515 } 516 517 // Create subslices from recorded field indices. 518 a := make([][]byte, len(spans)) 519 for i, span := range spans { 520 a[i] = s[span.start:span.end:span.end] 521 } 522 523 return a 524 } 525 526 // Join concatenates the elements of s to create a new byte slice. The separator 527 // sep is placed between elements in the resulting slice. 528 func Join(s [][]byte, sep []byte) []byte { 529 if len(s) == 0 { 530 return []byte{} 531 } 532 if len(s) == 1 { 533 // Just return a copy. 534 return append([]byte(nil), s[0]...) 535 } 536 537 var n int 538 if len(sep) > 0 { 539 if len(sep) >= maxInt/(len(s)-1) { 540 panic("bytes: Join output length overflow") 541 } 542 n += len(sep) * (len(s) - 1) 543 } 544 for _, v := range s { 545 if len(v) > maxInt-n { 546 panic("bytes: Join output length overflow") 547 } 548 n += len(v) 549 } 550 551 b := bytealg.MakeNoZero(n) 552 bp := copy(b, s[0]) 553 for _, v := range s[1:] { 554 bp += copy(b[bp:], sep) 555 bp += copy(b[bp:], v) 556 } 557 return b 558 } 559 560 // HasPrefix tests whether the byte slice s begins with prefix. 561 func HasPrefix(s, prefix []byte) bool { 562 return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) 563 } 564 565 // HasSuffix tests whether the byte slice s ends with suffix. 566 func HasSuffix(s, suffix []byte) bool { 567 return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) 568 } 569 570 // Map returns a copy of the byte slice s with all its characters modified 571 // according to the mapping function. If mapping returns a negative value, the character is 572 // dropped from the byte slice with no replacement. The characters in s and the 573 // output are interpreted as UTF-8-encoded code points. 574 func Map(mapping func(r rune) rune, s []byte) []byte { 575 // In the worst case, the slice can grow when mapped, making 576 // things unpleasant. But it's so rare we barge in assuming it's 577 // fine. It could also shrink but that falls out naturally. 578 b := make([]byte, 0, len(s)) 579 for i := 0; i < len(s); { 580 wid := 1 581 r := rune(s[i]) 582 if r >= utf8.RuneSelf { 583 r, wid = utf8.DecodeRune(s[i:]) 584 } 585 r = mapping(r) 586 if r >= 0 { 587 b = utf8.AppendRune(b, r) 588 } 589 i += wid 590 } 591 return b 592 } 593 594 // Repeat returns a new byte slice consisting of count copies of b. 595 // 596 // It panics if count is negative or if the result of (len(b) * count) 597 // overflows. 598 func Repeat(b []byte, count int) []byte { 599 if count == 0 { 600 return []byte{} 601 } 602 603 // Since we cannot return an error on overflow, 604 // we should panic if the repeat will generate an overflow. 605 // See golang.org/issue/16237. 606 if count < 0 { 607 panic("bytes: negative Repeat count") 608 } 609 if len(b) >= maxInt/count { 610 panic("bytes: Repeat output length overflow") 611 } 612 n := len(b) * count 613 614 if len(b) == 0 { 615 return []byte{} 616 } 617 618 // Past a certain chunk size it is counterproductive to use 619 // larger chunks as the source of the write, as when the source 620 // is too large we are basically just thrashing the CPU D-cache. 621 // So if the result length is larger than an empirically-found 622 // limit (8KB), we stop growing the source string once the limit 623 // is reached and keep reusing the same source string - that 624 // should therefore be always resident in the L1 cache - until we 625 // have completed the construction of the result. 626 // This yields significant speedups (up to +100%) in cases where 627 // the result length is large (roughly, over L2 cache size). 628 const chunkLimit = 8 * 1024 629 chunkMax := n 630 if chunkMax > chunkLimit { 631 chunkMax = chunkLimit / len(b) * len(b) 632 if chunkMax == 0 { 633 chunkMax = len(b) 634 } 635 } 636 nb := bytealg.MakeNoZero(n) 637 bp := copy(nb, b) 638 for bp < n { 639 chunk := bp 640 if chunk > chunkMax { 641 chunk = chunkMax 642 } 643 bp += copy(nb[bp:], nb[:chunk]) 644 } 645 return nb 646 } 647 648 // ToUpper returns a copy of the byte slice s with all Unicode letters mapped to 649 // their upper case. 650 func ToUpper(s []byte) []byte { 651 isASCII, hasLower := true, false 652 for i := 0; i < len(s); i++ { 653 c := s[i] 654 if c >= utf8.RuneSelf { 655 isASCII = false 656 break 657 } 658 hasLower = hasLower || ('a' <= c && c <= 'z') 659 } 660 661 if isASCII { // optimize for ASCII-only byte slices. 662 if !hasLower { 663 // Just return a copy. 664 return append([]byte(""), s...) 665 } 666 b := bytealg.MakeNoZero(len(s)) 667 for i := 0; i < len(s); i++ { 668 c := s[i] 669 if 'a' <= c && c <= 'z' { 670 c -= 'a' - 'A' 671 } 672 b[i] = c 673 } 674 return b 675 } 676 return Map(unicode.ToUpper, s) 677 } 678 679 // ToLower returns a copy of the byte slice s with all Unicode letters mapped to 680 // their lower case. 681 func ToLower(s []byte) []byte { 682 isASCII, hasUpper := true, false 683 for i := 0; i < len(s); i++ { 684 c := s[i] 685 if c >= utf8.RuneSelf { 686 isASCII = false 687 break 688 } 689 hasUpper = hasUpper || ('A' <= c && c <= 'Z') 690 } 691 692 if isASCII { // optimize for ASCII-only byte slices. 693 if !hasUpper { 694 return append([]byte(""), s...) 695 } 696 b := bytealg.MakeNoZero(len(s)) 697 for i := 0; i < len(s); i++ { 698 c := s[i] 699 if 'A' <= c && c <= 'Z' { 700 c += 'a' - 'A' 701 } 702 b[i] = c 703 } 704 return b 705 } 706 return Map(unicode.ToLower, s) 707 } 708 709 // ToTitle treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their title case. 710 func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) } 711 712 // ToUpperSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their 713 // upper case, giving priority to the special casing rules. 714 func ToUpperSpecial(c unicode.SpecialCase, s []byte) []byte { 715 return Map(c.ToUpper, s) 716 } 717 718 // ToLowerSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their 719 // lower case, giving priority to the special casing rules. 720 func ToLowerSpecial(c unicode.SpecialCase, s []byte) []byte { 721 return Map(c.ToLower, s) 722 } 723 724 // ToTitleSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their 725 // title case, giving priority to the special casing rules. 726 func ToTitleSpecial(c unicode.SpecialCase, s []byte) []byte { 727 return Map(c.ToTitle, s) 728 } 729 730 // ToValidUTF8 treats s as UTF-8-encoded bytes and returns a copy with each run of bytes 731 // representing invalid UTF-8 replaced with the bytes in replacement, which may be empty. 732 func ToValidUTF8(s, replacement []byte) []byte { 733 b := make([]byte, 0, len(s)+len(replacement)) 734 invalid := false // previous byte was from an invalid UTF-8 sequence 735 for i := 0; i < len(s); { 736 c := s[i] 737 if c < utf8.RuneSelf { 738 i++ 739 invalid = false 740 b = append(b, c) 741 continue 742 } 743 _, wid := utf8.DecodeRune(s[i:]) 744 if wid == 1 { 745 i++ 746 if !invalid { 747 invalid = true 748 b = append(b, replacement...) 749 } 750 continue 751 } 752 invalid = false 753 b = append(b, s[i:i+wid]...) 754 i += wid 755 } 756 return b 757 } 758 759 // isSeparator reports whether the rune could mark a word boundary. 760 // TODO: update when package unicode captures more of the properties. 761 func isSeparator(r rune) bool { 762 // ASCII alphanumerics and underscore are not separators 763 if r <= 0x7F { 764 switch { 765 case '0' <= r && r <= '9': 766 return false 767 case 'a' <= r && r <= 'z': 768 return false 769 case 'A' <= r && r <= 'Z': 770 return false 771 case r == '_': 772 return false 773 } 774 return true 775 } 776 // Letters and digits are not separators 777 if unicode.IsLetter(r) || unicode.IsDigit(r) { 778 return false 779 } 780 // Otherwise, all we can do for now is treat spaces as separators. 781 return unicode.IsSpace(r) 782 } 783 784 // Title treats s as UTF-8-encoded bytes and returns a copy with all Unicode letters that begin 785 // words mapped to their title case. 786 // 787 // Deprecated: The rule Title uses for word boundaries does not handle Unicode 788 // punctuation properly. Use golang.org/x/text/cases instead. 789 func Title(s []byte) []byte { 790 // Use a closure here to remember state. 791 // Hackish but effective. Depends on Map scanning in order and calling 792 // the closure once per rune. 793 prev := ' ' 794 return Map( 795 func(r rune) rune { 796 if isSeparator(prev) { 797 prev = r 798 return unicode.ToTitle(r) 799 } 800 prev = r 801 return r 802 }, 803 s) 804 } 805 806 // TrimLeftFunc treats s as UTF-8-encoded bytes and returns a subslice of s by slicing off 807 // all leading UTF-8-encoded code points c that satisfy f(c). 808 func TrimLeftFunc(s []byte, f func(r rune) bool) []byte { 809 i := indexFunc(s, f, false) 810 if i == -1 { 811 return nil 812 } 813 return s[i:] 814 } 815 816 // TrimRightFunc returns a subslice of s by slicing off all trailing 817 // UTF-8-encoded code points c that satisfy f(c). 818 func TrimRightFunc(s []byte, f func(r rune) bool) []byte { 819 i := lastIndexFunc(s, f, false) 820 if i >= 0 && s[i] >= utf8.RuneSelf { 821 _, wid := utf8.DecodeRune(s[i:]) 822 i += wid 823 } else { 824 i++ 825 } 826 return s[0:i] 827 } 828 829 // TrimFunc returns a subslice of s by slicing off all leading and trailing 830 // UTF-8-encoded code points c that satisfy f(c). 831 func TrimFunc(s []byte, f func(r rune) bool) []byte { 832 return TrimRightFunc(TrimLeftFunc(s, f), f) 833 } 834 835 // TrimPrefix returns s without the provided leading prefix string. 836 // If s doesn't start with prefix, s is returned unchanged. 837 func TrimPrefix(s, prefix []byte) []byte { 838 if HasPrefix(s, prefix) { 839 return s[len(prefix):] 840 } 841 return s 842 } 843 844 // TrimSuffix returns s without the provided trailing suffix string. 845 // If s doesn't end with suffix, s is returned unchanged. 846 func TrimSuffix(s, suffix []byte) []byte { 847 if HasSuffix(s, suffix) { 848 return s[:len(s)-len(suffix)] 849 } 850 return s 851 } 852 853 // IndexFunc interprets s as a sequence of UTF-8-encoded code points. 854 // It returns the byte index in s of the first Unicode 855 // code point satisfying f(c), or -1 if none do. 856 func IndexFunc(s []byte, f func(r rune) bool) int { 857 return indexFunc(s, f, true) 858 } 859 860 // LastIndexFunc interprets s as a sequence of UTF-8-encoded code points. 861 // It returns the byte index in s of the last Unicode 862 // code point satisfying f(c), or -1 if none do. 863 func LastIndexFunc(s []byte, f func(r rune) bool) int { 864 return lastIndexFunc(s, f, true) 865 } 866 867 // indexFunc is the same as IndexFunc except that if 868 // truth==false, the sense of the predicate function is 869 // inverted. 870 func indexFunc(s []byte, f func(r rune) bool, truth bool) int { 871 start := 0 872 for start < len(s) { 873 wid := 1 874 r := rune(s[start]) 875 if r >= utf8.RuneSelf { 876 r, wid = utf8.DecodeRune(s[start:]) 877 } 878 if f(r) == truth { 879 return start 880 } 881 start += wid 882 } 883 return -1 884 } 885 886 // lastIndexFunc is the same as LastIndexFunc except that if 887 // truth==false, the sense of the predicate function is 888 // inverted. 889 func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int { 890 for i := len(s); i > 0; { 891 r, size := rune(s[i-1]), 1 892 if r >= utf8.RuneSelf { 893 r, size = utf8.DecodeLastRune(s[0:i]) 894 } 895 i -= size 896 if f(r) == truth { 897 return i 898 } 899 } 900 return -1 901 } 902 903 // asciiSet is a 32-byte value, where each bit represents the presence of a 904 // given ASCII character in the set. The 128-bits of the lower 16 bytes, 905 // starting with the least-significant bit of the lowest word to the 906 // most-significant bit of the highest word, map to the full range of all 907 // 128 ASCII characters. The 128-bits of the upper 16 bytes will be zeroed, 908 // ensuring that any non-ASCII character will be reported as not in the set. 909 // This allocates a total of 32 bytes even though the upper half 910 // is unused to avoid bounds checks in asciiSet.contains. 911 type asciiSet [8]uint32 912 913 // makeASCIISet creates a set of ASCII characters and reports whether all 914 // characters in chars are ASCII. 915 func makeASCIISet(chars string) (as asciiSet, ok bool) { 916 for i := 0; i < len(chars); i++ { 917 c := chars[i] 918 if c >= utf8.RuneSelf { 919 return as, false 920 } 921 as[c/32] |= 1 << (c % 32) 922 } 923 return as, true 924 } 925 926 // contains reports whether c is inside the set. 927 func (as *asciiSet) contains(c byte) bool { 928 return (as[c/32] & (1 << (c % 32))) != 0 929 } 930 931 // containsRune is a simplified version of strings.ContainsRune 932 // to avoid importing the strings package. 933 // We avoid bytes.ContainsRune to avoid allocating a temporary copy of s. 934 func containsRune(s string, r rune) bool { 935 for _, c := range s { 936 if c == r { 937 return true 938 } 939 } 940 return false 941 } 942 943 // Trim returns a subslice of s by slicing off all leading and 944 // trailing UTF-8-encoded code points contained in cutset. 945 func Trim(s []byte, cutset string) []byte { 946 if len(s) == 0 { 947 // This is what we've historically done. 948 return nil 949 } 950 if cutset == "" { 951 return s 952 } 953 if len(cutset) == 1 && cutset[0] < utf8.RuneSelf { 954 return trimLeftByte(trimRightByte(s, cutset[0]), cutset[0]) 955 } 956 if as, ok := makeASCIISet(cutset); ok { 957 return trimLeftASCII(trimRightASCII(s, &as), &as) 958 } 959 return trimLeftUnicode(trimRightUnicode(s, cutset), cutset) 960 } 961 962 // TrimLeft returns a subslice of s by slicing off all leading 963 // UTF-8-encoded code points contained in cutset. 964 func TrimLeft(s []byte, cutset string) []byte { 965 if len(s) == 0 { 966 // This is what we've historically done. 967 return nil 968 } 969 if cutset == "" { 970 return s 971 } 972 if len(cutset) == 1 && cutset[0] < utf8.RuneSelf { 973 return trimLeftByte(s, cutset[0]) 974 } 975 if as, ok := makeASCIISet(cutset); ok { 976 return trimLeftASCII(s, &as) 977 } 978 return trimLeftUnicode(s, cutset) 979 } 980 981 func trimLeftByte(s []byte, c byte) []byte { 982 for len(s) > 0 && s[0] == c { 983 s = s[1:] 984 } 985 if len(s) == 0 { 986 // This is what we've historically done. 987 return nil 988 } 989 return s 990 } 991 992 func trimLeftASCII(s []byte, as *asciiSet) []byte { 993 for len(s) > 0 { 994 if !as.contains(s[0]) { 995 break 996 } 997 s = s[1:] 998 } 999 if len(s) == 0 { 1000 // This is what we've historically done. 1001 return nil 1002 } 1003 return s 1004 } 1005 1006 func trimLeftUnicode(s []byte, cutset string) []byte { 1007 for len(s) > 0 { 1008 r, n := rune(s[0]), 1 1009 if r >= utf8.RuneSelf { 1010 r, n = utf8.DecodeRune(s) 1011 } 1012 if !containsRune(cutset, r) { 1013 break 1014 } 1015 s = s[n:] 1016 } 1017 if len(s) == 0 { 1018 // This is what we've historically done. 1019 return nil 1020 } 1021 return s 1022 } 1023 1024 // TrimRight returns a subslice of s by slicing off all trailing 1025 // UTF-8-encoded code points that are contained in cutset. 1026 func TrimRight(s []byte, cutset string) []byte { 1027 if len(s) == 0 || cutset == "" { 1028 return s 1029 } 1030 if len(cutset) == 1 && cutset[0] < utf8.RuneSelf { 1031 return trimRightByte(s, cutset[0]) 1032 } 1033 if as, ok := makeASCIISet(cutset); ok { 1034 return trimRightASCII(s, &as) 1035 } 1036 return trimRightUnicode(s, cutset) 1037 } 1038 1039 func trimRightByte(s []byte, c byte) []byte { 1040 for len(s) > 0 && s[len(s)-1] == c { 1041 s = s[:len(s)-1] 1042 } 1043 return s 1044 } 1045 1046 func trimRightASCII(s []byte, as *asciiSet) []byte { 1047 for len(s) > 0 { 1048 if !as.contains(s[len(s)-1]) { 1049 break 1050 } 1051 s = s[:len(s)-1] 1052 } 1053 return s 1054 } 1055 1056 func trimRightUnicode(s []byte, cutset string) []byte { 1057 for len(s) > 0 { 1058 r, n := rune(s[len(s)-1]), 1 1059 if r >= utf8.RuneSelf { 1060 r, n = utf8.DecodeLastRune(s) 1061 } 1062 if !containsRune(cutset, r) { 1063 break 1064 } 1065 s = s[:len(s)-n] 1066 } 1067 return s 1068 } 1069 1070 // TrimSpace returns a subslice of s by slicing off all leading and 1071 // trailing white space, as defined by Unicode. 1072 func TrimSpace(s []byte) []byte { 1073 // Fast path for ASCII: look for the first ASCII non-space byte 1074 start := 0 1075 for ; start < len(s); start++ { 1076 c := s[start] 1077 if c >= utf8.RuneSelf { 1078 // If we run into a non-ASCII byte, fall back to the 1079 // slower unicode-aware method on the remaining bytes 1080 return TrimFunc(s[start:], unicode.IsSpace) 1081 } 1082 if asciiSpace[c] == 0 { 1083 break 1084 } 1085 } 1086 1087 // Now look for the first ASCII non-space byte from the end 1088 stop := len(s) 1089 for ; stop > start; stop-- { 1090 c := s[stop-1] 1091 if c >= utf8.RuneSelf { 1092 return TrimFunc(s[start:stop], unicode.IsSpace) 1093 } 1094 if asciiSpace[c] == 0 { 1095 break 1096 } 1097 } 1098 1099 // At this point s[start:stop] starts and ends with an ASCII 1100 // non-space bytes, so we're done. Non-ASCII cases have already 1101 // been handled above. 1102 if start == stop { 1103 // Special case to preserve previous TrimLeftFunc behavior, 1104 // returning nil instead of empty slice if all spaces. 1105 return nil 1106 } 1107 return s[start:stop] 1108 } 1109 1110 // Runes interprets s as a sequence of UTF-8-encoded code points. 1111 // It returns a slice of runes (Unicode code points) equivalent to s. 1112 func Runes(s []byte) []rune { 1113 t := make([]rune, utf8.RuneCount(s)) 1114 i := 0 1115 for len(s) > 0 { 1116 r, l := utf8.DecodeRune(s) 1117 t[i] = r 1118 i++ 1119 s = s[l:] 1120 } 1121 return t 1122 } 1123 1124 // Replace returns a copy of the slice s with the first n 1125 // non-overlapping instances of old replaced by new. 1126 // If old is empty, it matches at the beginning of the slice 1127 // and after each UTF-8 sequence, yielding up to k+1 replacements 1128 // for a k-rune slice. 1129 // If n < 0, there is no limit on the number of replacements. 1130 func Replace(s, old, new []byte, n int) []byte { 1131 m := 0 1132 if n != 0 { 1133 // Compute number of replacements. 1134 m = Count(s, old) 1135 } 1136 if m == 0 { 1137 // Just return a copy. 1138 return append([]byte(nil), s...) 1139 } 1140 if n < 0 || m < n { 1141 n = m 1142 } 1143 1144 // Apply replacements to buffer. 1145 t := make([]byte, len(s)+n*(len(new)-len(old))) 1146 w := 0 1147 start := 0 1148 for i := 0; i < n; i++ { 1149 j := start 1150 if len(old) == 0 { 1151 if i > 0 { 1152 _, wid := utf8.DecodeRune(s[start:]) 1153 j += wid 1154 } 1155 } else { 1156 j += Index(s[start:], old) 1157 } 1158 w += copy(t[w:], s[start:j]) 1159 w += copy(t[w:], new) 1160 start = j + len(old) 1161 } 1162 w += copy(t[w:], s[start:]) 1163 return t[0:w] 1164 } 1165 1166 // ReplaceAll returns a copy of the slice s with all 1167 // non-overlapping instances of old replaced by new. 1168 // If old is empty, it matches at the beginning of the slice 1169 // and after each UTF-8 sequence, yielding up to k+1 replacements 1170 // for a k-rune slice. 1171 func ReplaceAll(s, old, new []byte) []byte { 1172 return Replace(s, old, new, -1) 1173 } 1174 1175 // EqualFold reports whether s and t, interpreted as UTF-8 strings, 1176 // are equal under simple Unicode case-folding, which is a more general 1177 // form of case-insensitivity. 1178 func EqualFold(s, t []byte) bool { 1179 // ASCII fast path 1180 i := 0 1181 for ; i < len(s) && i < len(t); i++ { 1182 sr := s[i] 1183 tr := t[i] 1184 if sr|tr >= utf8.RuneSelf { 1185 goto hasUnicode 1186 } 1187 1188 // Easy case. 1189 if tr == sr { 1190 continue 1191 } 1192 1193 // Make sr < tr to simplify what follows. 1194 if tr < sr { 1195 tr, sr = sr, tr 1196 } 1197 // ASCII only, sr/tr must be upper/lower case 1198 if 'A' <= sr && sr <= 'Z' && tr == sr+'a'-'A' { 1199 continue 1200 } 1201 return false 1202 } 1203 // Check if we've exhausted both strings. 1204 return len(s) == len(t) 1205 1206 hasUnicode: 1207 s = s[i:] 1208 t = t[i:] 1209 for len(s) != 0 && len(t) != 0 { 1210 // Extract first rune from each. 1211 var sr, tr rune 1212 if s[0] < utf8.RuneSelf { 1213 sr, s = rune(s[0]), s[1:] 1214 } else { 1215 r, size := utf8.DecodeRune(s) 1216 sr, s = r, s[size:] 1217 } 1218 if t[0] < utf8.RuneSelf { 1219 tr, t = rune(t[0]), t[1:] 1220 } else { 1221 r, size := utf8.DecodeRune(t) 1222 tr, t = r, t[size:] 1223 } 1224 1225 // If they match, keep going; if not, return false. 1226 1227 // Easy case. 1228 if tr == sr { 1229 continue 1230 } 1231 1232 // Make sr < tr to simplify what follows. 1233 if tr < sr { 1234 tr, sr = sr, tr 1235 } 1236 // Fast check for ASCII. 1237 if tr < utf8.RuneSelf { 1238 // ASCII only, sr/tr must be upper/lower case 1239 if 'A' <= sr && sr <= 'Z' && tr == sr+'a'-'A' { 1240 continue 1241 } 1242 return false 1243 } 1244 1245 // General case. SimpleFold(x) returns the next equivalent rune > x 1246 // or wraps around to smaller values. 1247 r := unicode.SimpleFold(sr) 1248 for r != sr && r < tr { 1249 r = unicode.SimpleFold(r) 1250 } 1251 if r == tr { 1252 continue 1253 } 1254 return false 1255 } 1256 1257 // One string is empty. Are both? 1258 return len(s) == len(t) 1259 } 1260 1261 // Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. 1262 func Index(s, sep []byte) int { 1263 n := len(sep) 1264 switch { 1265 case n == 0: 1266 return 0 1267 case n == 1: 1268 return IndexByte(s, sep[0]) 1269 case n == len(s): 1270 if Equal(sep, s) { 1271 return 0 1272 } 1273 return -1 1274 case n > len(s): 1275 return -1 1276 case n <= bytealg.MaxLen: 1277 // Use brute force when s and sep both are small 1278 if len(s) <= bytealg.MaxBruteForce { 1279 return bytealg.Index(s, sep) 1280 } 1281 c0 := sep[0] 1282 c1 := sep[1] 1283 i := 0 1284 t := len(s) - n + 1 1285 fails := 0 1286 for i < t { 1287 if s[i] != c0 { 1288 // IndexByte is faster than bytealg.Index, so use it as long as 1289 // we're not getting lots of false positives. 1290 o := IndexByte(s[i+1:t], c0) 1291 if o < 0 { 1292 return -1 1293 } 1294 i += o + 1 1295 } 1296 if s[i+1] == c1 && Equal(s[i:i+n], sep) { 1297 return i 1298 } 1299 fails++ 1300 i++ 1301 // Switch to bytealg.Index when IndexByte produces too many false positives. 1302 if fails > bytealg.Cutover(i) { 1303 r := bytealg.Index(s[i:], sep) 1304 if r >= 0 { 1305 return r + i 1306 } 1307 return -1 1308 } 1309 } 1310 return -1 1311 } 1312 c0 := sep[0] 1313 c1 := sep[1] 1314 i := 0 1315 fails := 0 1316 t := len(s) - n + 1 1317 for i < t { 1318 if s[i] != c0 { 1319 o := IndexByte(s[i+1:t], c0) 1320 if o < 0 { 1321 break 1322 } 1323 i += o + 1 1324 } 1325 if s[i+1] == c1 && Equal(s[i:i+n], sep) { 1326 return i 1327 } 1328 i++ 1329 fails++ 1330 if fails >= 4+i>>4 && i < t { 1331 // Give up on IndexByte, it isn't skipping ahead 1332 // far enough to be better than Rabin-Karp. 1333 // Experiments (using IndexPeriodic) suggest 1334 // the cutover is about 16 byte skips. 1335 // TODO: if large prefixes of sep are matching 1336 // we should cutover at even larger average skips, 1337 // because Equal becomes that much more expensive. 1338 // This code does not take that effect into account. 1339 j := bytealg.IndexRabinKarpBytes(s[i:], sep) 1340 if j < 0 { 1341 return -1 1342 } 1343 return i + j 1344 } 1345 } 1346 return -1 1347 } 1348 1349 // Cut slices s around the first instance of sep, 1350 // returning the text before and after sep. 1351 // The found result reports whether sep appears in s. 1352 // If sep does not appear in s, cut returns s, nil, false. 1353 // 1354 // Cut returns slices of the original slice s, not copies. 1355 func Cut(s, sep []byte) (before, after []byte, found bool) { 1356 if i := Index(s, sep); i >= 0 { 1357 return s[:i], s[i+len(sep):], true 1358 } 1359 return s, nil, false 1360 } 1361 1362 // Clone returns a copy of b[:len(b)]. 1363 // The result may have additional unused capacity. 1364 // Clone(nil) returns nil. 1365 func Clone(b []byte) []byte { 1366 if b == nil { 1367 return nil 1368 } 1369 return append([]byte{}, b...) 1370 } 1371 1372 // CutPrefix returns s without the provided leading prefix byte slice 1373 // and reports whether it found the prefix. 1374 // If s doesn't start with prefix, CutPrefix returns s, false. 1375 // If prefix is the empty byte slice, CutPrefix returns s, true. 1376 // 1377 // CutPrefix returns slices of the original slice s, not copies. 1378 func CutPrefix(s, prefix []byte) (after []byte, found bool) { 1379 if !HasPrefix(s, prefix) { 1380 return s, false 1381 } 1382 return s[len(prefix):], true 1383 } 1384 1385 // CutSuffix returns s without the provided ending suffix byte slice 1386 // and reports whether it found the suffix. 1387 // If s doesn't end with suffix, CutSuffix returns s, false. 1388 // If suffix is the empty byte slice, CutSuffix returns s, true. 1389 // 1390 // CutSuffix returns slices of the original slice s, not copies. 1391 func CutSuffix(s, suffix []byte) (before []byte, found bool) { 1392 if !HasSuffix(s, suffix) { 1393 return s, false 1394 } 1395 return s[:len(s)-len(suffix)], true 1396 }