github.com/graybobo/golang.org-package-offline-cache@v0.0.0-20200626051047-6608995c132f/x/text/unicode/norm/maketables.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build ignore 6 7 // Normalization table generator. 8 // Data read from the web. 9 // See forminfo.go for a description of the trie values associated with each rune. 10 11 package main 12 13 import ( 14 "bytes" 15 "flag" 16 "fmt" 17 "io" 18 "log" 19 "sort" 20 "strconv" 21 "strings" 22 23 "golang.org/x/text/internal/gen" 24 "golang.org/x/text/internal/triegen" 25 "golang.org/x/text/internal/ucd" 26 ) 27 28 func main() { 29 gen.Init() 30 loadUnicodeData() 31 compactCCC() 32 loadCompositionExclusions() 33 completeCharFields(FCanonical) 34 completeCharFields(FCompatibility) 35 computeNonStarterCounts() 36 verifyComputed() 37 printChars() 38 if *test { 39 testDerived() 40 printTestdata() 41 } else { 42 makeTables() 43 } 44 } 45 46 var ( 47 tablelist = flag.String("tables", 48 "all", 49 "comma-separated list of which tables to generate; "+ 50 "can be 'decomp', 'recomp', 'info' and 'all'") 51 test = flag.Bool("test", 52 false, 53 "test existing tables against DerivedNormalizationProps and generate test data for regression testing") 54 verbose = flag.Bool("verbose", 55 false, 56 "write data to stdout as it is parsed") 57 ) 58 59 const MaxChar = 0x10FFFF // anything above this shouldn't exist 60 61 // Quick Check properties of runes allow us to quickly 62 // determine whether a rune may occur in a normal form. 63 // For a given normal form, a rune may be guaranteed to occur 64 // verbatim (QC=Yes), may or may not combine with another 65 // rune (QC=Maybe), or may not occur (QC=No). 66 type QCResult int 67 68 const ( 69 QCUnknown QCResult = iota 70 QCYes 71 QCNo 72 QCMaybe 73 ) 74 75 func (r QCResult) String() string { 76 switch r { 77 case QCYes: 78 return "Yes" 79 case QCNo: 80 return "No" 81 case QCMaybe: 82 return "Maybe" 83 } 84 return "***UNKNOWN***" 85 } 86 87 const ( 88 FCanonical = iota // NFC or NFD 89 FCompatibility // NFKC or NFKD 90 FNumberOfFormTypes 91 ) 92 93 const ( 94 MComposed = iota // NFC or NFKC 95 MDecomposed // NFD or NFKD 96 MNumberOfModes 97 ) 98 99 // This contains only the properties we're interested in. 100 type Char struct { 101 name string 102 codePoint rune // if zero, this index is not a valid code point. 103 ccc uint8 // canonical combining class 104 origCCC uint8 105 excludeInComp bool // from CompositionExclusions.txt 106 compatDecomp bool // it has a compatibility expansion 107 108 nTrailingNonStarters uint8 109 nLeadingNonStarters uint8 // must be equal to trailing if non-zero 110 111 forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility 112 113 state State 114 } 115 116 var chars = make([]Char, MaxChar+1) 117 var cccMap = make(map[uint8]uint8) 118 119 func (c Char) String() string { 120 buf := new(bytes.Buffer) 121 122 fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) 123 fmt.Fprintf(buf, " ccc: %v\n", c.ccc) 124 fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) 125 fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) 126 fmt.Fprintf(buf, " state: %v\n", c.state) 127 fmt.Fprintf(buf, " NFC:\n") 128 fmt.Fprint(buf, c.forms[FCanonical]) 129 fmt.Fprintf(buf, " NFKC:\n") 130 fmt.Fprint(buf, c.forms[FCompatibility]) 131 132 return buf.String() 133 } 134 135 // In UnicodeData.txt, some ranges are marked like this: 136 // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;; 137 // 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;; 138 // parseCharacter keeps a state variable indicating the weirdness. 139 type State int 140 141 const ( 142 SNormal State = iota // known to be zero for the type 143 SFirst 144 SLast 145 SMissing 146 ) 147 148 var lastChar = rune('\u0000') 149 150 func (c Char) isValid() bool { 151 return c.codePoint != 0 && c.state != SMissing 152 } 153 154 type FormInfo struct { 155 quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed 156 verified [MNumberOfModes]bool // index: MComposed or MDecomposed 157 158 combinesForward bool // May combine with rune on the right 159 combinesBackward bool // May combine with rune on the left 160 isOneWay bool // Never appears in result 161 inDecomp bool // Some decompositions result in this char. 162 decomp Decomposition 163 expandedDecomp Decomposition 164 } 165 166 func (f FormInfo) String() string { 167 buf := bytes.NewBuffer(make([]byte, 0)) 168 169 fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) 170 fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) 171 fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) 172 fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) 173 fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) 174 fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) 175 fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) 176 fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) 177 178 return buf.String() 179 } 180 181 type Decomposition []rune 182 183 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { 184 decomp := strings.Split(s, " ") 185 if len(decomp) > 0 && skipfirst { 186 decomp = decomp[1:] 187 } 188 for _, d := range decomp { 189 point, err := strconv.ParseUint(d, 16, 64) 190 if err != nil { 191 return a, err 192 } 193 a = append(a, rune(point)) 194 } 195 return a, nil 196 } 197 198 func loadUnicodeData() { 199 f := gen.OpenUCDFile("UnicodeData.txt") 200 defer f.Close() 201 p := ucd.New(f) 202 for p.Next() { 203 r := p.Rune(ucd.CodePoint) 204 char := &chars[r] 205 206 char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) 207 decmap := p.String(ucd.DecompMapping) 208 209 exp, err := parseDecomposition(decmap, false) 210 isCompat := false 211 if err != nil { 212 if len(decmap) > 0 { 213 exp, err = parseDecomposition(decmap, true) 214 if err != nil { 215 log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) 216 } 217 isCompat = true 218 } 219 } 220 221 char.name = p.String(ucd.Name) 222 char.codePoint = r 223 char.forms[FCompatibility].decomp = exp 224 if !isCompat { 225 char.forms[FCanonical].decomp = exp 226 } else { 227 char.compatDecomp = true 228 } 229 if len(decmap) > 0 { 230 char.forms[FCompatibility].decomp = exp 231 } 232 } 233 if err := p.Err(); err != nil { 234 log.Fatal(err) 235 } 236 } 237 238 // compactCCC converts the sparse set of CCC values to a continguous one, 239 // reducing the number of bits needed from 8 to 6. 240 func compactCCC() { 241 m := make(map[uint8]uint8) 242 for i := range chars { 243 c := &chars[i] 244 m[c.ccc] = 0 245 } 246 cccs := []int{} 247 for v, _ := range m { 248 cccs = append(cccs, int(v)) 249 } 250 sort.Ints(cccs) 251 for i, c := range cccs { 252 cccMap[uint8(i)] = uint8(c) 253 m[uint8(c)] = uint8(i) 254 } 255 for i := range chars { 256 c := &chars[i] 257 c.origCCC = c.ccc 258 c.ccc = m[c.ccc] 259 } 260 if len(m) >= 1<<6 { 261 log.Fatalf("too many difference CCC values: %d >= 64", len(m)) 262 } 263 } 264 265 // CompositionExclusions.txt has form: 266 // 0958 # ... 267 // See http://unicode.org/reports/tr44/ for full explanation 268 func loadCompositionExclusions() { 269 f := gen.OpenUCDFile("CompositionExclusions.txt") 270 defer f.Close() 271 p := ucd.New(f) 272 for p.Next() { 273 c := &chars[p.Rune(0)] 274 if c.excludeInComp { 275 log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) 276 } 277 c.excludeInComp = true 278 } 279 if e := p.Err(); e != nil { 280 log.Fatal(e) 281 } 282 } 283 284 // hasCompatDecomp returns true if any of the recursive 285 // decompositions contains a compatibility expansion. 286 // In this case, the character may not occur in NFK*. 287 func hasCompatDecomp(r rune) bool { 288 c := &chars[r] 289 if c.compatDecomp { 290 return true 291 } 292 for _, d := range c.forms[FCompatibility].decomp { 293 if hasCompatDecomp(d) { 294 return true 295 } 296 } 297 return false 298 } 299 300 // Hangul related constants. 301 const ( 302 HangulBase = 0xAC00 303 HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) 304 305 JamoLBase = 0x1100 306 JamoLEnd = 0x1113 307 JamoVBase = 0x1161 308 JamoVEnd = 0x1176 309 JamoTBase = 0x11A8 310 JamoTEnd = 0x11C3 311 312 JamoLVTCount = 19 * 21 * 28 313 JamoTCount = 28 314 ) 315 316 func isHangul(r rune) bool { 317 return HangulBase <= r && r < HangulEnd 318 } 319 320 func isHangulWithoutJamoT(r rune) bool { 321 if !isHangul(r) { 322 return false 323 } 324 r -= HangulBase 325 return r < JamoLVTCount && r%JamoTCount == 0 326 } 327 328 func ccc(r rune) uint8 { 329 return chars[r].ccc 330 } 331 332 // Insert a rune in a buffer, ordered by Canonical Combining Class. 333 func insertOrdered(b Decomposition, r rune) Decomposition { 334 n := len(b) 335 b = append(b, 0) 336 cc := ccc(r) 337 if cc > 0 { 338 // Use bubble sort. 339 for ; n > 0; n-- { 340 if ccc(b[n-1]) <= cc { 341 break 342 } 343 b[n] = b[n-1] 344 } 345 } 346 b[n] = r 347 return b 348 } 349 350 // Recursively decompose. 351 func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { 352 dcomp := chars[r].forms[form].decomp 353 if len(dcomp) == 0 { 354 return insertOrdered(d, r) 355 } 356 for _, c := range dcomp { 357 d = decomposeRecursive(form, c, d) 358 } 359 return d 360 } 361 362 func completeCharFields(form int) { 363 // Phase 0: pre-expand decomposition. 364 for i := range chars { 365 f := &chars[i].forms[form] 366 if len(f.decomp) == 0 { 367 continue 368 } 369 exp := make(Decomposition, 0) 370 for _, c := range f.decomp { 371 exp = decomposeRecursive(form, c, exp) 372 } 373 f.expandedDecomp = exp 374 } 375 376 // Phase 1: composition exclusion, mark decomposition. 377 for i := range chars { 378 c := &chars[i] 379 f := &c.forms[form] 380 381 // Marks script-specific exclusions and version restricted. 382 f.isOneWay = c.excludeInComp 383 384 // Singletons 385 f.isOneWay = f.isOneWay || len(f.decomp) == 1 386 387 // Non-starter decompositions 388 if len(f.decomp) > 1 { 389 chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 390 f.isOneWay = f.isOneWay || chk 391 } 392 393 // Runes that decompose into more than two runes. 394 f.isOneWay = f.isOneWay || len(f.decomp) > 2 395 396 if form == FCompatibility { 397 f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) 398 } 399 400 for _, r := range f.decomp { 401 chars[r].forms[form].inDecomp = true 402 } 403 } 404 405 // Phase 2: forward and backward combining. 406 for i := range chars { 407 c := &chars[i] 408 f := &c.forms[form] 409 410 if !f.isOneWay && len(f.decomp) == 2 { 411 f0 := &chars[f.decomp[0]].forms[form] 412 f1 := &chars[f.decomp[1]].forms[form] 413 if !f0.isOneWay { 414 f0.combinesForward = true 415 } 416 if !f1.isOneWay { 417 f1.combinesBackward = true 418 } 419 } 420 if isHangulWithoutJamoT(rune(i)) { 421 f.combinesForward = true 422 } 423 } 424 425 // Phase 3: quick check values. 426 for i := range chars { 427 c := &chars[i] 428 f := &c.forms[form] 429 430 switch { 431 case len(f.decomp) > 0: 432 f.quickCheck[MDecomposed] = QCNo 433 case isHangul(rune(i)): 434 f.quickCheck[MDecomposed] = QCNo 435 default: 436 f.quickCheck[MDecomposed] = QCYes 437 } 438 switch { 439 case f.isOneWay: 440 f.quickCheck[MComposed] = QCNo 441 case (i & 0xffff00) == JamoLBase: 442 f.quickCheck[MComposed] = QCYes 443 if JamoLBase <= i && i < JamoLEnd { 444 f.combinesForward = true 445 } 446 if JamoVBase <= i && i < JamoVEnd { 447 f.quickCheck[MComposed] = QCMaybe 448 f.combinesBackward = true 449 f.combinesForward = true 450 } 451 if JamoTBase <= i && i < JamoTEnd { 452 f.quickCheck[MComposed] = QCMaybe 453 f.combinesBackward = true 454 } 455 case !f.combinesBackward: 456 f.quickCheck[MComposed] = QCYes 457 default: 458 f.quickCheck[MComposed] = QCMaybe 459 } 460 } 461 } 462 463 func computeNonStarterCounts() { 464 // Phase 4: leading and trailing non-starter count 465 for i := range chars { 466 c := &chars[i] 467 468 runes := []rune{rune(i)} 469 // We always use FCompatibility so that the CGJ insertion points do not 470 // change for repeated normalizations with different forms. 471 if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { 472 runes = exp 473 } 474 // We consider runes that combine backwards to be non-starters for the 475 // purpose of Stream-Safe Text Processing. 476 for _, r := range runes { 477 if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { 478 break 479 } 480 c.nLeadingNonStarters++ 481 } 482 for i := len(runes) - 1; i >= 0; i-- { 483 if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { 484 break 485 } 486 c.nTrailingNonStarters++ 487 } 488 if c.nTrailingNonStarters > 3 { 489 log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) 490 } 491 492 if isHangul(rune(i)) { 493 c.nTrailingNonStarters = 2 494 if isHangulWithoutJamoT(rune(i)) { 495 c.nTrailingNonStarters = 1 496 } 497 } 498 499 if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { 500 log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) 501 } 502 if t := c.nTrailingNonStarters; t > 3 { 503 log.Fatalf("%U: number of trailing non-starters is %d > 3", t) 504 } 505 } 506 } 507 508 func printBytes(w io.Writer, b []byte, name string) { 509 fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) 510 fmt.Fprintf(w, "var %s = [...]byte {", name) 511 for i, c := range b { 512 switch { 513 case i%64 == 0: 514 fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) 515 case i%8 == 0: 516 fmt.Fprintf(w, "\n") 517 } 518 fmt.Fprintf(w, "0x%.2X, ", c) 519 } 520 fmt.Fprint(w, "\n}\n\n") 521 } 522 523 // See forminfo.go for format. 524 func makeEntry(f *FormInfo, c *Char) uint16 { 525 e := uint16(0) 526 if r := c.codePoint; HangulBase <= r && r < HangulEnd { 527 e |= 0x40 528 } 529 if f.combinesForward { 530 e |= 0x20 531 } 532 if f.quickCheck[MDecomposed] == QCNo { 533 e |= 0x4 534 } 535 switch f.quickCheck[MComposed] { 536 case QCYes: 537 case QCNo: 538 e |= 0x10 539 case QCMaybe: 540 e |= 0x18 541 default: 542 log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) 543 } 544 e |= uint16(c.nTrailingNonStarters) 545 return e 546 } 547 548 // decompSet keeps track of unique decompositions, grouped by whether 549 // the decomposition is followed by a trailing and/or leading CCC. 550 type decompSet [7]map[string]bool 551 552 const ( 553 normalDecomp = iota 554 firstMulti 555 firstCCC 556 endMulti 557 firstLeadingCCC 558 firstCCCZeroExcept 559 firstStarterWithNLead 560 lastDecomp 561 ) 562 563 var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} 564 565 func makeDecompSet() decompSet { 566 m := decompSet{} 567 for i := range m { 568 m[i] = make(map[string]bool) 569 } 570 return m 571 } 572 func (m *decompSet) insert(key int, s string) { 573 m[key][s] = true 574 } 575 576 func printCharInfoTables(w io.Writer) int { 577 mkstr := func(r rune, f *FormInfo) (int, string) { 578 d := f.expandedDecomp 579 s := string([]rune(d)) 580 if max := 1 << 6; len(s) >= max { 581 const msg = "%U: too many bytes in decomposition: %d >= %d" 582 log.Fatalf(msg, r, len(s), max) 583 } 584 head := uint8(len(s)) 585 if f.quickCheck[MComposed] != QCYes { 586 head |= 0x40 587 } 588 if f.combinesForward { 589 head |= 0x80 590 } 591 s = string([]byte{head}) + s 592 593 lccc := ccc(d[0]) 594 tccc := ccc(d[len(d)-1]) 595 cc := ccc(r) 596 if cc != 0 && lccc == 0 && tccc == 0 { 597 log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) 598 } 599 if tccc < lccc && lccc != 0 { 600 const msg = "%U: lccc (%d) must be <= tcc (%d)" 601 log.Fatalf(msg, r, lccc, tccc) 602 } 603 index := normalDecomp 604 nTrail := chars[r].nTrailingNonStarters 605 if tccc > 0 || lccc > 0 || nTrail > 0 { 606 tccc <<= 2 607 tccc |= nTrail 608 s += string([]byte{tccc}) 609 index = endMulti 610 for _, r := range d[1:] { 611 if ccc(r) == 0 { 612 index = firstCCC 613 } 614 } 615 if lccc > 0 { 616 s += string([]byte{lccc}) 617 if index == firstCCC { 618 log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) 619 } 620 index = firstLeadingCCC 621 } 622 if cc != lccc { 623 if cc != 0 { 624 log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) 625 } 626 index = firstCCCZeroExcept 627 } 628 } else if len(d) > 1 { 629 index = firstMulti 630 } 631 return index, s 632 } 633 634 decompSet := makeDecompSet() 635 const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail. 636 decompSet.insert(firstStarterWithNLead, nLeadStr) 637 638 // Store the uniqued decompositions in a byte buffer, 639 // preceded by their byte length. 640 for _, c := range chars { 641 for _, f := range c.forms { 642 if len(f.expandedDecomp) == 0 { 643 continue 644 } 645 if f.combinesBackward { 646 log.Fatalf("%U: combinesBackward and decompose", c.codePoint) 647 } 648 index, s := mkstr(c.codePoint, &f) 649 decompSet.insert(index, s) 650 } 651 } 652 653 decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) 654 size := 0 655 positionMap := make(map[string]uint16) 656 decompositions.WriteString("\000") 657 fmt.Fprintln(w, "const (") 658 for i, m := range decompSet { 659 sa := []string{} 660 for s := range m { 661 sa = append(sa, s) 662 } 663 sort.Strings(sa) 664 for _, s := range sa { 665 p := decompositions.Len() 666 decompositions.WriteString(s) 667 positionMap[s] = uint16(p) 668 } 669 if cname[i] != "" { 670 fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) 671 } 672 } 673 fmt.Fprintln(w, "maxDecomp = 0x8000") 674 fmt.Fprintln(w, ")") 675 b := decompositions.Bytes() 676 printBytes(w, b, "decomps") 677 size += len(b) 678 679 varnames := []string{"nfc", "nfkc"} 680 for i := 0; i < FNumberOfFormTypes; i++ { 681 trie := triegen.NewTrie(varnames[i]) 682 683 for r, c := range chars { 684 f := c.forms[i] 685 d := f.expandedDecomp 686 if len(d) != 0 { 687 _, key := mkstr(c.codePoint, &f) 688 trie.Insert(rune(r), uint64(positionMap[key])) 689 if c.ccc != ccc(d[0]) { 690 // We assume the lead ccc of a decomposition !=0 in this case. 691 if ccc(d[0]) == 0 { 692 log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) 693 } 694 } 695 } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { 696 // Handle cases where it can't be detected that the nLead should be equal 697 // to nTrail. 698 trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) 699 } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { 700 trie.Insert(c.codePoint, uint64(0x8000|v)) 701 } 702 } 703 sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) 704 if err != nil { 705 log.Fatal(err) 706 } 707 size += sz 708 } 709 return size 710 } 711 712 func contains(sa []string, s string) bool { 713 for _, a := range sa { 714 if a == s { 715 return true 716 } 717 } 718 return false 719 } 720 721 func makeTables() { 722 w := &bytes.Buffer{} 723 724 size := 0 725 if *tablelist == "" { 726 return 727 } 728 list := strings.Split(*tablelist, ",") 729 if *tablelist == "all" { 730 list = []string{"recomp", "info"} 731 } 732 733 // Compute maximum decomposition size. 734 max := 0 735 for _, c := range chars { 736 if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { 737 max = n 738 } 739 } 740 741 fmt.Fprintln(w, "const (") 742 fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") 743 fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) 744 fmt.Fprintln(w) 745 fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") 746 fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") 747 fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") 748 fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") 749 fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) 750 fmt.Fprintln(w, ")\n") 751 752 // Print the CCC remap table. 753 size += len(cccMap) 754 fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) 755 for i := 0; i < len(cccMap); i++ { 756 if i%8 == 0 { 757 fmt.Fprintln(w) 758 } 759 fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) 760 } 761 fmt.Fprintln(w, "\n}\n") 762 763 if contains(list, "info") { 764 size += printCharInfoTables(w) 765 } 766 767 if contains(list, "recomp") { 768 // Note that we use 32 bit keys, instead of 64 bit. 769 // This clips the bits of three entries, but we know 770 // this won't cause a collision. The compiler will catch 771 // any changes made to UnicodeData.txt that introduces 772 // a collision. 773 // Note that the recomposition map for NFC and NFKC 774 // are identical. 775 776 // Recomposition map 777 nrentries := 0 778 for _, c := range chars { 779 f := c.forms[FCanonical] 780 if !f.isOneWay && len(f.decomp) > 0 { 781 nrentries++ 782 } 783 } 784 sz := nrentries * 8 785 size += sz 786 fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) 787 fmt.Fprintln(w, "var recompMap = map[uint32]rune{") 788 for i, c := range chars { 789 f := c.forms[FCanonical] 790 d := f.decomp 791 if !f.isOneWay && len(d) > 0 { 792 key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) 793 fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) 794 } 795 } 796 fmt.Fprintf(w, "}\n\n") 797 } 798 799 fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) 800 gen.WriteGoFile("tables.go", "norm", w.Bytes()) 801 } 802 803 func printChars() { 804 if *verbose { 805 for _, c := range chars { 806 if !c.isValid() || c.state == SMissing { 807 continue 808 } 809 fmt.Println(c) 810 } 811 } 812 } 813 814 // verifyComputed does various consistency tests. 815 func verifyComputed() { 816 for i, c := range chars { 817 for _, f := range c.forms { 818 isNo := (f.quickCheck[MDecomposed] == QCNo) 819 if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { 820 log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) 821 } 822 823 isMaybe := f.quickCheck[MComposed] == QCMaybe 824 if f.combinesBackward != isMaybe { 825 log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) 826 } 827 if len(f.decomp) > 0 && f.combinesForward && isMaybe { 828 log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) 829 } 830 831 if len(f.expandedDecomp) != 0 { 832 continue 833 } 834 if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { 835 // We accept these runes to be treated differently (it only affects 836 // segment breaking in iteration, most likely on improper use), but 837 // reconsider if more characters are added. 838 // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;; 839 // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;; 840 // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; 841 // U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;; 842 // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; 843 // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;; 844 if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { 845 log.Fatalf("%U: nLead was %v; want %v", i, a, b) 846 } 847 } 848 } 849 nfc := c.forms[FCanonical] 850 nfkc := c.forms[FCompatibility] 851 if nfc.combinesBackward != nfkc.combinesBackward { 852 log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) 853 } 854 } 855 } 856 857 // Use values in DerivedNormalizationProps.txt to compare against the 858 // values we computed. 859 // DerivedNormalizationProps.txt has form: 860 // 00C0..00C5 ; NFD_QC; N # ... 861 // 0374 ; NFD_QC; N # ... 862 // See http://unicode.org/reports/tr44/ for full explanation 863 func testDerived() { 864 f := gen.OpenUCDFile("DerivedNormalizationProps.txt") 865 defer f.Close() 866 p := ucd.New(f) 867 for p.Next() { 868 r := p.Rune(0) 869 c := &chars[r] 870 871 var ftype, mode int 872 qt := p.String(1) 873 switch qt { 874 case "NFC_QC": 875 ftype, mode = FCanonical, MComposed 876 case "NFD_QC": 877 ftype, mode = FCanonical, MDecomposed 878 case "NFKC_QC": 879 ftype, mode = FCompatibility, MComposed 880 case "NFKD_QC": 881 ftype, mode = FCompatibility, MDecomposed 882 default: 883 continue 884 } 885 var qr QCResult 886 switch p.String(2) { 887 case "Y": 888 qr = QCYes 889 case "N": 890 qr = QCNo 891 case "M": 892 qr = QCMaybe 893 default: 894 log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) 895 } 896 if got := c.forms[ftype].quickCheck[mode]; got != qr { 897 log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) 898 } 899 c.forms[ftype].verified[mode] = true 900 } 901 if err := p.Err(); err != nil { 902 log.Fatal(err) 903 } 904 // Any unspecified value must be QCYes. Verify this. 905 for i, c := range chars { 906 for j, fd := range c.forms { 907 for k, qr := range fd.quickCheck { 908 if !fd.verified[k] && qr != QCYes { 909 m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" 910 log.Printf(m, i, j, k, qr, c.name) 911 } 912 } 913 } 914 } 915 } 916 917 var testHeader = `const ( 918 Yes = iota 919 No 920 Maybe 921 ) 922 923 type formData struct { 924 qc uint8 925 combinesForward bool 926 decomposition string 927 } 928 929 type runeData struct { 930 r rune 931 ccc uint8 932 nLead uint8 933 nTrail uint8 934 f [2]formData // 0: canonical; 1: compatibility 935 } 936 937 func f(qc uint8, cf bool, dec string) [2]formData { 938 return [2]formData{{qc, cf, dec}, {qc, cf, dec}} 939 } 940 941 func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { 942 return [2]formData{{qc, cf, d}, {qck, cfk, dk}} 943 } 944 945 var testData = []runeData{ 946 ` 947 948 func printTestdata() { 949 type lastInfo struct { 950 ccc uint8 951 nLead uint8 952 nTrail uint8 953 f string 954 } 955 956 last := lastInfo{} 957 w := &bytes.Buffer{} 958 fmt.Fprintf(w, testHeader) 959 for r, c := range chars { 960 f := c.forms[FCanonical] 961 qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) 962 f = c.forms[FCompatibility] 963 qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) 964 s := "" 965 if d == dk && qc == qck && cf == cfk { 966 s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) 967 } else { 968 s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) 969 } 970 current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} 971 if last != current { 972 fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) 973 last = current 974 } 975 } 976 fmt.Fprintln(w, "}") 977 gen.WriteGoFile("data_test.go", "norm", w.Bytes()) 978 }