github.com/graybobo/golang.org-package-offline-cache@v0.0.0-20200626051047-6608995c132f/x/text/unicode/norm/maketables.go (about)

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // +build ignore
     6  
     7  // Normalization table generator.
     8  // Data read from the web.
     9  // See forminfo.go for a description of the trie values associated with each rune.
    10  
    11  package main
    12  
    13  import (
    14  	"bytes"
    15  	"flag"
    16  	"fmt"
    17  	"io"
    18  	"log"
    19  	"sort"
    20  	"strconv"
    21  	"strings"
    22  
    23  	"golang.org/x/text/internal/gen"
    24  	"golang.org/x/text/internal/triegen"
    25  	"golang.org/x/text/internal/ucd"
    26  )
    27  
    28  func main() {
    29  	gen.Init()
    30  	loadUnicodeData()
    31  	compactCCC()
    32  	loadCompositionExclusions()
    33  	completeCharFields(FCanonical)
    34  	completeCharFields(FCompatibility)
    35  	computeNonStarterCounts()
    36  	verifyComputed()
    37  	printChars()
    38  	if *test {
    39  		testDerived()
    40  		printTestdata()
    41  	} else {
    42  		makeTables()
    43  	}
    44  }
    45  
    46  var (
    47  	tablelist = flag.String("tables",
    48  		"all",
    49  		"comma-separated list of which tables to generate; "+
    50  			"can be 'decomp', 'recomp', 'info' and 'all'")
    51  	test = flag.Bool("test",
    52  		false,
    53  		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
    54  	verbose = flag.Bool("verbose",
    55  		false,
    56  		"write data to stdout as it is parsed")
    57  )
    58  
    59  const MaxChar = 0x10FFFF // anything above this shouldn't exist
    60  
    61  // Quick Check properties of runes allow us to quickly
    62  // determine whether a rune may occur in a normal form.
    63  // For a given normal form, a rune may be guaranteed to occur
    64  // verbatim (QC=Yes), may or may not combine with another
    65  // rune (QC=Maybe), or may not occur (QC=No).
    66  type QCResult int
    67  
    68  const (
    69  	QCUnknown QCResult = iota
    70  	QCYes
    71  	QCNo
    72  	QCMaybe
    73  )
    74  
    75  func (r QCResult) String() string {
    76  	switch r {
    77  	case QCYes:
    78  		return "Yes"
    79  	case QCNo:
    80  		return "No"
    81  	case QCMaybe:
    82  		return "Maybe"
    83  	}
    84  	return "***UNKNOWN***"
    85  }
    86  
    87  const (
    88  	FCanonical     = iota // NFC or NFD
    89  	FCompatibility        // NFKC or NFKD
    90  	FNumberOfFormTypes
    91  )
    92  
    93  const (
    94  	MComposed   = iota // NFC or NFKC
    95  	MDecomposed        // NFD or NFKD
    96  	MNumberOfModes
    97  )
    98  
    99  // This contains only the properties we're interested in.
   100  type Char struct {
   101  	name          string
   102  	codePoint     rune  // if zero, this index is not a valid code point.
   103  	ccc           uint8 // canonical combining class
   104  	origCCC       uint8
   105  	excludeInComp bool // from CompositionExclusions.txt
   106  	compatDecomp  bool // it has a compatibility expansion
   107  
   108  	nTrailingNonStarters uint8
   109  	nLeadingNonStarters  uint8 // must be equal to trailing if non-zero
   110  
   111  	forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
   112  
   113  	state State
   114  }
   115  
   116  var chars = make([]Char, MaxChar+1)
   117  var cccMap = make(map[uint8]uint8)
   118  
   119  func (c Char) String() string {
   120  	buf := new(bytes.Buffer)
   121  
   122  	fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name)
   123  	fmt.Fprintf(buf, "  ccc: %v\n", c.ccc)
   124  	fmt.Fprintf(buf, "  excludeInComp: %v\n", c.excludeInComp)
   125  	fmt.Fprintf(buf, "  compatDecomp: %v\n", c.compatDecomp)
   126  	fmt.Fprintf(buf, "  state: %v\n", c.state)
   127  	fmt.Fprintf(buf, "  NFC:\n")
   128  	fmt.Fprint(buf, c.forms[FCanonical])
   129  	fmt.Fprintf(buf, "  NFKC:\n")
   130  	fmt.Fprint(buf, c.forms[FCompatibility])
   131  
   132  	return buf.String()
   133  }
   134  
   135  // In UnicodeData.txt, some ranges are marked like this:
   136  //	3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
   137  //	4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
   138  // parseCharacter keeps a state variable indicating the weirdness.
   139  type State int
   140  
   141  const (
   142  	SNormal State = iota // known to be zero for the type
   143  	SFirst
   144  	SLast
   145  	SMissing
   146  )
   147  
   148  var lastChar = rune('\u0000')
   149  
   150  func (c Char) isValid() bool {
   151  	return c.codePoint != 0 && c.state != SMissing
   152  }
   153  
   154  type FormInfo struct {
   155  	quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
   156  	verified   [MNumberOfModes]bool     // index: MComposed or MDecomposed
   157  
   158  	combinesForward  bool // May combine with rune on the right
   159  	combinesBackward bool // May combine with rune on the left
   160  	isOneWay         bool // Never appears in result
   161  	inDecomp         bool // Some decompositions result in this char.
   162  	decomp           Decomposition
   163  	expandedDecomp   Decomposition
   164  }
   165  
   166  func (f FormInfo) String() string {
   167  	buf := bytes.NewBuffer(make([]byte, 0))
   168  
   169  	fmt.Fprintf(buf, "    quickCheck[C]: %v\n", f.quickCheck[MComposed])
   170  	fmt.Fprintf(buf, "    quickCheck[D]: %v\n", f.quickCheck[MDecomposed])
   171  	fmt.Fprintf(buf, "    cmbForward: %v\n", f.combinesForward)
   172  	fmt.Fprintf(buf, "    cmbBackward: %v\n", f.combinesBackward)
   173  	fmt.Fprintf(buf, "    isOneWay: %v\n", f.isOneWay)
   174  	fmt.Fprintf(buf, "    inDecomp: %v\n", f.inDecomp)
   175  	fmt.Fprintf(buf, "    decomposition: %X\n", f.decomp)
   176  	fmt.Fprintf(buf, "    expandedDecomp: %X\n", f.expandedDecomp)
   177  
   178  	return buf.String()
   179  }
   180  
   181  type Decomposition []rune
   182  
   183  func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
   184  	decomp := strings.Split(s, " ")
   185  	if len(decomp) > 0 && skipfirst {
   186  		decomp = decomp[1:]
   187  	}
   188  	for _, d := range decomp {
   189  		point, err := strconv.ParseUint(d, 16, 64)
   190  		if err != nil {
   191  			return a, err
   192  		}
   193  		a = append(a, rune(point))
   194  	}
   195  	return a, nil
   196  }
   197  
   198  func loadUnicodeData() {
   199  	f := gen.OpenUCDFile("UnicodeData.txt")
   200  	defer f.Close()
   201  	p := ucd.New(f)
   202  	for p.Next() {
   203  		r := p.Rune(ucd.CodePoint)
   204  		char := &chars[r]
   205  
   206  		char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
   207  		decmap := p.String(ucd.DecompMapping)
   208  
   209  		exp, err := parseDecomposition(decmap, false)
   210  		isCompat := false
   211  		if err != nil {
   212  			if len(decmap) > 0 {
   213  				exp, err = parseDecomposition(decmap, true)
   214  				if err != nil {
   215  					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
   216  				}
   217  				isCompat = true
   218  			}
   219  		}
   220  
   221  		char.name = p.String(ucd.Name)
   222  		char.codePoint = r
   223  		char.forms[FCompatibility].decomp = exp
   224  		if !isCompat {
   225  			char.forms[FCanonical].decomp = exp
   226  		} else {
   227  			char.compatDecomp = true
   228  		}
   229  		if len(decmap) > 0 {
   230  			char.forms[FCompatibility].decomp = exp
   231  		}
   232  	}
   233  	if err := p.Err(); err != nil {
   234  		log.Fatal(err)
   235  	}
   236  }
   237  
   238  // compactCCC converts the sparse set of CCC values to a continguous one,
   239  // reducing the number of bits needed from 8 to 6.
   240  func compactCCC() {
   241  	m := make(map[uint8]uint8)
   242  	for i := range chars {
   243  		c := &chars[i]
   244  		m[c.ccc] = 0
   245  	}
   246  	cccs := []int{}
   247  	for v, _ := range m {
   248  		cccs = append(cccs, int(v))
   249  	}
   250  	sort.Ints(cccs)
   251  	for i, c := range cccs {
   252  		cccMap[uint8(i)] = uint8(c)
   253  		m[uint8(c)] = uint8(i)
   254  	}
   255  	for i := range chars {
   256  		c := &chars[i]
   257  		c.origCCC = c.ccc
   258  		c.ccc = m[c.ccc]
   259  	}
   260  	if len(m) >= 1<<6 {
   261  		log.Fatalf("too many difference CCC values: %d >= 64", len(m))
   262  	}
   263  }
   264  
   265  // CompositionExclusions.txt has form:
   266  // 0958    # ...
   267  // See http://unicode.org/reports/tr44/ for full explanation
   268  func loadCompositionExclusions() {
   269  	f := gen.OpenUCDFile("CompositionExclusions.txt")
   270  	defer f.Close()
   271  	p := ucd.New(f)
   272  	for p.Next() {
   273  		c := &chars[p.Rune(0)]
   274  		if c.excludeInComp {
   275  			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
   276  		}
   277  		c.excludeInComp = true
   278  	}
   279  	if e := p.Err(); e != nil {
   280  		log.Fatal(e)
   281  	}
   282  }
   283  
   284  // hasCompatDecomp returns true if any of the recursive
   285  // decompositions contains a compatibility expansion.
   286  // In this case, the character may not occur in NFK*.
   287  func hasCompatDecomp(r rune) bool {
   288  	c := &chars[r]
   289  	if c.compatDecomp {
   290  		return true
   291  	}
   292  	for _, d := range c.forms[FCompatibility].decomp {
   293  		if hasCompatDecomp(d) {
   294  			return true
   295  		}
   296  	}
   297  	return false
   298  }
   299  
   300  // Hangul related constants.
   301  const (
   302  	HangulBase = 0xAC00
   303  	HangulEnd  = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
   304  
   305  	JamoLBase = 0x1100
   306  	JamoLEnd  = 0x1113
   307  	JamoVBase = 0x1161
   308  	JamoVEnd  = 0x1176
   309  	JamoTBase = 0x11A8
   310  	JamoTEnd  = 0x11C3
   311  
   312  	JamoLVTCount = 19 * 21 * 28
   313  	JamoTCount   = 28
   314  )
   315  
   316  func isHangul(r rune) bool {
   317  	return HangulBase <= r && r < HangulEnd
   318  }
   319  
   320  func isHangulWithoutJamoT(r rune) bool {
   321  	if !isHangul(r) {
   322  		return false
   323  	}
   324  	r -= HangulBase
   325  	return r < JamoLVTCount && r%JamoTCount == 0
   326  }
   327  
   328  func ccc(r rune) uint8 {
   329  	return chars[r].ccc
   330  }
   331  
   332  // Insert a rune in a buffer, ordered by Canonical Combining Class.
   333  func insertOrdered(b Decomposition, r rune) Decomposition {
   334  	n := len(b)
   335  	b = append(b, 0)
   336  	cc := ccc(r)
   337  	if cc > 0 {
   338  		// Use bubble sort.
   339  		for ; n > 0; n-- {
   340  			if ccc(b[n-1]) <= cc {
   341  				break
   342  			}
   343  			b[n] = b[n-1]
   344  		}
   345  	}
   346  	b[n] = r
   347  	return b
   348  }
   349  
   350  // Recursively decompose.
   351  func decomposeRecursive(form int, r rune, d Decomposition) Decomposition {
   352  	dcomp := chars[r].forms[form].decomp
   353  	if len(dcomp) == 0 {
   354  		return insertOrdered(d, r)
   355  	}
   356  	for _, c := range dcomp {
   357  		d = decomposeRecursive(form, c, d)
   358  	}
   359  	return d
   360  }
   361  
   362  func completeCharFields(form int) {
   363  	// Phase 0: pre-expand decomposition.
   364  	for i := range chars {
   365  		f := &chars[i].forms[form]
   366  		if len(f.decomp) == 0 {
   367  			continue
   368  		}
   369  		exp := make(Decomposition, 0)
   370  		for _, c := range f.decomp {
   371  			exp = decomposeRecursive(form, c, exp)
   372  		}
   373  		f.expandedDecomp = exp
   374  	}
   375  
   376  	// Phase 1: composition exclusion, mark decomposition.
   377  	for i := range chars {
   378  		c := &chars[i]
   379  		f := &c.forms[form]
   380  
   381  		// Marks script-specific exclusions and version restricted.
   382  		f.isOneWay = c.excludeInComp
   383  
   384  		// Singletons
   385  		f.isOneWay = f.isOneWay || len(f.decomp) == 1
   386  
   387  		// Non-starter decompositions
   388  		if len(f.decomp) > 1 {
   389  			chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0
   390  			f.isOneWay = f.isOneWay || chk
   391  		}
   392  
   393  		// Runes that decompose into more than two runes.
   394  		f.isOneWay = f.isOneWay || len(f.decomp) > 2
   395  
   396  		if form == FCompatibility {
   397  			f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint)
   398  		}
   399  
   400  		for _, r := range f.decomp {
   401  			chars[r].forms[form].inDecomp = true
   402  		}
   403  	}
   404  
   405  	// Phase 2: forward and backward combining.
   406  	for i := range chars {
   407  		c := &chars[i]
   408  		f := &c.forms[form]
   409  
   410  		if !f.isOneWay && len(f.decomp) == 2 {
   411  			f0 := &chars[f.decomp[0]].forms[form]
   412  			f1 := &chars[f.decomp[1]].forms[form]
   413  			if !f0.isOneWay {
   414  				f0.combinesForward = true
   415  			}
   416  			if !f1.isOneWay {
   417  				f1.combinesBackward = true
   418  			}
   419  		}
   420  		if isHangulWithoutJamoT(rune(i)) {
   421  			f.combinesForward = true
   422  		}
   423  	}
   424  
   425  	// Phase 3: quick check values.
   426  	for i := range chars {
   427  		c := &chars[i]
   428  		f := &c.forms[form]
   429  
   430  		switch {
   431  		case len(f.decomp) > 0:
   432  			f.quickCheck[MDecomposed] = QCNo
   433  		case isHangul(rune(i)):
   434  			f.quickCheck[MDecomposed] = QCNo
   435  		default:
   436  			f.quickCheck[MDecomposed] = QCYes
   437  		}
   438  		switch {
   439  		case f.isOneWay:
   440  			f.quickCheck[MComposed] = QCNo
   441  		case (i & 0xffff00) == JamoLBase:
   442  			f.quickCheck[MComposed] = QCYes
   443  			if JamoLBase <= i && i < JamoLEnd {
   444  				f.combinesForward = true
   445  			}
   446  			if JamoVBase <= i && i < JamoVEnd {
   447  				f.quickCheck[MComposed] = QCMaybe
   448  				f.combinesBackward = true
   449  				f.combinesForward = true
   450  			}
   451  			if JamoTBase <= i && i < JamoTEnd {
   452  				f.quickCheck[MComposed] = QCMaybe
   453  				f.combinesBackward = true
   454  			}
   455  		case !f.combinesBackward:
   456  			f.quickCheck[MComposed] = QCYes
   457  		default:
   458  			f.quickCheck[MComposed] = QCMaybe
   459  		}
   460  	}
   461  }
   462  
   463  func computeNonStarterCounts() {
   464  	// Phase 4: leading and trailing non-starter count
   465  	for i := range chars {
   466  		c := &chars[i]
   467  
   468  		runes := []rune{rune(i)}
   469  		// We always use FCompatibility so that the CGJ insertion points do not
   470  		// change for repeated normalizations with different forms.
   471  		if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
   472  			runes = exp
   473  		}
   474  		// We consider runes that combine backwards to be non-starters for the
   475  		// purpose of Stream-Safe Text Processing.
   476  		for _, r := range runes {
   477  			if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
   478  				break
   479  			}
   480  			c.nLeadingNonStarters++
   481  		}
   482  		for i := len(runes) - 1; i >= 0; i-- {
   483  			if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
   484  				break
   485  			}
   486  			c.nTrailingNonStarters++
   487  		}
   488  		if c.nTrailingNonStarters > 3 {
   489  			log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
   490  		}
   491  
   492  		if isHangul(rune(i)) {
   493  			c.nTrailingNonStarters = 2
   494  			if isHangulWithoutJamoT(rune(i)) {
   495  				c.nTrailingNonStarters = 1
   496  			}
   497  		}
   498  
   499  		if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t {
   500  			log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t)
   501  		}
   502  		if t := c.nTrailingNonStarters; t > 3 {
   503  			log.Fatalf("%U: number of trailing non-starters is %d > 3", t)
   504  		}
   505  	}
   506  }
   507  
   508  func printBytes(w io.Writer, b []byte, name string) {
   509  	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
   510  	fmt.Fprintf(w, "var %s = [...]byte {", name)
   511  	for i, c := range b {
   512  		switch {
   513  		case i%64 == 0:
   514  			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
   515  		case i%8 == 0:
   516  			fmt.Fprintf(w, "\n")
   517  		}
   518  		fmt.Fprintf(w, "0x%.2X, ", c)
   519  	}
   520  	fmt.Fprint(w, "\n}\n\n")
   521  }
   522  
   523  // See forminfo.go for format.
   524  func makeEntry(f *FormInfo, c *Char) uint16 {
   525  	e := uint16(0)
   526  	if r := c.codePoint; HangulBase <= r && r < HangulEnd {
   527  		e |= 0x40
   528  	}
   529  	if f.combinesForward {
   530  		e |= 0x20
   531  	}
   532  	if f.quickCheck[MDecomposed] == QCNo {
   533  		e |= 0x4
   534  	}
   535  	switch f.quickCheck[MComposed] {
   536  	case QCYes:
   537  	case QCNo:
   538  		e |= 0x10
   539  	case QCMaybe:
   540  		e |= 0x18
   541  	default:
   542  		log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed])
   543  	}
   544  	e |= uint16(c.nTrailingNonStarters)
   545  	return e
   546  }
   547  
   548  // decompSet keeps track of unique decompositions, grouped by whether
   549  // the decomposition is followed by a trailing and/or leading CCC.
   550  type decompSet [7]map[string]bool
   551  
   552  const (
   553  	normalDecomp = iota
   554  	firstMulti
   555  	firstCCC
   556  	endMulti
   557  	firstLeadingCCC
   558  	firstCCCZeroExcept
   559  	firstStarterWithNLead
   560  	lastDecomp
   561  )
   562  
   563  var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"}
   564  
   565  func makeDecompSet() decompSet {
   566  	m := decompSet{}
   567  	for i := range m {
   568  		m[i] = make(map[string]bool)
   569  	}
   570  	return m
   571  }
   572  func (m *decompSet) insert(key int, s string) {
   573  	m[key][s] = true
   574  }
   575  
   576  func printCharInfoTables(w io.Writer) int {
   577  	mkstr := func(r rune, f *FormInfo) (int, string) {
   578  		d := f.expandedDecomp
   579  		s := string([]rune(d))
   580  		if max := 1 << 6; len(s) >= max {
   581  			const msg = "%U: too many bytes in decomposition: %d >= %d"
   582  			log.Fatalf(msg, r, len(s), max)
   583  		}
   584  		head := uint8(len(s))
   585  		if f.quickCheck[MComposed] != QCYes {
   586  			head |= 0x40
   587  		}
   588  		if f.combinesForward {
   589  			head |= 0x80
   590  		}
   591  		s = string([]byte{head}) + s
   592  
   593  		lccc := ccc(d[0])
   594  		tccc := ccc(d[len(d)-1])
   595  		cc := ccc(r)
   596  		if cc != 0 && lccc == 0 && tccc == 0 {
   597  			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
   598  		}
   599  		if tccc < lccc && lccc != 0 {
   600  			const msg = "%U: lccc (%d) must be <= tcc (%d)"
   601  			log.Fatalf(msg, r, lccc, tccc)
   602  		}
   603  		index := normalDecomp
   604  		nTrail := chars[r].nTrailingNonStarters
   605  		if tccc > 0 || lccc > 0 || nTrail > 0 {
   606  			tccc <<= 2
   607  			tccc |= nTrail
   608  			s += string([]byte{tccc})
   609  			index = endMulti
   610  			for _, r := range d[1:] {
   611  				if ccc(r) == 0 {
   612  					index = firstCCC
   613  				}
   614  			}
   615  			if lccc > 0 {
   616  				s += string([]byte{lccc})
   617  				if index == firstCCC {
   618  					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
   619  				}
   620  				index = firstLeadingCCC
   621  			}
   622  			if cc != lccc {
   623  				if cc != 0 {
   624  					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
   625  				}
   626  				index = firstCCCZeroExcept
   627  			}
   628  		} else if len(d) > 1 {
   629  			index = firstMulti
   630  		}
   631  		return index, s
   632  	}
   633  
   634  	decompSet := makeDecompSet()
   635  	const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
   636  	decompSet.insert(firstStarterWithNLead, nLeadStr)
   637  
   638  	// Store the uniqued decompositions in a byte buffer,
   639  	// preceded by their byte length.
   640  	for _, c := range chars {
   641  		for _, f := range c.forms {
   642  			if len(f.expandedDecomp) == 0 {
   643  				continue
   644  			}
   645  			if f.combinesBackward {
   646  				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
   647  			}
   648  			index, s := mkstr(c.codePoint, &f)
   649  			decompSet.insert(index, s)
   650  		}
   651  	}
   652  
   653  	decompositions := bytes.NewBuffer(make([]byte, 0, 10000))
   654  	size := 0
   655  	positionMap := make(map[string]uint16)
   656  	decompositions.WriteString("\000")
   657  	fmt.Fprintln(w, "const (")
   658  	for i, m := range decompSet {
   659  		sa := []string{}
   660  		for s := range m {
   661  			sa = append(sa, s)
   662  		}
   663  		sort.Strings(sa)
   664  		for _, s := range sa {
   665  			p := decompositions.Len()
   666  			decompositions.WriteString(s)
   667  			positionMap[s] = uint16(p)
   668  		}
   669  		if cname[i] != "" {
   670  			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
   671  		}
   672  	}
   673  	fmt.Fprintln(w, "maxDecomp = 0x8000")
   674  	fmt.Fprintln(w, ")")
   675  	b := decompositions.Bytes()
   676  	printBytes(w, b, "decomps")
   677  	size += len(b)
   678  
   679  	varnames := []string{"nfc", "nfkc"}
   680  	for i := 0; i < FNumberOfFormTypes; i++ {
   681  		trie := triegen.NewTrie(varnames[i])
   682  
   683  		for r, c := range chars {
   684  			f := c.forms[i]
   685  			d := f.expandedDecomp
   686  			if len(d) != 0 {
   687  				_, key := mkstr(c.codePoint, &f)
   688  				trie.Insert(rune(r), uint64(positionMap[key]))
   689  				if c.ccc != ccc(d[0]) {
   690  					// We assume the lead ccc of a decomposition !=0 in this case.
   691  					if ccc(d[0]) == 0 {
   692  						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
   693  					}
   694  				}
   695  			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
   696  				// Handle cases where it can't be detected that the nLead should be equal
   697  				// to nTrail.
   698  				trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
   699  			} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
   700  				trie.Insert(c.codePoint, uint64(0x8000|v))
   701  			}
   702  		}
   703  		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
   704  		if err != nil {
   705  			log.Fatal(err)
   706  		}
   707  		size += sz
   708  	}
   709  	return size
   710  }
   711  
   712  func contains(sa []string, s string) bool {
   713  	for _, a := range sa {
   714  		if a == s {
   715  			return true
   716  		}
   717  	}
   718  	return false
   719  }
   720  
   721  func makeTables() {
   722  	w := &bytes.Buffer{}
   723  
   724  	size := 0
   725  	if *tablelist == "" {
   726  		return
   727  	}
   728  	list := strings.Split(*tablelist, ",")
   729  	if *tablelist == "all" {
   730  		list = []string{"recomp", "info"}
   731  	}
   732  
   733  	// Compute maximum decomposition size.
   734  	max := 0
   735  	for _, c := range chars {
   736  		if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max {
   737  			max = n
   738  		}
   739  	}
   740  
   741  	fmt.Fprintln(w, "const (")
   742  	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
   743  	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
   744  	fmt.Fprintln(w)
   745  	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
   746  	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
   747  	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
   748  	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
   749  	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
   750  	fmt.Fprintln(w, ")\n")
   751  
   752  	// Print the CCC remap table.
   753  	size += len(cccMap)
   754  	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
   755  	for i := 0; i < len(cccMap); i++ {
   756  		if i%8 == 0 {
   757  			fmt.Fprintln(w)
   758  		}
   759  		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
   760  	}
   761  	fmt.Fprintln(w, "\n}\n")
   762  
   763  	if contains(list, "info") {
   764  		size += printCharInfoTables(w)
   765  	}
   766  
   767  	if contains(list, "recomp") {
   768  		// Note that we use 32 bit keys, instead of 64 bit.
   769  		// This clips the bits of three entries, but we know
   770  		// this won't cause a collision. The compiler will catch
   771  		// any changes made to UnicodeData.txt that introduces
   772  		// a collision.
   773  		// Note that the recomposition map for NFC and NFKC
   774  		// are identical.
   775  
   776  		// Recomposition map
   777  		nrentries := 0
   778  		for _, c := range chars {
   779  			f := c.forms[FCanonical]
   780  			if !f.isOneWay && len(f.decomp) > 0 {
   781  				nrentries++
   782  			}
   783  		}
   784  		sz := nrentries * 8
   785  		size += sz
   786  		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
   787  		fmt.Fprintln(w, "var recompMap = map[uint32]rune{")
   788  		for i, c := range chars {
   789  			f := c.forms[FCanonical]
   790  			d := f.decomp
   791  			if !f.isOneWay && len(d) > 0 {
   792  				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
   793  				fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i)
   794  			}
   795  		}
   796  		fmt.Fprintf(w, "}\n\n")
   797  	}
   798  
   799  	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
   800  	gen.WriteGoFile("tables.go", "norm", w.Bytes())
   801  }
   802  
   803  func printChars() {
   804  	if *verbose {
   805  		for _, c := range chars {
   806  			if !c.isValid() || c.state == SMissing {
   807  				continue
   808  			}
   809  			fmt.Println(c)
   810  		}
   811  	}
   812  }
   813  
   814  // verifyComputed does various consistency tests.
   815  func verifyComputed() {
   816  	for i, c := range chars {
   817  		for _, f := range c.forms {
   818  			isNo := (f.quickCheck[MDecomposed] == QCNo)
   819  			if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) {
   820  				log.Fatalf("%U: NF*D QC must be No if rune decomposes", i)
   821  			}
   822  
   823  			isMaybe := f.quickCheck[MComposed] == QCMaybe
   824  			if f.combinesBackward != isMaybe {
   825  				log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i)
   826  			}
   827  			if len(f.decomp) > 0 && f.combinesForward && isMaybe {
   828  				log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i)
   829  			}
   830  
   831  			if len(f.expandedDecomp) != 0 {
   832  				continue
   833  			}
   834  			if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
   835  				// We accept these runes to be treated differently (it only affects
   836  				// segment breaking in iteration, most likely on improper use), but
   837  				// reconsider if more characters are added.
   838  				// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
   839  				// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
   840  				// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
   841  				// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
   842  				// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
   843  				// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
   844  				if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
   845  					log.Fatalf("%U: nLead was %v; want %v", i, a, b)
   846  				}
   847  			}
   848  		}
   849  		nfc := c.forms[FCanonical]
   850  		nfkc := c.forms[FCompatibility]
   851  		if nfc.combinesBackward != nfkc.combinesBackward {
   852  			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
   853  		}
   854  	}
   855  }
   856  
   857  // Use values in DerivedNormalizationProps.txt to compare against the
   858  // values we computed.
   859  // DerivedNormalizationProps.txt has form:
   860  // 00C0..00C5    ; NFD_QC; N # ...
   861  // 0374          ; NFD_QC; N # ...
   862  // See http://unicode.org/reports/tr44/ for full explanation
   863  func testDerived() {
   864  	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
   865  	defer f.Close()
   866  	p := ucd.New(f)
   867  	for p.Next() {
   868  		r := p.Rune(0)
   869  		c := &chars[r]
   870  
   871  		var ftype, mode int
   872  		qt := p.String(1)
   873  		switch qt {
   874  		case "NFC_QC":
   875  			ftype, mode = FCanonical, MComposed
   876  		case "NFD_QC":
   877  			ftype, mode = FCanonical, MDecomposed
   878  		case "NFKC_QC":
   879  			ftype, mode = FCompatibility, MComposed
   880  		case "NFKD_QC":
   881  			ftype, mode = FCompatibility, MDecomposed
   882  		default:
   883  			continue
   884  		}
   885  		var qr QCResult
   886  		switch p.String(2) {
   887  		case "Y":
   888  			qr = QCYes
   889  		case "N":
   890  			qr = QCNo
   891  		case "M":
   892  			qr = QCMaybe
   893  		default:
   894  			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
   895  		}
   896  		if got := c.forms[ftype].quickCheck[mode]; got != qr {
   897  			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
   898  		}
   899  		c.forms[ftype].verified[mode] = true
   900  	}
   901  	if err := p.Err(); err != nil {
   902  		log.Fatal(err)
   903  	}
   904  	// Any unspecified value must be QCYes. Verify this.
   905  	for i, c := range chars {
   906  		for j, fd := range c.forms {
   907  			for k, qr := range fd.quickCheck {
   908  				if !fd.verified[k] && qr != QCYes {
   909  					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
   910  					log.Printf(m, i, j, k, qr, c.name)
   911  				}
   912  			}
   913  		}
   914  	}
   915  }
   916  
   917  var testHeader = `const (
   918  	Yes = iota
   919  	No
   920  	Maybe
   921  )
   922  
   923  type formData struct {
   924  	qc              uint8
   925  	combinesForward bool
   926  	decomposition   string
   927  }
   928  
   929  type runeData struct {
   930  	r      rune
   931  	ccc    uint8
   932  	nLead  uint8
   933  	nTrail uint8
   934  	f      [2]formData // 0: canonical; 1: compatibility
   935  }
   936  
   937  func f(qc uint8, cf bool, dec string) [2]formData {
   938  	return [2]formData{{qc, cf, dec}, {qc, cf, dec}}
   939  }
   940  
   941  func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData {
   942  	return [2]formData{{qc, cf, d}, {qck, cfk, dk}}
   943  }
   944  
   945  var testData = []runeData{
   946  `
   947  
   948  func printTestdata() {
   949  	type lastInfo struct {
   950  		ccc    uint8
   951  		nLead  uint8
   952  		nTrail uint8
   953  		f      string
   954  	}
   955  
   956  	last := lastInfo{}
   957  	w := &bytes.Buffer{}
   958  	fmt.Fprintf(w, testHeader)
   959  	for r, c := range chars {
   960  		f := c.forms[FCanonical]
   961  		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
   962  		f = c.forms[FCompatibility]
   963  		qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
   964  		s := ""
   965  		if d == dk && qc == qck && cf == cfk {
   966  			s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d)
   967  		} else {
   968  			s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk)
   969  		}
   970  		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
   971  		if last != current {
   972  			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
   973  			last = current
   974  		}
   975  	}
   976  	fmt.Fprintln(w, "}")
   977  	gen.WriteGoFile("data_test.go", "norm", w.Bytes())
   978  }