github.com/go-enjin/golang-org-x-text@v0.12.1-enjin.2/encoding/simplifiedchinese/maketables.go (about)

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  // +build ignore
     7  
     8  package main
     9  
    10  // This program generates tables.go:
    11  //	go run maketables.go | gofmt > tables.go
    12  
    13  import (
    14  	"bufio"
    15  	"fmt"
    16  	"log"
    17  	"net/http"
    18  	"sort"
    19  	"strings"
    20  )
    21  
    22  func main() {
    23  	fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
    24  	fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n")
    25  	fmt.Printf(`package simplifiedchinese // import "github.com/go-enjin/golang-org-x-text/encoding/simplifiedchinese"` + "\n\n")
    26  
    27  	printGB18030()
    28  	printGBK()
    29  }
    30  
    31  func printGB18030() {
    32  	res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt")
    33  	if err != nil {
    34  		log.Fatalf("Get: %v", err)
    35  	}
    36  	defer res.Body.Close()
    37  
    38  	fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n")
    39  	fmt.Printf("var gb18030 = [...][2]uint16{\n")
    40  	scanner := bufio.NewScanner(res.Body)
    41  	for scanner.Scan() {
    42  		s := strings.TrimSpace(scanner.Text())
    43  		if s == "" || s[0] == '#' {
    44  			continue
    45  		}
    46  		x, y := uint32(0), uint32(0)
    47  		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
    48  			log.Fatalf("could not parse %q", s)
    49  		}
    50  		if x < 0x10000 && y < 0x10000 {
    51  			fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y)
    52  		}
    53  	}
    54  	fmt.Printf("}\n\n")
    55  }
    56  
    57  func printGBK() {
    58  	res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt")
    59  	if err != nil {
    60  		log.Fatalf("Get: %v", err)
    61  	}
    62  	defer res.Body.Close()
    63  
    64  	mapping := [65536]uint16{}
    65  	reverse := [65536]uint16{}
    66  
    67  	scanner := bufio.NewScanner(res.Body)
    68  	for scanner.Scan() {
    69  		s := strings.TrimSpace(scanner.Text())
    70  		if s == "" || s[0] == '#' {
    71  			continue
    72  		}
    73  		x, y := uint16(0), uint16(0)
    74  		if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
    75  			log.Fatalf("could not parse %q", s)
    76  		}
    77  		if x < 0 || 126*190 <= x {
    78  			log.Fatalf("GBK code %d is out of range", x)
    79  		}
    80  		mapping[x] = y
    81  		if reverse[y] == 0 {
    82  			c0, c1 := x/190, x%190
    83  			if c1 >= 0x3f {
    84  				c1++
    85  			}
    86  			reverse[y] = (0x81+c0)<<8 | (0x40 + c1)
    87  		}
    88  	}
    89  	if err := scanner.Err(); err != nil {
    90  		log.Fatalf("scanner error: %v", err)
    91  	}
    92  
    93  	fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n")
    94  	fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n")
    95  	fmt.Printf("var decode = [...]uint16{\n")
    96  	for i, v := range mapping {
    97  		if v != 0 {
    98  			fmt.Printf("\t%d: 0x%04X,\n", i, v)
    99  		}
   100  	}
   101  	fmt.Printf("}\n\n")
   102  
   103  	// Any run of at least separation continuous zero entries in the reverse map will
   104  	// be a separate encode table.
   105  	const separation = 1024
   106  
   107  	intervals := []interval(nil)
   108  	low, high := -1, -1
   109  	for i, v := range reverse {
   110  		if v == 0 {
   111  			continue
   112  		}
   113  		if low < 0 {
   114  			low = i
   115  		} else if i-high >= separation {
   116  			if high >= 0 {
   117  				intervals = append(intervals, interval{low, high})
   118  			}
   119  			low = i
   120  		}
   121  		high = i + 1
   122  	}
   123  	if high >= 0 {
   124  		intervals = append(intervals, interval{low, high})
   125  	}
   126  	sort.Sort(byDecreasingLength(intervals))
   127  
   128  	fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
   129  	fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n")
   130  	fmt.Printf("// sorted by decreasing length.\n")
   131  	for i, v := range intervals {
   132  		fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
   133  	}
   134  	fmt.Printf("\n")
   135  
   136  	for i, v := range intervals {
   137  		fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
   138  		fmt.Printf("var encode%d = [...]uint16{\n", i)
   139  		for j := v.low; j < v.high; j++ {
   140  			x := reverse[j]
   141  			if x == 0 {
   142  				continue
   143  			}
   144  			fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
   145  		}
   146  		fmt.Printf("}\n\n")
   147  	}
   148  }
   149  
   150  // interval is a half-open interval [low, high).
   151  type interval struct {
   152  	low, high int
   153  }
   154  
   155  func (i interval) len() int { return i.high - i.low }
   156  
   157  // byDecreasingLength sorts intervals by decreasing length.
   158  type byDecreasingLength []interval
   159  
   160  func (b byDecreasingLength) Len() int           { return len(b) }
   161  func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
   162  func (b byDecreasingLength) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }