code.gitea.io/gitea@v1.19.3/modules/charset/ambiguous/generate.go (about)

     1  // Copyright 2022 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package main
     5  
     6  import (
     7  	"bytes"
     8  	"flag"
     9  	"fmt"
    10  	"go/format"
    11  	"os"
    12  	"sort"
    13  	"text/template"
    14  	"unicode"
    15  
    16  	"code.gitea.io/gitea/modules/json"
    17  
    18  	"golang.org/x/text/unicode/rangetable"
    19  )
    20  
    21  // ambiguous.json provides a one to one mapping of ambiguous characters to other characters
    22  // See https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json
    23  
    24  type AmbiguousTable struct {
    25  	Confusable []rune
    26  	With       []rune
    27  	Locale     string
    28  	RangeTable *unicode.RangeTable
    29  }
    30  
    31  type RunePair struct {
    32  	Confusable rune
    33  	With       rune
    34  }
    35  
    36  var verbose bool
    37  
    38  func main() {
    39  	flag.Usage = func() {
    40  		fmt.Fprintf(os.Stderr, `%s: Generate AmbiguousCharacter
    41  
    42  Usage: %[1]s [-v] [-o output.go] ambiguous.json
    43  `, os.Args[0])
    44  		flag.PrintDefaults()
    45  	}
    46  
    47  	output := ""
    48  	flag.BoolVar(&verbose, "v", false, "verbose output")
    49  	flag.StringVar(&output, "o", "ambiguous_gen.go", "file to output to")
    50  	flag.Parse()
    51  	input := flag.Arg(0)
    52  	if input == "" {
    53  		input = "ambiguous.json"
    54  	}
    55  
    56  	bs, err := os.ReadFile(input)
    57  	if err != nil {
    58  		fatalf("Unable to read: %s Err: %v", input, err)
    59  	}
    60  
    61  	var unwrapped string
    62  	if err := json.Unmarshal(bs, &unwrapped); err != nil {
    63  		fatalf("Unable to unwrap content in: %s Err: %v", input, err)
    64  	}
    65  
    66  	fromJSON := map[string][]uint32{}
    67  	if err := json.Unmarshal([]byte(unwrapped), &fromJSON); err != nil {
    68  		fatalf("Unable to unmarshal content in: %s Err: %v", input, err)
    69  	}
    70  
    71  	tables := make([]*AmbiguousTable, 0, len(fromJSON))
    72  	for locale, chars := range fromJSON {
    73  		table := &AmbiguousTable{Locale: locale}
    74  		table.Confusable = make([]rune, 0, len(chars)/2)
    75  		table.With = make([]rune, 0, len(chars)/2)
    76  		pairs := make([]RunePair, len(chars)/2)
    77  		for i := 0; i < len(chars); i += 2 {
    78  			pairs[i/2].Confusable, pairs[i/2].With = rune(chars[i]), rune(chars[i+1])
    79  		}
    80  		sort.Slice(pairs, func(i, j int) bool {
    81  			return pairs[i].Confusable < pairs[j].Confusable
    82  		})
    83  		for _, pair := range pairs {
    84  			table.Confusable = append(table.Confusable, pair.Confusable)
    85  			table.With = append(table.With, pair.With)
    86  		}
    87  		table.RangeTable = rangetable.New(table.Confusable...)
    88  		tables = append(tables, table)
    89  	}
    90  	sort.Slice(tables, func(i, j int) bool {
    91  		return tables[i].Locale < tables[j].Locale
    92  	})
    93  	data := map[string]interface{}{
    94  		"Tables": tables,
    95  	}
    96  
    97  	if err := runTemplate(generatorTemplate, output, &data); err != nil {
    98  		fatalf("Unable to run template: %v", err)
    99  	}
   100  }
   101  
   102  func runTemplate(t *template.Template, filename string, data interface{}) error {
   103  	buf := bytes.NewBuffer(nil)
   104  	if err := t.Execute(buf, data); err != nil {
   105  		return fmt.Errorf("unable to execute template: %w", err)
   106  	}
   107  	bs, err := format.Source(buf.Bytes())
   108  	if err != nil {
   109  		verbosef("Bad source:\n%s", buf.String())
   110  		return fmt.Errorf("unable to format source: %w", err)
   111  	}
   112  
   113  	old, err := os.ReadFile(filename)
   114  	if err != nil && !os.IsNotExist(err) {
   115  		return fmt.Errorf("failed to read old file %s because %w", filename, err)
   116  	} else if err == nil {
   117  		if bytes.Equal(bs, old) {
   118  			// files are the same don't rewrite it.
   119  			return nil
   120  		}
   121  	}
   122  
   123  	file, err := os.Create(filename)
   124  	if err != nil {
   125  		return fmt.Errorf("failed to create file %s because %w", filename, err)
   126  	}
   127  	defer file.Close()
   128  	_, err = file.Write(bs)
   129  	if err != nil {
   130  		return fmt.Errorf("unable to write generated source: %w", err)
   131  	}
   132  	return nil
   133  }
   134  
   135  var generatorTemplate = template.Must(template.New("ambiguousTemplate").Parse(`// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT
   136  // Copyright 2022 The Gitea Authors. All rights reserved.
   137  // SPDX-License-Identifier: MIT
   138  
   139  
   140  package charset
   141  
   142  import "unicode"
   143  
   144  // This file is generated from https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json
   145  
   146  // AmbiguousTable matches a confusable rune with its partner for the Locale
   147  type AmbiguousTable struct {
   148  	Confusable []rune
   149  	With       []rune
   150  	Locale     string
   151  	RangeTable *unicode.RangeTable
   152  }
   153  
   154  // AmbiguousCharacters provides a map by locale name to the confusable characters in that locale
   155  var AmbiguousCharacters = map[string]*AmbiguousTable{
   156  	{{range .Tables}}{{printf "%q:" .Locale}} {
   157  			Confusable: []rune{ {{range .Confusable}}{{.}},{{end}} },
   158  			With: []rune{ {{range .With}}{{.}},{{end}} },
   159  			Locale: {{printf "%q" .Locale}},
   160  			RangeTable: &unicode.RangeTable{
   161  				R16: []unicode.Range16{
   162  			{{range .RangeTable.R16 }}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
   163  			{{end}}	},
   164  				R32: []unicode.Range32{
   165  			{{range .RangeTable.R32}}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
   166  			{{end}}	},
   167  				LatinOffset: {{.RangeTable.LatinOffset}},
   168  			},
   169  		},
   170  	{{end}}
   171  }
   172  
   173  `))
   174  
   175  func logf(format string, args ...interface{}) {
   176  	fmt.Fprintf(os.Stderr, format+"\n", args...)
   177  }
   178  
   179  func verbosef(format string, args ...interface{}) {
   180  	if verbose {
   181  		logf(format, args...)
   182  	}
   183  }
   184  
   185  func fatalf(format string, args ...interface{}) {
   186  	logf("fatal: "+format+"\n", args...)
   187  	os.Exit(1)
   188  }