code.gitea.io/gitea@v1.19.3/modules/charset/ambiguous/generate.go (about) 1 // Copyright 2022 The Gitea Authors. All rights reserved. 2 // SPDX-License-Identifier: MIT 3 4 package main 5 6 import ( 7 "bytes" 8 "flag" 9 "fmt" 10 "go/format" 11 "os" 12 "sort" 13 "text/template" 14 "unicode" 15 16 "code.gitea.io/gitea/modules/json" 17 18 "golang.org/x/text/unicode/rangetable" 19 ) 20 21 // ambiguous.json provides a one to one mapping of ambiguous characters to other characters 22 // See https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json 23 24 type AmbiguousTable struct { 25 Confusable []rune 26 With []rune 27 Locale string 28 RangeTable *unicode.RangeTable 29 } 30 31 type RunePair struct { 32 Confusable rune 33 With rune 34 } 35 36 var verbose bool 37 38 func main() { 39 flag.Usage = func() { 40 fmt.Fprintf(os.Stderr, `%s: Generate AmbiguousCharacter 41 42 Usage: %[1]s [-v] [-o output.go] ambiguous.json 43 `, os.Args[0]) 44 flag.PrintDefaults() 45 } 46 47 output := "" 48 flag.BoolVar(&verbose, "v", false, "verbose output") 49 flag.StringVar(&output, "o", "ambiguous_gen.go", "file to output to") 50 flag.Parse() 51 input := flag.Arg(0) 52 if input == "" { 53 input = "ambiguous.json" 54 } 55 56 bs, err := os.ReadFile(input) 57 if err != nil { 58 fatalf("Unable to read: %s Err: %v", input, err) 59 } 60 61 var unwrapped string 62 if err := json.Unmarshal(bs, &unwrapped); err != nil { 63 fatalf("Unable to unwrap content in: %s Err: %v", input, err) 64 } 65 66 fromJSON := map[string][]uint32{} 67 if err := json.Unmarshal([]byte(unwrapped), &fromJSON); err != nil { 68 fatalf("Unable to unmarshal content in: %s Err: %v", input, err) 69 } 70 71 tables := make([]*AmbiguousTable, 0, len(fromJSON)) 72 for locale, chars := range fromJSON { 73 table := &AmbiguousTable{Locale: locale} 74 table.Confusable = make([]rune, 0, len(chars)/2) 75 table.With = make([]rune, 0, len(chars)/2) 76 pairs := make([]RunePair, len(chars)/2) 77 for i := 0; i < len(chars); i += 2 { 78 pairs[i/2].Confusable, pairs[i/2].With = rune(chars[i]), rune(chars[i+1]) 79 } 80 sort.Slice(pairs, func(i, j int) bool { 81 return pairs[i].Confusable < pairs[j].Confusable 82 }) 83 for _, pair := range pairs { 84 table.Confusable = append(table.Confusable, pair.Confusable) 85 table.With = append(table.With, pair.With) 86 } 87 table.RangeTable = rangetable.New(table.Confusable...) 88 tables = append(tables, table) 89 } 90 sort.Slice(tables, func(i, j int) bool { 91 return tables[i].Locale < tables[j].Locale 92 }) 93 data := map[string]interface{}{ 94 "Tables": tables, 95 } 96 97 if err := runTemplate(generatorTemplate, output, &data); err != nil { 98 fatalf("Unable to run template: %v", err) 99 } 100 } 101 102 func runTemplate(t *template.Template, filename string, data interface{}) error { 103 buf := bytes.NewBuffer(nil) 104 if err := t.Execute(buf, data); err != nil { 105 return fmt.Errorf("unable to execute template: %w", err) 106 } 107 bs, err := format.Source(buf.Bytes()) 108 if err != nil { 109 verbosef("Bad source:\n%s", buf.String()) 110 return fmt.Errorf("unable to format source: %w", err) 111 } 112 113 old, err := os.ReadFile(filename) 114 if err != nil && !os.IsNotExist(err) { 115 return fmt.Errorf("failed to read old file %s because %w", filename, err) 116 } else if err == nil { 117 if bytes.Equal(bs, old) { 118 // files are the same don't rewrite it. 119 return nil 120 } 121 } 122 123 file, err := os.Create(filename) 124 if err != nil { 125 return fmt.Errorf("failed to create file %s because %w", filename, err) 126 } 127 defer file.Close() 128 _, err = file.Write(bs) 129 if err != nil { 130 return fmt.Errorf("unable to write generated source: %w", err) 131 } 132 return nil 133 } 134 135 var generatorTemplate = template.Must(template.New("ambiguousTemplate").Parse(`// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT 136 // Copyright 2022 The Gitea Authors. All rights reserved. 137 // SPDX-License-Identifier: MIT 138 139 140 package charset 141 142 import "unicode" 143 144 // This file is generated from https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json 145 146 // AmbiguousTable matches a confusable rune with its partner for the Locale 147 type AmbiguousTable struct { 148 Confusable []rune 149 With []rune 150 Locale string 151 RangeTable *unicode.RangeTable 152 } 153 154 // AmbiguousCharacters provides a map by locale name to the confusable characters in that locale 155 var AmbiguousCharacters = map[string]*AmbiguousTable{ 156 {{range .Tables}}{{printf "%q:" .Locale}} { 157 Confusable: []rune{ {{range .Confusable}}{{.}},{{end}} }, 158 With: []rune{ {{range .With}}{{.}},{{end}} }, 159 Locale: {{printf "%q" .Locale}}, 160 RangeTable: &unicode.RangeTable{ 161 R16: []unicode.Range16{ 162 {{range .RangeTable.R16 }} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, 163 {{end}} }, 164 R32: []unicode.Range32{ 165 {{range .RangeTable.R32}} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, 166 {{end}} }, 167 LatinOffset: {{.RangeTable.LatinOffset}}, 168 }, 169 }, 170 {{end}} 171 } 172 173 `)) 174 175 func logf(format string, args ...interface{}) { 176 fmt.Fprintf(os.Stderr, format+"\n", args...) 177 } 178 179 func verbosef(format string, args ...interface{}) { 180 if verbose { 181 logf(format, args...) 182 } 183 } 184 185 func fatalf(format string, args ...interface{}) { 186 logf("fatal: "+format+"\n", args...) 187 os.Exit(1) 188 }