github.com/vugu/vugu@v0.3.6-0.20240430171613-3f6f402e014b/internal/htmlx/charset/charset_test.go (about) 1 // Copyright 2013 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package charset 6 7 import ( 8 "bytes" 9 "encoding/xml" 10 "io" 11 "os" 12 "runtime" 13 "strings" 14 "testing" 15 16 "golang.org/x/text/transform" 17 ) 18 19 func transformString(t transform.Transformer, s string) (string, error) { 20 r := transform.NewReader(strings.NewReader(s), t) 21 b, err := io.ReadAll(r) 22 return string(b), err 23 } 24 25 type testCase struct { 26 utf8, other, otherEncoding string 27 } 28 29 // testCases for encoding and decoding. 30 var testCases = []testCase{ 31 {"Résumé", "Résumé", "utf8"}, 32 {"Résumé", "R\xe9sum\xe9", "latin1"}, 33 {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"}, 34 {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"}, 35 {"Hello, world", "Hello, world", "ASCII"}, 36 {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"}, 37 {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"}, 38 {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"}, 39 {"latviešu", "latvie\xf0u", "ISO-8859-13"}, 40 {"Seònaid", "Se\xf2naid", "ISO-8859-14"}, 41 {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"}, 42 {"românește", "rom\xe2ne\xbate", "ISO-8859-16"}, 43 {"nutraĵo", "nutra\xbco", "ISO-8859-3"}, 44 {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"}, 45 {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"}, 46 {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"}, 47 {"Kağan", "Ka\xf0an", "ISO-8859-9"}, 48 {"Résumé", "R\x8esum\x8e", "macintosh"}, 49 {"Gdańsk", "Gda\xf1sk", "windows-1250"}, 50 {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"}, 51 {"Résumé", "R\xe9sum\xe9", "windows-1252"}, 52 {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"}, 53 {"Kağan", "Ka\xf0an", "windows-1254"}, 54 {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"}, 55 {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"}, 56 {"latviešu", "latvie\xf0u", "windows-1257"}, 57 {"Việt", "Vi\xea\xf2t", "windows-1258"}, 58 {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"}, 59 {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"}, 60 {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"}, 61 {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"}, 62 {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"}, 63 {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"}, 64 {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"}, 65 {"㧯", "\x82\x31\x89\x38", "gb18030"}, 66 {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"}, 67 {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"}, 68 {"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"}, 69 {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"}, 70 {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"}, 71 {"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"}, 72 } 73 74 func TestDecode(t *testing.T) { 75 testCases := append(testCases, []testCase{ 76 // Replace multi-byte maximum subpart of ill-formed subsequence with 77 // single replacement character (WhatWG requirement). 78 {"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"}, 79 }...) 80 for _, tc := range testCases { 81 e, _ := Lookup(tc.otherEncoding) 82 if e == nil { 83 t.Errorf("%s: not found", tc.otherEncoding) 84 continue 85 } 86 s, err := transformString(e.NewDecoder(), tc.other) 87 if err != nil { 88 t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err) 89 continue 90 } 91 if s != tc.utf8 { 92 t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8) 93 } 94 } 95 } 96 97 func TestEncode(t *testing.T) { 98 testCases := append(testCases, []testCase{ 99 // Use Go-style replacement. 100 {"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"}, 101 // U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding. 102 {"Gdańsk", "Gdańsk", "ISO-8859-11"}, 103 {"\ufffd", "�", "ISO-8859-11"}, 104 {"a\xe1\x80b", "a��b", "ISO-8859-11"}, 105 }...) 106 for _, tc := range testCases { 107 e, _ := Lookup(tc.otherEncoding) 108 if e == nil { 109 t.Errorf("%s: not found", tc.otherEncoding) 110 continue 111 } 112 s, err := transformString(e.NewEncoder(), tc.utf8) 113 if err != nil { 114 t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err) 115 continue 116 } 117 if s != tc.other { 118 t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other) 119 } 120 } 121 } 122 123 var sniffTestCases = []struct { 124 filename, declared, want string 125 }{ 126 {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, 127 {"UTF-16LE-BOM.html", "", "utf-16le"}, 128 {"UTF-16BE-BOM.html", "", "utf-16be"}, 129 {"meta-content-attribute.html", "text/html", "iso-8859-15"}, 130 {"meta-charset-attribute.html", "text/html", "iso-8859-15"}, 131 {"No-encoding-declaration.html", "text/html", "utf-8"}, 132 {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"}, 133 {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, 134 {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"}, 135 {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"}, 136 {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"}, 137 } 138 139 func TestSniff(t *testing.T) { 140 switch runtime.GOOS { 141 case "nacl": // platforms that don't permit direct file system access 142 t.Skipf("not supported on %q", runtime.GOOS) 143 } 144 145 for _, tc := range sniffTestCases { 146 content, err := os.ReadFile("testdata/" + tc.filename) 147 if err != nil { 148 t.Errorf("%s: error reading file: %v", tc.filename, err) 149 continue 150 } 151 152 _, name, _ := DetermineEncoding(content, tc.declared) 153 if name != tc.want { 154 t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want) 155 continue 156 } 157 } 158 } 159 160 func TestReader(t *testing.T) { 161 switch runtime.GOOS { 162 case "nacl": // platforms that don't permit direct file system access 163 t.Skipf("not supported on %q", runtime.GOOS) 164 } 165 166 for _, tc := range sniffTestCases { 167 content, err := os.ReadFile("testdata/" + tc.filename) 168 if err != nil { 169 t.Errorf("%s: error reading file: %v", tc.filename, err) 170 continue 171 } 172 173 r, err := NewReader(bytes.NewReader(content), tc.declared) 174 if err != nil { 175 t.Errorf("%s: error creating reader: %v", tc.filename, err) 176 continue 177 } 178 179 got, err := io.ReadAll(r) 180 if err != nil { 181 t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err) 182 continue 183 } 184 185 e, _ := Lookup(tc.want) 186 want, err := io.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder())) 187 if err != nil { 188 t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err) 189 continue 190 } 191 192 if !bytes.Equal(got, want) { 193 t.Errorf("%s: got %q, want %q", tc.filename, got, want) 194 continue 195 } 196 } 197 } 198 199 var metaTestCases = []struct { 200 meta, want string 201 }{ 202 {"", ""}, 203 {"text/html", ""}, 204 {"text/html; charset utf-8", ""}, 205 {"text/html; charset=latin-2", "latin-2"}, 206 {"text/html; charset; charset = utf-8", "utf-8"}, 207 {`charset="big5"`, "big5"}, 208 {"charset='shift_jis'", "shift_jis"}, 209 } 210 211 func TestFromMeta(t *testing.T) { 212 for _, tc := range metaTestCases { 213 got := fromMetaElement(tc.meta) 214 if got != tc.want { 215 t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want) 216 } 217 } 218 } 219 220 func TestXML(t *testing.T) { 221 const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><a><Word>r\xe9sum\xe9</Word></a>" 222 223 d := xml.NewDecoder(strings.NewReader(s)) 224 d.CharsetReader = NewReaderLabel 225 226 var a struct { 227 Word string 228 } 229 err := d.Decode(&a) 230 if err != nil { 231 t.Fatalf("Decode: %v", err) 232 } 233 234 want := "résumé" 235 if a.Word != want { 236 t.Errorf("got %q, want %q", a.Word, want) 237 } 238 }