github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/mahonia/mahonia_test.go (about) 1 package mahonia 2 3 import ( 4 "bytes" 5 "io/ioutil" 6 "testing" 7 ) 8 9 var nameTests = map[string]string{ 10 "utf8": "utf8", 11 "ISO 8859-1": "iso88591", 12 "Big5": "big5", 13 "": "", 14 } 15 16 func TestSimplifyName(t *testing.T) { 17 for name, simple := range nameTests { 18 if simple != simplifyName(name) { 19 t.Errorf("%s came out as %s instead of as %s", name, simplifyName(name), simple) 20 } 21 } 22 } 23 24 var testData = []struct { 25 utf8, other, otherEncoding string 26 }{ 27 {"Résumé", "Résumé", "utf8"}, 28 {"Résumé", "R\xe9sum\xe9", "latin-1"}, 29 {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"}, 30 {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"}, 31 {"これは漢字です。", "\xfe\xff0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16"}, 32 {"𝄢𝄞𝄪𝄫", "\xfe\xff\xd8\x34\xdd\x22\xd8\x34\xdd\x1e\xd8\x34\xdd\x2a\xd8\x34\xdd\x2b", "UTF-16"}, 33 {"Hello, world", "Hello, world", "ASCII"}, 34 {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"}, 35 {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"}, 36 {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"}, 37 {"latviešu", "latvie\xf0u", "ISO-8859-13"}, 38 {"Seònaid", "Se\xf2naid", "ISO-8859-14"}, 39 {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"}, 40 {"românește", "rom\xe2ne\xbate", "ISO-8859-16"}, 41 {"nutraĵo", "nutra\xbco", "ISO-8859-3"}, 42 {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"}, 43 {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"}, 44 {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"}, 45 {"Kağan", "Ka\xf0an", "ISO-8859-9"}, 46 {"Résumé", "R\x8esum\x8e", "macintosh"}, 47 {"Gdańsk", "Gda\xf1sk", "windows-1250"}, 48 {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"}, 49 {"Résumé", "R\xe9sum\xe9", "windows-1252"}, 50 {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"}, 51 {"Kağan", "Ka\xf0an", "windows-1254"}, 52 {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"}, 53 {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"}, 54 {"latviešu", "latvie\xf0u", "windows-1257"}, 55 {"Việt", "Vi\xea\xf2t", "windows-1258"}, 56 {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"}, 57 {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"}, 58 {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"}, 59 {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"}, 60 {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"}, 61 {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"}, 62 {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"}, 63 {"㧯", "\x82\x31\x89\x38", "gb18030"}, 64 {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"}, 65 {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"}, 66 {"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"}, 67 {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"}, 68 {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "CP51932"}, 69 {"Thông tin bạn đồng hànhỌ", "Th\xabng tin b\xb9n \xae\xe5ng h\xb5nhO\xe4", "TCVN3"}, 70 {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"}, 71 {"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"}, 72 } 73 74 func TestDecode(t *testing.T) { 75 for _, data := range testData { 76 d := NewDecoder(data.otherEncoding) 77 if d == nil { 78 t.Errorf("Could not create decoder for %s", data.otherEncoding) 79 continue 80 } 81 82 str := d.ConvertString(data.other) 83 84 if str != data.utf8 { 85 t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8) 86 } 87 } 88 } 89 90 func TestDecodeTranslate(t *testing.T) { 91 for _, data := range testData { 92 d := NewDecoder(data.otherEncoding) 93 if d == nil { 94 t.Errorf("Could not create decoder for %s", data.otherEncoding) 95 continue 96 } 97 98 _, cdata, _ := d.Translate([]byte(data.other), true) 99 str := string(cdata) 100 101 if str != data.utf8 { 102 t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8) 103 } 104 } 105 } 106 107 func TestEncode(t *testing.T) { 108 for _, data := range testData { 109 e := NewEncoder(data.otherEncoding) 110 if e == nil { 111 t.Errorf("Could not create encoder for %s", data.otherEncoding) 112 continue 113 } 114 115 str := e.ConvertString(data.utf8) 116 117 if str != data.other { 118 t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other) 119 } 120 } 121 } 122 123 func TestReader(t *testing.T) { 124 for _, data := range testData { 125 d := NewDecoder(data.otherEncoding) 126 if d == nil { 127 t.Errorf("Could not create decoder for %s", data.otherEncoding) 128 continue 129 } 130 131 b := bytes.NewBufferString(data.other) 132 r := d.NewReader(b) 133 result, _ := ioutil.ReadAll(r) 134 str := string(result) 135 136 if str != data.utf8 { 137 t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8) 138 } 139 } 140 } 141 142 func TestWriter(t *testing.T) { 143 for _, data := range testData { 144 e := NewEncoder(data.otherEncoding) 145 if e == nil { 146 t.Errorf("Could not create encoder for %s", data.otherEncoding) 147 continue 148 } 149 150 b := new(bytes.Buffer) 151 w := e.NewWriter(b) 152 w.Write([]byte(data.utf8)) 153 str := b.String() 154 155 if str != data.other { 156 t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other) 157 } 158 } 159 } 160 161 func TestFallback(t *testing.T) { 162 mixed := "résum\xe9 " // The space is needed because of the issue mentioned in the Note: in fallback.go 163 pure := "résumé " 164 d := FallbackDecoder(NewDecoder("utf8"), NewDecoder("ISO-8859-1")) 165 result := d.ConvertString(mixed) 166 if result != pure { 167 t.Errorf("Unexpected value: %#v (expected %#v)", result, pure) 168 } 169 } 170 171 func TestEntities(t *testing.T) { 172 escaped := "¬it; I'm ∉ I tell you‚ ≪⃒ " 173 plain := "¬it; I'm ∉ I tell you\u201a \u226A\u20D2 " 174 d := FallbackDecoder(EntityDecoder(), NewDecoder("ISO-8859-1")) 175 result := d.ConvertString(escaped) 176 if result != plain { 177 t.Errorf("Unexpected value: %#v (expected %#v)", result, plain) 178 } 179 } 180 181 func TestConvertStringOK(t *testing.T) { 182 d := NewDecoder("ASCII") 183 if d == nil { 184 t.Fatal("Could not create decoder for ASCII") 185 } 186 187 str, ok := d.ConvertStringOK("hello") 188 if !ok { 189 t.Error("Spurious error found while decoding") 190 } 191 if str != "hello" { 192 t.Errorf("expected %#v, got %#v", "hello", str) 193 } 194 195 str, ok = d.ConvertStringOK("\x80") 196 if ok { 197 t.Error(`Failed to detect error decoding "\x80"`) 198 } 199 200 e := NewEncoder("ISO-8859-3") 201 if e == nil { 202 t.Fatal("Could not create encoder for ISO-8859-1") 203 } 204 205 str, ok = e.ConvertStringOK("nutraĵo") 206 if !ok { 207 t.Error("spurious error while encoding") 208 } 209 if str != "nutra\xbco" { 210 t.Errorf("expected %#v, got %#v", "nutra\xbco", str) 211 } 212 213 str, ok = e.ConvertStringOK("\x80abc") 214 if ok { 215 t.Error("failed to detect invalid UTF-8 while encoding") 216 } 217 218 str, ok = e.ConvertStringOK("русский") 219 if ok { 220 t.Error("failed to detect characters that couldn't be encoded") 221 } 222 } 223 224 func TestBadCharset(t *testing.T) { 225 d := NewDecoder("this is not a valid charset") 226 if d != nil { 227 t.Fatal("got a non-nil decoder for an invalid charset") 228 } 229 }