github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/mahonia/mahonia_test.go (about)

     1  package mahonia
     2  
     3  import (
     4  	"bytes"
     5  	"io/ioutil"
     6  	"testing"
     7  )
     8  
     9  var nameTests = map[string]string{
    10  	"utf8":       "utf8",
    11  	"ISO 8859-1": "iso88591",
    12  	"Big5":       "big5",
    13  	"":           "",
    14  }
    15  
    16  func TestSimplifyName(t *testing.T) {
    17  	for name, simple := range nameTests {
    18  		if simple != simplifyName(name) {
    19  			t.Errorf("%s came out as %s instead of as %s", name, simplifyName(name), simple)
    20  		}
    21  	}
    22  }
    23  
    24  var testData = []struct {
    25  	utf8, other, otherEncoding string
    26  }{
    27  	{"Résumé", "Résumé", "utf8"},
    28  	{"Résumé", "R\xe9sum\xe9", "latin-1"},
    29  	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
    30  	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
    31  	{"これは漢字です。", "\xfe\xff0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16"},
    32  	{"𝄢𝄞𝄪𝄫", "\xfe\xff\xd8\x34\xdd\x22\xd8\x34\xdd\x1e\xd8\x34\xdd\x2a\xd8\x34\xdd\x2b", "UTF-16"},
    33  	{"Hello, world", "Hello, world", "ASCII"},
    34  	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
    35  	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
    36  	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
    37  	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
    38  	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
    39  	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
    40  	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
    41  	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
    42  	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
    43  	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
    44  	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
    45  	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
    46  	{"Résumé", "R\x8esum\x8e", "macintosh"},
    47  	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
    48  	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
    49  	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
    50  	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
    51  	{"Kağan", "Ka\xf0an", "windows-1254"},
    52  	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
    53  	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
    54  	{"latviešu", "latvie\xf0u", "windows-1257"},
    55  	{"Việt", "Vi\xea\xf2t", "windows-1258"},
    56  	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
    57  	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
    58  	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
    59  	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
    60  	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
    61  	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
    62  	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
    63  	{"㧯", "\x82\x31\x89\x38", "gb18030"},
    64  	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
    65  	{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
    66  	{"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
    67  	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
    68  	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "CP51932"},
    69  	{"Thông tin bạn đồng hànhỌ", "Th\xabng tin b\xb9n \xae\xe5ng h\xb5nhO\xe4", "TCVN3"},
    70  	{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
    71  	{"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"},
    72  }
    73  
    74  func TestDecode(t *testing.T) {
    75  	for _, data := range testData {
    76  		d := NewDecoder(data.otherEncoding)
    77  		if d == nil {
    78  			t.Errorf("Could not create decoder for %s", data.otherEncoding)
    79  			continue
    80  		}
    81  
    82  		str := d.ConvertString(data.other)
    83  
    84  		if str != data.utf8 {
    85  			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8)
    86  		}
    87  	}
    88  }
    89  
    90  func TestDecodeTranslate(t *testing.T) {
    91  	for _, data := range testData {
    92  		d := NewDecoder(data.otherEncoding)
    93  		if d == nil {
    94  			t.Errorf("Could not create decoder for %s", data.otherEncoding)
    95  			continue
    96  		}
    97  
    98  		_, cdata, _ := d.Translate([]byte(data.other), true)
    99  		str := string(cdata)
   100  
   101  		if str != data.utf8 {
   102  			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8)
   103  		}
   104  	}
   105  }
   106  
   107  func TestEncode(t *testing.T) {
   108  	for _, data := range testData {
   109  		e := NewEncoder(data.otherEncoding)
   110  		if e == nil {
   111  			t.Errorf("Could not create encoder for %s", data.otherEncoding)
   112  			continue
   113  		}
   114  
   115  		str := e.ConvertString(data.utf8)
   116  
   117  		if str != data.other {
   118  			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other)
   119  		}
   120  	}
   121  }
   122  
   123  func TestReader(t *testing.T) {
   124  	for _, data := range testData {
   125  		d := NewDecoder(data.otherEncoding)
   126  		if d == nil {
   127  			t.Errorf("Could not create decoder for %s", data.otherEncoding)
   128  			continue
   129  		}
   130  
   131  		b := bytes.NewBufferString(data.other)
   132  		r := d.NewReader(b)
   133  		result, _ := ioutil.ReadAll(r)
   134  		str := string(result)
   135  
   136  		if str != data.utf8 {
   137  			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8)
   138  		}
   139  	}
   140  }
   141  
   142  func TestWriter(t *testing.T) {
   143  	for _, data := range testData {
   144  		e := NewEncoder(data.otherEncoding)
   145  		if e == nil {
   146  			t.Errorf("Could not create encoder for %s", data.otherEncoding)
   147  			continue
   148  		}
   149  
   150  		b := new(bytes.Buffer)
   151  		w := e.NewWriter(b)
   152  		w.Write([]byte(data.utf8))
   153  		str := b.String()
   154  
   155  		if str != data.other {
   156  			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other)
   157  		}
   158  	}
   159  }
   160  
   161  func TestFallback(t *testing.T) {
   162  	mixed := "résum\xe9 " // The space is needed because of the issue mentioned in the Note: in fallback.go
   163  	pure := "résumé "
   164  	d := FallbackDecoder(NewDecoder("utf8"), NewDecoder("ISO-8859-1"))
   165  	result := d.ConvertString(mixed)
   166  	if result != pure {
   167  		t.Errorf("Unexpected value: %#v (expected %#v)", result, pure)
   168  	}
   169  }
   170  
   171  func TestEntities(t *testing.T) {
   172  	escaped := "&notit; I'm ∉ I tell you&#X82 ≪⃒ "
   173  	plain := "¬it; I'm ∉ I tell you\u201a \u226A\u20D2 "
   174  	d := FallbackDecoder(EntityDecoder(), NewDecoder("ISO-8859-1"))
   175  	result := d.ConvertString(escaped)
   176  	if result != plain {
   177  		t.Errorf("Unexpected value: %#v (expected %#v)", result, plain)
   178  	}
   179  }
   180  
   181  func TestConvertStringOK(t *testing.T) {
   182  	d := NewDecoder("ASCII")
   183  	if d == nil {
   184  		t.Fatal("Could not create decoder for ASCII")
   185  	}
   186  
   187  	str, ok := d.ConvertStringOK("hello")
   188  	if !ok {
   189  		t.Error("Spurious error found while decoding")
   190  	}
   191  	if str != "hello" {
   192  		t.Errorf("expected %#v, got %#v", "hello", str)
   193  	}
   194  
   195  	str, ok = d.ConvertStringOK("\x80")
   196  	if ok {
   197  		t.Error(`Failed to detect error decoding "\x80"`)
   198  	}
   199  
   200  	e := NewEncoder("ISO-8859-3")
   201  	if e == nil {
   202  		t.Fatal("Could not create encoder for ISO-8859-1")
   203  	}
   204  
   205  	str, ok = e.ConvertStringOK("nutraĵo")
   206  	if !ok {
   207  		t.Error("spurious error while encoding")
   208  	}
   209  	if str != "nutra\xbco" {
   210  		t.Errorf("expected %#v, got %#v", "nutra\xbco", str)
   211  	}
   212  
   213  	str, ok = e.ConvertStringOK("\x80abc")
   214  	if ok {
   215  		t.Error("failed to detect invalid UTF-8 while encoding")
   216  	}
   217  
   218  	str, ok = e.ConvertStringOK("русский")
   219  	if ok {
   220  		t.Error("failed to detect characters that couldn't be encoded")
   221  	}
   222  }
   223  
   224  func TestBadCharset(t *testing.T) {
   225  	d := NewDecoder("this is not a valid charset")
   226  	if d != nil {
   227  		t.Fatal("got a non-nil decoder for an invalid charset")
   228  	}
   229  }