github.com/graybobo/golang.org-package-offline-cache@v0.0.0-20200626051047-6608995c132f/x/text/width/transform_test.go (about)

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package width
     6  
     7  import (
     8  	"bytes"
     9  	"testing"
    10  
    11  	"golang.org/x/text/internal/testtext"
    12  	"golang.org/x/text/transform"
    13  )
    14  
    15  func foldRune(r rune) (folded rune, ok bool) {
    16  	alt, ok := mapRunes[r]
    17  	if ok && alt.e&tagNeedsFold != 0 {
    18  		return alt.r, true
    19  	}
    20  	return r, false
    21  }
    22  
    23  func widenRune(r rune) (wide rune, ok bool) {
    24  	alt, ok := mapRunes[r]
    25  	if k := alt.e.kind(); k == EastAsianHalfwidth || k == EastAsianNarrow {
    26  		return alt.r, true
    27  	}
    28  	return r, false
    29  }
    30  
    31  func narrowRune(r rune) (narrow rune, ok bool) {
    32  	alt, ok := mapRunes[r]
    33  	if k := alt.e.kind(); k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous {
    34  		return alt.r, true
    35  	}
    36  	return r, false
    37  }
    38  
    39  func TestFoldSingleRunes(t *testing.T) {
    40  	for r := rune(0); r < 0x1FFFF; r++ {
    41  		if loSurrogate <= r && r <= hiSurrogate {
    42  			continue
    43  		}
    44  		x, _ := foldRune(r)
    45  		want := string(x)
    46  		got := Fold.String(string(r))
    47  		if got != want {
    48  			t.Errorf("Fold().String(%U) = %+q; want %+q", r, got, want)
    49  		}
    50  	}
    51  }
    52  
    53  func TestFold(t *testing.T) {
    54  	for _, tc := range []struct {
    55  		desc  string
    56  		src   string
    57  		nDst  int
    58  		atEOF bool
    59  		dst   string
    60  		nSrc  int
    61  		err   error
    62  	}{{
    63  		desc:  "empty",
    64  		src:   "",
    65  		dst:   "",
    66  		nDst:  10,
    67  		nSrc:  0,
    68  		atEOF: false,
    69  		err:   nil,
    70  	}, {
    71  		desc:  "short source 1",
    72  		src:   "a\xc2",
    73  		dst:   "a",
    74  		nDst:  10,
    75  		nSrc:  1,
    76  		atEOF: false,
    77  		err:   transform.ErrShortSrc,
    78  	}, {
    79  		desc:  "short source 2",
    80  		src:   "a\xe0\x80",
    81  		dst:   "a",
    82  		nDst:  10,
    83  		nSrc:  1,
    84  		atEOF: false,
    85  		err:   transform.ErrShortSrc,
    86  	}, {
    87  		desc:  "incomplete but terminated source 1",
    88  		src:   "a\xc2",
    89  		dst:   "a\xc2",
    90  		nDst:  10,
    91  		nSrc:  2,
    92  		atEOF: true,
    93  		err:   nil,
    94  	}, {
    95  		desc:  "incomplete but terminated source 2",
    96  		src:   "a\xe0\x80",
    97  		dst:   "a\xe0\x80",
    98  		nDst:  10,
    99  		nSrc:  3,
   100  		atEOF: true,
   101  		err:   nil,
   102  	}, {
   103  		desc:  "exact fit dst",
   104  		src:   "a\uff01",
   105  		dst:   "a!",
   106  		nDst:  2,
   107  		nSrc:  4,
   108  		atEOF: false,
   109  		err:   nil,
   110  	}, {
   111  		desc:  "short dst 1",
   112  		src:   "a\uffe0",
   113  		dst:   "a",
   114  		nDst:  2,
   115  		nSrc:  1,
   116  		atEOF: false,
   117  		err:   transform.ErrShortDst,
   118  	}, {
   119  		desc:  "short dst 2",
   120  		src:   "不夠",
   121  		dst:   "不",
   122  		nDst:  3,
   123  		nSrc:  3,
   124  		atEOF: true,
   125  		err:   transform.ErrShortDst,
   126  	}, {
   127  		desc:  "short dst fast path",
   128  		src:   "fast",
   129  		dst:   "fas",
   130  		nDst:  3,
   131  		nSrc:  3,
   132  		atEOF: true,
   133  		err:   transform.ErrShortDst,
   134  	}, {
   135  		desc:  "fast path alternation",
   136  		src:   "fast路徑fast路徑",
   137  		dst:   "fast路徑fast路徑",
   138  		nDst:  20,
   139  		nSrc:  20,
   140  		atEOF: true,
   141  		err:   nil,
   142  	}} {
   143  		b := make([]byte, tc.nDst)
   144  		nDst, nSrc, err := Fold.Transform(b, []byte(tc.src), tc.atEOF)
   145  		if got := string(b[:nDst]); got != tc.dst {
   146  			t.Errorf("%s: dst was %+q; want %+q", tc.desc, got, tc.dst)
   147  		}
   148  		if nSrc != tc.nSrc {
   149  			t.Errorf("%s: nSrc was %d; want %d", tc.desc, nSrc, tc.nSrc)
   150  		}
   151  		if err != tc.err {
   152  			t.Errorf("%s: error was %v; want %v", tc.desc, err, tc.err)
   153  		}
   154  	}
   155  }
   156  
   157  func TestWidenSingleRunes(t *testing.T) {
   158  	for r := rune(0); r < 0x1FFFF; r++ {
   159  		if loSurrogate <= r && r <= hiSurrogate {
   160  			continue
   161  		}
   162  		alt, _ := widenRune(r)
   163  		want := string(alt)
   164  		got := Widen.String(string(r))
   165  		if got != want {
   166  			t.Errorf("Widen().String(%U) = %+q; want %+q", r, got, want)
   167  		}
   168  	}
   169  }
   170  
   171  func TestWiden(t *testing.T) {
   172  	for _, tc := range []struct {
   173  		desc  string
   174  		src   string
   175  		nDst  int
   176  		atEOF bool
   177  		dst   string
   178  		nSrc  int
   179  		err   error
   180  	}{{
   181  		desc:  "empty",
   182  		src:   "",
   183  		dst:   "",
   184  		nDst:  10,
   185  		nSrc:  0,
   186  		atEOF: false,
   187  		err:   nil,
   188  	}, {
   189  		desc:  "short source 1",
   190  		src:   "a\xc2",
   191  		dst:   "ａ",
   192  		nDst:  10,
   193  		nSrc:  1,
   194  		atEOF: false,
   195  		err:   transform.ErrShortSrc,
   196  	}, {
   197  		desc:  "short source 2",
   198  		src:   "a\xe0\x80",
   199  		dst:   "ａ",
   200  		nDst:  10,
   201  		nSrc:  1,
   202  		atEOF: false,
   203  		err:   transform.ErrShortSrc,
   204  	}, {
   205  		desc:  "incomplete but terminated source 1",
   206  		src:   "a\xc2",
   207  		dst:   "ａ\xc2",
   208  		nDst:  10,
   209  		nSrc:  2,
   210  		atEOF: true,
   211  		err:   nil,
   212  	}, {
   213  		desc:  "incomplete but terminated source 2",
   214  		src:   "a\xe0\x80",
   215  		dst:   "ａ\xe0\x80",
   216  		nDst:  10,
   217  		nSrc:  3,
   218  		atEOF: true,
   219  		err:   nil,
   220  	}, {
   221  		desc:  "exact fit dst",
   222  		src:   "a!",
   223  		dst:   "ａ\uff01",
   224  		nDst:  6,
   225  		nSrc:  2,
   226  		atEOF: false,
   227  		err:   nil,
   228  	}, {
   229  		desc:  "short dst 1",
   230  		src:   "a\uffe0",
   231  		dst:   "ａ",
   232  		nDst:  4,
   233  		nSrc:  1,
   234  		atEOF: false,
   235  		err:   transform.ErrShortDst,
   236  	}, {
   237  		desc:  "short dst 2",
   238  		src:   "不夠",
   239  		dst:   "不",
   240  		nDst:  3,
   241  		nSrc:  3,
   242  		atEOF: true,
   243  		err:   transform.ErrShortDst,
   244  	}, {
   245  		desc:  "short dst ascii",
   246  		src:   "ascii",
   247  		dst:   "\uff41",
   248  		nDst:  3,
   249  		nSrc:  1,
   250  		atEOF: true,
   251  		err:   transform.ErrShortDst,
   252  	}, {
   253  		desc:  "ambiguous",
   254  		src:   "\uffe9",
   255  		dst:   "\u2190",
   256  		nDst:  4,
   257  		nSrc:  3,
   258  		atEOF: false,
   259  		err:   nil,
   260  	}} {
   261  		b := make([]byte, tc.nDst)
   262  		nDst, nSrc, err := Widen.Transform(b, []byte(tc.src), tc.atEOF)
   263  		if got := string(b[:nDst]); got != tc.dst {
   264  			t.Errorf("%s: dst was %+q; want %+q", tc.desc, got, tc.dst)
   265  		}
   266  		if nSrc != tc.nSrc {
   267  			t.Errorf("%s: nSrc was %d; want %d", tc.desc, nSrc, tc.nSrc)
   268  		}
   269  		if err != tc.err {
   270  			t.Errorf("%s: error was %v; want %v", tc.desc, err, tc.err)
   271  		}
   272  	}
   273  }
   274  
   275  func TestNarrowSingleRunes(t *testing.T) {
   276  	for r := rune(0); r < 0x1FFFF; r++ {
   277  		if loSurrogate <= r && r <= hiSurrogate {
   278  			continue
   279  		}
   280  		alt, _ := narrowRune(r)
   281  		want := string(alt)
   282  		got := Narrow.String(string(r))
   283  		if got != want {
   284  			t.Errorf("Narrow().String(%U) = %+q; want %+q", r, got, want)
   285  		}
   286  	}
   287  }
   288  
   289  func TestNarrow(t *testing.T) {
   290  	for _, tc := range []struct {
   291  		desc  string
   292  		src   string
   293  		nDst  int
   294  		atEOF bool
   295  		dst   string
   296  		nSrc  int
   297  		err   error
   298  	}{{
   299  		desc:  "empty",
   300  		src:   "",
   301  		dst:   "",
   302  		nDst:  10,
   303  		nSrc:  0,
   304  		atEOF: false,
   305  		err:   nil,
   306  	}, {
   307  		desc:  "short source 1",
   308  		src:   "a\xc2",
   309  		dst:   "a",
   310  		nDst:  10,
   311  		nSrc:  1,
   312  		atEOF: false,
   313  		err:   transform.ErrShortSrc,
   314  	}, {
   315  		desc:  "short source 2",
   316  		src:   "ａ\xe0\x80",
   317  		dst:   "a",
   318  		nDst:  10,
   319  		nSrc:  3,
   320  		atEOF: false,
   321  		err:   transform.ErrShortSrc,
   322  	}, {
   323  		desc:  "incomplete but terminated source 1",
   324  		src:   "ａ\xc2",
   325  		dst:   "a\xc2",
   326  		nDst:  10,
   327  		nSrc:  4,
   328  		atEOF: true,
   329  		err:   nil,
   330  	}, {
   331  		desc:  "incomplete but terminated source 2",
   332  		src:   "ａ\xe0\x80",
   333  		dst:   "a\xe0\x80",
   334  		nDst:  10,
   335  		nSrc:  5,
   336  		atEOF: true,
   337  		err:   nil,
   338  	}, {
   339  		desc:  "exact fit dst",
   340  		src:   "ａ\uff01",
   341  		dst:   "a!",
   342  		nDst:  2,
   343  		nSrc:  6,
   344  		atEOF: false,
   345  		err:   nil,
   346  	}, {
   347  		desc:  "short dst 1",
   348  		src:   "ａ\uffe0",
   349  		dst:   "a",
   350  		nDst:  2,
   351  		nSrc:  3,
   352  		atEOF: false,
   353  		err:   transform.ErrShortDst,
   354  	}, {
   355  		desc:  "short dst 2",
   356  		src:   "不夠",
   357  		dst:   "不",
   358  		nDst:  3,
   359  		nSrc:  3,
   360  		atEOF: true,
   361  		err:   transform.ErrShortDst,
   362  	}, {
   363  		// Create a narrow variant of ambiguous runes, if they exist.
   364  		desc:  "ambiguous",
   365  		src:   "\u2190",
   366  		dst:   "\uffe9",
   367  		nDst:  4,
   368  		nSrc:  3,
   369  		atEOF: false,
   370  		err:   nil,
   371  	}, {
   372  		desc:  "short dst fast path",
   373  		src:   "fast",
   374  		dst:   "fas",
   375  		nDst:  3,
   376  		nSrc:  3,
   377  		atEOF: true,
   378  		err:   transform.ErrShortDst,
   379  	}, {
   380  		desc:  "fast path alternation",
   381  		src:   "fast路徑fast路徑",
   382  		dst:   "fast路徑fast路徑",
   383  		nDst:  20,
   384  		nSrc:  20,
   385  		atEOF: true,
   386  		err:   nil,
   387  	}} {
   388  		b := make([]byte, tc.nDst)
   389  		nDst, nSrc, err := Narrow.Transform(b, []byte(tc.src), tc.atEOF)
   390  		if got := string(b[:nDst]); got != tc.dst {
   391  			t.Errorf("%s: dst was %+q; want %+q", tc.desc, got, tc.dst)
   392  		}
   393  		if nSrc != tc.nSrc {
   394  			t.Errorf("%s: nSrc was %d; want %d", tc.desc, nSrc, tc.nSrc)
   395  		}
   396  		if err != tc.err {
   397  			t.Errorf("%s: error was %v; want %v", tc.desc, err, tc.err)
   398  		}
   399  	}
   400  }
   401  func bench(b *testing.B, t Transformer, s string) {
   402  	dst := make([]byte, 1024)
   403  	src := []byte(s)
   404  	b.SetBytes(int64(len(src)))
   405  	b.ResetTimer()
   406  	for i := 0; i < b.N; i++ {
   407  		t.Transform(dst, src, true)
   408  	}
   409  }
   410  
   411  func changingRunes(f func(r rune) (rune, bool)) string {
   412  	buf := &bytes.Buffer{}
   413  	for r := rune(0); r <= 0xFFFF; r++ {
   414  		if _, ok := foldRune(r); ok {
   415  			buf.WriteRune(r)
   416  		}
   417  	}
   418  	return buf.String()
   419  }
   420  
   421  func BenchmarkFoldASCII(b *testing.B) {
   422  	bench(b, Fold, testtext.ASCII)
   423  }
   424  
   425  func BenchmarkFoldCJK(b *testing.B) {
   426  	bench(b, Fold, testtext.CJK)
   427  }
   428  
   429  func BenchmarkFoldNonCanonical(b *testing.B) {
   430  	bench(b, Fold, changingRunes(foldRune))
   431  }
   432  
   433  func BenchmarkFoldOther(b *testing.B) {
   434  	bench(b, Fold, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
   435  }
   436  
   437  func BenchmarkWideASCII(b *testing.B) {
   438  	bench(b, Widen, testtext.ASCII)
   439  }
   440  
   441  func BenchmarkWideCJK(b *testing.B) {
   442  	bench(b, Widen, testtext.CJK)
   443  }
   444  
   445  func BenchmarkWideNonCanonical(b *testing.B) {
   446  	bench(b, Widen, changingRunes(widenRune))
   447  }
   448  
   449  func BenchmarkWideOther(b *testing.B) {
   450  	bench(b, Widen, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
   451  }
   452  
   453  func BenchmarkNarrowASCII(b *testing.B) {
   454  	bench(b, Narrow, testtext.ASCII)
   455  }
   456  
   457  func BenchmarkNarrowCJK(b *testing.B) {
   458  	bench(b, Narrow, testtext.CJK)
   459  }
   460  
   461  func BenchmarkNarrowNonCanonical(b *testing.B) {
   462  	bench(b, Narrow, changingRunes(narrowRune))
   463  }
   464  
   465  func BenchmarkNarrowOther(b *testing.B) {
   466  	bench(b, Narrow, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
   467  }