vitess.io/vitess@v0.16.2/go/mysql/collations/integration/charset_test.go (about)

     1  /*
     2  Copyright 2021 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package integration
    18  
    19  import (
    20  	"bytes"
    21  	"testing"
    22  	"unicode/utf8"
    23  
    24  	"github.com/stretchr/testify/require"
    25  
    26  	"vitess.io/vitess/go/mysql/collations"
    27  	"vitess.io/vitess/go/mysql/collations/internal/charset"
    28  	"vitess.io/vitess/go/mysql/collations/remote"
    29  )
    30  
    31  func TestLocalEncodings(t *testing.T) {
    32  	var cases = []struct {
    33  		collation string
    34  		input     []byte
    35  	}{
    36  		{
    37  			collation: "latin1_swedish_ci",
    38  			input:     []byte("abcdABCD01234"),
    39  		},
    40  	}
    41  
    42  	conn := mysqlconn(t)
    43  	defer conn.Close()
    44  
    45  	for _, tc := range cases {
    46  		local := collations.Local().LookupByName(tc.collation)
    47  		remote := remote.NewCollation(conn, tc.collation)
    48  		verifyTranscoding(t, local, remote, tc.input)
    49  	}
    50  }
    51  
    52  func TestCJKStress(t *testing.T) {
    53  	var universe [][]byte
    54  	for cp := rune(0); cp <= 0x10FFFF; cp++ {
    55  		if utf8.ValidRune(cp) {
    56  			var b [16]byte
    57  			l := utf8.EncodeRune(b[:], cp)
    58  
    59  			block := int(cp / 256)
    60  			for len(universe) <= block {
    61  				universe = append(universe, nil)
    62  			}
    63  			universe[block] = append(universe[block], b[:l]...)
    64  		}
    65  	}
    66  
    67  	var charsets = []charset.Charset{
    68  		charset.Charset_latin1{},
    69  		// charset.Charset_gb18030{},
    70  		charset.Charset_gb2312{},
    71  		charset.Charset_ujis{},
    72  		charset.Charset_eucjpms{},
    73  		charset.Charset_sjis{},
    74  		charset.Charset_cp932{},
    75  		charset.Charset_euckr{},
    76  	}
    77  
    78  	conn := mysqlconn(t)
    79  	defer conn.Close()
    80  
    81  	remoteUtf8mb4 := remote.NewCharset(conn, "utf8mb4")
    82  
    83  	for _, local := range charsets {
    84  		t.Run(local.Name(), func(t *testing.T) {
    85  			remote := remote.NewCharset(conn, local.Name())
    86  			convert := func(block []byte) ([]byte, []byte) {
    87  				t.Helper()
    88  				ours, _ := charset.ConvertFromUTF8(nil, local, block)
    89  				theirs, err := charset.ConvertFromUTF8(nil, remote, block)
    90  				require.NoError(t, err, "remote transcoding failed: %v", err)
    91  
    92  				return ours, theirs
    93  			}
    94  
    95  			unconvert := func(block []byte) ([]byte, []byte) {
    96  				t.Helper()
    97  				ours, _ := charset.Convert(nil, charset.Charset_utf8mb4{}, block, local)
    98  				theirs, err := charset.Convert(nil, remoteUtf8mb4, block, remote)
    99  				require.NoError(t, err, "remote transcoding failed: %v", err)
   100  
   101  				return ours, theirs
   102  			}
   103  
   104  			for _, block := range universe {
   105  				if len(block) == 0 {
   106  					continue
   107  				}
   108  
   109  				ours, theirs := convert(block)
   110  				if !bytes.Equal(ours, theirs) {
   111  					for _, cp := range string(block) {
   112  						input := string(cp)
   113  						ours, theirs := convert([]byte(input))
   114  						require.True(t, bytes.Equal(ours, theirs), "%s: bad conversion for %q (U+%04X). ours: %#v, theirs: %#v", local.Name(), input, cp, ours, theirs)
   115  
   116  					}
   117  					panic("???")
   118  				}
   119  
   120  				ours2, theirs2 := unconvert(ours)
   121  				if !bytes.Equal(ours2, theirs2) {
   122  					for _, cp := range string(block) {
   123  						input := string(cp)
   124  						ours, _ := charset.ConvertFromUTF8(nil, local, []byte(input))
   125  
   126  						ours2, theirs2 := unconvert(ours)
   127  						require.True(t, bytes.Equal(ours2, theirs2), "%s: bad return conversion for %q (U+%04X) %#v. ours: %#v, theirs: %#v", local.Name(), input, cp, ours, ours2, theirs2)
   128  
   129  					}
   130  					panic("???")
   131  				}
   132  			}
   133  		})
   134  	}
   135  }