vitess.io/vitess@v0.16.2/go/mysql/collations/uca_test.go (about) 1 /* 2 Copyright 2021 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package collations 18 19 import ( 20 "bytes" 21 "fmt" 22 "math/rand" 23 "sort" 24 "strings" 25 "sync" 26 "testing" 27 "unicode/utf8" 28 29 "github.com/stretchr/testify/assert" 30 "github.com/stretchr/testify/require" 31 "golang.org/x/exp/slices" 32 33 "vitess.io/vitess/go/mysql/collations/internal/charset" 34 ) 35 36 var testcollationMap map[string]Collation 37 var testcollationSlice []Collation 38 var testcollationOnce sync.Once 39 40 func testinit() { 41 testcollationOnce.Do(func() { 42 testcollationSlice = make([]Collation, 0, len(globalAllCollations)) 43 testcollationMap = make(map[string]Collation) 44 45 for _, collation := range globalAllCollations { 46 collation.Init() 47 testcollationMap[collation.Name()] = collation 48 testcollationSlice = append(testcollationSlice, collation) 49 } 50 51 sort.Slice(testcollationSlice, func(i, j int) bool { 52 return testcollationSlice[i].ID() < testcollationSlice[j].ID() 53 }) 54 }) 55 } 56 57 func testcollation(t testing.TB, name string) Collation { 58 t.Helper() 59 testinit() 60 coll := testcollationMap[name] 61 require.NotNil(t, coll, "missing collation: %s", name) 62 63 return coll 64 } 65 66 func testall() []Collation { 67 testinit() 68 return testcollationSlice 69 } 70 71 func TestWeightsForSpace(t *testing.T) { 72 for _, coll := range testall() { 73 var actual, expected uint16 74 switch coll := coll.(type) { 75 case *Collation_uca_legacy: 76 actual = coll.uca.WeightForSpace() 77 if strings.Contains(coll.name, "_520_") { 78 expected = 0x20A 79 } else { 80 expected = 0x209 81 } 82 case *Collation_utf8mb4_uca_0900: 83 actual = coll.uca.WeightForSpace() 84 expected = 0x209 85 default: 86 continue 87 } 88 assert.Equal(t, expected, actual, "expected Weight(' ') == 0x%X, got 0x%X", expected, actual) 89 90 } 91 } 92 93 func TestKanaSensitivity(t *testing.T) { 94 const Kana1 = "の東京ノ" 95 const Kana2 = "ノ東京の" 96 97 var cases = []struct { 98 collation string 99 equal bool 100 }{ 101 {"utf8mb4_0900_as_cs", false}, 102 {"utf8mb4_ja_0900_as_cs", true}, 103 {"utf8mb4_ja_0900_as_cs_ks", false}, 104 } 105 106 for _, tc := range cases { 107 t.Run(tc.collation, func(t *testing.T) { 108 collation := testcollation(t, tc.collation) 109 equal := collation.Collate([]byte(Kana1), []byte(Kana2), false) == 0 110 assert.Equal(t, tc.equal, equal, "expected %q == %q to be %v", Kana1, Kana2, tc.equal) 111 112 }) 113 } 114 } 115 116 func TestContractions(t *testing.T) { 117 var cases = []struct { 118 collation string 119 inputs []string 120 expected []byte 121 }{ 122 { 123 collation: "utf8mb4_es_trad_0900_ai_ci", 124 inputs: []string{"ch", "CH", "Ch"}, 125 expected: []byte{0x1c, 0x7a, 0x54, 0xa5}, 126 }, 127 { 128 collation: "utf8mb4_es_trad_0900_as_cs", 129 inputs: []string{"Ch"}, 130 expected: []byte{0x1c, 0x7a, 0x54, 0xa5, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x8, 0x0, 0x21}, 131 }, 132 } 133 134 for _, tc := range cases { 135 t.Run(tc.collation, func(t *testing.T) { 136 coll := testcollation(t, tc.collation) 137 138 for _, in := range tc.inputs { 139 weightString := coll.WeightString(nil, []byte(in), 0) 140 assert.True(t, bytes.Equal(weightString, tc.expected), "weight_string(%q) = %#v (expected %#v)", in, weightString, tc.expected) 141 142 } 143 }) 144 } 145 } 146 147 func TestReplacementCharacter(t *testing.T) { 148 var cases = []struct { 149 collation string 150 expected []byte 151 }{ 152 {"utf8mb4_0900_ai_ci", []byte{0xff, 0xfd}}, 153 {"utf8mb4_0900_as_ci", []byte{0xff, 0xfd, 0x0, 0x0, 0x0, 0x20}}, 154 {"utf8mb4_0900_as_cs", []byte{0xff, 0xfd, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x2}}, 155 } 156 157 for _, tc := range cases { 158 t.Run(tc.collation, func(t *testing.T) { 159 coll := testcollation(t, tc.collation) 160 weightString := coll.WeightString(nil, []byte(string(utf8.RuneError)), 0) 161 assert.True(t, bytes.Equal(weightString, tc.expected), "weight_string(\\uFFFD) = %#v (expected %#v)", weightString, tc.expected) 162 163 }) 164 } 165 } 166 167 func TestIsPrefix(t *testing.T) { 168 var collations = []string{ 169 "utf8mb4_0900_ai_ci", 170 "utf8mb4_0900_as_ci", 171 "utf8mb4_unicode_ci", 172 "utf8mb4_unicode_520_ci", 173 "utf8mb4_general_ci", 174 } 175 176 for _, collName := range collations { 177 coll := testcollation(t, collName) 178 input := []rune(strings.ToUpper(ExampleStringLong)) 179 180 for size := 0; size < len(input); size++ { 181 left := ExampleStringLong 182 right := string(input[:size]) 183 184 cmp := coll.Collate([]byte(left), []byte(right), true) 185 assert.Equal(t, 0, cmp, "IsPrefix(%q, %q) = %d (expected 0)", left, right, cmp) 186 187 } 188 } 189 } 190 191 func DebugUcaLegacyWeightString(t *testing.T, collname string, input, expected []byte) { 192 coll := testcollation(t, collname).(*Collation_uca_legacy) 193 iter := coll.uca.Iterator(input) 194 defer iter.Done() 195 196 for { 197 curcp, _ := iter.DebugCodepoint() 198 w, ok := iter.Next() 199 if !ok { 200 break 201 } 202 203 a := byte(w >> 8) 204 b := byte(w) 205 t.Logf("%q -> %d, %d | %d, %d", string(curcp), a, b, expected[0], expected[1]) 206 if a != expected[0] || b != expected[1] { 207 t.Errorf("mismatch on %q", string(curcp)) 208 } 209 expected = expected[2:] 210 } 211 } 212 213 const ExampleString = "abc æøå 日本語" 214 const ExampleStringLong = "Premature optimization is the root of all evil. " + 215 "Våre norske tegn bør æres. 日本語が少しわかります。 " + 216 "✌️🐶👩🏽" 217 const JapaneseString = "データの保存とアクセスを行うストレージエンジンがSQLパーサとは" + 218 "分離独立しており、用途に応じたストレージエンジンを選択できる" + 219 "「マルチストレージエンジン」方式を採用している。" 220 const WhitespaceString = "This is a\n prett\ny unrealist\nic case; a\nn " + 221 "Eng\nlish sente\nnce where\n we'\nve added a new\nline every te\nn " + 222 "bytes or\n so.\n" 223 const HungarianString = "A MySQL adatbázisok adminisztrációjára a mellékelt " + 224 "parancssori eszközöket (mysql és mysqladmin) használhatjuk." 225 const JapaneseString2 = "サーバー SQL モードの設定方法。この設定は、たとえば" + 226 "別のデータベースシステムからのコードとの互換性を保ったり、特定の状況に" + 227 "ついてのエラー処理を制御したりするために、SQL の構文およびセマンティクス" + 228 "の特定の側面を変更します。" 229 const ChineseString = "\xE9\x98\xBF\xE5\x92\x97\xF0\xAC\xBA\xA1" + 230 "\xC4\x81\x61\x62\xC5\xAB\x75\x55\xC7\x96\x5A\xF0\x94\x99\x86" + 231 "\xF0\x97\x86\xA0\xF0\xAC\xBA\xA2\xF0\xAE\xAF\xA0\xF0\xB3\x8C\xB3" 232 const ChineseString2 = "春江潮水连海平，海上明月共潮生。" + 233 "滟滟随波千万里，何处春江无月明！" + 234 "江流宛转绕芳甸，月照花林皆似霰；" + 235 "空里流霜不觉飞，汀上白沙看不见。" + 236 "江天一色无纤尘，皎皎空中孤月轮。" + 237 "江畔何人初见月？江月何年初照人？" + 238 "人生代代无穷已，江月年年只相似。" + 239 "不知江月待何人，但见长江送流水。" + 240 "白云一片去悠悠，青枫浦上不胜愁。" + 241 "谁家今夜扁舟子？何处相思明月楼？" 242 const SpanishString = "A mí se me hace cuento que empezó Buenos Aires: " + 243 "La juzgo tan eterna como el agua y el aire." 244 const EnglishString = "Dame Mary Rosa Alleyne Hunnings DBE (nee Berry; born 24 March 1935), " + 245 "known professionally as Mary Berry, is an English food writer, chef, baker and television presenter." 246 247 var AllTestStrings = []struct { 248 Name, Content string 249 }{ 250 {"Example", ExampleString}, 251 {"ExampleLong", ExampleStringLong}, 252 {"Japanese", JapaneseString}, 253 {"Whitespace", WhitespaceString}, 254 {"Hungarian", HungarianString}, 255 {"Japanese2", JapaneseString2}, 256 {"Chinese", ChineseString}, 257 {"Chinese2", ChineseString2}, 258 {"Spanish", SpanishString}, 259 {"English", EnglishString}, 260 } 261 262 var TestCases = []struct { 263 collation string 264 input string 265 expected []byte 266 }{ 267 { 268 collation: "utf8mb4_bin", 269 input: ExampleString, 270 expected: []byte{ 271 0x00, 0x00, 0x61, 0x00, 0x00, 0x62, 0x00, 0x00, 0x63, // abc 272 0x00, 0x00, 0x20, // space 273 0x00, 0x00, 0xe6, 0x00, 0x00, 0xf8, 0x00, 0x00, 0xe5, // æøå 274 0x00, 0x00, 0x20, // space 275 0x00, 0x65, 0xe5, 0x00, 0x67, 0x2c, 0x00, 0x8a, 0x9e, // 日本語 276 0x00, 0x00, 0x20, 0x00, 0x00, 0x20, 0x00, 0x00, 0x20, 277 0x00, 0x00, 0x20, 0x00, 0x00, 0x20, // space for padding 278 }, 279 }, 280 { 281 collation: "utf8mb4_0900_ai_ci", 282 input: ExampleString, 283 expected: []byte{ 284 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, // abc 285 0x02, 0x09, // space 286 0x1c, 0x47, 0x1c, 0xaa, 0x1d, 0xdd, 0x1c, 0x47, // æøå 287 0x02, 0x09, // space 288 0xfb, 0x40, 0xe5, 0xe5, 0xfb, 0x40, 0xe7, 0x2c, 289 0xfb, 0x41, 0x8a, 0x9e, // 日本語 290 }, 291 }, 292 { 293 collation: "utf8mb4_0900_as_ci", 294 input: ExampleString, 295 expected: []byte{ 296 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, // abc 297 0x02, 0x09, // space 298 0x1c, 0x47, 0x1c, 0xaa, 0x1d, 0xdd, 0x1c, 0x47, // æøå 299 0x02, 0x09, // space 300 0xfb, 0x40, 0xe5, 0xe5, 0xfb, 0x40, 0xe7, 0x2c, // 日本語 301 0xfb, 0x41, 0x8a, 0x9e, 0x00, 0x00, // level separator 302 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, // abc 303 0x00, 0x20, // space 304 0x00, 0x20, 0x01, 0x10, 0x00, 0x20, 0x00, 0x20, // æøå 305 0x00, 0x2F, 0x00, 0x20, 0x00, 0x29, 0x00, 0x20, // space 306 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, // 日本語 307 }, 308 }, 309 { 310 collation: "utf8mb4_0900_as_cs", 311 input: "abc ", 312 expected: []byte{ 313 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, // abc 314 0x02, 0x09, 0x02, 0x09, 0x02, 0x09, 315 0x02, 0x09, // Four spaces. 316 0x00, 0x00, // Level separator. 317 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, // Accents for abc. 318 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 319 0x00, 0x20, // Accents for four spaces. 320 0x00, 0x00, // Level separator. 321 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, // Case for abc. 322 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 323 0x00, 0x02, // Case for four spaces. 324 }, 325 }, 326 { 327 collation: "utf8mb4_0900_ai_ci", 328 input: ExampleStringLong, 329 expected: []byte{ 330 0x1e, 0x0c, 0x1e, 0x33, 0x1c, 0xaa, 0x1d, 0xaa, 0x1c, 0x47, 0x1e, 0x95, 331 0x1e, 0xb5, 0x1e, 0x33, 0x1c, 0xaa, 0x02, 0x09, 0x1d, 0xdd, 0x1e, 0x0c, 332 0x1e, 0x95, 0x1d, 0x32, 0x1d, 0xaa, 0x1d, 0x32, 0x1f, 0x21, 0x1c, 0x47, 333 0x1e, 0x95, 0x1d, 0x32, 0x1d, 0xdd, 0x1d, 0xb9, 0x02, 0x09, 0x1d, 0x32, 334 0x1e, 0x71, 0x02, 0x09, 0x1e, 0x95, 0x1d, 0x18, 0x1c, 0xaa, 0x02, 0x09, 335 0x1e, 0x33, 0x1d, 0xdd, 0x1d, 0xdd, 0x1e, 0x95, 0x02, 0x09, 0x1d, 0xdd, 336 0x1c, 0xe5, 0x02, 0x09, 0x1c, 0x47, 0x1d, 0x77, 0x1d, 0x77, 0x02, 0x09, 337 0x1c, 0xaa, 0x1e, 0xe3, 0x1d, 0x32, 0x1d, 0x77, 0x02, 0x77, 0x02, 0x09, 338 0x1e, 0xe3, 0x1c, 0x47, 0x1e, 0x33, 0x1c, 0xaa, 0x02, 0x09, 0x1d, 0xb9, 339 0x1d, 0xdd, 0x1e, 0x33, 0x1e, 0x71, 0x1d, 0x65, 0x1c, 0xaa, 0x02, 0x09, 340 0x1e, 0x95, 0x1c, 0xaa, 0x1c, 0xf4, 0x1d, 0xb9, 0x02, 0x09, 0x1c, 0x60, 341 0x1d, 0xdd, 0x1e, 0x33, 0x02, 0x09, 0x1c, 0x47, 0x1c, 0xaa, 0x1e, 0x33, 342 0x1c, 0xaa, 0x1e, 0x71, 0x02, 0x77, 0x02, 0x09, 0xfb, 0x40, 0xe5, 0xe5, 343 0xfb, 0x40, 0xe7, 0x2c, 0xfb, 0x41, 0x8a, 0x9e, 0x3d, 0x60, 0xfb, 0x40, 344 0xdc, 0x11, 0x3d, 0x66, 0x3d, 0x87, 0x3d, 0x60, 0x3d, 0x83, 0x3d, 0x79, 345 0x3d, 0x67, 0x02, 0x8a, 0x02, 0x09, 0x0a, 0x2d, 0x13, 0xdf, 0x14, 0x12, 346 0x13, 0xa6, 347 }, 348 }, 349 { 350 collation: "utf8mb4_0900_as_ci", 351 input: ExampleStringLong, 352 expected: []byte{ 353 0x1e, 0x0c, 0x1e, 0x33, 0x1c, 0xaa, 0x1d, 0xaa, 0x1c, 0x47, 0x1e, 0x95, 354 0x1e, 0xb5, 0x1e, 0x33, 0x1c, 0xaa, 0x02, 0x09, 0x1d, 0xdd, 0x1e, 0x0c, 355 0x1e, 0x95, 0x1d, 0x32, 0x1d, 0xaa, 0x1d, 0x32, 0x1f, 0x21, 0x1c, 0x47, 356 0x1e, 0x95, 0x1d, 0x32, 0x1d, 0xdd, 0x1d, 0xb9, 0x02, 0x09, 0x1d, 0x32, 357 0x1e, 0x71, 0x02, 0x09, 0x1e, 0x95, 0x1d, 0x18, 0x1c, 0xaa, 0x02, 0x09, 358 0x1e, 0x33, 0x1d, 0xdd, 0x1d, 0xdd, 0x1e, 0x95, 0x02, 0x09, 0x1d, 0xdd, 359 0x1c, 0xe5, 0x02, 0x09, 0x1c, 0x47, 0x1d, 0x77, 0x1d, 0x77, 0x02, 0x09, 360 0x1c, 0xaa, 0x1e, 0xe3, 0x1d, 0x32, 0x1d, 0x77, 0x02, 0x77, 0x02, 0x09, 361 0x1e, 0xe3, 0x1c, 0x47, 0x1e, 0x33, 0x1c, 0xaa, 0x02, 0x09, 0x1d, 0xb9, 362 0x1d, 0xdd, 0x1e, 0x33, 0x1e, 0x71, 0x1d, 0x65, 0x1c, 0xaa, 0x02, 0x09, 363 0x1e, 0x95, 0x1c, 0xaa, 0x1c, 0xf4, 0x1d, 0xb9, 0x02, 0x09, 0x1c, 0x60, 364 0x1d, 0xdd, 0x1e, 0x33, 0x02, 0x09, 0x1c, 0x47, 0x1c, 0xaa, 0x1e, 0x33, 365 0x1c, 0xaa, 0x1e, 0x71, 0x02, 0x77, 0x02, 0x09, 0xfb, 0x40, 0xe5, 0xe5, 366 0xfb, 0x40, 0xe7, 0x2c, 0xfb, 0x41, 0x8a, 0x9e, 0x3d, 0x60, 0xfb, 0x40, 367 0xdc, 0x11, 0x3d, 0x66, 0x3d, 0x87, 0x3d, 0x60, 0x3d, 0x83, 0x3d, 0x79, 368 0x3d, 0x67, 0x02, 0x8a, 0x02, 0x09, 0x0a, 0x2d, 0x13, 0xdf, 0x14, 0x12, 369 0x13, 0xa6, 0x00, 0x00, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 370 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 371 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 372 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 373 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 374 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 375 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 376 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 377 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x29, 0x00, 0x20, 378 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 379 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 380 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x2F, 0x00, 0x20, 381 0x00, 0x20, 0x00, 0x20, 0x01, 0x10, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 382 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 383 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 384 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 385 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 386 }, 387 }, 388 { 389 collation: "utf8mb4_0900_ai_ci", 390 input: JapaneseString, 391 expected: []byte{ 392 0x3d, 0x6d, 0x1c, 0x0e, 0x3d, 0x6a, 0x3d, 0x73, 0xfb, 0x40, 0xcf, 0xdd, 393 0xfb, 0x40, 0xdb, 0x58, 0x3d, 0x6e, 0x3d, 0x5a, 0x3d, 0x62, 0x3d, 0x68, 394 0x3d, 0x67, 0x3d, 0x8a, 0xfb, 0x41, 0x88, 0x4c, 0x3d, 0x5c, 0x3d, 0x67, 395 0x3d, 0x6e, 0x3d, 0x85, 0x1c, 0x0e, 0x3d, 0x66, 0x3d, 0x5e, 0x3d, 0x8b, 396 0x3d, 0x66, 0x3d, 0x8b, 0x3d, 0x60, 0x1e, 0x71, 0x1e, 0x21, 0x1d, 0x77, 397 0x3d, 0x74, 0x1c, 0x0e, 0x3d, 0x65, 0x3d, 0x6e, 0x3d, 0x74, 0xfb, 0x40, 398 0xd2, 0x06, 0xfb, 0x41, 0x96, 0xe2, 0xfb, 0x40, 0xf2, 0xec, 0xfb, 0x40, 399 0xfa, 0xcb, 0x3d, 0x66, 0x3d, 0x6d, 0x3d, 0x5f, 0x3d, 0x83, 0x02, 0x31, 400 0xfb, 0x40, 0xf5, 0x28, 0xfb, 0x41, 0x90, 0x14, 0x3d, 0x70, 0xfb, 0x40, 401 0xdf, 0xdc, 0x3d, 0x66, 0x3d, 0x6a, 0x3d, 0x67, 0x3d, 0x6e, 0x3d, 0x85, 402 0x1c, 0x0e, 0x3d, 0x66, 0x3d, 0x5e, 0x3d, 0x8b, 0x3d, 0x66, 0x3d, 0x8b, 403 0x3d, 0x8a, 0xfb, 0x41, 0x90, 0x78, 0xfb, 0x40, 0xe2, 0x9e, 0x3d, 0x6d, 404 0x3d, 0x61, 0x3d, 0x84, 0x03, 0x73, 0x3d, 0x79, 0x3d, 0x84, 0x3d, 0x6b, 405 0x3d, 0x67, 0x3d, 0x6e, 0x3d, 0x85, 0x1c, 0x0e, 0x3d, 0x66, 0x3d, 0x5e, 406 0x3d, 0x8b, 0x3d, 0x66, 0x3d, 0x8b, 0x03, 0x74, 0xfb, 0x40, 0xe5, 0xb9, 407 0xfb, 0x40, 0xdf, 0x0f, 0x3d, 0x8a, 0xfb, 0x40, 0xe3, 0xa1, 0xfb, 0x40, 408 0xf5, 0x28, 0x3d, 0x66, 0x3d, 0x6d, 0x3d, 0x5b, 0x3d, 0x84, 0x02, 0x8a, 409 }, 410 }, 411 { 412 collation: "utf8mb4_0900_ai_ci", 413 input: WhitespaceString, 414 expected: []byte{ 415 0x1e, 0x95, 0x1d, 0x18, 0x1d, 0x32, 0x1e, 0x71, 0x02, 0x09, 0x1d, 0x32, 416 0x1e, 0x71, 0x02, 0x09, 0x1c, 0x47, 0x02, 0x02, 0x02, 0x09, 0x1e, 0x0c, 417 0x1e, 0x33, 0x1c, 0xaa, 0x1e, 0x95, 0x1e, 0x95, 0x02, 0x02, 0x1f, 0x0b, 418 0x02, 0x09, 0x1e, 0xb5, 0x1d, 0xb9, 0x1e, 0x33, 0x1c, 0xaa, 0x1c, 0x47, 419 0x1d, 0x77, 0x1d, 0x32, 0x1e, 0x71, 0x1e, 0x95, 0x02, 0x02, 0x1d, 0x32, 420 0x1c, 0x7a, 0x02, 0x09, 0x1c, 0x7a, 0x1c, 0x47, 0x1e, 0x71, 0x1c, 0xaa, 421 0x02, 0x34, 0x02, 0x09, 0x1c, 0x47, 0x02, 0x02, 0x1d, 0xb9, 0x02, 0x09, 422 0x1c, 0xaa, 0x1d, 0xb9, 0x1c, 0xf4, 0x02, 0x02, 0x1d, 0x77, 0x1d, 0x32, 423 0x1e, 0x71, 0x1d, 0x18, 0x02, 0x09, 0x1e, 0x71, 0x1c, 0xaa, 0x1d, 0xb9, 424 0x1e, 0x95, 0x1c, 0xaa, 0x02, 0x02, 0x1d, 0xb9, 0x1c, 0x7a, 0x1c, 0xaa, 425 0x02, 0x09, 0x1e, 0xf5, 0x1d, 0x18, 0x1c, 0xaa, 0x1e, 0x33, 0x1c, 0xaa, 426 0x02, 0x02, 0x02, 0x09, 0x1e, 0xf5, 0x1c, 0xaa, 0x03, 0x05, 0x02, 0x02, 427 0x1e, 0xe3, 0x1c, 0xaa, 0x02, 0x09, 0x1c, 0x47, 0x1c, 0x8f, 0x1c, 0x8f, 428 0x1c, 0xaa, 0x1c, 0x8f, 0x02, 0x09, 0x1c, 0x47, 0x02, 0x09, 0x1d, 0xb9, 429 0x1c, 0xaa, 0x1e, 0xf5, 0x02, 0x02, 0x1d, 0x77, 0x1d, 0x32, 0x1d, 0xb9, 430 0x1c, 0xaa, 0x02, 0x09, 0x1c, 0xaa, 0x1e, 0xe3, 0x1c, 0xaa, 0x1e, 0x33, 431 0x1f, 0x0b, 0x02, 0x09, 0x1e, 0x95, 0x1c, 0xaa, 0x02, 0x02, 0x1d, 0xb9, 432 0x02, 0x09, 0x1c, 0x60, 0x1f, 0x0b, 0x1e, 0x95, 0x1c, 0xaa, 0x1e, 0x71, 433 0x02, 0x09, 0x1d, 0xdd, 0x1e, 0x33, 0x02, 0x02, 0x02, 0x09, 0x1e, 0x71, 434 0x1d, 0xdd, 0x02, 0x77, 0x02, 0x02, 435 }, 436 }, 437 { 438 collation: "utf8mb4_hu_0900_as_cs", 439 input: HungarianString, 440 expected: []byte{ 441 0x1c, 0x47, 0x02, 0x09, 0x1d, 0xaa, 0x1f, 0x0b, 0x1e, 0x71, 0x1e, 0x21, 442 0x1d, 0x77, 0x02, 0x09, 0x1c, 0x47, 0x1c, 0x8f, 0x1c, 0x47, 0x1e, 0x95, 443 0x1c, 0x60, 0x1c, 0x47, 0x1f, 0x21, 0x1d, 0x32, 0x1e, 0x71, 0x1d, 0xdd, 444 0x1d, 0x65, 0x02, 0x09, 0x1c, 0x47, 0x1c, 0x8f, 0x1d, 0xaa, 0x1d, 0x32, 445 0x1d, 0xb9, 0x1d, 0x32, 0x1e, 0x71, 0x54, 0xa5, 0x1e, 0x95, 0x1e, 0x33, 446 0x1c, 0x47, 0x1c, 0x7a, 0x1d, 0x32, 0x1d, 0xdd, 0x1d, 0x4c, 0x1c, 0x47, 447 0x1e, 0x33, 0x1c, 0x47, 0x02, 0x09, 0x1c, 0x47, 0x02, 0x09, 0x1d, 0xaa, 448 0x1c, 0xaa, 0x1d, 0x77, 0x1d, 0x77, 0x1c, 0xaa, 0x1d, 0x65, 0x1c, 0xaa, 449 0x1d, 0x77, 0x1e, 0x95, 0x02, 0x09, 0x1e, 0x0c, 0x1c, 0x47, 0x1e, 0x33, 450 0x1c, 0x47, 0x1d, 0xb9, 0x1c, 0x7a, 0x54, 0xa5, 0x1e, 0x71, 0x1d, 0xdd, 451 0x1e, 0x33, 0x1d, 0x32, 0x02, 0x09, 0x1c, 0xaa, 0x1e, 0x71, 0x54, 0xa5, 452 0x1d, 0x65, 0x1d, 0xdd, 0x54, 0xa5, 0x1f, 0x21, 0x1d, 0xdd, 0x54, 0xa5, 453 0x1d, 0x65, 0x1c, 0xaa, 0x1e, 0x95, 0x02, 0x09, 0x03, 0x17, 0x1d, 0xaa, 454 0x1f, 0x0b, 0x1e, 0x71, 0x1e, 0x21, 0x1d, 0x77, 0x02, 0x09, 0x1c, 0xaa, 455 0x1e, 0x71, 0x02, 0x09, 0x1d, 0xaa, 0x1f, 0x0b, 0x1e, 0x71, 0x1e, 0x21, 456 0x1d, 0x77, 0x1c, 0x47, 0x1c, 0x8f, 0x1d, 0xaa, 0x1d, 0x32, 0x1d, 0xb9, 457 0x03, 0x18, 0x02, 0x09, 0x1d, 0x18, 0x1c, 0x47, 0x1e, 0x71, 0x54, 0xa5, 458 0x1d, 0xb9, 0x1c, 0x47, 0x1d, 0x77, 0x1d, 0x18, 0x1c, 0x47, 0x1e, 0x95, 459 0x1d, 0x4c, 0x1e, 0xb5, 0x1d, 0x65, 0x02, 0x77, 0x00, 0x00, 0x00, 0x20, 460 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 461 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 462 0x00, 0x20, 0x00, 0x24, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 463 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 464 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 465 0x00, 0x24, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x24, 0x00, 0x20, 466 0x00, 0x20, 0x00, 0x24, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 467 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 468 0x00, 0x24, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 469 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 470 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 471 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 472 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 473 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x24, 474 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 475 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 476 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 477 0x00, 0x20, 0x00, 0x24, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 478 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x00, 0x00, 0x08, 479 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 480 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 481 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 482 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 483 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 484 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 485 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 486 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 487 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 488 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 489 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 490 0x00, 0x08, 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 491 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 492 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 493 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 494 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 495 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 0x00, 0x02, 496 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 497 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 498 }, 499 }, 500 { 501 collation: "utf8mb4_ja_0900_as_cs", 502 input: JapaneseString2, 503 expected: []byte{ 504 0x1F, 0xC1, 0x1F, 0xB6, 0x1F, 0xD0, 0x1F, 0xB6, 0x02, 0x09, 0x1E, 0x71, 505 0x1E, 0x21, 0x1D, 0x77, 0x02, 0x09, 0x1F, 0xD9, 0x1F, 0xBB, 0x1F, 0xCA, 506 0x1F, 0xCF, 0x5A, 0xC2, 0x5C, 0x45, 0x5E, 0x8C, 0x5E, 0x8E, 0x02, 0x8A, 507 0x1F, 0xC0, 0x1F, 0xCF, 0x5A, 0xC2, 0x5C, 0x45, 0x1F, 0xD0, 0x02, 0x31, 508 0x1F, 0xC6, 0x1F, 0xCA, 0x1F, 0xBA, 0x1F, 0xD0, 0x5E, 0x5B, 0x1F, 0xCF, 509 0x1F, 0xC9, 0x1F, 0xBA, 0x1F, 0xC6, 0x1F, 0xD3, 0x1F, 0xBA, 0x1F, 0xC3, 510 0x1F, 0xC2, 0x1F, 0xC3, 0x1F, 0xC9, 0x1F, 0xD7, 0x1F, 0xBC, 0x1F, 0xDE, 511 0x1F, 0xCF, 0x1F, 0xC0, 0x1F, 0xBB, 0x1F, 0xCA, 0x1F, 0xCA, 0x1F, 0xCF, 512 0x57, 0xD2, 0x56, 0x34, 0x5A, 0x90, 0x1F, 0xE6, 0x5E, 0x6C, 0x1F, 0xC8, 513 0x1F, 0xC6, 0x1F, 0xDF, 0x02, 0x31, 0x5C, 0xDA, 0x5C, 0x45, 0x1F, 0xCF, 514 0x5A, 0x1C, 0x56, 0xEE, 0x1F, 0xCC, 0x1F, 0xC8, 0x1F, 0xB7, 0x1F, 0xC9, 515 0x1F, 0xCF, 0x1F, 0xBA, 0x1F, 0xDE, 0x1F, 0xB6, 0x59, 0xB1, 0x5F, 0xA6, 516 0x1F, 0xE6, 0x5A, 0x8C, 0x57, 0xD9, 0x1F, 0xC2, 0x1F, 0xC6, 0x1F, 0xDF, 517 0x1F, 0xC3, 0x1F, 0xE0, 0x1F, 0xC6, 0x1F, 0xD8, 0x1F, 0xCC, 0x02, 0x31, 518 0x1E, 0x71, 0x1E, 0x21, 0x1D, 0x77, 0x02, 0x09, 0x1F, 0xCF, 0x58, 0x0E, 519 0x5E, 0x47, 0x1F, 0xBB, 0x1F, 0xDD, 0x1F, 0xD1, 0x1F, 0xC4, 0x1F, 0xD5, 520 0x1F, 0xE7, 0x1F, 0xC9, 0x1F, 0xB7, 0x1F, 0xBE, 0x1F, 0xC3, 0x1F, 0xCF, 521 0x5C, 0xDA, 0x5C, 0x45, 0x1F, 0xCF, 0x5B, 0x45, 0x5F, 0x17, 0x1F, 0xE6, 522 0x5E, 0x60, 0x58, 0x0A, 0x1F, 0xC2, 0x1F, 0xD5, 0x1F, 0xC3, 0x02, 0x8A, 523 0x00, 0x00, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 524 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 525 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 526 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 527 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 528 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 529 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 530 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 531 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 532 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 533 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 534 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 535 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 536 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 537 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 538 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 539 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 540 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 541 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 542 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 543 0x00, 0x20, 0x00, 0x20, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 544 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 0x00, 0x21, 0x00, 0x02, 0x00, 0x08, 545 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 546 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 547 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 548 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 549 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 550 0x00, 0x21, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 0x00, 0x21, 551 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 552 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 0x00, 0x0E, 553 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 554 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0D, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 555 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 556 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 557 0x00, 0x0C, 0x00, 0x21, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 558 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 559 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 560 0x00, 0x08, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 561 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 562 0x00, 0x0E, 0x00, 0x0D, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 563 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 564 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 565 }, 566 }, 567 { 568 collation: "utf8mb4_ja_0900_as_cs_ks", 569 input: JapaneseString2, 570 expected: []byte{ 571 0x1F, 0xC1, 0x1F, 0xB6, 0x1F, 0xD0, 0x1F, 0xB6, 0x02, 0x09, 0x1E, 0x71, 572 0x1E, 0x21, 0x1D, 0x77, 0x02, 0x09, 0x1F, 0xD9, 0x1F, 0xBB, 0x1F, 0xCA, 573 0x1F, 0xCF, 0x5A, 0xC2, 0x5C, 0x45, 0x5E, 0x8C, 0x5E, 0x8E, 0x02, 0x8A, 574 0x1F, 0xC0, 0x1F, 0xCF, 0x5A, 0xC2, 0x5C, 0x45, 0x1F, 0xD0, 0x02, 0x31, 575 0x1F, 0xC6, 0x1F, 0xCA, 0x1F, 0xBA, 0x1F, 0xD0, 0x5E, 0x5B, 0x1F, 0xCF, 576 0x1F, 0xC9, 0x1F, 0xBA, 0x1F, 0xC6, 0x1F, 0xD3, 0x1F, 0xBA, 0x1F, 0xC3, 577 0x1F, 0xC2, 0x1F, 0xC3, 0x1F, 0xC9, 0x1F, 0xD7, 0x1F, 0xBC, 0x1F, 0xDE, 578 0x1F, 0xCF, 0x1F, 0xC0, 0x1F, 0xBB, 0x1F, 0xCA, 0x1F, 0xCA, 0x1F, 0xCF, 579 0x57, 0xD2, 0x56, 0x34, 0x5A, 0x90, 0x1F, 0xE6, 0x5E, 0x6C, 0x1F, 0xC8, 580 0x1F, 0xC6, 0x1F, 0xDF, 0x02, 0x31, 0x5C, 0xDA, 0x5C, 0x45, 0x1F, 0xCF, 581 0x5A, 0x1C, 0x56, 0xEE, 0x1F, 0xCC, 0x1F, 0xC8, 0x1F, 0xB7, 0x1F, 0xC9, 582 0x1F, 0xCF, 0x1F, 0xBA, 0x1F, 0xDE, 0x1F, 0xB6, 0x59, 0xB1, 0x5F, 0xA6, 583 0x1F, 0xE6, 0x5A, 0x8C, 0x57, 0xD9, 0x1F, 0xC2, 0x1F, 0xC6, 0x1F, 0xDF, 584 0x1F, 0xC3, 0x1F, 0xE0, 0x1F, 0xC6, 0x1F, 0xD8, 0x1F, 0xCC, 0x02, 0x31, 585 0x1E, 0x71, 0x1E, 0x21, 0x1D, 0x77, 0x02, 0x09, 0x1F, 0xCF, 0x58, 0x0E, 586 0x5E, 0x47, 0x1F, 0xBB, 0x1F, 0xDD, 0x1F, 0xD1, 0x1F, 0xC4, 0x1F, 0xD5, 587 0x1F, 0xE7, 0x1F, 0xC9, 0x1F, 0xB7, 0x1F, 0xBE, 0x1F, 0xC3, 0x1F, 0xCF, 588 0x5C, 0xDA, 0x5C, 0x45, 0x1F, 0xCF, 0x5B, 0x45, 0x5F, 0x17, 0x1F, 0xE6, 589 0x5E, 0x60, 0x58, 0x0A, 0x1F, 0xC2, 0x1F, 0xD5, 0x1F, 0xC3, 0x02, 0x8A, 590 0x00, 0x00, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 591 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 592 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 593 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 594 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 595 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 596 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 0x00, 0x20, 597 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 598 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 0x00, 0x20, 599 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 600 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 601 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 602 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 603 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 604 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 605 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 606 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x37, 607 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 608 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 609 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 610 0x00, 0x20, 0x00, 0x20, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 611 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 0x00, 0x21, 0x00, 0x02, 0x00, 0x08, 612 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 613 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 614 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 615 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 616 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 617 0x00, 0x21, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0C, 0x00, 0x21, 618 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 619 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0C, 0x00, 0x21, 0x00, 0x0E, 620 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 621 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0D, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 622 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 623 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 624 0x00, 0x0C, 0x00, 0x21, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 625 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 626 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 627 0x00, 0x08, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 628 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 629 0x00, 0x0E, 0x00, 0x0D, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 630 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x02, 0x00, 0x0E, 0x00, 0x02, 631 0x00, 0x02, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x0E, 0x00, 0x02, 0x00, 0x00, 632 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 633 0x00, 0x08, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 634 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 635 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 636 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 637 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 638 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 639 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 640 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 641 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 642 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 643 0x00, 0x08, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 644 0x00, 0x02, 645 }, 646 }, 647 { 648 collation: "utf8mb4_zh_0900_as_cs", 649 input: ChineseString, 650 expected: []byte{ 651 0x1C, 0x47, 0xBD, 0xBE, 0xBD, 0xC3, 0xCE, 0xA1, 0xBD, 0xC4, 0xBD, 0xC4, 652 0xBD, 0xDD, 0xC0, 0x32, 0xC0, 0x32, 0xC0, 0x32, 0xC0, 0x32, 0xC0, 0x9E, 653 0xF6, 0x20, 0xF6, 0x21, 0x81, 0xA0, 0xF6, 0x27, 0xCE, 0xA2, 0xF6, 0x27, 654 0xEB, 0xE0, 0xF6, 0x28, 0xB3, 0x33, 0x00, 0x00, 0x00, 0x20, 0x00, 0x20, 655 0x00, 0x20, 0x00, 0x1F, 0x01, 0x16, 0x00, 0x20, 0x00, 0x20, 0x00, 0x1F, 656 0x01, 0x16, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x01, 0x16, 0x00, 0x20, 657 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x00, 658 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 659 0x00, 0x02, 0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x02, 660 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 661 }, 662 }, 663 { 664 collation: "utf8mb4_zh_0900_as_cs", 665 input: ChineseString2, 666 expected: []byte{ 667 0x2C, 0xD0, 0x4F, 0xF1, 0x28, 0x08, 0x87, 0xE8, 0x60, 0x4C, 0x42, 0xEF, 668 0x75, 0x93, 0x02, 0x22, 0x42, 0xEF, 0x83, 0x8A, 0x6C, 0x4F, 0xAF, 0x96, 669 0x3F, 0x58, 0x28, 0x08, 0x84, 0xCF, 0x02, 0x8A, 0xA3, 0xA4, 0xA3, 0xA4, 670 0x8A, 0x5F, 0x23, 0x71, 0x78, 0xA8, 0x93, 0x1A, 0x5E, 0xD9, 0x02, 0x22, 671 0x44, 0xAC, 0x2B, 0xD5, 0x2C, 0xD0, 0x4F, 0xF1, 0x96, 0x31, 0xAF, 0x96, 672 0x6C, 0x4F, 0x02, 0x60, 0x4F, 0xF1, 0x63, 0x7B, 0x92, 0xDD, 0xBA, 0x2E, 673 0x7F, 0x07, 0x39, 0x15, 0x32, 0xB2, 0x02, 0x22, 0xAF, 0x96, 0xB4, 0x41, 674 0x47, 0xD7, 0x62, 0x27, 0x51, 0x4C, 0x85, 0xE9, 0x81, 0x86, 0x02, 0x34, 675 0x59, 0x09, 0x5E, 0xD9, 0x63, 0x7B, 0x87, 0xBA, 0x24, 0x78, 0x56, 0x5A, 676 0x39, 0x48, 0x02, 0x22, 0x8F, 0x74, 0x83, 0x8A, 0x1E, 0x4D, 0x82, 0x46, 677 0x57, 0xD9, 0x24, 0x78, 0x4F, 0x79, 0x02, 0x8A, 0x4F, 0xF1, 0x8E, 0x8A, 678 0xA6, 0x3E, 0x81, 0xEE, 0x96, 0x31, 0x99, 0x9E, 0x28, 0x97, 0x02, 0x22, 679 0x50, 0xC2, 0x50, 0xC2, 0x59, 0x09, 0xB8, 0x20, 0x3F, 0xCC, 0xAF, 0x96, 680 0x66, 0xC9, 0x02, 0x8A, 0x4F, 0xF1, 0x72, 0xB6, 0x44, 0xAC, 0x7F, 0x11, 681 0x2B, 0x7B, 0x4F, 0x79, 0xAF, 0x96, 0x02, 0x66, 0x4F, 0xF1, 0xAF, 0x96, 682 0x44, 0xAC, 0x6F, 0xD5, 0x2B, 0x7B, 0xB4, 0x41, 0x7F, 0x11, 0x02, 0x66, 683 0x7F, 0x11, 0x84, 0xCF, 0x2F, 0xE2, 0x2F, 0xE2, 0x96, 0x31, 0x7B, 0xE1, 684 0xA7, 0x41, 0x02, 0x22, 0x4F, 0xF1, 0xAF, 0x96, 0x6F, 0xD5, 0x6F, 0xD5, 685 0xB6, 0xC3, 0x9B, 0x15, 0x85, 0xE9, 0x02, 0x8A, 0x24, 0x78, 0xB6, 0x2E, 686 0x4F, 0xF1, 0xAF, 0x96, 0x2F, 0xF4, 0x44, 0xAC, 0x7F, 0x11, 0x02, 0x22, 687 0x30, 0x86, 0x4F, 0x79, 0xB3, 0xDD, 0x4F, 0xF1, 0x89, 0x2A, 0x63, 0x7B, 688 0x87, 0xE8, 0x02, 0x8A, 0x1E, 0x4D, 0xB0, 0x1B, 0xA6, 0x3E, 0x75, 0x00, 689 0x7D, 0x93, 0xAB, 0xAF, 0xAB, 0xAF, 0x02, 0x22, 0x7B, 0x7D, 0x3A, 0x63, 690 0x76, 0xA2, 0x83, 0x8A, 0x24, 0x78, 0x85, 0x16, 0x2B, 0x2D, 0x02, 0x8A, 691 0x84, 0x30, 0x4D, 0xF3, 0x52, 0x63, 0xA5, 0xC7, 0x21, 0xE0, 0xB8, 0x87, 692 0xBC, 0x16, 0x02, 0x66, 0x44, 0xAC, 0x2B, 0xD5, 0x9B, 0x15, 0x88, 0x52, 693 0x6C, 0x4F, 0xAF, 0x96, 0x64, 0xA1, 0x02, 0x66, 0x00, 0x00, 0x00, 0x20, 694 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 695 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 696 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 697 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 698 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 699 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 700 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 701 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 702 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 703 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 704 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 705 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 706 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 707 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 708 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 709 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 710 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 711 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 712 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 713 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 714 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 715 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 716 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 717 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 718 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 719 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 720 0x00, 0x20, 0x00, 0x20, 0x00, 0x20, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 721 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 722 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 723 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 724 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 725 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 726 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 727 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 728 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 729 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 730 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 731 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 732 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 733 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 734 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 735 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 736 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 737 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 738 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 739 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 740 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 741 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 742 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 743 0x00, 0x02, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 744 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 745 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x03, 746 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 0x00, 0x02, 747 0x00, 0x02, 0x00, 0x03, 748 }, 749 }, 750 } 751 752 func TestUCAWeightStrings(t *testing.T) { 753 for _, tc := range TestCases { 754 t.Run(tc.collation, func(t *testing.T) { 755 collation := testcollation(t, tc.collation) 756 for maxlen := 0; maxlen < len(tc.expected); maxlen += 2 { 757 buf := make([]byte, 0, len(tc.expected)) 758 result := collation.WeightString(buf, []byte(tc.input), PadToMax) 759 if !bytes.Equal(tc.expected[:maxlen], result[:maxlen]) { 760 t.Errorf("mismatch at len=%d\ninput: %#v\nexpected: %#v\nactual: %#v", 761 maxlen, []byte(tc.input), tc.expected[:maxlen], result[:maxlen]) 762 break 763 } 764 } 765 }) 766 } 767 } 768 769 func BenchmarkAllUCAWeightStrings(b *testing.B) { 770 for _, tc := range TestCases { 771 b.Run(tc.collation, func(b *testing.B) { 772 collation := testcollation(b, tc.collation) 773 buf := make([]byte, 0, len(tc.expected)) 774 input := []byte(tc.input) 775 776 b.ResetTimer() 777 778 for i := 0; i < b.N; i++ { 779 _ = collation.WeightString(buf, input, 0) 780 } 781 }) 782 } 783 } 784 785 func TestCompareWithWeightString(t *testing.T) { 786 var cases = []struct { 787 left, right string 788 equal bool 789 }{ 790 {"Abc", "aBC", true}, 791 {"ǍḄÇ", "ÁḆĈ", false}, 792 {"\uA73A", "\uA738", false}, 793 {"\uAC00", "\u326E", true}, 794 } 795 796 collation := testcollation(t, "utf8mb4_0900_as_ci") 797 798 for _, tc := range cases { 799 left := collation.WeightString(nil, []byte(tc.left), 0) 800 right := collation.WeightString(nil, []byte(tc.right), 0) 801 assert.Equal(t, tc.equal, bytes.Equal(left, right), "expected %q / %v == %q / %v to be %v", tc.left, left, tc.right, right, tc.equal) 802 803 } 804 } 805 806 func TestFastIterators(t *testing.T) { 807 allASCIICharacters := make([]byte, 128) 808 for n := range allASCIICharacters { 809 allASCIICharacters[n] = byte(n) 810 } 811 allASCIICharacters[0] = 'A' 812 813 var cases = []struct { 814 collation string 815 expected []byte 816 }{ 817 { 818 collation: "utf8mb4_0900_as_cs", 819 expected: []byte{0x1c, 0x47, 0x2, 0x1, 0x2, 0x2, 0x2, 0x3, 0x2, 0x4, 0x2, 0x5, 0x2, 0x9, 0x2, 0x60, 0x3, 0xc, 0x3, 0x98, 0x1c, 0x12, 0x3, 0x99, 0x3, 0x96, 0x3, 0x5, 0x3, 0x17, 0x3, 0x18, 0x3, 0x8f, 0x6, 0x16, 0x2, 0x22, 0x2, 0xd, 0x2, 0x77, 0x3, 0x94, 0x1c, 0x3d, 0x1c, 0x3e, 0x1c, 0x3f, 0x1c, 0x40, 0x1c, 0x41, 0x1c, 0x42, 0x1c, 0x43, 0x1c, 0x44, 0x1c, 0x45, 0x1c, 0x46, 0x2, 0x39, 0x2, 0x34, 0x6, 0x1a, 0x6, 0x1b, 0x6, 0x1c, 0x2, 0x66, 0x3, 0x8e, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x19, 0x3, 0x95, 0x3, 0x1a, 0x4, 0x85, 0x2, 0xb, 0x4, 0x82, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x1b, 0x6, 0x1e, 0x3, 0x1c, 0x6, 0x20, 0x0, 0x0, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x0, 0x0, 0x8, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x8, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2, 0x0, 0x2}, 820 }, 821 { 822 collation: "utf8mb4_0900_as_ci", 823 expected: []byte{0x1c, 0x47, 0x2, 0x1, 0x2, 0x2, 0x2, 0x3, 0x2, 0x4, 0x2, 0x5, 0x2, 0x9, 0x2, 0x60, 0x3, 0xc, 0x3, 0x98, 0x1c, 0x12, 0x3, 0x99, 0x3, 0x96, 0x3, 0x5, 0x3, 0x17, 0x3, 0x18, 0x3, 0x8f, 0x6, 0x16, 0x2, 0x22, 0x2, 0xd, 0x2, 0x77, 0x3, 0x94, 0x1c, 0x3d, 0x1c, 0x3e, 0x1c, 0x3f, 0x1c, 0x40, 0x1c, 0x41, 0x1c, 0x42, 0x1c, 0x43, 0x1c, 0x44, 0x1c, 0x45, 0x1c, 0x46, 0x2, 0x39, 0x2, 0x34, 0x6, 0x1a, 0x6, 0x1b, 0x6, 0x1c, 0x2, 0x66, 0x3, 0x8e, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x19, 0x3, 0x95, 0x3, 0x1a, 0x4, 0x85, 0x2, 0xb, 0x4, 0x82, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x1b, 0x6, 0x1e, 0x3, 0x1c, 0x6, 0x20, 0x0, 0x0, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20, 0x0, 0x20}, 824 }, 825 { 826 collation: "utf8mb4_0900_ai_ci", 827 expected: []byte{0x1c, 0x47, 0x2, 0x1, 0x2, 0x2, 0x2, 0x3, 0x2, 0x4, 0x2, 0x5, 0x2, 0x9, 0x2, 0x60, 0x3, 0xc, 0x3, 0x98, 0x1c, 0x12, 0x3, 0x99, 0x3, 0x96, 0x3, 0x5, 0x3, 0x17, 0x3, 0x18, 0x3, 0x8f, 0x6, 0x16, 0x2, 0x22, 0x2, 0xd, 0x2, 0x77, 0x3, 0x94, 0x1c, 0x3d, 0x1c, 0x3e, 0x1c, 0x3f, 0x1c, 0x40, 0x1c, 0x41, 0x1c, 0x42, 0x1c, 0x43, 0x1c, 0x44, 0x1c, 0x45, 0x1c, 0x46, 0x2, 0x39, 0x2, 0x34, 0x6, 0x1a, 0x6, 0x1b, 0x6, 0x1c, 0x2, 0x66, 0x3, 0x8e, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x19, 0x3, 0x95, 0x3, 0x1a, 0x4, 0x85, 0x2, 0xb, 0x4, 0x82, 0x1c, 0x47, 0x1c, 0x60, 0x1c, 0x7a, 0x1c, 0x8f, 0x1c, 0xaa, 0x1c, 0xe5, 0x1c, 0xf4, 0x1d, 0x18, 0x1d, 0x32, 0x1d, 0x4c, 0x1d, 0x65, 0x1d, 0x77, 0x1d, 0xaa, 0x1d, 0xb9, 0x1d, 0xdd, 0x1e, 0xc, 0x1e, 0x21, 0x1e, 0x33, 0x1e, 0x71, 0x1e, 0x95, 0x1e, 0xb5, 0x1e, 0xe3, 0x1e, 0xf5, 0x1e, 0xff, 0x1f, 0xb, 0x1f, 0x21, 0x3, 0x1b, 0x6, 0x1e, 0x3, 0x1c, 0x6, 0x20}, 828 }, 829 } 830 831 for _, tc := range cases { 832 t.Run(tc.collation, func(t *testing.T) { 833 coll := testcollation(t, tc.collation) 834 result := coll.WeightString(nil, allASCIICharacters, 0) 835 assert.True(t, bytes.Equal(tc.expected, result), "weight_string(%q) = %#v (expected %#v)", allASCIICharacters, result, tc.expected) 836 837 }) 838 } 839 } 840 841 func TestUniqueHashes(t *testing.T) { 842 for _, teststr := range AllTestStrings { 843 t.Run(teststr.Name, func(t *testing.T) { 844 var hashes = make(map[uintptr]string) 845 for _, collation := range testall() { 846 trans, _ := charset.ConvertFromUTF8(nil, collation.Charset(), []byte(teststr.Content)) 847 h := collation.Hash(trans, 0) 848 if dupname, dup := hashes[h]; dup { 849 t.Fatalf("%s hashes to %d in %s and %s", teststr.Name, h, collation.Name(), dupname) 850 } 851 hashes[h] = collation.Name() 852 } 853 }) 854 } 855 } 856 857 type ConsistentCollation struct { 858 Collation 859 t testing.TB 860 } 861 862 func (c *ConsistentCollation) Collate(left, right []byte, isPrefix bool) int { 863 cmp := c.Collation.Collate(left, right, isPrefix) 864 if isPrefix { 865 return cmp 866 } 867 868 equal := cmp == 0 869 w1 := c.WeightString(nil, left, 0) 870 w2 := c.WeightString(nil, right, 0) 871 if bytes.Equal(w1, w2) != equal { 872 c.t.Errorf("ConsistentCollation(%s): expected WeightString %q / %v == %q / %v to be %v", c.Name(), left, w1, right, w2, equal) 873 } 874 875 h1 := c.Hash(left, 0) 876 h2 := c.Hash(right, 0) 877 if (h1 == h2) != equal { 878 c.t.Errorf("ConsistentCollation(%s): expected Hash %q / %v == %q / %v to be %v", c.Name(), left, h1, right, h2, equal) 879 } 880 881 return cmp 882 } 883 884 func TestEqualities(t *testing.T) { 885 var cases = []struct { 886 collation string 887 left, right string 888 equal bool 889 }{ 890 {"utf8mb4_0900_as_ci", "Abc", "aBC", true}, 891 {"utf8mb4_0900_as_ci", "ǍḄÇ", "ÁḆĈ", false}, 892 {"utf8mb4_0900_as_ci", "\uA73A", "\uA738", false}, 893 {"utf8mb4_0900_as_ci", "\uAC00", "\u326E", true}, 894 {"utf8mb4_0900_as_cs", "の東京ノ", "ノ東京の", false}, 895 {"utf8mb4_ja_0900_as_cs", "の東京ノ", "ノ東京の", true}, 896 {"utf8mb4_ja_0900_as_cs_ks", "の東京ノ", "ノ東京の", false}, 897 } 898 899 for _, tc := range cases { 900 collation := testcollation(t, tc.collation) 901 collation = &ConsistentCollation{Collation: collation, t: t} 902 903 cmp := collation.Collate([]byte(tc.left), []byte(tc.right), false) 904 assert.Equal(t, tc.equal, (cmp == 0), "expected %q == %q to be %v", tc.left, tc.right, tc.equal) 905 906 } 907 } 908 909 func TestUCACollationOrder(t *testing.T) { 910 var sorted = []string{ 911 "aaaa", 912 "bbbb", 913 "cccc", 914 "dddd", 915 "zzzz", 916 } 917 918 var collations = []string{ 919 "utf8mb4_0900_ai_ci", 920 "utf8mb4_0900_as_cs", 921 } 922 923 for _, colname := range collations { 924 col := testcollation(t, colname) 925 926 for _, a := range sorted { 927 for _, b := range sorted { 928 want := strings.Compare(a, b) < 0 929 got := col.Collate([]byte(a), []byte(b), false) < 0 930 require.Equalf(t, want, got, "failed to compare %q vs %q", a, b) 931 } 932 } 933 934 ary := slices.Clone(sorted) 935 for i := range ary { 936 j := rand.Intn(i + 1) 937 ary[i], ary[j] = ary[j], ary[i] 938 } 939 slices.SortFunc(ary, func(a, b string) bool { 940 return col.Collate([]byte(a), []byte(b), false) < 0 941 }) 942 require.Equal(t, sorted, ary) 943 } 944 } 945 946 func TestCaseChangeEqualities(t *testing.T) { 947 for _, teststr := range AllTestStrings { 948 str1 := []byte(teststr.Content) 949 str2 := []byte(strings.ToUpper(teststr.Content)) 950 str3 := []byte(strings.ToLower(teststr.Content)) 951 952 t.Run(teststr.Name, func(t *testing.T) { 953 for _, collation := range testall() { 954 collation = &ConsistentCollation{Collation: collation, t: t} 955 956 cs := collation.Charset() 957 trans1, _ := charset.ConvertFromUTF8(nil, cs, str1) 958 trans2, _ := charset.ConvertFromUTF8(nil, cs, str2) 959 trans3, _ := charset.ConvertFromUTF8(nil, cs, str3) 960 961 _ = collation.Collate(trans1, trans2, false) 962 _ = collation.Collate(trans1, trans3, false) 963 _ = collation.Collate(trans2, trans3, false) 964 } 965 }) 966 } 967 } 968 969 func BenchmarkUCA900Collation(b *testing.B) { 970 var Collations = []Collation{ 971 testcollation(b, "utf8mb4_0900_as_cs"), 972 testcollation(b, "utf8mb4_0900_as_ci"), 973 testcollation(b, "utf8mb4_0900_ai_ci"), 974 } 975 976 var BenchStrings = []struct { 977 Name, Content string 978 }{ 979 {"Long", ExampleStringLong}, 980 {"Spanish", SpanishString}, 981 {"English", EnglishString}, 982 {"Japanese", JapaneseString2}, 983 } 984 985 for _, teststr := range BenchStrings { 986 for _, length := range []int{1, 8} { 987 content := strings.Repeat(teststr.Content, length) 988 str1 := []byte(content) 989 str2 := []byte(strings.ToUpper(content)) 990 for _, collation := range Collations { 991 b.Run(fmt.Sprintf("%s/%d/%s", teststr.Name, length, collation.Name()), func(b *testing.B) { 992 for i := 0; i < b.N; i++ { 993 _ = collation.Collate(str1, str2, false) 994 } 995 }) 996 } 997 } 998 } 999 }