vitess.io/vitess@v0.16.2/go/mysql/collations/uca_tables_test.go (about)

     1  /*
     2  Copyright 2021 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package collations
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"os"
    23  	"reflect"
    24  	"strconv"
    25  	"testing"
    26  	"unsafe"
    27  
    28  	"github.com/stretchr/testify/require"
    29  	"gotest.tools/assert"
    30  
    31  	"vitess.io/vitess/go/mysql/collations/internal/charset"
    32  	"vitess.io/vitess/go/mysql/collations/internal/uca"
    33  )
    34  
    35  func verifyAllCodepoints(t *testing.T, expected map[rune][]uint16, weights uca.Weights, layout uca.Layout) {
    36  	t.Helper()
    37  
    38  	maxCodepoint := layout.MaxCodepoint()
    39  	for cp := rune(0); cp <= maxCodepoint; cp++ {
    40  		vitessWeights := layout.DebugWeights(weights, cp)
    41  		mysqlWeights, mysqlFound := expected[cp]
    42  
    43  		if len(vitessWeights) == 0 {
    44  			if mysqlFound {
    45  				t.Errorf("missing MySQL weight in Vitess' tables: U+%04X", cp)
    46  				continue
    47  			}
    48  		} else {
    49  			if !mysqlFound {
    50  				t.Errorf("missing Vitess weight in MySQL's tables: U+%04X", cp)
    51  				continue
    52  			}
    53  
    54  			if len(mysqlWeights) != len(vitessWeights) {
    55  				t.Errorf("wrong number of collation entities for U+%04X: mysql=%v vs vitess=%v", cp, mysqlWeights, vitessWeights)
    56  				continue
    57  			}
    58  
    59  			for i := range vitessWeights {
    60  				a, b := mysqlWeights[i], vitessWeights[i]
    61  				assert.Equal(t, b, a, "weight mismatch for U+%04X (collation entity %d): mysql=%v vitess=%v", cp, i+1, a, b)
    62  
    63  			}
    64  		}
    65  	}
    66  }
    67  
    68  func loadExpectedWeights(t *testing.T, weights string) map[rune][]uint16 {
    69  	fullpath := fmt.Sprintf("testdata/mysqldata/%s.json", weights)
    70  	weightsMysqlFile, err := os.Open(fullpath)
    71  	if err != nil {
    72  		t.Skipf("failed to load %q (did you run 'colldump' locally?)", fullpath)
    73  	}
    74  
    75  	var meta struct {
    76  		Weights map[string][]uint16
    77  	}
    78  	dec := json.NewDecoder(weightsMysqlFile)
    79  	require.NoError(t, dec.Decode(&meta))
    80  
    81  	var result = make(map[rune][]uint16, len(meta.Weights))
    82  	for key, w := range meta.Weights {
    83  		cp, err := strconv.ParseInt(key[2:], 16, 32)
    84  		if err != nil {
    85  			t.Fatal(err)
    86  		}
    87  		result[rune(cp)] = w
    88  	}
    89  	return result
    90  }
    91  
    92  func TestWeightsForAllCodepoints(t *testing.T) {
    93  	testWeightsFromMysql := loadExpectedWeights(t, "utf8mb4_0900_ai_ci")
    94  	verifyAllCodepoints(t, testWeightsFromMysql, weightTable_uca900, uca.Layout_uca900{})
    95  }
    96  
    97  func TestWeightTablesAreDeduplicated(t *testing.T) {
    98  	sliceptr := func(table uca.Weights) uintptr {
    99  		hdr := (*reflect.SliceHeader)(unsafe.Pointer(&table))
   100  		return hdr.Data
   101  	}
   102  
   103  	uniqueTables := make(map[uintptr]int)
   104  	for _, col := range testall() {
   105  		var weights uca.Weights
   106  		switch col := col.(type) {
   107  		case *Collation_uca_legacy:
   108  			weights, _ = col.uca.Weights()
   109  		case *Collation_utf8mb4_uca_0900:
   110  			weights, _ = col.uca.Weights()
   111  		default:
   112  			continue
   113  		}
   114  		uniqueTables[sliceptr(weights)]++
   115  	}
   116  
   117  	var total int
   118  	for _, count := range uniqueTables {
   119  		total += count
   120  	}
   121  	average := float64(total) / float64(len(uniqueTables))
   122  	if average < 10.0/3.0 {
   123  		t.Fatalf("weight tables are not deduplicated, average table reuse: %f", average)
   124  	}
   125  }
   126  
   127  func TestTailoringPatchApplication(t *testing.T) {
   128  	for _, col := range testall() {
   129  		var weightTable uca.Weights
   130  		var tableLayout uca.Layout
   131  
   132  		switch col := col.(type) {
   133  		case *Collation_uca_legacy:
   134  			if _, utf8 := col.charset.(charset.Charset_utf8mb4); !utf8 {
   135  				continue
   136  			}
   137  			weightTable, tableLayout = col.uca.Weights()
   138  		case *Collation_utf8mb4_uca_0900:
   139  			weightTable, tableLayout = col.uca.Weights()
   140  		default:
   141  			continue
   142  		}
   143  
   144  		t.Run(col.Name(), func(t *testing.T) {
   145  			expected := loadExpectedWeights(t, col.Name())
   146  			verifyAllCodepoints(t, expected, weightTable, tableLayout)
   147  		})
   148  	}
   149  }