github.com/lbryio/lbcd@v0.22.119/claimtrie/normalization/normalizer_icu_test.go (about)

     1  //go:build use_icu_normalization
     2  // +build use_icu_normalization
     3  
     4  package normalization
     5  
     6  import (
     7  	"bytes"
     8  	"encoding/hex"
     9  	"testing"
    10  	"unicode/utf8"
    11  
    12  	"github.com/stretchr/testify/assert"
    13  )
    14  
    15  func TestNormalizationICU(t *testing.T) {
    16  	testNormalization(t, normalizeICU)
    17  }
    18  
    19  func BenchmarkNormalizeICU(b *testing.B) {
    20  	benchmarkNormalize(b, normalizeICU)
    21  }
    22  
    23  var testStrings = []string{
    24  	"Les-Masques-Blancs-Die-Dead-place-Sathonay-28-Août",
    25  	"Bez-komentu-výbuch-z-vnútra,-radšej-pozri-video...-",
    26  	"၂-နစ်အကြာမှာ",
    27  	"ငရဲပြည်မှ-6",
    28  	"@happyvision",
    29  	"ကမ္ဘာပျက်ကိန်း-9",
    30  	"ဝိညာဉ်နား၊-3",
    31  	"un-amore-nuovo-o-un-ritorno-cosa-mi-dona",
    32  	"è-innamorato-di-me-anche-se-non-lo-dice",
    33  	"ပြင်ဆင်ပါ-no.1",
    34  	"ပြင်ဆင်ပါ-no.4",
    35  	"ပြင်ဆင်ပါ-no.2",
    36  	"ပြင်ဆင်ပါ-no.3",
    37  	"ငရဲပြည်မှ-5",
    38  	"ပြင်ဆင်ပါ-no.6",
    39  	"ပြင်ဆင်ပါ-no.5",
    40  	"ပြင်ဆင်ပါ-no.7",
    41  	"ပြင်ဆင်ပါ-no.8",
    42  	"အချိန်-2",
    43  	"ဝိညာဉ်နား၊-4",
    44  	"ပြင်ဆင်ပါ-no.-13",
    45  	"ပြင်ဆင်ပါ-no.15",
    46  	"ပြင်ဆင်ပါ-9",
    47  	"schilddrüsenhormonsubstitution-nach",
    48  	"Linxextremismus-JPzuG_UBtEg",
    49  	"Ꮖ-Ꮩ-Ꭺ-N--------Ꭺ-N-Ꮹ-Ꭼ-Ꮮ-Ꭺ-on-Instagram_-“Our-next-destination-is-East-and-Southeast-Asia--selfie--asia”",
    50  	"ABCDEFGHIJKLMNOPQRSTUVWXYZ",
    51  }
    52  
    53  func TestBlock760150_1020105(t *testing.T) {
    54  	test, _ := hex.DecodeString("43efbfbd")
    55  	assert.True(t, utf8.Valid(test))
    56  	a := normalizeGo(test)
    57  	b := normalizeICU(test)
    58  	assert.Equal(t, a, b)
    59  
    60  	for i, s := range testStrings {
    61  		a = normalizeGo([]byte(s))
    62  		b = normalizeICU([]byte(s))
    63  		assert.Equal(t, a, b, "%d: %s != %s", i, string(a), string(b))
    64  		// t.Logf("%s -> %s", s, string(b))
    65  	}
    66  }
    67  
    68  func TestBlock1085612(t *testing.T) {
    69  	s, err := hex.DecodeString("6eccb7cd9dcc92cd90cc86cc80cc80cd91cd9dcd8acd80cd92cc94cc85cc8fccbdcda0ccbdcd80cda0cd84cc94cc8ccc9acd84cc94cd9bcda0cca7cc99ccaccd99cca9cca7")
    70  	assert.NoError(t, err)
    71  	a := normalizeICU(s)
    72  	b := normalizeGo(s)
    73  	assert.Equal(t, a, b, "%s != %s, %v", string(a), string(b), bytes.Equal(b, s))
    74  }