github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/index/zonemap_test.go (about) 1 // Copyright 2021 Matrix Origin 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package index 16 17 import ( 18 "strconv" 19 "testing" 20 21 "github.com/RoaringBitmap/roaring" 22 "github.com/matrixorigin/matrixone/pkg/container/types" 23 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" 24 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/testutils" 25 "github.com/stretchr/testify/require" 26 ) 27 28 func TestZoneMapNumeric(t *testing.T) { 29 defer testutils.AfterTest(t)() 30 testutils.EnsureNoLeak(t) 31 typ := types.Type{Oid: types.T_int32} 32 zm := NewZoneMap(typ) 33 var yes bool 34 var err error 35 var visibility *roaring.Bitmap 36 yes = zm.Contains(int32(0)) 37 require.False(t, yes) 38 39 rows := 1000 40 ctx := new(KeysCtx) 41 ctx.Keys = containers.MockVector2(typ, rows, 0) 42 ctx.Count = rows 43 defer ctx.Keys.Close() 44 err = zm.BatchUpdate(ctx) 45 require.NoError(t, err) 46 47 yes = zm.Contains(int32(0)) 48 require.True(t, yes) 49 50 yes = zm.Contains(int32(999)) 51 require.True(t, yes) 52 53 yes = zm.Contains(int32(555)) 54 require.True(t, yes) 55 56 yes = zm.Contains(int32(1000)) 57 require.False(t, yes) 58 59 yes = zm.Contains(int32(-1)) 60 require.False(t, yes) 61 62 rows = 500 63 ctx.Keys = containers.MockVector2(typ, rows, 700) 64 ctx.Count = rows 65 defer ctx.Keys.Close() 66 err = zm.BatchUpdate(ctx) 67 require.NoError(t, err) 68 69 yes = zm.Contains(int32(1001)) 70 require.True(t, yes) 71 72 yes = zm.Contains(int32(1199)) 73 require.True(t, yes) 74 75 yes = zm.Contains(int32(1200)) 76 require.False(t, yes) 77 78 rows = 500 79 ctx.Keys = containers.MockVector2(typ, rows, -200) 80 ctx.Count = rows 81 defer ctx.Keys.Close() 82 err = zm.BatchUpdate(ctx) 83 require.NoError(t, err) 84 85 yes = zm.Contains(int32(-201)) 86 require.False(t, yes) 87 88 yes = zm.Contains(int32(-100)) 89 require.True(t, yes) 90 91 buf, err := zm.Marshal() 92 require.NoError(t, err) 93 zm1 := NewZoneMap(typ) 94 err = zm1.Unmarshal(buf) 95 require.NoError(t, err) 96 97 rows = 500 98 typ1 := typ 99 typ1.Oid = types.T_int64 100 ctx.Keys = containers.MockVector2(typ1, rows, 2000) 101 ctx.Count = rows 102 defer ctx.Keys.Close() 103 err = zm.BatchUpdate(ctx) 104 require.Error(t, err) 105 106 yes = zm1.Contains(int32(1234)) 107 require.False(t, yes) 108 109 yes = zm1.Contains(int32(1199)) 110 require.True(t, yes) 111 112 typ1.Oid = types.T_int32 113 vec := containers.MockVector2(typ1, rows, 3000) 114 defer vec.Close() 115 visibility, yes = zm1.ContainsAny(vec) 116 require.False(t, yes) 117 require.Equal(t, uint64(0), visibility.GetCardinality()) 118 119 vec = containers.MockVector2(typ1, rows, 0) 120 defer vec.Close() 121 visibility, yes = zm1.ContainsAny(vec) 122 require.True(t, yes) 123 require.Equal(t, uint64(rows), visibility.GetCardinality()) 124 125 err = zm1.Update(int32(999999)) 126 require.NoError(t, err) 127 128 yes = zm1.Contains(int32(99999)) 129 require.True(t, yes) 130 } 131 132 func TestZoneMapString(t *testing.T) { 133 defer testutils.AfterTest(t)() 134 testutils.EnsureNoLeak(t) 135 typ := types.Type{Oid: types.T_char} 136 zm := NewZoneMap(typ) 137 var yes bool 138 var err error 139 yes = zm.Contains([]byte(strconv.Itoa(0))) 140 require.False(t, yes) 141 142 rows := 1000 143 ctx := new(KeysCtx) 144 ctx.Keys = containers.MockVector2(typ, rows, 0) 145 ctx.Count = rows 146 defer ctx.Keys.Close() 147 err = zm.BatchUpdate(ctx) 148 require.NoError(t, err) 149 150 yes = zm.Contains([]byte(strconv.Itoa(500))) 151 require.True(t, yes) 152 153 yes = zm.Contains([]byte(strconv.Itoa(9999))) 154 require.False(t, yes) 155 156 yes = zm.Contains([]byte("/")) 157 require.False(t, yes) 158 159 err = zm.Update([]byte("z")) 160 require.NoError(t, err) 161 162 yes = zm.Contains([]byte(strconv.Itoa(999999))) 163 require.True(t, yes) 164 165 yes = zm.Contains([]byte("abcdefghijklmn")) 166 require.True(t, yes) 167 168 yes = zm.Contains([]byte("ydasdasda")) 169 require.True(t, yes) 170 171 yes = zm.Contains([]byte("z1")) 172 require.False(t, yes) 173 174 buf, err := zm.Marshal() 175 require.NoError(t, err) 176 zm1 := NewZoneMap(typ) 177 err = zm1.Unmarshal(buf) 178 require.NoError(t, err) 179 180 yes = zm.Contains([]byte("z1")) 181 require.False(t, yes) 182 183 yes = zm.Contains([]byte("z")) 184 require.True(t, yes) 185 186 yes = zm.Contains([]byte("/")) 187 require.False(t, yes) 188 } 189 190 func TestZMEmptyString(t *testing.T) { 191 defer testutils.AfterTest(t)() 192 typ := types.Type{Oid: types.T_varchar, Width: types.MaxVarcharLen} 193 zm := NewZoneMap(typ) 194 require.Equal(t, typ.Oid, zm.GetType().Oid) 195 // check not inited 196 require.False(t, zm.Contains(nil)) 197 _, existed := zm.ContainsAny(nil) 198 require.False(t, existed) 199 200 zmNoInit := NewZoneMap(typ) 201 data, _ := zm.Marshal() 202 _ = zmNoInit.Unmarshal(data) 203 require.False(t, zm.Contains(nil)) 204 _, existed = zm.ContainsAny(nil) 205 require.False(t, existed) 206 207 zm.SetMin([]byte{0, 0, 0}) 208 zm.SetMin([]byte("")) 209 210 require.Equal(t, []byte{0, 0, 0}, zm.GetMax()) 211 require.Equal(t, []byte{}, zm.GetMin()) 212 require.True(t, zm.Contains([]byte(""))) 213 require.True(t, zm.Contains([]byte{0, 0})) 214 require.False(t, zm.Contains([]byte{0, 0, 0, 0})) 215 216 data, _ = zm.Marshal() 217 zm1 := NewZoneMap(typ) 218 _ = zm1.Unmarshal(data) 219 220 require.True(t, zm1.Contains([]byte(""))) 221 require.True(t, zm1.Contains([]byte{0, 0})) 222 // this is a false positive 223 require.True(t, zm1.Contains([]byte{0, 0, 0, 0})) 224 } 225 226 func TestZMTruncatedString(t *testing.T) { 227 defer testutils.AfterTest(t)() 228 mockBytes := func(init byte, size int) []byte { 229 ret := make([]byte, size) 230 for i := 0; i < size; i++ { 231 ret[i] = init 232 } 233 return ret 234 } 235 236 typ := types.Type{Oid: types.T_varchar, Width: types.MaxVarcharLen} 237 zm := NewZoneMap(typ) 238 239 minv := mockBytes(0x00, 33) 240 maxv := mockBytes(0xff, 33) 241 maxv[3], maxv[32] = 0x00, 0x00 242 243 zm.SetMax(maxv) 244 zm.SetMin(minv) 245 246 require.False(t, zm.Contains([]byte(""))) 247 require.False(t, zm.Contains([]byte{0, 0})) 248 249 // not in original range, but in deserialized range 250 edgeMin := mockBytes(0x00, 32) 251 edgeMax := mockBytes(0xff, 33) 252 edgeMax[3], edgeMax[32] = 0x00, 0x11 253 254 require.False(t, zm.Contains(edgeMin)) 255 require.False(t, zm.Contains(edgeMax)) 256 257 data, _ := zm.Marshal() 258 zm1 := NewZoneMap(typ) 259 _ = zm1.Unmarshal(data) 260 261 require.True(t, zm1.Contains(edgeMin)) 262 require.True(t, zm1.Contains(edgeMax)) 263 264 zm.SetMax(mockBytes(0xff, 33)) // isInf is true 265 data, _ = zm.Marshal() 266 zm2 := NewZoneMap(typ) 267 zm2.Unmarshal(data) 268 require.True(t, zm2.isInf) 269 require.True(t, zm2.Contains(mockBytes(0xff, 100))) 270 271 }