github.com/etecs-ru/ristretto@v0.9.1/z/buffer_test.go (about) 1 /* 2 * Copyright 2020 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package z 18 19 import ( 20 "bytes" 21 "encoding/binary" 22 "encoding/hex" 23 "fmt" 24 "math/rand" 25 "sort" 26 "testing" 27 "time" 28 29 "github.com/stretchr/testify/require" 30 ) 31 32 func TestBuffer(t *testing.T) { 33 rand.Seed(time.Now().Unix()) 34 const capacity = 512 35 buffers := newTestBuffers(t, capacity) 36 37 for _, buf := range buffers { 38 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 39 t.Run(name, func(t *testing.T) { 40 // This is just for verifying result 41 var bytesBuf bytes.Buffer 42 bytesBuf.Grow(capacity) 43 44 // Writer small []byte 45 var smallData [256]byte 46 rand.Read(smallData[:]) 47 var bigData [1024]byte 48 rand.Read(bigData[:]) 49 50 _, err := buf.Write(smallData[:]) 51 require.NoError(t, err, "unable to write data to page buffer") 52 _, err = buf.Write(bigData[:]) 53 require.NoError(t, err, "unable to write data to page buffer") 54 55 // Write data to bytesBuffer also, just to match result. 56 bytesBuf.Write(smallData[:]) 57 bytesBuf.Write(bigData[:]) 58 require.Equal(t, buf.Bytes(), bytesBuf.Bytes()) 59 }) 60 } 61 } 62 63 func TestBufferWrite(t *testing.T) { 64 rand.Seed(time.Now().Unix()) 65 const capacity = 32 66 buffers := newTestBuffers(t, capacity) 67 68 for _, buf := range buffers { 69 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 70 t.Run(name, func(t *testing.T) { 71 var data [128]byte 72 rand.Read(data[:]) 73 bytesBuf := new(bytes.Buffer) 74 75 end := 32 76 for i := 0; i < 3; i++ { 77 n, err := buf.Write(data[:end]) 78 require.NoError(t, err, "unable to write bytes to buffer") 79 require.Equal(t, n, end, "length of buffer and length written should be equal") 80 81 // append to bb also for testing. 82 bytesBuf.Write(data[:end]) 83 84 require.Equal(t, buf.Bytes(), bytesBuf.Bytes()) 85 end = end * 2 86 } 87 }) 88 } 89 } 90 91 func TestBufferAutoMmap(t *testing.T) { 92 buf := NewBuffer(1<<20, "test").WithAutoMmap(64<<20, "") 93 defer func() { require.NoError(t, buf.Release()) }() 94 95 N := 128 << 10 96 var wb [1024]byte 97 for i := 0; i < N; i++ { 98 rand.Read(wb[:]) 99 b := buf.SliceAllocate(len(wb)) 100 copy(b, wb[:]) 101 } 102 t.Logf("Buffer size: %d\n", buf.LenWithPadding()) 103 104 buf.SortSlice(func(l, r []byte) bool { 105 return bytes.Compare(l, r) < 0 106 }) 107 t.Logf("sort done\n") 108 109 var count int 110 var last []byte 111 buf.SliceIterate(func(slice []byte) error { 112 require.True(t, bytes.Compare(slice, last) >= 0) 113 last = append(last[:0], slice...) 114 count++ 115 return nil 116 }) 117 require.Equal(t, N, count) 118 } 119 120 func TestBufferSimpleSort(t *testing.T) { 121 bufs := newTestBuffers(t, 1<<20) 122 for _, buf := range bufs { 123 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 124 t.Run(name, func(t *testing.T) { 125 for i := 0; i < 25600; i++ { 126 b := buf.SliceAllocate(4) 127 binary.BigEndian.PutUint32(b, uint32(rand.Int31n(256000))) 128 } 129 buf.SortSlice(func(ls, rs []byte) bool { 130 left := binary.BigEndian.Uint32(ls) 131 right := binary.BigEndian.Uint32(rs) 132 return left < right 133 }) 134 var last uint32 135 var i int 136 buf.SliceIterate(func(slice []byte) error { 137 num := binary.BigEndian.Uint32(slice) 138 if num < last { 139 fmt.Printf("num: %d idx: %d last: %d\n", num, i, last) 140 } 141 i++ 142 require.GreaterOrEqual(t, num, last) 143 last = num 144 // fmt.Printf("Got number: %d\n", num) 145 return nil 146 }) 147 }) 148 } 149 } 150 151 func TestBufferSlice(t *testing.T) { 152 const capacity = 32 153 buffers := newTestBuffers(t, capacity) 154 155 for _, buf := range buffers { 156 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 157 t.Run(name, func(t *testing.T) { 158 count := 10000 159 exp := make([][]byte, 0, count) 160 161 // Create "count" number of slices. 162 for i := 0; i < count; i++ { 163 sz := 1 + rand.Intn(8) 164 testBuf := make([]byte, sz) 165 rand.Read(testBuf) 166 167 newSlice := buf.SliceAllocate(sz) 168 require.Equal(t, sz, copy(newSlice, testBuf)) 169 170 // Save testBuf for verification. 171 exp = append(exp, testBuf) 172 } 173 174 compare := func() { 175 i := 0 176 buf.SliceIterate(func(slice []byte) error { 177 // All the slices returned by the buffer should be equal to what we 178 // inserted earlier. 179 if !bytes.Equal(exp[i], slice) { 180 fmt.Printf("exp: %s got: %s\n", hex.Dump(exp[i]), hex.Dump(slice)) 181 t.Fail() 182 } 183 require.Equal(t, exp[i], slice) 184 i++ 185 return nil 186 }) 187 require.Equal(t, len(exp), i) 188 } 189 compare() // same order as inserted. 190 191 t.Logf("Sorting using sort.Slice\n") 192 sort.Slice(exp, func(i, j int) bool { 193 return bytes.Compare(exp[i], exp[j]) < 0 194 }) 195 t.Logf("Sorting using buf.SortSlice\n") 196 buf.SortSlice(func(a, b []byte) bool { 197 return bytes.Compare(a, b) < 0 198 }) 199 t.Logf("Done sorting\n") 200 compare() // same order after sort. 201 }) 202 } 203 } 204 205 func TestBufferSort(t *testing.T) { 206 const capacity = 32 207 bufs := newTestBuffers(t, capacity) 208 209 for _, buf := range bufs { 210 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 211 t.Run(name, func(t *testing.T) { 212 const N = 10000 213 214 for i := 0; i < N; i++ { 215 newSlice := buf.SliceAllocate(8) 216 uid := uint64(rand.Int63()) 217 binary.BigEndian.PutUint64(newSlice, uid) 218 } 219 220 test := func(start, end int) { 221 start = buf.StartOffset() + 12*start 222 end = buf.StartOffset() + 12*end 223 buf.SortSliceBetween(start, end, func(ls, rs []byte) bool { 224 lhs := binary.BigEndian.Uint64(ls) 225 rhs := binary.BigEndian.Uint64(rs) 226 return lhs < rhs 227 }) 228 229 slice, next := []byte{}, start 230 var last uint64 231 var count int 232 for next >= 0 && next < end { 233 slice, next = buf.Slice(next) 234 uid := binary.BigEndian.Uint64(slice) 235 require.GreaterOrEqual(t, uid, last) 236 last = uid 237 count++ 238 } 239 require.Equal(t, (end-start)/12, count) 240 } 241 for i := 10; i <= N; i += 10 { 242 test(i-10, i) 243 } 244 test(0, N) 245 }) 246 } 247 } 248 249 // Test that the APIs returns the expected offsets. 250 func TestBufferPadding(t *testing.T) { 251 bufs := newTestBuffers(t, 1<<10) 252 for _, buf := range bufs { 253 name := fmt.Sprintf("Using buffer type: %s", buf.bufType) 254 t.Run(name, func(t *testing.T) { 255 sz := rand.Int31n(100) 256 257 writeOffset := buf.AllocateOffset(int(sz)) 258 require.Equal(t, buf.StartOffset(), writeOffset) 259 260 b := make([]byte, sz) 261 rand.Read(b) 262 263 copy(buf.Bytes(), b) 264 data := buf.Data(buf.StartOffset()) 265 require.Equal(t, b, data[:sz]) 266 }) 267 } 268 } 269 270 func newTestBuffers(t *testing.T, capacity int) []*Buffer { 271 var bufs []*Buffer 272 273 buf := NewBuffer(capacity, "test") 274 bufs = append(bufs, buf) 275 276 buf, err := NewBufferTmp("", capacity) 277 require.NoError(t, err) 278 bufs = append(bufs, buf) 279 280 t.Cleanup(func() { 281 for _, buf := range bufs { 282 require.NoError(t, buf.Release()) 283 } 284 }) 285 286 return bufs 287 } 288 289 func TestSmallBuffer(t *testing.T) { 290 buf := NewBuffer(5, "test") 291 t.Cleanup(func() { 292 require.NoError(t, buf.Release()) 293 }) 294 // Write something to buffer so sort actually happens. 295 buf.WriteSlice([]byte("abc")) 296 // This test fails if the buffer has offset > currSz. 297 require.NotPanics(t, func() { 298 buf.SortSlice(func(left, right []byte) bool { 299 return true 300 }) 301 }) 302 }