github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/merging_iter_test.go (about) 1 // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "fmt" 9 "strings" 10 "testing" 11 "time" 12 13 "github.com/petermattis/pebble/cache" 14 "github.com/petermattis/pebble/internal/base" 15 "github.com/petermattis/pebble/internal/datadriven" 16 "github.com/petermattis/pebble/sstable" 17 "github.com/petermattis/pebble/vfs" 18 "golang.org/x/exp/rand" 19 ) 20 21 func TestMergingIter(t *testing.T) { 22 newFunc := func(iters ...internalIterator) internalIterator { 23 return newMergingIter(DefaultComparer.Compare, iters...) 24 } 25 testIterator(t, newFunc, func(r *rand.Rand) [][]string { 26 // Shuffle testKeyValuePairs into one or more splits. Each individual 27 // split is in increasing order, but different splits may overlap in 28 // range. Some of the splits may be empty. 29 splits := make([][]string, 1+r.Intn(2+len(testKeyValuePairs))) 30 for _, kv := range testKeyValuePairs { 31 j := r.Intn(len(splits)) 32 splits[j] = append(splits[j], kv) 33 } 34 return splits 35 }) 36 } 37 38 func TestMergingIterSeek(t *testing.T) { 39 var def string 40 datadriven.RunTest(t, "testdata/merging_iter_seek", func(d *datadriven.TestData) string { 41 switch d.Cmd { 42 case "define": 43 def = d.Input 44 return "" 45 46 case "iter": 47 var iters []internalIterator 48 for _, line := range strings.Split(def, "\n") { 49 f := &fakeIter{} 50 for _, key := range strings.Fields(line) { 51 j := strings.Index(key, ":") 52 f.keys = append(f.keys, base.ParseInternalKey(key[:j])) 53 f.vals = append(f.vals, []byte(key[j+1:])) 54 } 55 iters = append(iters, f) 56 } 57 58 iter := newMergingIter(DefaultComparer.Compare, iters...) 59 defer iter.Close() 60 return runInternalIterCmd(d, iter) 61 62 default: 63 return fmt.Sprintf("unknown command: %s", d.Cmd) 64 } 65 }) 66 } 67 68 func TestMergingIterNextPrev(t *testing.T) { 69 // The data is the same in each of these cases, but divided up amongst the 70 // iterators differently. This data must match the definition in 71 // testdata/internal_iter. 72 iterCases := [][]string{ 73 []string{ 74 "a.SET.2:2 a.SET.1:1 b.SET.2:2 b.SET.1:1 c.SET.2:2 c.SET.1:1", 75 }, 76 []string{ 77 "a.SET.2:2 b.SET.2:2 c.SET.2:2", 78 "a.SET.1:1 b.SET.1:1 c.SET.1:1", 79 }, 80 []string{ 81 "a.SET.2:2 b.SET.2:2", 82 "a.SET.1:1 b.SET.1:1", 83 "c.SET.2:2 c.SET.1:1", 84 }, 85 []string{ 86 "a.SET.2:2", 87 "a.SET.1:1", 88 "b.SET.2:2", 89 "b.SET.1:1", 90 "c.SET.2:2", 91 "c.SET.1:1", 92 }, 93 } 94 95 for _, c := range iterCases { 96 t.Run("", func(t *testing.T) { 97 datadriven.RunTest(t, "testdata/internal_iter_next", func(d *datadriven.TestData) string { 98 switch d.Cmd { 99 case "define": 100 // Ignore. We've defined the iterator data above. 101 return "" 102 103 case "iter": 104 iters := make([]internalIterator, len(c)) 105 for i := range c { 106 f := &fakeIter{} 107 iters[i] = f 108 for _, key := range strings.Fields(c[i]) { 109 j := strings.Index(key, ":") 110 f.keys = append(f.keys, base.ParseInternalKey(key[:j])) 111 f.vals = append(f.vals, []byte(key[j+1:])) 112 } 113 } 114 115 iter := newMergingIter(DefaultComparer.Compare, iters...) 116 defer iter.Close() 117 return runInternalIterCmd(d, iter) 118 119 default: 120 return fmt.Sprintf("unknown command: %s", d.Cmd) 121 } 122 }) 123 }) 124 } 125 } 126 127 func buildMergingIterTables( 128 b *testing.B, blockSize, restartInterval, count int, 129 ) ([]*sstable.Reader, [][]byte) { 130 mem := vfs.NewMem() 131 files := make([]vfs.File, count) 132 for i := range files { 133 f, err := mem.Create(fmt.Sprintf("bench%d", i)) 134 if err != nil { 135 b.Fatal(err) 136 } 137 defer f.Close() 138 files[i] = f 139 } 140 141 writers := make([]*sstable.Writer, len(files)) 142 for i := range files { 143 writers[i] = sstable.NewWriter(files[i], nil, LevelOptions{ 144 BlockRestartInterval: restartInterval, 145 BlockSize: blockSize, 146 Compression: NoCompression, 147 }) 148 } 149 150 estimatedSize := func() uint64 { 151 var sum uint64 152 for _, w := range writers { 153 sum += w.EstimatedSize() 154 } 155 return sum 156 } 157 158 var keys [][]byte 159 var ikey InternalKey 160 targetSize := uint64(count * (2 << 20)) 161 for i := 0; estimatedSize() < targetSize; i++ { 162 key := []byte(fmt.Sprintf("%08d", i)) 163 keys = append(keys, key) 164 ikey.UserKey = key 165 j := rand.Intn(len(writers)) 166 w := writers[j] 167 w.Add(ikey, nil) 168 } 169 170 for _, w := range writers { 171 if err := w.Close(); err != nil { 172 b.Fatal(err) 173 } 174 } 175 176 cache := cache.New(128 << 20) 177 readers := make([]*sstable.Reader, len(files)) 178 for i := range files { 179 f, err := mem.Open(fmt.Sprintf("bench%d", i)) 180 if err != nil { 181 b.Fatal(err) 182 } 183 readers[i], err = sstable.NewReader(f, 0, uint64(i), &Options{ 184 Cache: cache, 185 }) 186 if err != nil { 187 b.Fatal(err) 188 } 189 } 190 return readers, keys 191 } 192 193 func BenchmarkMergingIterSeekGE(b *testing.B) { 194 const blockSize = 32 << 10 195 196 for _, restartInterval := range []int{16} { 197 b.Run(fmt.Sprintf("restart=%d", restartInterval), 198 func(b *testing.B) { 199 for _, count := range []int{1, 2, 3, 4, 5} { 200 b.Run(fmt.Sprintf("count=%d", count), 201 func(b *testing.B) { 202 readers, keys := buildMergingIterTables(b, blockSize, restartInterval, count) 203 iters := make([]internalIterator, len(readers)) 204 for i := range readers { 205 iters[i] = readers[i].NewIter(nil /* lower */, nil /* upper */) 206 } 207 m := newMergingIter(DefaultComparer.Compare, iters...) 208 rng := rand.New(rand.NewSource(uint64(time.Now().UnixNano()))) 209 210 b.ResetTimer() 211 for i := 0; i < b.N; i++ { 212 m.SeekGE(keys[rng.Intn(len(keys))]) 213 } 214 }) 215 } 216 }) 217 } 218 } 219 220 func BenchmarkMergingIterNext(b *testing.B) { 221 const blockSize = 32 << 10 222 223 for _, restartInterval := range []int{16} { 224 b.Run(fmt.Sprintf("restart=%d", restartInterval), 225 func(b *testing.B) { 226 for _, count := range []int{1, 2, 3, 4, 5} { 227 b.Run(fmt.Sprintf("count=%d", count), 228 func(b *testing.B) { 229 readers, _ := buildMergingIterTables(b, blockSize, restartInterval, count) 230 iters := make([]internalIterator, len(readers)) 231 for i := range readers { 232 iters[i] = readers[i].NewIter(nil /* lower */, nil /* upper */) 233 } 234 m := newMergingIter(DefaultComparer.Compare, iters...) 235 236 b.ResetTimer() 237 for i := 0; i < b.N; i++ { 238 key, _ := m.Next() 239 if key == nil { 240 key, _ = m.First() 241 } 242 _ = key 243 } 244 }) 245 } 246 }) 247 } 248 } 249 250 func BenchmarkMergingIterPrev(b *testing.B) { 251 const blockSize = 32 << 10 252 253 for _, restartInterval := range []int{16} { 254 b.Run(fmt.Sprintf("restart=%d", restartInterval), 255 func(b *testing.B) { 256 for _, count := range []int{1, 2, 3, 4, 5} { 257 b.Run(fmt.Sprintf("count=%d", count), 258 func(b *testing.B) { 259 readers, _ := buildMergingIterTables(b, blockSize, restartInterval, count) 260 iters := make([]internalIterator, len(readers)) 261 for i := range readers { 262 iters[i] = readers[i].NewIter(nil /* lower */, nil /* upper */) 263 } 264 m := newMergingIter(DefaultComparer.Compare, iters...) 265 266 b.ResetTimer() 267 for i := 0; i < b.N; i++ { 268 key, _ := m.Prev() 269 if key == nil { 270 key, _ = m.Last() 271 } 272 _ = key 273 } 274 }) 275 } 276 }) 277 } 278 }