github.com/coocood/badger@v1.5.1-0.20200528065104-c02ac3616d04/manifest_test.go (about) 1 /* 2 * Copyright 2017 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package badger 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "math/rand" 23 "os" 24 "path/filepath" 25 "sort" 26 "testing" 27 28 "github.com/coocood/badger/cache" 29 "github.com/coocood/badger/epoch" 30 "github.com/coocood/badger/options" 31 "github.com/coocood/badger/protos" 32 "github.com/coocood/badger/table/sstable" 33 "github.com/coocood/badger/y" 34 "github.com/stretchr/testify/require" 35 ) 36 37 func testCache() *cache.Cache { 38 c, err := cache.NewCache(&cache.Config{ 39 NumCounters: 1000000 * 10, 40 MaxCost: 1000000, 41 BufferItems: 64, 42 Metrics: true, 43 }) 44 y.Check(err) 45 return c 46 } 47 48 func TestManifestBasic(t *testing.T) { 49 dir, err := ioutil.TempDir("", "badger") 50 require.NoError(t, err) 51 defer os.RemoveAll(dir) 52 53 opt := getTestOptions(dir) 54 { 55 kv, err := Open(opt) 56 require.NoError(t, err) 57 n := 5000 58 for i := 0; i < n; i++ { 59 if (i % 10000) == 0 { 60 fmt.Printf("Putting i=%d\n", i) 61 } 62 k := []byte(fmt.Sprintf("%16x", rand.Int63())) 63 txnSet(t, kv, k, k, 0x00) 64 } 65 txnSet(t, kv, []byte("testkey"), []byte("testval"), 0x05) 66 kv.validate() 67 require.NoError(t, kv.Close()) 68 } 69 70 kv, err := Open(opt) 71 require.NoError(t, err) 72 73 require.NoError(t, kv.View(func(txn *Txn) error { 74 item, err := txn.Get([]byte("testkey")) 75 require.NoError(t, err) 76 require.EqualValues(t, "testval", string(getItemValue(t, item))) 77 require.EqualValues(t, []byte{0x05}, item.UserMeta()) 78 return nil 79 })) 80 require.NoError(t, kv.Close()) 81 } 82 83 func helpTestManifestFileCorruption(t *testing.T, off int64, errorContent string) { 84 dir, err := ioutil.TempDir("", "badger") 85 require.NoError(t, err) 86 defer os.RemoveAll(dir) 87 88 opt := getTestOptions(dir) 89 { 90 kv, err := Open(opt) 91 require.NoError(t, err) 92 require.NoError(t, kv.Close()) 93 } 94 fp, err := os.OpenFile(filepath.Join(dir, ManifestFilename), os.O_RDWR, 0) 95 require.NoError(t, err) 96 // Mess with magic value or version to force error 97 _, err = fp.WriteAt([]byte{'X'}, off) 98 require.NoError(t, err) 99 require.NoError(t, fp.Close()) 100 kv, err := Open(opt) 101 defer func() { 102 if kv != nil { 103 kv.Close() 104 } 105 }() 106 require.Error(t, err) 107 require.Contains(t, err.Error(), errorContent) 108 } 109 110 func TestManifestMagic(t *testing.T) { 111 helpTestManifestFileCorruption(t, 3, "bad magic") 112 } 113 114 func TestManifestVersion(t *testing.T) { 115 helpTestManifestFileCorruption(t, 4, "unsupported version") 116 } 117 118 func key(prefix string, i int) string { 119 return prefix + fmt.Sprintf("%04d", i) 120 } 121 122 func buildTestTable(t *testing.T, prefix string, n int) *os.File { 123 y.Assert(n <= 10000) 124 keyValues := make([][]string, n) 125 for i := 0; i < n; i++ { 126 k := key(prefix, i) 127 v := fmt.Sprintf("%d", i) 128 keyValues[i] = []string{k, v} 129 } 130 return buildTable(t, keyValues) 131 } 132 133 // TODO - Move these to somewhere where table package can also use it. 134 // keyValues is n by 2 where n is number of pairs. 135 func buildTable(t *testing.T, keyValues [][]string) *os.File { 136 // TODO: Add test for file garbage collection here. No files should be left after the tests here. 137 138 filename := fmt.Sprintf("%s%s%x.sst", os.TempDir(), string(os.PathSeparator), rand.Uint32()) 139 f, err := y.OpenSyncedFile(filename, false) 140 if t != nil { 141 require.NoError(t, err) 142 } else { 143 y.Check(err) 144 } 145 146 sort.Slice(keyValues, func(i, j int) bool { 147 return keyValues[i][0] < keyValues[j][0] 148 }) 149 150 opts := DefaultOptions.TableBuilderOptions 151 opts.CompressionPerLevel = getTestCompression(options.ZSTD) 152 b := sstable.NewTableBuilder(f, nil, 0, opts) 153 defer b.Close() 154 for _, kv := range keyValues { 155 y.Assert(len(kv) == 2) 156 err := b.Add(y.KeyWithTs([]byte(kv[0]), 10), y.ValueStruct{ 157 Value: []byte(kv[1]), 158 Meta: 'A', 159 }) 160 if t != nil { 161 require.NoError(t, err) 162 } else { 163 y.Check(err) 164 } 165 } 166 y.Check(b.Finish()) 167 f.Close() 168 f, _ = y.OpenSyncedFile(filename, true) 169 return f 170 } 171 172 func TestOverlappingKeyRangeError(t *testing.T) { 173 dir, err := ioutil.TempDir("", "badger") 174 require.NoError(t, err) 175 defer os.RemoveAll(dir) 176 opt := DefaultOptions 177 opt.Dir = dir 178 opt.ValueDir = dir 179 kv, err := Open(opt) 180 require.NoError(t, err) 181 defer kv.Close() 182 blkCache, idxCache := testCache(), testCache() 183 184 lh0 := newLevelHandler(kv, 0) 185 lh1 := newLevelHandler(kv, 1) 186 f := buildTestTable(t, "k", 2) 187 t1, err := sstable.OpenTable(f.Name(), blkCache, idxCache) 188 require.NoError(t, err) 189 defer t1.Delete() 190 191 done := lh0.tryAddLevel0Table(t1) 192 require.Equal(t, true, done) 193 194 cd := &compactDef{ 195 thisLevel: lh0, 196 nextLevel: lh1, 197 } 198 199 closer := y.NewCloser(0) 200 defer closer.SignalAndWait() 201 rm := epoch.NewResourceManager(closer, epoch.NoOpInspector{}) 202 g := rm.Acquire() 203 defer g.Done() 204 manifest := createManifest() 205 opts := DefaultOptions.TableBuilderOptions 206 lc, err := newLevelsController(kv, &manifest, rm, opts) 207 require.NoError(t, err) 208 done = lc.fillTablesL0(cd) 209 require.Equal(t, true, done) 210 lc.runCompactDef(0, cd, nil, g) 211 212 f = buildTestTable(t, "l", 2) 213 t2, err := sstable.OpenTable(f.Name(), blkCache, idxCache) 214 require.NoError(t, err) 215 defer t2.Delete() 216 done = lh0.tryAddLevel0Table(t2) 217 require.Equal(t, true, done) 218 219 cd = &compactDef{ 220 thisLevel: lh0, 221 nextLevel: lh1, 222 } 223 lc.fillTablesL0(cd) 224 lc.runCompactDef(0, cd, nil, g) 225 } 226 227 func TestManifestRewrite(t *testing.T) { 228 dir, err := ioutil.TempDir("", "badger") 229 require.NoError(t, err) 230 defer os.RemoveAll(dir) 231 deletionsThreshold := 10 232 mf, m, err := helpOpenOrCreateManifestFile(dir, false, deletionsThreshold) 233 defer func() { 234 if mf != nil { 235 mf.close() 236 } 237 }() 238 require.NoError(t, err) 239 require.Equal(t, 0, m.Creations) 240 require.Equal(t, 0, m.Deletions) 241 head := &protos.HeadInfo{ 242 Version: 1, 243 LogID: 1, 244 LogOffset: 1, 245 } 246 err = mf.addChanges([]*protos.ManifestChange{ 247 newCreateChange(0, 0), 248 }, head) 249 require.NoError(t, err) 250 require.NotNil(t, mf.manifest.Head) 251 252 for i := uint64(0); i < uint64(deletionsThreshold*3); i++ { 253 ch := []*protos.ManifestChange{ 254 newCreateChange(i+1, 0), 255 newDeleteChange(i), 256 } 257 // Only add head for some change set to make sure head is not overwritten to nil. 258 if i < uint64(deletionsThreshold) { 259 head = &protos.HeadInfo{ 260 Version: i, 261 LogID: uint32(i), 262 LogOffset: uint32(i), 263 } 264 err := mf.addChanges(ch, head) 265 require.NoError(t, err) 266 } else { 267 err := mf.addChanges(ch, nil) 268 require.NoError(t, err) 269 } 270 } 271 err = mf.close() 272 require.NoError(t, err) 273 mf = nil 274 mf, m, err = helpOpenOrCreateManifestFile(dir, false, deletionsThreshold) 275 require.NoError(t, err) 276 require.Equal(t, map[uint64]tableManifest{ 277 uint64(deletionsThreshold * 3): {Level: 0}, 278 }, m.Tables) 279 require.NotNil(t, m.Head) 280 require.Equal(t, *m.Head, *head) 281 }