github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/value_test.go (about) 1 /* 2 * Copyright 2017 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package badger 18 19 import ( 20 "bufio" 21 "bytes" 22 "io" 23 "io/ioutil" 24 "os" 25 "testing" 26 27 "github.com/pingcap/badger/y" 28 "github.com/stretchr/testify/require" 29 ) 30 31 func TestValueBasic(t *testing.T) { 32 dir, err := ioutil.TempDir("", "badger") 33 y.Check(err) 34 defer os.RemoveAll(dir) 35 36 kv, _ := Open(getTestOptions(dir)) 37 defer kv.Close() 38 log := &kv.vlog 39 40 // Use value big enough that the value log writes them even if SyncWrites is false. 41 const val1 = "sampleval012345678901234567890123" 42 const val2 = "samplevalb012345678901234567890123" 43 require.True(t, len(val1) >= kv.opt.ValueThreshold) 44 e := &Entry{ 45 Key: y.KeyWithTs([]byte("samplekey"), 100), 46 Value: []byte(val1), 47 meta: bitTxn, 48 } 49 e2 := &Entry{ 50 Key: y.KeyWithTs([]byte("samplekeyb"), 100), 51 Value: []byte(val2), 52 meta: bitTxn, 53 } 54 eFin := &Entry{ 55 Key: y.KeyWithTs(txnKey, 100), 56 meta: bitFinTxn, 57 } 58 offset := log.getMaxPtr() 59 b := new(request) 60 b.Entries = []*Entry{e, e2, eFin} 61 err = log.write([]*request{b}) 62 newOffset := log.getMaxPtr() 63 require.True(t, newOffset > offset) 64 require.Nil(t, err) 65 expectedEntries := []Entry{*e, *e2} 66 var i int 67 err = kv.IterateVLog(offset, func(e Entry) { 68 expectedEntry := expectedEntries[i] 69 expectedEntry.offset = e.offset 70 expectedEntry.logOffset = logOffset{} 71 require.Equal(t, expectedEntry, e) 72 i++ 73 }) 74 require.Nil(t, err) 75 require.Equal(t, 2, i) 76 } 77 78 func TestValueBasicManaged(t *testing.T) { 79 dir, err := ioutil.TempDir("", "badger") 80 y.Check(err) 81 defer os.RemoveAll(dir) 82 83 testOpt := getTestOptions(dir) 84 testOpt.ManagedTxns = true 85 kv, _ := Open(testOpt) 86 defer kv.Close() 87 log := &kv.vlog 88 89 // Use value big enough that the value log writes them even if SyncWrites is false. 90 const val1 = "sampleval012345678901234567890123" 91 const val2 = "samplevalb012345678901234567890123" 92 require.True(t, len(val1) >= kv.opt.ValueThreshold) 93 e := &Entry{ 94 Key: y.KeyWithTs([]byte("samplekey"), 100), 95 Value: []byte(val1), 96 meta: bitTxn, 97 } 98 e2 := &Entry{ 99 Key: y.KeyWithTs([]byte("samplekeyb"), 100), 100 Value: []byte(val2), 101 meta: bitTxn, 102 } 103 eFin := &Entry{ 104 Key: y.KeyWithTs(txnKey, 0), 105 meta: bitFinTxn, 106 } 107 e3 := &Entry{ 108 Key: y.KeyWithTs([]byte("samplekey"), 101), 109 Value: []byte(val1), 110 meta: bitTxn, 111 } 112 e4 := &Entry{ 113 Key: y.KeyWithTs([]byte("samplekeyb"), 101), 114 Value: []byte(val2), 115 meta: bitTxn, 116 } 117 eFin2 := &Entry{ 118 Key: y.KeyWithTs(txnKey, 0), 119 meta: bitFinTxn, 120 } 121 offset := log.getMaxPtr() 122 err = log.write([]*request{{ 123 Entries: []*Entry{e, e2, eFin}, 124 }}) 125 require.Nil(t, err) 126 err = log.write([]*request{{ 127 Entries: []*Entry{e3, e4, eFin2}, 128 }}) 129 newOffset := log.getMaxPtr() 130 require.True(t, newOffset > offset) 131 expectedEntries := []Entry{*e, *e2, *e3, *e4} 132 var i int 133 err = kv.IterateVLog(offset, func(e Entry) { 134 expectedEntry := expectedEntries[i] 135 expectedEntry.offset = e.offset 136 expectedEntry.logOffset = logOffset{} 137 require.Equal(t, expectedEntry, e) 138 i++ 139 }) 140 require.Nil(t, err) 141 require.Equal(t, 4, i) 142 } 143 144 func TestChecksums(t *testing.T) { 145 dir, err := ioutil.TempDir("", "badger") 146 require.NoError(t, err) 147 defer os.RemoveAll(dir) 148 149 // Set up SST with K1=V1 150 opts := getTestOptions(dir) 151 opts.Truncate = true 152 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb 153 kv, err := Open(opts) 154 require.NoError(t, err) 155 require.NoError(t, kv.Close()) 156 157 var ( 158 k0 = []byte("k0") 159 k1 = []byte("k1") 160 k2 = []byte("k2") 161 k3 = []byte("k3") 162 v0 = []byte("value0-012345678901234567890123012345678901234567890123") 163 v1 = []byte("value1-012345678901234567890123012345678901234567890123") 164 v2 = []byte("value2-012345678901234567890123012345678901234567890123") 165 v3 = []byte("value3-012345678901234567890123012345678901234567890123") 166 ) 167 // Make sure the value log would actually store the item 168 require.True(t, len(v0) >= kv.opt.ValueThreshold) 169 170 // Use a vlog with K0=V0 and a (corrupted) second transaction(k1,k2) 171 buf := createVlog(t, []*Entry{ 172 {Key: y.KeyWithTs(k0, 0), Value: y.Copy(v0)}, 173 {Key: y.KeyWithTs(k1, 0), Value: y.Copy(v1)}, 174 {Key: y.KeyWithTs(k2, 0), Value: y.Copy(v2)}, 175 }) 176 buf[len(buf)-1]++ // Corrupt last byte 177 require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) 178 179 // K1 should exist, but K2 shouldn't. 180 kv, err = Open(opts) 181 require.NoError(t, err) 182 183 require.NoError(t, kv.View(func(txn *Txn) error { 184 item, err := txn.Get(k0) 185 require.NoError(t, err) 186 require.Equal(t, getItemValue(t, item), v0) 187 188 _, err = txn.Get(k1) 189 require.Equal(t, ErrKeyNotFound, err) 190 191 _, err = txn.Get(k2) 192 require.Equal(t, ErrKeyNotFound, err) 193 return nil 194 })) 195 196 // Write K3 at the end of the vlog. 197 txnSet(t, kv, k3, y.Copy(v3), 0) 198 require.NoError(t, kv.Close()) 199 200 // The vlog should contain K0 and K3 (K1 and k2 was lost when Badger started up 201 // last due to checksum failure). 202 kv, err = Open(opts) 203 require.NoError(t, err) 204 205 { 206 txn := kv.NewTransaction(false) 207 208 iter := txn.NewIterator(DefaultIteratorOptions) 209 iter.Seek(k0) 210 require.True(t, iter.Valid()) 211 it := iter.Item() 212 require.Equal(t, it.Key(), k0) 213 require.Equal(t, getItemValue(t, it), v0) 214 iter.Next() 215 require.True(t, iter.Valid()) 216 it = iter.Item() 217 require.Equal(t, it.Key(), k3) 218 require.Equal(t, getItemValue(t, it), v3) 219 220 iter.Close() 221 txn.Discard() 222 } 223 224 require.NoError(t, kv.Close()) 225 } 226 227 func TestPartialAppendToValueLog(t *testing.T) { 228 dir, err := ioutil.TempDir("", "badger") 229 require.NoError(t, err) 230 defer os.RemoveAll(dir) 231 232 // Create skeleton files. 233 opts := getTestOptions(dir) 234 opts.Truncate = true 235 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb 236 kv, err := Open(opts) 237 require.NoError(t, err) 238 require.NoError(t, kv.Close()) 239 240 var ( 241 k0 = []byte("k0") 242 k1 = []byte("k1") 243 k2 = []byte("k2") 244 k3 = []byte("k3") 245 v0 = []byte("value0-01234567890123456789012012345678901234567890123") 246 v1 = []byte("value1-01234567890123456789012012345678901234567890123") 247 v2 = []byte("value2-01234567890123456789012012345678901234567890123") 248 v3 = []byte("value3-01234567890123456789012012345678901234567890123") 249 ) 250 // Values need to be long enough to actually get written to value log. 251 require.True(t, len(v3) >= kv.opt.ValueThreshold) 252 253 // Create truncated vlog to simulate a partial append. 254 // k0 - single transaction, k1 and k2 in another transaction 255 buf := createVlog(t, []*Entry{ 256 {Key: y.KeyWithTs(k0, 0), Value: y.Copy(v0)}, 257 {Key: y.KeyWithTs(k1, 0), Value: y.Copy(v1)}, 258 {Key: y.KeyWithTs(k2, 0), Value: y.Copy(v2)}, 259 }) 260 buf = buf[:len(buf)-6] 261 require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) 262 263 // Badger should now start up 264 kv, err = Open(opts) 265 require.NoError(t, err) 266 267 require.NoError(t, kv.View(func(txn *Txn) error { 268 item, err := txn.Get(k0) 269 require.NoError(t, err) 270 require.Equal(t, v0, getItemValue(t, item)) 271 272 _, err = txn.Get(k1) 273 require.Equal(t, ErrKeyNotFound, err) 274 _, err = txn.Get(k2) 275 require.Equal(t, ErrKeyNotFound, err) 276 return nil 277 })) 278 279 // When K3 is set, it should be persisted after a restart. 280 txnSet(t, kv, k3, v3, 0) 281 require.NoError(t, kv.Close()) 282 kv, err = Open(getTestOptions(dir)) 283 require.NoError(t, err) 284 checkKeys(t, kv, [][]byte{k3}) 285 286 // Replay value log from beginning, badger head is past k2. 287 kv.vlog.Replay(logOffset{fid: 0}, replayFunction(kv)) 288 require.NoError(t, kv.Close()) 289 } 290 291 func TestReadOnlyOpenWithPartialAppendToValueLog(t *testing.T) { 292 dir, err := ioutil.TempDir("", "badger") 293 require.NoError(t, err) 294 defer os.RemoveAll(dir) 295 296 // Create skeleton files. 297 opts := getTestOptions(dir) 298 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb 299 kv, err := Open(opts) 300 require.NoError(t, err) 301 require.NoError(t, kv.Close()) 302 303 var ( 304 k0 = []byte("k0") 305 k1 = []byte("k1") 306 k2 = []byte("k2") 307 v0 = []byte("value0-012345678901234567890123") 308 v1 = []byte("value1-012345678901234567890123") 309 v2 = []byte("value2-012345678901234567890123") 310 ) 311 312 // Create truncated vlog to simulate a partial append. 313 // k0 - single transaction, k1 and k2 in another transaction 314 buf := createVlog(t, []*Entry{ 315 {Key: y.KeyWithTs(k0, 0), Value: v0}, 316 {Key: y.KeyWithTs(k1, 0), Value: v1}, 317 {Key: y.KeyWithTs(k2, 0), Value: v2}, 318 }) 319 buf = buf[:len(buf)-6] 320 require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) 321 322 opts.ReadOnly = true 323 // Badger should fail a read-only open with values to replay 324 kv, err = Open(opts) 325 require.Error(t, err) 326 require.Regexp(t, "Database was not properly closed, cannot open read-only|Read-only mode is not supported on Windows", err.Error()) 327 } 328 329 func createVlog(t *testing.T, entries []*Entry) []byte { 330 dir, err := ioutil.TempDir("", "badger") 331 require.NoError(t, err) 332 defer os.RemoveAll(dir) 333 334 opts := getTestOptions(dir) 335 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb 336 kv, err := Open(opts) 337 require.NoError(t, err) 338 txnSet(t, kv, entries[0].Key.UserKey, entries[0].Value, entries[0].meta) 339 entries = entries[1:] 340 txn := kv.NewTransaction(true) 341 for _, entry := range entries { 342 require.NoError(t, txn.SetWithMeta(entry.Key.UserKey, entry.Value, entry.meta)) 343 } 344 require.NoError(t, txn.Commit()) 345 require.NoError(t, kv.Close()) 346 347 filename := vlogFilePath(dir, 0) 348 buf, err := ioutil.ReadFile(filename) 349 require.NoError(t, err) 350 351 read, reader := &safeRead{}, bufio.NewReader(bytes.NewReader(buf)) 352 var offset int 353 for { 354 e, err := read.Entry(reader) 355 if err == io.EOF { 356 break 357 } 358 require.NoError(t, err) 359 offset += headerBufSize + len(e.Key.UserKey) + len(e.Value) + 4 360 } 361 return buf[:offset] 362 } 363 364 func checkKeys(t *testing.T, kv *DB, keys [][]byte) { 365 i := 0 366 txn := kv.NewTransaction(false) 367 iter := txn.NewIterator(IteratorOptions{}) 368 defer iter.Close() 369 for iter.Seek(keys[0]); iter.Valid(); iter.Next() { 370 require.Equal(t, iter.Item().Key(), keys[i]) 371 i++ 372 } 373 require.Equal(t, i, len(keys)) 374 }