github.com/palisadeinc/bor@v0.0.0-20230615125219-ab7196213d15/core/rawdb/freezer_test.go (about) 1 // Copyright 2021 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "io/ioutil" 24 "math/big" 25 "math/rand" 26 "os" 27 "path" 28 "sync" 29 "testing" 30 31 "github.com/ethereum/go-ethereum/ethdb" 32 "github.com/ethereum/go-ethereum/rlp" 33 "github.com/stretchr/testify/require" 34 ) 35 36 var freezerTestTableDef = map[string]bool{"test": true} 37 38 func TestFreezerModify(t *testing.T) { 39 t.Parallel() 40 41 // Create test data. 42 var valuesRaw [][]byte 43 var valuesRLP []*big.Int 44 for x := 0; x < 100; x++ { 45 v := getChunk(256, x) 46 valuesRaw = append(valuesRaw, v) 47 iv := big.NewInt(int64(x)) 48 iv = iv.Exp(iv, iv, nil) 49 valuesRLP = append(valuesRLP, iv) 50 } 51 52 tables := map[string]bool{"raw": true, "rlp": false} 53 f, dir := newFreezerForTesting(t, tables) 54 defer os.RemoveAll(dir) 55 defer f.Close() 56 57 // Commit test data. 58 _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { 59 for i := range valuesRaw { 60 if err := op.AppendRaw("raw", uint64(i), valuesRaw[i]); err != nil { 61 return err 62 } 63 if err := op.Append("rlp", uint64(i), valuesRLP[i]); err != nil { 64 return err 65 } 66 } 67 return nil 68 }) 69 if err != nil { 70 t.Fatal("ModifyAncients failed:", err) 71 } 72 73 // Dump indexes. 74 for _, table := range f.tables { 75 t.Log(table.name, "index:", table.dumpIndexString(0, int64(len(valuesRaw)))) 76 } 77 78 // Read back test data. 79 checkAncientCount(t, f, "raw", uint64(len(valuesRaw))) 80 checkAncientCount(t, f, "rlp", uint64(len(valuesRLP))) 81 for i := range valuesRaw { 82 v, _ := f.Ancient("raw", uint64(i)) 83 if !bytes.Equal(v, valuesRaw[i]) { 84 t.Fatalf("wrong raw value at %d: %x", i, v) 85 } 86 ivEnc, _ := f.Ancient("rlp", uint64(i)) 87 want, _ := rlp.EncodeToBytes(valuesRLP[i]) 88 if !bytes.Equal(ivEnc, want) { 89 t.Fatalf("wrong RLP value at %d: %x", i, ivEnc) 90 } 91 } 92 } 93 94 // This checks that ModifyAncients rolls back freezer updates 95 // when the function passed to it returns an error. 96 func TestFreezerModifyRollback(t *testing.T) { 97 t.Parallel() 98 99 f, dir := newFreezerForTesting(t, freezerTestTableDef) 100 defer os.RemoveAll(dir) 101 102 theError := errors.New("oops") 103 _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { 104 // Append three items. This creates two files immediately, 105 // because the table size limit of the test freezer is 2048. 106 require.NoError(t, op.AppendRaw("test", 0, make([]byte, 2048))) 107 require.NoError(t, op.AppendRaw("test", 1, make([]byte, 2048))) 108 require.NoError(t, op.AppendRaw("test", 2, make([]byte, 2048))) 109 return theError 110 }) 111 if err != theError { 112 t.Errorf("ModifyAncients returned wrong error %q", err) 113 } 114 checkAncientCount(t, f, "test", 0) 115 f.Close() 116 117 // Reopen and check that the rolled-back data doesn't reappear. 118 tables := map[string]bool{"test": true} 119 f2, err := newFreezer(dir, "", false, 2049, tables) 120 if err != nil { 121 t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err) 122 } 123 defer f2.Close() 124 checkAncientCount(t, f2, "test", 0) 125 } 126 127 // This test runs ModifyAncients and Ancient concurrently with each other. 128 func TestFreezerConcurrentModifyRetrieve(t *testing.T) { 129 t.Parallel() 130 131 f, dir := newFreezerForTesting(t, freezerTestTableDef) 132 defer os.RemoveAll(dir) 133 defer f.Close() 134 135 var ( 136 numReaders = 5 137 writeBatchSize = uint64(50) 138 written = make(chan uint64, numReaders*6) 139 wg sync.WaitGroup 140 ) 141 wg.Add(numReaders + 1) 142 143 // Launch the writer. It appends 10000 items in batches. 144 go func() { 145 defer wg.Done() 146 defer close(written) 147 for item := uint64(0); item < 10000; item += writeBatchSize { 148 _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { 149 for i := uint64(0); i < writeBatchSize; i++ { 150 item := item + i 151 value := getChunk(32, int(item)) 152 if err := op.AppendRaw("test", item, value); err != nil { 153 return err 154 } 155 } 156 return nil 157 }) 158 if err != nil { 159 panic(err) 160 } 161 for i := 0; i < numReaders; i++ { 162 written <- item + writeBatchSize 163 } 164 } 165 }() 166 167 // Launch the readers. They read random items from the freezer up to the 168 // current frozen item count. 169 for i := 0; i < numReaders; i++ { 170 go func() { 171 defer wg.Done() 172 for frozen := range written { 173 for rc := 0; rc < 80; rc++ { 174 num := uint64(rand.Intn(int(frozen))) 175 value, err := f.Ancient("test", num) 176 if err != nil { 177 panic(fmt.Errorf("error reading %d (frozen %d): %v", num, frozen, err)) 178 } 179 if !bytes.Equal(value, getChunk(32, int(num))) { 180 panic(fmt.Errorf("wrong value at %d", num)) 181 } 182 } 183 } 184 }() 185 } 186 187 wg.Wait() 188 } 189 190 // This test runs ModifyAncients and TruncateHead concurrently with each other. 191 func TestFreezerConcurrentModifyTruncate(t *testing.T) { 192 f, dir := newFreezerForTesting(t, freezerTestTableDef) 193 defer os.RemoveAll(dir) 194 defer f.Close() 195 196 var item = make([]byte, 256) 197 198 for i := 0; i < 1000; i++ { 199 // First reset and write 100 items. 200 if err := f.TruncateHead(0); err != nil { 201 t.Fatal("truncate failed:", err) 202 } 203 _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { 204 for i := uint64(0); i < 100; i++ { 205 if err := op.AppendRaw("test", i, item); err != nil { 206 return err 207 } 208 } 209 return nil 210 }) 211 if err != nil { 212 t.Fatal("modify failed:", err) 213 } 214 checkAncientCount(t, f, "test", 100) 215 216 // Now append 100 more items and truncate concurrently. 217 var ( 218 wg sync.WaitGroup 219 truncateErr error 220 modifyErr error 221 ) 222 wg.Add(3) 223 go func() { 224 _, modifyErr = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { 225 for i := uint64(100); i < 200; i++ { 226 if err := op.AppendRaw("test", i, item); err != nil { 227 return err 228 } 229 } 230 return nil 231 }) 232 wg.Done() 233 }() 234 go func() { 235 truncateErr = f.TruncateHead(10) 236 wg.Done() 237 }() 238 go func() { 239 f.AncientSize("test") 240 wg.Done() 241 }() 242 wg.Wait() 243 244 // Now check the outcome. If the truncate operation went through first, the append 245 // fails, otherwise it succeeds. In either case, the freezer should be positioned 246 // at 10 after both operations are done. 247 if truncateErr != nil { 248 t.Fatal("concurrent truncate failed:", err) 249 } 250 if !(errors.Is(modifyErr, nil) || errors.Is(modifyErr, errOutOrderInsertion)) { 251 t.Fatal("wrong error from concurrent modify:", modifyErr) 252 } 253 checkAncientCount(t, f, "test", 10) 254 } 255 } 256 257 func TestFreezerReadonlyValidate(t *testing.T) { 258 tables := map[string]bool{"a": true, "b": true} 259 dir, err := ioutil.TempDir("", "freezer") 260 if err != nil { 261 t.Fatal(err) 262 } 263 defer os.RemoveAll(dir) 264 // Open non-readonly freezer and fill individual tables 265 // with different amount of data. 266 f, err := newFreezer(dir, "", false, 2049, tables) 267 if err != nil { 268 t.Fatal("can't open freezer", err) 269 } 270 var item = make([]byte, 1024) 271 aBatch := f.tables["a"].newBatch() 272 require.NoError(t, aBatch.AppendRaw(0, item)) 273 require.NoError(t, aBatch.AppendRaw(1, item)) 274 require.NoError(t, aBatch.AppendRaw(2, item)) 275 require.NoError(t, aBatch.commit()) 276 bBatch := f.tables["b"].newBatch() 277 require.NoError(t, bBatch.AppendRaw(0, item)) 278 require.NoError(t, bBatch.commit()) 279 if f.tables["a"].items != 3 { 280 t.Fatalf("unexpected number of items in table") 281 } 282 if f.tables["b"].items != 1 { 283 t.Fatalf("unexpected number of items in table") 284 } 285 require.NoError(t, f.Close()) 286 287 // Re-openening as readonly should fail when validating 288 // table lengths. 289 f, err = newFreezer(dir, "", true, 2049, tables) 290 if err == nil { 291 t.Fatal("readonly freezer should fail with differing table lengths") 292 } 293 } 294 295 func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) { 296 t.Helper() 297 298 dir, err := ioutil.TempDir("", "freezer") 299 if err != nil { 300 t.Fatal(err) 301 } 302 // note: using low max table size here to ensure the tests actually 303 // switch between multiple files. 304 f, err := newFreezer(dir, "", false, 2049, tables) 305 if err != nil { 306 t.Fatal("can't open freezer", err) 307 } 308 return f, dir 309 } 310 311 // checkAncientCount verifies that the freezer contains n items. 312 func checkAncientCount(t *testing.T, f *freezer, kind string, n uint64) { 313 t.Helper() 314 315 if frozen, _ := f.Ancients(); frozen != n { 316 t.Fatalf("Ancients() returned %d, want %d", frozen, n) 317 } 318 319 // Check at index n-1. 320 if n > 0 { 321 index := n - 1 322 if ok, _ := f.HasAncient(kind, index); !ok { 323 t.Errorf("HasAncient(%q, %d) returned false unexpectedly", kind, index) 324 } 325 if _, err := f.Ancient(kind, index); err != nil { 326 t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) 327 } 328 } 329 330 // Check at index n. 331 index := n 332 if ok, _ := f.HasAncient(kind, index); ok { 333 t.Errorf("HasAncient(%q, %d) returned true unexpectedly", kind, index) 334 } 335 if _, err := f.Ancient(kind, index); err == nil { 336 t.Errorf("Ancient(%q, %d) didn't return expected error", kind, index) 337 } else if err != errOutOfBounds { 338 t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) 339 } 340 } 341 342 func TestRenameWindows(t *testing.T) { 343 var ( 344 fname = "file.bin" 345 fname2 = "file2.bin" 346 data = []byte{1, 2, 3, 4} 347 data2 = []byte{2, 3, 4, 5} 348 data3 = []byte{3, 5, 6, 7} 349 dataLen = 4 350 ) 351 352 // Create 2 temp dirs 353 dir1, err := os.MkdirTemp("", "rename-test") 354 if err != nil { 355 t.Fatal(err) 356 } 357 defer os.Remove(dir1) 358 dir2, err := os.MkdirTemp("", "rename-test") 359 if err != nil { 360 t.Fatal(err) 361 } 362 defer os.Remove(dir2) 363 364 // Create file in dir1 and fill with data 365 f, err := os.Create(path.Join(dir1, fname)) 366 if err != nil { 367 t.Fatal(err) 368 } 369 f2, err := os.Create(path.Join(dir1, fname2)) 370 if err != nil { 371 t.Fatal(err) 372 } 373 f3, err := os.Create(path.Join(dir2, fname2)) 374 if err != nil { 375 t.Fatal(err) 376 } 377 if _, err := f.Write(data); err != nil { 378 t.Fatal(err) 379 } 380 if _, err := f2.Write(data2); err != nil { 381 t.Fatal(err) 382 } 383 if _, err := f3.Write(data3); err != nil { 384 t.Fatal(err) 385 } 386 if err := f.Close(); err != nil { 387 t.Fatal(err) 388 } 389 if err := f2.Close(); err != nil { 390 t.Fatal(err) 391 } 392 if err := f3.Close(); err != nil { 393 t.Fatal(err) 394 } 395 if err := os.Rename(f.Name(), path.Join(dir2, fname)); err != nil { 396 t.Fatal(err) 397 } 398 if err := os.Rename(f2.Name(), path.Join(dir2, fname2)); err != nil { 399 t.Fatal(err) 400 } 401 402 // Check file contents 403 f, err = os.Open(path.Join(dir2, fname)) 404 if err != nil { 405 t.Fatal(err) 406 } 407 defer f.Close() 408 defer os.Remove(f.Name()) 409 buf := make([]byte, dataLen) 410 if _, err := f.Read(buf); err != nil { 411 t.Fatal(err) 412 } 413 if !bytes.Equal(buf, data) { 414 t.Errorf("unexpected file contents. Got %v\n", buf) 415 } 416 417 f, err = os.Open(path.Join(dir2, fname2)) 418 if err != nil { 419 t.Fatal(err) 420 } 421 defer f.Close() 422 defer os.Remove(f.Name()) 423 if _, err := f.Read(buf); err != nil { 424 t.Fatal(err) 425 } 426 if !bytes.Equal(buf, data2) { 427 t.Errorf("unexpected file contents. Got %v\n", buf) 428 } 429 }