github.com/DFWallet/tendermint-cosmos@v0.0.2/statesync/chunks_test.go (about) 1 package statesync 2 3 import ( 4 "io/ioutil" 5 "os" 6 "testing" 7 8 "github.com/stretchr/testify/assert" 9 "github.com/stretchr/testify/require" 10 11 "github.com/DFWallet/tendermint-cosmos/p2p" 12 ) 13 14 func setupChunkQueue(t *testing.T) (*chunkQueue, func()) { 15 snapshot := &snapshot{ 16 Height: 3, 17 Format: 1, 18 Chunks: 5, 19 Hash: []byte{7}, 20 Metadata: nil, 21 } 22 queue, err := newChunkQueue(snapshot, "") 23 require.NoError(t, err) 24 teardown := func() { 25 err := queue.Close() 26 require.NoError(t, err) 27 } 28 return queue, teardown 29 } 30 31 func TestNewChunkQueue_TempDir(t *testing.T) { 32 snapshot := &snapshot{ 33 Height: 3, 34 Format: 1, 35 Chunks: 5, 36 Hash: []byte{7}, 37 Metadata: nil, 38 } 39 dir, err := ioutil.TempDir("", "newchunkqueue") 40 require.NoError(t, err) 41 defer os.RemoveAll(dir) 42 queue, err := newChunkQueue(snapshot, dir) 43 require.NoError(t, err) 44 45 files, err := ioutil.ReadDir(dir) 46 require.NoError(t, err) 47 assert.Len(t, files, 1) 48 49 err = queue.Close() 50 require.NoError(t, err) 51 52 files, err = ioutil.ReadDir(dir) 53 require.NoError(t, err) 54 assert.Len(t, files, 0) 55 } 56 57 func TestChunkQueue(t *testing.T) { 58 queue, teardown := setupChunkQueue(t) 59 defer teardown() 60 61 // Adding the first chunk should be fine 62 added, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}) 63 require.NoError(t, err) 64 assert.True(t, added) 65 66 // Adding the last chunk should also be fine 67 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}}) 68 require.NoError(t, err) 69 assert.True(t, added) 70 71 // Adding the first or last chunks again should return false 72 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}) 73 require.NoError(t, err) 74 assert.False(t, added) 75 76 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}}) 77 require.NoError(t, err) 78 assert.False(t, added) 79 80 // Adding the remaining chunks in reverse should be fine 81 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}}) 82 require.NoError(t, err) 83 assert.True(t, added) 84 85 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}}) 86 require.NoError(t, err) 87 assert.True(t, added) 88 89 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}}) 90 require.NoError(t, err) 91 assert.True(t, added) 92 93 // At this point, we should be able to retrieve them all via Next 94 for i := 0; i < 5; i++ { 95 c, err := queue.Next() 96 require.NoError(t, err) 97 assert.Equal(t, &chunk{Height: 3, Format: 1, Index: uint32(i), Chunk: []byte{3, 1, byte(i)}}, c) 98 } 99 _, err = queue.Next() 100 require.Error(t, err) 101 assert.Equal(t, errDone, err) 102 103 // It should still be possible to try to add chunks (which will be ignored) 104 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}) 105 require.NoError(t, err) 106 assert.False(t, added) 107 108 // After closing the queue it will also return false 109 err = queue.Close() 110 require.NoError(t, err) 111 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}) 112 require.NoError(t, err) 113 assert.False(t, added) 114 115 // Closing the queue again should also be fine 116 err = queue.Close() 117 require.NoError(t, err) 118 } 119 120 func TestChunkQueue_Add_ChunkErrors(t *testing.T) { 121 testcases := map[string]struct { 122 chunk *chunk 123 }{ 124 "nil chunk": {nil}, 125 "nil body": {&chunk{Height: 3, Format: 1, Index: 0, Chunk: nil}}, 126 "wrong height": {&chunk{Height: 9, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}}, 127 "wrong format": {&chunk{Height: 3, Format: 9, Index: 0, Chunk: []byte{3, 1, 0}}}, 128 "invalid index": {&chunk{Height: 3, Format: 1, Index: 5, Chunk: []byte{3, 1, 0}}}, 129 } 130 for name, tc := range testcases { 131 tc := tc 132 t.Run(name, func(t *testing.T) { 133 queue, teardown := setupChunkQueue(t) 134 defer teardown() 135 _, err := queue.Add(tc.chunk) 136 require.Error(t, err) 137 }) 138 } 139 } 140 141 func TestChunkQueue_Allocate(t *testing.T) { 142 queue, teardown := setupChunkQueue(t) 143 defer teardown() 144 145 for i := uint32(0); i < queue.Size(); i++ { 146 index, err := queue.Allocate() 147 require.NoError(t, err) 148 assert.EqualValues(t, i, index) 149 } 150 151 _, err := queue.Allocate() 152 require.Error(t, err) 153 assert.Equal(t, errDone, err) 154 155 for i := uint32(0); i < queue.Size(); i++ { 156 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}}) 157 require.NoError(t, err) 158 } 159 160 // After all chunks have been allocated and retrieved, discarding a chunk will reallocate it. 161 err = queue.Discard(2) 162 require.NoError(t, err) 163 164 index, err := queue.Allocate() 165 require.NoError(t, err) 166 assert.EqualValues(t, 2, index) 167 _, err = queue.Allocate() 168 require.Error(t, err) 169 assert.Equal(t, errDone, err) 170 171 // Discarding a chunk the closing the queue will return errDone. 172 err = queue.Discard(2) 173 require.NoError(t, err) 174 err = queue.Close() 175 require.NoError(t, err) 176 _, err = queue.Allocate() 177 require.Error(t, err) 178 assert.Equal(t, errDone, err) 179 } 180 181 func TestChunkQueue_Discard(t *testing.T) { 182 queue, teardown := setupChunkQueue(t) 183 defer teardown() 184 185 // Add a few chunks to the queue and fetch a couple 186 _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{byte(0)}}) 187 require.NoError(t, err) 188 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{byte(1)}}) 189 require.NoError(t, err) 190 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{byte(2)}}) 191 require.NoError(t, err) 192 193 c, err := queue.Next() 194 require.NoError(t, err) 195 assert.EqualValues(t, 0, c.Index) 196 c, err = queue.Next() 197 require.NoError(t, err) 198 assert.EqualValues(t, 1, c.Index) 199 200 // Discarding the first chunk and re-adding it should cause it to be returned 201 // immediately by Next(), before procceeding with chunk 2 202 err = queue.Discard(0) 203 require.NoError(t, err) 204 added, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{byte(0)}}) 205 require.NoError(t, err) 206 assert.True(t, added) 207 c, err = queue.Next() 208 require.NoError(t, err) 209 assert.EqualValues(t, 0, c.Index) 210 c, err = queue.Next() 211 require.NoError(t, err) 212 assert.EqualValues(t, 2, c.Index) 213 214 // Discard then allocate, add and fetch all chunks 215 for i := uint32(0); i < queue.Size(); i++ { 216 err := queue.Discard(i) 217 require.NoError(t, err) 218 } 219 for i := uint32(0); i < queue.Size(); i++ { 220 _, err := queue.Allocate() 221 require.NoError(t, err) 222 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}}) 223 require.NoError(t, err) 224 c, err = queue.Next() 225 require.NoError(t, err) 226 assert.EqualValues(t, i, c.Index) 227 } 228 229 // Discarding a non-existent chunk does nothing. 230 err = queue.Discard(99) 231 require.NoError(t, err) 232 233 // When discard a couple of chunks, we should be able to allocate, add, and fetch them again. 234 err = queue.Discard(3) 235 require.NoError(t, err) 236 err = queue.Discard(1) 237 require.NoError(t, err) 238 239 index, err := queue.Allocate() 240 require.NoError(t, err) 241 assert.EqualValues(t, 1, index) 242 index, err = queue.Allocate() 243 require.NoError(t, err) 244 assert.EqualValues(t, 3, index) 245 246 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3}}) 247 require.NoError(t, err) 248 assert.True(t, added) 249 added, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{1}}) 250 require.NoError(t, err) 251 assert.True(t, added) 252 253 chunk, err := queue.Next() 254 require.NoError(t, err) 255 assert.EqualValues(t, 1, chunk.Index) 256 257 chunk, err = queue.Next() 258 require.NoError(t, err) 259 assert.EqualValues(t, 3, chunk.Index) 260 261 _, err = queue.Next() 262 require.Error(t, err) 263 assert.Equal(t, errDone, err) 264 265 // After closing the queue, discarding does nothing 266 err = queue.Close() 267 require.NoError(t, err) 268 err = queue.Discard(2) 269 require.NoError(t, err) 270 } 271 272 func TestChunkQueue_DiscardSender(t *testing.T) { 273 queue, teardown := setupChunkQueue(t) 274 defer teardown() 275 276 // Allocate and add all chunks to the queue 277 senders := []p2p.ID{"a", "b", "c"} 278 for i := uint32(0); i < queue.Size(); i++ { 279 _, err := queue.Allocate() 280 require.NoError(t, err) 281 _, err = queue.Add(&chunk{ 282 Height: 3, 283 Format: 1, 284 Index: i, 285 Chunk: []byte{byte(i)}, 286 Sender: senders[int(i)%len(senders)], 287 }) 288 require.NoError(t, err) 289 } 290 291 // Fetch the first three chunks 292 for i := uint32(0); i < 3; i++ { 293 _, err := queue.Next() 294 require.NoError(t, err) 295 } 296 297 // Discarding an unknown sender should do nothing 298 err := queue.DiscardSender("x") 299 require.NoError(t, err) 300 _, err = queue.Allocate() 301 assert.Equal(t, errDone, err) 302 303 // Discarding sender b should discard chunk 4, but not chunk 1 which has already been 304 // returned. 305 err = queue.DiscardSender("b") 306 require.NoError(t, err) 307 index, err := queue.Allocate() 308 require.NoError(t, err) 309 assert.EqualValues(t, 4, index) 310 _, err = queue.Allocate() 311 assert.Equal(t, errDone, err) 312 } 313 314 func TestChunkQueue_GetSender(t *testing.T) { 315 queue, teardown := setupChunkQueue(t) 316 defer teardown() 317 318 _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{1}, Sender: p2p.ID("a")}) 319 require.NoError(t, err) 320 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{2}, Sender: p2p.ID("b")}) 321 require.NoError(t, err) 322 323 assert.EqualValues(t, "a", queue.GetSender(0)) 324 assert.EqualValues(t, "b", queue.GetSender(1)) 325 assert.EqualValues(t, "", queue.GetSender(2)) 326 327 // After the chunk has been processed, we should still know who the sender was 328 chunk, err := queue.Next() 329 require.NoError(t, err) 330 require.NotNil(t, chunk) 331 require.EqualValues(t, 0, chunk.Index) 332 assert.EqualValues(t, "a", queue.GetSender(0)) 333 } 334 335 func TestChunkQueue_Next(t *testing.T) { 336 queue, teardown := setupChunkQueue(t) 337 defer teardown() 338 339 // Next should block waiting for the next chunks, even when given out of order. 340 chNext := make(chan *chunk, 10) 341 go func() { 342 for { 343 c, err := queue.Next() 344 if err == errDone { 345 close(chNext) 346 break 347 } 348 require.NoError(t, err) 349 chNext <- c 350 } 351 }() 352 353 assert.Empty(t, chNext) 354 _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}, Sender: p2p.ID("b")}) 355 require.NoError(t, err) 356 select { 357 case <-chNext: 358 assert.Fail(t, "channel should be empty") 359 default: 360 } 361 362 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}, Sender: p2p.ID("a")}) 363 require.NoError(t, err) 364 365 assert.Equal(t, 366 &chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}, Sender: p2p.ID("a")}, 367 <-chNext) 368 assert.Equal(t, 369 &chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}, Sender: p2p.ID("b")}, 370 <-chNext) 371 372 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}, Sender: p2p.ID("e")}) 373 require.NoError(t, err) 374 select { 375 case <-chNext: 376 assert.Fail(t, "channel should be empty") 377 default: 378 } 379 380 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}, Sender: p2p.ID("c")}) 381 require.NoError(t, err) 382 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}, Sender: p2p.ID("d")}) 383 require.NoError(t, err) 384 385 assert.Equal(t, 386 &chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}, Sender: p2p.ID("c")}, 387 <-chNext) 388 assert.Equal(t, 389 &chunk{Height: 3, Format: 1, Index: 3, Chunk: []byte{3, 1, 3}, Sender: p2p.ID("d")}, 390 <-chNext) 391 assert.Equal(t, 392 &chunk{Height: 3, Format: 1, Index: 4, Chunk: []byte{3, 1, 4}, Sender: p2p.ID("e")}, 393 <-chNext) 394 395 _, ok := <-chNext 396 assert.False(t, ok, "channel should be closed") 397 398 // Calling next on a finished queue should return done 399 _, err = queue.Next() 400 assert.Equal(t, errDone, err) 401 } 402 403 func TestChunkQueue_Next_Closed(t *testing.T) { 404 queue, teardown := setupChunkQueue(t) 405 defer teardown() 406 407 // Calling Next on a closed queue should return done 408 _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}}) 409 require.NoError(t, err) 410 err = queue.Close() 411 require.NoError(t, err) 412 413 _, err = queue.Next() 414 assert.Equal(t, errDone, err) 415 } 416 417 func TestChunkQueue_Retry(t *testing.T) { 418 queue, teardown := setupChunkQueue(t) 419 defer teardown() 420 421 // Allocate and add all chunks to the queue 422 for i := uint32(0); i < queue.Size(); i++ { 423 _, err := queue.Allocate() 424 require.NoError(t, err) 425 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}}) 426 require.NoError(t, err) 427 _, err = queue.Next() 428 require.NoError(t, err) 429 } 430 431 // Retrying a couple of chunks makes Next() return them, but they are not allocatable 432 queue.Retry(3) 433 queue.Retry(1) 434 435 _, err := queue.Allocate() 436 assert.Equal(t, errDone, err) 437 438 chunk, err := queue.Next() 439 require.NoError(t, err) 440 assert.EqualValues(t, 1, chunk.Index) 441 442 chunk, err = queue.Next() 443 require.NoError(t, err) 444 assert.EqualValues(t, 3, chunk.Index) 445 446 _, err = queue.Next() 447 assert.Equal(t, errDone, err) 448 } 449 450 func TestChunkQueue_RetryAll(t *testing.T) { 451 queue, teardown := setupChunkQueue(t) 452 defer teardown() 453 454 // Allocate and add all chunks to the queue 455 for i := uint32(0); i < queue.Size(); i++ { 456 _, err := queue.Allocate() 457 require.NoError(t, err) 458 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: i, Chunk: []byte{byte(i)}}) 459 require.NoError(t, err) 460 _, err = queue.Next() 461 require.NoError(t, err) 462 } 463 464 _, err := queue.Next() 465 assert.Equal(t, errDone, err) 466 467 queue.RetryAll() 468 469 _, err = queue.Allocate() 470 assert.Equal(t, errDone, err) 471 472 for i := uint32(0); i < queue.Size(); i++ { 473 chunk, err := queue.Next() 474 require.NoError(t, err) 475 assert.EqualValues(t, i, chunk.Index) 476 } 477 478 _, err = queue.Next() 479 assert.Equal(t, errDone, err) 480 } 481 482 func TestChunkQueue_Size(t *testing.T) { 483 queue, teardown := setupChunkQueue(t) 484 defer teardown() 485 486 assert.EqualValues(t, 5, queue.Size()) 487 488 err := queue.Close() 489 require.NoError(t, err) 490 assert.EqualValues(t, 0, queue.Size()) 491 } 492 493 func TestChunkQueue_WaitFor(t *testing.T) { 494 queue, teardown := setupChunkQueue(t) 495 defer teardown() 496 497 waitFor1 := queue.WaitFor(1) 498 waitFor4 := queue.WaitFor(4) 499 500 // Adding 0 and 2 should not trigger waiters 501 _, err := queue.Add(&chunk{Height: 3, Format: 1, Index: 0, Chunk: []byte{3, 1, 0}}) 502 require.NoError(t, err) 503 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 2, Chunk: []byte{3, 1, 2}}) 504 require.NoError(t, err) 505 select { 506 case <-waitFor1: 507 require.Fail(t, "WaitFor(1) should not trigger on 0 or 2") 508 case <-waitFor4: 509 require.Fail(t, "WaitFor(4) should not trigger on 0 or 2") 510 default: 511 } 512 513 // Adding 1 should trigger WaitFor(1), but not WaitFor(4). The channel should be closed. 514 _, err = queue.Add(&chunk{Height: 3, Format: 1, Index: 1, Chunk: []byte{3, 1, 1}}) 515 require.NoError(t, err) 516 assert.EqualValues(t, 1, <-waitFor1) 517 _, ok := <-waitFor1 518 assert.False(t, ok) 519 select { 520 case <-waitFor4: 521 require.Fail(t, "WaitFor(4) should not trigger on 0 or 2") 522 default: 523 } 524 525 // Fetch the first chunk. At this point, waiting for either 0 (retrieved from pool) or 1 526 // (queued in pool) should immediately return true. 527 c, err := queue.Next() 528 require.NoError(t, err) 529 assert.EqualValues(t, 0, c.Index) 530 531 w := queue.WaitFor(0) 532 assert.EqualValues(t, 0, <-w) 533 _, ok = <-w 534 assert.False(t, ok) 535 536 w = queue.WaitFor(1) 537 assert.EqualValues(t, 1, <-w) 538 _, ok = <-w 539 assert.False(t, ok) 540 541 // Close the queue. This should cause the waiter for 4 to close, and also cause any future 542 // waiters to get closed channels. 543 err = queue.Close() 544 require.NoError(t, err) 545 _, ok = <-waitFor4 546 assert.False(t, ok) 547 548 w = queue.WaitFor(3) 549 _, ok = <-w 550 assert.False(t, ok) 551 }