github.com/koko1123/flow-go-1@v0.29.6/engine/consensus/sealing/core_test.go (about) 1 package sealing 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/stretchr/testify/mock" 9 "github.com/stretchr/testify/require" 10 "github.com/stretchr/testify/suite" 11 12 "github.com/koko1123/flow-go-1/engine" 13 "github.com/koko1123/flow-go-1/engine/consensus/approvals" 14 "github.com/koko1123/flow-go-1/engine/consensus/approvals/tracker" 15 "github.com/koko1123/flow-go-1/model/chunks" 16 "github.com/koko1123/flow-go-1/model/flow" 17 realmodule "github.com/koko1123/flow-go-1/module" 18 "github.com/koko1123/flow-go-1/module/metrics" 19 module "github.com/koko1123/flow-go-1/module/mock" 20 "github.com/koko1123/flow-go-1/module/trace" 21 "github.com/koko1123/flow-go-1/module/updatable_configs" 22 mockstate "github.com/koko1123/flow-go-1/state/protocol/mock" 23 storage "github.com/koko1123/flow-go-1/storage/mock" 24 "github.com/koko1123/flow-go-1/utils/unittest" 25 ) 26 27 // TestApprovalProcessingCore performs testing of approval processing core 28 // Core is responsible for delegating processing to assignment collectorTree for each separate execution result 29 // Core performs height based checks and decides if approval or incorporated result has to be processed at all 30 // or rejected as outdated or unverifiable. 31 // Core maintains a LRU cache of known approvals that cannot be verified at the moment/ 32 func TestApprovalProcessingCore(t *testing.T) { 33 suite.Run(t, new(ApprovalProcessingCoreTestSuite)) 34 } 35 36 // RequiredApprovalsForSealConstructionTestingValue defines the number of approvals that are 37 // required to construct a seal for testing purposes. Thereby, the default production value 38 // can be set independently without changing test behaviour. 39 const RequiredApprovalsForSealConstructionTestingValue = 1 40 41 type ApprovalProcessingCoreTestSuite struct { 42 approvals.BaseAssignmentCollectorTestSuite 43 44 sealsDB *storage.Seals 45 rootHeader *flow.Header 46 core *Core 47 setter realmodule.SealingConfigsSetter 48 } 49 50 func (s *ApprovalProcessingCoreTestSuite) TearDownTest() { 51 s.BaseAssignmentCollectorTestSuite.TearDownTest() 52 } 53 54 func (s *ApprovalProcessingCoreTestSuite) SetupTest() { 55 s.BaseAssignmentCollectorTestSuite.SetupTest() 56 57 s.sealsDB = &storage.Seals{} 58 59 s.rootHeader = unittest.GenesisFixture().Header 60 params := new(mockstate.Params) 61 s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil)).Maybe() 62 s.State.On("Params").Return(params) 63 params.On("Root").Return( 64 func() *flow.Header { return s.rootHeader }, 65 func() error { return nil }, 66 ) 67 68 metrics := metrics.NewNoopCollector() 69 tracer := trace.NewNoopTracer() 70 71 setter := unittest.NewSealingConfigs(flow.DefaultChunkAssignmentAlpha) 72 var err error 73 s.core, err = NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), s.Headers, s.State, s.sealsDB, s.Assigner, s.SigHasher, s.SealsPL, s.Conduit, setter) 74 require.NoError(s.T(), err) 75 s.setter = setter 76 } 77 78 // TestOnBlockFinalized_RejectOutdatedApprovals tests that approvals will be rejected as outdated 79 // for block that is already sealed 80 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOutdatedApprovals() { 81 approval := unittest.ResultApprovalFixture(unittest.WithApproverID(s.VerID), 82 unittest.WithChunk(s.Chunks[0].Index), 83 unittest.WithBlockID(s.Block.ID())) 84 err := s.core.processApproval(approval) 85 require.NoError(s.T(), err) 86 87 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block)) 88 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 89 90 err = s.core.ProcessFinalizedBlock(s.Block.ID()) 91 require.NoError(s.T(), err) 92 93 err = s.core.processApproval(approval) 94 require.Error(s.T(), err) 95 require.True(s.T(), engine.IsOutdatedInputError(err)) 96 } 97 98 // TestOnBlockFinalized_RejectOutdatedExecutionResult tests that incorporated result will be rejected as outdated 99 // if the block which is targeted by execution result is already sealed. 100 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOutdatedExecutionResult() { 101 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block)) 102 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 103 104 err := s.core.ProcessFinalizedBlock(s.Block.ID()) 105 require.NoError(s.T(), err) 106 107 err = s.core.processIncorporatedResult(s.IncorporatedResult) 108 require.Error(s.T(), err) 109 require.True(s.T(), engine.IsOutdatedInputError(err)) 110 } 111 112 // TestOnBlockFinalized_RejectUnverifiableEntries tests that core will reject both execution results 113 // and approvals for blocks that we have no information about. 114 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectUnverifiableEntries() { 115 s.IncorporatedResult.Result.BlockID = unittest.IdentifierFixture() // replace blockID with random one 116 err := s.core.processIncorporatedResult(s.IncorporatedResult) 117 require.Error(s.T(), err) 118 require.True(s.T(), engine.IsUnverifiableInputError(err)) 119 120 approval := unittest.ResultApprovalFixture(unittest.WithApproverID(s.VerID), 121 unittest.WithChunk(s.Chunks[0].Index)) 122 123 err = s.core.processApproval(approval) 124 require.Error(s.T(), err) 125 require.True(s.T(), engine.IsUnverifiableInputError(err)) 126 } 127 128 // TestOnBlockFinalized_RejectOrphanIncorporatedResults tests that execution results incorporated in orphan blocks 129 // are rejected as outdated in next situation 130 // 131 // A <- B_1 132 // <- B_2 133 // 134 // B_1 is finalized rendering B_2 as orphan, submitting IR[ER[A], B_1] is a success, submitting IR[ER[A], B_2] is an outdated incorporated result 135 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOrphanIncorporatedResults() { 136 blockB1 := unittest.BlockHeaderWithParentFixture(s.Block) 137 blockB2 := unittest.BlockHeaderWithParentFixture(s.Block) 138 139 s.Blocks[blockB1.ID()] = blockB1 140 s.Blocks[blockB2.ID()] = blockB2 141 142 IR1 := unittest.IncorporatedResult.Fixture( 143 unittest.IncorporatedResult.WithIncorporatedBlockID(blockB1.ID()), 144 unittest.IncorporatedResult.WithResult(s.IncorporatedResult.Result)) 145 146 IR2 := unittest.IncorporatedResult.Fixture( 147 unittest.IncorporatedResult.WithIncorporatedBlockID(blockB2.ID()), 148 unittest.IncorporatedResult.WithResult(s.IncorporatedResult.Result)) 149 150 s.MarkFinalized(blockB1) 151 152 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock)) 153 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 154 155 // blockB1 becomes finalized 156 err := s.core.ProcessFinalizedBlock(blockB1.ID()) 157 require.NoError(s.T(), err) 158 159 err = s.core.processIncorporatedResult(IR1) 160 require.NoError(s.T(), err) 161 162 err = s.core.processIncorporatedResult(IR2) 163 require.Error(s.T(), err) 164 require.True(s.T(), engine.IsOutdatedInputError(err)) 165 } 166 167 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOldFinalizedBlock() { 168 blockB1 := unittest.BlockHeaderWithParentFixture(s.Block) 169 blockB2 := unittest.BlockHeaderWithParentFixture(blockB1) 170 171 s.Blocks[blockB1.ID()] = blockB1 172 s.Blocks[blockB2.ID()] = blockB2 173 174 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block)) 175 // should only call it once 176 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 177 s.MarkFinalized(blockB1) 178 s.MarkFinalized(blockB2) 179 180 // blockB1 becomes finalized 181 err := s.core.ProcessFinalizedBlock(blockB2.ID()) 182 require.NoError(s.T(), err) 183 184 err = s.core.ProcessFinalizedBlock(blockB1.ID()) 185 require.NoError(s.T(), err) 186 } 187 188 // TestProcessFinalizedBlock_CollectorsCleanup tests that stale collectorTree are cleaned up for 189 // already sealed blocks. 190 func (s *ApprovalProcessingCoreTestSuite) TestProcessFinalizedBlock_CollectorsCleanup() { 191 blockID := s.Block.ID() 192 numResults := uint(10) 193 for i := uint(0); i < numResults; i++ { 194 // all results incorporated in different blocks 195 incorporatedBlock := unittest.BlockHeaderWithParentFixture(s.IncorporatedBlock) 196 s.Blocks[incorporatedBlock.ID()] = incorporatedBlock 197 // create different incorporated results for same block ID 198 result := unittest.ExecutionResultFixture() 199 result.BlockID = blockID 200 result.PreviousResultID = s.IncorporatedResult.Result.ID() 201 incorporatedResult := unittest.IncorporatedResult.Fixture( 202 unittest.IncorporatedResult.WithResult(result), 203 unittest.IncorporatedResult.WithIncorporatedBlockID(incorporatedBlock.ID())) 204 err := s.core.processIncorporatedResult(incorporatedResult) 205 require.NoError(s.T(), err) 206 } 207 require.Equal(s.T(), uint64(numResults), s.core.collectorTree.GetSize()) 208 209 candidate := unittest.BlockHeaderWithParentFixture(s.Block) 210 s.Blocks[candidate.ID()] = candidate 211 212 // candidate becomes new sealed and finalized block, it means that 213 // we will need to cleanup our tree till new height, removing all outdated collectors 214 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(candidate)) 215 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 216 217 s.MarkFinalized(candidate) 218 err := s.core.ProcessFinalizedBlock(candidate.ID()) 219 require.NoError(s.T(), err) 220 require.Equal(s.T(), uint64(0), s.core.collectorTree.GetSize()) 221 } 222 223 // TestProcessIncorporated_ApprovalsBeforeResult tests a scenario when first we have received approvals for unknown 224 // execution result and after that we discovered execution result. In this scenario we should be able 225 // to create a seal right after discovering execution result since all approvals should be cached.(if cache capacity is big enough) 226 func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalsBeforeResult() { 227 s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) 228 229 for _, chunk := range s.Chunks { 230 for verID := range s.AuthorizedVerifiers { 231 approval := unittest.ResultApprovalFixture(unittest.WithChunk(chunk.Index), 232 unittest.WithApproverID(verID), 233 unittest.WithBlockID(s.Block.ID()), 234 unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID())) 235 err := s.core.processApproval(approval) 236 require.NoError(s.T(), err) 237 } 238 } 239 240 s.SealsPL.On("Add", mock.Anything).Return(true, nil).Once() 241 242 err := s.core.processIncorporatedResult(s.IncorporatedResult) 243 require.NoError(s.T(), err) 244 245 s.SealsPL.AssertCalled(s.T(), "Add", mock.Anything) 246 } 247 248 // TestProcessIncorporated_ApprovalsAfterResult tests a scenario when first we have discovered execution result 249 // and after that we started receiving approvals. In this scenario we should be able to create a seal right 250 // after processing last needed approval to meet `RequiredApprovalsForSealConstruction` threshold. 251 func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalsAfterResult() { 252 s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) 253 254 s.SealsPL.On("Add", mock.Anything).Return(true, nil).Once() 255 256 err := s.core.processIncorporatedResult(s.IncorporatedResult) 257 require.NoError(s.T(), err) 258 259 for _, chunk := range s.Chunks { 260 for verID := range s.AuthorizedVerifiers { 261 approval := unittest.ResultApprovalFixture(unittest.WithChunk(chunk.Index), 262 unittest.WithApproverID(verID), 263 unittest.WithBlockID(s.Block.ID()), 264 unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID())) 265 err := s.core.processApproval(approval) 266 require.NoError(s.T(), err) 267 } 268 } 269 270 s.SealsPL.AssertCalled(s.T(), "Add", mock.Anything) 271 } 272 273 // TestProcessIncorporated_ProcessingInvalidApproval tests that processing invalid approval when result is discovered 274 // is correctly handled in case of sentinel error 275 func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ProcessingInvalidApproval() { 276 // fail signature verification for first approval 277 s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(false, nil).Once() 278 279 // generate approvals for first chunk 280 approval := unittest.ResultApprovalFixture(unittest.WithChunk(s.Chunks[0].Index), 281 unittest.WithApproverID(s.VerID), 282 unittest.WithBlockID(s.Block.ID()), 283 unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID())) 284 285 // this approval has to be cached since execution result is not known yet 286 err := s.core.processApproval(approval) 287 require.NoError(s.T(), err) 288 289 // at this point approval has to be processed, even if it's invalid 290 // if it's an expected sentinel error, it has to be handled internally 291 err = s.core.processIncorporatedResult(s.IncorporatedResult) 292 require.NoError(s.T(), err) 293 } 294 295 // TestProcessIncorporated_ApprovalVerificationException tests that processing invalid approval when result is discovered 296 // is correctly handled in case of exception 297 func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalVerificationException() { 298 // fail signature verification with exception 299 s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(false, fmt.Errorf("exception")).Once() 300 301 // generate approvals for first chunk 302 approval := unittest.ResultApprovalFixture(unittest.WithChunk(s.Chunks[0].Index), 303 unittest.WithApproverID(s.VerID), 304 unittest.WithBlockID(s.Block.ID()), 305 unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID())) 306 307 // this approval has to be cached since execution result is not known yet 308 err := s.core.processApproval(approval) 309 require.NoError(s.T(), err) 310 311 // at this point approval has to be processed, even if it's invalid 312 // if it's an expected sentinel error, it has to be handled internally 313 err = s.core.processIncorporatedResult(s.IncorporatedResult) 314 require.Error(s.T(), err) 315 } 316 317 // TestOnBlockFinalized_EmergencySealing tests that emergency sealing kicks in to resolve sealing halt 318 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_EmergencySealing() { 319 320 metrics := metrics.NewNoopCollector() 321 tracer := trace.NewNoopTracer() 322 323 setter, err := updatable_configs.NewSealingConfigs( 324 flow.DefaultRequiredApprovalsForSealConstruction, 325 flow.DefaultRequiredApprovalsForSealValidation, 326 flow.DefaultChunkAssignmentAlpha, 327 true, // enable emergency sealing 328 ) 329 require.NoError(s.T(), err) 330 s.core, err = NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), s.Headers, s.State, s.sealsDB, s.Assigner, s.SigHasher, s.SealsPL, s.Conduit, setter) 331 require.NoError(s.T(), err) 332 s.setter = setter 333 334 s.SealsPL.On("ByID", mock.Anything).Return(nil, false).Maybe() 335 s.SealsPL.On("Add", mock.Anything).Run( 336 func(args mock.Arguments) { 337 seal := args.Get(0).(*flow.IncorporatedResultSeal) 338 require.Equal(s.T(), s.Block.ID(), seal.Seal.BlockID) 339 require.Equal(s.T(), s.IncorporatedResult.Result.ID(), seal.Seal.ResultID) 340 }, 341 ).Return(true, nil).Once() 342 343 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock)) 344 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Times(approvals.DefaultEmergencySealingThresholdForFinalization) 345 s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil)) 346 347 err = s.core.ProcessIncorporatedResult(s.IncorporatedResult) 348 require.NoError(s.T(), err) 349 350 lastFinalizedBlock := s.IncorporatedBlock 351 s.MarkFinalized(lastFinalizedBlock) 352 for i := 0; i < approvals.DefaultEmergencySealingThresholdForFinalization; i++ { 353 finalizedBlock := unittest.BlockHeaderWithParentFixture(lastFinalizedBlock) 354 s.Blocks[finalizedBlock.ID()] = finalizedBlock 355 s.MarkFinalized(finalizedBlock) 356 err := s.core.ProcessFinalizedBlock(finalizedBlock.ID()) 357 require.NoError(s.T(), err) 358 lastFinalizedBlock = finalizedBlock 359 } 360 361 s.SealsPL.AssertExpectations(s.T()) 362 } 363 364 // TestOnBlockFinalized_ProcessingOrphanApprovals tests that approvals for orphan forks are rejected as outdated entries without processing 365 // 366 // A <- B_1 <- C_1{ IER[B_1] } 367 // <- B_2 <- C_2{ IER[B_2] } <- D_2{ IER[C_2] } 368 // <- B_3 <- C_3{ IER[B_3] } <- D_3{ IER[C_3] } <- E_3{ IER[D_3] } 369 // 370 // B_1 becomes finalized rendering forks starting at B_2 and B_3 as orphans 371 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ProcessingOrphanApprovals() { 372 forks := make([][]*flow.Block, 3) 373 forkResults := make([][]*flow.ExecutionResult, len(forks)) 374 375 for forkIndex := range forks { 376 forks[forkIndex] = unittest.ChainFixtureFrom(forkIndex+2, s.ParentBlock) 377 fork := forks[forkIndex] 378 379 previousResult := s.IncorporatedResult.Result 380 for blockIndex, block := range fork { 381 s.Blocks[block.ID()] = block.Header 382 s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers 383 384 // create and incorporate result for every block in fork except first one 385 if blockIndex > 0 { 386 // create a result 387 result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*previousResult)) 388 result.BlockID = block.Header.ParentID 389 result.Chunks = s.Chunks 390 forkResults[forkIndex] = append(forkResults[forkIndex], result) 391 previousResult = result 392 393 // incorporate in fork 394 IR := unittest.IncorporatedResult.Fixture( 395 unittest.IncorporatedResult.WithIncorporatedBlockID(block.ID()), 396 unittest.IncorporatedResult.WithResult(result)) 397 398 err := s.core.processIncorporatedResult(IR) 399 require.NoError(s.T(), err) 400 } 401 } 402 } 403 404 // same block sealed 405 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock)) 406 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 407 408 // block B_1 becomes finalized 409 finalized := forks[0][0].Header 410 s.MarkFinalized(finalized) 411 err := s.core.ProcessFinalizedBlock(finalized.ID()) 412 require.NoError(s.T(), err) 413 414 // verify will be called twice for every approval in first fork 415 s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Times(len(forkResults[0]) * 2) 416 417 // try submitting approvals for each result 418 for _, results := range forkResults { 419 for _, result := range results { 420 executedBlockID := result.BlockID 421 resultID := result.ID() 422 423 approval := unittest.ResultApprovalFixture(unittest.WithChunk(0), 424 unittest.WithApproverID(s.VerID), 425 unittest.WithBlockID(executedBlockID), 426 unittest.WithExecutionResultID(resultID)) 427 428 err := s.core.processApproval(approval) 429 require.NoError(s.T(), err) 430 } 431 } 432 } 433 434 // TestOnBlockFinalized_ExtendingUnprocessableFork tests that extending orphan fork results in non processable collectors 435 // 436 // . - X <- Y <- Z 437 // . / 438 // . <- A <- B <- C <- D <- E 439 // . | 440 // . finalized 441 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ExtendingUnprocessableFork() { 442 forks := make([][]*flow.Block, 2) 443 444 for forkIndex := range forks { 445 forks[forkIndex] = unittest.ChainFixtureFrom(forkIndex+3, s.Block) 446 fork := forks[forkIndex] 447 for _, block := range fork { 448 s.Blocks[block.ID()] = block.Header 449 s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers 450 } 451 } 452 453 finalized := forks[1][0].Header 454 455 s.MarkFinalized(finalized) 456 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock)) 457 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 458 459 // finalize block B 460 err := s.core.ProcessFinalizedBlock(finalized.ID()) 461 require.NoError(s.T(), err) 462 463 // create incorporated result for each block in main fork 464 for forkIndex, fork := range forks { 465 previousResult := s.IncorporatedResult.Result 466 for blockIndex, block := range fork { 467 result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*previousResult)) 468 result.BlockID = block.Header.ParentID 469 result.Chunks = s.Chunks 470 previousResult = result 471 472 // incorporate in fork 473 IR := unittest.IncorporatedResult.Fixture( 474 unittest.IncorporatedResult.WithIncorporatedBlockID(block.ID()), 475 unittest.IncorporatedResult.WithResult(result)) 476 err := s.core.processIncorporatedResult(IR) 477 collector := s.core.collectorTree.GetCollector(result.ID()) 478 if forkIndex > 0 { 479 require.NoError(s.T(), err) 480 require.Equal(s.T(), approvals.VerifyingApprovals, collector.ProcessingStatus()) 481 } else { 482 if blockIndex == 0 { 483 require.Error(s.T(), err) 484 require.True(s.T(), engine.IsOutdatedInputError(err)) 485 } else { 486 require.Equal(s.T(), approvals.CachingApprovals, collector.ProcessingStatus()) 487 } 488 } 489 } 490 } 491 } 492 493 // TestOnBlockFinalized_ExtendingSealedResult tests if assignment collector tree accepts collector which extends sealed result 494 func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ExtendingSealedResult() { 495 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block)) 496 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once() 497 498 unsealedBlock := unittest.BlockHeaderWithParentFixture(s.Block) 499 s.Blocks[unsealedBlock.ID()] = unsealedBlock 500 s.IdentitiesCache[unsealedBlock.ID()] = s.AuthorizedVerifiers 501 result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*s.IncorporatedResult.Result)) 502 result.BlockID = unsealedBlock.ID() 503 504 s.MarkFinalized(unsealedBlock) 505 err := s.core.ProcessFinalizedBlock(unsealedBlock.ID()) 506 require.NoError(s.T(), err) 507 508 incorporatedBlock := unittest.BlockHeaderWithParentFixture(unsealedBlock) 509 s.Blocks[incorporatedBlock.ID()] = incorporatedBlock 510 s.IdentitiesCache[incorporatedBlock.ID()] = s.AuthorizedVerifiers 511 IR := unittest.IncorporatedResult.Fixture( 512 unittest.IncorporatedResult.WithIncorporatedBlockID(incorporatedBlock.ID()), 513 unittest.IncorporatedResult.WithResult(result)) 514 err = s.core.processIncorporatedResult(IR) 515 require.NoError(s.T(), err) 516 517 s.sealsDB.AssertExpectations(s.T()) 518 } 519 520 // TestRequestPendingApprovals checks that requests are sent only for chunks 521 // that have not collected enough approvals yet, and are sent only to the 522 // verifiers assigned to those chunks. It also checks that the threshold and 523 // rate limiting is respected. 524 func (s *ApprovalProcessingCoreTestSuite) TestRequestPendingApprovals() { 525 s.core.requestTracker = approvals.NewRequestTracker(s.core.headers, 1, 3) 526 s.SealsPL.On("ByID", mock.Anything).Return(nil, false) 527 528 // n is the total number of blocks and incorporated-results we add to the 529 // chain and mempool 530 n := 100 531 532 // create blocks 533 unsealedFinalizedBlocks := make([]flow.Block, 0, n) 534 parentBlock := s.ParentBlock 535 for i := 0; i < n; i++ { 536 block := unittest.BlockWithParentFixture(parentBlock) 537 s.Blocks[block.ID()] = block.Header 538 s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers 539 unsealedFinalizedBlocks = append(unsealedFinalizedBlocks, *block) 540 parentBlock = block.Header 541 } 542 543 // progress latest sealed and latest finalized: 544 //s.LatestSealedBlock = unsealedFinalizedBlocks[0] 545 //s.LatestFinalizedBlock = &unsealedFinalizedBlocks[n-1] 546 547 // add an unfinalized block; it shouldn't require an approval request 548 unfinalizedBlock := unittest.BlockWithParentFixture(parentBlock) 549 s.Blocks[unfinalizedBlock.ID()] = unfinalizedBlock.Header 550 551 // we will assume that all chunks are assigned to the same two verifiers. 552 verifiers := make([]flow.Identifier, 0) 553 for nodeID := range s.AuthorizedVerifiers { 554 if len(verifiers) > 2 { 555 break 556 } 557 verifiers = append(verifiers, nodeID) 558 } 559 560 // the sealing Core requires approvals from both verifiers for each chunk 561 err := s.setter.SetRequiredApprovalsForSealingConstruction(2) 562 require.NoError(s.T(), err) 563 564 // populate the incorporated-results tree with: 565 // - 50 that have collected two signatures per chunk 566 // - 25 that have collected only one signature 567 // - 25 that have collected no signatures 568 // 569 // 570 // sealed unsealed/finalized 571 // | || | 572 // 1 <- 2 <- .. <- s <- s+1 <- .. <- n-t <- n 573 // | | 574 // expected reqs 575 prevResult := s.IncorporatedResult.Result 576 resultIDs := make([]flow.Identifier, 0, n) 577 chunkCount := 2 578 for i := 0; i < n-1; i++ { 579 580 // Create an incorporated result for unsealedFinalizedBlocks[i]. 581 // By default the result will contain 17 chunks. 582 ir := unittest.IncorporatedResult.Fixture( 583 unittest.IncorporatedResult.WithResult( 584 unittest.ExecutionResultFixture( 585 unittest.WithBlock(&unsealedFinalizedBlocks[i]), 586 unittest.WithPreviousResult(*prevResult), 587 unittest.WithChunks(uint(chunkCount)), 588 ), 589 ), 590 unittest.IncorporatedResult.WithIncorporatedBlockID( 591 unsealedFinalizedBlocks[i+1].ID(), 592 ), 593 ) 594 595 prevResult = ir.Result 596 597 s.ChunksAssignment = chunks.NewAssignment() 598 599 for _, chunk := range ir.Result.Chunks { 600 // assign the verifier to this chunk 601 s.ChunksAssignment.Add(chunk, verifiers) 602 } 603 604 err := s.core.processIncorporatedResult(ir) 605 require.NoError(s.T(), err) 606 607 resultIDs = append(resultIDs, ir.Result.ID()) 608 } 609 610 // sealed block doesn't change 611 seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock)) 612 s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil) 613 614 s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil)) 615 616 // start delivering finalization events 617 lastProcessedIndex := 0 618 for ; lastProcessedIndex < int(s.core.sealingConfigsGetter.ApprovalRequestsThresholdConst()); lastProcessedIndex++ { 619 finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header 620 s.MarkFinalized(finalized) 621 err := s.core.ProcessFinalizedBlock(finalized.ID()) 622 require.NoError(s.T(), err) 623 } 624 625 require.Empty(s.T(), s.core.requestTracker.GetAllIds()) 626 627 // process two more blocks, this will trigger requesting approvals for lastSealed + 1 height 628 // but they will be in blackout period 629 for i := 0; i < 2; i++ { 630 finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header 631 s.MarkFinalized(finalized) 632 err := s.core.ProcessFinalizedBlock(finalized.ID()) 633 require.NoError(s.T(), err) 634 lastProcessedIndex += 1 635 } 636 637 require.ElementsMatch(s.T(), s.core.requestTracker.GetAllIds(), resultIDs[:1]) 638 639 // wait for the max blackout period to elapse 640 time.Sleep(3 * time.Second) 641 642 // our setup is for 5 verification nodes 643 s.Conduit.On("Publish", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). 644 Return(nil).Times(chunkCount) 645 646 // process next block 647 finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header 648 s.MarkFinalized(finalized) 649 err = s.core.ProcessFinalizedBlock(finalized.ID()) 650 require.NoError(s.T(), err) 651 652 // now 2 results should be pending 653 require.ElementsMatch(s.T(), s.core.requestTracker.GetAllIds(), resultIDs[:2]) 654 655 s.Conduit.AssertExpectations(s.T()) 656 } 657 658 // TestRepopulateAssignmentCollectorTree tests that the 659 // collectors tree will contain execution results and assignment collectors will be created. 660 // 661 // P <- A[ER{P}] <- B[ER{A}] <- C[ER{B}] <- D[ER{C}] <- E[ER{D}] 662 // | <- F[ER{A}] <- G[ER{B}] <- H[ER{G}] 663 // finalized 664 // 665 // collectors tree has to be repopulated with incorporated results from blocks [A, B, C, D, F, G] 666 // E, H shouldn't be considered since 667 func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree() { 668 metrics := metrics.NewNoopCollector() 669 tracer := trace.NewNoopTracer() 670 assigner := &module.ChunkAssigner{} 671 672 // setup mocks 673 payloads := &storage.Payloads{} 674 expectedResults := []*flow.IncorporatedResult{s.IncorporatedResult} 675 blockChildren := make([]flow.Identifier, 0) 676 677 rootSnapshot := unittest.StateSnapshotForKnownBlock(s.rootHeader, nil) 678 s.Snapshots[s.rootHeader.ID()] = rootSnapshot 679 rootSnapshot.On("SealingSegment").Return( 680 &flow.SealingSegment{ 681 Blocks: []*flow.Block{{ 682 Header: s.rootHeader, 683 Payload: &flow.Payload{}, 684 }}, 685 }, nil) 686 687 s.sealsDB.On("HighestInFork", s.IncorporatedBlock.ID()).Return( 688 unittest.Seal.Fixture( 689 unittest.Seal.WithBlock(s.ParentBlock)), nil) 690 691 // the incorporated block contains the result for the sealing candidate block 692 incorporatedBlockPayload := unittest.PayloadFixture( 693 unittest.WithReceipts( 694 unittest.ExecutionReceiptFixture( 695 unittest.WithResult(s.IncorporatedResult.Result)))) 696 payloads.On("ByBlockID", s.IncorporatedBlock.ID()).Return(&incorporatedBlockPayload, nil) 697 698 emptyPayload := flow.EmptyPayload() 699 payloads.On("ByBlockID", s.Block.ID()).Return(&emptyPayload, nil) 700 701 s.IdentitiesCache[s.IncorporatedBlock.ID()] = s.AuthorizedVerifiers 702 703 assigner.On("Assign", s.IncorporatedResult.Result, mock.Anything).Return(s.ChunksAssignment, nil) 704 705 // two forks 706 for i := 0; i < 2; i++ { 707 fork := unittest.ChainFixtureFrom(i+3, s.IncorporatedBlock) 708 prevResult := s.IncorporatedResult.Result 709 // create execution results for all blocks except last one, since it won't be valid by definition 710 for blockIndex, block := range fork { 711 blockID := block.ID() 712 713 // create execution result for previous block in chain 714 // this result will be incorporated in current block. 715 result := unittest.ExecutionResultFixture( 716 unittest.WithPreviousResult(*prevResult), 717 ) 718 result.BlockID = block.Header.ParentID 719 720 // update caches 721 s.Blocks[blockID] = block.Header 722 s.IdentitiesCache[blockID] = s.AuthorizedVerifiers 723 blockChildren = append(blockChildren, blockID) 724 725 IR := unittest.IncorporatedResult.Fixture( 726 unittest.IncorporatedResult.WithResult(result), 727 unittest.IncorporatedResult.WithIncorporatedBlockID(blockID)) 728 729 // TODO: change this test for phase 3, assigner should expect incorporated block ID, not executed 730 if blockIndex < len(fork)-1 { 731 assigner.On("Assign", result, blockID).Return(s.ChunksAssignment, nil) 732 expectedResults = append(expectedResults, IR) 733 } else { 734 assigner.On("Assign", result, blockID).Return(nil, fmt.Errorf("no assignment for block without valid child")) 735 } 736 737 payload := unittest.PayloadFixture() 738 payload.Results = append(payload.Results, result) 739 payloads.On("ByBlockID", blockID).Return(&payload, nil) 740 741 prevResult = result 742 } 743 } 744 745 // ValidDescendants has to return all valid descendants from finalized block 746 finalSnapShot := unittest.StateSnapshotForKnownBlock(s.IncorporatedBlock, nil) 747 finalSnapShot.On("ValidDescendants").Return(blockChildren, nil) 748 s.State.On("Final").Return(finalSnapShot) 749 750 core, err := NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), 751 s.Headers, s.State, s.sealsDB, assigner, s.SigHasher, s.SealsPL, s.Conduit, s.setter) 752 require.NoError(s.T(), err) 753 754 err = core.RepopulateAssignmentCollectorTree(payloads) 755 require.NoError(s.T(), err) 756 757 // check collector tree, after repopulating we should have all collectors for execution results that we have 758 // traversed and they have to be processable. 759 for _, incorporatedResult := range expectedResults { 760 collector, err := core.collectorTree.GetOrCreateCollector(incorporatedResult.Result) 761 require.NoError(s.T(), err) 762 require.False(s.T(), collector.Created) 763 require.Equal(s.T(), approvals.VerifyingApprovals, collector.Collector.ProcessingStatus()) 764 } 765 } 766 767 // TestRepopulateAssignmentCollectorTree_RootSealingSegment tests that the sealing 768 // engine will be initialized correctly when bootstrapping with a root sealing 769 // segment with multiple blocks, as is the case when joining the network at an epoch 770 // boundary. 771 // 772 // In particular, the assignment collector tree population step should ignore 773 // unknown block references below the root height. 774 func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree_RootSealingSegment() { 775 metrics := metrics.NewNoopCollector() 776 tracer := trace.NewNoopTracer() 777 assigner := &module.ChunkAssigner{} 778 payloads := &storage.Payloads{} 779 780 // setup mocks 781 s.rootHeader = s.IncorporatedBlock 782 expectedResults := []*flow.IncorporatedResult{s.IncorporatedResult} 783 784 s.sealsDB.On("HighestInFork", s.IncorporatedBlock.ID()).Return( 785 unittest.Seal.Fixture( 786 unittest.Seal.WithBlock(s.ParentBlock)), nil) 787 788 // the incorporated block contains the result for the sealing candidate block 789 incorporatedBlockPayload := unittest.PayloadFixture( 790 unittest.WithReceipts( 791 unittest.ExecutionReceiptFixture( 792 unittest.WithResult(s.IncorporatedResult.Result)))) 793 payloads.On("ByBlockID", s.IncorporatedBlock.ID()).Return(&incorporatedBlockPayload, nil) 794 795 // the sealing candidate block (S) is the lowest block in the segment under consideration here 796 // initially, this block would represent the lowest block in a node's root sealing segment, 797 // meaning that all earlier blocks are not known. In this case we should ignore results and seals 798 // referencing unknown blocks (tested here by adding such a result+seal to the candidate payload). 799 candidatePayload := unittest.PayloadFixture( 800 unittest.WithReceipts(unittest.ExecutionReceiptFixture()), // receipt referencing pre-root block 801 unittest.WithSeals(unittest.Seal.Fixture()), // seal referencing pre-root block 802 ) 803 payloads.On("ByBlockID", s.Block.ID()).Return(&candidatePayload, nil) 804 805 s.IdentitiesCache[s.IncorporatedBlock.ID()] = s.AuthorizedVerifiers 806 807 assigner.On("Assign", s.IncorporatedResult.Result, mock.Anything).Return(s.ChunksAssignment, nil) 808 809 finalSnapShot := unittest.StateSnapshotForKnownBlock(s.rootHeader, nil) 810 s.Snapshots[s.rootHeader.ID()] = finalSnapShot 811 // root snapshot has no pending children 812 finalSnapShot.On("ValidDescendants").Return(nil, nil) 813 // set up sealing segment 814 finalSnapShot.On("SealingSegment").Return( 815 &flow.SealingSegment{ 816 Blocks: []*flow.Block{{ 817 Header: s.Block, 818 Payload: &candidatePayload, 819 }, { 820 Header: s.ParentBlock, 821 Payload: &flow.Payload{}, 822 }, { 823 Header: s.IncorporatedBlock, 824 Payload: &incorporatedBlockPayload, 825 }}, 826 }, nil) 827 s.State.On("Final").Return(finalSnapShot) 828 829 core, err := NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), 830 s.Headers, s.State, s.sealsDB, assigner, s.SigHasher, s.SealsPL, s.Conduit, s.setter) 831 require.NoError(s.T(), err) 832 833 err = core.RepopulateAssignmentCollectorTree(payloads) 834 require.NoError(s.T(), err) 835 836 // check collector tree, after repopulating we should have all collectors for execution results that we have 837 // traversed and they have to be processable. 838 for _, incorporatedResult := range expectedResults { 839 collector, err := core.collectorTree.GetOrCreateCollector(incorporatedResult.Result) 840 require.NoError(s.T(), err) 841 require.False(s.T(), collector.Created) 842 require.Equal(s.T(), approvals.VerifyingApprovals, collector.Collector.ProcessingStatus()) 843 } 844 }