github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/core/ledger/pvtdatastorage/reconcile_missing_pvtdata_test.go (about) 1 /* 2 Copyright hechain. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package pvtdatastorage 8 9 import ( 10 "fmt" 11 "testing" 12 13 "github.com/hechain20/hechain/core/ledger" 14 btltestutil "github.com/hechain20/hechain/core/ledger/pvtdatapolicy/testutil" 15 "github.com/stretchr/testify/require" 16 "github.com/willf/bitset" 17 ) 18 19 type blockTxPvtDataInfoForTest struct { 20 blkNum uint64 21 txNum uint64 22 pvtDataPresent map[string][]string 23 pvtDataMissing map[string][]string 24 } 25 26 type pvtDataForTest struct { 27 pvtData []*ledger.TxPvtData 28 dataKeys []*dataKey 29 missingDataInfo ledger.TxMissingPvtData 30 } 31 32 func TestCommitPvtDataOfOldBlocks(t *testing.T) { 33 btlPolicy := btltestutil.SampleBTLPolicy( 34 map[[2]string]uint64{ 35 {"ns-1", "coll-1"}: 0, 36 {"ns-1", "coll-2"}: 0, 37 {"ns-2", "coll-1"}: 0, 38 {"ns-2", "coll-2"}: 0, 39 }, 40 ) 41 env := NewTestStoreEnv(t, "TestCommitPvtDataOfOldBlocks", btlPolicy, pvtDataConf()) 42 defer env.Cleanup() 43 store := env.TestStore 44 45 blockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 46 { 47 blkNum: 1, 48 txNum: 1, 49 pvtDataMissing: map[string][]string{ 50 "ns-1": {"coll-1", "coll-2"}, 51 "ns-2": {"coll-1", "coll-2"}, 52 }, 53 }, 54 { 55 blkNum: 1, 56 txNum: 2, 57 pvtDataPresent: map[string][]string{ 58 "ns-2": {"coll-1", "coll-2"}, 59 }, 60 pvtDataMissing: map[string][]string{ 61 "ns-1": {"coll-1", "coll-2"}, 62 }, 63 }, 64 { 65 blkNum: 1, 66 txNum: 4, 67 pvtDataPresent: map[string][]string{ 68 "ns-1": {"coll-1", "coll-2"}, 69 "ns-2": {"coll-1", "coll-2"}, 70 }, 71 }, 72 { 73 blkNum: 2, 74 txNum: 1, 75 pvtDataMissing: map[string][]string{ 76 "ns-1": {"coll-1", "coll-2"}, 77 }, 78 }, 79 { 80 blkNum: 2, 81 txNum: 3, 82 pvtDataMissing: map[string][]string{ 83 "ns-1": {"coll-1"}, 84 }, 85 }, 86 } 87 88 blocksPvtData, missingDataSummary := constructPvtDataForTest(t, blockTxPvtDataInfo) 89 90 require.NoError(t, store.Commit(0, nil, nil)) 91 require.NoError(t, store.Commit(1, blocksPvtData[1].pvtData, blocksPvtData[1].missingDataInfo)) 92 require.NoError(t, store.Commit(2, blocksPvtData[2].pvtData, blocksPvtData[2].missingDataInfo)) 93 94 assertMissingDataInfo(t, store, missingDataSummary, 2) 95 96 // COMMIT some of the missing data in the block 1 and block 2 97 oldBlockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 98 { 99 blkNum: 1, 100 txNum: 1, 101 pvtDataPresent: map[string][]string{ 102 "ns-1": {"coll-1"}, 103 "ns-2": {"coll-1"}, 104 }, 105 pvtDataMissing: map[string][]string{ 106 "ns-1": {"coll-2"}, 107 "ns-2": {"coll-2"}, 108 }, 109 }, 110 { 111 blkNum: 1, 112 txNum: 2, 113 pvtDataPresent: map[string][]string{ 114 "ns-1": {"coll-1"}, 115 }, 116 pvtDataMissing: map[string][]string{ 117 "ns-1": {"coll-2"}, 118 }, 119 }, 120 { 121 blkNum: 2, 122 txNum: 1, 123 pvtDataMissing: map[string][]string{ 124 "ns-1": {"coll-1", "coll-2"}, 125 }, 126 }, 127 { 128 blkNum: 2, 129 txNum: 3, 130 pvtDataPresent: map[string][]string{ 131 "ns-1": {"coll-1"}, 132 }, 133 }, 134 } 135 136 blocksPvtData, missingDataSummary = constructPvtDataForTest(t, oldBlockTxPvtDataInfo) 137 oldBlocksPvtData := map[uint64][]*ledger.TxPvtData{ 138 1: blocksPvtData[1].pvtData, 139 2: blocksPvtData[2].pvtData, 140 } 141 require.NoError(t, store.CommitPvtDataOfOldBlocks(oldBlocksPvtData, nil)) 142 143 for _, b := range blocksPvtData { 144 for _, dkey := range b.dataKeys { 145 require.True(t, testDataKeyExists(t, store, dkey)) 146 } 147 } 148 assertMissingDataInfo(t, store, missingDataSummary, 2) 149 } 150 151 func TestCommitPvtDataOfOldBlocksWithBTL(t *testing.T) { 152 setup := func(store *Store) { 153 blockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 154 { 155 blkNum: 1, 156 txNum: 1, 157 pvtDataMissing: map[string][]string{ 158 "ns-1": {"coll-1"}, 159 "ns-2": {"coll-1"}, 160 }, 161 }, 162 { 163 blkNum: 1, 164 txNum: 2, 165 pvtDataMissing: map[string][]string{ 166 "ns-1": {"coll-1"}, 167 "ns-2": {"coll-1"}, 168 }, 169 }, 170 { 171 blkNum: 1, 172 txNum: 3, 173 pvtDataMissing: map[string][]string{ 174 "ns-1": {"coll-1"}, 175 "ns-2": {"coll-1"}, 176 }, 177 }, 178 } 179 180 blocksPvtData, missingDataSummary := constructPvtDataForTest(t, blockTxPvtDataInfo) 181 182 require.NoError(t, store.Commit(0, nil, nil)) 183 require.NoError(t, store.Commit(1, blocksPvtData[1].pvtData, blocksPvtData[1].missingDataInfo)) 184 185 assertMissingDataInfo(t, store, missingDataSummary, 1) 186 187 // COMMIT BLOCK 2 & 3 WITH NO PVTDATA 188 require.NoError(t, store.Commit(2, nil, nil)) 189 require.NoError(t, store.Commit(3, nil, nil)) 190 } 191 192 t.Run("expired but not purged", func(t *testing.T) { 193 btlPolicy := btltestutil.SampleBTLPolicy( 194 map[[2]string]uint64{ 195 {"ns-1", "coll-1"}: 1, 196 {"ns-2", "coll-1"}: 1, 197 }, 198 ) 199 env := NewTestStoreEnv(t, "TestCommitPvtDataOfOldBlocksWithBTL", btlPolicy, pvtDataConf()) 200 defer env.Cleanup() 201 store := env.TestStore 202 203 setup(store) 204 // in block 1, ns-1:coll-1 and ns-2:coll-2 should have expired but not purged. 205 // hence, the commit of pvtdata of block 1 transaction 1 should create entries 206 // in the store 207 oldBlockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 208 { 209 blkNum: 1, 210 txNum: 1, 211 pvtDataPresent: map[string][]string{ 212 "ns-1": {"coll-1"}, 213 "ns-2": {"coll-1"}, 214 }, 215 }, 216 } 217 218 blocksPvtData, _ := constructPvtDataForTest(t, oldBlockTxPvtDataInfo) 219 oldBlocksPvtData := map[uint64][]*ledger.TxPvtData{ 220 1: blocksPvtData[1].pvtData, 221 } 222 deprioritizedList := ledger.MissingPvtDataInfo{ 223 1: ledger.MissingBlockPvtdataInfo{ 224 2: { 225 { 226 Namespace: "ns-1", 227 Collection: "coll-1", 228 }, 229 { 230 Namespace: "ns-2", 231 Collection: "coll-1", 232 }, 233 }, 234 }, 235 } 236 require.NoError(t, store.CommitPvtDataOfOldBlocks(oldBlocksPvtData, deprioritizedList)) 237 238 for _, b := range blocksPvtData { 239 for _, dkey := range b.dataKeys { 240 require.True(t, testDataKeyExists(t, store, dkey)) 241 } 242 } 243 // as all missing data are expired, get missing info would return nil though 244 // it is not purged yet 245 assertMissingDataInfo(t, store, make(ledger.MissingPvtDataInfo), 1) 246 247 // deprioritized list should be present 248 tests := []struct { 249 key nsCollBlk 250 expectedBitmap *bitset.BitSet 251 }{ 252 { 253 key: nsCollBlk{ 254 ns: "ns-1", 255 coll: "coll-1", 256 blkNum: 1, 257 }, 258 expectedBitmap: constructBitSetForTest(2), 259 }, 260 { 261 key: nsCollBlk{ 262 ns: "ns-2", 263 coll: "coll-1", 264 blkNum: 1, 265 }, 266 expectedBitmap: constructBitSetForTest(2), 267 }, 268 } 269 270 for _, tt := range tests { 271 encKey := encodeElgDeprioMissingDataKey(&missingDataKey{tt.key}) 272 missingData, err := store.db.Get(encKey) 273 require.NoError(t, err) 274 275 expectedMissingData, err := encodeMissingDataValue(tt.expectedBitmap) 276 require.NoError(t, err) 277 require.Equal(t, expectedMissingData, missingData) 278 } 279 }) 280 281 t.Run("expired and purged", func(t *testing.T) { 282 btlPolicy := btltestutil.SampleBTLPolicy( 283 map[[2]string]uint64{ 284 {"ns-1", "coll-1"}: 1, 285 {"ns-2", "coll-1"}: 1, 286 }, 287 ) 288 env := NewTestStoreEnv(t, "TestCommitPvtDataOfOldBlocksWithBTL", btlPolicy, pvtDataConf()) 289 defer env.Cleanup() 290 store := env.TestStore 291 292 setup(store) 293 require.NoError(t, store.Commit(4, nil, nil)) 294 295 testWaitForPurgerRoutineToFinish(store) 296 297 // in block 1, ns-1:coll-1 and ns-2:coll-2 should have expired and purged. 298 // hence, the commit of pvtdata of block 1 transaction 2 should not create 299 // entries in the store 300 oldBlockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 301 { 302 blkNum: 1, 303 txNum: 2, 304 pvtDataPresent: map[string][]string{ 305 "ns-1": {"coll-1"}, 306 "ns-2": {"coll-1"}, 307 }, 308 }, 309 } 310 blocksPvtData, _ := constructPvtDataForTest(t, oldBlockTxPvtDataInfo) 311 oldBlocksPvtData := map[uint64][]*ledger.TxPvtData{ 312 1: blocksPvtData[1].pvtData, 313 } 314 deprioritizedList := ledger.MissingPvtDataInfo{ 315 1: ledger.MissingBlockPvtdataInfo{ 316 3: { 317 { 318 Namespace: "ns-1", 319 Collection: "coll-1", 320 }, 321 { 322 Namespace: "ns-2", 323 Collection: "coll-1", 324 }, 325 }, 326 }, 327 } 328 require.NoError(t, store.CommitPvtDataOfOldBlocks(oldBlocksPvtData, deprioritizedList)) 329 330 for _, b := range blocksPvtData { 331 for _, dkey := range b.dataKeys { 332 require.False(t, testDataKeyExists(t, store, dkey)) 333 } 334 } 335 336 // deprioritized list should not be present 337 keys := []nsCollBlk{ 338 { 339 ns: "ns-1", 340 coll: "coll-1", 341 blkNum: 1, 342 }, 343 { 344 ns: "ns-2", 345 coll: "coll-1", 346 blkNum: 1, 347 }, 348 } 349 350 for _, k := range keys { 351 encKey := encodeElgDeprioMissingDataKey(&missingDataKey{k}) 352 missingData, err := store.db.Get(encKey) 353 require.NoError(t, err) 354 require.Nil(t, missingData) 355 } 356 }) 357 } 358 359 func TestCommitPvtDataOfOldBlocksWithDeprioritization(t *testing.T) { 360 blockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 361 { 362 blkNum: 1, 363 txNum: 1, 364 pvtDataMissing: map[string][]string{ 365 "ns-1": {"coll-1"}, 366 "ns-2": {"coll-1"}, 367 }, 368 }, 369 { 370 blkNum: 1, 371 txNum: 2, 372 pvtDataMissing: map[string][]string{ 373 "ns-1": {"coll-1"}, 374 "ns-2": {"coll-1"}, 375 }, 376 }, 377 { 378 blkNum: 2, 379 txNum: 1, 380 pvtDataMissing: map[string][]string{ 381 "ns-1": {"coll-1"}, 382 "ns-2": {"coll-1"}, 383 }, 384 }, 385 { 386 blkNum: 2, 387 txNum: 2, 388 pvtDataMissing: map[string][]string{ 389 "ns-1": {"coll-1"}, 390 "ns-2": {"coll-1"}, 391 }, 392 }, 393 } 394 395 blocksPvtData, missingDataSummary := constructPvtDataForTest(t, blockTxPvtDataInfo) 396 397 tests := []struct { 398 name string 399 deprioritizedList ledger.MissingPvtDataInfo 400 expectedPrioMissingDataKeys ledger.MissingPvtDataInfo 401 }{ 402 { 403 name: "all keys deprioritized", 404 deprioritizedList: missingDataSummary, 405 expectedPrioMissingDataKeys: make(ledger.MissingPvtDataInfo), 406 }, 407 { 408 name: "some keys deprioritized", 409 deprioritizedList: ledger.MissingPvtDataInfo{ 410 1: ledger.MissingBlockPvtdataInfo{ 411 2: { 412 { 413 Namespace: "ns-1", 414 Collection: "coll-1", 415 }, 416 }, 417 }, 418 2: ledger.MissingBlockPvtdataInfo{ 419 2: { 420 { 421 Namespace: "ns-1", 422 Collection: "coll-1", 423 }, 424 }, 425 }, 426 }, 427 expectedPrioMissingDataKeys: ledger.MissingPvtDataInfo{ 428 1: ledger.MissingBlockPvtdataInfo{ 429 1: { 430 { 431 Namespace: "ns-1", 432 Collection: "coll-1", 433 }, 434 { 435 Namespace: "ns-2", 436 Collection: "coll-1", 437 }, 438 }, 439 2: { 440 { 441 Namespace: "ns-2", 442 Collection: "coll-1", 443 }, 444 }, 445 }, 446 2: ledger.MissingBlockPvtdataInfo{ 447 1: { 448 { 449 Namespace: "ns-1", 450 Collection: "coll-1", 451 }, 452 { 453 Namespace: "ns-2", 454 Collection: "coll-1", 455 }, 456 }, 457 2: { 458 { 459 Namespace: "ns-2", 460 Collection: "coll-1", 461 }, 462 }, 463 }, 464 }, 465 }, 466 } 467 468 for _, tt := range tests { 469 t.Run(tt.name, func(t *testing.T) { 470 btlPolicy := btltestutil.SampleBTLPolicy( 471 map[[2]string]uint64{ 472 {"ns-1", "coll-1"}: 0, 473 {"ns-2", "coll-1"}: 0, 474 }, 475 ) 476 env := NewTestStoreEnv(t, "TestCommitPvtDataOfOldBlocksWithDeprio", btlPolicy, pvtDataConf()) 477 defer env.Cleanup() 478 store := env.TestStore 479 480 // COMMIT BLOCK 0 WITH NO DATA 481 require.NoError(t, store.Commit(0, nil, nil)) 482 require.NoError(t, store.Commit(1, blocksPvtData[1].pvtData, blocksPvtData[1].missingDataInfo)) 483 require.NoError(t, store.Commit(2, blocksPvtData[2].pvtData, blocksPvtData[2].missingDataInfo)) 484 485 assertMissingDataInfo(t, store, missingDataSummary, 2) 486 487 require.NoError(t, store.CommitPvtDataOfOldBlocks(nil, tt.deprioritizedList)) 488 489 prioMissingData, err := store.getMissingData(elgPrioritizedMissingDataGroup, 3) 490 require.NoError(t, err) 491 require.Equal(t, len(tt.expectedPrioMissingDataKeys), len(prioMissingData)) 492 for blkNum, txsMissingData := range tt.expectedPrioMissingDataKeys { 493 for txNum, expectedMissingData := range txsMissingData { 494 require.ElementsMatch(t, expectedMissingData, prioMissingData[blkNum][txNum]) 495 } 496 } 497 498 deprioMissingData, err := store.getMissingData(elgDeprioritizedMissingDataGroup, 3) 499 require.NoError(t, err) 500 require.Equal(t, len(tt.deprioritizedList), len(deprioMissingData)) 501 for blkNum, txsMissingData := range tt.deprioritizedList { 502 for txNum, expectedMissingData := range txsMissingData { 503 require.ElementsMatch(t, expectedMissingData, deprioMissingData[blkNum][txNum]) 504 } 505 } 506 507 oldBlockTxPvtDataInfo := []*blockTxPvtDataInfoForTest{ 508 { 509 blkNum: 1, 510 txNum: 1, 511 pvtDataPresent: map[string][]string{ 512 "ns-1": {"coll-1"}, 513 "ns-2": {"coll-1"}, 514 }, 515 }, 516 { 517 blkNum: 1, 518 txNum: 2, 519 pvtDataPresent: map[string][]string{ 520 "ns-1": {"coll-1"}, 521 "ns-2": {"coll-1"}, 522 }, 523 }, 524 { 525 blkNum: 2, 526 txNum: 1, 527 pvtDataPresent: map[string][]string{ 528 "ns-1": {"coll-1"}, 529 "ns-2": {"coll-1"}, 530 }, 531 }, 532 { 533 blkNum: 2, 534 txNum: 2, 535 pvtDataPresent: map[string][]string{ 536 "ns-1": {"coll-1"}, 537 "ns-2": {"coll-1"}, 538 }, 539 }, 540 } 541 542 pvtDataOfOldBlocks, _ := constructPvtDataForTest(t, oldBlockTxPvtDataInfo) 543 oldBlocksPvtData := map[uint64][]*ledger.TxPvtData{ 544 1: pvtDataOfOldBlocks[1].pvtData, 545 2: pvtDataOfOldBlocks[2].pvtData, 546 } 547 require.NoError(t, store.CommitPvtDataOfOldBlocks(oldBlocksPvtData, nil)) 548 549 prioMissingData, err = store.getMissingData(elgPrioritizedMissingDataGroup, 3) 550 require.NoError(t, err) 551 require.Equal(t, make(ledger.MissingPvtDataInfo), prioMissingData) 552 553 deprioMissingData, err = store.getMissingData(elgDeprioritizedMissingDataGroup, 3) 554 require.NoError(t, err) 555 require.Equal(t, make(ledger.MissingPvtDataInfo), deprioMissingData) 556 }) 557 } 558 } 559 560 func constructPvtDataForTest(t *testing.T, blockInfo []*blockTxPvtDataInfoForTest) (map[uint64]*pvtDataForTest, ledger.MissingPvtDataInfo) { 561 blocksPvtData := make(map[uint64]*pvtDataForTest) 562 missingPvtDataInfoSummary := make(ledger.MissingPvtDataInfo) 563 564 for _, b := range blockInfo { 565 p, ok := blocksPvtData[b.blkNum] 566 if !ok { 567 p = &pvtDataForTest{ 568 missingDataInfo: make(ledger.TxMissingPvtData), 569 } 570 blocksPvtData[b.blkNum] = p 571 } 572 573 for ns, colls := range b.pvtDataMissing { 574 for _, coll := range colls { 575 p.missingDataInfo.Add(b.txNum, ns, coll, true) 576 missingPvtDataInfoSummary.Add(b.blkNum, b.txNum, ns, coll) 577 } 578 } 579 580 var nsColls []string 581 for ns, colls := range b.pvtDataPresent { 582 for _, coll := range colls { 583 nsColls = append(nsColls, fmt.Sprintf("%s:%s", ns, coll)) 584 p.dataKeys = append(p.dataKeys, &dataKey{ 585 nsCollBlk: nsCollBlk{ 586 ns: ns, 587 coll: coll, 588 blkNum: b.blkNum, 589 }, 590 txNum: b.txNum, 591 }) 592 } 593 } 594 595 if len(nsColls) == 0 { 596 continue 597 } 598 p.pvtData = append( 599 p.pvtData, 600 produceSamplePvtdata(t, b.txNum, nsColls), 601 ) 602 } 603 604 return blocksPvtData, missingPvtDataInfoSummary 605 } 606 607 func assertMissingDataInfo(t *testing.T, store *Store, expected ledger.MissingPvtDataInfo, numRecentBlocks int) { 608 missingPvtDataInfo, err := store.GetMissingPvtDataInfoForMostRecentBlocks(numRecentBlocks) 609 require.NoError(t, err) 610 require.Equal(t, len(expected), len(missingPvtDataInfo)) 611 for blkNum, txsMissingData := range expected { 612 for txNum, expectedMissingData := range txsMissingData { 613 require.ElementsMatch(t, expectedMissingData, missingPvtDataInfo[blkNum][txNum]) 614 } 615 } 616 } 617 618 func constructBitSetForTest(txNums ...uint) *bitset.BitSet { 619 bitmap := &bitset.BitSet{} 620 for _, txNum := range txNums { 621 bitmap.Set(txNum) 622 } 623 return bitmap 624 }