github.com/palisadeinc/bor@v0.0.0-20230615125219-ab7196213d15/core/tests/blockchain_repair_test.go (about)

     1  // Copyright 2020 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // Tests that abnormal program termination (i.e.crash) and restart doesn't leave
    18  // the database in some strange state with gaps in the chain, nor with block data
    19  // dangling in the future.
    20  
    21  package tests
    22  
    23  import (
    24  	"io/ioutil"
    25  	"math/big"
    26  	"os"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/golang/mock/gomock"
    31  
    32  	"github.com/ethereum/go-ethereum/common"
    33  	"github.com/ethereum/go-ethereum/consensus/bor"
    34  	"github.com/ethereum/go-ethereum/consensus/bor/api"
    35  	"github.com/ethereum/go-ethereum/consensus/bor/valset"
    36  	"github.com/ethereum/go-ethereum/consensus/ethash"
    37  	"github.com/ethereum/go-ethereum/core"
    38  	"github.com/ethereum/go-ethereum/core/rawdb"
    39  	"github.com/ethereum/go-ethereum/core/types"
    40  	"github.com/ethereum/go-ethereum/core/vm"
    41  	"github.com/ethereum/go-ethereum/crypto"
    42  	"github.com/ethereum/go-ethereum/miner"
    43  	"github.com/ethereum/go-ethereum/params"
    44  	"github.com/ethereum/go-ethereum/tests/bor/mocks"
    45  )
    46  
    47  // Tests a recovery for a short canonical chain where a recent block was already
    48  // committed to disk and then the process crashed. In this case we expect the full
    49  // chain to be rolled back to the committed block, but the chain data itself left
    50  // in the database for replaying.
    51  func TestShortRepair(t *testing.T)              { testShortRepair(t, false) }
    52  func TestShortRepairWithSnapshots(t *testing.T) { testShortRepair(t, true) }
    53  
    54  func testShortRepair(t *testing.T, snapshots bool) {
    55  	// Chain:
    56  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
    57  	//
    58  	// Frozen: none
    59  	// Commit: G, C4
    60  	// Pivot : none
    61  	//
    62  	// CRASH
    63  	//
    64  	// ------------------------------
    65  	//
    66  	// Expected in leveldb:
    67  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
    68  	//
    69  	// Expected head header    : C8
    70  	// Expected head fast block: C8
    71  	// Expected head block     : C4
    72  	testRepair(t, &rewindTest{
    73  		canonicalBlocks:    8,
    74  		sidechainBlocks:    0,
    75  		freezeThreshold:    16,
    76  		commitBlock:        4,
    77  		pivotBlock:         nil,
    78  		expCanonicalBlocks: 8,
    79  		expSidechainBlocks: 0,
    80  		expFrozen:          0,
    81  		expHeadHeader:      8,
    82  		expHeadFastBlock:   8,
    83  		expHeadBlock:       4,
    84  	}, snapshots)
    85  }
    86  
    87  // Tests a recovery for a short canonical chain where the fast sync pivot point was
    88  // already committed, after which the process crashed. In this case we expect the full
    89  // chain to be rolled back to the committed block, but the chain data itself left in
    90  // the database for replaying.
    91  func TestShortSnapSyncedRepair(t *testing.T)              { testShortSnapSyncedRepair(t, false) }
    92  func TestShortSnapSyncedRepairWithSnapshots(t *testing.T) { testShortSnapSyncedRepair(t, true) }
    93  
    94  func testShortSnapSyncedRepair(t *testing.T, snapshots bool) {
    95  	// Chain:
    96  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
    97  	//
    98  	// Frozen: none
    99  	// Commit: G, C4
   100  	// Pivot : C4
   101  	//
   102  	// CRASH
   103  	//
   104  	// ------------------------------
   105  	//
   106  	// Expected in leveldb:
   107  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   108  	//
   109  	// Expected head header    : C8
   110  	// Expected head fast block: C8
   111  	// Expected head block     : C4
   112  	testRepair(t, &rewindTest{
   113  		canonicalBlocks:    8,
   114  		sidechainBlocks:    0,
   115  		freezeThreshold:    16,
   116  		commitBlock:        4,
   117  		pivotBlock:         uint64ptr(4),
   118  		expCanonicalBlocks: 8,
   119  		expSidechainBlocks: 0,
   120  		expFrozen:          0,
   121  		expHeadHeader:      8,
   122  		expHeadFastBlock:   8,
   123  		expHeadBlock:       4,
   124  	}, snapshots)
   125  }
   126  
   127  // Tests a recovery for a short canonical chain where the fast sync pivot point was
   128  // not yet committed, but the process crashed. In this case we expect the chain to
   129  // detect that it was fast syncing and not delete anything, since we can just pick
   130  // up directly where we left off.
   131  func TestShortSnapSyncingRepair(t *testing.T)              { testShortSnapSyncingRepair(t, false) }
   132  func TestShortSnapSyncingRepairWithSnapshots(t *testing.T) { testShortSnapSyncingRepair(t, true) }
   133  
   134  func testShortSnapSyncingRepair(t *testing.T, snapshots bool) {
   135  	// Chain:
   136  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   137  	//
   138  	// Frozen: none
   139  	// Commit: G
   140  	// Pivot : C4
   141  	//
   142  	// CRASH
   143  	//
   144  	// ------------------------------
   145  	//
   146  	// Expected in leveldb:
   147  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   148  	//
   149  	// Expected head header    : C8
   150  	// Expected head fast block: C8
   151  	// Expected head block     : G
   152  	testRepair(t, &rewindTest{
   153  		canonicalBlocks:    8,
   154  		sidechainBlocks:    0,
   155  		freezeThreshold:    16,
   156  		commitBlock:        0,
   157  		pivotBlock:         uint64ptr(4),
   158  		expCanonicalBlocks: 8,
   159  		expSidechainBlocks: 0,
   160  		expFrozen:          0,
   161  		expHeadHeader:      8,
   162  		expHeadFastBlock:   8,
   163  		expHeadBlock:       0,
   164  	}, snapshots)
   165  }
   166  
   167  // Tests a recovery for a short canonical chain and a shorter side chain, where a
   168  // recent block was already committed to disk and then the process crashed. In this
   169  // test scenario the side chain is below the committed block. In this case we expect
   170  // the canonical chain to be rolled back to the committed block, but the chain data
   171  // itself left in the database for replaying.
   172  func TestShortOldForkedRepair(t *testing.T)              { testShortOldForkedRepair(t, false) }
   173  func TestShortOldForkedRepairWithSnapshots(t *testing.T) { testShortOldForkedRepair(t, true) }
   174  
   175  func testShortOldForkedRepair(t *testing.T, snapshots bool) {
   176  	// Chain:
   177  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   178  	//   └->S1->S2->S3
   179  	//
   180  	// Frozen: none
   181  	// Commit: G, C4
   182  	// Pivot : none
   183  	//
   184  	// CRASH
   185  	//
   186  	// ------------------------------
   187  	//
   188  	// Expected in leveldb:
   189  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   190  	//   └->S1->S2->S3
   191  	//
   192  	// Expected head header    : C8
   193  	// Expected head fast block: C8
   194  	// Expected head block     : C4
   195  	testRepair(t, &rewindTest{
   196  		canonicalBlocks:    8,
   197  		sidechainBlocks:    3,
   198  		freezeThreshold:    16,
   199  		commitBlock:        4,
   200  		pivotBlock:         nil,
   201  		expCanonicalBlocks: 8,
   202  		expSidechainBlocks: 3,
   203  		expFrozen:          0,
   204  		expHeadHeader:      8,
   205  		expHeadFastBlock:   8,
   206  		expHeadBlock:       4,
   207  	}, snapshots)
   208  }
   209  
   210  // Tests a recovery for a short canonical chain and a shorter side chain, where
   211  // the fast sync pivot point was already committed to disk and then the process
   212  // crashed. In this test scenario the side chain is below the committed block. In
   213  // this case we expect the canonical chain to be rolled back to the committed block,
   214  // but the chain data itself left in the database for replaying.
   215  func TestShortOldForkedSnapSyncedRepair(t *testing.T) {
   216  	testShortOldForkedSnapSyncedRepair(t, false)
   217  }
   218  func TestShortOldForkedSnapSyncedRepairWithSnapshots(t *testing.T) {
   219  	testShortOldForkedSnapSyncedRepair(t, true)
   220  }
   221  
   222  func testShortOldForkedSnapSyncedRepair(t *testing.T, snapshots bool) {
   223  	// Chain:
   224  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   225  	//   └->S1->S2->S3
   226  	//
   227  	// Frozen: none
   228  	// Commit: G, C4
   229  	// Pivot : C4
   230  	//
   231  	// CRASH
   232  	//
   233  	// ------------------------------
   234  	//
   235  	// Expected in leveldb:
   236  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   237  	//   └->S1->S2->S3
   238  	//
   239  	// Expected head header    : C8
   240  	// Expected head fast block: C8
   241  	// Expected head block     : C4
   242  	testRepair(t, &rewindTest{
   243  		canonicalBlocks:    8,
   244  		sidechainBlocks:    3,
   245  		freezeThreshold:    16,
   246  		commitBlock:        4,
   247  		pivotBlock:         uint64ptr(4),
   248  		expCanonicalBlocks: 8,
   249  		expSidechainBlocks: 3,
   250  		expFrozen:          0,
   251  		expHeadHeader:      8,
   252  		expHeadFastBlock:   8,
   253  		expHeadBlock:       4,
   254  	}, snapshots)
   255  }
   256  
   257  // Tests a recovery for a short canonical chain and a shorter side chain, where
   258  // the fast sync pivot point was not yet committed, but the process crashed. In this
   259  // test scenario the side chain is below the committed block. In this case we expect
   260  // the chain to detect that it was fast syncing and not delete anything, since we
   261  // can just pick up directly where we left off.
   262  func TestShortOldForkedSnapSyncingRepair(t *testing.T) {
   263  	testShortOldForkedSnapSyncingRepair(t, false)
   264  }
   265  func TestShortOldForkedSnapSyncingRepairWithSnapshots(t *testing.T) {
   266  	testShortOldForkedSnapSyncingRepair(t, true)
   267  }
   268  
   269  func testShortOldForkedSnapSyncingRepair(t *testing.T, snapshots bool) {
   270  	// Chain:
   271  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   272  	//   └->S1->S2->S3
   273  	//
   274  	// Frozen: none
   275  	// Commit: G
   276  	// Pivot : C4
   277  	//
   278  	// CRASH
   279  	//
   280  	// ------------------------------
   281  	//
   282  	// Expected in leveldb:
   283  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   284  	//   └->S1->S2->S3
   285  	//
   286  	// Expected head header    : C8
   287  	// Expected head fast block: C8
   288  	// Expected head block     : G
   289  	testRepair(t, &rewindTest{
   290  		canonicalBlocks:    8,
   291  		sidechainBlocks:    3,
   292  		freezeThreshold:    16,
   293  		commitBlock:        0,
   294  		pivotBlock:         uint64ptr(4),
   295  		expCanonicalBlocks: 8,
   296  		expSidechainBlocks: 3,
   297  		expFrozen:          0,
   298  		expHeadHeader:      8,
   299  		expHeadFastBlock:   8,
   300  		expHeadBlock:       0,
   301  	}, snapshots)
   302  }
   303  
   304  // Tests a recovery for a short canonical chain and a shorter side chain, where a
   305  // recent block was already committed to disk and then the process crashed. In this
   306  // test scenario the side chain reaches above the committed block. In this case we
   307  // expect the canonical chain to be rolled back to the committed block, but the
   308  // chain data itself left in the database for replaying.
   309  func TestShortNewlyForkedRepair(t *testing.T)              { testShortNewlyForkedRepair(t, false) }
   310  func TestShortNewlyForkedRepairWithSnapshots(t *testing.T) { testShortNewlyForkedRepair(t, true) }
   311  
   312  func testShortNewlyForkedRepair(t *testing.T, snapshots bool) {
   313  	// Chain:
   314  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   315  	//   └->S1->S2->S3->S4->S5->S6
   316  	//
   317  	// Frozen: none
   318  	// Commit: G, C4
   319  	// Pivot : none
   320  	//
   321  	// CRASH
   322  	//
   323  	// ------------------------------
   324  	//
   325  	// Expected in leveldb:
   326  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   327  	//   └->S1->S2->S3->S4->S5->S6
   328  	//
   329  	// Expected head header    : C8
   330  	// Expected head fast block: C8
   331  	// Expected head block     : C4
   332  	testRepair(t, &rewindTest{
   333  		canonicalBlocks:    8,
   334  		sidechainBlocks:    6,
   335  		freezeThreshold:    16,
   336  		commitBlock:        4,
   337  		pivotBlock:         nil,
   338  		expCanonicalBlocks: 8,
   339  		expSidechainBlocks: 6,
   340  		expFrozen:          0,
   341  		expHeadHeader:      8,
   342  		expHeadFastBlock:   8,
   343  		expHeadBlock:       4,
   344  	}, snapshots)
   345  }
   346  
   347  // Tests a recovery for a short canonical chain and a shorter side chain, where
   348  // the fast sync pivot point was already committed to disk and then the process
   349  // crashed. In this test scenario the side chain reaches above the committed block.
   350  // In this case we expect the canonical chain to be rolled back to the committed
   351  // block, but the chain data itself left in the database for replaying.
   352  func TestShortNewlyForkedSnapSyncedRepair(t *testing.T) {
   353  	testShortNewlyForkedSnapSyncedRepair(t, false)
   354  }
   355  func TestShortNewlyForkedSnapSyncedRepairWithSnapshots(t *testing.T) {
   356  	testShortNewlyForkedSnapSyncedRepair(t, true)
   357  }
   358  
   359  func testShortNewlyForkedSnapSyncedRepair(t *testing.T, snapshots bool) {
   360  	// Chain:
   361  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   362  	//   └->S1->S2->S3->S4->S5->S6
   363  	//
   364  	// Frozen: none
   365  	// Commit: G, C4
   366  	// Pivot : C4
   367  	//
   368  	// CRASH
   369  	//
   370  	// ------------------------------
   371  	//
   372  	// Expected in leveldb:
   373  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   374  	//   └->S1->S2->S3->S4->S5->S6
   375  	//
   376  	// Expected head header    : C8
   377  	// Expected head fast block: C8
   378  	// Expected head block     : C4
   379  	testRepair(t, &rewindTest{
   380  		canonicalBlocks:    8,
   381  		sidechainBlocks:    6,
   382  		freezeThreshold:    16,
   383  		commitBlock:        4,
   384  		pivotBlock:         uint64ptr(4),
   385  		expCanonicalBlocks: 8,
   386  		expSidechainBlocks: 6,
   387  		expFrozen:          0,
   388  		expHeadHeader:      8,
   389  		expHeadFastBlock:   8,
   390  		expHeadBlock:       4,
   391  	}, snapshots)
   392  }
   393  
   394  // Tests a recovery for a short canonical chain and a shorter side chain, where
   395  // the fast sync pivot point was not yet committed, but the process crashed. In
   396  // this test scenario the side chain reaches above the committed block. In this
   397  // case we expect the chain to detect that it was fast syncing and not delete
   398  // anything, since we can just pick up directly where we left off.
   399  func TestShortNewlyForkedSnapSyncingRepair(t *testing.T) {
   400  	testShortNewlyForkedSnapSyncingRepair(t, false)
   401  }
   402  func TestShortNewlyForkedSnapSyncingRepairWithSnapshots(t *testing.T) {
   403  	testShortNewlyForkedSnapSyncingRepair(t, true)
   404  }
   405  
   406  func testShortNewlyForkedSnapSyncingRepair(t *testing.T, snapshots bool) {
   407  	// Chain:
   408  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   409  	//   └->S1->S2->S3->S4->S5->S6
   410  	//
   411  	// Frozen: none
   412  	// Commit: G
   413  	// Pivot : C4
   414  	//
   415  	// CRASH
   416  	//
   417  	// ------------------------------
   418  	//
   419  	// Expected in leveldb:
   420  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   421  	//   └->S1->S2->S3->S4->S5->S6
   422  	//
   423  	// Expected head header    : C8
   424  	// Expected head fast block: C8
   425  	// Expected head block     : G
   426  	testRepair(t, &rewindTest{
   427  		canonicalBlocks:    8,
   428  		sidechainBlocks:    6,
   429  		freezeThreshold:    16,
   430  		commitBlock:        0,
   431  		pivotBlock:         uint64ptr(4),
   432  		expCanonicalBlocks: 8,
   433  		expSidechainBlocks: 6,
   434  		expFrozen:          0,
   435  		expHeadHeader:      8,
   436  		expHeadFastBlock:   8,
   437  		expHeadBlock:       0,
   438  	}, snapshots)
   439  }
   440  
   441  // Tests a recovery for a short canonical chain and a longer side chain, where a
   442  // recent block was already committed to disk and then the process crashed. In this
   443  // case we expect the canonical chain to be rolled back to the committed block, but
   444  // the chain data itself left in the database for replaying.
   445  func TestShortReorgedRepair(t *testing.T)              { testShortReorgedRepair(t, false) }
   446  func TestShortReorgedRepairWithSnapshots(t *testing.T) { testShortReorgedRepair(t, true) }
   447  
   448  func testShortReorgedRepair(t *testing.T, snapshots bool) {
   449  	// Chain:
   450  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   451  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   452  	//
   453  	// Frozen: none
   454  	// Commit: G, C4
   455  	// Pivot : none
   456  	//
   457  	// CRASH
   458  	//
   459  	// ------------------------------
   460  	//
   461  	// Expected in leveldb:
   462  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   463  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   464  	//
   465  	// Expected head header    : C8
   466  	// Expected head fast block: C8
   467  	// Expected head block     : C4
   468  	testRepair(t, &rewindTest{
   469  		canonicalBlocks:    8,
   470  		sidechainBlocks:    10,
   471  		freezeThreshold:    16,
   472  		commitBlock:        4,
   473  		pivotBlock:         nil,
   474  		expCanonicalBlocks: 8,
   475  		expSidechainBlocks: 10,
   476  		expFrozen:          0,
   477  		expHeadHeader:      8,
   478  		expHeadFastBlock:   8,
   479  		expHeadBlock:       4,
   480  	}, snapshots)
   481  }
   482  
   483  // Tests a recovery for a short canonical chain and a longer side chain, where
   484  // the fast sync pivot point was already committed to disk and then the process
   485  // crashed. In this case we expect the canonical chain to be rolled back to the
   486  // committed block, but the chain data itself left in the database for replaying.
   487  func TestShortReorgedSnapSyncedRepair(t *testing.T) {
   488  	testShortReorgedSnapSyncedRepair(t, false)
   489  }
   490  func TestShortReorgedSnapSyncedRepairWithSnapshots(t *testing.T) {
   491  	testShortReorgedSnapSyncedRepair(t, true)
   492  }
   493  
   494  func testShortReorgedSnapSyncedRepair(t *testing.T, snapshots bool) {
   495  	// Chain:
   496  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   497  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   498  	//
   499  	// Frozen: none
   500  	// Commit: G, C4
   501  	// Pivot : C4
   502  	//
   503  	// CRASH
   504  	//
   505  	// ------------------------------
   506  	//
   507  	// Expected in leveldb:
   508  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   509  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   510  	//
   511  	// Expected head header    : C8
   512  	// Expected head fast block: C8
   513  	// Expected head block     : C4
   514  	testRepair(t, &rewindTest{
   515  		canonicalBlocks:    8,
   516  		sidechainBlocks:    10,
   517  		freezeThreshold:    16,
   518  		commitBlock:        4,
   519  		pivotBlock:         uint64ptr(4),
   520  		expCanonicalBlocks: 8,
   521  		expSidechainBlocks: 10,
   522  		expFrozen:          0,
   523  		expHeadHeader:      8,
   524  		expHeadFastBlock:   8,
   525  		expHeadBlock:       4,
   526  	}, snapshots)
   527  }
   528  
   529  // Tests a recovery for a short canonical chain and a longer side chain, where
   530  // the fast sync pivot point was not yet committed, but the process crashed. In
   531  // this case we expect the chain to detect that it was fast syncing and not delete
   532  // anything, since we can just pick up directly where we left off.
   533  func TestShortReorgedSnapSyncingRepair(t *testing.T) {
   534  	testShortReorgedSnapSyncingRepair(t, false)
   535  }
   536  func TestShortReorgedSnapSyncingRepairWithSnapshots(t *testing.T) {
   537  	testShortReorgedSnapSyncingRepair(t, true)
   538  }
   539  
   540  func testShortReorgedSnapSyncingRepair(t *testing.T, snapshots bool) {
   541  	// Chain:
   542  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   543  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   544  	//
   545  	// Frozen: none
   546  	// Commit: G
   547  	// Pivot : C4
   548  	//
   549  	// CRASH
   550  	//
   551  	// ------------------------------
   552  	//
   553  	// Expected in leveldb:
   554  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   555  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
   556  	//
   557  	// Expected head header    : C8
   558  	// Expected head fast block: C8
   559  	// Expected head block     : G
   560  	testRepair(t, &rewindTest{
   561  		canonicalBlocks:    8,
   562  		sidechainBlocks:    10,
   563  		freezeThreshold:    16,
   564  		commitBlock:        0,
   565  		pivotBlock:         uint64ptr(4),
   566  		expCanonicalBlocks: 8,
   567  		expSidechainBlocks: 10,
   568  		expFrozen:          0,
   569  		expHeadHeader:      8,
   570  		expHeadFastBlock:   8,
   571  		expHeadBlock:       0,
   572  	}, snapshots)
   573  }
   574  
   575  // Tests a recovery for a long canonical chain with frozen blocks where a recent
   576  // block - newer than the ancient limit - was already committed to disk and then
   577  // the process crashed. In this case we expect the chain to be rolled back to the
   578  // committed block, with everything afterwads kept as fast sync data.
   579  func TestLongShallowRepair(t *testing.T)              { testLongShallowRepair(t, false) }
   580  func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) }
   581  
   582  func testLongShallowRepair(t *testing.T, snapshots bool) {
   583  	// Chain:
   584  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
   585  	//
   586  	// Frozen:
   587  	//   G->C1->C2
   588  	//
   589  	// Commit: G, C4
   590  	// Pivot : none
   591  	//
   592  	// CRASH
   593  	//
   594  	// ------------------------------
   595  	//
   596  	// Expected in freezer:
   597  	//   G->C1->C2
   598  	//
   599  	// Expected in leveldb:
   600  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
   601  	//
   602  	// Expected head header    : C18
   603  	// Expected head fast block: C18
   604  	// Expected head block     : C4
   605  	testRepair(t, &rewindTest{
   606  		canonicalBlocks:    18,
   607  		sidechainBlocks:    0,
   608  		freezeThreshold:    16,
   609  		commitBlock:        4,
   610  		pivotBlock:         nil,
   611  		expCanonicalBlocks: 18,
   612  		expSidechainBlocks: 0,
   613  		expFrozen:          3,
   614  		expHeadHeader:      18,
   615  		expHeadFastBlock:   18,
   616  		expHeadBlock:       4,
   617  	}, snapshots)
   618  }
   619  
   620  // Tests a recovery for a long canonical chain with frozen blocks where a recent
   621  // block - older than the ancient limit - was already committed to disk and then
   622  // the process crashed. In this case we expect the chain to be rolled back to the
   623  // committed block, with everything afterwads deleted.
   624  func TestLongDeepRepair(t *testing.T)              { testLongDeepRepair(t, false) }
   625  func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) }
   626  
   627  func testLongDeepRepair(t *testing.T, snapshots bool) {
   628  	// Chain:
   629  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
   630  	//
   631  	// Frozen:
   632  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   633  	//
   634  	// Commit: G, C4
   635  	// Pivot : none
   636  	//
   637  	// CRASH
   638  	//
   639  	// ------------------------------
   640  	//
   641  	// Expected in freezer:
   642  	//   G->C1->C2->C3->C4
   643  	//
   644  	// Expected in leveldb: none
   645  	//
   646  	// Expected head header    : C4
   647  	// Expected head fast block: C4
   648  	// Expected head block     : C4
   649  	testRepair(t, &rewindTest{
   650  		canonicalBlocks:    24,
   651  		sidechainBlocks:    0,
   652  		freezeThreshold:    16,
   653  		commitBlock:        4,
   654  		pivotBlock:         nil,
   655  		expCanonicalBlocks: 4,
   656  		expSidechainBlocks: 0,
   657  		expFrozen:          5,
   658  		expHeadHeader:      4,
   659  		expHeadFastBlock:   4,
   660  		expHeadBlock:       4,
   661  	}, snapshots)
   662  }
   663  
   664  // Tests a recovery for a long canonical chain with frozen blocks where the fast
   665  // sync pivot point - newer than the ancient limit - was already committed, after
   666  // which the process crashed. In this case we expect the chain to be rolled back
   667  // to the committed block, with everything afterwads kept as fast sync data.
   668  func TestLongSnapSyncedShallowRepair(t *testing.T) {
   669  	testLongSnapSyncedShallowRepair(t, false)
   670  }
   671  func TestLongSnapSyncedShallowRepairWithSnapshots(t *testing.T) {
   672  	testLongSnapSyncedShallowRepair(t, true)
   673  }
   674  
   675  func testLongSnapSyncedShallowRepair(t *testing.T, snapshots bool) {
   676  	// Chain:
   677  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
   678  	//
   679  	// Frozen:
   680  	//   G->C1->C2
   681  	//
   682  	// Commit: G, C4
   683  	// Pivot : C4
   684  	//
   685  	// CRASH
   686  	//
   687  	// ------------------------------
   688  	//
   689  	// Expected in freezer:
   690  	//   G->C1->C2
   691  	//
   692  	// Expected in leveldb:
   693  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
   694  	//
   695  	// Expected head header    : C18
   696  	// Expected head fast block: C18
   697  	// Expected head block     : C4
   698  	testRepair(t, &rewindTest{
   699  		canonicalBlocks:    18,
   700  		sidechainBlocks:    0,
   701  		freezeThreshold:    16,
   702  		commitBlock:        4,
   703  		pivotBlock:         uint64ptr(4),
   704  		expCanonicalBlocks: 18,
   705  		expSidechainBlocks: 0,
   706  		expFrozen:          3,
   707  		expHeadHeader:      18,
   708  		expHeadFastBlock:   18,
   709  		expHeadBlock:       4,
   710  	}, snapshots)
   711  }
   712  
   713  // Tests a recovery for a long canonical chain with frozen blocks where the fast
   714  // sync pivot point - older than the ancient limit - was already committed, after
   715  // which the process crashed. In this case we expect the chain to be rolled back
   716  // to the committed block, with everything afterwads deleted.
   717  func TestLongSnapSyncedDeepRepair(t *testing.T)              { testLongSnapSyncedDeepRepair(t, false) }
   718  func TestLongSnapSyncedDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncedDeepRepair(t, true) }
   719  
   720  func testLongSnapSyncedDeepRepair(t *testing.T, snapshots bool) {
   721  	// Chain:
   722  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
   723  	//
   724  	// Frozen:
   725  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   726  	//
   727  	// Commit: G, C4
   728  	// Pivot : C4
   729  	//
   730  	// CRASH
   731  	//
   732  	// ------------------------------
   733  	//
   734  	// Expected in freezer:
   735  	//   G->C1->C2->C3->C4
   736  	//
   737  	// Expected in leveldb: none
   738  	//
   739  	// Expected head header    : C4
   740  	// Expected head fast block: C4
   741  	// Expected head block     : C4
   742  	testRepair(t, &rewindTest{
   743  		canonicalBlocks:    24,
   744  		sidechainBlocks:    0,
   745  		freezeThreshold:    16,
   746  		commitBlock:        4,
   747  		pivotBlock:         uint64ptr(4),
   748  		expCanonicalBlocks: 4,
   749  		expSidechainBlocks: 0,
   750  		expFrozen:          5,
   751  		expHeadHeader:      4,
   752  		expHeadFastBlock:   4,
   753  		expHeadBlock:       4,
   754  	}, snapshots)
   755  }
   756  
   757  // Tests a recovery for a long canonical chain with frozen blocks where the fast
   758  // sync pivot point - older than the ancient limit - was not yet committed, but the
   759  // process crashed. In this case we expect the chain to detect that it was fast
   760  // syncing and not delete anything, since we can just pick up directly where we
   761  // left off.
   762  func TestLongSnapSyncingShallowRepair(t *testing.T) {
   763  	testLongSnapSyncingShallowRepair(t, false)
   764  }
   765  func TestLongSnapSyncingShallowRepairWithSnapshots(t *testing.T) {
   766  	testLongSnapSyncingShallowRepair(t, true)
   767  }
   768  
   769  func testLongSnapSyncingShallowRepair(t *testing.T, snapshots bool) {
   770  	// Chain:
   771  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
   772  	//
   773  	// Frozen:
   774  	//   G->C1->C2
   775  	//
   776  	// Commit: G
   777  	// Pivot : C4
   778  	//
   779  	// CRASH
   780  	//
   781  	// ------------------------------
   782  	//
   783  	// Expected in freezer:
   784  	//   G->C1->C2
   785  	//
   786  	// Expected in leveldb:
   787  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
   788  	//
   789  	// Expected head header    : C18
   790  	// Expected head fast block: C18
   791  	// Expected head block     : G
   792  	testRepair(t, &rewindTest{
   793  		canonicalBlocks:    18,
   794  		sidechainBlocks:    0,
   795  		freezeThreshold:    16,
   796  		commitBlock:        0,
   797  		pivotBlock:         uint64ptr(4),
   798  		expCanonicalBlocks: 18,
   799  		expSidechainBlocks: 0,
   800  		expFrozen:          3,
   801  		expHeadHeader:      18,
   802  		expHeadFastBlock:   18,
   803  		expHeadBlock:       0,
   804  	}, snapshots)
   805  }
   806  
   807  // Tests a recovery for a long canonical chain with frozen blocks where the fast
   808  // sync pivot point - newer than the ancient limit - was not yet committed, but the
   809  // process crashed. In this case we expect the chain to detect that it was fast
   810  // syncing and not delete anything, since we can just pick up directly where we
   811  // left off.
   812  func TestLongSnapSyncingDeepRepair(t *testing.T)              { testLongSnapSyncingDeepRepair(t, false) }
   813  func TestLongSnapSyncingDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncingDeepRepair(t, true) }
   814  
   815  func testLongSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
   816  	// Chain:
   817  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
   818  	//
   819  	// Frozen:
   820  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   821  	//
   822  	// Commit: G
   823  	// Pivot : C4
   824  	//
   825  	// CRASH
   826  	//
   827  	// ------------------------------
   828  	//
   829  	// Expected in freezer:
   830  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   831  	//
   832  	// Expected in leveldb:
   833  	//   C8)->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24
   834  	//
   835  	// Expected head header    : C24
   836  	// Expected head fast block: C24
   837  	// Expected head block     : G
   838  	testRepair(t, &rewindTest{
   839  		canonicalBlocks:    24,
   840  		sidechainBlocks:    0,
   841  		freezeThreshold:    16,
   842  		commitBlock:        0,
   843  		pivotBlock:         uint64ptr(4),
   844  		expCanonicalBlocks: 24,
   845  		expSidechainBlocks: 0,
   846  		expFrozen:          9,
   847  		expHeadHeader:      24,
   848  		expHeadFastBlock:   24,
   849  		expHeadBlock:       0,
   850  	}, snapshots)
   851  }
   852  
   853  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
   854  // side chain, where a recent block - newer than the ancient limit - was already
   855  // committed to disk and then the process crashed. In this test scenario the side
   856  // chain is below the committed block. In this case we expect the chain to be
   857  // rolled back to the committed block, with everything afterwads kept as fast
   858  // sync data; the side chain completely nuked by the freezer.
   859  func TestLongOldForkedShallowRepair(t *testing.T) {
   860  	testLongOldForkedShallowRepair(t, false)
   861  }
   862  func TestLongOldForkedShallowRepairWithSnapshots(t *testing.T) {
   863  	testLongOldForkedShallowRepair(t, true)
   864  }
   865  
   866  func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) {
   867  	// Chain:
   868  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
   869  	//   └->S1->S2->S3
   870  	//
   871  	// Frozen:
   872  	//   G->C1->C2
   873  	//
   874  	// Commit: G, C4
   875  	// Pivot : none
   876  	//
   877  	// CRASH
   878  	//
   879  	// ------------------------------
   880  	//
   881  	// Expected in freezer:
   882  	//   G->C1->C2
   883  	//
   884  	// Expected in leveldb:
   885  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
   886  	//
   887  	// Expected head header    : C18
   888  	// Expected head fast block: C18
   889  	// Expected head block     : C4
   890  	testRepair(t, &rewindTest{
   891  		canonicalBlocks:    18,
   892  		sidechainBlocks:    3,
   893  		freezeThreshold:    16,
   894  		commitBlock:        4,
   895  		pivotBlock:         nil,
   896  		expCanonicalBlocks: 18,
   897  		expSidechainBlocks: 0,
   898  		expFrozen:          3,
   899  		expHeadHeader:      18,
   900  		expHeadFastBlock:   18,
   901  		expHeadBlock:       4,
   902  	}, snapshots)
   903  }
   904  
   905  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
   906  // side chain, where a recent block - older than the ancient limit - was already
   907  // committed to disk and then the process crashed. In this test scenario the side
   908  // chain is below the committed block. In this case we expect the canonical chain
   909  // to be rolled back to the committed block, with everything afterwads deleted;
   910  // the side chain completely nuked by the freezer.
   911  func TestLongOldForkedDeepRepair(t *testing.T)              { testLongOldForkedDeepRepair(t, false) }
   912  func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) }
   913  
   914  func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) {
   915  	// Chain:
   916  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
   917  	//   └->S1->S2->S3
   918  	//
   919  	// Frozen:
   920  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   921  	//
   922  	// Commit: G, C4
   923  	// Pivot : none
   924  	//
   925  	// CRASH
   926  	//
   927  	// ------------------------------
   928  	//
   929  	// Expected in freezer:
   930  	//   G->C1->C2->C3->C4
   931  	//
   932  	// Expected in leveldb: none
   933  	//
   934  	// Expected head header    : C4
   935  	// Expected head fast block: C4
   936  	// Expected head block     : C4
   937  	testRepair(t, &rewindTest{
   938  		canonicalBlocks:    24,
   939  		sidechainBlocks:    3,
   940  		freezeThreshold:    16,
   941  		commitBlock:        4,
   942  		pivotBlock:         nil,
   943  		expCanonicalBlocks: 4,
   944  		expSidechainBlocks: 0,
   945  		expFrozen:          5,
   946  		expHeadHeader:      4,
   947  		expHeadFastBlock:   4,
   948  		expHeadBlock:       4,
   949  	}, snapshots)
   950  }
   951  
   952  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
   953  // side chain, where the fast sync pivot point - newer than the ancient limit -
   954  // was already committed to disk and then the process crashed. In this test scenario
   955  // the side chain is below the committed block. In this case we expect the chain
   956  // to be rolled back to the committed block, with everything afterwads kept as
   957  // fast sync data; the side chain completely nuked by the freezer.
   958  func TestLongOldForkedSnapSyncedShallowRepair(t *testing.T) {
   959  	testLongOldForkedSnapSyncedShallowRepair(t, false)
   960  }
   961  func TestLongOldForkedSnapSyncedShallowRepairWithSnapshots(t *testing.T) {
   962  	testLongOldForkedSnapSyncedShallowRepair(t, true)
   963  }
   964  
   965  func testLongOldForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) {
   966  	// Chain:
   967  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
   968  	//   └->S1->S2->S3
   969  	//
   970  	// Frozen:
   971  	//   G->C1->C2
   972  	//
   973  	// Commit: G, C4
   974  	// Pivot : C4
   975  	//
   976  	// CRASH
   977  	//
   978  	// ------------------------------
   979  	//
   980  	// Expected in freezer:
   981  	//   G->C1->C2
   982  	//
   983  	// Expected in leveldb:
   984  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
   985  	//
   986  	// Expected head header    : C18
   987  	// Expected head fast block: C18
   988  	// Expected head block     : C4
   989  	testRepair(t, &rewindTest{
   990  		canonicalBlocks:    18,
   991  		sidechainBlocks:    3,
   992  		freezeThreshold:    16,
   993  		commitBlock:        4,
   994  		pivotBlock:         uint64ptr(4),
   995  		expCanonicalBlocks: 18,
   996  		expSidechainBlocks: 0,
   997  		expFrozen:          3,
   998  		expHeadHeader:      18,
   999  		expHeadFastBlock:   18,
  1000  		expHeadBlock:       4,
  1001  	}, snapshots)
  1002  }
  1003  
  1004  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1005  // side chain, where the fast sync pivot point - older than the ancient limit -
  1006  // was already committed to disk and then the process crashed. In this test scenario
  1007  // the side chain is below the committed block. In this case we expect the canonical
  1008  // chain to be rolled back to the committed block, with everything afterwads deleted;
  1009  // the side chain completely nuked by the freezer.
  1010  func TestLongOldForkedSnapSyncedDeepRepair(t *testing.T) {
  1011  	testLongOldForkedSnapSyncedDeepRepair(t, false)
  1012  }
  1013  func TestLongOldForkedSnapSyncedDeepRepairWithSnapshots(t *testing.T) {
  1014  	testLongOldForkedSnapSyncedDeepRepair(t, true)
  1015  }
  1016  
  1017  func testLongOldForkedSnapSyncedDeepRepair(t *testing.T, snapshots bool) {
  1018  	// Chain:
  1019  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1020  	//   └->S1->S2->S3
  1021  	//
  1022  	// Frozen:
  1023  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1024  	//
  1025  	// Commit: G, C4
  1026  	// Pivot : C4
  1027  	//
  1028  	// CRASH
  1029  	//
  1030  	// ------------------------------
  1031  	//
  1032  	// Expected in freezer:
  1033  	//   G->C1->C2->C3->C4
  1034  	//
  1035  	// Expected in leveldb: none
  1036  	//
  1037  	// Expected head header    : C4
  1038  	// Expected head fast block: C4
  1039  	// Expected head block     : C4
  1040  	testRepair(t, &rewindTest{
  1041  		canonicalBlocks:    24,
  1042  		sidechainBlocks:    3,
  1043  		freezeThreshold:    16,
  1044  		commitBlock:        4,
  1045  		pivotBlock:         uint64ptr(4),
  1046  		expCanonicalBlocks: 4,
  1047  		expSidechainBlocks: 0,
  1048  		expFrozen:          5,
  1049  		expHeadHeader:      4,
  1050  		expHeadFastBlock:   4,
  1051  		expHeadBlock:       4,
  1052  	}, snapshots)
  1053  }
  1054  
  1055  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1056  // side chain, where the fast sync pivot point - older than the ancient limit -
  1057  // was not yet committed, but the process crashed. In this test scenario the side
  1058  // chain is below the committed block. In this case we expect the chain to detect
  1059  // that it was fast syncing and not delete anything. The side chain is completely
  1060  // nuked by the freezer.
  1061  func TestLongOldForkedSnapSyncingShallowRepair(t *testing.T) {
  1062  	testLongOldForkedSnapSyncingShallowRepair(t, false)
  1063  }
  1064  func TestLongOldForkedSnapSyncingShallowRepairWithSnapshots(t *testing.T) {
  1065  	testLongOldForkedSnapSyncingShallowRepair(t, true)
  1066  }
  1067  
  1068  func testLongOldForkedSnapSyncingShallowRepair(t *testing.T, snapshots bool) {
  1069  	// Chain:
  1070  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1071  	//   └->S1->S2->S3
  1072  	//
  1073  	// Frozen:
  1074  	//   G->C1->C2
  1075  	//
  1076  	// Commit: G
  1077  	// Pivot : C4
  1078  	//
  1079  	// CRASH
  1080  	//
  1081  	// ------------------------------
  1082  	//
  1083  	// Expected in freezer:
  1084  	//   G->C1->C2
  1085  	//
  1086  	// Expected in leveldb:
  1087  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1088  	//
  1089  	// Expected head header    : C18
  1090  	// Expected head fast block: C18
  1091  	// Expected head block     : G
  1092  	testRepair(t, &rewindTest{
  1093  		canonicalBlocks:    18,
  1094  		sidechainBlocks:    3,
  1095  		freezeThreshold:    16,
  1096  		commitBlock:        0,
  1097  		pivotBlock:         uint64ptr(4),
  1098  		expCanonicalBlocks: 18,
  1099  		expSidechainBlocks: 0,
  1100  		expFrozen:          3,
  1101  		expHeadHeader:      18,
  1102  		expHeadFastBlock:   18,
  1103  		expHeadBlock:       0,
  1104  	}, snapshots)
  1105  }
  1106  
  1107  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1108  // side chain, where the fast sync pivot point - older than the ancient limit -
  1109  // was not yet committed, but the process crashed. In this test scenario the side
  1110  // chain is below the committed block. In this case we expect the chain to detect
  1111  // that it was fast syncing and not delete anything. The side chain is completely
  1112  // nuked by the freezer.
  1113  func TestLongOldForkedSnapSyncingDeepRepair(t *testing.T) {
  1114  	testLongOldForkedSnapSyncingDeepRepair(t, false)
  1115  }
  1116  func TestLongOldForkedSnapSyncingDeepRepairWithSnapshots(t *testing.T) {
  1117  	testLongOldForkedSnapSyncingDeepRepair(t, true)
  1118  }
  1119  
  1120  func testLongOldForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
  1121  	// Chain:
  1122  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1123  	//   └->S1->S2->S3
  1124  	//
  1125  	// Frozen:
  1126  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1127  	//
  1128  	// Commit: G
  1129  	// Pivot : C4
  1130  	//
  1131  	// CRASH
  1132  	//
  1133  	// ------------------------------
  1134  	//
  1135  	// Expected in freezer:
  1136  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1137  	//
  1138  	// Expected in leveldb:
  1139  	//   C8)->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24
  1140  	//
  1141  	// Expected head header    : C24
  1142  	// Expected head fast block: C24
  1143  	// Expected head block     : G
  1144  	testRepair(t, &rewindTest{
  1145  		canonicalBlocks:    24,
  1146  		sidechainBlocks:    3,
  1147  		freezeThreshold:    16,
  1148  		commitBlock:        0,
  1149  		pivotBlock:         uint64ptr(4),
  1150  		expCanonicalBlocks: 24,
  1151  		expSidechainBlocks: 0,
  1152  		expFrozen:          9,
  1153  		expHeadHeader:      24,
  1154  		expHeadFastBlock:   24,
  1155  		expHeadBlock:       0,
  1156  	}, snapshots)
  1157  }
  1158  
  1159  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1160  // side chain, where a recent block - newer than the ancient limit - was already
  1161  // committed to disk and then the process crashed. In this test scenario the side
  1162  // chain is above the committed block. In this case we expect the chain to be
  1163  // rolled back to the committed block, with everything afterwads kept as fast
  1164  // sync data; the side chain completely nuked by the freezer.
  1165  func TestLongNewerForkedShallowRepair(t *testing.T) {
  1166  	testLongNewerForkedShallowRepair(t, false)
  1167  }
  1168  func TestLongNewerForkedShallowRepairWithSnapshots(t *testing.T) {
  1169  	testLongNewerForkedShallowRepair(t, true)
  1170  }
  1171  
  1172  func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) {
  1173  	// Chain:
  1174  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1175  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1176  	//
  1177  	// Frozen:
  1178  	//   G->C1->C2
  1179  	//
  1180  	// Commit: G, C4
  1181  	// Pivot : none
  1182  	//
  1183  	// CRASH
  1184  	//
  1185  	// ------------------------------
  1186  	//
  1187  	// Expected in freezer:
  1188  	//   G->C1->C2
  1189  	//
  1190  	// Expected in leveldb:
  1191  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1192  	//
  1193  	// Expected head header    : C18
  1194  	// Expected head fast block: C18
  1195  	// Expected head block     : C4
  1196  	testRepair(t, &rewindTest{
  1197  		canonicalBlocks:    18,
  1198  		sidechainBlocks:    12,
  1199  		freezeThreshold:    16,
  1200  		commitBlock:        4,
  1201  		pivotBlock:         nil,
  1202  		expCanonicalBlocks: 18,
  1203  		expSidechainBlocks: 0,
  1204  		expFrozen:          3,
  1205  		expHeadHeader:      18,
  1206  		expHeadFastBlock:   18,
  1207  		expHeadBlock:       4,
  1208  	}, snapshots)
  1209  }
  1210  
  1211  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1212  // side chain, where a recent block - older than the ancient limit - was already
  1213  // committed to disk and then the process crashed. In this test scenario the side
  1214  // chain is above the committed block. In this case we expect the canonical chain
  1215  // to be rolled back to the committed block, with everything afterwads deleted;
  1216  // the side chain completely nuked by the freezer.
  1217  func TestLongNewerForkedDeepRepair(t *testing.T)              { testLongNewerForkedDeepRepair(t, false) }
  1218  func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) }
  1219  
  1220  func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) {
  1221  	// Chain:
  1222  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1223  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1224  	//
  1225  	// Frozen:
  1226  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1227  	//
  1228  	// Commit: G, C4
  1229  	// Pivot : none
  1230  	//
  1231  	// CRASH
  1232  	//
  1233  	// ------------------------------
  1234  	//
  1235  	// Expected in freezer:
  1236  	//   G->C1->C2->C3->C4
  1237  	//
  1238  	// Expected in leveldb: none
  1239  	//
  1240  	// Expected head header    : C4
  1241  	// Expected head fast block: C4
  1242  	// Expected head block     : C4
  1243  	testRepair(t, &rewindTest{
  1244  		canonicalBlocks:    24,
  1245  		sidechainBlocks:    12,
  1246  		freezeThreshold:    16,
  1247  		commitBlock:        4,
  1248  		pivotBlock:         nil,
  1249  		expCanonicalBlocks: 4,
  1250  		expSidechainBlocks: 0,
  1251  		expFrozen:          5,
  1252  		expHeadHeader:      4,
  1253  		expHeadFastBlock:   4,
  1254  		expHeadBlock:       4,
  1255  	}, snapshots)
  1256  }
  1257  
  1258  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1259  // side chain, where the fast sync pivot point - newer than the ancient limit -
  1260  // was already committed to disk and then the process crashed. In this test scenario
  1261  // the side chain is above the committed block. In this case we expect the chain
  1262  // to be rolled back to the committed block, with everything afterwads kept as fast
  1263  // sync data; the side chain completely nuked by the freezer.
  1264  func TestLongNewerForkedSnapSyncedShallowRepair(t *testing.T) {
  1265  	testLongNewerForkedSnapSyncedShallowRepair(t, false)
  1266  }
  1267  func TestLongNewerForkedSnapSyncedShallowRepairWithSnapshots(t *testing.T) {
  1268  	testLongNewerForkedSnapSyncedShallowRepair(t, true)
  1269  }
  1270  
  1271  func testLongNewerForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) {
  1272  	// Chain:
  1273  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1274  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1275  	//
  1276  	// Frozen:
  1277  	//   G->C1->C2
  1278  	//
  1279  	// Commit: G, C4
  1280  	// Pivot : C4
  1281  	//
  1282  	// CRASH
  1283  	//
  1284  	// ------------------------------
  1285  	//
  1286  	// Expected in freezer:
  1287  	//   G->C1->C2
  1288  	//
  1289  	// Expected in leveldb:
  1290  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1291  	//
  1292  	// Expected head header    : C18
  1293  	// Expected head fast block: C18
  1294  	// Expected head block     : C4
  1295  	testRepair(t, &rewindTest{
  1296  		canonicalBlocks:    18,
  1297  		sidechainBlocks:    12,
  1298  		freezeThreshold:    16,
  1299  		commitBlock:        4,
  1300  		pivotBlock:         uint64ptr(4),
  1301  		expCanonicalBlocks: 18,
  1302  		expSidechainBlocks: 0,
  1303  		expFrozen:          3,
  1304  		expHeadHeader:      18,
  1305  		expHeadFastBlock:   18,
  1306  		expHeadBlock:       4,
  1307  	}, snapshots)
  1308  }
  1309  
  1310  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1311  // side chain, where the fast sync pivot point - older than the ancient limit -
  1312  // was already committed to disk and then the process crashed. In this test scenario
  1313  // the side chain is above the committed block. In this case we expect the canonical
  1314  // chain to be rolled back to the committed block, with everything afterwads deleted;
  1315  // the side chain completely nuked by the freezer.
  1316  func TestLongNewerForkedSnapSyncedDeepRepair(t *testing.T) {
  1317  	testLongNewerForkedSnapSyncedDeepRepair(t, false)
  1318  }
  1319  func TestLongNewerForkedSnapSyncedDeepRepairWithSnapshots(t *testing.T) {
  1320  	testLongNewerForkedSnapSyncedDeepRepair(t, true)
  1321  }
  1322  
  1323  func testLongNewerForkedSnapSyncedDeepRepair(t *testing.T, snapshots bool) {
  1324  	// Chain:
  1325  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1326  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1327  	//
  1328  	// Frozen:
  1329  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1330  	//
  1331  	// Commit: G, C4
  1332  	// Pivot : C4
  1333  	//
  1334  	// CRASH
  1335  	//
  1336  	// ------------------------------
  1337  	//
  1338  	// Expected in freezer:
  1339  	//   G->C1->C2->C3->C4
  1340  	//
  1341  	// Expected in leveldb: none
  1342  	//
  1343  	// Expected head header    : C4
  1344  	// Expected head fast block: C4
  1345  	// Expected head block     : C4
  1346  	testRepair(t, &rewindTest{
  1347  		canonicalBlocks:    24,
  1348  		sidechainBlocks:    12,
  1349  		freezeThreshold:    16,
  1350  		commitBlock:        4,
  1351  		pivotBlock:         uint64ptr(4),
  1352  		expCanonicalBlocks: 4,
  1353  		expSidechainBlocks: 0,
  1354  		expFrozen:          5,
  1355  		expHeadHeader:      4,
  1356  		expHeadFastBlock:   4,
  1357  		expHeadBlock:       4,
  1358  	}, snapshots)
  1359  }
  1360  
  1361  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1362  // side chain, where the fast sync pivot point - older than the ancient limit -
  1363  // was not yet committed, but the process crashed. In this test scenario the side
  1364  // chain is above the committed block. In this case we expect the chain to detect
  1365  // that it was fast syncing and not delete anything. The side chain is completely
  1366  // nuked by the freezer.
  1367  func TestLongNewerForkedSnapSyncingShallowRepair(t *testing.T) {
  1368  	testLongNewerForkedSnapSyncingShallowRepair(t, false)
  1369  }
  1370  func TestLongNewerForkedSnapSyncingShallowRepairWithSnapshots(t *testing.T) {
  1371  	testLongNewerForkedSnapSyncingShallowRepair(t, true)
  1372  }
  1373  
  1374  func testLongNewerForkedSnapSyncingShallowRepair(t *testing.T, snapshots bool) {
  1375  	// Chain:
  1376  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1377  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1378  	//
  1379  	// Frozen:
  1380  	//   G->C1->C2
  1381  	//
  1382  	// Commit: G
  1383  	// Pivot : C4
  1384  	//
  1385  	// CRASH
  1386  	//
  1387  	// ------------------------------
  1388  	//
  1389  	// Expected in freezer:
  1390  	//   G->C1->C2
  1391  	//
  1392  	// Expected in leveldb:
  1393  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1394  	//
  1395  	// Expected head header    : C18
  1396  	// Expected head fast block: C18
  1397  	// Expected head block     : G
  1398  	testRepair(t, &rewindTest{
  1399  		canonicalBlocks:    18,
  1400  		sidechainBlocks:    12,
  1401  		freezeThreshold:    16,
  1402  		commitBlock:        0,
  1403  		pivotBlock:         uint64ptr(4),
  1404  		expCanonicalBlocks: 18,
  1405  		expSidechainBlocks: 0,
  1406  		expFrozen:          3,
  1407  		expHeadHeader:      18,
  1408  		expHeadFastBlock:   18,
  1409  		expHeadBlock:       0,
  1410  	}, snapshots)
  1411  }
  1412  
  1413  // Tests a recovery for a long canonical chain with frozen blocks and a shorter
  1414  // side chain, where the fast sync pivot point - older than the ancient limit -
  1415  // was not yet committed, but the process crashed. In this test scenario the side
  1416  // chain is above the committed block. In this case we expect the chain to detect
  1417  // that it was fast syncing and not delete anything. The side chain is completely
  1418  // nuked by the freezer.
  1419  func TestLongNewerForkedSnapSyncingDeepRepair(t *testing.T) {
  1420  	testLongNewerForkedSnapSyncingDeepRepair(t, false)
  1421  }
  1422  func TestLongNewerForkedSnapSyncingDeepRepairWithSnapshots(t *testing.T) {
  1423  	testLongNewerForkedSnapSyncingDeepRepair(t, true)
  1424  }
  1425  
  1426  func testLongNewerForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
  1427  	// Chain:
  1428  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1429  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
  1430  	//
  1431  	// Frozen:
  1432  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1433  	//
  1434  	// Commit: G
  1435  	// Pivot : C4
  1436  	//
  1437  	// CRASH
  1438  	//
  1439  	// ------------------------------
  1440  	//
  1441  	// Expected in freezer:
  1442  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1443  	//
  1444  	// Expected in leveldb:
  1445  	//   C8)->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24
  1446  	//
  1447  	// Expected head header    : C24
  1448  	// Expected head fast block: C24
  1449  	// Expected head block     : G
  1450  	testRepair(t, &rewindTest{
  1451  		canonicalBlocks:    24,
  1452  		sidechainBlocks:    12,
  1453  		freezeThreshold:    16,
  1454  		commitBlock:        0,
  1455  		pivotBlock:         uint64ptr(4),
  1456  		expCanonicalBlocks: 24,
  1457  		expSidechainBlocks: 0,
  1458  		expFrozen:          9,
  1459  		expHeadHeader:      24,
  1460  		expHeadFastBlock:   24,
  1461  		expHeadBlock:       0,
  1462  	}, snapshots)
  1463  }
  1464  
  1465  // Tests a recovery for a long canonical chain with frozen blocks and a longer side
  1466  // chain, where a recent block - newer than the ancient limit - was already committed
  1467  // to disk and then the process crashed. In this case we expect the chain to be
  1468  // rolled back to the committed block, with everything afterwads kept as fast sync
  1469  // data. The side chain completely nuked by the freezer.
  1470  func TestLongReorgedShallowRepair(t *testing.T)              { testLongReorgedShallowRepair(t, false) }
  1471  func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) }
  1472  
  1473  func testLongReorgedShallowRepair(t *testing.T, snapshots bool) {
  1474  	// Chain:
  1475  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1476  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1477  	//
  1478  	// Frozen:
  1479  	//   G->C1->C2
  1480  	//
  1481  	// Commit: G, C4
  1482  	// Pivot : none
  1483  	//
  1484  	// CRASH
  1485  	//
  1486  	// ------------------------------
  1487  	//
  1488  	// Expected in freezer:
  1489  	//   G->C1->C2
  1490  	//
  1491  	// Expected in leveldb:
  1492  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1493  	//
  1494  	// Expected head header    : C18
  1495  	// Expected head fast block: C18
  1496  	// Expected head block     : C4
  1497  	testRepair(t, &rewindTest{
  1498  		canonicalBlocks:    18,
  1499  		sidechainBlocks:    26,
  1500  		freezeThreshold:    16,
  1501  		commitBlock:        4,
  1502  		pivotBlock:         nil,
  1503  		expCanonicalBlocks: 18,
  1504  		expSidechainBlocks: 0,
  1505  		expFrozen:          3,
  1506  		expHeadHeader:      18,
  1507  		expHeadFastBlock:   18,
  1508  		expHeadBlock:       4,
  1509  	}, snapshots)
  1510  }
  1511  
  1512  // Tests a recovery for a long canonical chain with frozen blocks and a longer side
  1513  // chain, where a recent block - older than the ancient limit - was already committed
  1514  // to disk and then the process crashed. In this case we expect the canonical chains
  1515  // to be rolled back to the committed block, with everything afterwads deleted. The
  1516  // side chain completely nuked by the freezer.
  1517  func TestLongReorgedDeepRepair(t *testing.T)              { testLongReorgedDeepRepair(t, false) }
  1518  func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) }
  1519  
  1520  func testLongReorgedDeepRepair(t *testing.T, snapshots bool) {
  1521  	// Chain:
  1522  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1523  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1524  	//
  1525  	// Frozen:
  1526  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1527  	//
  1528  	// Commit: G, C4
  1529  	// Pivot : none
  1530  	//
  1531  	// CRASH
  1532  	//
  1533  	// ------------------------------
  1534  	//
  1535  	// Expected in freezer:
  1536  	//   G->C1->C2->C3->C4
  1537  	//
  1538  	// Expected in leveldb: none
  1539  	//
  1540  	// Expected head header    : C4
  1541  	// Expected head fast block: C4
  1542  	// Expected head block     : C4
  1543  	testRepair(t, &rewindTest{
  1544  		canonicalBlocks:    24,
  1545  		sidechainBlocks:    26,
  1546  		freezeThreshold:    16,
  1547  		commitBlock:        4,
  1548  		pivotBlock:         nil,
  1549  		expCanonicalBlocks: 4,
  1550  		expSidechainBlocks: 0,
  1551  		expFrozen:          5,
  1552  		expHeadHeader:      4,
  1553  		expHeadFastBlock:   4,
  1554  		expHeadBlock:       4,
  1555  	}, snapshots)
  1556  }
  1557  
  1558  // Tests a recovery for a long canonical chain with frozen blocks and a longer
  1559  // side chain, where the fast sync pivot point - newer than the ancient limit -
  1560  // was already committed to disk and then the process crashed. In this case we
  1561  // expect the chain to be rolled back to the committed block, with everything
  1562  // afterwads kept as fast sync data. The side chain completely nuked by the
  1563  // freezer.
  1564  func TestLongReorgedSnapSyncedShallowRepair(t *testing.T) {
  1565  	testLongReorgedSnapSyncedShallowRepair(t, false)
  1566  }
  1567  func TestLongReorgedSnapSyncedShallowRepairWithSnapshots(t *testing.T) {
  1568  	testLongReorgedSnapSyncedShallowRepair(t, true)
  1569  }
  1570  
  1571  func testLongReorgedSnapSyncedShallowRepair(t *testing.T, snapshots bool) {
  1572  	// Chain:
  1573  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1574  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1575  	//
  1576  	// Frozen:
  1577  	//   G->C1->C2
  1578  	//
  1579  	// Commit: G, C4
  1580  	// Pivot : C4
  1581  	//
  1582  	// CRASH
  1583  	//
  1584  	// ------------------------------
  1585  	//
  1586  	// Expected in freezer:
  1587  	//   G->C1->C2
  1588  	//
  1589  	// Expected in leveldb:
  1590  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1591  	//
  1592  	// Expected head header    : C18
  1593  	// Expected head fast block: C18
  1594  	// Expected head block     : C4
  1595  	testRepair(t, &rewindTest{
  1596  		canonicalBlocks:    18,
  1597  		sidechainBlocks:    26,
  1598  		freezeThreshold:    16,
  1599  		commitBlock:        4,
  1600  		pivotBlock:         uint64ptr(4),
  1601  		expCanonicalBlocks: 18,
  1602  		expSidechainBlocks: 0,
  1603  		expFrozen:          3,
  1604  		expHeadHeader:      18,
  1605  		expHeadFastBlock:   18,
  1606  		expHeadBlock:       4,
  1607  	}, snapshots)
  1608  }
  1609  
  1610  // Tests a recovery for a long canonical chain with frozen blocks and a longer
  1611  // side chain, where the fast sync pivot point - older than the ancient limit -
  1612  // was already committed to disk and then the process crashed. In this case we
  1613  // expect the canonical chains to be rolled back to the committed block, with
  1614  // everything afterwads deleted. The side chain completely nuked by the freezer.
  1615  func TestLongReorgedSnapSyncedDeepRepair(t *testing.T) {
  1616  	testLongReorgedSnapSyncedDeepRepair(t, false)
  1617  }
  1618  func TestLongReorgedSnapSyncedDeepRepairWithSnapshots(t *testing.T) {
  1619  	testLongReorgedSnapSyncedDeepRepair(t, true)
  1620  }
  1621  
  1622  func testLongReorgedSnapSyncedDeepRepair(t *testing.T, snapshots bool) {
  1623  	// Chain:
  1624  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1625  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1626  	//
  1627  	// Frozen:
  1628  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1629  	//
  1630  	// Commit: G, C4
  1631  	// Pivot : C4
  1632  	//
  1633  	// CRASH
  1634  	//
  1635  	// ------------------------------
  1636  	//
  1637  	// Expected in freezer:
  1638  	//   G->C1->C2->C3->C4
  1639  	//
  1640  	// Expected in leveldb: none
  1641  	//
  1642  	// Expected head header    : C4
  1643  	// Expected head fast block: C4
  1644  	// Expected head block     : C4
  1645  	testRepair(t, &rewindTest{
  1646  		canonicalBlocks:    24,
  1647  		sidechainBlocks:    26,
  1648  		freezeThreshold:    16,
  1649  		commitBlock:        4,
  1650  		pivotBlock:         uint64ptr(4),
  1651  		expCanonicalBlocks: 4,
  1652  		expSidechainBlocks: 0,
  1653  		expFrozen:          5,
  1654  		expHeadHeader:      4,
  1655  		expHeadFastBlock:   4,
  1656  		expHeadBlock:       4,
  1657  	}, snapshots)
  1658  }
  1659  
  1660  // Tests a recovery for a long canonical chain with frozen blocks and a longer
  1661  // side chain, where the fast sync pivot point - newer than the ancient limit -
  1662  // was not yet committed, but the process crashed. In this case we expect the
  1663  // chain to detect that it was fast syncing and not delete anything, since we
  1664  // can just pick up directly where we left off.
  1665  func TestLongReorgedSnapSyncingShallowRepair(t *testing.T) {
  1666  	testLongReorgedSnapSyncingShallowRepair(t, false)
  1667  }
  1668  func TestLongReorgedSnapSyncingShallowRepairWithSnapshots(t *testing.T) {
  1669  	testLongReorgedSnapSyncingShallowRepair(t, true)
  1670  }
  1671  
  1672  func testLongReorgedSnapSyncingShallowRepair(t *testing.T, snapshots bool) {
  1673  	// Chain:
  1674  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
  1675  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1676  	//
  1677  	// Frozen:
  1678  	//   G->C1->C2
  1679  	//
  1680  	// Commit: G
  1681  	// Pivot : C4
  1682  	//
  1683  	// CRASH
  1684  	//
  1685  	// ------------------------------
  1686  	//
  1687  	// Expected in freezer:
  1688  	//   G->C1->C2
  1689  	//
  1690  	// Expected in leveldb:
  1691  	//   C2)->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18
  1692  	//
  1693  	// Expected head header    : C18
  1694  	// Expected head fast block: C18
  1695  	// Expected head block     : G
  1696  	testRepair(t, &rewindTest{
  1697  		canonicalBlocks:    18,
  1698  		sidechainBlocks:    26,
  1699  		freezeThreshold:    16,
  1700  		commitBlock:        0,
  1701  		pivotBlock:         uint64ptr(4),
  1702  		expCanonicalBlocks: 18,
  1703  		expSidechainBlocks: 0,
  1704  		expFrozen:          3,
  1705  		expHeadHeader:      18,
  1706  		expHeadFastBlock:   18,
  1707  		expHeadBlock:       0,
  1708  	}, snapshots)
  1709  }
  1710  
  1711  // Tests a recovery for a long canonical chain with frozen blocks and a longer
  1712  // side chain, where the fast sync pivot point - older than the ancient limit -
  1713  // was not yet committed, but the process crashed. In this case we expect the
  1714  // chain to detect that it was fast syncing and not delete anything, since we
  1715  // can just pick up directly where we left off.
  1716  func TestLongReorgedSnapSyncingDeepRepair(t *testing.T) {
  1717  	testLongReorgedSnapSyncingDeepRepair(t, false)
  1718  }
  1719  func TestLongReorgedSnapSyncingDeepRepairWithSnapshots(t *testing.T) {
  1720  	testLongReorgedSnapSyncingDeepRepair(t, true)
  1721  }
  1722  
  1723  func testLongReorgedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
  1724  	// Chain:
  1725  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
  1726  	//   └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
  1727  	//
  1728  	// Frozen:
  1729  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1730  	//
  1731  	// Commit: G
  1732  	// Pivot : C4
  1733  	//
  1734  	// CRASH
  1735  	//
  1736  	// ------------------------------
  1737  	//
  1738  	// Expected in freezer:
  1739  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
  1740  	//
  1741  	// Expected in leveldb:
  1742  	//   C8)->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24
  1743  	//
  1744  	// Expected head header    : C24
  1745  	// Expected head fast block: C24
  1746  	// Expected head block     : G
  1747  	testRepair(t, &rewindTest{
  1748  		canonicalBlocks:    24,
  1749  		sidechainBlocks:    26,
  1750  		freezeThreshold:    16,
  1751  		commitBlock:        0,
  1752  		pivotBlock:         uint64ptr(4),
  1753  		expCanonicalBlocks: 24,
  1754  		expSidechainBlocks: 0,
  1755  		expFrozen:          9,
  1756  		expHeadHeader:      24,
  1757  		expHeadFastBlock:   24,
  1758  		expHeadBlock:       0,
  1759  	}, snapshots)
  1760  }
  1761  
  1762  var (
  1763  	testKey1, _  = crypto.GenerateKey()
  1764  	testAddress1 = crypto.PubkeyToAddress(testKey1.PublicKey)
  1765  
  1766  	testKey2, _  = crypto.GenerateKey()
  1767  	testAddress2 = crypto.PubkeyToAddress(testKey2.PublicKey) //nolint:unused,varcheck
  1768  )
  1769  
  1770  func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
  1771  	t.Skip("need to add a proper signer for Bor consensus")
  1772  
  1773  	// It's hard to follow the test case, visualize the input
  1774  	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
  1775  	// fmt.Println(tt.dump(true))
  1776  
  1777  	// Create a temporary persistent database
  1778  	datadir, err := ioutil.TempDir("", "")
  1779  	if err != nil {
  1780  		t.Fatalf("Failed to create temporary datadir: %v", err)
  1781  	}
  1782  	os.RemoveAll(datadir)
  1783  
  1784  	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
  1785  	if err != nil {
  1786  		t.Fatalf("Failed to create persistent database: %v", err)
  1787  	}
  1788  	defer db.Close() // Might double close, should be fine
  1789  
  1790  	chainConfig := params.BorUnittestChainConfig
  1791  
  1792  	ctrl := gomock.NewController(t)
  1793  	defer ctrl.Finish()
  1794  
  1795  	ethAPIMock := api.NewMockCaller(ctrl)
  1796  	ethAPIMock.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
  1797  
  1798  	spanner := bor.NewMockSpanner(ctrl)
  1799  	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{
  1800  		{
  1801  			ID:               0,
  1802  			Address:          miner.TestBankAddress,
  1803  			VotingPower:      100,
  1804  			ProposerPriority: 0,
  1805  		},
  1806  	}, nil).AnyTimes()
  1807  
  1808  	heimdallClientMock := mocks.NewMockIHeimdallClient(ctrl)
  1809  	heimdallClientMock.EXPECT().Close().Times(1)
  1810  
  1811  	contractMock := bor.NewMockGenesisContract(ctrl)
  1812  
  1813  	engine := miner.NewFakeBor(t, db, chainConfig, ethAPIMock, spanner, heimdallClientMock, contractMock)
  1814  	defer engine.Close()
  1815  
  1816  	chainConfig.LondonBlock = big.NewInt(0)
  1817  
  1818  	_, back, closeFn := miner.NewTestWorker(t, chainConfig, engine, db, 0, 0, 0, 0)
  1819  	defer closeFn()
  1820  
  1821  	genesis := back.BlockChain().Genesis()
  1822  
  1823  	// If sidechain blocks are needed, make a light chain and import it
  1824  	var sideblocks types.Blocks
  1825  	if tt.sidechainBlocks > 0 {
  1826  		sideblocks, _ = core.GenerateChain(params.BorUnittestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, func(i int, b *core.BlockGen) {
  1827  			b.SetCoinbase(testAddress1)
  1828  
  1829  			if bor.IsSprintStart(b.Number().Uint64(), params.BorUnittestChainConfig.Bor.CalculateSprint(b.Number().Uint64())) {
  1830  				b.SetExtra(back.Genesis.ExtraData)
  1831  			} else {
  1832  				b.SetExtra(make([]byte, 32+crypto.SignatureLength))
  1833  			}
  1834  		})
  1835  		if _, err := back.BlockChain().InsertChain(sideblocks); err != nil {
  1836  			t.Fatalf("Failed to import side chain: %v", err)
  1837  		}
  1838  	}
  1839  
  1840  	canonblocks, _ := core.GenerateChain(params.BorUnittestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, func(i int, b *core.BlockGen) {
  1841  		b.SetCoinbase(miner.TestBankAddress)
  1842  		b.SetDifficulty(big.NewInt(1000000))
  1843  
  1844  		if bor.IsSprintStart(b.Number().Uint64(), params.BorUnittestChainConfig.Bor.CalculateSprint(b.Number().Uint64())) {
  1845  			b.SetExtra(back.Genesis.ExtraData)
  1846  		} else {
  1847  			b.SetExtra(make([]byte, 32+crypto.SignatureLength))
  1848  		}
  1849  	})
  1850  	if _, err := back.BlockChain().InsertChain(canonblocks[:tt.commitBlock]); err != nil {
  1851  		t.Fatalf("Failed to import canonical chain start: %v", err)
  1852  	}
  1853  	if tt.commitBlock > 0 {
  1854  		err = back.BlockChain().StateCache().TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
  1855  		if err != nil {
  1856  			t.Fatal("on trieDB.Commit", err)
  1857  		}
  1858  
  1859  		if snapshots {
  1860  			if err := back.BlockChain().Snaps().Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
  1861  				t.Fatalf("Failed to flatten snapshots: %v", err)
  1862  			}
  1863  		}
  1864  	}
  1865  
  1866  	if _, err := back.BlockChain().InsertChain(canonblocks[tt.commitBlock:]); err != nil {
  1867  		t.Fatalf("Failed to import canonical chain tail: %v", err)
  1868  	}
  1869  
  1870  	// Force run a freeze cycle
  1871  	type freezer interface {
  1872  		Freeze(threshold uint64) error
  1873  		Ancients() (uint64, error)
  1874  	}
  1875  
  1876  	db.(freezer).Freeze(tt.freezeThreshold)
  1877  
  1878  	// Set the simulated pivot block
  1879  	if tt.pivotBlock != nil {
  1880  		rawdb.WriteLastPivotNumber(db, *tt.pivotBlock)
  1881  	}
  1882  
  1883  	// Pull the plug on the database, simulating a hard crash
  1884  	db.Close()
  1885  
  1886  	// Start a new blockchain back up and see where the repair leads us
  1887  	db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
  1888  	if err != nil {
  1889  		t.Fatalf("Failed to reopen persistent database: %v", err)
  1890  	}
  1891  
  1892  	defer db.Close()
  1893  
  1894  	newChain, err := core.NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, nil)
  1895  	if err != nil {
  1896  		t.Fatalf("Failed to recreate chain: %v", err)
  1897  	}
  1898  
  1899  	defer newChain.Stop()
  1900  
  1901  	// Iterate over all the remaining blocks and ensure there are no gaps
  1902  	verifyNoGaps(t, newChain, true, canonblocks)
  1903  	verifyNoGaps(t, newChain, false, sideblocks)
  1904  	verifyCutoff(t, newChain, true, canonblocks, tt.expCanonicalBlocks)
  1905  	verifyCutoff(t, newChain, false, sideblocks, tt.expSidechainBlocks)
  1906  
  1907  	if head := newChain.CurrentHeader(); head.Number.Uint64() != tt.expHeadHeader {
  1908  		t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadHeader)
  1909  	}
  1910  
  1911  	if head := newChain.CurrentFastBlock(); head.NumberU64() != tt.expHeadFastBlock {
  1912  		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadFastBlock)
  1913  	}
  1914  
  1915  	if head := newChain.CurrentBlock(); head.NumberU64() != tt.expHeadBlock {
  1916  		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadBlock)
  1917  	}
  1918  
  1919  	if frozen, err := db.(freezer).Ancients(); err != nil {
  1920  		t.Errorf("Failed to retrieve ancient count: %v\n", err)
  1921  	} else if int(frozen) != tt.expFrozen {
  1922  		t.Errorf("Frozen block count mismatch: have %d, want %d", frozen, tt.expFrozen)
  1923  	}
  1924  }
  1925  
  1926  // TestIssue23496 tests scenario described in https://github.com/ethereum/go-ethereum/pull/23496#issuecomment-926393893
  1927  // Credits to @zzyalbert for finding the issue.
  1928  //
  1929  // Local chain owns these blocks:
  1930  // G  B1  B2  B3  B4
  1931  // B1: state committed
  1932  // B2: snapshot disk layer
  1933  // B3: state committed
  1934  // B4: head block
  1935  //
  1936  // Crash happens without fully persisting snapshot and in-memory states,
  1937  // chain rewinds itself to the B1 (skip B3 in order to recover snapshot)
  1938  // In this case the snapshot layer of B3 is not created because of existent
  1939  // state.
  1940  func TestIssue23496(t *testing.T) {
  1941  	// It's hard to follow the test case, visualize the input
  1942  	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
  1943  
  1944  	// Create a temporary persistent database
  1945  	datadir, err := ioutil.TempDir("", "")
  1946  	if err != nil {
  1947  		t.Fatalf("Failed to create temporary datadir: %v", err)
  1948  	}
  1949  
  1950  	os.RemoveAll(datadir)
  1951  
  1952  	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
  1953  	if err != nil {
  1954  		t.Fatalf("Failed to create persistent database: %v", err)
  1955  	}
  1956  
  1957  	defer db.Close() // Might double close, should be fine
  1958  
  1959  	// Initialize a fresh chain
  1960  	var (
  1961  		genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db)
  1962  		engine  = ethash.NewFullFaker()
  1963  		config  = &core.CacheConfig{
  1964  			TrieCleanLimit: 256,
  1965  			TrieDirtyLimit: 256,
  1966  			TrieTimeLimit:  5 * time.Minute,
  1967  			SnapshotLimit:  256,
  1968  			SnapshotWait:   true,
  1969  		}
  1970  	)
  1971  
  1972  	chain, err := core.NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, nil)
  1973  	if err != nil {
  1974  		t.Fatalf("Failed to create chain: %v", err)
  1975  	}
  1976  
  1977  	blocks, _ := core.GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), 4, func(i int, b *core.BlockGen) {
  1978  		b.SetCoinbase(common.Address{0x02})
  1979  		b.SetDifficulty(big.NewInt(1000000))
  1980  	})
  1981  
  1982  	// Insert block B1 and commit the state into disk
  1983  	if _, err := chain.InsertChain(blocks[:1]); err != nil {
  1984  		t.Fatalf("Failed to import canonical chain start: %v", err)
  1985  	}
  1986  
  1987  	err = chain.StateCache().TrieDB().Commit(blocks[0].Root(), true, nil)
  1988  	if err != nil {
  1989  		t.Fatal("on trieDB.Commit", err)
  1990  	}
  1991  
  1992  	// Insert block B2 and commit the snapshot into disk
  1993  	if _, err := chain.InsertChain(blocks[1:2]); err != nil {
  1994  		t.Fatalf("Failed to import canonical chain start: %v", err)
  1995  	}
  1996  
  1997  	if err := chain.Snaps().Cap(blocks[1].Root(), 0); err != nil {
  1998  		t.Fatalf("Failed to flatten snapshots: %v", err)
  1999  	}
  2000  
  2001  	// Insert block B3 and commit the state into disk
  2002  	if _, err := chain.InsertChain(blocks[2:3]); err != nil {
  2003  		t.Fatalf("Failed to import canonical chain start: %v", err)
  2004  	}
  2005  
  2006  	err = chain.StateCache().TrieDB().Commit(blocks[2].Root(), true, nil)
  2007  	if err != nil {
  2008  		t.Fatal("on trieDB.Commit", err)
  2009  	}
  2010  
  2011  	// Insert the remaining blocks
  2012  	if _, err := chain.InsertChain(blocks[3:]); err != nil {
  2013  		t.Fatalf("Failed to import canonical chain tail: %v", err)
  2014  	}
  2015  
  2016  	// Pull the plug on the database, simulating a hard crash
  2017  	db.Close()
  2018  
  2019  	// Start a new blockchain back up and see where the repair leads us
  2020  	db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
  2021  	if err != nil {
  2022  		t.Fatalf("Failed to reopen persistent database: %v", err)
  2023  	}
  2024  
  2025  	defer db.Close()
  2026  
  2027  	chain, err = core.NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, nil)
  2028  	if err != nil {
  2029  		t.Fatalf("Failed to recreate chain: %v", err)
  2030  	}
  2031  
  2032  	defer chain.Stop()
  2033  
  2034  	if head := chain.CurrentHeader(); head.Number.Uint64() != uint64(4) {
  2035  		t.Errorf("Head header mismatch: have %d, want %d", head.Number, 4)
  2036  	}
  2037  
  2038  	if head := chain.CurrentFastBlock(); head.NumberU64() != uint64(4) {
  2039  		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), uint64(4))
  2040  	}
  2041  
  2042  	if head := chain.CurrentBlock(); head.NumberU64() != uint64(1) {
  2043  		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), uint64(1))
  2044  	}
  2045  
  2046  	// Reinsert B2-B4
  2047  	if _, err := chain.InsertChain(blocks[1:]); err != nil {
  2048  		t.Fatalf("Failed to import canonical chain tail: %v", err)
  2049  	}
  2050  
  2051  	if head := chain.CurrentHeader(); head.Number.Uint64() != uint64(4) {
  2052  		t.Errorf("Head header mismatch: have %d, want %d", head.Number, 4)
  2053  	}
  2054  
  2055  	if head := chain.CurrentFastBlock(); head.NumberU64() != uint64(4) {
  2056  		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), uint64(4))
  2057  	}
  2058  
  2059  	if head := chain.CurrentBlock(); head.NumberU64() != uint64(4) {
  2060  		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), uint64(4))
  2061  	}
  2062  
  2063  	if layer := chain.Snapshots().Snapshot(blocks[2].Root()); layer == nil {
  2064  		t.Error("Failed to regenerate the snapshot of known state")
  2065  	}
  2066  }