github.com/yacovm/fabric@v2.0.0-alpha.0.20191128145320-c5d4087dc723+incompatible/core/ledger/kvledger/tests/v1x_test.go

/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package tests

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"testing"
	"time"

	protopeer "github.com/hyperledger/fabric-protos-go/peer"
	"github.com/hyperledger/fabric/common/ledger/testutil"
	"github.com/hyperledger/fabric/core/ledger/kvledger"
	"github.com/hyperledger/fabric/core/ledger/mock"
	"github.com/hyperledger/fabric/core/ledger/util/couchdb"
	"github.com/stretchr/testify/require"
)

// TestV11 tests that a ledgersData folder created by v1.1 can be used with future releases in a backward compatible way
// The test data was generated by v1.1 code https://github.com/hyperledger/fabric/blob/release-1.1/core/ledger/kvledger/tests/v11_generate_test.go#L22
func TestV11(t *testing.T) {
	env := newEnv(t)
	defer env.cleanup()

	ledgerFSRoot := env.initializer.Config.RootFSPath
	// pass false so that 'ledgersData' directory will not be created when unzipped to ledgerFSRoot
	require.NoError(t, testutil.Unzip("testdata/v11/sample_ledgers/ledgersData.zip", ledgerFSRoot, false))

	// verify init ledger panic and drop the corresponding db after each panic
	checkInitLedgerPanicAndDropDBs(t, env, ledgerFSRoot, nil)

	t.Logf("All stores are reformatted/dropped, now ledgermgmt initialization should not panic")
	env.initLedgerMgmt()

	h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
	dataHelper := &v1xSampleDataHelper{sampleDataVersion: "v1.1", t: t}

	dataHelper.verify(h1)
	dataHelper.verify(h2)

	env.closeAllLedgersAndDrop(rebuildableStatedb + rebuildableBlockIndex + rebuildableConfigHistory + rebuildableHistoryDB)
	h1, h2 = env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
	dataHelper.verify(h1)
	dataHelper.verify(h2)
}

// TestV13WithStateCouchdb tests that a ledgersData folder and couchdb data created by v1.3 can be read by latest fabric version in a backward compatible way
// The test data was generated by v1.3 code https://gerrit.hyperledger.org/r/#/c/fabric/+/34078/3/core/ledger/kvledger/tests/v13_generate_test.go@60
func TestV13WithStateCouchdb(t *testing.T) {
	env := newEnv(t)
	defer env.cleanup()

	ledgerFSRoot := env.initializer.Config.RootFSPath
	// pass false so that 'ledgersData' directory will not be created when unzipped to ledgerFSRoot
	require.NoError(t, testutil.Unzip("testdata/v13_statecouchdb/sample_ledgers/ledgersData.zip", ledgerFSRoot, false))

	// unzip couchdb data to prepare the mount dir
	couchdbDataUnzipDir := filepath.Join(ledgerFSRoot, "couchdbData")
	require.NoError(t, os.Mkdir(couchdbDataUnzipDir, os.ModePerm))
	require.NoError(t, testutil.Unzip("testdata/v13_statecouchdb/sample_ledgers/couchdbData.zip", couchdbDataUnzipDir, false))

	// prepare the local.d mount dir to overwrite the number of shards and nodes so that they match the couchdb data generated from v1.3
	localdHostDir := filepath.Join(ledgerFSRoot, "local.d")
	require.NoError(t, os.MkdirAll(localdHostDir, os.ModePerm))
	testutil.CopyDir("testdata/v13_statecouchdb/couchdb_etc/local.d", localdHostDir, true)

	// start couchdb using couchdbDataUnzipDir and localdHostDir as mount dirs
	couchAddress, cleanup := couchDBSetup(t, couchdbDataUnzipDir, localdHostDir)
	defer cleanup()

	// set required config data to use state couchdb
	couchdbConfig := &couchdb.Config{
		Address:             couchAddress,
		Username:            "",
		Password:            "",
		MaxRetries:          3,
		MaxRetriesOnStartup: 3,
		RequestTimeout:      10 * time.Second,
		RedoLogPath:         filepath.Join(ledgerFSRoot, "couchdbRedoLogs"),
	}
	env.initializer.Config.StateDBConfig.StateDatabase = "CouchDB"
	env.initializer.Config.StateDBConfig.CouchDB = couchdbConfig
	env.initializer.HealthCheckRegistry = &mock.HealthCheckRegistry{}
	env.initializer.ChaincodeLifecycleEventProvider = &mock.ChaincodeLifecycleEventProvider{}

	// verify init ledger panic and drop the corresponding db after each panic
	checkInitLedgerPanicAndDropDBs(t, env, ledgerFSRoot, couchdbConfig)

	t.Logf("All stores are reformatted/dropped, now ledgermgmt initialization should not panic")
	env.initLedgerMgmt()

	h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
	dataHelper := &v1xSampleDataHelper{sampleDataVersion: "v1.3", t: t}

	dataHelper.verify(h1)
	dataHelper.verify(h2)

	// drop couchDB and other rebuildable dbs to test db rebuild
	dropCouchDBs(t, couchdbConfig)
	env.closeAllLedgersAndDrop(rebuildableBlockIndex + rebuildableConfigHistory + rebuildableHistoryDB)
	h1, h2 = env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
	dataHelper.verify(h1)
	dataHelper.verify(h2)
}

func checkInitLedgerPanicAndDropDBs(t *testing.T, env *env, ledgerFSRoot string, couchdbConfig *couchdb.Config) {
	t.Logf("verifying that a panic occurs because idStore has old format and then reformat the idstore to proceed")
	idStorePath := kvledger.LedgerProviderPath(ledgerFSRoot)
	require.PanicsWithValue(
		t,
		fmt.Sprintf("Error in instantiating ledger provider: unexpected format. db info = [leveldb for channel-IDs at [%s]], data format = [], expected format = [2.0]",
			idStorePath),
		func() { env.initLedgerMgmt() },
		"A panic should occur because idstore is in format 1.x",
	)
	require.NoError(t, kvledger.UpgradeIDStoreFormat(ledgerFSRoot))

	t.Logf("verifying that a panic occurs because blockstore index has old format and then drop the idstore to proceed")
	blkIndexPath := path.Join(kvledger.BlockStorePath(ledgerFSRoot), "index")
	require.PanicsWithValue(
		t,
		fmt.Sprintf("Error in instantiating ledger provider: unexpected format. db info = [leveldb at [%s]], data format = [], expected format = [2.0]",
			blkIndexPath),
		func() { env.initLedgerMgmt() },
		"A panic should occur because block store index is in format 1.x",
	)
	require.NoError(t, os.RemoveAll(blkIndexPath))

	t.Logf("verifying that a panic occurs because historydb has old format and then drop the historydb to proceed")
	historyDBPath := kvledger.HistoryDBPath(ledgerFSRoot)
	require.PanicsWithValue(
		t,
		fmt.Sprintf("Error in instantiating ledger provider: unexpected format. db info = [leveldb at [%s]], data format = [], expected format = [2.0]",
			historyDBPath),
		func() { env.initLedgerMgmt() },
		"A panic should occur because history is in format 1.x",
	)
	require.NoError(t, os.RemoveAll(historyDBPath))

	if couchdbConfig == nil {
		t.Logf("verifying that a panic occurs because stateleveldb has old format and then drop the statedb to proceed")
		stateLevelDBPath := kvledger.StateDBPath(ledgerFSRoot)
		require.PanicsWithValue(
			t,
			fmt.Sprintf(
				"Error in instantiating ledger provider: unexpected format. db info = [leveldb at [%s]], data format = [], expected format = [2.0]",
				stateLevelDBPath,
			),
			func() { env.initLedgerMgmt() },
			"A panic should occur because statedb is in format 1.x",
		)
		require.NoError(t, os.RemoveAll(stateLevelDBPath))
	} else {
		t.Logf("verifying that a panic occurs because statecouchdb has old format and then drop the statedb to proceed")
		require.PanicsWithValue(
			t,
			fmt.Sprintf(
				"Error in instantiating ledger provider: unexpected format. db info = [CouchDB for state database], data format = [], expected format = [2.0]"),
			func() { env.initLedgerMgmt() },
			"A panic should occur because statedb is in format 1.x",
		)
		dropCouchDBs(t, couchdbConfig)
	}
}

// v1xSampleDataHelper provides a set of functions to verify the ledger (after upgraded to latest data format).
// It verifies the ledger under the assumption that the ledger was generated by the specific generation code from v1.1 or v1.3.
// For v1.1, the sample ledger data was generated by https://github.com/hyperledger/fabric/blob/release-1.1/core/ledger/kvledger/tests/v11_generate_test.go#L22
// This generate function constructed two ledgers and populateed the ledgers using this code
// (https://github.com/hyperledger/fabric/blob/release-1.1/core/ledger/kvledger/tests/sample_data_helper.go#L55)
// For v1.3, the sample ledger data was generated by CR (https://gerrit.hyperledger.org/r/#/c/fabric/+/34078/3/core/ledger/kvledger/tests/v13_generate_test.go@60).
// This generate function constructed two ledgers and populated the ledgers using this code
// (https://github.com/hyperledger/fabric/blob/release-1.3/core/ledger/kvledger/tests/sample_data_helper.go#L55)
type v1xSampleDataHelper struct {
	sampleDataVersion string
	t                 *testing.T
}

func (d *v1xSampleDataHelper) verify(h *testhelper) {
	d.verifyState(h)
	d.verifyBlockAndPvtdata(h)
	d.verifyGetTransactionByID(h)
	d.verifyConfigHistory(h)
	d.verifyHistory(h)
}

func (d *v1xSampleDataHelper) verifyState(h *testhelper) {
	lgrid := h.lgrid
	h.verifyPubState("cc1", "key1", d.sampleVal("value13", lgrid))
	h.verifyPubState("cc1", "key2", "")
	h.verifyPvtState("cc1", "coll1", "key3", d.sampleVal("value14", lgrid))
	h.verifyPvtState("cc1", "coll1", "key4", "")
	h.verifyPvtState("cc1", "coll2", "key3", d.sampleVal("value09", lgrid))
	h.verifyPvtState("cc1", "coll2", "key4", d.sampleVal("value10", lgrid))

	h.verifyPubState("cc2", "key1", d.sampleVal("value03", lgrid))
	h.verifyPubState("cc2", "key2", d.sampleVal("value04", lgrid))
	h.verifyPvtState("cc2", "coll1", "key3", d.sampleVal("value07", lgrid))
	h.verifyPvtState("cc2", "coll1", "key4", d.sampleVal("value08", lgrid))
	h.verifyPvtState("cc2", "coll2", "key3", d.sampleVal("value11", lgrid))
	h.verifyPvtState("cc2", "coll2", "key4", d.sampleVal("value12", lgrid))
}

func (d *v1xSampleDataHelper) verifyHistory(h *testhelper) {
	lgrid := h.lgrid
	expectedHistoryCC1Key1 := []string{
		d.sampleVal("value13", lgrid),
		d.sampleVal("value01", lgrid),
	}
	h.verifyHistory("cc1", "key1", expectedHistoryCC1Key1)
}

func (d *v1xSampleDataHelper) verifyConfigHistory(h *testhelper) {
	lgrid := h.lgrid
	h.verifyMostRecentCollectionConfigBelow(10, "cc1",
		&expectedCollConfInfo{5, d.sampleCollConf2(lgrid, "cc1")})

	h.verifyMostRecentCollectionConfigBelow(5, "cc1",
		&expectedCollConfInfo{3, d.sampleCollConf1(lgrid, "cc1")})

	h.verifyMostRecentCollectionConfigBelow(10, "cc2",
		&expectedCollConfInfo{5, d.sampleCollConf2(lgrid, "cc2")})

	h.verifyMostRecentCollectionConfigBelow(5, "cc2",
		&expectedCollConfInfo{3, d.sampleCollConf1(lgrid, "cc2")})
}

func (d *v1xSampleDataHelper) verifyConfigHistoryDoesNotExist(h *testhelper) {
	h.verifyMostRecentCollectionConfigBelow(10, "cc1", nil)
	h.verifyMostRecentCollectionConfigBelow(10, "cc2", nil)
}

func (d *v1xSampleDataHelper) verifyBlockAndPvtdata(h *testhelper) {
	lgrid := h.lgrid
	h.verifyBlockAndPvtData(2, nil, func(r *retrievedBlockAndPvtdata) {
		r.hasNumTx(2)
		r.hasNoPvtdata()
	})

	h.verifyBlockAndPvtData(4, nil, func(r *retrievedBlockAndPvtdata) {
		r.hasNumTx(2)
		r.pvtdataShouldContain(0, "cc1", "coll1", "key3", d.sampleVal("value05", lgrid))
		r.pvtdataShouldContain(1, "cc2", "coll1", "key3", d.sampleVal("value07", lgrid))
	})
}

func (d *v1xSampleDataHelper) verifyGetTransactionByID(h *testhelper) {
	h.verifyTxValidationCode("txid7", protopeer.TxValidationCode_VALID)
	h.verifyTxValidationCode("txid8", protopeer.TxValidationCode_MVCC_READ_CONFLICT)
}

func (d *v1xSampleDataHelper) sampleVal(val, ledgerid string) string {
	return fmt.Sprintf("%s:%s", val, ledgerid)
}

func (d *v1xSampleDataHelper) sampleCollConf1(ledgerid, ccName string) []*collConf {
	switch d.sampleDataVersion {
	case "v1.1":
		return []*collConf{
			{name: "coll1", members: []string{"org1", "org2"}},
			{name: ledgerid, members: []string{"org1", "org2"}},
			{name: ccName, members: []string{"org1", "org2"}},
		}
	case "v1.3":
		return []*collConf{
			{name: "coll1"},
			{name: ledgerid},
			{name: ccName},
		}
	default:
		// should not happen
		require.Failf(d.t, "sample data version %s is wrong", d.sampleDataVersion)
		return nil
	}
}

func (d *v1xSampleDataHelper) sampleCollConf2(ledgerid string, ccName string) []*collConf {
	switch d.sampleDataVersion {
	case "v1.1":
		return []*collConf{
			{name: "coll1", members: []string{"org1", "org2"}},
			{name: "coll2", members: []string{"org1", "org2"}},
			{name: ledgerid, members: []string{"org1", "org2"}},
			{name: ccName, members: []string{"org1", "org2"}},
		}
	case "v1.3":
		return []*collConf{
			{name: "coll1"},
			{name: "coll2"},
			{name: ledgerid},
			{name: ccName},
		}
	default:
		// should not happen
		require.Failf(d.t, "sample data version %s is wrong", d.sampleDataVersion)
		return nil
	}
}