gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/skylinkdatasource_test.go

package renter

import (
	"bytes"
	"context"
	"encoding/json"
	"reflect"
	"strings"
	"testing"
	"time"

	"gitlab.com/NebulousLabs/errors"
	"gitlab.com/NebulousLabs/fastrand"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/siad/crypto"
	"go.sia.tech/siad/modules"
	"go.sia.tech/siad/types"
)

// mockProjectChunkWorkerSet is a mock object implementing the chunkFetcher
// interface
type mockProjectChunkWorkerSet struct {
	staticDownloadResponseChan chan *downloadResponse
	staticDownloadData         []byte
	staticEC                   skymodules.ErasureCoder
	staticErr                  error
}

// newTestDataSource is a helper that creates a mostly valid skylinkDataSource
// for testing. It doesn't have a valid renter or chunk fetchers.
func newTestDataSource(fileName string, data []byte) *skylinkDataSource {
	// create renter
	renter := new(Renter)
	renter.staticBaseSectorDownloadStats = skymodules.NewSectorDownloadStats()
	renter.staticFanoutSectorDownloadStats = skymodules.NewSectorDownloadStats()

	// Create some metadata.
	fileSize := uint64(len(data))
	md := TusSkyfileMetadata(fileName, "", fileSize, 0777)
	mdRaw, err := json.Marshal(md)
	if err != nil {
		panic(err)
	}

	// Create the layout and sector.
	sl := skymodules.NewSkyfileLayout(fileSize, uint64(len(mdRaw)), 0, skymodules.NewPassthroughErasureCoder(), crypto.TypePlain)
	sector, fetchLen, _ := skymodules.BuildBaseSector(sl.Encode(), nil, mdRaw, data)

	// Create the skylink.
	sectorRoot := crypto.MerkleRoot(sector)
	skylink, err := skymodules.NewSkylinkV1(sectorRoot, 0, fetchLen)
	if err != nil {
		panic(err)
	}

	ctx, cancel := context.WithCancel(renter.tg.StopCtx())
	return &skylinkDataSource{
		staticID:                     skylink.DataSourceID(),
		staticLayout:                 sl,
		staticMetadata:               md,
		staticRawMetadata:            mdRaw,
		staticSkylink:                skylink,
		staticSkylinkSector:          sector,
		staticLayoutOffset:           0,
		staticDecryptedSkylinkSector: sector,
		staticChunkFetchers:          make([]chunkFetcher, 0),

		staticCancelFunc: cancel,
		staticCtx:        ctx,
	}
}

// newTestDownloadResponse creates a new downloadResponse for testing.
func newTestDownloadResponse(ec skymodules.ErasureCoder, fullData []byte, offset, length uint64) (*downloadResponse, error) {
	// Erasure Code the full data.
	fullData = append([]byte{}, fullData...)
	data, err := ec.Encode(fullData)
	if err != nil {
		return nil, err
	}
	// Cut off all segments before the ones we are interested in. The actual
	// download code will also only download these.
	toCut := uint64(offset) / (crypto.SegmentSize * uint64(ec.MinPieces()))
	for i := 0; i < len(data); i++ {
		data[i] = data[i][toCut*crypto.SegmentSize:]
	}
	// Create response and send it.
	return newDownloadResponse(offset, length, ec, data, nil, nil), nil
}

// Download implements the chunkFetcher interface.
func (m *mockProjectChunkWorkerSet) Download(ctx context.Context, pricePerMS types.Currency, offset, length uint64, _ bool) (chan *downloadResponse, error) {
	dr, err := newTestDownloadResponse(m.staticEC, m.staticDownloadData, offset, length)
	if err != nil {
		return nil, err
	}
	m.staticDownloadResponseChan <- dr
	return m.staticDownloadResponseChan, m.staticErr
}

// newChunkFetcher returns a chunk fetcher.
func newChunkFetcher(data []byte, err error, ec skymodules.ErasureCoder) chunkFetcher {
	// For convenience the test only passes the data they are interested in
	// but we need to make sure it's sector size aligned.
	if uint64(len(data)) < modules.SectorSize {
		data = append(data, make([]byte, modules.SectorSize-uint64(len(data)))...)
	}
	responseChan := make(chan *downloadResponse, 1)
	return &mockProjectChunkWorkerSet{
		staticDownloadResponseChan: responseChan,
		staticDownloadData:         data,
		staticEC:                   ec,
		staticErr:                  err,
	}
}

// TestSkylinkDataSource is a unit test that verifies the behaviour of a
// SkylinkDataSource. Note that we are using mocked data, testing of the
// datasource with live PCWSs attached will happen through integration tests.
func TestSkylinkDataSource(t *testing.T) {
	t.Parallel()
	t.Run("small", testSkylinkDataSourceSmallFile)
	t.Run("large", testSkylinkDataSourceLargeFile)
	t.Run("managedSkylinkDataSource", testManagedSkylinkDataSource)
	t.Run("managedReadLayout", testSkylinkDataSourceReadLayout)
}

// testSkylinkDataSourceSmallFile verifies we can read from a datasource for a
// small skyfile.
func testSkylinkDataSourceSmallFile(t *testing.T) {
	data := fastrand.Bytes(int(modules.SectorSize) / 2)
	datasize := uint64(len(data))

	sds := newTestDataSource("thisisafilename", data)
	skylink := sds.staticSkylink

	if sds.DataSize() != datasize {
		t.Fatal("unexpected", sds.DataSize(), datasize)
	}
	if sds.ID() != skylink.DataSourceID() {
		t.Fatal("unexpected")
	}
	if !reflect.DeepEqual(sds.Metadata(), TusSkyfileMetadata("thisisafilename", "", datasize, 0777)) {
		t.Fatal("unexpected")
	}
	if sds.RequestSize() != SkylinkDataSourceRequestSize {
		t.Fatal("unexpected")
	}

	// verify invalid index.
	_, err := sds.ReadSection(context.Background(), (uint64(len(data))/uint64(sds.RequestSize()) + 1), types.ZeroCurrency)
	if err == nil || !strings.Contains(err.Error(), "ReadSection: offset out-of-bounds 2560 >= 2048") {
		t.Fatal(err)
	}

	index := uint64(1)
	responseChan, err := sds.ReadSection(context.Background(), index, types.ZeroCurrency)
	if err != nil {
		t.Fatal(err)
	}
	select {
	case resp := <-responseChan:
		dd, err := resp.Data()
		if resp == nil || err != nil {
			t.Fatal("unexpected")
		}
		respData, err := dd.Recover()
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(respData, data[sds.RequestSize():2*sds.RequestSize()]) {
			t.Log("expected: ", data[sds.RequestSize():2*sds.RequestSize()], sds.RequestSize())
			t.Log("actual:   ", respData, len(respData))
			t.Fatal("unexepected data")
		}
	case <-time.After(time.Second):
		t.Fatal("unexpected")
	}

	index = uint64(len(data)/int(sds.RequestSize())) - 1
	responseChan, err = sds.ReadSection(context.Background(), index, types.ZeroCurrency)
	if err != nil {
		t.Fatal(err)
	}
	select {
	case resp := <-responseChan:
		dd, err := resp.Data()
		if resp == nil || err != nil {
			t.Fatal("unexpected")
		}
		respData, err := dd.Recover()
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(respData, data[index*sds.RequestSize():]) {
			t.Log("expected: ", data[index*sds.RequestSize():], len(data[sds.RequestSize():]))
			t.Log("actual:   ", respData, len(respData))
			t.Fatal("unexepected data")
		}
	case <-time.After(time.Second):
		t.Fatal("unexpected")
	}

	select {
	case <-sds.staticCtx.Done():
		t.Fatal("unexpected")
	case <-time.After(10 * time.Millisecond):
		sds.SilentClose()
	}
	select {
	case <-sds.staticCtx.Done():
	case <-time.After(10 * time.Millisecond):
		t.Fatal("unexpected")
	}
}

// testSkylinkDataSourceLargeFile verifies we can read from a datasource for a
// large skyfile.
func testSkylinkDataSourceLargeFile(t *testing.T) {
	fanoutChunk1 := fastrand.Bytes(int(modules.SectorSize))
	fanoutChunk2 := fastrand.Bytes(int(modules.SectorSize) / 2)
	fanoutChunks := [][]byte{fanoutChunk1, fanoutChunk2}
	allData := append(fanoutChunk1, fanoutChunk2...)
	datasize := uint64(len(allData))
	fanoutDataPieces := uint64(1)
	fanoutParityPieces := uint64(9)
	ec, err := skymodules.NewRSSubCode(int(fanoutDataPieces), int(fanoutParityPieces), crypto.SegmentSize)
	if err != nil {
		t.Fatal(err)
	}

	// create renter
	renter := new(Renter)
	renter.staticBaseSectorDownloadStats = skymodules.NewSectorDownloadStats()
	renter.staticFanoutSectorDownloadStats = skymodules.NewSectorDownloadStats()

	ctx, cancel := context.WithCancel(renter.tg.StopCtx())

	// create chunk fetcher and no-op loaders
	chunkFetchers := make([]chunkFetcher, 2)
	chunkFetchersAvailable := make([]chan struct{}, 2)
	chunkFetcherLoaders := make([]chunkFetcherLoaderFn, 2)
	for i := 0; i < 2; i++ {
		chunkFetchers[i] = newChunkFetcher(fanoutChunks[i], nil, ec)
		chunkFetchersAvailable[i] = make(chan struct{})
		close(chunkFetchersAvailable[i])
		chunkFetcherLoaders[i] = func() {}
	}

	sds := &skylinkDataSource{
		staticID: skymodules.DataSourceID(crypto.Hash{1, 2, 3}),
		staticLayout: skymodules.SkyfileLayout{
			Version:            skymodules.SkyfileVersion,
			Filesize:           datasize,
			MetadataSize:       14e3,
			FanoutSize:         75e3,
			FanoutDataPieces:   uint8(fanoutDataPieces),
			FanoutParityPieces: uint8(fanoutParityPieces),
			CipherType:         crypto.TypePlain,
		},
		staticMetadata: skymodules.SkyfileMetadata{
			Filename: "thisisafilename",
			Length:   datasize,
		},

		staticSkylinkSector:          make([]byte, 0),
		staticChunkFetcherLoaders:    chunkFetcherLoaders,
		staticChunkFetchers:          chunkFetchers,
		staticChunkFetchersAvailable: chunkFetchersAvailable,
		staticChunkErrs:              []error{nil, nil},

		staticCancelFunc: cancel,
		staticCtx:        ctx,
	}

	if sds.DataSize() != datasize {
		t.Fatal("unexpected", sds.DataSize(), datasize)
	}
	if sds.ID() != skymodules.DataSourceID(crypto.Hash{1, 2, 3}) {
		t.Fatal("unexpected")
	}
	if !reflect.DeepEqual(sds.Metadata(), skymodules.SkyfileMetadata{
		Filename: "thisisafilename",
		Length:   datasize,
	}) {
		t.Fatal("unexpected")
	}
	if sds.RequestSize() != SkylinkDataSourceRequestSize {
		t.Fatal("unexpected")
	}

	responseChan, err := sds.ReadSection(context.Background(), 0, types.ZeroCurrency)
	if err != nil {
		t.Fatal(err)
	}
	select {
	case resp := <-responseChan:
		dd, err := resp.Data()
		if resp == nil || err != nil {
			t.Fatal("unexpected", err)
		}
		respData, err := dd.Recover()
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(respData, allData[:sds.RequestSize()]) {
			t.Log("expected: ", allData[:sds.RequestSize()], sds.RequestSize())
			t.Log("actual:   ", respData, len(respData))
			t.Fatal("unexepected data")
		}
	case <-time.After(time.Second):
		t.Fatal("unexpected")
	}

	select {
	case <-sds.staticCtx.Done():
		t.Fatal("unexpected")
	case <-time.After(10 * time.Millisecond):
		sds.SilentClose()
	}
	select {
	case <-sds.staticCtx.Done():
	case <-time.After(10 * time.Millisecond):
		t.Fatal("unexpected")
	}
}

// testManagedSkylinkDataSource is a unit test that verifies the chunk fetchers
// are preloaded up until a certain point, and all remaining chunk fetchers get
// lazy loaded when ReadStream is called
func testManagedSkylinkDataSource(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	// Create a renter tester
	rt, err := newRenterTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := rt.Close(); err != nil {
			t.Fatal(err)
		}
	}()
	r := rt.renter

	// Set an allowance.
	err = r.staticHostContractor.SetAllowance(skymodules.DefaultAllowance)
	if err != nil {
		t.Fatal(err)
	}

	// Add some hosts
	_, err1 := rt.addHost(t.Name() + "1")
	_, err2 := rt.addHost(t.Name() + "2")
	_, err3 := rt.addHost(t.Name() + "3")
	if err = errors.Compose(err1, err2, err3); err != nil {
		t.Fatal(err)
	}

	// Create upload params for a large file, ensure it has 4 fanout chunks,
	// which is one more than 'chunkFetchersMaximumPreload'
	data := fastrand.Bytes(int(4 * modules.SectorSize))
	sup := skymodules.SkyfileUploadParameters{
		SiaPath:             skymodules.RandomSiaPath(),
		Force:               false,
		Root:                false,
		BaseChunkRedundancy: 2,
		Filename:            "file.tt",
		Reader:              bytes.NewReader(data),
	}

	// Upload the skyfile
	reader := skymodules.NewSkyfileReader(sup.Reader, sup)
	sl, err := r.managedUploadSkyfileLargeFile(ctx, sup, reader)
	if err != nil {
		t.Fatal(err)
	}

	// Create a data source from the skylink
	ppms := skymodules.DefaultSkynetPricePerMS
	sbds, err := r.managedSkylinkDataSource(ctx, sl, ppms)
	if err != nil {
		t.Fatal(err)
	}
	sds, ok := sbds.(*skylinkDataSource)
	if !ok {
		t.Fatal("could not cast to skylinkDataSource")
	}

	// Create a helper that checks if the chunk is available w/o blocking
	checkAvailable := func(index int) bool {
		select {
		case <-sds.staticChunkFetchersAvailable[index]:
		case <-time.After(5 * time.Second):
			return false
		}
		return true
	}
	if !checkAvailable(0) || !checkAvailable(1) || !checkAvailable(2) {
		t.Fatal("expected the first three chunks to be available")
	}
	if checkAvailable(3) {
		t.Fatal("expected the fourth chunk to not be available")
	}

	// Create a helper function that waits for the read response
	checkData := func(c <-chan *downloadResponse) []byte {
		var data []byte
		select {
		case resp := <-c:
			dd, err := resp.Data()
			if err != nil {
				t.Fatal(err)
			}
			data, err = dd.Recover()
			if err != nil {
				t.Fatal(err)
			}
		case <-time.After(5 * time.Second):
			t.Fatal("read timed out")
		}
		return data
	}

	// Read from the fourth chunk
	index := 3 * modules.SectorSize / sds.RequestSize()
	resp, err := sds.ReadSection(ctx, index, ppms)
	if err != nil {
		t.Fatal(err)
	}
	download := checkData(resp)
	if len(download) != int(sds.RequestSize()) {
		t.Fatal("unexpected data", len(download), sds.RequestSize())
	}

	// Assert the chunk fetcher was lazy loaded
	if !checkAvailable(3) {
		t.Fatal("expected the fourth chunk to be available")
	}
}

// testSkylinkDataSourceReadLayout is a unit-test for managedReadLayout.
func testSkylinkDataSourceReadLayout(t *testing.T) {
	sds := newTestDataSource("somefile", []byte{1, 2, 3})

	layout, data, proof := sds.managedReadLayout()

	// Check returned layout.
	if !reflect.DeepEqual(sds.staticLayout, layout) {
		t.Fatal("layout mismatch")
	}

	// Data should be segment-aligned. That means 2 segments since the
	// layout is always 99 bytes big.
	if len(data) != 2*crypto.SegmentSize {
		t.Fatalf("unexpected data length: %v != %v", len(data), 2*crypto.SegmentSize)
	}

	// The merkle proof for 2 segments should be 5 hashes for a merkle tree
	// with 64 segments.
	if len(proof) != 5 {
		t.Fatalf("unexpected proof length %v != %v", len(proof), 5)
	}

	// Verify the proof.
	valid := crypto.VerifyRangeProof(data, proof, 0, 2, sds.staticSkylink.MerkleRoot())
	if !valid {
		t.Fatal("proof is not valid")
	}
}