gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/skylinkdatasource.go

package renter

import (
	"context"
	"fmt"
	"sync"

	"github.com/opentracing/opentracing-go"
	"gitlab.com/SkynetLabs/skyd/build"
	"gitlab.com/SkynetLabs/skyd/skykey"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/siad/crypto"
	"go.sia.tech/siad/modules"
	"go.sia.tech/siad/types"

	"gitlab.com/NebulousLabs/errors"
)

var (
	// SkylinkDataSourceRequestSize is the size that is suggested by the data
	// source to be used when reading data from it.
	SkylinkDataSourceRequestSize = build.Select(build.Var{
		Dev:      uint64(1 << 18), // 256 KiB
		Standard: uint64(1 << 20), // 1 MiB
		Testing:  uint64(1 << 9),  // 512 B
	}).(uint64)

	// chunkFetchersMaximumPreload defines the amount of chunk fetchers we
	// preload when the data source gets created. The remaining chunk fetchers
	// are lazy loaded when data sections get created, which in turn are also
	// created before they are being read from, giving those PCWS some buffer to
	// execute their has sector jobs before being used to download the chunk.
	chunkFetchersMaximumPreload = build.Select(build.Var{
		Dev:      3,
		Standard: 16, // ~32MiB worth of PCWS
		Testing:  3,
	}).(int)
)

type (
	// skylinkDataSource implements streamBufferDataSource on a Skylink.
	// Notably, it creates a pcws for every single chunk in the Skylink and
	// keeps them in memory, to reduce latency on seeking through the file.
	skylinkDataSource struct {
		// Metadata.
		staticID          skymodules.DataSourceID
		staticLayout      skymodules.SkyfileLayout
		staticMetadata    skymodules.SkyfileMetadata
		staticRawMetadata []byte
		staticSkylink     skymodules.Skylink

		// staticSkylinkSector will contain the raw data of the sector
		// referenced by the skylink. That way we can provide proofs for
		// arbitrary ranges of the sector later.
		// NOTE: The skylink sector is never decrypted to make sure the
		// merkle proofs can still be verified against the sector root.
		staticSkylinkSector []byte

		// staticLayoutOffset is the offset of the layout within the
		// sector.
		staticLayoutOffset int

		staticDecryptedSkylinkSector []byte

		// staticChunkFetcherLoaders holds a loader function for every chunk in
		// the fanout. This loaders creates a PCWS for that chunk, and thus
		// spins up a bunch of has sector jobs. On initial read, we want to
		// limit the amount of PCWS created, to avoid unnecessary network load.
		// That is why we only preload a certain amount, and lazy load the
		// others when a new data section gets created.
		//
		// staticChunkFetchers holds the PCWS after it became available
		//
		// staticChunkFetchersAvailable is a channel that gets closed by the
		// loader after the PCWS got created and its state was updated for the
		// first time
		//
		// staticChunkErrs contains a potential error, this error should be
		// checked when the chunk fetcher becomes available
		staticChunkFetcherLoaders    []chunkFetcherLoaderFn
		staticChunkFetchersAvailable []chan struct{}
		staticChunkFetchers          []chunkFetcher
		staticChunkErrs              []error

		// Utilities
		staticCtx        context.Context
		staticCancelFunc context.CancelFunc
	}

	// chunkFetcherLoaderFn is a helper type that represents a function that
	// loads a chunk fetcher. Every chunk in the fanout will have a
	// corresponding loader function that creates the PCWS for that chunk.
	chunkFetcherLoaderFn func()
)

// DataSize implements streamBufferDataSource
func (sds *skylinkDataSource) DataSize() uint64 {
	return sds.staticLayout.Filesize
}

// ID implements streamBufferDataSource
func (sds *skylinkDataSource) ID() skymodules.DataSourceID {
	return sds.staticID
}

// ReadBaseSectorPayload reads data from the data source's base sector payload.
// This will return an error when called on anything but a small skyfile.
func (sds *skylinkDataSource) ReadBaseSectorPayload(off, length uint64) (*downloadResponse, error) {
	return sds.managedReadSmallFileSection(off, length)
}

// HasRecursiveFanout returns whether or not the datasource belongs to a skylink
// with a recursive fanout.
func (sds *skylinkDataSource) HasRecursiveFanout() bool {
	return sds.staticLayout.HasRecursiveFanout(uint64(sds.staticLayoutOffset))
}

// Layout implements streamBufferDataSource
func (sds *skylinkDataSource) Layout() skymodules.SkyfileLayout {
	return sds.staticLayout
}

// RawLayout implements streamBufferDataSource
func (sds *skylinkDataSource) RawLayout() (skymodules.SkyfileLayout, []byte, []crypto.Hash) {
	return sds.managedReadLayout()
}

// Metadata implements streamBufferDataSource
func (sds *skylinkDataSource) Metadata() skymodules.SkyfileMetadata {
	return sds.staticMetadata
}

// RawMetadata implements streamBufferDataSource
func (sds *skylinkDataSource) RawMetadata() []byte {
	return sds.staticRawMetadata
}

// RequestSize implements streamBufferDataSource
func (sds *skylinkDataSource) RequestSize() uint64 {
	return SkylinkDataSourceRequestSize
}

// Skylink implements streamBufferDataSource
func (sds *skylinkDataSource) Skylink() skymodules.Skylink {
	return sds.staticSkylink
}

// SilentClose implements streamBufferDataSource
func (sds *skylinkDataSource) SilentClose() {
	// Canceling the context for the data source should be sufficient. As
	// all child processes (such as the pcws for each chunk) should be using
	// contexts derived from the sds context.
	sds.staticCancelFunc()
}

// managedReadSection downloads a section for the given section index from the
// data source. It returns a response that can be awaited,
func (sds *skylinkDataSource) managedReadSection(ctx context.Context, sectionIndex uint64, pricePerMS types.Currency) (<-chan *downloadResponse, uint64, error) {
	// Translate input to offset and fetchSize within DataSource.
	off := sectionIndex * sds.RequestSize()
	if off >= sds.staticLayout.Filesize {
		return nil, 0, fmt.Errorf("ReadSection: offset out-of-bounds %v >= %v", off, sds.staticLayout.Filesize)
	}

	// We tolerate out-of-bounds if at least the offset was within bounds.
	// That just means we are at the end of the file and there is not a full
	// section left.
	fetchSize := sds.RequestSize()
	if off+fetchSize > sds.staticLayout.Filesize {
		fetchSize = sds.staticLayout.Filesize - off
	}

	// If we are dealing with a small skyfile without fanout bytes, we can
	// simply read from that and return early.
	if sds.staticLayout.FanoutSize == 0 {
		dr, err := sds.managedReadSmallFileSection(off, fetchSize)
		respChan := make(chan *downloadResponse, 1)
		respChan <- dr
		close(respChan)
		return respChan, 0, err
	}

	// Determine how large each uploaded chunk is.
	chunkSize := skymodules.ChunkSize(sds.staticLayout.CipherType, uint64(sds.staticLayout.FanoutDataPieces))

	// The fetchSize should never exceed the size of a chunk. That way we
	// only need to deal with a single set of proofs per data section and
	// not multiple.
	if fetchSize > chunkSize {
		err := fmt.Errorf("ReadSection: fetchSize > SectorSize %v > %v", fetchSize, modules.SectorSize)
		build.Critical(err)
		return nil, 0, err
	}

	// Determine which chunk the offset is currently in.
	chunkIndex := off / chunkSize
	offsetInChunk := off % chunkSize

	// Trigger the loader, this function will only be executed once and
	// ensures the chunk fetcher is being initialized. Consecutive calls are
	// essentially a no-op
	sds.staticChunkFetcherLoaders[chunkIndex]()

	// Wait until the chunk fetcher is ready, and check if there was any
	// error in initializing the chunk fetcher.
	select {
	case <-sds.staticChunkFetchersAvailable[chunkIndex]:
	case <-sds.staticCtx.Done():
		return nil, 0, errors.New("stream fetch aborted because of cancel")
	}
	if sds.staticChunkErrs[chunkIndex] != nil {
		return nil, 0, errors.AddContext(sds.staticChunkErrs[chunkIndex], "unable to start download")
	}

	// Schedule the download.
	respChan, err := sds.staticChunkFetchers[chunkIndex].Download(ctx, pricePerMS, offsetInChunk, fetchSize, false)
	if err != nil {
		return nil, 0, errors.AddContext(err, "unable to start download")
	}
	return respChan, chunkIndex, nil
}

// ReadFanout reads a single piece root from the fanout of the datasource and
// returns the proof for that root as well as the offset within the sector.
func (sds *skylinkDataSource) ReadFanout(chunkIndex, pieceIndex uint64) ([]byte, []crypto.Hash, uint32, error) {
	layout := sds.staticLayout

	// Get the offset of where the fanout starts within the base sector.
	fanoutOff := layout.FanoutOffset(uint64(sds.staticLayoutOffset))

	// Get the offset of the right piece root within the fanout. If it is
	// compressed, each chunk only contains 1 piece. So we can ignore the
	// pieceIndex.
	chunkOff := fanoutOff + layout.FanoutPiecesPerChunk()*chunkIndex*crypto.HashSize
	rootOff := chunkOff + pieceIndex*crypto.HashSize

	dr, err := sds.managedReadBaseSector(rootOff, crypto.HashSize)
	if err != nil {
		return nil, nil, 0, err
	}
	return dr.externDownloadedData.LogicalChunkData[0], dr.externDownloadedData.Proofs[0], uint32(rootOff), nil
}

// ReadSection implements streamBufferDataSource
func (sds *skylinkDataSource) ReadSection(ctx context.Context, sectionIndex uint64, pricePerMS types.Currency) (<-chan *downloadResponse, error) {
	respChan, _, err := sds.managedReadSection(ctx, sectionIndex, pricePerMS)
	return respChan, err
}

// managedDownloadByRoot will fetch data using the merkle root of that data.
func (r *Renter) managedDownloadByRoot(ctx context.Context, root crypto.Hash, offset, length uint64, pricePerMS types.Currency) ([]byte, *downloadedData, *pcwsWorkerState, error) {
	// Create a context that dies when the function ends, this will cancel all
	// of the worker jobs that get created by this function.
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// Capture the base sector download in a new span.
	span, ctx := opentracing.StartSpanFromContext(ctx, "managedDownloadByRoot")
	span.SetTag("root", root)
	defer span.Finish()

	// Create the pcws for the first chunk. We use a passthrough cipher and
	// erasure coder. If the base sector is encrypted, we will notice and be
	// able to decrypt it once we have fully downloaded it and are able to
	// access the layout. We can make the assumption on the erasure coding being
	// of 1-N seeing as we currently always upload the basechunk using 1-N
	// redundancy.
	ptec := skymodules.NewPassthroughErasureCoder()
	tpsk, err := crypto.NewSiaKey(crypto.TypePlain, nil)
	if err != nil {
		return nil, nil, nil, errors.AddContext(err, "unable to create plain skykey")
	}
	pcws, err := r.newPCWSByRoots(ctx, []crypto.Hash{root}, ptec, tpsk, 0)
	if err != nil {
		return nil, nil, nil, errors.AddContext(err, "unable to create the worker set for this skylink")
	}

	// Download the base sector. The base sector contains the metadata, without
	// it we can't provide a completed data source.
	//
	// NOTE: we pass in the provided context here, if the user imposed a timeout
	// on the download request, this will fire if it takes too long.
	respChan, err := pcws.managedDownload(ctx, pricePerMS, offset, length, false)
	if err != nil {
		return nil, nil, nil, errors.AddContext(err, "unable to start download")
	}
	resp := <-respChan
	dd, err := resp.Data()
	if err != nil {
		return nil, nil, nil, errors.AddContext(resp.err, "base sector download did not succeed")
	}
	data, err := dd.Recover()
	return data, dd, pcws.managedWorkerState(), err
}

// managedReadLayout returns the data sources layout, the segment-aligned raw
// layout and a proof for the raw layout.
func (sds *skylinkDataSource) managedReadLayout() (skymodules.SkyfileLayout, []byte, []crypto.Hash) {
	// Get offset and length of the layout.
	layoutOff := sds.staticLayoutOffset
	layoutLen := skymodules.SkyfileLayoutSize

	// The downloaded data needs to be segment aligned for the proof.
	layoutOffAligned := layoutOff
	if mod := layoutOffAligned % crypto.SegmentSize; mod != 0 {
		layoutOffAligned -= mod
	}
	endLayoutOffAligned := layoutOff + layoutLen
	if mod := endLayoutOffAligned % crypto.SegmentSize; mod != 0 {
		endLayoutOffAligned += (crypto.SegmentSize - mod)
	}

	// Get the requested, segment-aligned data.
	data := sds.staticSkylinkSector[layoutOffAligned:endLayoutOffAligned]

	// Create a range proof for the payload.
	proofStart := int(layoutOffAligned / crypto.SegmentSize)
	proofEnd := int(endLayoutOffAligned / crypto.SegmentSize)
	proof := crypto.MerkleRangeProof(sds.staticSkylinkSector, proofStart, proofEnd)

	// Sanity check the proof.
	if build.Release == "testing" {
		if !crypto.VerifyRangeProof(data, proof, proofStart, proofEnd, sds.Skylink().MerkleRoot()) {
			build.Critical("managedReadLayout: created merkle proof is invalid")
		}
	}
	return sds.staticLayout, data, proof
}

// managedReadBaseSector handles reading a part of a base sector. It returns a
// proof for the section that was read.
func (sds *skylinkDataSource) managedReadBaseSector(off, fetchSize uint64) (*downloadResponse, error) {
	if off > modules.SectorSize {
		return nil, fmt.Errorf("managedReadBaseSectorRange: off can't be greater than sector size: %v > %v", off, modules.SectorSize)
	}
	bytesLeft := modules.SectorSize - off
	if fetchSize > bytesLeft {
		return nil, fmt.Errorf("managedReadBaseSectorRange: read is out-of-bounds for off %v fetchSize %v and sector size %v", off, fetchSize, modules.SectorSize)
	}

	// Convenience var.
	offsetInChunk := off

	// The data we grab needs to be segment aligned so we adjust the offset
	// and length that we use.
	offsetInChunkAligned := offsetInChunk
	if mod := offsetInChunkAligned % crypto.SegmentSize; mod != 0 {
		offsetInChunkAligned -= mod
	}
	endOffsetInChunkAligned := offsetInChunk + fetchSize
	if mod := endOffsetInChunkAligned % crypto.SegmentSize; mod != 0 {
		endOffsetInChunkAligned += (crypto.SegmentSize - mod)
	}

	// Prepare slice for returning the segment-aligned data. We just copy
	// the raw, decrypted sector data of the range we want to return.
	data := append([]byte{}, sds.staticDecryptedSkylinkSector[offsetInChunkAligned:endOffsetInChunkAligned]...)

	// Create a range proof for the payload. The range proof is created over
	// the encrypted data because otherwise the proof doesn't add up to the
	// root in the skylink.
	proofStart := int(offsetInChunkAligned / crypto.SegmentSize)
	proofEnd := int(endOffsetInChunkAligned / crypto.SegmentSize)
	proof := crypto.MerkleRangeProof(sds.staticSkylinkSector, proofStart, proofEnd)

	// Sanity check the proof. We do this only for unencrypted sectors. For
	// encrypted ones, the client needs to encrypt the data again for it to
	// match the proof.
	if build.Release == "testing" && !skymodules.IsEncryptedLayout(sds.staticLayout) {
		if !crypto.VerifyRangeProof(data, proof, proofStart, proofEnd, sds.Skylink().MerkleRoot()) {
			build.Critical("managedReadSmallFileSection: created merkle proof is invalid")
		}
	}

	// Create the response.
	dr := newDownloadResponse(offsetInChunk, fetchSize, skymodules.NewPassthroughErasureCoder(), [][]byte{data}, [][]crypto.Hash{proof}, nil)
	return dr, nil
}

// managedReadSmallFileSection handles reading a section from the buffered
// sector in-memory in case the file is a small one.
func (sds *skylinkDataSource) managedReadSmallFileSection(off, fetchSize uint64) (*downloadResponse, error) {
	if !sds.staticLayout.IsSmallFile() {
		return nil, errors.New("file is not a small file")
	}
	if off > sds.staticLayout.Filesize {
		return nil, fmt.Errorf("managedReadSmallFileSection: off can't be greater than payload length: %v > %v", off, sds.staticLayout.Filesize)
	}
	bytesLeft := sds.staticLayout.Filesize - off
	if fetchSize > bytesLeft {
		fetchSize = bytesLeft
	}

	// Get the offset of the payload within the sector.
	payloadOff := uint64(sds.staticLayoutOffset + skymodules.SkyfileLayoutSize + len(sds.staticRawMetadata))
	return sds.managedReadBaseSector(payloadOff+off, fetchSize)
}

// managedSkylinkDataSource will create a streamBufferDataSource for the data
// contained inside of a Skylink. The function will not return until the base
// sector and all skyfile metadata has been retrieved.
//
// NOTE: Skylink data sources are cached and outlive the user's request because
// multiple different callers may want to use the same data source. We do have
// to pass in a context though to adhere to a possible user-imposed request
// timeout. This can be optimized to always create the data source when it was
// requested, but we should only do so after gathering some real world feedback
// that indicates we would benefit from this.
func (r *Renter) managedSkylinkDataSource(ctx context.Context, skylink skymodules.Skylink, pricePerMS types.Currency) (streamBufferDataSource, error) {
	skylinkSector, dd, _, err := r.managedDownloadByRoot(ctx, skylink.MerkleRoot(), 0, modules.SectorSize, pricePerMS)
	if err != nil {
		return nil, errors.AddContext(err, "failed to download baseSector")
	}

	// Store the skylink sector in the cache. We call this even if the sector
	// was cached. That way we count the cache hit and refresh the lru.
	err = r.staticStreamBufferSet.staticCache.Put(skylink.DataSourceID(), baseSectorSectionIndex, dd)
	if err != nil {
		return nil, errors.AddContext(err, "failed to update the cache")
	}

	// Get the offset and fetchsize of the base sector within the full
	// sector.
	slOffset, slFetchSize, err := skylink.OffsetAndFetchSize()
	if err != nil {
		return nil, errors.AddContext(err, "failed to get offset and fetchsize from skylink")
	}
	baseSector := skylinkSector[slOffset : slOffset+slFetchSize]
	decryptedSkylinkSector := skylinkSector

	// Check if the base sector is encrypted, and attempt to decrypt it.
	// This will fail if we don't have the decryption key.
	var fileSpecificSkykey skykey.Skykey
	if skymodules.IsEncryptedBaseSector(baseSector) {
		// Deep-copy the sector to avoid decrypting the
		// original skylinkSector.
		decryptedSkylinkSector = append([]byte{}, decryptedSkylinkSector...)
		baseSector = decryptedSkylinkSector[slOffset : slOffset+slFetchSize]
		fileSpecificSkykey, err = r.managedDecryptBaseSector(baseSector)
		if err != nil {
			return nil, errors.AddContext(err, "unable to decrypt skyfile base sector")
		}
	}

	// Parse out the metadata of the skyfile.
	// TODO: (f/u?) it might be better to resolve the parts of the fanout we
	// need on demand. But that's quite the undertaking by itself.
	// e.g. if we don't start resolving the recursive fanout right away we
	// lose the benefit of the workerset, because we will add some latency
	// later once we actually know what the user wants to download.
	layout, fanoutBytes, metadata, rawMetadata, _, _, err := r.ParseSkyfileMetadata(baseSector)
	if err != nil {
		return nil, errors.AddContext(err, "unable to parse skyfile metadata")
	}

	// Create the context for the data source - a child of the renter
	// threadgroup but otherwise independent.
	dsCtx, cancelFunc := context.WithCancel(r.tg.StopCtx())

	// Tag the span with its size. We tag it with 64kb, 1mb, 4mb and 10mb as
	// those are the size increments used by the benchmark tool. This way we can
	// run the benchmark and then filter the results using these tags.
	//
	// NOTE: the sizes used are "exact sizes", meaning they are as close as
	// possible to their eventual size after taking into account the size of the
	// metadata. See cmd/skynet-benchmark/dl.go for more info.
	if span := opentracing.SpanFromContext(ctx); span != nil {
		switch length := metadata.Length; {
		case length <= 61e3:
			span.SetTag("length", "64kb")
		case length <= 982e3:
			span.SetTag("length", "1mb")
		case length <= 3931e3:
			span.SetTag("length", "4mb")
		default:
			span.SetTag("length", "10mb")
		}

		// Attach the span to the ctx
		dsCtx = opentracing.ContextWithSpan(dsCtx, span)
	}

	// If there's a fanout create a PCWS for every chunk.
	var fanoutChunkFetcherLoaders []chunkFetcherLoaderFn
	var fanoutChunkFetchersAvailable []chan struct{}
	var fanoutChunkFetchers []chunkFetcher
	var fanoutChunkErrs []error
	var ec skymodules.ErasureCoder
	if len(fanoutBytes) > 0 {
		// Derive the fanout key
		fanoutKey, err := skymodules.DeriveFanoutKey(&layout, fileSpecificSkykey)
		if err != nil {
			cancelFunc()
			return nil, errors.AddContext(err, "unable to derive encryption key")
		}

		// Create the erasure coder
		ec, err = skymodules.NewRSSubCode(int(layout.FanoutDataPieces), int(layout.FanoutParityPieces), crypto.SegmentSize)
		if err != nil {
			cancelFunc()
			return nil, errors.AddContext(err, "unable to derive erasure coding settings for fanout")
		}

		// Create the list of chunks from the fanout.
		fanoutChunks, err := layout.DecodeFanoutIntoChunks(fanoutBytes)
		if err != nil {
			cancelFunc()
			return nil, errors.AddContext(err, "error parsing skyfile fanout")
		}

		// Initialize the fanout chunk fetcher loaders. Note that we only
		// initialize the loaders here and we do not block on initializing the
		// actual PCWS objects for every chunk. We will preload a certain amount
		// to improve TTFB, we do not however want to precreate a PCWS for every
		// chunk on initial read because that might put too much strain on the
		// network because of the amount of has sector jobs. The loaders will
		// send a PCWS down the channel when it is ready, the caller (Stream)
		// can block on that channel.
		numChunks := len(fanoutChunks)
		fanoutChunkFetcherLoaders = make([]chunkFetcherLoaderFn, numChunks)
		fanoutChunkFetchersAvailable = make([]chan struct{}, numChunks)
		fanoutChunkFetchers = make([]chunkFetcher, numChunks)
		fanoutChunkErrs = make([]error, numChunks)
		for i := 0; i < numChunks; i++ {
			fanoutChunkFetchersAvailable[i] = make(chan struct{})
		}

		// Create the PCWS loaders
		for i, chunk := range fanoutChunks {
			chunkIndex := uint64(i)
			chunkCopy := chunk
			var once sync.Once
			fanoutChunkFetcherLoaders[i] = func() {
				once.Do(func() {
					pcws, err := r.newPCWSByRoots(dsCtx, chunkCopy, ec, fanoutKey, chunkIndex)
					fanoutChunkErrs[chunkIndex] = err
					fanoutChunkFetchers[chunkIndex] = pcws
					close(fanoutChunkFetchersAvailable[chunkIndex])
				})
			}
		}

		// Launch a portion of the loaders in a goroutine, this makes it so we
		// can return the data source faster and we also preload the first
		// couple of PCWSs, the others are lazy loaded when new data sections
		// get created. Since we use a `minimumLookahead` there, we will load
		// the PCWSs before they are used, giving them some time to execute
		// their has sector jobs prior to being downloaded from.
		err = r.tg.Launch(func() {
			for i := 0; i < len(fanoutChunks) && i < chunkFetchersMaximumPreload; i++ {
				fanoutChunkFetcherLoaders[i]()
			}
		})
		if err != nil {
			cancelFunc()
			return nil, errors.AddContext(err, "unable to launch thread to preload one of the initial chunk fetchers")
		}
	}

	sds := &skylinkDataSource{
		staticID:           skylink.DataSourceID(),
		staticLayout:       layout,
		staticLayoutOffset: int(slOffset),
		staticMetadata:     metadata,
		staticRawMetadata:  rawMetadata,
		staticSkylink:      skylink,

		staticSkylinkSector:          skylinkSector,
		staticDecryptedSkylinkSector: decryptedSkylinkSector,

		staticChunkFetcherLoaders:    fanoutChunkFetcherLoaders,
		staticChunkFetchersAvailable: fanoutChunkFetchersAvailable,
		staticChunkFetchers:          fanoutChunkFetchers,
		staticChunkErrs:              fanoutChunkErrs,

		staticCtx:        dsCtx,
		staticCancelFunc: cancelFunc,
	}
	return sds, nil
}