gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/streambuffer.go

package renter

// NOTE: This stream buffer is uninfished in a couple of ways. The first way is
// that it's not possible to cancel fetches. The second way is that fetches are
// not prioritized, there should be a higher priority on data that is closer to
// the current stream offset. The third is that the amount of data which gets
// fetched is not dynamically adjusted. The streamer really should be monitoring
// the total amount of time it takes for a call to the data source to return
// some data, and should buffer accordingly. If auto-adjusting the lookahead
// size, care needs to be taken to ensure not to exceed the
// bytesBufferedPerStream size, as exceeding that will cause issues with the
// lru, and cause data fetches to be evicted before they become useful.

import (
	"context"
	"encoding/hex"
	"io"
	"sync"
	"time"

	"github.com/opentracing/opentracing-go"
	"gitlab.com/SkynetLabs/skyd/build"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/siad/crypto"
	"go.sia.tech/siad/types"

	"gitlab.com/NebulousLabs/errors"
	"gitlab.com/NebulousLabs/fastrand"
	"gitlab.com/NebulousLabs/threadgroup"
)

const (
	// minimumDataSections is set to two because the streamer always tries to
	// buffer at least the current data section and the next data section for
	// the current offset of a stream.
	//
	// Three as a number was considered so that in addition to buffering one
	// piece ahead, a previous piece could also be cached. This was considered
	// to be less valuable than keeping memory requirements low -
	// minimumDataSections is only at play if there is not enough room for
	// multiple cache nodes in the bytesBufferedPerStream.
	minimumDataSections = 2

	// longDownloadThreshold specifies when a download is considered to be
	// taking long. This value might change in the future, it is based on the
	// p99 values for downloads, which is above 3s on some of our servers in
	// production currently.
	longDownloadThreshold = time.Second * 3
)

var (
	// errTimeout is returned when the context cancels before the data is
	// available.
	errTimeout = errors.New("could not get data from data section, context timed out")

	// bytesBufferedPerStream is the total amount of data that gets allocated
	// per stream. If the RequestSize of a stream buffer is less than three
	// times the bytesBufferedPerStream, that much data will be allocated
	// instead.
	//
	// For example, if the RequestSize is 10kb and the bytesBufferedPerStream is
	// 100kb, then each stream is going to buffer 10 segments that are each 10kb
	// long in the LRU.
	//
	// But if the RequestSize is 50kb and the bytesBufferedPerStream is 100kb,
	// then each stream is going to buffer 3 segments that are each 50kb long in
	// the LRU, for a total of 150kb.
	bytesBufferedPerStream = build.Select(build.Var{
		Dev:      uint64(1 << 25), // 32 MiB
		Standard: uint64(1 << 25), // 32 MiB
		Testing:  uint64(1 << 8),  // 256 bytes
	}).(uint64)

	// keepOldBuffersDuration specifies how long a stream buffer will stay in
	// the buffer set after the final stream is closed. This gives some buffer
	// time for a new request to the same resource, without having the data
	// source fully cleared out. This optimization is particularly useful for
	// certain video players and web applications.
	keepOldBuffersDuration = build.Select(build.Var{
		Dev:      time.Second * 15,
		Standard: time.Second * 60,
		Testing:  time.Second * 2,
	}).(time.Duration)

	// minimumLookahead defines the minimum amount that the stream will fetch
	// ahead of the current seek position in a stream.
	//
	// Note that there is a throughput vs. latency tradeoff here. The maximum
	// speed of a stream has an upper bound of the lookahead / latency. So if it
	// takes 1 second to fetch data and the lookahead is 2 MB, the maximum speed
	// of a single stream is going to be 2 MB/s. When Sia is healthy, the
	// latency on a fetch should be under 200ms, which means with a 2 MB
	// lookahead a single stream should be able to do more than 10 MB/s.
	//
	// A smaller minimum lookahead means that less data is being buffered
	// simultaneously, so seek times should be lower. A smaller minimum
	// lookahead becomes less important if we get some way to ensure the earlier
	// parts are prioritized, but we don't have control over that at the moment.
	minimumLookahead = build.Select(build.Var{
		Dev:      uint64(1 << 21), // 2 MiB
		Standard: uint64(1 << 23), // 8 MiB
		Testing:  uint64(1 << 6),  // 64 bytes
	}).(uint64)

	// newDataSectionTimeout is the timeout we enforce when downloading the
	// data for a new data section we just created.
	newDataSectionTimeout = build.Select(build.Var{
		Dev:      30 * time.Second,
		Standard: 2 * time.Minute,
		Testing:  30 * time.Second,
	}).(time.Duration)
)

// streamBufferDataSource is an interface that the stream buffer uses to fetch
// data. This type is internal to the renter as there are plans to expand on the
// type.
type streamBufferDataSource interface {
	// DataSize should return the size of the data. When the streamBuffer is
	// reading from the data source, it will ensure that none of the read calls
	// go beyond the boundary of the data source.
	DataSize() uint64

	// ID returns the ID of the data source. This should be unique to the data
	// source - that is, every data source that returns the same ID should have
	// identical data and be fully interchangeable.
	ID() skymodules.DataSourceID

	// HasRecursiveFanout returns 'true' if the datasource belongs to a
	// skyfile with recursive fanout.
	HasRecursiveFanout() bool

	// Metadata returns the Skyfile metadata of a data source.
	Metadata() skymodules.SkyfileMetadata

	// RawMetadata returns the raw metadata of a data source.
	RawMetadata() []byte

	// ReadBaseSectorPayload reads data from the data source's base sector
	// payload.  This will return an error when called on anything but a
	// small skyfile.
	ReadBaseSectorPayload(off, length uint64) (*downloadResponse, error)

	// ReadFanout reads a single piece root from the fanout of the
	// datasource and returns the proof for that root as well as the offset
	// within the sector.
	ReadFanout(chunkIndex, pieceIndex uint64) ([]byte, []crypto.Hash, uint32, error)

	// Layout returns the Skyfile layout of a data source.
	Layout() skymodules.SkyfileLayout

	// Layout returns the Skyfile layout of a data source.
	RawLayout() (skymodules.SkyfileLayout, []byte, []crypto.Hash)

	// RequestSize should return the request size that the dataSource expects
	// the streamBuffer to use. The streamBuffer will always make ReadAt calls
	// that are of the suggested request size and byte aligned.
	//
	// If the request size is small, many ReadAt calls will be made in parallel.
	// If the dataSource can handle high parallelism, a smaller request size
	// should be recommended to the streamBuffer, because that will reduce
	// latency. If the dataSource cannot handle high parallelism, a larger
	// request size should be used to optimize for total throughput.
	//
	// A general rule of thumb is that the streamer should be able to
	// comfortably handle 100 mbps (high end 4K video) if the user's local
	// connection has that much throughput.
	RequestSize() uint64

	// SilentClose is an io.Closer that does not return an error. The data
	// source is expected to handle any logging or reporting that is necessary
	// if the closing fails.
	SilentClose()

	// Skylink returns the skylink of the datasource.
	Skylink() skymodules.Skylink

	// ReadSection allows the stream buffer to request a specific data
	// section from the data source. It returns a channel containing a
	// download response.
	ReadSection(context.Context, uint64, types.Currency) (<-chan *downloadResponse, error)
}

// dataSection represents a section of data from a data source. The data section
// includes a refcount of how many different streams have the data in their LRU.
// If the refCount is ever set to 0, the data section should be deleted. Because
// the dataSection has no mutex, the refCount falls under the consistency domain
// of the object holding it, which should always be a streamBuffer.
type dataSection struct {
	// staticID uniquely identifies this data section.
	staticID string

	// dataAvailable, externData, externDuration, and externErr work together.
	// The data and error are not allowed to be accessed by external threads
	// until the data available channel has been closed. Once the dataAvailable
	// channel has been closed, externData, externDuration and externErr are to
	// be treated like static fields.
	dataAvailable  chan struct{}
	externDuration time.Duration
	externData     *downloadedData
	externErr      error

	refCount uint64
}

// stream is a single stream that uses a stream buffer. The stream implements
// io.ReadSeeker and io.Closer, and must be closed when it is done being used.
// The stream will cache data, both data that has been accessed recently as well
// as data that is in front of the current read head. The stream buffer is a
// common cache that is used between all streams that are using the same data
// source, allowing each stream to depend on the other streams if data has
// already been loaded.
type stream struct {
	lru    *leastRecentlyUsedCache
	offset uint64

	mu                 sync.Mutex
	staticStreamBuffer *streamBuffer

	staticCacheRatio  float64
	staticContext     context.Context
	staticRepair      bool
	staticSpan        opentracing.Span
	staticReadTimeout time.Duration
}

// streamBuffer is a buffer for a single dataSource.
//
// The streamBuffer uses a threadgroup to ensure that it does not call ReadAt
// after calling SilentClose.
type streamBuffer struct {
	dataSections map[uint64]*dataSection

	// externRefCount is in the same consistency domain as the streamBufferSet,
	// it needs to be incremented and decremented simultaneously with the
	// creation and deletion of the streamBuffer.
	externRefCount uint64

	mu                    sync.Mutex
	staticCache           PersistedLRU
	staticTG              threadgroup.ThreadGroup
	staticDataSource      streamBufferDataSource
	staticStreamBufferSet *streamBufferSet
	staticStreamID        skymodules.DataSourceID
	staticPricePerMS      types.Currency
	staticSpan            opentracing.Span
}

// streamBufferSet tracks all of the stream buffers that are currently active.
// When a new stream is created, the stream buffer set is referenced to check
// whether another stream using the same data source already exists.
type streamBufferSet struct {
	streams map[skymodules.DataSourceID]*streamBuffer

	staticCache          PersistedLRU
	staticStatsCollector *skymodules.DistributionTracker
	staticTG             *threadgroup.ThreadGroup
	mu                   sync.Mutex
}

// newStreamBufferSet initializes and returns a stream buffer set.
func newStreamBufferSet(statsCollector *skymodules.DistributionTracker, tg *threadgroup.ThreadGroup, cache PersistedLRU) *streamBufferSet {
	return &streamBufferSet{
		streams: make(map[skymodules.DataSourceID]*streamBuffer),

		staticCache:          cache,
		staticStatsCollector: statsCollector,
		staticTG:             tg,
	}
}

// callNewStream will create a stream that implements io.Close and
// io.ReadSeeker. A dataSource must be provided for the stream so that the
// stream can fetch data in advance of calls to 'Read' and attempt to provide a
// smooth streaming experience.
//
// The 'sourceID' is a unique identifier for the dataSource which allows
// multiple streams fetching data from the same source to combine their cache.
// This shared cache only comes into play if the streams are simultaneously
// accessing the same data, allowing the buffer to save on memory and access
// latency.
//
// Each stream has a separate LRU for determining what data to buffer. Because
// the LRU is distinct to the stream, the shared cache feature will not result
// in one stream evicting data from another stream's LRU.
func (sbs *streamBufferSet) callNewStream(ctx context.Context, dataSource streamBufferDataSource, initialOffset uint64, timeout time.Duration, pricePerMS types.Currency, cachedSections uint64, repair bool) *stream {
	// Grab the streamBuffer for the provided sourceID. If no streamBuffer for
	// the sourceID exists, create a new one.
	sourceID := dataSource.ID()
	sbs.mu.Lock()
	streamBuf, exists := sbs.streams[sourceID]
	if !exists {
		streamBuf = &streamBuffer{
			dataSections: make(map[uint64]*dataSection),

			staticCache:           sbs.staticCache,
			staticDataSource:      dataSource,
			staticPricePerMS:      pricePerMS,
			staticStreamBufferSet: sbs,
			staticStreamID:        sourceID,
			staticSpan:            opentracing.SpanFromContext(ctx),
		}
		sbs.streams[sourceID] = streamBuf
	} else {
		// Another data source already exists for this content which will be
		// used instead of the input data source. Close the input source.
		dataSource.SilentClose()
	}
	streamBuf.externRefCount++
	sbs.mu.Unlock()
	return streamBuf.managedPrepareNewStream(ctx, initialOffset, timeout, cachedSections, repair)
}

// callNewStreamFromID will check the stream buffer set to see if a stream
// buffer exists for the given data source id. If so, a new stream will be
// created using the data source, and the bool will be set to 'true'. Otherwise,
// the stream returned will be nil and the bool will be set to 'false'.
func (sbs *streamBufferSet) callNewStreamFromID(ctx context.Context, id skymodules.DataSourceID, initialOffset uint64, timeout time.Duration, cachedSections uint64, repair bool) (*stream, bool) {
	sbs.mu.Lock()
	streamBuf, exists := sbs.streams[id]
	if !exists {
		sbs.mu.Unlock()
		return nil, false
	}
	streamBuf.externRefCount++
	sbs.mu.Unlock()
	return streamBuf.managedPrepareNewStream(ctx, initialOffset, timeout, cachedSections, repair), true
}

// managedData will block until the data for a data section is available, and
// then return the data. The data is not safe to modify.
func (ds *dataSection) managedData(ctx context.Context) (_ *downloadedData, err error) {
	start := time.Now()

	// Trace info.
	var duration time.Duration
	if span := opentracing.SpanFromContext(ctx); span != nil {
		span.SetTag("datasection", ds.staticID)
		defer func() {
			span.SetTag("success", err == nil)
			span.SetTag("duration", duration)
			if err != nil {
				span.LogKV("error", err)
				if errors.Contains(err, errTimeout) {
					span.SetTag("timeout", true)
				}
			}
		}()
	}

	select {
	case <-ds.dataAvailable:
		duration = time.Since(start)
	case <-ctx.Done():
		return nil, errTimeout
	}
	return ds.externData, ds.externErr
}

// CacheRatio returns the percentage of data served from  the persisted cache by
// this stream.
func (s *stream) CacheRatio() float64 {
	return s.staticCacheRatio
}

// Close will release all of the resources held by a stream.
//
// Before removing the stream, this function will sleep for some time. This is
// specifically to address the use case where an application may be using the
// same file or resource continuously, but doing so by repeatedly opening new
// connections to siad rather than keeping a single stable connection. Some
// video players do this. On Skynet, most javascript applications do this, as
// the javascript application does not realize that multiple files within the
// app are all part of the same resource. This sleep here to delay the release
// of a resource substantially improves performance in practice, in many cases
// causing a 4x reduction in response latency.
func (s *stream) Close() error {
	// Finish the span
	s.staticSpan.Finish()

	s.staticStreamBuffer.staticStreamBufferSet.staticTG.Launch(func() {
		// Convenience variables.
		sb := s.staticStreamBuffer
		sbs := sb.staticStreamBufferSet
		// Keep the memory for a while after closing unless this stream
		// was used for repairs.
		if !s.staticRepair {
			sbs.staticTG.Sleep(keepOldBuffersDuration)
		}

		// Drop all nodes from the lru.
		s.lru.callEvictAll()

		// Remove the stream from the streamBuffer.
		sbs.managedRemoveStream(sb)
	})
	return nil
}

// Metadata returns the skyfile metadata associated with this stream.
func (s *stream) Metadata() skymodules.SkyfileMetadata {
	return s.staticStreamBuffer.staticDataSource.Metadata()
}

// RawMetadata returns the skyfile metadata associated with this stream.
func (s *stream) RawMetadata() []byte {
	return s.staticStreamBuffer.staticDataSource.RawMetadata()
}

// Layout returns the skyfile layout associated with this stream.
func (s *stream) Layout() skymodules.SkyfileLayout {
	return s.staticStreamBuffer.staticDataSource.Layout()
}

// Layout returns the skyfile layout associated with this stream.
func (s *stream) RawLayout() (skymodules.SkyfileLayout, []byte, []crypto.Hash) {
	return s.staticStreamBuffer.staticDataSource.RawLayout()
}

// Skylink returns the skylink associated with this stream.
func (s *stream) Skylink() skymodules.Skylink {
	return s.staticStreamBuffer.staticDataSource.Skylink()
}

// Read will read data into 'b', returning the number of bytes read and any
// errors. Read will not fill 'b' up all the way if only part of the data is
// available.
func (s *stream) Read(b []byte) (int, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Create a context.
	ctx := s.staticContext
	if s.staticReadTimeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, s.staticReadTimeout)
		defer cancel()
	}

	// Create a child span.
	spanRef := opentracing.ChildOf(s.staticSpan.Context())
	span := opentracing.StartSpan("Read", spanRef)
	defer span.Finish()

	// Attach the span to the ctx.
	ctx = opentracing.ContextWithSpan(ctx, span)

	// Convenience variables.
	dataSize := s.staticStreamBuffer.staticDataSource.DataSize()
	dataSectionSize := s.staticStreamBuffer.staticDataSource.RequestSize()
	sb := s.staticStreamBuffer

	// Check for EOF.
	if s.offset == dataSize {
		return 0, io.EOF
	}

	// Get the index of the current section and the offset within the current
	// section.
	currentSection := s.offset / dataSectionSize
	offsetInSection := s.offset % dataSectionSize

	// Determine how many bytes are remaining within the current section, this
	// forms an upper bound on how many bytes can be read.
	var bytesRemaining uint64
	lastSection := (currentSection+1)*dataSectionSize >= dataSize
	if !lastSection {
		bytesRemaining = dataSectionSize - offsetInSection
	} else {
		bytesRemaining = dataSize - s.offset
	}

	// Determine how many bytes should be read.
	var bytesToRead uint64
	if bytesRemaining > uint64(len(b)) {
		bytesToRead = uint64(len(b))
	} else {
		bytesToRead = bytesRemaining
	}

	// Fetch the dataSection that has the data we want to read.
	sb.mu.Lock()
	dataSection, exists := sb.dataSections[currentSection]
	sb.mu.Unlock()
	if !exists {
		err := errors.New("data section should always in the stream buffer for the current offset of a stream")
		build.Critical(err)
		return 0, err
	}

	// Block until the data is available.
	dd, err := dataSection.managedData(ctx)
	if err != nil {
		return 0, errors.AddContext(err, "read call failed because data section fetch failed")
	}

	// Recover the data into b.
	b = b[:bytesToRead]
	err = dd.RecoverTo(b, int(offsetInSection))
	if err != nil {
		return 0, errors.AddContext(err, "failed to recover data")
	}
	s.offset += bytesToRead

	// Put the section in the cache. This needs to be called every time we
	// access a section for the hit counter to increment. Put only caches
	// once a certain threshold is reached.
	if err := sb.staticCache.Put(sb.staticDataSource.ID(), currentSection, dd); err != nil {
		build.Critical("failed to store response data in cache", err)
	}

	// Send the call to prepare the next data section.
	s.prepareOffset()
	return int(bytesToRead), nil
}

// Seek will move the read head of the stream to the provided offset.
func (s *stream) Seek(offset int64, whence int) (int64, error) {
	// Input checking.
	if offset < 0 {
		return int64(s.offset), errors.New("offset cannot be negative in call to seek")
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	// Update the offset of the stream according to the inputs.
	dataSize := s.staticStreamBuffer.staticDataSource.DataSize()
	switch whence {
	case io.SeekStart:
		s.offset = uint64(offset)
	case io.SeekCurrent:
		newOffset := s.offset + uint64(offset)
		if newOffset > dataSize {
			return int64(s.offset), errors.New("offset cannot seek beyond the bounds of the file")
		}
		s.offset = newOffset
	case io.SeekEnd:
		if uint64(offset) > dataSize {
			return int64(s.offset), errors.New("cannot seek before the front of the file")
		}
		s.offset = dataSize - uint64(offset)
	default:
		return int64(s.offset), errors.New("invalid value for 'whence' in call to seek")
	}

	// Prepare the fetch of the updated offset.
	s.prepareOffset()
	return int64(s.offset), nil
}

// prepareOffset will ensure that the dataSection containing the offset is made
// available in the LRU, and that the following dataSection is also available.
func (s *stream) prepareOffset() {
	// Convenience variables.
	dataSize := s.staticStreamBuffer.staticDataSource.DataSize()
	dataSectionSize := s.staticStreamBuffer.staticDataSource.RequestSize()

	// If the offset is already at the end of the data, there is nothing to do.
	if s.offset == dataSize {
		return
	}

	// Update the current data section. The update call will trigger the
	// streamBuffer to fetch the dataSection if the dataSection is not already
	// in the streamBuffer cache.
	index := s.offset / dataSectionSize
	s.lru.callUpdate(index)

	// If there is a following data section, update that as well. This update is
	// done regardless of the minimumLookahead, we always want to buffer at
	// least one more piece than the current piece.
	nextIndex := index + 1
	if nextIndex*dataSectionSize < dataSize {
		s.lru.callUpdate(nextIndex)
	}

	// For repair streams we don't prepare anymore sections since we usually
	// only download one chunk at-a-time anyway and repairs don't require
	// high latency guarantees.
	if s.staticRepair {
		return
	}

	// Keep adding more pieces to the buffer until we have buffered at least
	// minimumLookahead total data or have reached the end of the stream.
	nextIndex++
	for i := dataSectionSize * 2; i < minimumLookahead && nextIndex*dataSectionSize < dataSize; i += dataSectionSize {
		s.lru.callUpdate(nextIndex)
		nextIndex++
	}
}

// callFetchDataSection will increment the refcount of a dataSection in the
// stream buffer. If the dataSection is not currently available in the stream
// buffer, the data section will be fetched from the dataSource.
func (sb *streamBuffer) callFetchDataSection(index uint64) {
	sb.mu.Lock()
	defer sb.mu.Unlock()

	// Fetch the relevant dataSection, creating a new one if necessary.
	dataSection, exists := sb.dataSections[index]

	// If the data section exists, check if the data is valid.
	if exists {
		var replace bool
		select {
		case <-dataSection.dataAvailable:
			// If the cached section is invalid, replace it.
			replace = dataSection.externErr != nil
		default:
		}
		// If the section exists and shouldn't be replaced, just increment the
		// refcount.
		if !replace {
			// Increment the refcount of the dataSection.
			dataSection.refCount++
			return
		}
	}

	// Otherwise we create a new datasection and either set the refcount to
	// 1 or the previous count + 1.
	refCount := uint64(1)
	if exists {
		refCount = dataSection.refCount + 1
	}

	dataSection = sb.newDataSection(index)
	dataSection.refCount = refCount
}

// callRemoveDataSection will decrement the refcount of a data section in the
// stream buffer. If the refcount reaches zero, the data section will be deleted
// from the stream buffer.
func (sb *streamBuffer) callRemoveDataSection(index uint64) {
	sb.mu.Lock()
	defer sb.mu.Unlock()

	// Fetch the data section.
	ds, exists := sb.dataSections[index]
	if !exists {
		build.Critical("remove called on data section that does not exist")
		return
	}
	// Decrement the refcount.
	ds.refCount--
	// Delete the data section if the refcount has fallen to zero.
	if ds.refCount == 0 {
		// Also set the section in the map to nil for the garabage
		// collector.
		sb.dataSections[index] = nil
		delete(sb.dataSections, index)
	}
	// If there are no more sections we recreate the map to allow for the
	// old one to be garbage collected.
	if len(sb.dataSections) == 0 {
		sb.dataSections = make(map[uint64]*dataSection)
	}
}

// managedPrepareNewStream creates a new stream from an existing stream buffer.
// The ref count for the buffer needs to be incremented under the
// streamBufferSet lock, before this method is called.
func (sb *streamBuffer) managedPrepareNewStream(ctx context.Context, initialOffset uint64, timeout time.Duration, cachedSections uint64, repair bool) *stream {
	// Determine how many data sections the stream should cache.
	dataSize := sb.staticDataSource.DataSize()
	sectionSize := sb.staticDataSource.RequestSize()
	dataSectionsToCache := bytesBufferedPerStream / sectionSize
	if dataSectionsToCache < minimumDataSections {
		dataSectionsToCache = minimumDataSections
	}

	// Check how many sections are already in the persisted cache relative
	// to all potential sections we could cache.
	totalSections := dataSize / sectionSize
	if dataSize%sectionSize != 0 {
		totalSections++
	}
	// We add +1 to the total sections since there is one section reserved
	// for the base sector as well.
	totalSections++

	var cacheRatio float64
	if totalSections > 0 {
		cacheRatio = float64(100*cachedSections/totalSections) / 100
	}

	// Create a stream that points to the stream buffer.
	stream := &stream{
		lru:    newLeastRecentlyUsedCache(dataSectionsToCache, sb),
		offset: initialOffset,

		staticCacheRatio:   cacheRatio,
		staticContext:      sb.staticTG.StopCtx(),
		staticReadTimeout:  timeout,
		staticRepair:       repair,
		staticStreamBuffer: sb,
		staticSpan:         opentracing.SpanFromContext(ctx),
	}
	stream.prepareOffset()
	return stream
}

// newDataSection will create a new data section for the streamBuffer and spin
// up a goroutine to pull the data from the data source.
func (sb *streamBuffer) newDataSection(index uint64) *dataSection {
	// Create a random identifier
	var id [8]byte
	fastrand.Read(id[:])

	// Create the data section, allocating the right number of bytes for the
	// ReadAt call to fill out.
	ds := &dataSection{
		staticID: hex.EncodeToString(id[:]),

		dataAvailable: make(chan struct{}),
	}
	sb.dataSections[index] = ds

	// See if we can fill the data section from the cache.
	lru := sb.staticCache
	data, cached, err := lru.Get(sb.staticDataSource.ID(), index)
	if err != nil {
		build.Critical("failed to read from cache", err)
	}
	if err == nil && cached {
		ds.externData = data
		close(ds.dataAvailable)
		return ds
	}

	// If not, perform the data fetch in a goroutine. The dataAvailable
	// channel will be closed when the data is available.
	go func() {
		defer close(ds.dataAvailable)

		// Create a child span for the data section
		spanRef := opentracing.ChildOf(sb.staticSpan.Context())
		span := opentracing.StartSpan("newDataSection", spanRef)
		span.LogKV("index", index)
		defer func() {
			if ds.externErr != nil {
				span.LogKV("error", ds.externErr)
			}
			span.SetTag("success", ds.externErr == nil)
			span.SetTag("long", ds.externDuration >= longDownloadThreshold)
			span.Finish()
		}()

		// Ensure that the streambuffer has not closed.
		err := sb.staticTG.Add()
		if err != nil {
			ds.externErr = errors.AddContext(err, "stream buffer has been shut down")
			return
		}
		defer sb.staticTG.Done()

		// Limit the time we wait for the section to be downloaded.
		ctx, cancel := context.WithTimeout(sb.staticTG.StopCtx(), newDataSectionTimeout)
		defer cancel()

		// Create a context from our span
		ctx = opentracing.ContextWithSpan(ctx, span)

		// Grab the data from the data source.
		start := time.Now()
		responseChan, err := sb.staticDataSource.ReadSection(ctx, index, sb.staticPricePerMS)
		if err != nil {
			ds.externErr = errors.AddContext(err, "failed to read data section")
			return
		}

		select {
		case response := <-responseChan:
			dd, err := response.Data()
			ds.externErr = errors.AddContext(err, "data section ReadStream failed")
			ds.externDuration = time.Since(start)
			ds.externData = dd

			if ds.externErr == nil {
				// Add datapoint to stats.
				sb.staticStreamBufferSet.staticStatsCollector.AddDataPoint(ds.externDuration)
			}
		case <-sb.staticTG.StopChan():
			ds.externErr = errors.AddContext(errTimeout, "failed to read response from ReadStream")
		}
	}()
	return ds
}

// managedRemoveStream will remove a stream from a stream buffer. If the total
// number of streams using that stream buffer reaches zero, the stream buffer
// will be removed from the stream buffer set.
//
// The reference counter for a stream buffer needs to be in the domain of the
// stream buffer set because the stream buffer needs to be deleted from the
// stream buffer set simultaneously with the reference counter reaching zero.
func (sbs *streamBufferSet) managedRemoveStream(sb *streamBuffer) {
	// Decrement the refcount of the streamBuffer.
	sbs.mu.Lock()
	sb.externRefCount--
	if sb.externRefCount > 0 {
		// streamBuffer still in use, nothing to do.
		sbs.mu.Unlock()
		return
	}
	// Before deletion, nil the entry for the GC.
	sbs.streams[sb.staticStreamID] = nil
	delete(sbs.streams, sb.staticStreamID)

	// Reallocate the map if it is empty to free more memory.
	if len(sbs.streams) == 0 {
		sbs.streams = make(map[skymodules.DataSourceID]*streamBuffer)
	}
	sbs.mu.Unlock()

	// Close out the streamBuffer and its data source. Calling Stop() will block
	// any new calls to ReadAt from executing, and will block until all existing
	// calls are completed. This prevents any issues that could be caused by the
	// data source being accessed after it has been closed.
	sb.staticTG.Stop()
	sb.staticDataSource.SilentClose()
}