gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/trustlessstreamer.go

package renter

import (
	"encoding/binary"
	"fmt"
	"io"

	"gitlab.com/NebulousLabs/errors"
	"gitlab.com/SkynetLabs/skyd/build"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/core/net/rhp"
	"go.sia.tech/siad/crypto"
)

// BuildSubSectionProof builds a merkle proof by combining a subSectionProof
// leading up to a section root and a sectionProof leading up to a sector root.
// The provided sectionIndex describes the index of the datasection within the
// streambuffer.
func BuildSubSectionProof(sectionIndexInChunk uint64, subSectionProof, sectionProof []crypto.Hash) []crypto.Hash {
	i := rhp.RangeProofSize(sectionIndexInChunk+1, sectionIndexInChunk, sectionIndexInChunk+1)
	left := append([]crypto.Hash{}, sectionProof[:i]...)
	right := append([]crypto.Hash{}, sectionProof[i:]...)
	proof := append(left, subSectionProof...)
	proof = append(proof, right...)
	return proof
}

// writeTrustlessLargeRecursive handles downloading from a large skyfile. Since
// this is more involved than downloading a small file this will be implemented
// in a f/u.
func (s *stream) writeTrustlessLarge(w io.Writer, length uint64) error {
	// Convenience vars.
	sb := s.staticStreamBuffer
	dataSize := sb.staticDataSource.DataSize()

	// Check that the read is within bounds.
	if s.offset+length > dataSize {
		return fmt.Errorf("out-of-bounds read: %v + %v > %v", s.offset, length, dataSize)
	}

	// Write sections one-by-one.
	for length > 0 {
		n, err := s.writeTrustlessLargeSection(w, s.offset, length)
		if err != nil {
			return fmt.Errorf("failed to write section %v: %v", s.offset, err)
		}
		length -= n
	}
	return nil
}

// writeTrustlessLargeSection writes the logical data required for the next
// length bytes of a section plus proves to the connection.
func (s *stream) writeTrustlessLargeSection(w io.Writer, currentOffset, length uint64) (uint64, error) {
	// Helper vars.
	sb := s.staticStreamBuffer
	ds := sb.staticDataSource
	layout := sb.staticDataSource.Layout()

	// Various helper vars related to indices and offsets within chunks/sections.
	currentSection := currentOffset / sb.staticDataSource.RequestSize()
	offsetInSection := currentOffset % sb.staticDataSource.RequestSize()
	chunkSize := skymodules.ChunkSize(layout.CipherType, uint64(layout.FanoutDataPieces))
	sectionSize := sb.staticDataSource.RequestSize()
	chunkIndex := currentOffset / chunkSize
	offsetInChunk := currentOffset % chunkSize
	sectionsPerChunk := chunkSize / sectionSize
	if chunkSize%sectionSize != 0 {
		return 0, fmt.Errorf("chunkSize must be section aligned %v mod %v == %v", chunkSize, sectionSize, chunkSize%sectionSize)
	}

	// Cap the length at a section size.
	if offsetInSection+length > sb.staticDataSource.RequestSize() {
		length = sb.staticDataSource.RequestSize() - offsetInSection
	}

	// Get an erasure coder for the fanout.
	fanoutEC, err := skymodules.NewRSSubCode(int(layout.FanoutDataPieces), int(layout.FanoutParityPieces), crypto.SegmentSize)
	if err != nil {
		return 0, err
	}

	// Compute the sectionIndexInChunk.
	sectionIndexInChunk := currentSection % sectionsPerChunk
	sectionOffInChunk, _ := GetPieceOffsetAndLen(fanoutEC, offsetInChunk/sectionSize*sectionSize, sectionSize)

	// Get the datasection for the current section.
	sb.mu.Lock()
	dataSection, exists := sb.dataSections[currentSection]
	sb.mu.Unlock()
	if !exists {
		err := errors.New("writeTrustLessLarge: data section should always be present in the stream buffer for the current offset of a stream")
		build.Critical(err)
		return 0, err
	}

	// Wait for the data.
	dd, err := dataSection.managedData(s.staticContext)
	if err != nil {
		return 0, err
	}

	// We need to return only the minimum number of pieces
	neededPieces := layout.FanoutDataPieces

	// Write a piece, followed by the proof.
	piecesSent := uint8(0)
	for pieceIndex, piece := range dd.LogicalChunkData {
		if piecesSent >= neededPieces {
			break // don't need more than min pieces even if we got more
		}
		if len(piece) == 0 {
			continue
		}
		proof := dd.Proofs[pieceIndex]
		if len(proof) == 0 {
			build.Critical("shouldn't have an empty proof for a non-empty piece")
			continue
		}

		// Fetch the root of the piece within the fanout, the proof for
		// that root as well as the offset.
		logicalPieceRoot, pieceRootProof, pieceRootOff, err := ds.ReadFanout(chunkIndex, uint64(pieceIndex))
		if err != nil {
			return 0, err
		}
		var pieceRoot crypto.Hash
		copy(pieceRoot[:], logicalPieceRoot[pieceRootOff%crypto.SegmentSize:][:crypto.HashSize])

		// Write the offset of the piece root, followed by the piece root and
		// the proof for the piece root.
		err = binary.Write(w, binary.LittleEndian, pieceRootOff)
		if err != nil {
			return 0, err
		}
		_, err = w.Write(logicalPieceRoot)
		if err != nil {
			return 0, err
		}
		for _, h := range pieceRootProof {
			_, err = w.Write(h[:])
			if err != nil {
				return 0, err
			}
		}

		// Trim the piece according to the requested offset and length.
		pieceOff, pieceLen := GetPieceOffsetAndLen(fanoutEC, offsetInChunk, length)
		pieceOffInChunk := pieceOff - sectionOffInChunk
		trimmedPiece := piece[pieceOffInChunk:][:pieceLen]

		// Finally write the piece and the piece proof.
		_, err = w.Write(trimmedPiece)
		if err != nil {
			return 0, err
		}

		subSectionProofStart := pieceOffInChunk / crypto.SegmentSize
		subSectionProofEnd := (pieceOffInChunk + pieceLen) / crypto.SegmentSize
		subSectionProof := crypto.MerkleRangeProof(piece, int(subSectionProofStart), int(subSectionProofEnd))
		fullPieceProof := BuildSubSectionProof(sectionIndexInChunk, subSectionProof, proof)
		for _, h := range fullPieceProof {
			_, err = w.Write(h[:])
			if err != nil {
				return 0, err
			}
		}

		// Piece sent successfully.
		piecesSent++
	}
	if piecesSent < neededPieces {
		return 0, errors.New("not enough pieces sent")
	}

	// Update offset.
	s.offset += length
	s.prepareOffset()
	return length, nil
}

// writeTrustlessLargeRecursive handles downloading from a large skyfile with a
// recursive fanout.
// NOTE: This is not supported right at the moment.
func (s *stream) writeTrustlessLargeRecursive() error {
	return errors.New("trustless download of recursive fanout skylinks not yet supported")
}

// writeTrustlessSmall handles downloading from a small skyfile.
func (s *stream) writeTrustlessSmall(w io.Writer, length uint64) error {
	sb := s.staticStreamBuffer
	ds := sb.staticDataSource
	dr, err := ds.ReadBaseSectorPayload(s.offset, length)
	if err != nil {
		return err
	}
	dd := dr.externDownloadedData

	// We are sending payload from a base sector. So we should not need to send more
	// than a single piece.
	for pieceIndex, piece := range dd.LogicalChunkData {
		if piece == nil {
			continue
		}
		// Write the piece.
		_, err := w.Write(piece)
		if err != nil {
			return err
		}

		// Write the corresponding proof.
		for _, h := range dd.Proofs[pieceIndex] {
			_, err = w.Write(h[:])
			if err != nil {
				return err
			}
		}
		return nil
	}
	err = errors.New("Failed to send a piece to the client. This should never happen.")
	build.Critical(err)
	return err
}

// TrustlessWrite writes the trustless download of a certain length to the
// writer.
func (s *stream) TrustlessWrite(w io.Writer, length uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	sb := s.staticStreamBuffer
	ds := sb.staticDataSource

	// Fetch the layout and its proof.
	layout, layoutData, proof := ds.RawLayout()

	// Can't request no data.
	if length == 0 {
		return errors.New("can't request 0 byte")
	}

	// Check for out-of-bounds read.
	if s.offset+length > layout.Filesize {
		return fmt.Errorf("read at offset %v and length %v is out-of-bounds for file of size %v", s.offset, length, layout.Filesize)
	}

	// Write metadata first.
	err := binary.Write(w, binary.LittleEndian, uint64(ds.RequestSize()))
	if err != nil {
		return err
	}

	// Write the layout and proof.
	_, err = w.Write(layoutData)
	if err != nil {
		return errors.AddContext(err, "failed to write layout data")
	}

	for _, h := range proof {
		_, err = w.Write(h[:])
		if err != nil {
			return errors.AddContext(err, "failed to write layout proof")
		}
	}

	// Call appropriate handler for the actual data.
	if layout.IsSmallFile() {
		return s.writeTrustlessSmall(w, length)
	}
	if ds.HasRecursiveFanout() {
		return s.writeTrustlessLargeRecursive()
	}
	return s.writeTrustlessLarge(w, length)
}