github.com/0chain/gosdk@v1.17.11/zboxcore/sdk/chunked_upload_form_builder.go

package sdk

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"io"
	"mime/multipart"
	"sync"

	"github.com/0chain/gosdk/zboxcore/client"

	"golang.org/x/crypto/sha3"
)

// ChunkedUploadFormBuilder build form data for uploading
type ChunkedUploadFormBuilder interface {
	// build form data
	Build(
		fileMeta *FileMeta, hasher Hasher, connectionID string,
		chunkSize int64, chunkStartIndex, chunkEndIndex int,
		isFinal bool, encryptedKey, encryptedKeyPoint string, fileChunksData [][]byte,
		thumbnailChunkData []byte, shardSize int64,
	) (blobberData, error)
}

// ChunkedUploadFormMetadata upload form metadata
type ChunkedUploadFormMetadata struct {
	FileBytesLen         int
	ThumbnailBytesLen    int
	ContentType          string
	FixedMerkleRoot      string
	ValidationRoot       string
	ThumbnailContentHash string
}

// CreateChunkedUploadFormBuilder create ChunkedUploadFormBuilder instance
func CreateChunkedUploadFormBuilder() ChunkedUploadFormBuilder {
	return &chunkedUploadFormBuilder{}
}

type chunkedUploadFormBuilder struct {
}

const MAX_BLOCKS = 80 // 5MB(CHUNK_SIZE*80)

func (b *chunkedUploadFormBuilder) Build(
	fileMeta *FileMeta, hasher Hasher, connectionID string,
	chunkSize int64, chunkStartIndex, chunkEndIndex int,
	isFinal bool, encryptedKey, encryptedKeyPoint string, fileChunksData [][]byte,
	thumbnailChunkData []byte, shardSize int64,
) (blobberData, error) {

	metadata := ChunkedUploadFormMetadata{
		ThumbnailBytesLen: len(thumbnailChunkData),
	}

	var res blobberData

	if len(fileChunksData) == 0 {
		return res, nil
	}

	numBodies := len(fileChunksData) / MAX_BLOCKS
	if len(fileChunksData)%MAX_BLOCKS > 0 {
		numBodies++
	}
	dataBuffers := make([]*bytes.Buffer, 0, numBodies)
	contentSlice := make([]string, 0, numBodies)

	formData := UploadFormData{
		ConnectionID: connectionID,
		Filename:     fileMeta.RemoteName,
		Path:         fileMeta.RemotePath,

		ActualSize: fileMeta.ActualSize,

		ActualThumbHash: fileMeta.ActualThumbnailHash,
		ActualThumbSize: fileMeta.ActualThumbnailSize,

		MimeType: fileMeta.MimeType,

		// IsFinal:           isFinal,
		ChunkSize:         chunkSize,
		ChunkStartIndex:   chunkStartIndex,
		ChunkEndIndex:     chunkEndIndex,
		UploadOffset:      chunkSize * int64(chunkStartIndex),
		Size:              shardSize,
		EncryptedKeyPoint: encryptedKeyPoint,
		EncryptedKey:      encryptedKey,
		CustomMeta:        fileMeta.CustomMeta,
	}

	for i := 0; i < numBodies; i++ {

		startRange := i * MAX_BLOCKS
		endRange := startRange + MAX_BLOCKS
		if endRange > len(fileChunksData) {
			endRange = len(fileChunksData)
		}
		buff := formDataPool.Get()
		bufSize := (CHUNK_SIZE * (endRange - startRange)) + 1024
		if cap(buff.B) < bufSize {
			buff.B = make([]byte, 0, bufSize)
		}
		bodyBuf := buff.B

		body := bytes.NewBuffer(bodyBuf)
		formWriter := multipart.NewWriter(body)
		defer formWriter.Close()

		uploadFile, err := formWriter.CreateFormFile("uploadFile", formData.Filename)
		if err != nil {
			return res, err
		}

		for _, chunkBytes := range fileChunksData[startRange:endRange] {
			_, err = uploadFile.Write(chunkBytes)
			if err != nil {
				return res, err
			}

			err = hasher.WriteToFixedMT(chunkBytes)
			if err != nil {
				return res, err
			}

			err = hasher.WriteToValidationMT(chunkBytes)
			if err != nil {
				return res, err
			}

			metadata.FileBytesLen += len(chunkBytes)
		}

		if isFinal && i == numBodies-1 {
			err = hasher.Finalize()
			if err != nil {
				return res, err
			}

			var (
				wg      sync.WaitGroup
				errChan = make(chan error, 2)
			)
			wg.Add(2)
			go func() {
				formData.FixedMerkleRoot, err = hasher.GetFixedMerkleRoot()
				if err != nil {
					errChan <- err
				}
				wg.Done()
			}()
			go func() {
				formData.ValidationRoot, err = hasher.GetValidationRoot()
				if err != nil {
					errChan <- err
				}
				wg.Done()
			}()
			wg.Wait()
			close(errChan)
			for err := range errChan {
				return res, err
			}
			actualHashSignature, err := client.Sign(fileMeta.ActualHash)
			if err != nil {
				return res, err
			}

			validationRootSignature, err := client.Sign(actualHashSignature + formData.ValidationRoot)
			if err != nil {
				return res, err
			}

			formData.ActualHash = fileMeta.ActualHash
			formData.ActualFileHashSignature = actualHashSignature
			formData.ValidationRootSignature = validationRootSignature
			formData.ActualSize = fileMeta.ActualSize

		}

		thumbnailSize := len(thumbnailChunkData)
		if thumbnailSize > 0 && i == 0 {

			uploadThumbnailFile, err := formWriter.CreateFormFile("uploadThumbnailFile", fileMeta.RemoteName+".thumb")
			if err != nil {

				return res, err
			}

			thumbnailHash := sha3.New256()
			thumbnailWriters := io.MultiWriter(uploadThumbnailFile, thumbnailHash)
			_, err = thumbnailWriters.Write(thumbnailChunkData)
			if err != nil {
				return res, err
			}
			_, err = thumbnailHash.Write([]byte(fileMeta.RemotePath))
			if err != nil {
				return res, err
			}
			formData.ActualThumbSize = fileMeta.ActualThumbnailSize
			formData.ThumbnailContentHash = hex.EncodeToString(thumbnailHash.Sum(nil))

		}
		if i > 0 {
			formData.UploadOffset = formData.UploadOffset + chunkSize*int64(MAX_BLOCKS)
		}

		err = formWriter.WriteField("connection_id", connectionID)
		if err != nil {
			return res, err
		}

		if isFinal && i == numBodies-1 {
			formData.IsFinal = true
		}

		uploadMeta, err := json.Marshal(formData)
		if err != nil {
			return res, err
		}

		err = formWriter.WriteField("uploadMeta", string(uploadMeta))
		if err != nil {
			return res, err
		}

		contentSlice = append(contentSlice, formWriter.FormDataContentType())
		dataBuffers = append(dataBuffers, body)
	}
	metadata.FixedMerkleRoot = formData.FixedMerkleRoot
	metadata.ValidationRoot = formData.ValidationRoot
	metadata.ThumbnailContentHash = formData.ThumbnailContentHash
	res.dataBuffers = dataBuffers
	res.contentSlice = contentSlice
	res.formData = metadata
	return res, nil
}