github.com/0chain/gosdk@v1.17.11/zboxcore/sdk/chunked_upload_model.go (about)

     1  package sdk
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/json"
     7  	"hash/fnv"
     8  	"io"
     9  	"strconv"
    10  	"sync"
    11  	"time"
    12  
    13  	"github.com/0chain/gosdk/core/common"
    14  	"github.com/0chain/gosdk/zboxcore/allocationchange"
    15  	"github.com/0chain/gosdk/zboxcore/encryption"
    16  	"github.com/0chain/gosdk/zboxcore/fileref"
    17  	"github.com/0chain/gosdk/zboxcore/zboxutil"
    18  	"github.com/google/uuid"
    19  	"github.com/klauspost/reedsolomon"
    20  	"golang.org/x/crypto/sha3"
    21  )
    22  
    23  // ChunkedUpload upload manager with chunked upload feature
    24  type ChunkedUpload struct {
    25  	consensus Consensus
    26  
    27  	workdir string
    28  
    29  	allocationObj  *Allocation
    30  	progress       UploadProgress
    31  	progressStorer ChunkedUploadProgressStorer
    32  	client         zboxutil.HttpClient
    33  
    34  	uploadMask zboxutil.Uint128
    35  
    36  	// httpMethod POST = Upload File / PUT = Update file
    37  	httpMethod  string
    38  	buildChange func(ref *fileref.FileRef,
    39  		uid uuid.UUID, timestamp common.Timestamp) allocationchange.AllocationChange
    40  
    41  	fileMeta           FileMeta
    42  	fileReader         io.Reader
    43  	fileErasureEncoder reedsolomon.Encoder
    44  	fileEncscheme      encryption.EncryptionScheme
    45  	fileHasher         Hasher
    46  
    47  	thumbnailBytes         []byte
    48  	thumbailErasureEncoder reedsolomon.Encoder
    49  
    50  	chunkReader ChunkedUploadChunkReader
    51  	formBuilder ChunkedUploadFormBuilder
    52  
    53  	// encryptOnUpload encrypt data on upload or not.
    54  	encryptOnUpload bool
    55  	// webStreaming whether data has to be encoded.
    56  	webStreaming bool
    57  	// chunkSize how much bytes a chunk has. 64KB is default value.
    58  	chunkSize int64
    59  	// chunkNumber the number of chunks in a http upload request. 100 is default value
    60  	chunkNumber int
    61  
    62  	// shardUploadedSize how much bytes a shard has. it is original size
    63  	shardUploadedSize int64
    64  	// shardUploadedThumbnailSize how much thumbnail bytes a shard has. it is original size
    65  	shardUploadedThumbnailSize int64
    66  	// size of shard
    67  	shardSize int64
    68  
    69  	// statusCallback trigger progress on StatusCallback
    70  	statusCallback StatusCallback
    71  
    72  	blobbers []*ChunkedUploadBlobber
    73  
    74  	writeMarkerMutex *WriteMarkerMutex
    75  
    76  	// isRepair identifies if upload is repair operation
    77  	isRepair bool
    78  
    79  	opCode            int
    80  	uploadTimeOut     time.Duration
    81  	commitTimeOut     time.Duration
    82  	maskMu            *sync.Mutex
    83  	ctx               context.Context
    84  	ctxCncl           context.CancelCauseFunc
    85  	addConsensus      int32
    86  	encryptedKeyPoint string
    87  	encryptedKey      string
    88  	uploadChan        chan UploadData
    89  	uploadWG          sync.WaitGroup
    90  	uploadWorkers     int
    91  	//used in wasm check chunked_upload_process_js.go
    92  	listenChan chan struct{} //nolint:unused
    93  	//used in wasm check chunked_upload_process_js.go
    94  	processMap map[int]zboxutil.Uint128 //nolint:unused
    95  	//used in wasm check chunked_upload_process_js.go
    96  	processMapLock sync.Mutex //nolint:unused
    97  }
    98  
    99  // FileMeta metadata of stream input/local
   100  type FileMeta struct {
   101  	// Mimetype mime type of source file
   102  	MimeType string
   103  
   104  	// Path local path of source file
   105  	Path string
   106  	// ThumbnailPath local path of source thumbnail
   107  	ThumbnailPath string
   108  
   109  	// ActualHash hash of original file (un-encoded, un-encrypted)
   110  	ActualHash string
   111  	// ActualSize total bytes of  original file (unencoded, un-encrypted).  it is 0 if input is live stream.
   112  	ActualSize int64
   113  	// ActualThumbnailSize total bytes of original thumbnail (un-encoded, un-encrypted)
   114  	ActualThumbnailSize int64
   115  	// ActualThumbnailHash hash of original thumbnail (un-encoded, un-encrypted)
   116  	ActualThumbnailHash string
   117  
   118  	//RemoteName remote file name
   119  	RemoteName string
   120  	// RemotePath remote path
   121  	RemotePath string
   122  	// CustomMeta custom meta data
   123  	CustomMeta string
   124  }
   125  
   126  // FileID generate id of progress on local cache
   127  func (meta *FileMeta) FileID() string {
   128  
   129  	hash := fnv.New64a()
   130  	hash.Write([]byte(meta.Path + "_" + meta.RemotePath))
   131  
   132  	return strconv.FormatUint(hash.Sum64(), 36) + "_" + meta.RemoteName
   133  }
   134  
   135  // UploadFormData form data of upload
   136  type UploadFormData struct {
   137  	ConnectionID string `json:"connection_id,omitempty"`
   138  	// Filename remote file name
   139  	Filename string `json:"filename,omitempty"`
   140  	// Path remote path
   141  	Path string `json:"filepath,omitempty"`
   142  
   143  	// ValidationRoot is merkle root of sha256 of 64KB as leaf
   144  	ValidationRoot          string `json:"validation_root,omitempty"`
   145  	ValidationRootSignature string `json:"validation_root_signature,omitempty"`
   146  	// Hash hash of shard thumbnail  (encoded,encrypted)
   147  	ThumbnailContentHash string `json:"thumbnail_content_hash,omitempty"`
   148  
   149  	// ChallengeHash challenge hash of shard data (encoded, encrypted)
   150  	FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"`
   151  
   152  	// ActualHash hash of original file (un-encoded, un-encrypted)
   153  	ActualHash              string `json:"actual_hash,omitempty"`
   154  	ActualFileHashSignature string `json:"actual_file_hash_signature,omitempty"`
   155  	// ActualSize total bytes of original file (un-encoded, un-encrypted)
   156  	ActualSize int64 `json:"actual_size,omitempty"`
   157  	// ActualThumbnailSize total bytes of original thumbnail (un-encoded, un-encrypted)
   158  	ActualThumbSize int64 `json:"actual_thumb_size,omitempty"`
   159  	// ActualThumbnailHash hash of original thumbnail (un-encoded, un-encrypted)
   160  	ActualThumbHash string `json:"actual_thumb_hash,omitempty"`
   161  
   162  	MimeType          string `json:"mimetype,omitempty"`
   163  	CustomMeta        string `json:"custom_meta,omitempty"`
   164  	EncryptedKey      string `json:"encrypted_key,omitempty"`
   165  	EncryptedKeyPoint string `json:"encrypted_key_point,omitempty"`
   166  
   167  	IsFinal         bool  `json:"is_final,omitempty"`          // all of chunks are uploaded
   168  	ChunkStartIndex int   `json:"chunk_start_index,omitempty"` // start index of chunks.
   169  	ChunkEndIndex   int   `json:"chunk_end_index,omitempty"`   // end index of chunks. all chunks MUST be uploaded one by one because of streaming merkle hash
   170  	ChunkSize       int64 `json:"chunk_size,omitempty"`        // the size of a chunk. 64*1024 is default
   171  	UploadOffset    int64 `json:"upload_offset,omitempty"`     // It is next position that new incoming chunk should be append to
   172  	Size            int64 `json:"size"`                        // total size of shard
   173  
   174  }
   175  
   176  // UploadProgress progress of upload
   177  type UploadProgress struct {
   178  	ID string `json:"id"`
   179  	// Lat updated time
   180  	LastUpdated common.Timestamp `json:"last_updated,omitempty"`
   181  	// ChunkSize size of chunk
   182  	ChunkSize   int64 `json:"chunk_size,omitempty"`
   183  	ActualSize  int64 `json:"actual_size,omitempty"`
   184  	ChunkNumber int   `json:"chunk_number,omitempty"`
   185  	// EncryptOnUpload encrypt data on upload or not
   186  	EncryptOnUpload   bool   `json:"is_encrypted,omitempty"`
   187  	EncryptPrivateKey string `json:"-"`
   188  	EncryptedKeyPoint string `json:"encrypted_key_point,omitempty"`
   189  
   190  	// ConnectionID chunked upload connection_id
   191  	ConnectionID string `json:"connection_id,omitempty"`
   192  	// ChunkIndex index of last updated chunk
   193  	ChunkIndex int `json:"chunk_index,omitempty"`
   194  	// UploadLength total bytes that has been uploaded to blobbers
   195  	UploadLength int64 `json:"-"`
   196  	// ReadLength total bytes that has been read from original reader (un-encoded, un-encrypted)
   197  	ReadLength int64            `json:"-"`
   198  	UploadMask zboxutil.Uint128 `json:"upload_mask"`
   199  
   200  	Blobbers []*UploadBlobberStatus `json:"-"`
   201  }
   202  
   203  // UploadBlobberStatus the status of blobber's upload progress
   204  type UploadBlobberStatus struct {
   205  	Hasher Hasher
   206  
   207  	// UploadLength total bytes that has been uploaded to blobbers
   208  	UploadLength int64 `json:"upload_length,omitempty"`
   209  }
   210  
   211  //			err = b.sendUploadRequest(ctx, su, chunkEndIndex, isFinal, su.encryptedKey, body, formData, pos)
   212  
   213  type UploadData struct {
   214  	chunkStartIndex int
   215  	chunkEndIndex   int
   216  	isFinal         bool
   217  	uploadLength    int64
   218  	uploadBody      []blobberData
   219  }
   220  
   221  type blobberData struct {
   222  	dataBuffers  []*bytes.Buffer
   223  	formData     ChunkedUploadFormMetadata
   224  	contentSlice []string
   225  }
   226  
   227  type status struct {
   228  	Hasher       hasher
   229  	UploadLength int64 `json:"upload_length,omitempty"`
   230  }
   231  
   232  func (s *UploadBlobberStatus) UnmarshalJSON(b []byte) error {
   233  	if s == nil {
   234  		return nil
   235  	}
   236  	//fixed Hasher doesn't work in UnmarshalJSON
   237  	status := &status{}
   238  
   239  	if err := json.Unmarshal(b, status); err != nil {
   240  		return err
   241  	}
   242  
   243  	status.Hasher.File = sha3.New256()
   244  
   245  	s.Hasher = &status.Hasher
   246  	s.UploadLength = status.UploadLength
   247  
   248  	return nil
   249  }
   250  
   251  type blobberShards [][]byte
   252  
   253  // batchChunksData chunks data
   254  type batchChunksData struct {
   255  	// chunkStartIndex start index of chunks
   256  	chunkStartIndex int
   257  	// chunkEndIndex end index of chunks
   258  	chunkEndIndex int
   259  	// isFinal last chunk or not
   260  	isFinal bool
   261  	// ReadSize total size read from original reader (un-encoded, un-encrypted)
   262  	totalReadSize int64
   263  	// FragmentSize total fragment size for a blobber (un-encrypted)
   264  	totalFragmentSize int64
   265  
   266  	fileShards      []blobberShards
   267  	thumbnailShards blobberShards
   268  }