github.com/cloudreve/Cloudreve/v3@v3.0.0-20240224133659-3edb00a6484c/pkg/filesystem/chunk/chunk.go (about) 1 package chunk 2 3 import ( 4 "context" 5 "fmt" 6 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" 7 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" 8 "github.com/cloudreve/Cloudreve/v3/pkg/request" 9 "github.com/cloudreve/Cloudreve/v3/pkg/util" 10 "io" 11 "os" 12 ) 13 14 const bufferTempPattern = "cdChunk.*.tmp" 15 16 // ChunkProcessFunc callback function for processing a chunk 17 type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error 18 19 // ChunkGroup manage groups of chunks 20 type ChunkGroup struct { 21 file fsctx.FileHeader 22 chunkSize uint64 23 backoff backoff.Backoff 24 enableRetryBuffer bool 25 26 fileInfo *fsctx.UploadTaskInfo 27 currentIndex int 28 chunkNum uint64 29 bufferTemp *os.File 30 } 31 32 func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup { 33 c := &ChunkGroup{ 34 file: file, 35 chunkSize: chunkSize, 36 backoff: backoff, 37 fileInfo: file.Info(), 38 currentIndex: -1, 39 enableRetryBuffer: useBuffer, 40 } 41 42 if c.chunkSize == 0 { 43 c.chunkSize = c.fileInfo.Size 44 } 45 46 if c.fileInfo.Size == 0 { 47 c.chunkNum = 1 48 } else { 49 c.chunkNum = c.fileInfo.Size / c.chunkSize 50 if c.fileInfo.Size%c.chunkSize != 0 { 51 c.chunkNum++ 52 } 53 } 54 55 return c 56 } 57 58 // TempAvailable returns if current chunk temp file is available to be read 59 func (c *ChunkGroup) TempAvailable() bool { 60 if c.bufferTemp != nil { 61 state, _ := c.bufferTemp.Stat() 62 return state != nil && state.Size() == c.Length() 63 } 64 65 return false 66 } 67 68 // Process a chunk with retry logic 69 func (c *ChunkGroup) Process(processor ChunkProcessFunc) error { 70 reader := io.LimitReader(c.file, c.Length()) 71 72 // If useBuffer is enabled, tee the reader to a temp file 73 if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() { 74 c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern) 75 reader = io.TeeReader(reader, c.bufferTemp) 76 } 77 78 if c.bufferTemp != nil { 79 defer func() { 80 if c.bufferTemp != nil { 81 c.bufferTemp.Close() 82 os.Remove(c.bufferTemp.Name()) 83 c.bufferTemp = nil 84 } 85 }() 86 87 // if temp buffer file is available, use it 88 if c.TempAvailable() { 89 if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil { 90 return fmt.Errorf("failed to seek temp file back to chunk start: %w", err) 91 } 92 93 util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name()) 94 reader = io.NopCloser(c.bufferTemp) 95 } 96 } 97 98 err := processor(c, reader) 99 if err != nil { 100 if c.enableRetryBuffer { 101 request.BlackHole(reader) 102 } 103 104 if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next(err) { 105 if c.file.Seekable() { 106 if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil { 107 return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err) 108 } 109 } 110 111 util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err) 112 return c.Process(processor) 113 } 114 115 return err 116 } 117 118 util.Log().Debug("Chunk %d processed", c.currentIndex) 119 return nil 120 } 121 122 // Start returns the byte index of current chunk 123 func (c *ChunkGroup) Start() int64 { 124 return int64(uint64(c.Index()) * c.chunkSize) 125 } 126 127 // Total returns the total length 128 func (c *ChunkGroup) Total() int64 { 129 return int64(c.fileInfo.Size) 130 } 131 132 // Num returns the total chunk number 133 func (c *ChunkGroup) Num() int { 134 return int(c.chunkNum) 135 } 136 137 // RangeHeader returns header value of Content-Range 138 func (c *ChunkGroup) RangeHeader() string { 139 return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total()) 140 } 141 142 // Index returns current chunk index, starts from 0 143 func (c *ChunkGroup) Index() int { 144 return c.currentIndex 145 } 146 147 // Next switch to next chunk, returns whether all chunks are processed 148 func (c *ChunkGroup) Next() bool { 149 c.currentIndex++ 150 c.backoff.Reset() 151 return c.currentIndex < int(c.chunkNum) 152 } 153 154 // Length returns the length of current chunk 155 func (c *ChunkGroup) Length() int64 { 156 contentLength := c.chunkSize 157 if c.Index() == int(c.chunkNum-1) { 158 contentLength = c.fileInfo.Size - c.chunkSize*(c.chunkNum-1) 159 } 160 161 return int64(contentLength) 162 } 163 164 // IsLast returns if current chunk is the last one 165 func (c *ChunkGroup) IsLast() bool { 166 return c.Index() == int(c.chunkNum-1) 167 }