github.com/voedger/voedger@v0.0.0-20240520144910-273e84102129/pkg/iblobstoragestg/impl.go (about) 1 /* 2 * Copyright (c) 2021-present Sigma-Soft, Ltd. 3 */ 4 5 package iblobstoragestg 6 7 import ( 8 "bytes" 9 "context" 10 "encoding/binary" 11 "encoding/json" 12 "errors" 13 "fmt" 14 "io" 15 "reflect" 16 17 "github.com/voedger/voedger/pkg/iblobstorage" 18 "github.com/voedger/voedger/pkg/istructs" 19 coreutils "github.com/voedger/voedger/pkg/utils" 20 ) 21 22 type bStorageType struct { 23 appStorage BlobAppStoragePtr 24 now coreutils.TimeFunc 25 } 26 27 func (b *bStorageType) WriteBLOB(ctx context.Context, key iblobstorage.KeyType, descr iblobstorage.DescrType, reader io.Reader, maxSize int64) (err error) { 28 var ( 29 bytesRead int64 30 cCol uint64 31 bucketNumber uint64 = 1 32 pKeyBuf *bytes.Buffer 33 ) 34 state := iblobstorage.BLOBState{ 35 Descr: descr, 36 StartedAt: istructs.UnixMilli(b.now().UnixMilli()), 37 Status: iblobstorage.BLOBStatus_InProcess, 38 } 39 40 err = b.writeState(key, &state) 41 if err != nil { 42 return err 43 } 44 45 buf := make([]byte, 0, chunkSize) 46 47 if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, bucketNumber); err != nil { 48 return 49 } 50 51 for err == nil { 52 var chunkBytes int 53 if ctx.Err() != nil { 54 state.Error = ctx.Err().Error() 55 state.Status = iblobstorage.BLOBStatus_Unknown 56 break 57 } 58 chunkBytes, err = reader.Read(buf[:cap(buf)]) 59 60 if chunkBytes > 0 { 61 buf = buf[:chunkBytes] 62 bytesRead += int64(len(buf)) 63 if bytesRead > maxSize { 64 err = iblobstorage.ErrBLOBSizeQuotaExceeded 65 break 66 } 67 if err = b.writeChunk(pKeyBuf, cCol, &bucketNumber, bytesRead, &buf); err != nil { 68 break 69 } 70 cCol++ 71 } 72 } 73 if errors.Is(err, io.EOF) { 74 err = nil 75 } 76 state.FinishedAt = istructs.UnixMilli(b.now().UnixMilli()) 77 state.Status = iblobstorage.BLOBStatus_Completed 78 state.Size = bytesRead 79 if err != nil { 80 state.Error = err.Error() 81 state.Status = iblobstorage.BLOBStatus_Unknown 82 } 83 if errStatus := b.writeState(key, &state); errStatus != nil { 84 err = errStatus 85 } 86 return err 87 } 88 89 func (b *bStorageType) writeChunk(pKeyBuf *bytes.Buffer, cCol uint64, bucketNumber *uint64, bytesRead int64, buf *[]byte) (err error) { 90 var cColBuf *bytes.Buffer 91 if uint64(bytesRead) > chunkSize*bucketSize*(*bucketNumber) { 92 *bucketNumber++ 93 if errBucket := addBucket(pKeyBuf, *bucketNumber); errBucket != nil { 94 err = errBucket 95 return 96 } 97 } 98 if cColBuf, err = createKey(cCol); err != nil { 99 return 100 } 101 err = (*(b.appStorage)).Put(pKeyBuf.Bytes(), cColBuf.Bytes(), *buf) 102 return 103 } 104 105 func addBucket(pKeyBuf *bytes.Buffer, bucketNumber uint64) (err error) { 106 bucketKey := bytes.NewBuffer(pKeyBuf.Bytes()[:keyLength]) 107 err = binary.Write(bucketKey, binary.LittleEndian, bucketNumber) 108 return 109 } 110 111 func (b *bStorageType) ReadBLOB(ctx context.Context, key iblobstorage.KeyType, stateWriter func(state iblobstorage.BLOBState) error, writer io.Writer) (err error) { 112 var ( 113 bucketNumber uint64 = 1 114 isFound bool 115 state iblobstorage.BLOBState 116 pKeyBuf *bytes.Buffer 117 ) 118 if stateWriter != nil { 119 errState := b.readState(key, &state) 120 if errState != nil { 121 err = errState 122 return 123 } 124 isFound = true 125 errWriterState := stateWriter(state) 126 if errWriterState != nil { 127 err = errWriterState 128 return 129 } 130 } 131 if writer != nil { 132 for ctx.Err() == nil { 133 if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, bucketNumber); err != nil { 134 return err 135 } 136 var n int 137 err = (*(b.appStorage)).Read(ctx, pKeyBuf.Bytes(), nil, nil, 138 func(ccols []byte, viewRecord []byte) (err error) { 139 isFound = true 140 n, err = writer.Write(viewRecord) 141 if err != nil { 142 return err 143 } 144 return nil 145 }) 146 if err != nil { 147 break 148 } 149 if n > 0 { 150 bucketNumber++ 151 } else { 152 break 153 } 154 } 155 } 156 157 if !isFound && err == nil { 158 err = iblobstorage.ErrBLOBNotFound 159 return err 160 } 161 return err 162 } 163 164 func (b *bStorageType) QueryBLOBState(ctx context.Context, key iblobstorage.KeyType) (state iblobstorage.BLOBState, err error) { 165 err = b.ReadBLOB(ctx, key, 166 func(blobState iblobstorage.BLOBState) (err error) { 167 state = blobState 168 return nil 169 }, 170 nil) 171 return 172 } 173 174 func createKey(columns ...interface{}) (buf *bytes.Buffer, err error) { 175 buf = new(bytes.Buffer) 176 for _, col := range columns { 177 switch v := col.(type) { 178 case nil: 179 return nil, fmt.Errorf("key column with type «%s» is missed: %w", reflect.ValueOf(col).Type(), errPKeyCreateError) 180 case appType, istructs.ClusterAppID, istructs.WSID, istructs.RecordID, uint64: 181 if errWrite := binary.Write(buf, binary.LittleEndian, v); errWrite != nil { 182 err = errWrite 183 return nil, fmt.Errorf("error create key: %w", err) 184 } 185 default: 186 return nil, fmt.Errorf("unsupported data type %s: %w", reflect.ValueOf(col).Type(), errPKeyCreateError) 187 } 188 } 189 return buf, nil 190 } 191 192 func (b *bStorageType) readState(key iblobstorage.KeyType, state *iblobstorage.BLOBState) (err error) { 193 var ( 194 currentState []byte 195 ok bool 196 pKeyBuf *bytes.Buffer 197 cColBuf *bytes.Buffer 198 ) 199 if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, zeroBucket); err != nil { 200 return 201 } 202 if cColBuf, err = createKey(zeroCcCol); err != nil { 203 return 204 } 205 if ok, err = (*(b.appStorage)).Get( 206 pKeyBuf.Bytes(), 207 cColBuf.Bytes(), 208 ¤tState); ok { 209 err = json.Unmarshal(currentState, &state) 210 return 211 } 212 if err != nil { 213 return 214 } 215 err = iblobstorage.ErrBLOBNotFound 216 return 217 } 218 219 func (b *bStorageType) writeState(key iblobstorage.KeyType, s interface{}) (err error) { 220 var ( 221 value []byte 222 pKeyBuf *bytes.Buffer 223 cColBuf *bytes.Buffer 224 ) 225 if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, zeroBucket); err != nil { 226 return 227 } 228 if cColBuf, err = createKey(zeroCcCol); err != nil { 229 return 230 } 231 value, err = json.Marshal(s) 232 if err != nil { 233 return fmt.Errorf("error write meta information of blob appType: %d, wsid: %d, blobid: %d, error: %w - marshal to JSON failed ", 234 key.AppID, key.WSID, key.ID, err) 235 } 236 if err = (*(b.appStorage)).Put( 237 pKeyBuf.Bytes(), 238 cColBuf.Bytes(), 239 value); err != nil { 240 return 241 } 242 return nil 243 }