github.com/voedger/voedger@v0.0.0-20240520144910-273e84102129/pkg/iblobstoragestg/impl.go (about)

     1  /*
     2   * Copyright (c) 2021-present Sigma-Soft, Ltd.
     3   */
     4  
     5  package iblobstoragestg
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/binary"
    11  	"encoding/json"
    12  	"errors"
    13  	"fmt"
    14  	"io"
    15  	"reflect"
    16  
    17  	"github.com/voedger/voedger/pkg/iblobstorage"
    18  	"github.com/voedger/voedger/pkg/istructs"
    19  	coreutils "github.com/voedger/voedger/pkg/utils"
    20  )
    21  
    22  type bStorageType struct {
    23  	appStorage BlobAppStoragePtr
    24  	now        coreutils.TimeFunc
    25  }
    26  
    27  func (b *bStorageType) WriteBLOB(ctx context.Context, key iblobstorage.KeyType, descr iblobstorage.DescrType, reader io.Reader, maxSize int64) (err error) {
    28  	var (
    29  		bytesRead    int64
    30  		cCol         uint64
    31  		bucketNumber uint64 = 1
    32  		pKeyBuf      *bytes.Buffer
    33  	)
    34  	state := iblobstorage.BLOBState{
    35  		Descr:     descr,
    36  		StartedAt: istructs.UnixMilli(b.now().UnixMilli()),
    37  		Status:    iblobstorage.BLOBStatus_InProcess,
    38  	}
    39  
    40  	err = b.writeState(key, &state)
    41  	if err != nil {
    42  		return err
    43  	}
    44  
    45  	buf := make([]byte, 0, chunkSize)
    46  
    47  	if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, bucketNumber); err != nil {
    48  		return
    49  	}
    50  
    51  	for err == nil {
    52  		var chunkBytes int
    53  		if ctx.Err() != nil {
    54  			state.Error = ctx.Err().Error()
    55  			state.Status = iblobstorage.BLOBStatus_Unknown
    56  			break
    57  		}
    58  		chunkBytes, err = reader.Read(buf[:cap(buf)])
    59  
    60  		if chunkBytes > 0 {
    61  			buf = buf[:chunkBytes]
    62  			bytesRead += int64(len(buf))
    63  			if bytesRead > maxSize {
    64  				err = iblobstorage.ErrBLOBSizeQuotaExceeded
    65  				break
    66  			}
    67  			if err = b.writeChunk(pKeyBuf, cCol, &bucketNumber, bytesRead, &buf); err != nil {
    68  				break
    69  			}
    70  			cCol++
    71  		}
    72  	}
    73  	if errors.Is(err, io.EOF) {
    74  		err = nil
    75  	}
    76  	state.FinishedAt = istructs.UnixMilli(b.now().UnixMilli())
    77  	state.Status = iblobstorage.BLOBStatus_Completed
    78  	state.Size = bytesRead
    79  	if err != nil {
    80  		state.Error = err.Error()
    81  		state.Status = iblobstorage.BLOBStatus_Unknown
    82  	}
    83  	if errStatus := b.writeState(key, &state); errStatus != nil {
    84  		err = errStatus
    85  	}
    86  	return err
    87  }
    88  
    89  func (b *bStorageType) writeChunk(pKeyBuf *bytes.Buffer, cCol uint64, bucketNumber *uint64, bytesRead int64, buf *[]byte) (err error) {
    90  	var cColBuf *bytes.Buffer
    91  	if uint64(bytesRead) > chunkSize*bucketSize*(*bucketNumber) {
    92  		*bucketNumber++
    93  		if errBucket := addBucket(pKeyBuf, *bucketNumber); errBucket != nil {
    94  			err = errBucket
    95  			return
    96  		}
    97  	}
    98  	if cColBuf, err = createKey(cCol); err != nil {
    99  		return
   100  	}
   101  	err = (*(b.appStorage)).Put(pKeyBuf.Bytes(), cColBuf.Bytes(), *buf)
   102  	return
   103  }
   104  
   105  func addBucket(pKeyBuf *bytes.Buffer, bucketNumber uint64) (err error) {
   106  	bucketKey := bytes.NewBuffer(pKeyBuf.Bytes()[:keyLength])
   107  	err = binary.Write(bucketKey, binary.LittleEndian, bucketNumber)
   108  	return
   109  }
   110  
   111  func (b *bStorageType) ReadBLOB(ctx context.Context, key iblobstorage.KeyType, stateWriter func(state iblobstorage.BLOBState) error, writer io.Writer) (err error) {
   112  	var (
   113  		bucketNumber uint64 = 1
   114  		isFound      bool
   115  		state        iblobstorage.BLOBState
   116  		pKeyBuf      *bytes.Buffer
   117  	)
   118  	if stateWriter != nil {
   119  		errState := b.readState(key, &state)
   120  		if errState != nil {
   121  			err = errState
   122  			return
   123  		}
   124  		isFound = true
   125  		errWriterState := stateWriter(state)
   126  		if errWriterState != nil {
   127  			err = errWriterState
   128  			return
   129  		}
   130  	}
   131  	if writer != nil {
   132  		for ctx.Err() == nil {
   133  			if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, bucketNumber); err != nil {
   134  				return err
   135  			}
   136  			var n int
   137  			err = (*(b.appStorage)).Read(ctx, pKeyBuf.Bytes(), nil, nil,
   138  				func(ccols []byte, viewRecord []byte) (err error) {
   139  					isFound = true
   140  					n, err = writer.Write(viewRecord)
   141  					if err != nil {
   142  						return err
   143  					}
   144  					return nil
   145  				})
   146  			if err != nil {
   147  				break
   148  			}
   149  			if n > 0 {
   150  				bucketNumber++
   151  			} else {
   152  				break
   153  			}
   154  		}
   155  	}
   156  
   157  	if !isFound && err == nil {
   158  		err = iblobstorage.ErrBLOBNotFound
   159  		return err
   160  	}
   161  	return err
   162  }
   163  
   164  func (b *bStorageType) QueryBLOBState(ctx context.Context, key iblobstorage.KeyType) (state iblobstorage.BLOBState, err error) {
   165  	err = b.ReadBLOB(ctx, key,
   166  		func(blobState iblobstorage.BLOBState) (err error) {
   167  			state = blobState
   168  			return nil
   169  		},
   170  		nil)
   171  	return
   172  }
   173  
   174  func createKey(columns ...interface{}) (buf *bytes.Buffer, err error) {
   175  	buf = new(bytes.Buffer)
   176  	for _, col := range columns {
   177  		switch v := col.(type) {
   178  		case nil:
   179  			return nil, fmt.Errorf("key column with type «%s» is missed: %w", reflect.ValueOf(col).Type(), errPKeyCreateError)
   180  		case appType, istructs.ClusterAppID, istructs.WSID, istructs.RecordID, uint64:
   181  			if errWrite := binary.Write(buf, binary.LittleEndian, v); errWrite != nil {
   182  				err = errWrite
   183  				return nil, fmt.Errorf("error create key: %w", err)
   184  			}
   185  		default:
   186  			return nil, fmt.Errorf("unsupported data type %s:  %w", reflect.ValueOf(col).Type(), errPKeyCreateError)
   187  		}
   188  	}
   189  	return buf, nil
   190  }
   191  
   192  func (b *bStorageType) readState(key iblobstorage.KeyType, state *iblobstorage.BLOBState) (err error) {
   193  	var (
   194  		currentState []byte
   195  		ok           bool
   196  		pKeyBuf      *bytes.Buffer
   197  		cColBuf      *bytes.Buffer
   198  	)
   199  	if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, zeroBucket); err != nil {
   200  		return
   201  	}
   202  	if cColBuf, err = createKey(zeroCcCol); err != nil {
   203  		return
   204  	}
   205  	if ok, err = (*(b.appStorage)).Get(
   206  		pKeyBuf.Bytes(),
   207  		cColBuf.Bytes(),
   208  		&currentState); ok {
   209  		err = json.Unmarshal(currentState, &state)
   210  		return
   211  	}
   212  	if err != nil {
   213  		return
   214  	}
   215  	err = iblobstorage.ErrBLOBNotFound
   216  	return
   217  }
   218  
   219  func (b *bStorageType) writeState(key iblobstorage.KeyType, s interface{}) (err error) {
   220  	var (
   221  		value   []byte
   222  		pKeyBuf *bytes.Buffer
   223  		cColBuf *bytes.Buffer
   224  	)
   225  	if pKeyBuf, err = createKey(blobberAppID, key.AppID, key.WSID, key.ID, zeroBucket); err != nil {
   226  		return
   227  	}
   228  	if cColBuf, err = createKey(zeroCcCol); err != nil {
   229  		return
   230  	}
   231  	value, err = json.Marshal(s)
   232  	if err != nil {
   233  		return fmt.Errorf("error write meta information of blob appType: %d, wsid: %d, blobid: %d,  error: %w - marshal to JSON failed ",
   234  			key.AppID, key.WSID, key.ID, err)
   235  	}
   236  	if err = (*(b.appStorage)).Put(
   237  		pKeyBuf.Bytes(),
   238  		cColBuf.Bytes(),
   239  		value); err != nil {
   240  		return
   241  	}
   242  	return nil
   243  }