github.com/koko1123/flow-go-1@v0.29.6/module/executiondatasync/execution_data/store.go (about) 1 package execution_data 2 3 import ( 4 "bytes" 5 "context" 6 "errors" 7 "fmt" 8 9 "github.com/ipfs/go-cid" 10 11 "github.com/koko1123/flow-go-1/model/flow" 12 "github.com/koko1123/flow-go-1/module/blobs" 13 ) 14 15 // ExecutionDataStore handles adding / getting execution data to / from a local blobstore 16 type ExecutionDataStore interface { 17 // GetExecutionData gets the BlockExecutionData for the given root ID from the blobstore. 18 // The returned error will be: 19 // - MalformedDataError if some level of the blob tree cannot be properly deserialized 20 // - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore 21 GetExecutionData(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) 22 23 // AddExecutionData constructs a blob tree for the given BlockExecutionData and adds it to the 24 // blobstore, and then returns the root CID. 25 AddExecutionData(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) 26 } 27 28 type ExecutionDataStoreOption func(*store) 29 30 // WithMaxBlobSize configures the maximum blob size of the store 31 func WithMaxBlobSize(size int) ExecutionDataStoreOption { 32 return func(s *store) { 33 s.maxBlobSize = size 34 } 35 } 36 37 type store struct { 38 blobstore blobs.Blobstore 39 serializer Serializer 40 maxBlobSize int 41 } 42 43 // NewExecutionDataStore creates a new Execution Data Store. 44 func NewExecutionDataStore(blobstore blobs.Blobstore, serializer Serializer, opts ...ExecutionDataStoreOption) *store { 45 s := &store{ 46 blobstore: blobstore, 47 serializer: serializer, 48 maxBlobSize: DefaultMaxBlobSize, 49 } 50 51 for _, opt := range opts { 52 opt(s) 53 } 54 55 return s 56 } 57 58 func (s *store) AddExecutionData(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) { 59 executionDataRoot := &BlockExecutionDataRoot{ 60 BlockID: executionData.BlockID, 61 ChunkExecutionDataIDs: make([]cid.Cid, len(executionData.ChunkExecutionDatas)), 62 } 63 64 for i, chunkExecutionData := range executionData.ChunkExecutionDatas { 65 chunkExecutionDataID, err := s.addChunkExecutionData(ctx, chunkExecutionData) 66 if err != nil { 67 return flow.ZeroID, fmt.Errorf("could not add chunk execution data at index %d: %w", i, err) 68 } 69 70 executionDataRoot.ChunkExecutionDataIDs[i] = chunkExecutionDataID 71 } 72 73 buf := new(bytes.Buffer) 74 if err := s.serializer.Serialize(buf, executionDataRoot); err != nil { 75 return flow.ZeroID, fmt.Errorf("could not serialize execution data root: %w", err) 76 } 77 78 if buf.Len() > s.maxBlobSize { 79 return flow.ZeroID, errors.New("root blob exceeds blob size limit") 80 } 81 82 rootBlob := blobs.NewBlob(buf.Bytes()) 83 if err := s.blobstore.Put(ctx, rootBlob); err != nil { 84 return flow.ZeroID, fmt.Errorf("could not add execution data root: %w", err) 85 } 86 87 rootID, err := flow.CidToId(rootBlob.Cid()) 88 if err != nil { 89 return flow.ZeroID, fmt.Errorf("could not get root ID: %w", err) 90 } 91 92 return rootID, nil 93 } 94 95 func (s *store) addChunkExecutionData(ctx context.Context, chunkExecutionData *ChunkExecutionData) (cid.Cid, error) { 96 var v interface{} = chunkExecutionData 97 98 for i := 0; ; i++ { 99 cids, err := s.addBlobs(ctx, v) 100 if err != nil { 101 return cid.Undef, fmt.Errorf("failed to add blob tree level at height %d: %w", i, err) 102 } 103 104 if len(cids) == 1 { 105 return cids[0], nil 106 } 107 108 v = cids 109 } 110 } 111 112 func (s *store) addBlobs(ctx context.Context, v interface{}) ([]cid.Cid, error) { 113 buf := new(bytes.Buffer) 114 if err := s.serializer.Serialize(buf, v); err != nil { 115 return nil, fmt.Errorf("could not serialize execution data root: %w", err) 116 } 117 118 data := buf.Bytes() 119 var cids []cid.Cid 120 var blbs []blobs.Blob 121 122 for len(data) > 0 { 123 blobLen := s.maxBlobSize 124 if len(data) < blobLen { 125 blobLen = len(data) 126 } 127 128 blob := blobs.NewBlob(data[:blobLen]) 129 data = data[blobLen:] 130 blbs = append(blbs, blob) 131 cids = append(cids, blob.Cid()) 132 } 133 134 if err := s.blobstore.PutMany(ctx, blbs); err != nil { 135 return nil, fmt.Errorf("could not add blobs: %w", err) 136 } 137 138 return cids, nil 139 } 140 141 func (s *store) GetExecutionData(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) { 142 rootCid := flow.IdToCid(rootID) 143 144 rootBlob, err := s.blobstore.Get(ctx, rootCid) 145 if err != nil { 146 if errors.Is(err, blobs.ErrNotFound) { 147 return nil, NewBlobNotFoundError(rootCid) 148 } 149 150 return nil, fmt.Errorf("failed to get root blob: %w", err) 151 } 152 153 rootData, err := s.serializer.Deserialize(bytes.NewBuffer(rootBlob.RawData())) 154 if err != nil { 155 return nil, NewMalformedDataError(err) 156 } 157 158 executionDataRoot, ok := rootData.(*BlockExecutionDataRoot) 159 if !ok { 160 return nil, NewMalformedDataError(fmt.Errorf("root blob does not deserialize to a BlockExecutionDataRoot, got %T instead", rootData)) 161 } 162 163 blockExecutionData := &BlockExecutionData{ 164 BlockID: executionDataRoot.BlockID, 165 ChunkExecutionDatas: make([]*ChunkExecutionData, len(executionDataRoot.ChunkExecutionDataIDs)), 166 } 167 168 for i, chunkExecutionDataID := range executionDataRoot.ChunkExecutionDataIDs { 169 chunkExecutionData, err := s.getChunkExecutionData(ctx, chunkExecutionDataID) 170 if err != nil { 171 return nil, fmt.Errorf("could not get chunk execution data at index %d: %w", i, err) 172 } 173 174 blockExecutionData.ChunkExecutionDatas[i] = chunkExecutionData 175 } 176 177 return blockExecutionData, nil 178 } 179 180 func (s *store) getChunkExecutionData(ctx context.Context, chunkExecutionDataID cid.Cid) (*ChunkExecutionData, error) { 181 cids := []cid.Cid{chunkExecutionDataID} 182 183 for i := 0; ; i++ { 184 v, err := s.getBlobs(ctx, cids) 185 if err != nil { 186 return nil, fmt.Errorf("failed to get blob tree level at depth %d: %w", i, err) 187 } 188 189 switch v := v.(type) { 190 case *ChunkExecutionData: 191 return v, nil 192 case *[]cid.Cid: 193 cids = *v 194 default: 195 return nil, NewMalformedDataError(fmt.Errorf("blob tree contains unexpected type %T at level %d", v, i)) 196 } 197 } 198 } 199 200 func (s *store) getBlobs(ctx context.Context, cids []cid.Cid) (interface{}, error) { 201 buf := new(bytes.Buffer) 202 203 for _, cid := range cids { 204 blob, err := s.blobstore.Get(ctx, cid) 205 if err != nil { 206 if errors.Is(err, blobs.ErrNotFound) { 207 return nil, NewBlobNotFoundError(cid) 208 } 209 210 return nil, fmt.Errorf("failed to get blob: %w", err) 211 } 212 213 _, err = buf.Write(blob.RawData()) 214 if err != nil { 215 return nil, fmt.Errorf("failed to write blob %s to deserialization buffer: %w", cid.String(), err) 216 } 217 } 218 219 v, err := s.serializer.Deserialize(buf) 220 if err != nil { 221 return nil, NewMalformedDataError(err) 222 } 223 224 return v, nil 225 } 226 227 // MalformedDataError is returned when malformed data is found at some level of the requested 228 // blob tree. It likely indicates that the tree was generated incorrectly, and hence the request 229 // should not be retried. 230 type MalformedDataError struct { 231 err error 232 } 233 234 func NewMalformedDataError(err error) *MalformedDataError { 235 return &MalformedDataError{err: err} 236 } 237 238 func (e *MalformedDataError) Error() string { 239 return fmt.Sprintf("malformed data: %v", e.err) 240 } 241 242 func (e *MalformedDataError) Unwrap() error { return e.err } 243 244 // IsMalformedDataError returns whether an error is MalformedDataError 245 func IsMalformedDataError(err error) bool { 246 var malformedDataErr *MalformedDataError 247 return errors.As(err, &malformedDataErr) 248 } 249 250 // BlobNotFoundError is returned when a blob could not be found. 251 type BlobNotFoundError struct { 252 cid cid.Cid 253 } 254 255 func NewBlobNotFoundError(cid cid.Cid) *BlobNotFoundError { 256 return &BlobNotFoundError{cid: cid} 257 } 258 259 func (e *BlobNotFoundError) Error() string { 260 return fmt.Sprintf("blob %v not found", e.cid.String()) 261 }