github.com/koko1123/flow-go-1@v0.29.6/engine/execution/ingestion/uploader/gcp_uploader.go (about)

     1  package uploader
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	"cloud.google.com/go/storage"
     8  	"github.com/hashicorp/go-multierror"
     9  	"github.com/rs/zerolog"
    10  
    11  	"github.com/koko1123/flow-go-1/engine/execution"
    12  )
    13  
    14  type GCPBucketUploader struct {
    15  	log    zerolog.Logger
    16  	bucket *storage.BucketHandle
    17  	ctx    context.Context
    18  }
    19  
    20  func NewGCPBucketUploader(ctx context.Context, bucketName string, log zerolog.Logger) (*GCPBucketUploader, error) {
    21  
    22  	// no need to close the client according to documentation
    23  	// https://pkg.go.dev/cloud.google.com/go/storage#Client.Close
    24  	client, err := storage.NewClient(ctx)
    25  	if err != nil {
    26  		return nil, fmt.Errorf("cannot create GCP Bucket client: %w", err)
    27  	}
    28  	bucket := client.Bucket(bucketName)
    29  
    30  	// try accessing buckets to validate settings
    31  	_, err = bucket.Attrs(ctx)
    32  	if err != nil {
    33  		return nil, fmt.Errorf("error while listing bucket attributes: %w", err)
    34  	}
    35  
    36  	return &GCPBucketUploader{
    37  		bucket: bucket,
    38  		log:    log.With().Str("subcomponent", "gcp_bucket_uploader").Logger(),
    39  		ctx:    ctx,
    40  	}, nil
    41  }
    42  
    43  // Upload uploads the computation result to the configured GCP bucket.
    44  // All errors returned from this function can be considered benign.
    45  func (u *GCPBucketUploader) Upload(computationResult *execution.ComputationResult) error {
    46  	var errs *multierror.Error
    47  
    48  	objectName := GCPBlockDataObjectName(computationResult)
    49  	object := u.bucket.Object(objectName)
    50  
    51  	writer := object.NewWriter(u.ctx)
    52  
    53  	// serialize and write computation result to upload stream
    54  	err := WriteComputationResultsTo(computationResult, writer)
    55  
    56  	if err != nil {
    57  		errs = multierror.Append(errs, fmt.Errorf("error while writing computation result to GCP object: %w", err))
    58  		// fall through because we always want to close the writer
    59  	}
    60  
    61  	// flush and close the stream
    62  	err = writer.Close()
    63  
    64  	// this occasionally fails with HTTP 5xx errors due to flakiness on the network/GCP API
    65  	if err != nil {
    66  		errs = multierror.Append(errs, fmt.Errorf("error while closing GCP object: %w", err))
    67  	}
    68  
    69  	return errs.ErrorOrNil()
    70  }
    71  
    72  func GCPBlockDataObjectName(computationResult *execution.ComputationResult) string {
    73  	return fmt.Sprintf("%s.cbor", computationResult.ExecutableBlock.ID().String())
    74  }