github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/blobstore/gcs.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package blobstore
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  	"path"
    21  	"strconv"
    22  
    23  	"cloud.google.com/go/storage"
    24  	"google.golang.org/api/googleapi"
    25  )
    26  
    27  const (
    28  	precondFailCode = 412
    29  )
    30  
    31  // GCSBlobstore provides a GCS implementation of the Blobstore interface
    32  type GCSBlobstore struct {
    33  	bucket     *storage.BucketHandle
    34  	bucketName string
    35  	prefix     string
    36  }
    37  
    38  // NewGCSBlobstore creates a new instance of a GCSBlobstare
    39  func NewGCSBlobstore(gcs *storage.Client, bucketName, prefix string) *GCSBlobstore {
    40  	for len(prefix) > 0 && prefix[0] == '/' {
    41  		prefix = prefix[1:]
    42  	}
    43  
    44  	bucket := gcs.Bucket(bucketName)
    45  	return &GCSBlobstore{bucket, bucketName, prefix}
    46  }
    47  
    48  // Exists returns true if a blob exists for the given key, and false if it does not.
    49  // For InMemoryBlobstore instances error should never be returned (though other
    50  // implementations of this interface can)
    51  func (bs *GCSBlobstore) Exists(ctx context.Context, key string) (bool, error) {
    52  	absKey := path.Join(bs.prefix, key)
    53  	oh := bs.bucket.Object(absKey)
    54  	_, err := oh.Attrs(ctx)
    55  
    56  	if err == storage.ErrObjectNotExist {
    57  		return false, nil
    58  	}
    59  
    60  	return err == nil, err
    61  }
    62  
    63  // Get retrieves an io.reader for the portion of a blob specified by br along with
    64  // its version
    65  func (bs *GCSBlobstore) Get(ctx context.Context, key string, br BlobRange) (io.ReadCloser, string, error) {
    66  	absKey := path.Join(bs.prefix, key)
    67  	oh := bs.bucket.Object(absKey)
    68  	attrs, err := oh.Attrs(ctx)
    69  
    70  	if err == storage.ErrObjectNotExist {
    71  		return nil, "", NotFound{"gs://" + path.Join(bs.bucketName, absKey)}
    72  	} else if err != nil {
    73  		return nil, "", err
    74  	}
    75  
    76  	generation := attrs.Generation
    77  
    78  	var reader *storage.Reader
    79  	if br.isAllRange() {
    80  		reader, err = oh.Generation(generation).NewReader(ctx)
    81  	} else {
    82  		posBr := br.positiveRange(attrs.Size)
    83  		reader, err = oh.Generation(generation).NewRangeReader(ctx, posBr.offset, posBr.length)
    84  	}
    85  
    86  	if err != nil {
    87  		return nil, "", err
    88  	}
    89  
    90  	return reader, strconv.FormatInt(generation, 16), nil
    91  }
    92  
    93  func writeObj(writer *storage.Writer, reader io.Reader) (string, error) {
    94  	writeErr, closeErr := func() (writeErr error, closeErr error) {
    95  		defer func() {
    96  			closeErr = writer.Close()
    97  		}()
    98  		_, writeErr = io.Copy(writer, reader)
    99  
   100  		return
   101  	}()
   102  
   103  	if writeErr != nil {
   104  		return "", writeErr
   105  	} else if closeErr != nil {
   106  		return "", closeErr
   107  	}
   108  
   109  	generation := writer.Attrs().Generation
   110  
   111  	return strconv.FormatInt(generation, 16), nil
   112  }
   113  
   114  // Put sets the blob and the version for a key
   115  func (bs *GCSBlobstore) Put(ctx context.Context, key string, reader io.Reader) (string, error) {
   116  	absKey := path.Join(bs.prefix, key)
   117  	oh := bs.bucket.Object(absKey)
   118  	writer := oh.NewWriter(ctx)
   119  
   120  	return writeObj(writer, reader)
   121  }
   122  
   123  // CheckAndPut will check the current version of a blob against an expectedVersion, and if the
   124  // versions match it will update the data and version associated with the key
   125  func (bs *GCSBlobstore) CheckAndPut(ctx context.Context, expectedVersion, key string, reader io.Reader) (string, error) {
   126  	absKey := path.Join(bs.prefix, key)
   127  	oh := bs.bucket.Object(absKey)
   128  
   129  	var conditionalHandle *storage.ObjectHandle
   130  	if expectedVersion != "" {
   131  		expectedGen, err := strconv.ParseInt(expectedVersion, 16, 64)
   132  
   133  		if err != nil {
   134  			panic("Invalid expected Version")
   135  		}
   136  
   137  		conditionalHandle = oh.If(storage.Conditions{GenerationMatch: expectedGen})
   138  	} else {
   139  		conditionalHandle = oh.If(storage.Conditions{DoesNotExist: true})
   140  	}
   141  
   142  	writer := conditionalHandle.NewWriter(ctx)
   143  
   144  	ver, err := writeObj(writer, reader)
   145  
   146  	if err != nil {
   147  		apiErr, ok := err.(*googleapi.Error)
   148  
   149  		if ok {
   150  			if apiErr.Code == precondFailCode {
   151  				return "", CheckAndPutError{key, expectedVersion, "unknown (Not supported in GCS implementation)"}
   152  			}
   153  		}
   154  	}
   155  
   156  	return ver, err
   157  }