github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/dynamo_manifest.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package nbs
    23  
    24  import (
    25  	"context"
    26  	"errors"
    27  	"fmt"
    28  	"strings"
    29  	"time"
    30  
    31  	"github.com/aws/aws-sdk-go/aws"
    32  	"github.com/aws/aws-sdk-go/aws/awserr"
    33  	"github.com/aws/aws-sdk-go/aws/request"
    34  	"github.com/aws/aws-sdk-go/service/dynamodb"
    35  
    36  	"github.com/dolthub/dolt/go/store/d"
    37  	"github.com/dolthub/dolt/go/store/hash"
    38  )
    39  
    40  const (
    41  	// DynamoManifest does not yet include GC Generation
    42  	AWSStorageVersion = "4"
    43  
    44  	dbAttr                      = "db"
    45  	lockAttr                    = "lck" // 'lock' is a reserved word in dynamo
    46  	rootAttr                    = "root"
    47  	versAttr                    = "vers"
    48  	nbsVersAttr                 = "nbsVers"
    49  	tableSpecsAttr              = "specs"
    50  	appendixAttr                = "appendix"
    51  	prevLockExpressionValuesKey = ":prev"
    52  	versExpressionValuesKey     = ":vers"
    53  )
    54  
    55  var (
    56  	valueEqualsExpression            = fmt.Sprintf("(%s = %s) and (%s = %s)", lockAttr, prevLockExpressionValuesKey, versAttr, versExpressionValuesKey)
    57  	valueNotExistsOrEqualsExpression = fmt.Sprintf("attribute_not_exists("+lockAttr+") or %s", valueEqualsExpression)
    58  )
    59  
    60  type ddbsvc interface {
    61  	GetItemWithContext(ctx aws.Context, input *dynamodb.GetItemInput, opts ...request.Option) (*dynamodb.GetItemOutput, error)
    62  	PutItemWithContext(ctx aws.Context, input *dynamodb.PutItemInput, opts ...request.Option) (*dynamodb.PutItemOutput, error)
    63  }
    64  
    65  // dynamoManifest assumes the existence of a DynamoDB table whose primary partition key is in String format and named `db`.
    66  type dynamoManifest struct {
    67  	table, db string
    68  	ddbsvc    ddbsvc
    69  }
    70  
    71  func newDynamoManifest(table, namespace string, ddb ddbsvc) manifest {
    72  	d.PanicIfTrue(table == "")
    73  	d.PanicIfTrue(namespace == "")
    74  	return dynamoManifest{table, namespace, ddb}
    75  }
    76  
    77  func (dm dynamoManifest) Name() string {
    78  	return dm.table + dm.db
    79  }
    80  
    81  func (dm dynamoManifest) ParseIfExists(ctx context.Context, stats *Stats, readHook func() error) (bool, manifestContents, error) {
    82  	t1 := time.Now()
    83  	defer func() { stats.ReadManifestLatency.SampleTimeSince(t1) }()
    84  
    85  	var exists bool
    86  	var contents manifestContents
    87  
    88  	result, err := dm.ddbsvc.GetItemWithContext(ctx, &dynamodb.GetItemInput{
    89  		ConsistentRead: aws.Bool(true), // This doubles the cost :-(
    90  		TableName:      aws.String(dm.table),
    91  		Key: map[string]*dynamodb.AttributeValue{
    92  			dbAttr: {S: aws.String(dm.db)},
    93  		},
    94  	})
    95  
    96  	if err != nil {
    97  		return false, manifestContents{}, fmt.Errorf("failed to get dynamo table: '%s' - %w", dm.table, err)
    98  	}
    99  
   100  	// !exists(dbAttr) => unitialized store
   101  	if len(result.Item) > 0 {
   102  		valid, hasSpecs, hasAppendix := validateManifest(result.Item)
   103  		if !valid {
   104  			return false, contents, ErrCorruptManifest
   105  		}
   106  
   107  		exists = true
   108  		contents.vers = *result.Item[versAttr].S
   109  		contents.root = hash.New(result.Item[rootAttr].B)
   110  		copy(contents.lock[:], result.Item[lockAttr].B)
   111  		if hasSpecs {
   112  			contents.specs, err = parseSpecs(strings.Split(*result.Item[tableSpecsAttr].S, ":"))
   113  			if err != nil {
   114  				return false, manifestContents{}, ErrCorruptManifest
   115  			}
   116  		}
   117  
   118  		if hasAppendix {
   119  			contents.appendix, err = parseSpecs(strings.Split(*result.Item[appendixAttr].S, ":"))
   120  			if err != nil {
   121  				return false, manifestContents{}, ErrCorruptManifest
   122  			}
   123  		}
   124  	}
   125  
   126  	return exists, contents, nil
   127  }
   128  
   129  func validateManifest(item map[string]*dynamodb.AttributeValue) (valid, hasSpecs, hasAppendix bool) {
   130  	if item[nbsVersAttr] != nil && item[nbsVersAttr].S != nil &&
   131  		AWSStorageVersion == *item[nbsVersAttr].S &&
   132  		item[versAttr] != nil && item[versAttr].S != nil &&
   133  		item[lockAttr] != nil && item[lockAttr].B != nil &&
   134  		item[rootAttr] != nil && item[rootAttr].B != nil {
   135  		if len(item) == 6 || len(item) == 7 {
   136  			if item[tableSpecsAttr] != nil && item[tableSpecsAttr].S != nil {
   137  				hasSpecs = true
   138  			}
   139  			if item[appendixAttr] != nil && item[appendixAttr].S != nil {
   140  				hasAppendix = true
   141  			}
   142  			return true, hasSpecs, hasAppendix
   143  		}
   144  		return len(item) == 5, false, false
   145  	}
   146  	return false, false, false
   147  }
   148  
   149  func (dm dynamoManifest) Update(ctx context.Context, lastLock addr, newContents manifestContents, stats *Stats, writeHook func() error) (manifestContents, error) {
   150  	t1 := time.Now()
   151  	defer func() { stats.WriteManifestLatency.SampleTimeSince(t1) }()
   152  
   153  	putArgs := dynamodb.PutItemInput{
   154  		TableName: aws.String(dm.table),
   155  		Item: map[string]*dynamodb.AttributeValue{
   156  			dbAttr:      {S: aws.String(dm.db)},
   157  			nbsVersAttr: {S: aws.String(AWSStorageVersion)},
   158  			versAttr:    {S: aws.String(newContents.vers)},
   159  			rootAttr:    {B: newContents.root[:]},
   160  			lockAttr:    {B: newContents.lock[:]},
   161  		},
   162  	}
   163  
   164  	if len(newContents.specs) > 0 {
   165  		tableInfo := make([]string, 2*len(newContents.specs))
   166  		formatSpecs(newContents.specs, tableInfo)
   167  		putArgs.Item[tableSpecsAttr] = &dynamodb.AttributeValue{S: aws.String(strings.Join(tableInfo, ":"))}
   168  	}
   169  
   170  	if len(newContents.appendix) > 0 {
   171  		tableInfo := make([]string, 2*len(newContents.appendix))
   172  		formatSpecs(newContents.appendix, tableInfo)
   173  		putArgs.Item[appendixAttr] = &dynamodb.AttributeValue{S: aws.String(strings.Join(tableInfo, ":"))}
   174  	}
   175  
   176  	expr := valueEqualsExpression
   177  	if lastLock == (addr{}) {
   178  		expr = valueNotExistsOrEqualsExpression
   179  	}
   180  
   181  	putArgs.ConditionExpression = aws.String(expr)
   182  	putArgs.ExpressionAttributeValues = map[string]*dynamodb.AttributeValue{
   183  		prevLockExpressionValuesKey: {B: lastLock[:]},
   184  		versExpressionValuesKey:     {S: aws.String(newContents.vers)},
   185  	}
   186  
   187  	_, ddberr := dm.ddbsvc.PutItemWithContext(ctx, &putArgs)
   188  	if ddberr != nil {
   189  		if errIsConditionalCheckFailed(ddberr) {
   190  			exists, upstream, err := dm.ParseIfExists(ctx, stats, nil)
   191  
   192  			if err != nil {
   193  				return manifestContents{}, err
   194  			}
   195  
   196  			if !exists {
   197  				return manifestContents{}, errors.New("manifest not found")
   198  			}
   199  
   200  			if upstream.vers != newContents.vers {
   201  				return manifestContents{}, errors.New("version mismatch")
   202  			}
   203  
   204  			return upstream, nil
   205  		}
   206  
   207  		if ddberr != nil {
   208  			return manifestContents{}, ddberr
   209  		}
   210  	}
   211  
   212  	return newContents, nil
   213  }
   214  
   215  func errIsConditionalCheckFailed(err error) bool {
   216  	awsErr, ok := err.(awserr.Error)
   217  	return ok && awsErr.Code() == "ConditionalCheckFailedException"
   218  }