github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/dynamo_manifest.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package nbs 23 24 import ( 25 "context" 26 "errors" 27 "fmt" 28 "strings" 29 "time" 30 31 "github.com/aws/aws-sdk-go/aws" 32 "github.com/aws/aws-sdk-go/aws/awserr" 33 "github.com/aws/aws-sdk-go/aws/request" 34 "github.com/aws/aws-sdk-go/service/dynamodb" 35 36 "github.com/dolthub/dolt/go/store/d" 37 "github.com/dolthub/dolt/go/store/hash" 38 ) 39 40 const ( 41 // DynamoManifest does not yet include GC Generation 42 AWSStorageVersion = "4" 43 44 dbAttr = "db" 45 lockAttr = "lck" // 'lock' is a reserved word in dynamo 46 rootAttr = "root" 47 versAttr = "vers" 48 nbsVersAttr = "nbsVers" 49 tableSpecsAttr = "specs" 50 appendixAttr = "appendix" 51 prevLockExpressionValuesKey = ":prev" 52 versExpressionValuesKey = ":vers" 53 ) 54 55 var ( 56 valueEqualsExpression = fmt.Sprintf("(%s = %s) and (%s = %s)", lockAttr, prevLockExpressionValuesKey, versAttr, versExpressionValuesKey) 57 valueNotExistsOrEqualsExpression = fmt.Sprintf("attribute_not_exists("+lockAttr+") or %s", valueEqualsExpression) 58 ) 59 60 type ddbsvc interface { 61 GetItemWithContext(ctx aws.Context, input *dynamodb.GetItemInput, opts ...request.Option) (*dynamodb.GetItemOutput, error) 62 PutItemWithContext(ctx aws.Context, input *dynamodb.PutItemInput, opts ...request.Option) (*dynamodb.PutItemOutput, error) 63 } 64 65 // dynamoManifest assumes the existence of a DynamoDB table whose primary partition key is in String format and named `db`. 66 type dynamoManifest struct { 67 table, db string 68 ddbsvc ddbsvc 69 } 70 71 func newDynamoManifest(table, namespace string, ddb ddbsvc) manifest { 72 d.PanicIfTrue(table == "") 73 d.PanicIfTrue(namespace == "") 74 return dynamoManifest{table, namespace, ddb} 75 } 76 77 func (dm dynamoManifest) Name() string { 78 return dm.table + dm.db 79 } 80 81 func (dm dynamoManifest) ParseIfExists(ctx context.Context, stats *Stats, readHook func() error) (bool, manifestContents, error) { 82 t1 := time.Now() 83 defer func() { stats.ReadManifestLatency.SampleTimeSince(t1) }() 84 85 var exists bool 86 var contents manifestContents 87 88 result, err := dm.ddbsvc.GetItemWithContext(ctx, &dynamodb.GetItemInput{ 89 ConsistentRead: aws.Bool(true), // This doubles the cost :-( 90 TableName: aws.String(dm.table), 91 Key: map[string]*dynamodb.AttributeValue{ 92 dbAttr: {S: aws.String(dm.db)}, 93 }, 94 }) 95 96 if err != nil { 97 return false, manifestContents{}, fmt.Errorf("failed to get dynamo table: '%s' - %w", dm.table, err) 98 } 99 100 // !exists(dbAttr) => unitialized store 101 if len(result.Item) > 0 { 102 valid, hasSpecs, hasAppendix := validateManifest(result.Item) 103 if !valid { 104 return false, contents, ErrCorruptManifest 105 } 106 107 exists = true 108 contents.vers = *result.Item[versAttr].S 109 contents.root = hash.New(result.Item[rootAttr].B) 110 copy(contents.lock[:], result.Item[lockAttr].B) 111 if hasSpecs { 112 contents.specs, err = parseSpecs(strings.Split(*result.Item[tableSpecsAttr].S, ":")) 113 if err != nil { 114 return false, manifestContents{}, ErrCorruptManifest 115 } 116 } 117 118 if hasAppendix { 119 contents.appendix, err = parseSpecs(strings.Split(*result.Item[appendixAttr].S, ":")) 120 if err != nil { 121 return false, manifestContents{}, ErrCorruptManifest 122 } 123 } 124 } 125 126 return exists, contents, nil 127 } 128 129 func validateManifest(item map[string]*dynamodb.AttributeValue) (valid, hasSpecs, hasAppendix bool) { 130 if item[nbsVersAttr] != nil && item[nbsVersAttr].S != nil && 131 AWSStorageVersion == *item[nbsVersAttr].S && 132 item[versAttr] != nil && item[versAttr].S != nil && 133 item[lockAttr] != nil && item[lockAttr].B != nil && 134 item[rootAttr] != nil && item[rootAttr].B != nil { 135 if len(item) == 6 || len(item) == 7 { 136 if item[tableSpecsAttr] != nil && item[tableSpecsAttr].S != nil { 137 hasSpecs = true 138 } 139 if item[appendixAttr] != nil && item[appendixAttr].S != nil { 140 hasAppendix = true 141 } 142 return true, hasSpecs, hasAppendix 143 } 144 return len(item) == 5, false, false 145 } 146 return false, false, false 147 } 148 149 func (dm dynamoManifest) Update(ctx context.Context, lastLock addr, newContents manifestContents, stats *Stats, writeHook func() error) (manifestContents, error) { 150 t1 := time.Now() 151 defer func() { stats.WriteManifestLatency.SampleTimeSince(t1) }() 152 153 putArgs := dynamodb.PutItemInput{ 154 TableName: aws.String(dm.table), 155 Item: map[string]*dynamodb.AttributeValue{ 156 dbAttr: {S: aws.String(dm.db)}, 157 nbsVersAttr: {S: aws.String(AWSStorageVersion)}, 158 versAttr: {S: aws.String(newContents.vers)}, 159 rootAttr: {B: newContents.root[:]}, 160 lockAttr: {B: newContents.lock[:]}, 161 }, 162 } 163 164 if len(newContents.specs) > 0 { 165 tableInfo := make([]string, 2*len(newContents.specs)) 166 formatSpecs(newContents.specs, tableInfo) 167 putArgs.Item[tableSpecsAttr] = &dynamodb.AttributeValue{S: aws.String(strings.Join(tableInfo, ":"))} 168 } 169 170 if len(newContents.appendix) > 0 { 171 tableInfo := make([]string, 2*len(newContents.appendix)) 172 formatSpecs(newContents.appendix, tableInfo) 173 putArgs.Item[appendixAttr] = &dynamodb.AttributeValue{S: aws.String(strings.Join(tableInfo, ":"))} 174 } 175 176 expr := valueEqualsExpression 177 if lastLock == (addr{}) { 178 expr = valueNotExistsOrEqualsExpression 179 } 180 181 putArgs.ConditionExpression = aws.String(expr) 182 putArgs.ExpressionAttributeValues = map[string]*dynamodb.AttributeValue{ 183 prevLockExpressionValuesKey: {B: lastLock[:]}, 184 versExpressionValuesKey: {S: aws.String(newContents.vers)}, 185 } 186 187 _, ddberr := dm.ddbsvc.PutItemWithContext(ctx, &putArgs) 188 if ddberr != nil { 189 if errIsConditionalCheckFailed(ddberr) { 190 exists, upstream, err := dm.ParseIfExists(ctx, stats, nil) 191 192 if err != nil { 193 return manifestContents{}, err 194 } 195 196 if !exists { 197 return manifestContents{}, errors.New("manifest not found") 198 } 199 200 if upstream.vers != newContents.vers { 201 return manifestContents{}, errors.New("version mismatch") 202 } 203 204 return upstream, nil 205 } 206 207 if ddberr != nil { 208 return manifestContents{}, ddberr 209 } 210 } 211 212 return newContents, nil 213 } 214 215 func errIsConditionalCheckFailed(err error) bool { 216 awsErr, ok := err.(awserr.Error) 217 return ok && awsErr.Code() == "ConditionalCheckFailedException" 218 }