zotregistry.io/zot@v1.4.4-0.20231124084042-02a8ed785457/pkg/storage/cache/dynamodb.go (about) 1 package cache 2 3 import ( 4 "context" 5 "strings" 6 7 "github.com/aws/aws-sdk-go-v2/aws" 8 "github.com/aws/aws-sdk-go-v2/config" 9 "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" 10 "github.com/aws/aws-sdk-go-v2/service/dynamodb" 11 "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" 12 godigest "github.com/opencontainers/go-digest" 13 14 zerr "zotregistry.io/zot/errors" 15 zlog "zotregistry.io/zot/pkg/log" 16 ) 17 18 type DynamoDBDriver struct { 19 client *dynamodb.Client 20 log zlog.Logger 21 tableName string 22 } 23 24 type DynamoDBDriverParameters struct { 25 Endpoint, Region, TableName string 26 } 27 28 type Blob struct { 29 Digest string `dynamodbav:"Digest,string"` 30 DuplicateBlobPath []string `dynamodbav:"DuplicateBlobPath,stringset"` 31 OriginalBlobPath string `dynamodbav:"OriginalBlobPath,string"` 32 } 33 34 func (d *DynamoDBDriver) NewTable(tableName string) error { 35 //nolint:gomnd 36 _, err := d.client.CreateTable(context.TODO(), &dynamodb.CreateTableInput{ 37 TableName: &tableName, 38 AttributeDefinitions: []types.AttributeDefinition{ 39 { 40 AttributeName: aws.String("Digest"), 41 AttributeType: types.ScalarAttributeTypeS, 42 }, 43 }, 44 KeySchema: []types.KeySchemaElement{ 45 { 46 AttributeName: aws.String("Digest"), 47 KeyType: types.KeyTypeHash, 48 }, 49 }, 50 ProvisionedThroughput: &types.ProvisionedThroughput{ 51 ReadCapacityUnits: aws.Int64(10), 52 WriteCapacityUnits: aws.Int64(5), 53 }, 54 }) 55 if err != nil && !strings.Contains(err.Error(), "Table already exists") { 56 return err 57 } 58 59 d.tableName = tableName 60 61 return nil 62 } 63 64 func NewDynamoDBCache(parameters interface{}, log zlog.Logger) (*DynamoDBDriver, error) { 65 properParameters, ok := parameters.(DynamoDBDriverParameters) 66 if !ok { 67 log.Error().Err(zerr.ErrTypeAssertionFailed).Msgf("expected type '%T' but got '%T'", 68 BoltDBDriverParameters{}, parameters) 69 70 return nil, zerr.ErrTypeAssertionFailed 71 } 72 73 // custom endpoint resolver to point to localhost 74 customResolver := aws.EndpointResolverWithOptionsFunc( 75 func(service, region string, options ...interface{}) (aws.Endpoint, error) { 76 return aws.Endpoint{ 77 PartitionID: "aws", 78 URL: properParameters.Endpoint, 79 SigningRegion: region, 80 }, nil 81 }) 82 83 // Using the SDK's default configuration, loading additional config 84 // and credentials values from the environment variables, shared 85 // credentials, and shared configuration files 86 cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(properParameters.Region), 87 config.WithEndpointResolverWithOptions(customResolver)) 88 if err != nil { 89 log.Error().Err(err).Msg("unable to load AWS SDK config for dynamodb") 90 91 return nil, err 92 } 93 94 driver := &DynamoDBDriver{client: dynamodb.NewFromConfig(cfg), tableName: properParameters.TableName, log: log} 95 96 err = driver.NewTable(driver.tableName) 97 if err != nil { 98 log.Error().Err(err).Str("tableName", driver.tableName).Msg("unable to create table for cache") 99 100 return nil, err 101 } 102 103 // Using the Config value, create the DynamoDB client 104 return driver, nil 105 } 106 107 func (d *DynamoDBDriver) SetTableName(table string) { 108 d.tableName = table 109 } 110 111 func (d *DynamoDBDriver) UsesRelativePaths() bool { 112 return false 113 } 114 115 func (d *DynamoDBDriver) Name() string { 116 return "dynamodb" 117 } 118 119 // Returns the original blob. 120 func (d *DynamoDBDriver) GetBlob(digest godigest.Digest) (string, error) { 121 resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ 122 TableName: aws.String(d.tableName), 123 Key: map[string]types.AttributeValue{ 124 "Digest": &types.AttributeValueMemberS{Value: digest.String()}, 125 }, 126 }) 127 if err != nil { 128 d.log.Error().Err(err).Str("tableName", d.tableName).Msg("failed to get blob") 129 130 return "", err 131 } 132 133 out := Blob{} 134 135 if resp.Item == nil { 136 return "", zerr.ErrCacheMiss 137 } 138 139 _ = attributevalue.UnmarshalMap(resp.Item, &out) 140 141 return out.OriginalBlobPath, nil 142 } 143 144 func (d *DynamoDBDriver) PutBlob(digest godigest.Digest, path string) error { 145 if path == "" { 146 d.log.Error().Err(zerr.ErrEmptyValue).Str("digest", digest.String()).Msg("empty path provided") 147 148 return zerr.ErrEmptyValue 149 } 150 151 if originBlob, _ := d.GetBlob(digest); originBlob == "" { 152 // first entry, so add original blob 153 if err := d.putOriginBlob(digest, path); err != nil { 154 return err 155 } 156 } 157 158 expression := "ADD DuplicateBlobPath :i" 159 attrPath := types.AttributeValueMemberSS{Value: []string{path}} 160 161 if err := d.updateItem(digest, expression, map[string]types.AttributeValue{":i": &attrPath}); err != nil { 162 d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("unable to put blob") 163 164 return err 165 } 166 167 return nil 168 } 169 170 func (d *DynamoDBDriver) HasBlob(digest godigest.Digest, path string) bool { 171 resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ 172 TableName: aws.String(d.tableName), 173 Key: map[string]types.AttributeValue{ 174 "Digest": &types.AttributeValueMemberS{Value: digest.String()}, 175 }, 176 }) 177 if err != nil { 178 d.log.Error().Err(err).Str("tableName", d.tableName).Msg("failed to get blob") 179 180 return false 181 } 182 183 out := Blob{} 184 185 if resp.Item == nil { 186 d.log.Debug().Err(zerr.ErrCacheMiss).Str("digest", string(digest)).Msg("unable to find blob in cache") 187 188 return false 189 } 190 191 _ = attributevalue.UnmarshalMap(resp.Item, &out) 192 193 if out.OriginalBlobPath == path { 194 return true 195 } 196 197 for _, item := range out.DuplicateBlobPath { 198 if item == path { 199 return true 200 } 201 } 202 203 d.log.Debug().Err(zerr.ErrCacheMiss).Str("digest", string(digest)).Msg("unable to find blob in cache") 204 205 return false 206 } 207 208 func (d *DynamoDBDriver) DeleteBlob(digest godigest.Digest, path string) error { 209 marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()}) 210 211 expression := "DELETE DuplicateBlobPath :i" 212 attrPath := types.AttributeValueMemberSS{Value: []string{path}} 213 214 if err := d.updateItem(digest, expression, map[string]types.AttributeValue{":i": &attrPath}); err != nil { 215 d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("unable to delete") 216 217 return err 218 } 219 220 originBlob, _ := d.GetBlob(digest) 221 // if original blob is the one deleted 222 if originBlob == path { 223 // move duplicate blob to original, storage will move content here 224 originBlob, _ = d.GetDuplicateBlob(digest) 225 if originBlob != "" { 226 if err := d.putOriginBlob(digest, originBlob); err != nil { 227 return err 228 } 229 } 230 } 231 232 if originBlob == "" { 233 d.log.Debug().Str("digest", digest.String()).Str("path", path).Msg("deleting empty bucket") 234 235 _, _ = d.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{ 236 Key: marshaledKey, 237 TableName: &d.tableName, 238 }) 239 } 240 241 return nil 242 } 243 244 func (d *DynamoDBDriver) GetDuplicateBlob(digest godigest.Digest) (string, error) { 245 resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ 246 TableName: aws.String(d.tableName), 247 Key: map[string]types.AttributeValue{ 248 "Digest": &types.AttributeValueMemberS{Value: digest.String()}, 249 }, 250 }) 251 if err != nil { 252 d.log.Error().Err(err).Str("tableName", d.tableName).Msg("failed to get blob") 253 254 return "", err 255 } 256 257 out := Blob{} 258 259 if resp.Item == nil { 260 return "", zerr.ErrCacheMiss 261 } 262 263 _ = attributevalue.UnmarshalMap(resp.Item, &out) 264 265 if len(out.DuplicateBlobPath) == 0 { 266 return "", nil 267 } 268 269 return out.DuplicateBlobPath[0], nil 270 } 271 272 func (d *DynamoDBDriver) putOriginBlob(digest godigest.Digest, path string) error { 273 expression := "SET OriginalBlobPath = :s" 274 attrPath := types.AttributeValueMemberS{Value: path} 275 276 if err := d.updateItem(digest, expression, map[string]types.AttributeValue{":s": &attrPath}); err != nil { 277 d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("unable to put original blob") 278 279 return err 280 } 281 282 return nil 283 } 284 285 func (d *DynamoDBDriver) updateItem(digest godigest.Digest, expression string, 286 expressionAttVals map[string]types.AttributeValue, 287 ) error { 288 marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()}) 289 290 _, err := d.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ 291 Key: marshaledKey, 292 TableName: &d.tableName, 293 UpdateExpression: &expression, 294 ExpressionAttributeValues: expressionAttVals, 295 }) 296 297 return err 298 }