storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-metadata-sys.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "errors" 23 "fmt" 24 "sync" 25 26 "github.com/minio/minio-go/v7/pkg/tags" 27 28 "storj.io/minio/cmd/crypto" 29 "storj.io/minio/cmd/logger" 30 bucketsse "storj.io/minio/pkg/bucket/encryption" 31 "storj.io/minio/pkg/bucket/lifecycle" 32 objectlock "storj.io/minio/pkg/bucket/object/lock" 33 "storj.io/minio/pkg/bucket/policy" 34 "storj.io/minio/pkg/bucket/replication" 35 "storj.io/minio/pkg/bucket/versioning" 36 "storj.io/minio/pkg/event" 37 "storj.io/minio/pkg/madmin" 38 "storj.io/minio/pkg/sync/errgroup" 39 ) 40 41 // BucketMetadataSys captures all bucket metadata for a given cluster. 42 type BucketMetadataSys struct { 43 sync.RWMutex 44 metadataMap map[string]BucketMetadata 45 } 46 47 // Remove bucket metadata from memory. 48 func (sys *BucketMetadataSys) Remove(bucket string) { 49 if GlobalIsGateway { 50 return 51 } 52 sys.Lock() 53 delete(sys.metadataMap, bucket) 54 globalBucketMonitor.DeleteBucket(bucket) 55 sys.Unlock() 56 } 57 58 // Set - sets a new metadata in-memory. 59 // Only a shallow copy is saved and fields with references 60 // cannot be modified without causing a race condition, 61 // so they should be replaced atomically and not appended to, etc. 62 // Data is not persisted to disk. 63 func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) { 64 if GlobalIsGateway { 65 return 66 } 67 68 if bucket != minioMetaBucket { 69 sys.Lock() 70 sys.metadataMap[bucket] = meta 71 sys.Unlock() 72 } 73 } 74 75 // Update update bucket metadata for the specified config file. 76 // The configData data should not be modified after being sent here. 77 func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error { 78 objAPI := newObjectLayerFn() 79 if objAPI == nil { 80 return errServerNotInitialized 81 } 82 83 if GlobalIsGateway { 84 // This code is needed only for gateway implementations. 85 switch configFile { 86 case bucketSSEConfig: 87 if globalGatewayName == NASBackendGateway { 88 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 89 if err != nil { 90 return err 91 } 92 meta.EncryptionConfigXML = configData 93 return meta.Save(GlobalContext, objAPI) 94 } 95 case bucketLifecycleConfig: 96 if globalGatewayName == NASBackendGateway { 97 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 98 if err != nil { 99 return err 100 } 101 meta.LifecycleConfigXML = configData 102 return meta.Save(GlobalContext, objAPI) 103 } 104 case bucketTaggingConfig: 105 if globalGatewayName == NASBackendGateway { 106 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 107 if err != nil { 108 return err 109 } 110 meta.TaggingConfigXML = configData 111 return meta.Save(GlobalContext, objAPI) 112 } 113 case bucketNotificationConfig: 114 if globalGatewayName == NASBackendGateway { 115 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 116 if err != nil { 117 return err 118 } 119 meta.NotificationConfigXML = configData 120 return meta.Save(GlobalContext, objAPI) 121 } 122 case bucketPolicyConfig: 123 if configData == nil { 124 return objAPI.DeleteBucketPolicy(GlobalContext, bucket) 125 } 126 config, err := policy.ParseConfig(bytes.NewReader(configData), bucket) 127 if err != nil { 128 return err 129 } 130 return objAPI.SetBucketPolicy(GlobalContext, bucket, config) 131 } 132 return NotImplemented{} 133 } 134 135 if bucket == minioMetaBucket { 136 return errInvalidArgument 137 } 138 139 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 140 if err != nil { 141 return err 142 } 143 144 switch configFile { 145 case bucketPolicyConfig: 146 meta.PolicyConfigJSON = configData 147 case bucketNotificationConfig: 148 meta.NotificationConfigXML = configData 149 case bucketLifecycleConfig: 150 meta.LifecycleConfigXML = configData 151 case bucketSSEConfig: 152 meta.EncryptionConfigXML = configData 153 case bucketTaggingConfig: 154 meta.TaggingConfigXML = configData 155 case bucketQuotaConfigFile: 156 meta.QuotaConfigJSON = configData 157 case objectLockConfig: 158 if !globalIsErasure && !globalIsDistErasure { 159 return NotImplemented{} 160 } 161 meta.ObjectLockConfigXML = configData 162 case bucketVersioningConfig: 163 if !globalIsErasure && !globalIsDistErasure { 164 return NotImplemented{} 165 } 166 meta.VersioningConfigXML = configData 167 case bucketReplicationConfig: 168 if !globalIsErasure && !globalIsDistErasure { 169 return NotImplemented{} 170 } 171 meta.ReplicationConfigXML = configData 172 case bucketTargetsFile: 173 meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{ 174 bucket: meta.Name, 175 bucketTargetsFile: bucketTargetsFile, 176 }) 177 if err != nil { 178 return fmt.Errorf("Error encrypting bucket target metadata %w", err) 179 } 180 default: 181 return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile) 182 } 183 184 if err := meta.Save(GlobalContext, objAPI); err != nil { 185 return err 186 } 187 188 sys.Set(bucket, meta) 189 GlobalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) 190 191 return nil 192 } 193 194 // Get metadata for a bucket. 195 // If no metadata exists errConfigNotFound is returned and a new metadata is returned. 196 // Only a shallow copy is returned, so referenced data should not be modified, 197 // but can be replaced atomically. 198 // 199 // This function should only be used with 200 // - GetBucketInfo 201 // - ListBuckets 202 // For all other bucket specific metadata, use the relevant 203 // calls implemented specifically for each of those features. 204 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) { 205 if GlobalIsGateway || bucket == minioMetaBucket { 206 return newBucketMetadata(bucket), errConfigNotFound 207 } 208 209 sys.RLock() 210 defer sys.RUnlock() 211 212 meta, ok := sys.metadataMap[bucket] 213 if !ok { 214 return newBucketMetadata(bucket), errConfigNotFound 215 } 216 217 return meta, nil 218 } 219 220 // GetVersioningConfig returns configured versioning config 221 // The returned object may not be modified. 222 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) { 223 meta, err := sys.GetConfig(bucket) 224 if err != nil { 225 return nil, err 226 } 227 return meta.versioningConfig, nil 228 } 229 230 // GetTaggingConfig returns configured tagging config 231 // The returned object may not be modified. 232 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) { 233 meta, err := sys.GetConfig(bucket) 234 if err != nil { 235 if errors.Is(err, errConfigNotFound) { 236 return nil, BucketTaggingNotFound{Bucket: bucket} 237 } 238 return nil, err 239 } 240 if meta.taggingConfig == nil { 241 return nil, BucketTaggingNotFound{Bucket: bucket} 242 } 243 return meta.taggingConfig, nil 244 } 245 246 // GetObjectLockConfig returns configured object lock config 247 // The returned object may not be modified. 248 func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) { 249 meta, err := sys.GetConfig(bucket) 250 if err != nil { 251 if errors.Is(err, errConfigNotFound) { 252 return nil, BucketObjectLockConfigNotFound{Bucket: bucket} 253 } 254 return nil, err 255 } 256 if meta.objectLockConfig == nil { 257 return nil, BucketObjectLockConfigNotFound{Bucket: bucket} 258 } 259 return meta.objectLockConfig, nil 260 } 261 262 // GetLifecycleConfig returns configured lifecycle config 263 // The returned object may not be modified. 264 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) { 265 meta, err := sys.GetConfig(bucket) 266 if err != nil { 267 if errors.Is(err, errConfigNotFound) { 268 return nil, BucketLifecycleNotFound{Bucket: bucket} 269 } 270 return nil, err 271 } 272 if meta.lifecycleConfig == nil { 273 return nil, BucketLifecycleNotFound{Bucket: bucket} 274 } 275 return meta.lifecycleConfig, nil 276 } 277 278 // GetNotificationConfig returns configured notification config 279 // The returned object may not be modified. 280 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) { 281 if GlobalIsGateway && globalGatewayName == NASBackendGateway { 282 // Only needed in case of NAS gateway. 283 objAPI := newObjectLayerFn() 284 if objAPI == nil { 285 return nil, errServerNotInitialized 286 } 287 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 288 if err != nil { 289 return nil, err 290 } 291 return meta.notificationConfig, nil 292 } 293 294 meta, err := sys.GetConfig(bucket) 295 if err != nil { 296 return nil, err 297 } 298 return meta.notificationConfig, nil 299 } 300 301 // GetSSEConfig returns configured SSE config 302 // The returned object may not be modified. 303 func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) { 304 meta, err := sys.GetConfig(bucket) 305 if err != nil { 306 if errors.Is(err, errConfigNotFound) { 307 return nil, BucketSSEConfigNotFound{Bucket: bucket} 308 } 309 return nil, err 310 } 311 if meta.sseConfig == nil { 312 return nil, BucketSSEConfigNotFound{Bucket: bucket} 313 } 314 return meta.sseConfig, nil 315 } 316 317 // GetPolicyConfig returns configured bucket policy 318 // The returned object may not be modified. 319 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) { 320 if GlobalIsGateway { 321 objAPI := newObjectLayerFn() 322 if objAPI == nil { 323 return nil, errServerNotInitialized 324 } 325 return objAPI.GetBucketPolicy(GlobalContext, bucket) 326 } 327 328 meta, err := sys.GetConfig(bucket) 329 if err != nil { 330 if errors.Is(err, errConfigNotFound) { 331 return nil, BucketPolicyNotFound{Bucket: bucket} 332 } 333 return nil, err 334 } 335 if meta.policyConfig == nil { 336 return nil, BucketPolicyNotFound{Bucket: bucket} 337 } 338 return meta.policyConfig, nil 339 } 340 341 // GetQuotaConfig returns configured bucket quota 342 // The returned object may not be modified. 343 func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) { 344 meta, err := sys.GetConfig(bucket) 345 if err != nil { 346 return nil, err 347 } 348 return meta.quotaConfig, nil 349 } 350 351 // GetReplicationConfig returns configured bucket replication config 352 // The returned object may not be modified. 353 func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) { 354 meta, err := sys.GetConfig(bucket) 355 if err != nil { 356 if errors.Is(err, errConfigNotFound) { 357 return nil, BucketReplicationConfigNotFound{Bucket: bucket} 358 } 359 return nil, err 360 } 361 362 if meta.replicationConfig == nil { 363 return nil, BucketReplicationConfigNotFound{Bucket: bucket} 364 } 365 return meta.replicationConfig, nil 366 } 367 368 // GetBucketTargetsConfig returns configured bucket targets for this bucket 369 // The returned object may not be modified. 370 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) { 371 meta, err := sys.GetConfig(bucket) 372 if err != nil { 373 return nil, err 374 } 375 if meta.bucketTargetConfig == nil { 376 return nil, BucketRemoteTargetNotFound{Bucket: bucket} 377 } 378 return meta.bucketTargetConfig, nil 379 } 380 381 // GetBucketTarget returns the target for the bucket and arn. 382 func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) { 383 targets, err := sys.GetBucketTargetsConfig(bucket) 384 if err != nil { 385 return madmin.BucketTarget{}, err 386 } 387 for _, t := range targets.Targets { 388 if t.Arn == arn { 389 return t, nil 390 } 391 } 392 return madmin.BucketTarget{}, errConfigNotFound 393 } 394 395 // GetConfig returns a specific configuration from the bucket metadata. 396 // The returned object may not be modified. 397 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) { 398 objAPI := newObjectLayerFn() 399 if objAPI == nil { 400 return newBucketMetadata(bucket), errServerNotInitialized 401 } 402 403 if GlobalIsGateway { 404 return newBucketMetadata(bucket), NotImplemented{} 405 } 406 407 if bucket == minioMetaBucket { 408 return newBucketMetadata(bucket), errInvalidArgument 409 } 410 411 sys.RLock() 412 meta, ok := sys.metadataMap[bucket] 413 sys.RUnlock() 414 if ok { 415 return meta, nil 416 } 417 meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket) 418 if err != nil { 419 return meta, err 420 } 421 sys.Lock() 422 sys.metadataMap[bucket] = meta 423 sys.Unlock() 424 return meta, nil 425 } 426 427 // Init - initializes bucket metadata system for all buckets. 428 func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error { 429 if objAPI == nil { 430 return errServerNotInitialized 431 } 432 433 // In gateway mode, we don't need to load the policies 434 // from the backend. 435 if GlobalIsGateway { 436 return nil 437 } 438 439 // Load bucket metadata sys in background 440 go sys.load(ctx, buckets, objAPI) 441 return nil 442 } 443 444 // concurrently load bucket metadata to speed up loading bucket metadata. 445 func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) { 446 g := errgroup.WithNErrs(len(buckets)) 447 for index := range buckets { 448 index := index 449 g.Go(func() error { 450 _, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{ 451 // Ensure heal opts for bucket metadata be deep healed all the time. 452 ScanMode: madmin.HealDeepScan, 453 }) 454 meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name) 455 if err != nil { 456 return err 457 } 458 sys.Lock() 459 sys.metadataMap[buckets[index].Name] = meta 460 sys.Unlock() 461 return nil 462 }, index) 463 } 464 for _, err := range g.Wait() { 465 if err != nil { 466 logger.LogIf(ctx, err) 467 } 468 } 469 } 470 471 // Loads bucket metadata for all buckets into BucketMetadataSys. 472 func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) { 473 count := 100 // load 100 bucket metadata at a time. 474 for { 475 if len(buckets) < count { 476 sys.concurrentLoad(ctx, buckets, objAPI) 477 return 478 } 479 sys.concurrentLoad(ctx, buckets[:count], objAPI) 480 buckets = buckets[count:] 481 } 482 } 483 484 // Reset the state of the BucketMetadataSys. 485 func (sys *BucketMetadataSys) Reset() { 486 sys.Lock() 487 for k := range sys.metadataMap { 488 delete(sys.metadataMap, k) 489 } 490 sys.Unlock() 491 } 492 493 // NewBucketMetadataSys - creates new policy system. 494 func NewBucketMetadataSys() *BucketMetadataSys { 495 return &BucketMetadataSys{ 496 metadataMap: make(map[string]BucketMetadata), 497 } 498 }