storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-metadata-sys.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"sync"
    25  
    26  	"github.com/minio/minio-go/v7/pkg/tags"
    27  
    28  	"storj.io/minio/cmd/crypto"
    29  	"storj.io/minio/cmd/logger"
    30  	bucketsse "storj.io/minio/pkg/bucket/encryption"
    31  	"storj.io/minio/pkg/bucket/lifecycle"
    32  	objectlock "storj.io/minio/pkg/bucket/object/lock"
    33  	"storj.io/minio/pkg/bucket/policy"
    34  	"storj.io/minio/pkg/bucket/replication"
    35  	"storj.io/minio/pkg/bucket/versioning"
    36  	"storj.io/minio/pkg/event"
    37  	"storj.io/minio/pkg/madmin"
    38  	"storj.io/minio/pkg/sync/errgroup"
    39  )
    40  
    41  // BucketMetadataSys captures all bucket metadata for a given cluster.
    42  type BucketMetadataSys struct {
    43  	sync.RWMutex
    44  	metadataMap map[string]BucketMetadata
    45  }
    46  
    47  // Remove bucket metadata from memory.
    48  func (sys *BucketMetadataSys) Remove(bucket string) {
    49  	if GlobalIsGateway {
    50  		return
    51  	}
    52  	sys.Lock()
    53  	delete(sys.metadataMap, bucket)
    54  	globalBucketMonitor.DeleteBucket(bucket)
    55  	sys.Unlock()
    56  }
    57  
    58  // Set - sets a new metadata in-memory.
    59  // Only a shallow copy is saved and fields with references
    60  // cannot be modified without causing a race condition,
    61  // so they should be replaced atomically and not appended to, etc.
    62  // Data is not persisted to disk.
    63  func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
    64  	if GlobalIsGateway {
    65  		return
    66  	}
    67  
    68  	if bucket != minioMetaBucket {
    69  		sys.Lock()
    70  		sys.metadataMap[bucket] = meta
    71  		sys.Unlock()
    72  	}
    73  }
    74  
    75  // Update update bucket metadata for the specified config file.
    76  // The configData data should not be modified after being sent here.
    77  func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
    78  	objAPI := newObjectLayerFn()
    79  	if objAPI == nil {
    80  		return errServerNotInitialized
    81  	}
    82  
    83  	if GlobalIsGateway {
    84  		// This code is needed only for gateway implementations.
    85  		switch configFile {
    86  		case bucketSSEConfig:
    87  			if globalGatewayName == NASBackendGateway {
    88  				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
    89  				if err != nil {
    90  					return err
    91  				}
    92  				meta.EncryptionConfigXML = configData
    93  				return meta.Save(GlobalContext, objAPI)
    94  			}
    95  		case bucketLifecycleConfig:
    96  			if globalGatewayName == NASBackendGateway {
    97  				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
    98  				if err != nil {
    99  					return err
   100  				}
   101  				meta.LifecycleConfigXML = configData
   102  				return meta.Save(GlobalContext, objAPI)
   103  			}
   104  		case bucketTaggingConfig:
   105  			if globalGatewayName == NASBackendGateway {
   106  				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
   107  				if err != nil {
   108  					return err
   109  				}
   110  				meta.TaggingConfigXML = configData
   111  				return meta.Save(GlobalContext, objAPI)
   112  			}
   113  		case bucketNotificationConfig:
   114  			if globalGatewayName == NASBackendGateway {
   115  				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
   116  				if err != nil {
   117  					return err
   118  				}
   119  				meta.NotificationConfigXML = configData
   120  				return meta.Save(GlobalContext, objAPI)
   121  			}
   122  		case bucketPolicyConfig:
   123  			if configData == nil {
   124  				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
   125  			}
   126  			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
   127  			if err != nil {
   128  				return err
   129  			}
   130  			return objAPI.SetBucketPolicy(GlobalContext, bucket, config)
   131  		}
   132  		return NotImplemented{}
   133  	}
   134  
   135  	if bucket == minioMetaBucket {
   136  		return errInvalidArgument
   137  	}
   138  
   139  	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
   140  	if err != nil {
   141  		return err
   142  	}
   143  
   144  	switch configFile {
   145  	case bucketPolicyConfig:
   146  		meta.PolicyConfigJSON = configData
   147  	case bucketNotificationConfig:
   148  		meta.NotificationConfigXML = configData
   149  	case bucketLifecycleConfig:
   150  		meta.LifecycleConfigXML = configData
   151  	case bucketSSEConfig:
   152  		meta.EncryptionConfigXML = configData
   153  	case bucketTaggingConfig:
   154  		meta.TaggingConfigXML = configData
   155  	case bucketQuotaConfigFile:
   156  		meta.QuotaConfigJSON = configData
   157  	case objectLockConfig:
   158  		if !globalIsErasure && !globalIsDistErasure {
   159  			return NotImplemented{}
   160  		}
   161  		meta.ObjectLockConfigXML = configData
   162  	case bucketVersioningConfig:
   163  		if !globalIsErasure && !globalIsDistErasure {
   164  			return NotImplemented{}
   165  		}
   166  		meta.VersioningConfigXML = configData
   167  	case bucketReplicationConfig:
   168  		if !globalIsErasure && !globalIsDistErasure {
   169  			return NotImplemented{}
   170  		}
   171  		meta.ReplicationConfigXML = configData
   172  	case bucketTargetsFile:
   173  		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{
   174  			bucket:            meta.Name,
   175  			bucketTargetsFile: bucketTargetsFile,
   176  		})
   177  		if err != nil {
   178  			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
   179  		}
   180  	default:
   181  		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
   182  	}
   183  
   184  	if err := meta.Save(GlobalContext, objAPI); err != nil {
   185  		return err
   186  	}
   187  
   188  	sys.Set(bucket, meta)
   189  	GlobalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
   190  
   191  	return nil
   192  }
   193  
   194  // Get metadata for a bucket.
   195  // If no metadata exists errConfigNotFound is returned and a new metadata is returned.
   196  // Only a shallow copy is returned, so referenced data should not be modified,
   197  // but can be replaced atomically.
   198  //
   199  // This function should only be used with
   200  // - GetBucketInfo
   201  // - ListBuckets
   202  // For all other bucket specific metadata, use the relevant
   203  // calls implemented specifically for each of those features.
   204  func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
   205  	if GlobalIsGateway || bucket == minioMetaBucket {
   206  		return newBucketMetadata(bucket), errConfigNotFound
   207  	}
   208  
   209  	sys.RLock()
   210  	defer sys.RUnlock()
   211  
   212  	meta, ok := sys.metadataMap[bucket]
   213  	if !ok {
   214  		return newBucketMetadata(bucket), errConfigNotFound
   215  	}
   216  
   217  	return meta, nil
   218  }
   219  
   220  // GetVersioningConfig returns configured versioning config
   221  // The returned object may not be modified.
   222  func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
   223  	meta, err := sys.GetConfig(bucket)
   224  	if err != nil {
   225  		return nil, err
   226  	}
   227  	return meta.versioningConfig, nil
   228  }
   229  
   230  // GetTaggingConfig returns configured tagging config
   231  // The returned object may not be modified.
   232  func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
   233  	meta, err := sys.GetConfig(bucket)
   234  	if err != nil {
   235  		if errors.Is(err, errConfigNotFound) {
   236  			return nil, BucketTaggingNotFound{Bucket: bucket}
   237  		}
   238  		return nil, err
   239  	}
   240  	if meta.taggingConfig == nil {
   241  		return nil, BucketTaggingNotFound{Bucket: bucket}
   242  	}
   243  	return meta.taggingConfig, nil
   244  }
   245  
   246  // GetObjectLockConfig returns configured object lock config
   247  // The returned object may not be modified.
   248  func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
   249  	meta, err := sys.GetConfig(bucket)
   250  	if err != nil {
   251  		if errors.Is(err, errConfigNotFound) {
   252  			return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
   253  		}
   254  		return nil, err
   255  	}
   256  	if meta.objectLockConfig == nil {
   257  		return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
   258  	}
   259  	return meta.objectLockConfig, nil
   260  }
   261  
   262  // GetLifecycleConfig returns configured lifecycle config
   263  // The returned object may not be modified.
   264  func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
   265  	meta, err := sys.GetConfig(bucket)
   266  	if err != nil {
   267  		if errors.Is(err, errConfigNotFound) {
   268  			return nil, BucketLifecycleNotFound{Bucket: bucket}
   269  		}
   270  		return nil, err
   271  	}
   272  	if meta.lifecycleConfig == nil {
   273  		return nil, BucketLifecycleNotFound{Bucket: bucket}
   274  	}
   275  	return meta.lifecycleConfig, nil
   276  }
   277  
   278  // GetNotificationConfig returns configured notification config
   279  // The returned object may not be modified.
   280  func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
   281  	if GlobalIsGateway && globalGatewayName == NASBackendGateway {
   282  		// Only needed in case of NAS gateway.
   283  		objAPI := newObjectLayerFn()
   284  		if objAPI == nil {
   285  			return nil, errServerNotInitialized
   286  		}
   287  		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
   288  		if err != nil {
   289  			return nil, err
   290  		}
   291  		return meta.notificationConfig, nil
   292  	}
   293  
   294  	meta, err := sys.GetConfig(bucket)
   295  	if err != nil {
   296  		return nil, err
   297  	}
   298  	return meta.notificationConfig, nil
   299  }
   300  
   301  // GetSSEConfig returns configured SSE config
   302  // The returned object may not be modified.
   303  func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
   304  	meta, err := sys.GetConfig(bucket)
   305  	if err != nil {
   306  		if errors.Is(err, errConfigNotFound) {
   307  			return nil, BucketSSEConfigNotFound{Bucket: bucket}
   308  		}
   309  		return nil, err
   310  	}
   311  	if meta.sseConfig == nil {
   312  		return nil, BucketSSEConfigNotFound{Bucket: bucket}
   313  	}
   314  	return meta.sseConfig, nil
   315  }
   316  
   317  // GetPolicyConfig returns configured bucket policy
   318  // The returned object may not be modified.
   319  func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
   320  	if GlobalIsGateway {
   321  		objAPI := newObjectLayerFn()
   322  		if objAPI == nil {
   323  			return nil, errServerNotInitialized
   324  		}
   325  		return objAPI.GetBucketPolicy(GlobalContext, bucket)
   326  	}
   327  
   328  	meta, err := sys.GetConfig(bucket)
   329  	if err != nil {
   330  		if errors.Is(err, errConfigNotFound) {
   331  			return nil, BucketPolicyNotFound{Bucket: bucket}
   332  		}
   333  		return nil, err
   334  	}
   335  	if meta.policyConfig == nil {
   336  		return nil, BucketPolicyNotFound{Bucket: bucket}
   337  	}
   338  	return meta.policyConfig, nil
   339  }
   340  
   341  // GetQuotaConfig returns configured bucket quota
   342  // The returned object may not be modified.
   343  func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) {
   344  	meta, err := sys.GetConfig(bucket)
   345  	if err != nil {
   346  		return nil, err
   347  	}
   348  	return meta.quotaConfig, nil
   349  }
   350  
   351  // GetReplicationConfig returns configured bucket replication config
   352  // The returned object may not be modified.
   353  func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) {
   354  	meta, err := sys.GetConfig(bucket)
   355  	if err != nil {
   356  		if errors.Is(err, errConfigNotFound) {
   357  			return nil, BucketReplicationConfigNotFound{Bucket: bucket}
   358  		}
   359  		return nil, err
   360  	}
   361  
   362  	if meta.replicationConfig == nil {
   363  		return nil, BucketReplicationConfigNotFound{Bucket: bucket}
   364  	}
   365  	return meta.replicationConfig, nil
   366  }
   367  
   368  // GetBucketTargetsConfig returns configured bucket targets for this bucket
   369  // The returned object may not be modified.
   370  func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
   371  	meta, err := sys.GetConfig(bucket)
   372  	if err != nil {
   373  		return nil, err
   374  	}
   375  	if meta.bucketTargetConfig == nil {
   376  		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
   377  	}
   378  	return meta.bucketTargetConfig, nil
   379  }
   380  
   381  // GetBucketTarget returns the target for the bucket and arn.
   382  func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
   383  	targets, err := sys.GetBucketTargetsConfig(bucket)
   384  	if err != nil {
   385  		return madmin.BucketTarget{}, err
   386  	}
   387  	for _, t := range targets.Targets {
   388  		if t.Arn == arn {
   389  			return t, nil
   390  		}
   391  	}
   392  	return madmin.BucketTarget{}, errConfigNotFound
   393  }
   394  
   395  // GetConfig returns a specific configuration from the bucket metadata.
   396  // The returned object may not be modified.
   397  func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
   398  	objAPI := newObjectLayerFn()
   399  	if objAPI == nil {
   400  		return newBucketMetadata(bucket), errServerNotInitialized
   401  	}
   402  
   403  	if GlobalIsGateway {
   404  		return newBucketMetadata(bucket), NotImplemented{}
   405  	}
   406  
   407  	if bucket == minioMetaBucket {
   408  		return newBucketMetadata(bucket), errInvalidArgument
   409  	}
   410  
   411  	sys.RLock()
   412  	meta, ok := sys.metadataMap[bucket]
   413  	sys.RUnlock()
   414  	if ok {
   415  		return meta, nil
   416  	}
   417  	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
   418  	if err != nil {
   419  		return meta, err
   420  	}
   421  	sys.Lock()
   422  	sys.metadataMap[bucket] = meta
   423  	sys.Unlock()
   424  	return meta, nil
   425  }
   426  
   427  // Init - initializes bucket metadata system for all buckets.
   428  func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
   429  	if objAPI == nil {
   430  		return errServerNotInitialized
   431  	}
   432  
   433  	// In gateway mode, we don't need to load the policies
   434  	// from the backend.
   435  	if GlobalIsGateway {
   436  		return nil
   437  	}
   438  
   439  	// Load bucket metadata sys in background
   440  	go sys.load(ctx, buckets, objAPI)
   441  	return nil
   442  }
   443  
   444  // concurrently load bucket metadata to speed up loading bucket metadata.
   445  func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
   446  	g := errgroup.WithNErrs(len(buckets))
   447  	for index := range buckets {
   448  		index := index
   449  		g.Go(func() error {
   450  			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
   451  				// Ensure heal opts for bucket metadata be deep healed all the time.
   452  				ScanMode: madmin.HealDeepScan,
   453  			})
   454  			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
   455  			if err != nil {
   456  				return err
   457  			}
   458  			sys.Lock()
   459  			sys.metadataMap[buckets[index].Name] = meta
   460  			sys.Unlock()
   461  			return nil
   462  		}, index)
   463  	}
   464  	for _, err := range g.Wait() {
   465  		if err != nil {
   466  			logger.LogIf(ctx, err)
   467  		}
   468  	}
   469  }
   470  
   471  // Loads bucket metadata for all buckets into BucketMetadataSys.
   472  func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
   473  	count := 100 // load 100 bucket metadata at a time.
   474  	for {
   475  		if len(buckets) < count {
   476  			sys.concurrentLoad(ctx, buckets, objAPI)
   477  			return
   478  		}
   479  		sys.concurrentLoad(ctx, buckets[:count], objAPI)
   480  		buckets = buckets[count:]
   481  	}
   482  }
   483  
   484  // Reset the state of the BucketMetadataSys.
   485  func (sys *BucketMetadataSys) Reset() {
   486  	sys.Lock()
   487  	for k := range sys.metadataMap {
   488  		delete(sys.metadataMap, k)
   489  	}
   490  	sys.Unlock()
   491  }
   492  
   493  // NewBucketMetadataSys - creates new policy system.
   494  func NewBucketMetadataSys() *BucketMetadataSys {
   495  	return &BucketMetadataSys{
   496  		metadataMap: make(map[string]BucketMetadata),
   497  	}
   498  }