storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-quota.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"time"
    24  
    25  	"storj.io/minio/cmd/logger"
    26  	"storj.io/minio/pkg/event"
    27  	"storj.io/minio/pkg/madmin"
    28  )
    29  
    30  // BucketQuotaSys - map of bucket and quota configuration.
    31  type BucketQuotaSys struct {
    32  	bucketStorageCache timedValue
    33  }
    34  
    35  // Get - Get quota configuration.
    36  func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
    37  	if GlobalIsGateway {
    38  		objAPI := newObjectLayerFn()
    39  		if objAPI == nil {
    40  			return nil, errServerNotInitialized
    41  		}
    42  		return &madmin.BucketQuota{}, nil
    43  	}
    44  
    45  	return globalBucketMetadataSys.GetQuotaConfig(bucketName)
    46  }
    47  
    48  // NewBucketQuotaSys returns initialized BucketQuotaSys
    49  func NewBucketQuotaSys() *BucketQuotaSys {
    50  	return &BucketQuotaSys{}
    51  }
    52  
    53  // parseBucketQuota parses BucketQuota from json
    54  func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
    55  	quotaCfg = &madmin.BucketQuota{}
    56  	if err = json.Unmarshal(data, quotaCfg); err != nil {
    57  		return quotaCfg, err
    58  	}
    59  	if !quotaCfg.IsValid() {
    60  		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
    61  	}
    62  	return
    63  }
    64  
    65  func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
    66  	objAPI := newObjectLayerFn()
    67  	if objAPI == nil {
    68  		return errServerNotInitialized
    69  	}
    70  
    71  	sys.bucketStorageCache.Once.Do(func() {
    72  		sys.bucketStorageCache.TTL = 1 * time.Second
    73  		sys.bucketStorageCache.Update = func() (interface{}, error) {
    74  			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
    75  			defer done()
    76  			return loadDataUsageFromBackend(ctx, objAPI)
    77  		}
    78  	})
    79  
    80  	q, err := sys.Get(bucket)
    81  	if err != nil {
    82  		return err
    83  	}
    84  
    85  	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
    86  		v, err := sys.bucketStorageCache.Get()
    87  		if err != nil {
    88  			return err
    89  		}
    90  
    91  		dui := v.(madmin.DataUsageInfo)
    92  
    93  		bui, ok := dui.BucketsUsage[bucket]
    94  		if !ok {
    95  			// bucket not found, cannot enforce quota
    96  			// call will fail anyways later.
    97  			return nil
    98  		}
    99  
   100  		if (bui.Size + uint64(size)) >= q.Quota {
   101  			return BucketQuotaExceeded{Bucket: bucket}
   102  		}
   103  	}
   104  
   105  	return nil
   106  }
   107  
   108  func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
   109  	if size < 0 {
   110  		return nil
   111  	}
   112  
   113  	return GlobalBucketQuotaSys.check(ctx, bucket, size)
   114  }
   115  
   116  // enforceFIFOQuota deletes objects in FIFO order until sufficient objects
   117  // have been deleted so as to bring bucket usage within quota.
   118  func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui madmin.BucketUsageInfo) {
   119  	// Check if the current bucket has quota restrictions, if not skip it
   120  	cfg, err := GlobalBucketQuotaSys.Get(bucket)
   121  	if err != nil {
   122  		return
   123  	}
   124  
   125  	if cfg.Type != madmin.FIFOQuota {
   126  		return
   127  	}
   128  
   129  	var toFree uint64
   130  	if bui.Size > cfg.Quota && cfg.Quota > 0 {
   131  		toFree = bui.Size - cfg.Quota
   132  	}
   133  
   134  	if toFree <= 0 {
   135  		return
   136  	}
   137  
   138  	// Allocate new results channel to receive ObjectInfo.
   139  	objInfoCh := make(chan ObjectInfo)
   140  
   141  	versioned := globalBucketVersioningSys.Enabled(bucket)
   142  
   143  	// Walk through all objects
   144  	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
   145  		logger.LogIf(ctx, err)
   146  		return
   147  	}
   148  
   149  	// reuse the fileScorer used by disk cache to score entries by
   150  	// ModTime to find the oldest objects in bucket to delete. In
   151  	// the context of bucket quota enforcement - number of hits are
   152  	// irrelevant.
   153  	scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
   154  	if err != nil {
   155  		logger.LogIf(ctx, err)
   156  		return
   157  	}
   158  
   159  	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
   160  	for obj := range objInfoCh {
   161  		if obj.DeleteMarker {
   162  			// Delete markers are automatically added for FIFO purge.
   163  			scorer.addFileWithObjInfo(obj, 1)
   164  			continue
   165  		}
   166  		// skip objects currently under retention
   167  		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
   168  			continue
   169  		}
   170  		scorer.addFileWithObjInfo(obj, 1)
   171  	}
   172  
   173  	// If we saw less than quota we are good.
   174  	if scorer.seenBytes <= cfg.Quota {
   175  		return
   176  	}
   177  	// Calculate how much we want to delete now.
   178  	toFreeNow := scorer.seenBytes - cfg.Quota
   179  	// We were less over quota than we thought. Adjust so we delete less.
   180  	// If we are more over, leave it for the next run to pick up.
   181  	if toFreeNow < toFree {
   182  		if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) {
   183  			// We got below or at quota.
   184  			return
   185  		}
   186  	}
   187  
   188  	var objects []ObjectToDelete
   189  	numKeys := len(scorer.fileObjInfos())
   190  	for i, obj := range scorer.fileObjInfos() {
   191  		objects = append(objects, ObjectToDelete{
   192  			ObjectName: obj.Name,
   193  			VersionID:  obj.VersionID,
   194  		})
   195  		if len(objects) < maxDeleteList && (i < numKeys-1) {
   196  			// skip deletion until maxDeleteList or end of slice
   197  			continue
   198  		}
   199  
   200  		if len(objects) == 0 {
   201  			break
   202  		}
   203  
   204  		// Deletes a list of objects.
   205  		_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
   206  			Versioned: versioned,
   207  		})
   208  		for i := range deleteErrs {
   209  			if deleteErrs[i] != nil {
   210  				logger.LogIf(ctx, deleteErrs[i])
   211  				continue
   212  			}
   213  
   214  			// Notify object deleted event.
   215  			sendEvent(eventArgs{
   216  				EventName:  event.ObjectRemovedDelete,
   217  				BucketName: bucket,
   218  				Object:     obj,
   219  				Host:       "Internal: [FIFO-QUOTA-EXPIRY]",
   220  			})
   221  		}
   222  		objects = nil
   223  	}
   224  }