storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-quota.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "time" 24 25 "storj.io/minio/cmd/logger" 26 "storj.io/minio/pkg/event" 27 "storj.io/minio/pkg/madmin" 28 ) 29 30 // BucketQuotaSys - map of bucket and quota configuration. 31 type BucketQuotaSys struct { 32 bucketStorageCache timedValue 33 } 34 35 // Get - Get quota configuration. 36 func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) { 37 if GlobalIsGateway { 38 objAPI := newObjectLayerFn() 39 if objAPI == nil { 40 return nil, errServerNotInitialized 41 } 42 return &madmin.BucketQuota{}, nil 43 } 44 45 return globalBucketMetadataSys.GetQuotaConfig(bucketName) 46 } 47 48 // NewBucketQuotaSys returns initialized BucketQuotaSys 49 func NewBucketQuotaSys() *BucketQuotaSys { 50 return &BucketQuotaSys{} 51 } 52 53 // parseBucketQuota parses BucketQuota from json 54 func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) { 55 quotaCfg = &madmin.BucketQuota{} 56 if err = json.Unmarshal(data, quotaCfg); err != nil { 57 return quotaCfg, err 58 } 59 if !quotaCfg.IsValid() { 60 return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg) 61 } 62 return 63 } 64 65 func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error { 66 objAPI := newObjectLayerFn() 67 if objAPI == nil { 68 return errServerNotInitialized 69 } 70 71 sys.bucketStorageCache.Once.Do(func() { 72 sys.bucketStorageCache.TTL = 1 * time.Second 73 sys.bucketStorageCache.Update = func() (interface{}, error) { 74 ctx, done := context.WithTimeout(context.Background(), 5*time.Second) 75 defer done() 76 return loadDataUsageFromBackend(ctx, objAPI) 77 } 78 }) 79 80 q, err := sys.Get(bucket) 81 if err != nil { 82 return err 83 } 84 85 if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 { 86 v, err := sys.bucketStorageCache.Get() 87 if err != nil { 88 return err 89 } 90 91 dui := v.(madmin.DataUsageInfo) 92 93 bui, ok := dui.BucketsUsage[bucket] 94 if !ok { 95 // bucket not found, cannot enforce quota 96 // call will fail anyways later. 97 return nil 98 } 99 100 if (bui.Size + uint64(size)) >= q.Quota { 101 return BucketQuotaExceeded{Bucket: bucket} 102 } 103 } 104 105 return nil 106 } 107 108 func enforceBucketQuota(ctx context.Context, bucket string, size int64) error { 109 if size < 0 { 110 return nil 111 } 112 113 return GlobalBucketQuotaSys.check(ctx, bucket, size) 114 } 115 116 // enforceFIFOQuota deletes objects in FIFO order until sufficient objects 117 // have been deleted so as to bring bucket usage within quota. 118 func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui madmin.BucketUsageInfo) { 119 // Check if the current bucket has quota restrictions, if not skip it 120 cfg, err := GlobalBucketQuotaSys.Get(bucket) 121 if err != nil { 122 return 123 } 124 125 if cfg.Type != madmin.FIFOQuota { 126 return 127 } 128 129 var toFree uint64 130 if bui.Size > cfg.Quota && cfg.Quota > 0 { 131 toFree = bui.Size - cfg.Quota 132 } 133 134 if toFree <= 0 { 135 return 136 } 137 138 // Allocate new results channel to receive ObjectInfo. 139 objInfoCh := make(chan ObjectInfo) 140 141 versioned := globalBucketVersioningSys.Enabled(bucket) 142 143 // Walk through all objects 144 if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil { 145 logger.LogIf(ctx, err) 146 return 147 } 148 149 // reuse the fileScorer used by disk cache to score entries by 150 // ModTime to find the oldest objects in bucket to delete. In 151 // the context of bucket quota enforcement - number of hits are 152 // irrelevant. 153 scorer, err := newFileScorer(toFree, time.Now().Unix(), 1) 154 if err != nil { 155 logger.LogIf(ctx, err) 156 return 157 } 158 159 rcfg, _ := globalBucketObjectLockSys.Get(bucket) 160 for obj := range objInfoCh { 161 if obj.DeleteMarker { 162 // Delete markers are automatically added for FIFO purge. 163 scorer.addFileWithObjInfo(obj, 1) 164 continue 165 } 166 // skip objects currently under retention 167 if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) { 168 continue 169 } 170 scorer.addFileWithObjInfo(obj, 1) 171 } 172 173 // If we saw less than quota we are good. 174 if scorer.seenBytes <= cfg.Quota { 175 return 176 } 177 // Calculate how much we want to delete now. 178 toFreeNow := scorer.seenBytes - cfg.Quota 179 // We were less over quota than we thought. Adjust so we delete less. 180 // If we are more over, leave it for the next run to pick up. 181 if toFreeNow < toFree { 182 if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) { 183 // We got below or at quota. 184 return 185 } 186 } 187 188 var objects []ObjectToDelete 189 numKeys := len(scorer.fileObjInfos()) 190 for i, obj := range scorer.fileObjInfos() { 191 objects = append(objects, ObjectToDelete{ 192 ObjectName: obj.Name, 193 VersionID: obj.VersionID, 194 }) 195 if len(objects) < maxDeleteList && (i < numKeys-1) { 196 // skip deletion until maxDeleteList or end of slice 197 continue 198 } 199 200 if len(objects) == 0 { 201 break 202 } 203 204 // Deletes a list of objects. 205 _, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{ 206 Versioned: versioned, 207 }) 208 for i := range deleteErrs { 209 if deleteErrs[i] != nil { 210 logger.LogIf(ctx, deleteErrs[i]) 211 continue 212 } 213 214 // Notify object deleted event. 215 sendEvent(eventArgs{ 216 EventName: event.ObjectRemovedDelete, 217 BucketName: bucket, 218 Object: obj, 219 Host: "Internal: [FIFO-QUOTA-EXPIRY]", 220 }) 221 } 222 objects = nil 223 } 224 }