storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache-bucket.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "errors" 23 "fmt" 24 "net/http" 25 "path" 26 "runtime/debug" 27 "sort" 28 "strings" 29 "sync" 30 "time" 31 32 "github.com/klauspost/compress/s2" 33 "github.com/tinylib/msgp/msgp" 34 35 "storj.io/minio/cmd/logger" 36 "storj.io/minio/pkg/console" 37 "storj.io/minio/pkg/hash" 38 ) 39 40 //go:generate msgp -file $GOFILE -unexported 41 42 // a bucketMetacache keeps track of all caches generated 43 // for a bucket. 44 type bucketMetacache struct { 45 // Name of bucket 46 bucket string 47 48 // caches indexed by id. 49 caches map[string]metacache 50 // cache ids indexed by root paths 51 cachesRoot map[string][]string `msg:"-"` 52 53 // Internal state 54 mu sync.RWMutex `msg:"-"` 55 updated bool `msg:"-"` 56 transient bool `msg:"-"` // bucket used for non-persisted caches. 57 } 58 59 // newBucketMetacache creates a new bucketMetacache. 60 // Optionally remove all existing caches. 61 func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache { 62 if cleanup { 63 // Recursively delete all caches. 64 objAPI := newObjectLayerFn() 65 ez, ok := objAPI.(*erasureServerPools) 66 if ok { 67 ctx := context.Background() 68 ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator)) 69 } 70 } 71 return &bucketMetacache{ 72 bucket: bucket, 73 caches: make(map[string]metacache, 10), 74 cachesRoot: make(map[string][]string, 10), 75 } 76 } 77 78 func (b *bucketMetacache) debugf(format string, data ...interface{}) { 79 if serverDebugLog { 80 console.Debugf(format+"\n", data...) 81 } 82 } 83 84 // loadBucketMetaCache will load the cache from the object layer. 85 // If the cache cannot be found a new one is created. 86 func loadBucketMetaCache(ctx context.Context, bucket string) (*bucketMetacache, error) { 87 objAPI := newObjectLayerFn() 88 for objAPI == nil { 89 select { 90 case <-ctx.Done(): 91 return nil, ctx.Err() 92 default: 93 time.Sleep(250 * time.Millisecond) 94 } 95 objAPI = newObjectLayerFn() 96 if objAPI == nil { 97 logger.LogIf(ctx, fmt.Errorf("loadBucketMetaCache: object layer not ready. bucket: %q", bucket)) 98 } 99 } 100 101 var meta bucketMetacache 102 var decErr error 103 // Use global context for this. 104 r, err := objAPI.GetObjectNInfo(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), nil, http.Header{}, readLock, ObjectOptions{}) 105 if err == nil { 106 dec := s2DecPool.Get().(*s2.Reader) 107 dec.Reset(r) 108 decErr = meta.DecodeMsg(msgp.NewReader(dec)) 109 dec.Reset(nil) 110 r.Close() 111 s2DecPool.Put(dec) 112 } 113 if err != nil { 114 switch err.(type) { 115 case ObjectNotFound: 116 err = nil 117 case InsufficientReadQuorum: 118 // Cache is likely lost. Clean up and return new. 119 return newBucketMetacache(bucket, true), nil 120 default: 121 logger.LogIf(ctx, err) 122 } 123 return newBucketMetacache(bucket, false), err 124 } 125 if decErr != nil { 126 if errors.Is(err, context.Canceled) { 127 return newBucketMetacache(bucket, false), err 128 } 129 // Log the error, but assume the data is lost and return a fresh bucket. 130 // Otherwise a broken cache will never recover. 131 logger.LogIf(ctx, decErr) 132 return newBucketMetacache(bucket, true), nil 133 } 134 // Sanity check... 135 if meta.bucket != bucket { 136 logger.Info("loadBucketMetaCache: loaded cache name mismatch, want %s, got %s. Discarding.", bucket, meta.bucket) 137 return newBucketMetacache(bucket, true), nil 138 } 139 meta.cachesRoot = make(map[string][]string, len(meta.caches)/10) 140 // Index roots 141 for id, cache := range meta.caches { 142 meta.cachesRoot[cache.root] = append(meta.cachesRoot[cache.root], id) 143 } 144 return &meta, nil 145 } 146 147 // save the bucket cache to the object storage. 148 func (b *bucketMetacache) save(ctx context.Context) error { 149 if b.transient { 150 return nil 151 } 152 objAPI := newObjectLayerFn() 153 if objAPI == nil { 154 return errServerNotInitialized 155 } 156 157 // Keep lock while we marshal. 158 // We need a write lock since we update 'updated' 159 b.mu.Lock() 160 if !b.updated { 161 b.mu.Unlock() 162 return nil 163 } 164 // Save as s2 compressed msgpack 165 tmp := bytes.NewBuffer(make([]byte, 0, b.Msgsize())) 166 enc := s2.NewWriter(tmp) 167 err := msgp.Encode(enc, b) 168 if err != nil { 169 b.mu.Unlock() 170 return err 171 } 172 err = enc.Close() 173 if err != nil { 174 b.mu.Unlock() 175 return err 176 } 177 b.updated = false 178 b.mu.Unlock() 179 180 hr, err := hash.NewReader(tmp, int64(tmp.Len()), "", "", int64(tmp.Len())) 181 if err != nil { 182 return err 183 } 184 _, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr), ObjectOptions{}) 185 logger.LogIf(ctx, err) 186 return err 187 } 188 189 // findCache will attempt to find a matching cache for the provided options. 190 // If a cache with the same ID exists already it will be returned. 191 // If none can be found a new is created with the provided ID. 192 func (b *bucketMetacache) findCache(o listPathOptions) metacache { 193 if b == nil { 194 logger.Info("bucketMetacache.findCache: nil cache for bucket %s", o.Bucket) 195 return metacache{} 196 } 197 198 if o.Bucket != b.bucket && !b.transient { 199 logger.Info("bucketMetacache.findCache: bucket %s does not match this bucket %s", o.Bucket, b.bucket) 200 debug.PrintStack() 201 return metacache{} 202 } 203 204 extend := globalAPIConfig.getExtendListLife() 205 206 // Grab a write lock, since we create one if we cannot find one. 207 if o.Create { 208 b.mu.Lock() 209 defer b.mu.Unlock() 210 } else { 211 b.mu.RLock() 212 defer b.mu.RUnlock() 213 } 214 215 // Check if exists already. 216 if c, ok := b.caches[o.ID]; ok { 217 b.debugf("returning existing %v", o.ID) 218 return c 219 } 220 // No need to do expensive checks on transients. 221 if b.transient { 222 if !o.Create { 223 return metacache{ 224 id: o.ID, 225 bucket: o.Bucket, 226 status: scanStateNone, 227 } 228 } 229 230 // Create new 231 best := o.newMetacache() 232 b.caches[o.ID] = best 233 b.updated = true 234 b.debugf("returning new cache %s, bucket: %v", best.id, best.bucket) 235 return best 236 } 237 238 var best metacache 239 rootSplit := strings.Split(o.BaseDir, slashSeparator) 240 for i := range rootSplit { 241 interesting := b.cachesRoot[path.Join(rootSplit[:i+1]...)] 242 243 for _, id := range interesting { 244 cached, ok := b.caches[id] 245 if !ok { 246 continue 247 } 248 if !cached.matches(&o, extend) { 249 continue 250 } 251 if cached.started.Before(best.started) { 252 b.debugf("cache %s disregarded - we have a better", cached.id) 253 // If we already have a newer, keep that. 254 continue 255 } 256 best = cached 257 } 258 } 259 if !best.started.IsZero() { 260 if o.Create { 261 best.lastHandout = UTCNow() 262 b.caches[best.id] = best 263 b.updated = true 264 } 265 b.debugf("returning cached %s, status: %v, ended: %v", best.id, best.status, best.ended) 266 return best 267 } 268 if !o.Create { 269 return metacache{ 270 id: o.ID, 271 bucket: o.Bucket, 272 status: scanStateNone, 273 } 274 } 275 276 // Create new and add. 277 best = o.newMetacache() 278 b.caches[o.ID] = best 279 b.cachesRoot[best.root] = append(b.cachesRoot[best.root], best.id) 280 b.updated = true 281 b.debugf("returning new cache %s, bucket: %v", best.id, best.bucket) 282 return best 283 } 284 285 // cleanup removes redundant and outdated entries. 286 func (b *bucketMetacache) cleanup() { 287 // Entries to remove. 288 remove := make(map[string]struct{}) 289 currentCycle := intDataUpdateTracker.current() 290 291 // Test on a copy 292 // cleanup is the only one deleting caches. 293 caches, rootIdx := b.cloneCaches() 294 295 for id, cache := range caches { 296 if b.transient && time.Since(cache.lastUpdate) > 10*time.Minute && time.Since(cache.lastHandout) > 10*time.Minute { 297 // Keep transient caches only for 15 minutes. 298 remove[id] = struct{}{} 299 continue 300 } 301 if !cache.worthKeeping(currentCycle) { 302 b.debugf("cache %s not worth keeping", id) 303 remove[id] = struct{}{} 304 continue 305 } 306 if cache.id != id { 307 logger.Info("cache ID mismatch %s != %s", id, cache.id) 308 remove[id] = struct{}{} 309 continue 310 } 311 if cache.bucket != b.bucket && !b.transient { 312 logger.Info("cache bucket mismatch %s != %s", b.bucket, cache.bucket) 313 remove[id] = struct{}{} 314 continue 315 } 316 } 317 318 // Check all non-deleted against eachother. 319 // O(n*n), but should still be rather quick. 320 for id, cache := range caches { 321 if b.transient { 322 break 323 } 324 if _, ok := remove[id]; ok { 325 continue 326 } 327 328 interesting := interestingCaches(cache.root, rootIdx) 329 for _, id2 := range interesting { 330 if _, ok := remove[id2]; ok || id2 == id { 331 // Don't check against one we are already removing 332 continue 333 } 334 cache2, ok := caches[id2] 335 if !ok { 336 continue 337 } 338 339 if cache.canBeReplacedBy(&cache2) { 340 b.debugf("cache %s can be replaced by %s", id, cache2.id) 341 remove[id] = struct{}{} 342 break 343 } else { 344 b.debugf("cache %s can be NOT replaced by %s", id, cache2.id) 345 } 346 } 347 } 348 349 // If above limit, remove the caches with the oldest handout time. 350 if len(caches)-len(remove) > metacacheMaxEntries { 351 remainCaches := make([]metacache, 0, len(caches)-len(remove)) 352 for id, cache := range caches { 353 if _, ok := remove[id]; ok { 354 continue 355 } 356 remainCaches = append(remainCaches, cache) 357 } 358 if len(remainCaches) > metacacheMaxEntries { 359 // Sort oldest last... 360 sort.Slice(remainCaches, func(i, j int) bool { 361 return remainCaches[i].lastHandout.Before(remainCaches[j].lastHandout) 362 }) 363 // Keep first metacacheMaxEntries... 364 for _, cache := range remainCaches[metacacheMaxEntries:] { 365 if time.Since(cache.lastHandout) > 30*time.Minute { 366 remove[cache.id] = struct{}{} 367 } 368 } 369 } 370 } 371 372 for id := range remove { 373 b.deleteCache(id) 374 } 375 } 376 377 // Potentially interesting caches. 378 // Will only add root if request is for root. 379 func interestingCaches(root string, cachesRoot map[string][]string) []string { 380 var interesting []string 381 rootSplit := strings.Split(root, slashSeparator) 382 for i := range rootSplit { 383 want := path.Join(rootSplit[:i+1]...) 384 interesting = append(interesting, cachesRoot[want]...) 385 } 386 return interesting 387 } 388 389 // updateCache will update a cache by id. 390 // If the cache cannot be found nil is returned. 391 // The bucket cache will be locked until the done . 392 func (b *bucketMetacache) updateCache(id string) (cache *metacache, done func()) { 393 b.mu.Lock() 394 c, ok := b.caches[id] 395 if !ok { 396 b.mu.Unlock() 397 return nil, func() {} 398 } 399 return &c, func() { 400 c.lastUpdate = UTCNow() 401 b.caches[id] = c 402 b.mu.Unlock() 403 } 404 } 405 406 // updateCacheEntry will update a cache. 407 // Returns the updated status. 408 func (b *bucketMetacache) updateCacheEntry(update metacache) (metacache, error) { 409 b.mu.Lock() 410 defer b.mu.Unlock() 411 existing, ok := b.caches[update.id] 412 if !ok { 413 return update, errFileNotFound 414 } 415 existing.update(update) 416 b.caches[update.id] = existing 417 b.updated = true 418 return existing, nil 419 } 420 421 // cloneCaches will return a clone of all current caches. 422 func (b *bucketMetacache) cloneCaches() (map[string]metacache, map[string][]string) { 423 b.mu.RLock() 424 defer b.mu.RUnlock() 425 dst := make(map[string]metacache, len(b.caches)) 426 for k, v := range b.caches { 427 dst[k] = v 428 } 429 // Copy indexes 430 dst2 := make(map[string][]string, len(b.cachesRoot)) 431 for k, v := range b.cachesRoot { 432 tmp := make([]string, len(v)) 433 copy(tmp, v) 434 dst2[k] = tmp 435 } 436 437 return dst, dst2 438 } 439 440 // getCache will return a clone of a specific metacache. 441 // Will return nil if the cache doesn't exist. 442 func (b *bucketMetacache) getCache(id string) *metacache { 443 b.mu.RLock() 444 c, ok := b.caches[id] 445 b.mu.RUnlock() 446 if !ok { 447 return nil 448 } 449 return &c 450 } 451 452 // deleteAll will delete all on disk data for ALL caches. 453 // Deletes are performed concurrently. 454 func (b *bucketMetacache) deleteAll() { 455 ctx := context.Background() 456 ez, ok := newObjectLayerFn().(*erasureServerPools) 457 if !ok { 458 logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasurePools")) 459 return 460 } 461 462 b.mu.Lock() 463 defer b.mu.Unlock() 464 465 b.updated = true 466 if !b.transient { 467 // Delete all. 468 ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator)) 469 b.caches = make(map[string]metacache, 10) 470 b.cachesRoot = make(map[string][]string, 10) 471 return 472 } 473 474 // Transient are in different buckets. 475 var wg sync.WaitGroup 476 for id := range b.caches { 477 wg.Add(1) 478 go func(cache metacache) { 479 defer wg.Done() 480 ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(cache.bucket, cache.id)) 481 }(b.caches[id]) 482 } 483 wg.Wait() 484 b.caches = make(map[string]metacache, 10) 485 } 486 487 // deleteCache will delete a specific cache and all files related to it across the cluster. 488 func (b *bucketMetacache) deleteCache(id string) { 489 b.mu.Lock() 490 c, ok := b.caches[id] 491 if ok { 492 // Delete from root map. 493 list := b.cachesRoot[c.root] 494 for i, lid := range list { 495 if id == lid { 496 list = append(list[:i], list[i+1:]...) 497 break 498 } 499 } 500 b.cachesRoot[c.root] = list 501 delete(b.caches, id) 502 b.updated = true 503 } 504 b.mu.Unlock() 505 if ok { 506 c.delete(context.Background()) 507 } 508 }