storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache-bucket.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"net/http"
    25  	"path"
    26  	"runtime/debug"
    27  	"sort"
    28  	"strings"
    29  	"sync"
    30  	"time"
    31  
    32  	"github.com/klauspost/compress/s2"
    33  	"github.com/tinylib/msgp/msgp"
    34  
    35  	"storj.io/minio/cmd/logger"
    36  	"storj.io/minio/pkg/console"
    37  	"storj.io/minio/pkg/hash"
    38  )
    39  
    40  //go:generate msgp -file $GOFILE -unexported
    41  
    42  // a bucketMetacache keeps track of all caches generated
    43  // for a bucket.
    44  type bucketMetacache struct {
    45  	// Name of bucket
    46  	bucket string
    47  
    48  	// caches indexed by id.
    49  	caches map[string]metacache
    50  	// cache ids indexed by root paths
    51  	cachesRoot map[string][]string `msg:"-"`
    52  
    53  	// Internal state
    54  	mu        sync.RWMutex `msg:"-"`
    55  	updated   bool         `msg:"-"`
    56  	transient bool         `msg:"-"` // bucket used for non-persisted caches.
    57  }
    58  
    59  // newBucketMetacache creates a new bucketMetacache.
    60  // Optionally remove all existing caches.
    61  func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache {
    62  	if cleanup {
    63  		// Recursively delete all caches.
    64  		objAPI := newObjectLayerFn()
    65  		ez, ok := objAPI.(*erasureServerPools)
    66  		if ok {
    67  			ctx := context.Background()
    68  			ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
    69  		}
    70  	}
    71  	return &bucketMetacache{
    72  		bucket:     bucket,
    73  		caches:     make(map[string]metacache, 10),
    74  		cachesRoot: make(map[string][]string, 10),
    75  	}
    76  }
    77  
    78  func (b *bucketMetacache) debugf(format string, data ...interface{}) {
    79  	if serverDebugLog {
    80  		console.Debugf(format+"\n", data...)
    81  	}
    82  }
    83  
    84  // loadBucketMetaCache will load the cache from the object layer.
    85  // If the cache cannot be found a new one is created.
    86  func loadBucketMetaCache(ctx context.Context, bucket string) (*bucketMetacache, error) {
    87  	objAPI := newObjectLayerFn()
    88  	for objAPI == nil {
    89  		select {
    90  		case <-ctx.Done():
    91  			return nil, ctx.Err()
    92  		default:
    93  			time.Sleep(250 * time.Millisecond)
    94  		}
    95  		objAPI = newObjectLayerFn()
    96  		if objAPI == nil {
    97  			logger.LogIf(ctx, fmt.Errorf("loadBucketMetaCache: object layer not ready. bucket: %q", bucket))
    98  		}
    99  	}
   100  
   101  	var meta bucketMetacache
   102  	var decErr error
   103  	// Use global context for this.
   104  	r, err := objAPI.GetObjectNInfo(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), nil, http.Header{}, readLock, ObjectOptions{})
   105  	if err == nil {
   106  		dec := s2DecPool.Get().(*s2.Reader)
   107  		dec.Reset(r)
   108  		decErr = meta.DecodeMsg(msgp.NewReader(dec))
   109  		dec.Reset(nil)
   110  		r.Close()
   111  		s2DecPool.Put(dec)
   112  	}
   113  	if err != nil {
   114  		switch err.(type) {
   115  		case ObjectNotFound:
   116  			err = nil
   117  		case InsufficientReadQuorum:
   118  			// Cache is likely lost. Clean up and return new.
   119  			return newBucketMetacache(bucket, true), nil
   120  		default:
   121  			logger.LogIf(ctx, err)
   122  		}
   123  		return newBucketMetacache(bucket, false), err
   124  	}
   125  	if decErr != nil {
   126  		if errors.Is(err, context.Canceled) {
   127  			return newBucketMetacache(bucket, false), err
   128  		}
   129  		// Log the error, but assume the data is lost and return a fresh bucket.
   130  		// Otherwise a broken cache will never recover.
   131  		logger.LogIf(ctx, decErr)
   132  		return newBucketMetacache(bucket, true), nil
   133  	}
   134  	// Sanity check...
   135  	if meta.bucket != bucket {
   136  		logger.Info("loadBucketMetaCache: loaded cache name mismatch, want %s, got %s. Discarding.", bucket, meta.bucket)
   137  		return newBucketMetacache(bucket, true), nil
   138  	}
   139  	meta.cachesRoot = make(map[string][]string, len(meta.caches)/10)
   140  	// Index roots
   141  	for id, cache := range meta.caches {
   142  		meta.cachesRoot[cache.root] = append(meta.cachesRoot[cache.root], id)
   143  	}
   144  	return &meta, nil
   145  }
   146  
   147  // save the bucket cache to the object storage.
   148  func (b *bucketMetacache) save(ctx context.Context) error {
   149  	if b.transient {
   150  		return nil
   151  	}
   152  	objAPI := newObjectLayerFn()
   153  	if objAPI == nil {
   154  		return errServerNotInitialized
   155  	}
   156  
   157  	// Keep lock while we marshal.
   158  	// We need a write lock since we update 'updated'
   159  	b.mu.Lock()
   160  	if !b.updated {
   161  		b.mu.Unlock()
   162  		return nil
   163  	}
   164  	// Save as s2 compressed msgpack
   165  	tmp := bytes.NewBuffer(make([]byte, 0, b.Msgsize()))
   166  	enc := s2.NewWriter(tmp)
   167  	err := msgp.Encode(enc, b)
   168  	if err != nil {
   169  		b.mu.Unlock()
   170  		return err
   171  	}
   172  	err = enc.Close()
   173  	if err != nil {
   174  		b.mu.Unlock()
   175  		return err
   176  	}
   177  	b.updated = false
   178  	b.mu.Unlock()
   179  
   180  	hr, err := hash.NewReader(tmp, int64(tmp.Len()), "", "", int64(tmp.Len()))
   181  	if err != nil {
   182  		return err
   183  	}
   184  	_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr), ObjectOptions{})
   185  	logger.LogIf(ctx, err)
   186  	return err
   187  }
   188  
   189  // findCache will attempt to find a matching cache for the provided options.
   190  // If a cache with the same ID exists already it will be returned.
   191  // If none can be found a new is created with the provided ID.
   192  func (b *bucketMetacache) findCache(o listPathOptions) metacache {
   193  	if b == nil {
   194  		logger.Info("bucketMetacache.findCache: nil cache for bucket %s", o.Bucket)
   195  		return metacache{}
   196  	}
   197  
   198  	if o.Bucket != b.bucket && !b.transient {
   199  		logger.Info("bucketMetacache.findCache: bucket %s does not match this bucket %s", o.Bucket, b.bucket)
   200  		debug.PrintStack()
   201  		return metacache{}
   202  	}
   203  
   204  	extend := globalAPIConfig.getExtendListLife()
   205  
   206  	// Grab a write lock, since we create one if we cannot find one.
   207  	if o.Create {
   208  		b.mu.Lock()
   209  		defer b.mu.Unlock()
   210  	} else {
   211  		b.mu.RLock()
   212  		defer b.mu.RUnlock()
   213  	}
   214  
   215  	// Check if exists already.
   216  	if c, ok := b.caches[o.ID]; ok {
   217  		b.debugf("returning existing %v", o.ID)
   218  		return c
   219  	}
   220  	// No need to do expensive checks on transients.
   221  	if b.transient {
   222  		if !o.Create {
   223  			return metacache{
   224  				id:     o.ID,
   225  				bucket: o.Bucket,
   226  				status: scanStateNone,
   227  			}
   228  		}
   229  
   230  		// Create new
   231  		best := o.newMetacache()
   232  		b.caches[o.ID] = best
   233  		b.updated = true
   234  		b.debugf("returning new cache %s, bucket: %v", best.id, best.bucket)
   235  		return best
   236  	}
   237  
   238  	var best metacache
   239  	rootSplit := strings.Split(o.BaseDir, slashSeparator)
   240  	for i := range rootSplit {
   241  		interesting := b.cachesRoot[path.Join(rootSplit[:i+1]...)]
   242  
   243  		for _, id := range interesting {
   244  			cached, ok := b.caches[id]
   245  			if !ok {
   246  				continue
   247  			}
   248  			if !cached.matches(&o, extend) {
   249  				continue
   250  			}
   251  			if cached.started.Before(best.started) {
   252  				b.debugf("cache %s disregarded - we have a better", cached.id)
   253  				// If we already have a newer, keep that.
   254  				continue
   255  			}
   256  			best = cached
   257  		}
   258  	}
   259  	if !best.started.IsZero() {
   260  		if o.Create {
   261  			best.lastHandout = UTCNow()
   262  			b.caches[best.id] = best
   263  			b.updated = true
   264  		}
   265  		b.debugf("returning cached %s, status: %v, ended: %v", best.id, best.status, best.ended)
   266  		return best
   267  	}
   268  	if !o.Create {
   269  		return metacache{
   270  			id:     o.ID,
   271  			bucket: o.Bucket,
   272  			status: scanStateNone,
   273  		}
   274  	}
   275  
   276  	// Create new and add.
   277  	best = o.newMetacache()
   278  	b.caches[o.ID] = best
   279  	b.cachesRoot[best.root] = append(b.cachesRoot[best.root], best.id)
   280  	b.updated = true
   281  	b.debugf("returning new cache %s, bucket: %v", best.id, best.bucket)
   282  	return best
   283  }
   284  
   285  // cleanup removes redundant and outdated entries.
   286  func (b *bucketMetacache) cleanup() {
   287  	// Entries to remove.
   288  	remove := make(map[string]struct{})
   289  	currentCycle := intDataUpdateTracker.current()
   290  
   291  	// Test on a copy
   292  	// cleanup is the only one deleting caches.
   293  	caches, rootIdx := b.cloneCaches()
   294  
   295  	for id, cache := range caches {
   296  		if b.transient && time.Since(cache.lastUpdate) > 10*time.Minute && time.Since(cache.lastHandout) > 10*time.Minute {
   297  			// Keep transient caches only for 15 minutes.
   298  			remove[id] = struct{}{}
   299  			continue
   300  		}
   301  		if !cache.worthKeeping(currentCycle) {
   302  			b.debugf("cache %s not worth keeping", id)
   303  			remove[id] = struct{}{}
   304  			continue
   305  		}
   306  		if cache.id != id {
   307  			logger.Info("cache ID mismatch %s != %s", id, cache.id)
   308  			remove[id] = struct{}{}
   309  			continue
   310  		}
   311  		if cache.bucket != b.bucket && !b.transient {
   312  			logger.Info("cache bucket mismatch %s != %s", b.bucket, cache.bucket)
   313  			remove[id] = struct{}{}
   314  			continue
   315  		}
   316  	}
   317  
   318  	// Check all non-deleted against eachother.
   319  	// O(n*n), but should still be rather quick.
   320  	for id, cache := range caches {
   321  		if b.transient {
   322  			break
   323  		}
   324  		if _, ok := remove[id]; ok {
   325  			continue
   326  		}
   327  
   328  		interesting := interestingCaches(cache.root, rootIdx)
   329  		for _, id2 := range interesting {
   330  			if _, ok := remove[id2]; ok || id2 == id {
   331  				// Don't check against one we are already removing
   332  				continue
   333  			}
   334  			cache2, ok := caches[id2]
   335  			if !ok {
   336  				continue
   337  			}
   338  
   339  			if cache.canBeReplacedBy(&cache2) {
   340  				b.debugf("cache %s can be replaced by %s", id, cache2.id)
   341  				remove[id] = struct{}{}
   342  				break
   343  			} else {
   344  				b.debugf("cache %s can be NOT replaced by %s", id, cache2.id)
   345  			}
   346  		}
   347  	}
   348  
   349  	// If above limit, remove the caches with the oldest handout time.
   350  	if len(caches)-len(remove) > metacacheMaxEntries {
   351  		remainCaches := make([]metacache, 0, len(caches)-len(remove))
   352  		for id, cache := range caches {
   353  			if _, ok := remove[id]; ok {
   354  				continue
   355  			}
   356  			remainCaches = append(remainCaches, cache)
   357  		}
   358  		if len(remainCaches) > metacacheMaxEntries {
   359  			// Sort oldest last...
   360  			sort.Slice(remainCaches, func(i, j int) bool {
   361  				return remainCaches[i].lastHandout.Before(remainCaches[j].lastHandout)
   362  			})
   363  			// Keep first metacacheMaxEntries...
   364  			for _, cache := range remainCaches[metacacheMaxEntries:] {
   365  				if time.Since(cache.lastHandout) > 30*time.Minute {
   366  					remove[cache.id] = struct{}{}
   367  				}
   368  			}
   369  		}
   370  	}
   371  
   372  	for id := range remove {
   373  		b.deleteCache(id)
   374  	}
   375  }
   376  
   377  // Potentially interesting caches.
   378  // Will only add root if request is for root.
   379  func interestingCaches(root string, cachesRoot map[string][]string) []string {
   380  	var interesting []string
   381  	rootSplit := strings.Split(root, slashSeparator)
   382  	for i := range rootSplit {
   383  		want := path.Join(rootSplit[:i+1]...)
   384  		interesting = append(interesting, cachesRoot[want]...)
   385  	}
   386  	return interesting
   387  }
   388  
   389  // updateCache will update a cache by id.
   390  // If the cache cannot be found nil is returned.
   391  // The bucket cache will be locked until the done .
   392  func (b *bucketMetacache) updateCache(id string) (cache *metacache, done func()) {
   393  	b.mu.Lock()
   394  	c, ok := b.caches[id]
   395  	if !ok {
   396  		b.mu.Unlock()
   397  		return nil, func() {}
   398  	}
   399  	return &c, func() {
   400  		c.lastUpdate = UTCNow()
   401  		b.caches[id] = c
   402  		b.mu.Unlock()
   403  	}
   404  }
   405  
   406  // updateCacheEntry will update a cache.
   407  // Returns the updated status.
   408  func (b *bucketMetacache) updateCacheEntry(update metacache) (metacache, error) {
   409  	b.mu.Lock()
   410  	defer b.mu.Unlock()
   411  	existing, ok := b.caches[update.id]
   412  	if !ok {
   413  		return update, errFileNotFound
   414  	}
   415  	existing.update(update)
   416  	b.caches[update.id] = existing
   417  	b.updated = true
   418  	return existing, nil
   419  }
   420  
   421  // cloneCaches will return a clone of all current caches.
   422  func (b *bucketMetacache) cloneCaches() (map[string]metacache, map[string][]string) {
   423  	b.mu.RLock()
   424  	defer b.mu.RUnlock()
   425  	dst := make(map[string]metacache, len(b.caches))
   426  	for k, v := range b.caches {
   427  		dst[k] = v
   428  	}
   429  	// Copy indexes
   430  	dst2 := make(map[string][]string, len(b.cachesRoot))
   431  	for k, v := range b.cachesRoot {
   432  		tmp := make([]string, len(v))
   433  		copy(tmp, v)
   434  		dst2[k] = tmp
   435  	}
   436  
   437  	return dst, dst2
   438  }
   439  
   440  // getCache will return a clone of a specific metacache.
   441  // Will return nil if the cache doesn't exist.
   442  func (b *bucketMetacache) getCache(id string) *metacache {
   443  	b.mu.RLock()
   444  	c, ok := b.caches[id]
   445  	b.mu.RUnlock()
   446  	if !ok {
   447  		return nil
   448  	}
   449  	return &c
   450  }
   451  
   452  // deleteAll will delete all on disk data for ALL caches.
   453  // Deletes are performed concurrently.
   454  func (b *bucketMetacache) deleteAll() {
   455  	ctx := context.Background()
   456  	ez, ok := newObjectLayerFn().(*erasureServerPools)
   457  	if !ok {
   458  		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasurePools"))
   459  		return
   460  	}
   461  
   462  	b.mu.Lock()
   463  	defer b.mu.Unlock()
   464  
   465  	b.updated = true
   466  	if !b.transient {
   467  		// Delete all.
   468  		ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator))
   469  		b.caches = make(map[string]metacache, 10)
   470  		b.cachesRoot = make(map[string][]string, 10)
   471  		return
   472  	}
   473  
   474  	// Transient are in different buckets.
   475  	var wg sync.WaitGroup
   476  	for id := range b.caches {
   477  		wg.Add(1)
   478  		go func(cache metacache) {
   479  			defer wg.Done()
   480  			ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(cache.bucket, cache.id))
   481  		}(b.caches[id])
   482  	}
   483  	wg.Wait()
   484  	b.caches = make(map[string]metacache, 10)
   485  }
   486  
   487  // deleteCache will delete a specific cache and all files related to it across the cluster.
   488  func (b *bucketMetacache) deleteCache(id string) {
   489  	b.mu.Lock()
   490  	c, ok := b.caches[id]
   491  	if ok {
   492  		// Delete from root map.
   493  		list := b.cachesRoot[c.root]
   494  		for i, lid := range list {
   495  			if id == lid {
   496  				list = append(list[:i], list[i+1:]...)
   497  				break
   498  			}
   499  		}
   500  		b.cachesRoot[c.root] = list
   501  		delete(b.caches, id)
   502  		b.updated = true
   503  	}
   504  	b.mu.Unlock()
   505  	if ok {
   506  		c.delete(context.Background())
   507  	}
   508  }