storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/format-disk-cache.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"io"
    24  	"io/ioutil"
    25  	"os"
    26  	"path"
    27  	"path/filepath"
    28  	"reflect"
    29  	"strings"
    30  
    31  	jsoniter "github.com/json-iterator/go"
    32  	"github.com/minio/sio"
    33  
    34  	"storj.io/minio/cmd/logger"
    35  )
    36  
    37  const (
    38  	// Represents Cache format json holding details on all other cache drives in use.
    39  	formatCache = "cache"
    40  
    41  	// formatCacheV1.Cache.Version
    42  	formatCacheVersionV1 = "1"
    43  	formatCacheVersionV2 = "2"
    44  
    45  	formatMetaVersion1 = "1"
    46  
    47  	formatCacheV1DistributionAlgo = "CRCMOD"
    48  )
    49  
    50  // Represents the current cache structure with list of
    51  // disks comprising the disk cache
    52  // formatCacheV1 - structure holds format config version '1'.
    53  type formatCacheV1 struct {
    54  	formatMetaV1
    55  	Cache struct {
    56  		Version string `json:"version"` // Version of 'cache' format.
    57  		This    string `json:"this"`    // This field carries assigned disk uuid.
    58  		// Disks field carries the input disk order generated the first
    59  		// time when fresh disks were supplied.
    60  		Disks []string `json:"disks"`
    61  		// Distribution algorithm represents the hashing algorithm
    62  		// to pick the right set index for an object.
    63  		DistributionAlgo string `json:"distributionAlgo"`
    64  	} `json:"cache"` // Cache field holds cache format.
    65  }
    66  
    67  // formatCacheV2 is same as formatCacheV1
    68  type formatCacheV2 = formatCacheV1
    69  
    70  // Used to detect the version of "cache" format.
    71  type formatCacheVersionDetect struct {
    72  	Cache struct {
    73  		Version string `json:"version"`
    74  	} `json:"cache"`
    75  }
    76  
    77  // Return a slice of format, to be used to format uninitialized disks.
    78  func newFormatCacheV2(drives []string) []*formatCacheV2 {
    79  	diskCount := len(drives)
    80  	var disks = make([]string, diskCount)
    81  
    82  	var formats = make([]*formatCacheV2, diskCount)
    83  
    84  	for i := 0; i < diskCount; i++ {
    85  		format := &formatCacheV2{}
    86  		format.Version = formatMetaVersion1
    87  		format.Format = formatCache
    88  		format.Cache.Version = formatCacheVersionV2
    89  		format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
    90  		format.Cache.This = mustGetUUID()
    91  		formats[i] = format
    92  		disks[i] = formats[i].Cache.This
    93  	}
    94  	for i := 0; i < diskCount; i++ {
    95  		format := formats[i]
    96  		format.Cache.Disks = disks
    97  	}
    98  	return formats
    99  }
   100  
   101  // Returns formatCache.Cache.Version
   102  func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
   103  	format := &formatCacheVersionDetect{}
   104  	if err := jsonLoad(r, format); err != nil {
   105  		return "", err
   106  	}
   107  	return format.Cache.Version, nil
   108  }
   109  
   110  // Creates a new cache format.json if unformatted.
   111  func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
   112  	// open file using READ & WRITE permission
   113  	var file, err = os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600)
   114  	if err != nil {
   115  		return err
   116  	}
   117  	// Close the locked file upon return.
   118  	defer file.Close()
   119  
   120  	fi, err := file.Stat()
   121  	if err != nil {
   122  		return err
   123  	}
   124  	if fi.Size() != 0 {
   125  		// format.json already got created because of another minio process's createFormatCache()
   126  		return nil
   127  	}
   128  	return jsonSave(file, format)
   129  }
   130  
   131  // This function creates a cache format file on disk and returns a slice
   132  // of format cache config
   133  func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
   134  	nformats := newFormatCacheV2(drives)
   135  	for i, drive := range drives {
   136  		if err = os.MkdirAll(pathJoin(drive, minioMetaBucket), 0777); err != nil {
   137  			logger.GetReqInfo(ctx).AppendTags("drive", drive)
   138  			logger.LogIf(ctx, err)
   139  			return nil, err
   140  		}
   141  		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
   142  		// Fresh disk - create format.json for this cfs
   143  		if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil {
   144  			logger.GetReqInfo(ctx).AppendTags("drive", drive)
   145  			logger.LogIf(ctx, err)
   146  			return nil, err
   147  		}
   148  	}
   149  	return nformats, nil
   150  }
   151  
   152  func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
   153  	formats := make([]*formatCacheV2, len(drives))
   154  	var formatV2 *formatCacheV2
   155  	migrating := false
   156  	for i, drive := range drives {
   157  		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
   158  		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
   159  
   160  		if err != nil {
   161  			if osIsNotExist(err) {
   162  				continue
   163  			}
   164  			logger.LogIf(ctx, err)
   165  			return nil, migrating, err
   166  		}
   167  		defer f.Close()
   168  		format, err := formatMetaCacheV1(f)
   169  		if err != nil {
   170  			continue
   171  		}
   172  		formatV2 = format
   173  		if format.Cache.Version != formatCacheVersionV2 {
   174  			migrating = true
   175  		}
   176  		formats[i] = formatV2
   177  	}
   178  	return formats, migrating, nil
   179  }
   180  
   181  // unmarshalls the cache format.json into formatCacheV1
   182  func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
   183  	format := &formatCacheV1{}
   184  	if err := jsonLoad(r, format); err != nil {
   185  		return nil, err
   186  	}
   187  	return format, nil
   188  }
   189  
   190  func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
   191  	if format.Format != formatCache {
   192  		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
   193  	}
   194  
   195  	// during migration one or more cache drive(s) formats can be out of sync
   196  	if migrating {
   197  		// Validate format version and format type.
   198  		if format.Version != formatMetaVersion1 {
   199  			return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
   200  		}
   201  		if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
   202  			return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
   203  		}
   204  		return nil
   205  	}
   206  	// Validate format version and format type.
   207  	if format.Version != formatMetaVersion1 {
   208  		return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
   209  	}
   210  	if format.Cache.Version != formatCacheVersionV2 {
   211  		return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
   212  	}
   213  	return nil
   214  }
   215  
   216  func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
   217  	for i, formatCache := range formats {
   218  		if formatCache == nil {
   219  			continue
   220  		}
   221  		if err := checkFormatCacheValue(formatCache, migrating); err != nil {
   222  			return i, err
   223  		}
   224  		if len(formats) != len(formatCache.Cache.Disks) {
   225  			return i, fmt.Errorf("Expected number of cache drives %d , got  %d",
   226  				len(formatCache.Cache.Disks), len(formats))
   227  		}
   228  	}
   229  	return -1, nil
   230  }
   231  
   232  // checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
   233  // across disks.
   234  func checkCacheDiskConsistency(formats []*formatCacheV2) error {
   235  	var disks = make([]string, len(formats))
   236  	// Collect currently available disk uuids.
   237  	for index, format := range formats {
   238  		if format == nil {
   239  			disks[index] = ""
   240  			continue
   241  		}
   242  		disks[index] = format.Cache.This
   243  	}
   244  	for i, format := range formats {
   245  		if format == nil {
   246  			continue
   247  		}
   248  		j := findCacheDiskIndex(disks[i], format.Cache.Disks)
   249  		if j == -1 {
   250  			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s", i, j, disks[i])
   251  		}
   252  		if i != j {
   253  			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s got %s", i, j, disks[i], format.Cache.Disks[j])
   254  		}
   255  	}
   256  	return nil
   257  }
   258  
   259  // checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
   260  func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
   261  	var sentinelDisks []string
   262  	// Extract first valid Disks slice.
   263  	for _, format := range formats {
   264  		if format == nil {
   265  			continue
   266  		}
   267  		sentinelDisks = format.Cache.Disks
   268  		break
   269  	}
   270  	for _, format := range formats {
   271  		if format == nil {
   272  			continue
   273  		}
   274  		currentDisks := format.Cache.Disks
   275  		if !reflect.DeepEqual(sentinelDisks, currentDisks) {
   276  			return errors.New("inconsistent cache drives found")
   277  		}
   278  	}
   279  	return nil
   280  }
   281  
   282  // findCacheDiskIndex returns position of cache disk in JBOD.
   283  func findCacheDiskIndex(disk string, disks []string) int {
   284  	for index, uuid := range disks {
   285  		if uuid == disk {
   286  			return index
   287  		}
   288  	}
   289  	return -1
   290  }
   291  
   292  // validate whether cache drives order has changed
   293  func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
   294  	count := 0
   295  	for _, format := range formats {
   296  		if format == nil {
   297  			count++
   298  		}
   299  	}
   300  	if count == len(formats) {
   301  		return errors.New("Cache format files missing on all drives")
   302  	}
   303  	if _, err := checkFormatCacheValues(migrating, formats); err != nil {
   304  		logger.LogIf(ctx, err)
   305  		return err
   306  	}
   307  	if err := checkCacheDisksSliceConsistency(formats); err != nil {
   308  		logger.LogIf(ctx, err)
   309  		return err
   310  	}
   311  	err := checkCacheDiskConsistency(formats)
   312  	logger.LogIf(ctx, err)
   313  	return err
   314  }
   315  
   316  // return true if all of the list of cache drives are
   317  // fresh disks
   318  func cacheDrivesUnformatted(drives []string) bool {
   319  	count := 0
   320  	for _, drive := range drives {
   321  		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
   322  		if _, err := os.Stat(cacheFormatPath); osIsNotExist(err) {
   323  			count++
   324  		}
   325  	}
   326  	return count == len(drives)
   327  }
   328  
   329  // create format.json for each cache drive if fresh disk or load format from disk
   330  // Then validate the format for all drives in the cache to ensure order
   331  // of cache drives has not changed.
   332  func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
   333  	if cacheDrivesUnformatted(drives) {
   334  		formats, err = initFormatCache(ctx, drives)
   335  	} else {
   336  		formats, migrating, err = loadFormatCache(ctx, drives)
   337  	}
   338  	if err != nil {
   339  		return nil, false, err
   340  	}
   341  	if err = validateCacheFormats(ctx, migrating, formats); err != nil {
   342  		return nil, false, err
   343  	}
   344  	return formats, migrating, nil
   345  }
   346  
   347  // reads cached object on disk and writes it back after adding bitrot
   348  // hashsum per block as per the new disk cache format.
   349  func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error {
   350  	st, err := os.Stat(oldfile)
   351  	if err != nil {
   352  		err = osErrToFileErr(err)
   353  		return err
   354  	}
   355  	readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
   356  	if err != nil {
   357  		return err
   358  	}
   359  	var reader io.Reader = readCloser
   360  
   361  	actualSize := uint64(st.Size())
   362  	if globalCacheKMS != nil {
   363  		reader, err = newCacheEncryptReader(readCloser, bucket, object, metadata)
   364  		if err != nil {
   365  			return err
   366  		}
   367  		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
   368  	}
   369  	_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
   370  	return err
   371  }
   372  
   373  // migrate cache contents from old cacheFS format to new backend format
   374  // new format is flat
   375  //  sha(bucket,object)/  <== dir name
   376  //      - part.1         <== data
   377  //      - cache.json     <== metadata
   378  func migrateOldCache(ctx context.Context, c *diskCache) error {
   379  	oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
   380  	cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
   381  
   382  	if _, err := os.Stat(oldCacheBucketsPath); err != nil {
   383  		// remove .minio.sys sub directories
   384  		removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
   385  		removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
   386  		removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
   387  		removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
   388  		// just migrate cache format
   389  		return migrateCacheFormatJSON(cacheFormatPath)
   390  	}
   391  
   392  	buckets, err := readDir(oldCacheBucketsPath)
   393  	if err != nil {
   394  		return err
   395  	}
   396  
   397  	for _, bucket := range buckets {
   398  		bucket = strings.TrimSuffix(bucket, SlashSeparator)
   399  		var objMetaPaths []string
   400  		root := path.Join(oldCacheBucketsPath, bucket)
   401  		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
   402  			if strings.HasSuffix(path, cacheMetaJSONFile) {
   403  				objMetaPaths = append(objMetaPaths, path)
   404  			}
   405  			return nil
   406  		})
   407  		if err != nil {
   408  			return err
   409  		}
   410  		for _, oMeta := range objMetaPaths {
   411  			objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
   412  			object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
   413  			object = strings.TrimSuffix(object, "/")
   414  
   415  			destdir := getCacheSHADir(c.dir, bucket, object)
   416  			if err := os.MkdirAll(destdir, 0777); err != nil {
   417  				return err
   418  			}
   419  			prevCachedPath := path.Join(c.dir, bucket, object)
   420  
   421  			// get old cached metadata
   422  			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
   423  			metaPath := pathJoin(destdir, cacheMetaJSONFile)
   424  			metaBytes, err := ioutil.ReadFile(oldMetaPath)
   425  			if err != nil {
   426  				return err
   427  			}
   428  			// marshal cache metadata after adding version and stat info
   429  			meta := &cacheMeta{}
   430  			var json = jsoniter.ConfigCompatibleWithStandardLibrary
   431  			if err = json.Unmarshal(metaBytes, &meta); err != nil {
   432  				return err
   433  			}
   434  			// move cached object to new cache directory path
   435  			// migrate cache data and add bit-rot protection hash sum
   436  			// at the start of each block
   437  			if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil {
   438  				continue
   439  			}
   440  			stat, err := os.Stat(prevCachedPath)
   441  			if err != nil {
   442  				if err == errFileNotFound {
   443  					continue
   444  				}
   445  				logger.LogIf(ctx, err)
   446  				return err
   447  			}
   448  			// old cached file can now be removed
   449  			if err := os.Remove(prevCachedPath); err != nil {
   450  				return err
   451  			}
   452  			// move cached metadata after changing cache metadata version
   453  			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
   454  			meta.Version = cacheMetaVersion
   455  			meta.Stat.Size = stat.Size()
   456  			meta.Stat.ModTime = stat.ModTime()
   457  			jsonData, err := json.Marshal(meta)
   458  			if err != nil {
   459  				return err
   460  			}
   461  
   462  			if err = ioutil.WriteFile(metaPath, jsonData, 0644); err != nil {
   463  				return err
   464  			}
   465  		}
   466  
   467  		// delete old bucket from cache, now that all contents are cleared
   468  		removeAll(path.Join(c.dir, bucket))
   469  	}
   470  
   471  	// remove .minio.sys sub directories
   472  	removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
   473  	removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
   474  	removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
   475  	removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
   476  
   477  	return migrateCacheFormatJSON(cacheFormatPath)
   478  
   479  }
   480  
   481  func migrateCacheFormatJSON(cacheFormatPath string) error {
   482  	// now migrate format.json
   483  	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
   484  	if err != nil {
   485  		return err
   486  	}
   487  	defer f.Close()
   488  	formatV1 := formatCacheV1{}
   489  	if err := jsonLoad(f, &formatV1); err != nil {
   490  		return err
   491  	}
   492  
   493  	formatV2 := &formatCacheV2{}
   494  	formatV2.formatMetaV1 = formatV1.formatMetaV1
   495  	formatV2.Version = formatMetaVersion1
   496  	formatV2.Cache = formatV1.Cache
   497  	formatV2.Cache.Version = formatCacheVersionV2
   498  	if err := jsonSave(f, formatV2); err != nil {
   499  		return err
   500  	}
   501  	return nil
   502  }