storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/format-disk-cache.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2018 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "os" 26 "path" 27 "path/filepath" 28 "reflect" 29 "strings" 30 31 jsoniter "github.com/json-iterator/go" 32 "github.com/minio/sio" 33 34 "storj.io/minio/cmd/logger" 35 ) 36 37 const ( 38 // Represents Cache format json holding details on all other cache drives in use. 39 formatCache = "cache" 40 41 // formatCacheV1.Cache.Version 42 formatCacheVersionV1 = "1" 43 formatCacheVersionV2 = "2" 44 45 formatMetaVersion1 = "1" 46 47 formatCacheV1DistributionAlgo = "CRCMOD" 48 ) 49 50 // Represents the current cache structure with list of 51 // disks comprising the disk cache 52 // formatCacheV1 - structure holds format config version '1'. 53 type formatCacheV1 struct { 54 formatMetaV1 55 Cache struct { 56 Version string `json:"version"` // Version of 'cache' format. 57 This string `json:"this"` // This field carries assigned disk uuid. 58 // Disks field carries the input disk order generated the first 59 // time when fresh disks were supplied. 60 Disks []string `json:"disks"` 61 // Distribution algorithm represents the hashing algorithm 62 // to pick the right set index for an object. 63 DistributionAlgo string `json:"distributionAlgo"` 64 } `json:"cache"` // Cache field holds cache format. 65 } 66 67 // formatCacheV2 is same as formatCacheV1 68 type formatCacheV2 = formatCacheV1 69 70 // Used to detect the version of "cache" format. 71 type formatCacheVersionDetect struct { 72 Cache struct { 73 Version string `json:"version"` 74 } `json:"cache"` 75 } 76 77 // Return a slice of format, to be used to format uninitialized disks. 78 func newFormatCacheV2(drives []string) []*formatCacheV2 { 79 diskCount := len(drives) 80 var disks = make([]string, diskCount) 81 82 var formats = make([]*formatCacheV2, diskCount) 83 84 for i := 0; i < diskCount; i++ { 85 format := &formatCacheV2{} 86 format.Version = formatMetaVersion1 87 format.Format = formatCache 88 format.Cache.Version = formatCacheVersionV2 89 format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo 90 format.Cache.This = mustGetUUID() 91 formats[i] = format 92 disks[i] = formats[i].Cache.This 93 } 94 for i := 0; i < diskCount; i++ { 95 format := formats[i] 96 format.Cache.Disks = disks 97 } 98 return formats 99 } 100 101 // Returns formatCache.Cache.Version 102 func formatCacheGetVersion(r io.ReadSeeker) (string, error) { 103 format := &formatCacheVersionDetect{} 104 if err := jsonLoad(r, format); err != nil { 105 return "", err 106 } 107 return format.Cache.Version, nil 108 } 109 110 // Creates a new cache format.json if unformatted. 111 func createFormatCache(fsFormatPath string, format *formatCacheV1) error { 112 // open file using READ & WRITE permission 113 var file, err = os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600) 114 if err != nil { 115 return err 116 } 117 // Close the locked file upon return. 118 defer file.Close() 119 120 fi, err := file.Stat() 121 if err != nil { 122 return err 123 } 124 if fi.Size() != 0 { 125 // format.json already got created because of another minio process's createFormatCache() 126 return nil 127 } 128 return jsonSave(file, format) 129 } 130 131 // This function creates a cache format file on disk and returns a slice 132 // of format cache config 133 func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) { 134 nformats := newFormatCacheV2(drives) 135 for i, drive := range drives { 136 if err = os.MkdirAll(pathJoin(drive, minioMetaBucket), 0777); err != nil { 137 logger.GetReqInfo(ctx).AppendTags("drive", drive) 138 logger.LogIf(ctx, err) 139 return nil, err 140 } 141 cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile) 142 // Fresh disk - create format.json for this cfs 143 if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil { 144 logger.GetReqInfo(ctx).AppendTags("drive", drive) 145 logger.LogIf(ctx, err) 146 return nil, err 147 } 148 } 149 return nformats, nil 150 } 151 152 func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) { 153 formats := make([]*formatCacheV2, len(drives)) 154 var formatV2 *formatCacheV2 155 migrating := false 156 for i, drive := range drives { 157 cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile) 158 f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0) 159 160 if err != nil { 161 if osIsNotExist(err) { 162 continue 163 } 164 logger.LogIf(ctx, err) 165 return nil, migrating, err 166 } 167 defer f.Close() 168 format, err := formatMetaCacheV1(f) 169 if err != nil { 170 continue 171 } 172 formatV2 = format 173 if format.Cache.Version != formatCacheVersionV2 { 174 migrating = true 175 } 176 formats[i] = formatV2 177 } 178 return formats, migrating, nil 179 } 180 181 // unmarshalls the cache format.json into formatCacheV1 182 func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) { 183 format := &formatCacheV1{} 184 if err := jsonLoad(r, format); err != nil { 185 return nil, err 186 } 187 return format, nil 188 } 189 190 func checkFormatCacheValue(format *formatCacheV2, migrating bool) error { 191 if format.Format != formatCache { 192 return fmt.Errorf("Unsupported cache format [%s] found", format.Format) 193 } 194 195 // during migration one or more cache drive(s) formats can be out of sync 196 if migrating { 197 // Validate format version and format type. 198 if format.Version != formatMetaVersion1 { 199 return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version) 200 } 201 if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 { 202 return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version) 203 } 204 return nil 205 } 206 // Validate format version and format type. 207 if format.Version != formatMetaVersion1 { 208 return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version) 209 } 210 if format.Cache.Version != formatCacheVersionV2 { 211 return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version) 212 } 213 return nil 214 } 215 216 func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) { 217 for i, formatCache := range formats { 218 if formatCache == nil { 219 continue 220 } 221 if err := checkFormatCacheValue(formatCache, migrating); err != nil { 222 return i, err 223 } 224 if len(formats) != len(formatCache.Cache.Disks) { 225 return i, fmt.Errorf("Expected number of cache drives %d , got %d", 226 len(formatCache.Cache.Disks), len(formats)) 227 } 228 } 229 return -1, nil 230 } 231 232 // checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices 233 // across disks. 234 func checkCacheDiskConsistency(formats []*formatCacheV2) error { 235 var disks = make([]string, len(formats)) 236 // Collect currently available disk uuids. 237 for index, format := range formats { 238 if format == nil { 239 disks[index] = "" 240 continue 241 } 242 disks[index] = format.Cache.This 243 } 244 for i, format := range formats { 245 if format == nil { 246 continue 247 } 248 j := findCacheDiskIndex(disks[i], format.Cache.Disks) 249 if j == -1 { 250 return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s", i, j, disks[i]) 251 } 252 if i != j { 253 return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s got %s", i, j, disks[i], format.Cache.Disks[j]) 254 } 255 } 256 return nil 257 } 258 259 // checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent. 260 func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error { 261 var sentinelDisks []string 262 // Extract first valid Disks slice. 263 for _, format := range formats { 264 if format == nil { 265 continue 266 } 267 sentinelDisks = format.Cache.Disks 268 break 269 } 270 for _, format := range formats { 271 if format == nil { 272 continue 273 } 274 currentDisks := format.Cache.Disks 275 if !reflect.DeepEqual(sentinelDisks, currentDisks) { 276 return errors.New("inconsistent cache drives found") 277 } 278 } 279 return nil 280 } 281 282 // findCacheDiskIndex returns position of cache disk in JBOD. 283 func findCacheDiskIndex(disk string, disks []string) int { 284 for index, uuid := range disks { 285 if uuid == disk { 286 return index 287 } 288 } 289 return -1 290 } 291 292 // validate whether cache drives order has changed 293 func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error { 294 count := 0 295 for _, format := range formats { 296 if format == nil { 297 count++ 298 } 299 } 300 if count == len(formats) { 301 return errors.New("Cache format files missing on all drives") 302 } 303 if _, err := checkFormatCacheValues(migrating, formats); err != nil { 304 logger.LogIf(ctx, err) 305 return err 306 } 307 if err := checkCacheDisksSliceConsistency(formats); err != nil { 308 logger.LogIf(ctx, err) 309 return err 310 } 311 err := checkCacheDiskConsistency(formats) 312 logger.LogIf(ctx, err) 313 return err 314 } 315 316 // return true if all of the list of cache drives are 317 // fresh disks 318 func cacheDrivesUnformatted(drives []string) bool { 319 count := 0 320 for _, drive := range drives { 321 cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile) 322 if _, err := os.Stat(cacheFormatPath); osIsNotExist(err) { 323 count++ 324 } 325 } 326 return count == len(drives) 327 } 328 329 // create format.json for each cache drive if fresh disk or load format from disk 330 // Then validate the format for all drives in the cache to ensure order 331 // of cache drives has not changed. 332 func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) { 333 if cacheDrivesUnformatted(drives) { 334 formats, err = initFormatCache(ctx, drives) 335 } else { 336 formats, migrating, err = loadFormatCache(ctx, drives) 337 } 338 if err != nil { 339 return nil, false, err 340 } 341 if err = validateCacheFormats(ctx, migrating, formats); err != nil { 342 return nil, false, err 343 } 344 return formats, migrating, nil 345 } 346 347 // reads cached object on disk and writes it back after adding bitrot 348 // hashsum per block as per the new disk cache format. 349 func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error { 350 st, err := os.Stat(oldfile) 351 if err != nil { 352 err = osErrToFileErr(err) 353 return err 354 } 355 readCloser, err := readCacheFileStream(oldfile, 0, st.Size()) 356 if err != nil { 357 return err 358 } 359 var reader io.Reader = readCloser 360 361 actualSize := uint64(st.Size()) 362 if globalCacheKMS != nil { 363 reader, err = newCacheEncryptReader(readCloser, bucket, object, metadata) 364 if err != nil { 365 return err 366 } 367 actualSize, _ = sio.EncryptedSize(uint64(st.Size())) 368 } 369 _, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize) 370 return err 371 } 372 373 // migrate cache contents from old cacheFS format to new backend format 374 // new format is flat 375 // sha(bucket,object)/ <== dir name 376 // - part.1 <== data 377 // - cache.json <== metadata 378 func migrateOldCache(ctx context.Context, c *diskCache) error { 379 oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets") 380 cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile) 381 382 if _, err := os.Stat(oldCacheBucketsPath); err != nil { 383 // remove .minio.sys sub directories 384 removeAll(path.Join(c.dir, minioMetaBucket, "multipart")) 385 removeAll(path.Join(c.dir, minioMetaBucket, "tmp")) 386 removeAll(path.Join(c.dir, minioMetaBucket, "trash")) 387 removeAll(path.Join(c.dir, minioMetaBucket, "buckets")) 388 // just migrate cache format 389 return migrateCacheFormatJSON(cacheFormatPath) 390 } 391 392 buckets, err := readDir(oldCacheBucketsPath) 393 if err != nil { 394 return err 395 } 396 397 for _, bucket := range buckets { 398 bucket = strings.TrimSuffix(bucket, SlashSeparator) 399 var objMetaPaths []string 400 root := path.Join(oldCacheBucketsPath, bucket) 401 err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 402 if strings.HasSuffix(path, cacheMetaJSONFile) { 403 objMetaPaths = append(objMetaPaths, path) 404 } 405 return nil 406 }) 407 if err != nil { 408 return err 409 } 410 for _, oMeta := range objMetaPaths { 411 objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2) 412 object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket)) 413 object = strings.TrimSuffix(object, "/") 414 415 destdir := getCacheSHADir(c.dir, bucket, object) 416 if err := os.MkdirAll(destdir, 0777); err != nil { 417 return err 418 } 419 prevCachedPath := path.Join(c.dir, bucket, object) 420 421 // get old cached metadata 422 oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile) 423 metaPath := pathJoin(destdir, cacheMetaJSONFile) 424 metaBytes, err := ioutil.ReadFile(oldMetaPath) 425 if err != nil { 426 return err 427 } 428 // marshal cache metadata after adding version and stat info 429 meta := &cacheMeta{} 430 var json = jsoniter.ConfigCompatibleWithStandardLibrary 431 if err = json.Unmarshal(metaBytes, &meta); err != nil { 432 return err 433 } 434 // move cached object to new cache directory path 435 // migrate cache data and add bit-rot protection hash sum 436 // at the start of each block 437 if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil { 438 continue 439 } 440 stat, err := os.Stat(prevCachedPath) 441 if err != nil { 442 if err == errFileNotFound { 443 continue 444 } 445 logger.LogIf(ctx, err) 446 return err 447 } 448 // old cached file can now be removed 449 if err := os.Remove(prevCachedPath); err != nil { 450 return err 451 } 452 // move cached metadata after changing cache metadata version 453 meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize} 454 meta.Version = cacheMetaVersion 455 meta.Stat.Size = stat.Size() 456 meta.Stat.ModTime = stat.ModTime() 457 jsonData, err := json.Marshal(meta) 458 if err != nil { 459 return err 460 } 461 462 if err = ioutil.WriteFile(metaPath, jsonData, 0644); err != nil { 463 return err 464 } 465 } 466 467 // delete old bucket from cache, now that all contents are cleared 468 removeAll(path.Join(c.dir, bucket)) 469 } 470 471 // remove .minio.sys sub directories 472 removeAll(path.Join(c.dir, minioMetaBucket, "multipart")) 473 removeAll(path.Join(c.dir, minioMetaBucket, "tmp")) 474 removeAll(path.Join(c.dir, minioMetaBucket, "trash")) 475 removeAll(path.Join(c.dir, minioMetaBucket, "buckets")) 476 477 return migrateCacheFormatJSON(cacheFormatPath) 478 479 } 480 481 func migrateCacheFormatJSON(cacheFormatPath string) error { 482 // now migrate format.json 483 f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0) 484 if err != nil { 485 return err 486 } 487 defer f.Close() 488 formatV1 := formatCacheV1{} 489 if err := jsonLoad(f, &formatV1); err != nil { 490 return err 491 } 492 493 formatV2 := &formatCacheV2{} 494 formatV2.formatMetaV1 = formatV1.formatMetaV1 495 formatV2.Version = formatMetaVersion1 496 formatV2.Cache = formatV1.Cache 497 formatV2.Cache.Version = formatCacheVersionV2 498 if err := jsonSave(f, formatV2); err != nil { 499 return err 500 } 501 return nil 502 }