storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/xl-storage-disk-id-check.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "io" 22 "strings" 23 "sync" 24 "sync/atomic" 25 "time" 26 27 ewma "github.com/VividCortex/ewma" 28 29 trace "storj.io/minio/pkg/trace" 30 ) 31 32 //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE 33 34 type storageMetric uint8 35 36 const ( 37 storageMetricMakeVolBulk storageMetric = iota 38 storageMetricMakeVol 39 storageMetricListVols 40 storageMetricStatVol 41 storageMetricDeleteVol 42 storageMetricWalkDir 43 storageMetricListDir 44 storageMetricReadFile 45 storageMetricAppendFile 46 storageMetricCreateFile 47 storageMetricReadFileStream 48 storageMetricRenameFile 49 storageMetricRenameData 50 storageMetricCheckParts 51 storageMetricCheckFile 52 storageMetricDelete 53 storageMetricDeleteVersions 54 storageMetricVerifyFile 55 storageMetricWriteAll 56 storageMetricDeleteVersion 57 storageMetricWriteMetadata 58 storageMetricUpdateMetadata 59 storageMetricReadVersion 60 storageMetricReadAll 61 62 // .... add more 63 64 storageMetricLast 65 ) 66 67 // Detects change in underlying disk. 68 type xlStorageDiskIDCheck struct { 69 // fields position optimized for memory please 70 // do not re-order them, if you add new fields 71 // please use `fieldalignment ./...` to check 72 // if your changes are not causing any problems. 73 storage StorageAPI 74 apiLatencies [storageMetricLast]ewma.MovingAverage 75 diskID string 76 apiCalls [storageMetricLast]uint64 77 } 78 79 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics { 80 diskMetric := DiskMetrics{ 81 APILatencies: make(map[string]string), 82 APICalls: make(map[string]uint64), 83 } 84 for i, v := range p.apiLatencies { 85 diskMetric.APILatencies[storageMetric(i).String()] = time.Duration(v.Value()).String() 86 } 87 for i := range p.apiCalls { 88 diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i]) 89 } 90 return diskMetric 91 } 92 93 type lockedSimpleEWMA struct { 94 sync.RWMutex 95 *ewma.SimpleEWMA 96 } 97 98 func (e *lockedSimpleEWMA) Add(value float64) { 99 e.Lock() 100 defer e.Unlock() 101 e.SimpleEWMA.Add(value) 102 } 103 104 func (e *lockedSimpleEWMA) Set(value float64) { 105 e.Lock() 106 defer e.Unlock() 107 108 e.SimpleEWMA.Set(value) 109 } 110 111 func (e *lockedSimpleEWMA) Value() float64 { 112 e.RLock() 113 defer e.RUnlock() 114 return e.SimpleEWMA.Value() 115 } 116 117 func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck { 118 xl := xlStorageDiskIDCheck{ 119 storage: storage, 120 } 121 for i := range xl.apiLatencies[:] { 122 xl.apiLatencies[i] = &lockedSimpleEWMA{ 123 SimpleEWMA: new(ewma.SimpleEWMA), 124 } 125 } 126 return &xl 127 } 128 129 func (p *xlStorageDiskIDCheck) String() string { 130 return p.storage.String() 131 } 132 133 func (p *xlStorageDiskIDCheck) IsOnline() bool { 134 storedDiskID, err := p.storage.GetDiskID() 135 if err != nil { 136 return false 137 } 138 return storedDiskID == p.diskID 139 } 140 141 func (p *xlStorageDiskIDCheck) IsLocal() bool { 142 return p.storage.IsLocal() 143 } 144 145 func (p *xlStorageDiskIDCheck) Endpoint() Endpoint { 146 return p.storage.Endpoint() 147 } 148 149 func (p *xlStorageDiskIDCheck) Hostname() string { 150 return p.storage.Hostname() 151 } 152 153 func (p *xlStorageDiskIDCheck) Healing() *healingTracker { 154 return p.storage.Healing() 155 } 156 157 func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) { 158 select { 159 case <-ctx.Done(): 160 return dataUsageCache{}, ctx.Err() 161 default: 162 } 163 164 if err := p.checkDiskStale(); err != nil { 165 return dataUsageCache{}, err 166 } 167 return p.storage.NSScanner(ctx, cache) 168 } 169 170 func (p *xlStorageDiskIDCheck) GetDiskLoc() (poolIdx, setIdx, diskIdx int) { 171 return p.storage.GetDiskLoc() 172 } 173 174 func (p *xlStorageDiskIDCheck) SetDiskLoc(poolIdx, setIdx, diskIdx int) { 175 p.storage.SetDiskLoc(poolIdx, setIdx, diskIdx) 176 } 177 178 func (p *xlStorageDiskIDCheck) Close() error { 179 return p.storage.Close() 180 } 181 182 func (p *xlStorageDiskIDCheck) GetDiskID() (string, error) { 183 return p.storage.GetDiskID() 184 } 185 186 func (p *xlStorageDiskIDCheck) SetDiskID(id string) { 187 p.diskID = id 188 } 189 190 func (p *xlStorageDiskIDCheck) checkDiskStale() error { 191 if p.diskID == "" { 192 // For empty disk-id we allow the call as the server might be 193 // coming up and trying to read format.json or create format.json 194 return nil 195 } 196 storedDiskID, err := p.storage.GetDiskID() 197 if err != nil { 198 // return any error generated while reading `format.json` 199 return err 200 } 201 if err == nil && p.diskID == storedDiskID { 202 return nil 203 } 204 // not the same disk we remember, take it offline. 205 return errDiskNotFound 206 } 207 208 func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) { 209 select { 210 case <-ctx.Done(): 211 return DiskInfo{}, ctx.Err() 212 default: 213 } 214 215 info, err = p.storage.DiskInfo(ctx) 216 if err != nil { 217 return info, err 218 } 219 220 info.Metrics = p.getMetrics() 221 // check cached diskID against backend 222 // only if its non-empty. 223 if p.diskID != "" { 224 if p.diskID != info.ID { 225 return info, errDiskNotFound 226 } 227 } 228 return info, nil 229 } 230 231 func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { 232 defer p.updateStorageMetrics(storageMetricMakeVolBulk, volumes...)() 233 234 select { 235 case <-ctx.Done(): 236 return ctx.Err() 237 default: 238 } 239 240 if err = p.checkDiskStale(); err != nil { 241 return err 242 } 243 return p.storage.MakeVolBulk(ctx, volumes...) 244 } 245 246 func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) { 247 defer p.updateStorageMetrics(storageMetricMakeVol, volume)() 248 249 select { 250 case <-ctx.Done(): 251 return ctx.Err() 252 default: 253 } 254 255 if err = p.checkDiskStale(); err != nil { 256 return err 257 } 258 return p.storage.MakeVol(ctx, volume) 259 } 260 261 func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) { 262 defer p.updateStorageMetrics(storageMetricListVols, "/")() 263 264 select { 265 case <-ctx.Done(): 266 return nil, ctx.Err() 267 default: 268 } 269 270 if err := p.checkDiskStale(); err != nil { 271 return nil, err 272 } 273 return p.storage.ListVols(ctx) 274 } 275 276 func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { 277 defer p.updateStorageMetrics(storageMetricStatVol, volume)() 278 279 select { 280 case <-ctx.Done(): 281 return VolInfo{}, ctx.Err() 282 default: 283 } 284 285 if err = p.checkDiskStale(); err != nil { 286 return vol, err 287 } 288 return p.storage.StatVol(ctx, volume) 289 } 290 291 func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { 292 defer p.updateStorageMetrics(storageMetricDeleteVol, volume)() 293 294 select { 295 case <-ctx.Done(): 296 return ctx.Err() 297 default: 298 } 299 300 if err = p.checkDiskStale(); err != nil { 301 return err 302 } 303 return p.storage.DeleteVol(ctx, volume, forceDelete) 304 } 305 306 func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) { 307 defer p.updateStorageMetrics(storageMetricListDir, volume, dirPath)() 308 309 select { 310 case <-ctx.Done(): 311 return nil, ctx.Err() 312 default: 313 } 314 315 if err := p.checkDiskStale(); err != nil { 316 return nil, err 317 } 318 319 return p.storage.ListDir(ctx, volume, dirPath, count) 320 } 321 322 func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { 323 defer p.updateStorageMetrics(storageMetricReadFile, volume, path)() 324 325 select { 326 case <-ctx.Done(): 327 return 0, ctx.Err() 328 default: 329 } 330 331 if err := p.checkDiskStale(); err != nil { 332 return 0, err 333 } 334 335 return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier) 336 } 337 338 func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) { 339 defer p.updateStorageMetrics(storageMetricAppendFile, volume, path)() 340 341 select { 342 case <-ctx.Done(): 343 return ctx.Err() 344 default: 345 } 346 347 if err = p.checkDiskStale(); err != nil { 348 return err 349 } 350 351 return p.storage.AppendFile(ctx, volume, path, buf) 352 } 353 354 func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error { 355 defer p.updateStorageMetrics(storageMetricCreateFile, volume, path)() 356 357 select { 358 case <-ctx.Done(): 359 return ctx.Err() 360 default: 361 } 362 363 if err := p.checkDiskStale(); err != nil { 364 return err 365 } 366 367 return p.storage.CreateFile(ctx, volume, path, size, reader) 368 } 369 370 func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { 371 defer p.updateStorageMetrics(storageMetricReadFileStream, volume, path)() 372 373 select { 374 case <-ctx.Done(): 375 return nil, ctx.Err() 376 default: 377 } 378 379 if err := p.checkDiskStale(); err != nil { 380 return nil, err 381 } 382 383 return p.storage.ReadFileStream(ctx, volume, path, offset, length) 384 } 385 386 func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error { 387 defer p.updateStorageMetrics(storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)() 388 389 select { 390 case <-ctx.Done(): 391 return ctx.Err() 392 default: 393 } 394 395 if err := p.checkDiskStale(); err != nil { 396 return err 397 } 398 399 return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) 400 } 401 402 func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) error { 403 defer p.updateStorageMetrics(storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)() 404 405 select { 406 case <-ctx.Done(): 407 return ctx.Err() 408 default: 409 } 410 411 if err := p.checkDiskStale(); err != nil { 412 return err 413 } 414 415 return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath) 416 } 417 418 func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) { 419 defer p.updateStorageMetrics(storageMetricCheckParts, volume, path)() 420 421 select { 422 case <-ctx.Done(): 423 return ctx.Err() 424 default: 425 } 426 427 if err = p.checkDiskStale(); err != nil { 428 return err 429 } 430 431 return p.storage.CheckParts(ctx, volume, path, fi) 432 } 433 434 func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) { 435 defer p.updateStorageMetrics(storageMetricCheckFile, volume, path)() 436 437 select { 438 case <-ctx.Done(): 439 return ctx.Err() 440 default: 441 } 442 443 if err = p.checkDiskStale(); err != nil { 444 return err 445 } 446 447 return p.storage.CheckFile(ctx, volume, path) 448 } 449 450 func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) { 451 defer p.updateStorageMetrics(storageMetricDelete, volume, path)() 452 453 select { 454 case <-ctx.Done(): 455 return ctx.Err() 456 default: 457 } 458 459 if err = p.checkDiskStale(); err != nil { 460 return err 461 } 462 463 return p.storage.Delete(ctx, volume, path, recursive) 464 } 465 466 // DeleteVersions deletes slice of versions, it can be same object 467 // or multiple objects. 468 func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) { 469 // Mererly for tracing storage 470 path := "" 471 if len(versions) > 0 { 472 path = versions[0].Name 473 } 474 475 defer p.updateStorageMetrics(storageMetricDeleteVersions, volume, path)() 476 477 errs = make([]error, len(versions)) 478 479 select { 480 case <-ctx.Done(): 481 for i := range errs { 482 errs[i] = ctx.Err() 483 } 484 return errs 485 default: 486 } 487 488 if err := p.checkDiskStale(); err != nil { 489 for i := range errs { 490 errs[i] = err 491 } 492 return errs 493 } 494 495 return p.storage.DeleteVersions(ctx, volume, versions) 496 } 497 498 func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { 499 defer p.updateStorageMetrics(storageMetricVerifyFile, volume, path)() 500 501 select { 502 case <-ctx.Done(): 503 return ctx.Err() 504 default: 505 } 506 507 if err := p.checkDiskStale(); err != nil { 508 return err 509 } 510 511 return p.storage.VerifyFile(ctx, volume, path, fi) 512 } 513 514 func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) { 515 defer p.updateStorageMetrics(storageMetricWriteAll, volume, path)() 516 517 select { 518 case <-ctx.Done(): 519 return ctx.Err() 520 default: 521 } 522 523 if err = p.checkDiskStale(); err != nil { 524 return err 525 } 526 527 return p.storage.WriteAll(ctx, volume, path, b) 528 } 529 530 func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) { 531 defer p.updateStorageMetrics(storageMetricDeleteVersion, volume, path)() 532 533 select { 534 case <-ctx.Done(): 535 return ctx.Err() 536 default: 537 } 538 539 if err = p.checkDiskStale(); err != nil { 540 return err 541 } 542 543 return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker) 544 } 545 546 func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) { 547 defer p.updateStorageMetrics(storageMetricUpdateMetadata, volume, path)() 548 549 select { 550 case <-ctx.Done(): 551 return ctx.Err() 552 default: 553 } 554 555 if err = p.checkDiskStale(); err != nil { 556 return err 557 } 558 559 return p.storage.UpdateMetadata(ctx, volume, path, fi) 560 } 561 562 func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) { 563 defer p.updateStorageMetrics(storageMetricWriteMetadata, volume, path)() 564 565 select { 566 case <-ctx.Done(): 567 return ctx.Err() 568 default: 569 } 570 571 if err = p.checkDiskStale(); err != nil { 572 return err 573 } 574 575 return p.storage.WriteMetadata(ctx, volume, path, fi) 576 } 577 578 func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) { 579 defer p.updateStorageMetrics(storageMetricReadVersion, volume, path)() 580 581 select { 582 case <-ctx.Done(): 583 return fi, ctx.Err() 584 default: 585 } 586 587 if err = p.checkDiskStale(); err != nil { 588 return fi, err 589 } 590 591 return p.storage.ReadVersion(ctx, volume, path, versionID, readData) 592 } 593 594 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { 595 defer p.updateStorageMetrics(storageMetricReadAll, volume, path)() 596 597 select { 598 case <-ctx.Done(): 599 return nil, ctx.Err() 600 default: 601 } 602 603 if err = p.checkDiskStale(); err != nil { 604 return nil, err 605 } 606 607 return p.storage.ReadAll(ctx, volume, path) 608 } 609 610 func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) trace.Info { 611 return trace.Info{ 612 TraceType: trace.Storage, 613 Time: startTime, 614 NodeName: globalLocalNodeName, 615 FuncName: "storage." + s.String(), 616 StorageStats: trace.StorageStats{ 617 Duration: duration, 618 Path: path, 619 }, 620 } 621 } 622 623 // Update storage metrics 624 func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func() { 625 startTime := time.Now() 626 trace := globalTrace.NumSubscribers() > 0 627 return func() { 628 duration := time.Since(startTime) 629 630 atomic.AddUint64(&p.apiCalls[s], 1) 631 p.apiLatencies[s].Add(float64(duration)) 632 633 if trace { 634 globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "))) 635 } 636 } 637 }