storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/xl-storage-disk-id-check.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  	"strings"
    23  	"sync"
    24  	"sync/atomic"
    25  	"time"
    26  
    27  	ewma "github.com/VividCortex/ewma"
    28  
    29  	trace "storj.io/minio/pkg/trace"
    30  )
    31  
    32  //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE
    33  
    34  type storageMetric uint8
    35  
    36  const (
    37  	storageMetricMakeVolBulk storageMetric = iota
    38  	storageMetricMakeVol
    39  	storageMetricListVols
    40  	storageMetricStatVol
    41  	storageMetricDeleteVol
    42  	storageMetricWalkDir
    43  	storageMetricListDir
    44  	storageMetricReadFile
    45  	storageMetricAppendFile
    46  	storageMetricCreateFile
    47  	storageMetricReadFileStream
    48  	storageMetricRenameFile
    49  	storageMetricRenameData
    50  	storageMetricCheckParts
    51  	storageMetricCheckFile
    52  	storageMetricDelete
    53  	storageMetricDeleteVersions
    54  	storageMetricVerifyFile
    55  	storageMetricWriteAll
    56  	storageMetricDeleteVersion
    57  	storageMetricWriteMetadata
    58  	storageMetricUpdateMetadata
    59  	storageMetricReadVersion
    60  	storageMetricReadAll
    61  
    62  	// .... add more
    63  
    64  	storageMetricLast
    65  )
    66  
    67  // Detects change in underlying disk.
    68  type xlStorageDiskIDCheck struct {
    69  	// fields position optimized for memory please
    70  	// do not re-order them, if you add new fields
    71  	// please use `fieldalignment ./...` to check
    72  	// if your changes are not causing any problems.
    73  	storage      StorageAPI
    74  	apiLatencies [storageMetricLast]ewma.MovingAverage
    75  	diskID       string
    76  	apiCalls     [storageMetricLast]uint64
    77  }
    78  
    79  func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
    80  	diskMetric := DiskMetrics{
    81  		APILatencies: make(map[string]string),
    82  		APICalls:     make(map[string]uint64),
    83  	}
    84  	for i, v := range p.apiLatencies {
    85  		diskMetric.APILatencies[storageMetric(i).String()] = time.Duration(v.Value()).String()
    86  	}
    87  	for i := range p.apiCalls {
    88  		diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i])
    89  	}
    90  	return diskMetric
    91  }
    92  
    93  type lockedSimpleEWMA struct {
    94  	sync.RWMutex
    95  	*ewma.SimpleEWMA
    96  }
    97  
    98  func (e *lockedSimpleEWMA) Add(value float64) {
    99  	e.Lock()
   100  	defer e.Unlock()
   101  	e.SimpleEWMA.Add(value)
   102  }
   103  
   104  func (e *lockedSimpleEWMA) Set(value float64) {
   105  	e.Lock()
   106  	defer e.Unlock()
   107  
   108  	e.SimpleEWMA.Set(value)
   109  }
   110  
   111  func (e *lockedSimpleEWMA) Value() float64 {
   112  	e.RLock()
   113  	defer e.RUnlock()
   114  	return e.SimpleEWMA.Value()
   115  }
   116  
   117  func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck {
   118  	xl := xlStorageDiskIDCheck{
   119  		storage: storage,
   120  	}
   121  	for i := range xl.apiLatencies[:] {
   122  		xl.apiLatencies[i] = &lockedSimpleEWMA{
   123  			SimpleEWMA: new(ewma.SimpleEWMA),
   124  		}
   125  	}
   126  	return &xl
   127  }
   128  
   129  func (p *xlStorageDiskIDCheck) String() string {
   130  	return p.storage.String()
   131  }
   132  
   133  func (p *xlStorageDiskIDCheck) IsOnline() bool {
   134  	storedDiskID, err := p.storage.GetDiskID()
   135  	if err != nil {
   136  		return false
   137  	}
   138  	return storedDiskID == p.diskID
   139  }
   140  
   141  func (p *xlStorageDiskIDCheck) IsLocal() bool {
   142  	return p.storage.IsLocal()
   143  }
   144  
   145  func (p *xlStorageDiskIDCheck) Endpoint() Endpoint {
   146  	return p.storage.Endpoint()
   147  }
   148  
   149  func (p *xlStorageDiskIDCheck) Hostname() string {
   150  	return p.storage.Hostname()
   151  }
   152  
   153  func (p *xlStorageDiskIDCheck) Healing() *healingTracker {
   154  	return p.storage.Healing()
   155  }
   156  
   157  func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
   158  	select {
   159  	case <-ctx.Done():
   160  		return dataUsageCache{}, ctx.Err()
   161  	default:
   162  	}
   163  
   164  	if err := p.checkDiskStale(); err != nil {
   165  		return dataUsageCache{}, err
   166  	}
   167  	return p.storage.NSScanner(ctx, cache)
   168  }
   169  
   170  func (p *xlStorageDiskIDCheck) GetDiskLoc() (poolIdx, setIdx, diskIdx int) {
   171  	return p.storage.GetDiskLoc()
   172  }
   173  
   174  func (p *xlStorageDiskIDCheck) SetDiskLoc(poolIdx, setIdx, diskIdx int) {
   175  	p.storage.SetDiskLoc(poolIdx, setIdx, diskIdx)
   176  }
   177  
   178  func (p *xlStorageDiskIDCheck) Close() error {
   179  	return p.storage.Close()
   180  }
   181  
   182  func (p *xlStorageDiskIDCheck) GetDiskID() (string, error) {
   183  	return p.storage.GetDiskID()
   184  }
   185  
   186  func (p *xlStorageDiskIDCheck) SetDiskID(id string) {
   187  	p.diskID = id
   188  }
   189  
   190  func (p *xlStorageDiskIDCheck) checkDiskStale() error {
   191  	if p.diskID == "" {
   192  		// For empty disk-id we allow the call as the server might be
   193  		// coming up and trying to read format.json or create format.json
   194  		return nil
   195  	}
   196  	storedDiskID, err := p.storage.GetDiskID()
   197  	if err != nil {
   198  		// return any error generated while reading `format.json`
   199  		return err
   200  	}
   201  	if err == nil && p.diskID == storedDiskID {
   202  		return nil
   203  	}
   204  	// not the same disk we remember, take it offline.
   205  	return errDiskNotFound
   206  }
   207  
   208  func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
   209  	select {
   210  	case <-ctx.Done():
   211  		return DiskInfo{}, ctx.Err()
   212  	default:
   213  	}
   214  
   215  	info, err = p.storage.DiskInfo(ctx)
   216  	if err != nil {
   217  		return info, err
   218  	}
   219  
   220  	info.Metrics = p.getMetrics()
   221  	// check cached diskID against backend
   222  	// only if its non-empty.
   223  	if p.diskID != "" {
   224  		if p.diskID != info.ID {
   225  			return info, errDiskNotFound
   226  		}
   227  	}
   228  	return info, nil
   229  }
   230  
   231  func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
   232  	defer p.updateStorageMetrics(storageMetricMakeVolBulk, volumes...)()
   233  
   234  	select {
   235  	case <-ctx.Done():
   236  		return ctx.Err()
   237  	default:
   238  	}
   239  
   240  	if err = p.checkDiskStale(); err != nil {
   241  		return err
   242  	}
   243  	return p.storage.MakeVolBulk(ctx, volumes...)
   244  }
   245  
   246  func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
   247  	defer p.updateStorageMetrics(storageMetricMakeVol, volume)()
   248  
   249  	select {
   250  	case <-ctx.Done():
   251  		return ctx.Err()
   252  	default:
   253  	}
   254  
   255  	if err = p.checkDiskStale(); err != nil {
   256  		return err
   257  	}
   258  	return p.storage.MakeVol(ctx, volume)
   259  }
   260  
   261  func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
   262  	defer p.updateStorageMetrics(storageMetricListVols, "/")()
   263  
   264  	select {
   265  	case <-ctx.Done():
   266  		return nil, ctx.Err()
   267  	default:
   268  	}
   269  
   270  	if err := p.checkDiskStale(); err != nil {
   271  		return nil, err
   272  	}
   273  	return p.storage.ListVols(ctx)
   274  }
   275  
   276  func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
   277  	defer p.updateStorageMetrics(storageMetricStatVol, volume)()
   278  
   279  	select {
   280  	case <-ctx.Done():
   281  		return VolInfo{}, ctx.Err()
   282  	default:
   283  	}
   284  
   285  	if err = p.checkDiskStale(); err != nil {
   286  		return vol, err
   287  	}
   288  	return p.storage.StatVol(ctx, volume)
   289  }
   290  
   291  func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
   292  	defer p.updateStorageMetrics(storageMetricDeleteVol, volume)()
   293  
   294  	select {
   295  	case <-ctx.Done():
   296  		return ctx.Err()
   297  	default:
   298  	}
   299  
   300  	if err = p.checkDiskStale(); err != nil {
   301  		return err
   302  	}
   303  	return p.storage.DeleteVol(ctx, volume, forceDelete)
   304  }
   305  
   306  func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
   307  	defer p.updateStorageMetrics(storageMetricListDir, volume, dirPath)()
   308  
   309  	select {
   310  	case <-ctx.Done():
   311  		return nil, ctx.Err()
   312  	default:
   313  	}
   314  
   315  	if err := p.checkDiskStale(); err != nil {
   316  		return nil, err
   317  	}
   318  
   319  	return p.storage.ListDir(ctx, volume, dirPath, count)
   320  }
   321  
   322  func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
   323  	defer p.updateStorageMetrics(storageMetricReadFile, volume, path)()
   324  
   325  	select {
   326  	case <-ctx.Done():
   327  		return 0, ctx.Err()
   328  	default:
   329  	}
   330  
   331  	if err := p.checkDiskStale(); err != nil {
   332  		return 0, err
   333  	}
   334  
   335  	return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
   336  }
   337  
   338  func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
   339  	defer p.updateStorageMetrics(storageMetricAppendFile, volume, path)()
   340  
   341  	select {
   342  	case <-ctx.Done():
   343  		return ctx.Err()
   344  	default:
   345  	}
   346  
   347  	if err = p.checkDiskStale(); err != nil {
   348  		return err
   349  	}
   350  
   351  	return p.storage.AppendFile(ctx, volume, path, buf)
   352  }
   353  
   354  func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
   355  	defer p.updateStorageMetrics(storageMetricCreateFile, volume, path)()
   356  
   357  	select {
   358  	case <-ctx.Done():
   359  		return ctx.Err()
   360  	default:
   361  	}
   362  
   363  	if err := p.checkDiskStale(); err != nil {
   364  		return err
   365  	}
   366  
   367  	return p.storage.CreateFile(ctx, volume, path, size, reader)
   368  }
   369  
   370  func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
   371  	defer p.updateStorageMetrics(storageMetricReadFileStream, volume, path)()
   372  
   373  	select {
   374  	case <-ctx.Done():
   375  		return nil, ctx.Err()
   376  	default:
   377  	}
   378  
   379  	if err := p.checkDiskStale(); err != nil {
   380  		return nil, err
   381  	}
   382  
   383  	return p.storage.ReadFileStream(ctx, volume, path, offset, length)
   384  }
   385  
   386  func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
   387  	defer p.updateStorageMetrics(storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)()
   388  
   389  	select {
   390  	case <-ctx.Done():
   391  		return ctx.Err()
   392  	default:
   393  	}
   394  
   395  	if err := p.checkDiskStale(); err != nil {
   396  		return err
   397  	}
   398  
   399  	return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
   400  }
   401  
   402  func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) error {
   403  	defer p.updateStorageMetrics(storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)()
   404  
   405  	select {
   406  	case <-ctx.Done():
   407  		return ctx.Err()
   408  	default:
   409  	}
   410  
   411  	if err := p.checkDiskStale(); err != nil {
   412  		return err
   413  	}
   414  
   415  	return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
   416  }
   417  
   418  func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
   419  	defer p.updateStorageMetrics(storageMetricCheckParts, volume, path)()
   420  
   421  	select {
   422  	case <-ctx.Done():
   423  		return ctx.Err()
   424  	default:
   425  	}
   426  
   427  	if err = p.checkDiskStale(); err != nil {
   428  		return err
   429  	}
   430  
   431  	return p.storage.CheckParts(ctx, volume, path, fi)
   432  }
   433  
   434  func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) {
   435  	defer p.updateStorageMetrics(storageMetricCheckFile, volume, path)()
   436  
   437  	select {
   438  	case <-ctx.Done():
   439  		return ctx.Err()
   440  	default:
   441  	}
   442  
   443  	if err = p.checkDiskStale(); err != nil {
   444  		return err
   445  	}
   446  
   447  	return p.storage.CheckFile(ctx, volume, path)
   448  }
   449  
   450  func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
   451  	defer p.updateStorageMetrics(storageMetricDelete, volume, path)()
   452  
   453  	select {
   454  	case <-ctx.Done():
   455  		return ctx.Err()
   456  	default:
   457  	}
   458  
   459  	if err = p.checkDiskStale(); err != nil {
   460  		return err
   461  	}
   462  
   463  	return p.storage.Delete(ctx, volume, path, recursive)
   464  }
   465  
   466  // DeleteVersions deletes slice of versions, it can be same object
   467  // or multiple objects.
   468  func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
   469  	// Mererly for tracing storage
   470  	path := ""
   471  	if len(versions) > 0 {
   472  		path = versions[0].Name
   473  	}
   474  
   475  	defer p.updateStorageMetrics(storageMetricDeleteVersions, volume, path)()
   476  
   477  	errs = make([]error, len(versions))
   478  
   479  	select {
   480  	case <-ctx.Done():
   481  		for i := range errs {
   482  			errs[i] = ctx.Err()
   483  		}
   484  		return errs
   485  	default:
   486  	}
   487  
   488  	if err := p.checkDiskStale(); err != nil {
   489  		for i := range errs {
   490  			errs[i] = err
   491  		}
   492  		return errs
   493  	}
   494  
   495  	return p.storage.DeleteVersions(ctx, volume, versions)
   496  }
   497  
   498  func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
   499  	defer p.updateStorageMetrics(storageMetricVerifyFile, volume, path)()
   500  
   501  	select {
   502  	case <-ctx.Done():
   503  		return ctx.Err()
   504  	default:
   505  	}
   506  
   507  	if err := p.checkDiskStale(); err != nil {
   508  		return err
   509  	}
   510  
   511  	return p.storage.VerifyFile(ctx, volume, path, fi)
   512  }
   513  
   514  func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
   515  	defer p.updateStorageMetrics(storageMetricWriteAll, volume, path)()
   516  
   517  	select {
   518  	case <-ctx.Done():
   519  		return ctx.Err()
   520  	default:
   521  	}
   522  
   523  	if err = p.checkDiskStale(); err != nil {
   524  		return err
   525  	}
   526  
   527  	return p.storage.WriteAll(ctx, volume, path, b)
   528  }
   529  
   530  func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
   531  	defer p.updateStorageMetrics(storageMetricDeleteVersion, volume, path)()
   532  
   533  	select {
   534  	case <-ctx.Done():
   535  		return ctx.Err()
   536  	default:
   537  	}
   538  
   539  	if err = p.checkDiskStale(); err != nil {
   540  		return err
   541  	}
   542  
   543  	return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
   544  }
   545  
   546  func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
   547  	defer p.updateStorageMetrics(storageMetricUpdateMetadata, volume, path)()
   548  
   549  	select {
   550  	case <-ctx.Done():
   551  		return ctx.Err()
   552  	default:
   553  	}
   554  
   555  	if err = p.checkDiskStale(); err != nil {
   556  		return err
   557  	}
   558  
   559  	return p.storage.UpdateMetadata(ctx, volume, path, fi)
   560  }
   561  
   562  func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
   563  	defer p.updateStorageMetrics(storageMetricWriteMetadata, volume, path)()
   564  
   565  	select {
   566  	case <-ctx.Done():
   567  		return ctx.Err()
   568  	default:
   569  	}
   570  
   571  	if err = p.checkDiskStale(); err != nil {
   572  		return err
   573  	}
   574  
   575  	return p.storage.WriteMetadata(ctx, volume, path, fi)
   576  }
   577  
   578  func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
   579  	defer p.updateStorageMetrics(storageMetricReadVersion, volume, path)()
   580  
   581  	select {
   582  	case <-ctx.Done():
   583  		return fi, ctx.Err()
   584  	default:
   585  	}
   586  
   587  	if err = p.checkDiskStale(); err != nil {
   588  		return fi, err
   589  	}
   590  
   591  	return p.storage.ReadVersion(ctx, volume, path, versionID, readData)
   592  }
   593  
   594  func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
   595  	defer p.updateStorageMetrics(storageMetricReadAll, volume, path)()
   596  
   597  	select {
   598  	case <-ctx.Done():
   599  		return nil, ctx.Err()
   600  	default:
   601  	}
   602  
   603  	if err = p.checkDiskStale(); err != nil {
   604  		return nil, err
   605  	}
   606  
   607  	return p.storage.ReadAll(ctx, volume, path)
   608  }
   609  
   610  func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) trace.Info {
   611  	return trace.Info{
   612  		TraceType: trace.Storage,
   613  		Time:      startTime,
   614  		NodeName:  globalLocalNodeName,
   615  		FuncName:  "storage." + s.String(),
   616  		StorageStats: trace.StorageStats{
   617  			Duration: duration,
   618  			Path:     path,
   619  		},
   620  	}
   621  }
   622  
   623  // Update storage metrics
   624  func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func() {
   625  	startTime := time.Now()
   626  	trace := globalTrace.NumSubscribers() > 0
   627  	return func() {
   628  		duration := time.Since(startTime)
   629  
   630  		atomic.AddUint64(&p.apiCalls[s], 1)
   631  		p.apiLatencies[s].Add(float64(duration))
   632  
   633  		if trace {
   634  			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " ")))
   635  		}
   636  	}
   637  }