github.com/matrixorigin/matrixone@v1.2.0/pkg/fileservice/s3_fs_restore.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package fileservice
    16  
    17  import (
    18  	"context"
    19  	"flag"
    20  	"os"
    21  	"path/filepath"
    22  
    23  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    24  	"github.com/matrixorigin/matrixone/pkg/logutil"
    25  	"github.com/matrixorigin/matrixone/pkg/perfcounter"
    26  	"go.uber.org/zap"
    27  )
    28  
    29  var fixMissingFlag = flag.Bool(
    30  	"fs-fix-missing",
    31  	false,
    32  	"indicates file services to try their best to fix missing files",
    33  )
    34  
    35  var fixMissingFromEnv = os.Getenv("MO_FS_FIX_MISSING") != ""
    36  
    37  func (s *S3FS) restoreFromDiskCache(ctx context.Context) {
    38  	if s.diskCache == nil {
    39  		return
    40  	}
    41  	cache := s.diskCache
    42  	logutil.Info("restore S3FS from disk cache",
    43  		zap.Any("fs name", s.Name()),
    44  		zap.Any("cache dir", cache.path),
    45  	)
    46  
    47  	counterSet := new(perfcounter.CounterSet)
    48  	ctx = perfcounter.WithCounterSet(ctx, counterSet)
    49  	numS3Write := counterSet.FileService.S3.Put.Load()
    50  
    51  	err := filepath.WalkDir(cache.path, func(diskPath string, entry os.DirEntry, err error) error {
    52  
    53  		// ignore error
    54  		if err != nil {
    55  			logutil.Info("restore from disk cache error",
    56  				zap.Any("do", "WalkDir entry"),
    57  				zap.Any("error", err),
    58  			)
    59  			return nil
    60  		}
    61  
    62  		// ignore entry
    63  		if entry.IsDir() {
    64  			return nil
    65  		}
    66  
    67  		path, err := cache.decodeFilePath(diskPath)
    68  		if err != nil {
    69  			logutil.Info("restore from disk cache error",
    70  				zap.Any("do", "decode file path"),
    71  				zap.Any("error", err),
    72  			)
    73  			// ignore
    74  			return nil
    75  		}
    76  
    77  		f, err := os.Open(diskPath)
    78  		if err != nil {
    79  			logutil.Info("restore from disk cache error",
    80  				zap.Any("do", "open file"),
    81  				zap.Any("error", err),
    82  			)
    83  			return nil
    84  		}
    85  		defer f.Close()
    86  
    87  		err = s.Write(ctx, IOVector{
    88  			FilePath: path,
    89  			Entries: []IOEntry{
    90  				{
    91  					Size:           -1,
    92  					ReaderForWrite: f,
    93  				},
    94  			},
    95  		})
    96  		if err != nil {
    97  			if moerr.IsMoErrCode(err, moerr.ErrFileAlreadyExists) {
    98  				// already exists
    99  				return nil
   100  			}
   101  			logutil.Info("restore from disk cache error",
   102  				zap.Any("do", "write"),
   103  				zap.Any("error", err),
   104  			)
   105  			return nil
   106  		}
   107  
   108  		n := counterSet.FileService.S3.Put.Load()
   109  		if n > numS3Write {
   110  			logutil.Info("restore from disk cache",
   111  				zap.Any("path", path),
   112  			)
   113  			numS3Write = n
   114  		}
   115  
   116  		return nil
   117  	})
   118  	if err != nil {
   119  		logutil.Info("restore from disk cache error",
   120  			zap.Any("do", "WalkDir"),
   121  			zap.Any("error", err),
   122  		)
   123  	}
   124  
   125  }