github.com/matrixorigin/matrixone@v1.2.0/pkg/fileservice/config.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package fileservice
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strings"
    21  
    22  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    23  	"github.com/matrixorigin/matrixone/pkg/perfcounter"
    24  )
    25  
    26  const (
    27  	memFileServiceBackend     = "MEM"
    28  	diskFileServiceBackend    = "DISK"
    29  	diskETLFileServiceBackend = "DISK-ETL"
    30  	s3FileServiceBackend      = "S3"
    31  	minioFileServiceBackend   = "MINIO"
    32  )
    33  
    34  // Config fileService config
    35  type Config struct {
    36  	// Name name of fileservice, describe what an instance of fileservice is used for
    37  	Name string `toml:"name"`
    38  	// Backend fileservice backend. [MEM|DISK|DISK-ETL|S3|MINIO]
    39  	Backend string `toml:"backend"`
    40  	// S3 used to create fileservice using s3 as the backend
    41  	S3 ObjectStorageArguments `toml:"s3"`
    42  	// Cache specifies configs for cache
    43  	Cache CacheConfig `toml:"cache"`
    44  	// DataDir used to create fileservice using DISK as the backend
    45  	DataDir string `toml:"data-dir"`
    46  	// FixMissing inidicates the file service to try its best to fix missing files
    47  	FixMissing bool `toml:"fix-missing"`
    48  }
    49  
    50  // NewFileServicesFunc creates a new *FileServices
    51  type NewFileServicesFunc = func(defaultName string) (*FileServices, error)
    52  
    53  // NewFileService create file service from config
    54  func NewFileService(
    55  	ctx context.Context, cfg Config, perfCounterSets []*perfcounter.CounterSet,
    56  ) (FileService, error) {
    57  	if cfg.Name == "" {
    58  		panic("empty name")
    59  	}
    60  	switch strings.ToUpper(cfg.Backend) {
    61  	case memFileServiceBackend:
    62  		return newMemFileService(cfg, perfCounterSets)
    63  	case diskFileServiceBackend:
    64  		return newDiskFileService(ctx, cfg, perfCounterSets)
    65  	case diskETLFileServiceBackend:
    66  		return newDiskETLFileService(cfg, perfCounterSets)
    67  	case minioFileServiceBackend:
    68  		return newMinioFileService(ctx, cfg, perfCounterSets)
    69  	case s3FileServiceBackend:
    70  		return newS3FileService(ctx, cfg, perfCounterSets)
    71  	default:
    72  		return nil, moerr.NewInternalErrorNoCtx("file service backend %s not implemented", cfg.Backend)
    73  	}
    74  }
    75  
    76  func newMemFileService(cfg Config, perfCounters []*perfcounter.CounterSet) (FileService, error) {
    77  	fs, err := NewMemoryFS(
    78  		cfg.Name,
    79  		cfg.Cache,
    80  		perfCounters,
    81  	)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  	return fs, nil
    86  }
    87  
    88  func newDiskFileService(ctx context.Context, cfg Config, perfCounters []*perfcounter.CounterSet) (FileService, error) {
    89  	if cfg.DataDir == "" {
    90  		panic(fmt.Sprintf("empty data dir: %+v", cfg))
    91  	}
    92  	fs, err := NewLocalFS(
    93  		ctx,
    94  		cfg.Name,
    95  		cfg.DataDir,
    96  		cfg.Cache,
    97  		perfCounters,
    98  	)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  	return fs, nil
   103  }
   104  
   105  func newDiskETLFileService(cfg Config, _ []*perfcounter.CounterSet) (FileService, error) {
   106  	fs, err := NewLocalETLFS(
   107  		cfg.Name,
   108  		cfg.DataDir,
   109  	)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  	return fs, nil
   114  }
   115  
   116  func newMinioFileService(
   117  	ctx context.Context, cfg Config, perfCounters []*perfcounter.CounterSet,
   118  ) (FileService, error) {
   119  	cfg.S3.Name = cfg.Name
   120  	cfg.S3.IsMinio = true
   121  	fs, err := NewS3FS(
   122  		ctx,
   123  		cfg.S3,
   124  		cfg.Cache,
   125  		perfCounters,
   126  		false,
   127  		false,
   128  	)
   129  	if err != nil {
   130  		return nil, err
   131  	}
   132  	return fs, nil
   133  }
   134  
   135  func newS3FileService(
   136  	ctx context.Context, cfg Config, perfCounters []*perfcounter.CounterSet,
   137  ) (FileService, error) {
   138  
   139  	cfg.S3.Name = cfg.Name
   140  	fs, err := NewS3FS(
   141  		ctx,
   142  		cfg.S3,
   143  		cfg.Cache,
   144  		perfCounters,
   145  		false,
   146  		false,
   147  	)
   148  	if err != nil {
   149  		return nil, err
   150  	}
   151  
   152  	if cfg.FixMissing || *fixMissingFlag || fixMissingFromEnv {
   153  		//TODO use context.WithoutCancel(ctx)
   154  		go fs.restoreFromDiskCache(context.Background())
   155  	}
   156  
   157  	return fs, nil
   158  }