github.com/matrixorigin/matrixone@v0.7.0/pkg/fileservice/config.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package fileservice
    16  
    17  import (
    18  	"strings"
    19  
    20  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    21  )
    22  
    23  const (
    24  	memFileServiceBackend     = "MEM"
    25  	diskFileServiceBackend    = "DISK"
    26  	diskETLFileServiceBackend = "DISK-ETL"
    27  	s3FileServiceBackend      = "S3"
    28  	minioFileServiceBackend   = "MINIO"
    29  )
    30  
    31  // Config fileService config
    32  type Config struct {
    33  	// Name name of fileservice, describe what an instance of fileservice is used for
    34  	Name string `toml:"name"`
    35  	// Backend fileservice backend. [MEM|DISK|DISK-ETL|S3|MINIO]
    36  	Backend string `toml:"backend"`
    37  	// S3 used to create fileservice using s3 as the backend
    38  	S3 S3Config `toml:"s3"`
    39  	// Cache specifies configs for cache
    40  	Cache CacheConfig `toml:"cache"`
    41  	// DataDir used to create fileservice using DISK as the backend
    42  	DataDir string `toml:"data-dir"`
    43  }
    44  
    45  // NewFileServicesFunc creates a new *FileServices
    46  type NewFileServicesFunc = func(defaultName string) (*FileServices, error)
    47  
    48  // NewFileService create file service from config
    49  func NewFileService(cfg Config) (FileService, error) {
    50  	switch strings.ToUpper(cfg.Backend) {
    51  	case memFileServiceBackend:
    52  		return newMemFileService(cfg)
    53  	case diskFileServiceBackend:
    54  		return newDiskFileService(cfg)
    55  	case diskETLFileServiceBackend:
    56  		return newDiskETLFileService(cfg)
    57  	case minioFileServiceBackend:
    58  		return newMinioFileService(cfg)
    59  	case s3FileServiceBackend:
    60  		return newS3FileService(cfg)
    61  	default:
    62  		return nil, moerr.NewInternalErrorNoCtx("file service backend %s not implemented", cfg.Backend)
    63  	}
    64  }
    65  
    66  func newMemFileService(cfg Config) (FileService, error) {
    67  	fs, err := NewMemoryFS(cfg.Name)
    68  	if err != nil {
    69  		return nil, err
    70  	}
    71  	return fs, nil
    72  }
    73  
    74  func newDiskFileService(cfg Config) (FileService, error) {
    75  	fs, err := NewLocalFS(
    76  		cfg.Name,
    77  		cfg.DataDir,
    78  		int64(cfg.Cache.MemoryCapacity),
    79  	)
    80  	if err != nil {
    81  		return nil, err
    82  	}
    83  	return fs, nil
    84  }
    85  
    86  func newDiskETLFileService(cfg Config) (FileService, error) {
    87  	fs, err := NewLocalETLFS(
    88  		cfg.Name,
    89  		cfg.DataDir,
    90  	)
    91  	if err != nil {
    92  		return nil, err
    93  	}
    94  	return fs, nil
    95  }
    96  
    97  func newMinioFileService(cfg Config) (FileService, error) {
    98  	fs, err := NewS3FSOnMinio(
    99  		cfg.S3.SharedConfigProfile,
   100  		cfg.Name,
   101  		cfg.S3.Endpoint,
   102  		cfg.S3.Bucket,
   103  		cfg.S3.KeyPrefix,
   104  		int64(cfg.Cache.MemoryCapacity),
   105  		int64(cfg.Cache.DiskCapacity),
   106  		cfg.Cache.DiskPath,
   107  	)
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	return fs, nil
   112  }
   113  
   114  func newS3FileService(cfg Config) (FileService, error) {
   115  	fs, err := NewS3FS(
   116  		cfg.S3.SharedConfigProfile,
   117  		cfg.Name,
   118  		cfg.S3.Endpoint,
   119  		cfg.S3.Bucket,
   120  		cfg.S3.KeyPrefix,
   121  		int64(cfg.Cache.MemoryCapacity),
   122  		int64(cfg.Cache.DiskCapacity),
   123  		cfg.Cache.DiskPath,
   124  	)
   125  	if err != nil {
   126  		return nil, err
   127  	}
   128  	return fs, nil
   129  }