github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/pkg/externalresource/model/config.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package model
    15  
    16  import (
    17  	"path/filepath"
    18  
    19  	brStorage "github.com/pingcap/tidb/br/pkg/storage"
    20  	"github.com/pingcap/tiflow/pkg/errors"
    21  )
    22  
    23  const defaultLocalStorageDirPrefix = "/tmp/dfe-storage"
    24  
    25  // DefaultConfig defines the default configuration for external storage
    26  var DefaultConfig = Config{
    27  	Local: LocalFileConfig{BaseDir: ""},
    28  	S3: S3Config{
    29  		S3BackendOptions: brStorage.S3BackendOptions{
    30  			ForcePathStyle: true,
    31  		},
    32  		Bucket: "",
    33  		Prefix: "",
    34  	},
    35  	GCS: GCSConfig{
    36  		Bucket: "",
    37  		Prefix: "",
    38  	},
    39  }
    40  
    41  // Config defines configurations for an external storage resource
    42  type Config struct {
    43  	Local LocalFileConfig `json:"local" toml:"local"`
    44  	S3    S3Config        `json:"s3" toml:"s3"`
    45  	GCS   GCSConfig       `json:"gcs" toml:"gcs"`
    46  }
    47  
    48  // LocalEnabled returns true if the local storage is enabled
    49  func (c Config) LocalEnabled() bool {
    50  	return c.Local.BaseDir != ""
    51  }
    52  
    53  // S3Enabled returns true if the S3 storage is enabled
    54  func (c Config) S3Enabled() bool {
    55  	return c.S3.Bucket != ""
    56  }
    57  
    58  // GCSEnabled returns true if the GCS storage is enabled
    59  func (c Config) GCSEnabled() bool {
    60  	return c.GCS.Bucket != ""
    61  }
    62  
    63  // Adjust adjusts the configuration
    64  func (c *Config) Adjust(executorID ExecutorID) {
    65  	c.Local.Adjust(executorID)
    66  }
    67  
    68  // Validate implements the validation.Validatable interface
    69  func (c Config) Validate() error {
    70  	if c.S3Enabled() && c.GCSEnabled() {
    71  		return errors.ErrInvalidArgument.GenWithStackByArgs("both s3 and gcs are enabled")
    72  	}
    73  
    74  	return nil
    75  }
    76  
    77  // ToBrBackendOptions return BackendOptions for brStorage
    78  // Make sure the Config is a valid config
    79  func (c Config) ToBrBackendOptions() (opts *brStorage.BackendOptions, bucket, prefix string, tp ResourceType) {
    80  	if c.S3Enabled() {
    81  		return &brStorage.BackendOptions{
    82  			S3: c.S3.S3BackendOptions,
    83  		}, c.S3.Bucket, c.S3.Prefix, ResourceTypeS3
    84  	}
    85  
    86  	if c.GCSEnabled() {
    87  		return &brStorage.BackendOptions{
    88  			GCS: c.GCS.GCSBackendOptions,
    89  		}, c.GCS.Bucket, c.GCS.Prefix, ResourceTypeGCS
    90  	}
    91  
    92  	return &brStorage.BackendOptions{}, "", "", ResourceTypeNone
    93  }
    94  
    95  // LocalFileConfig defines configurations for a local file based resource
    96  type LocalFileConfig struct {
    97  	BaseDir string `json:"base-dir" toml:"base-dir"`
    98  }
    99  
   100  // Adjust adjusts the local file config
   101  func (c *LocalFileConfig) Adjust(executorID ExecutorID) {
   102  	if c.BaseDir == "" {
   103  		c.BaseDir = defaultLocalStorageDirPrefix
   104  	}
   105  	c.BaseDir = filepath.Join(c.BaseDir, string(executorID))
   106  }
   107  
   108  // S3Config defines configurations for s3 based resources
   109  type S3Config struct {
   110  	brStorage.S3BackendOptions
   111  	Bucket string `json:"bucket" toml:"bucket"`
   112  	Prefix string `json:"prefix" toml:"prefix"`
   113  }
   114  
   115  // GCSConfig defines configurations for gcs based resources
   116  type GCSConfig struct {
   117  	brStorage.GCSBackendOptions
   118  	Bucket string `json:"bucket" toml:"bucket"`
   119  	Prefix string `json:"prefix" toml:"prefix"`
   120  }