github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/pkg/externalresource/internal/bucket/storage_creator.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package bucket
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"net/url"
    20  
    21  	brStorage "github.com/pingcap/tidb/br/pkg/storage"
    22  	"github.com/pingcap/tiflow/engine/pkg/externalresource/internal"
    23  	"github.com/pingcap/tiflow/engine/pkg/externalresource/model"
    24  	"github.com/pingcap/tiflow/pkg/errors"
    25  	"github.com/pingcap/tiflow/pkg/util"
    26  )
    27  
    28  // Creator represents a creator used to create
    29  // brStorage.ExternalStorage.
    30  // Implementing mock or stub Creator will make
    31  // unit testing easier.
    32  type Creator interface {
    33  	newBucketForScope(ctx context.Context, scope internal.ResourceScope) (brStorage.ExternalStorage, error)
    34  	newBucketFromURI(ctx context.Context, uri string) (brStorage.ExternalStorage, error)
    35  	baseURI() string
    36  	resourceType() model.ResourceType
    37  }
    38  
    39  // CreatorImpl implements Creator.
    40  // It is exported for testing purposes.
    41  type CreatorImpl struct {
    42  	// Bucket represents a name of an s3 bucket.
    43  	Bucket string
    44  	// Prefix is an optional prefix in the S3 file path.
    45  	// It can be useful when a shared bucket is used for testing purposes.
    46  	Prefix string
    47  	// Options provide necessary information such as endpoints and access key
    48  	// for creating an s3 client.
    49  	Options *brStorage.BackendOptions
    50  	// ResourceType is the bucket type of this creator
    51  	ResourceType model.ResourceType
    52  }
    53  
    54  // NewCreator creates a new Creator
    55  func NewCreator(
    56  	config *model.Config,
    57  ) *CreatorImpl {
    58  	options, bucket, prefix, tp := config.ToBrBackendOptions()
    59  	return &CreatorImpl{Prefix: prefix, Bucket: bucket, Options: options, ResourceType: tp}
    60  }
    61  
    62  func (f *CreatorImpl) newBucketForScope(
    63  	ctx context.Context, scope internal.ResourceScope,
    64  ) (brStorage.ExternalStorage, error) {
    65  	// full uri path is like: `s3://bucket/prefix/executorID/workerID`
    66  	uri := fmt.Sprintf("%s/%s", f.baseURI(), scope.BuildResPath())
    67  	return GetExternalStorageFromURI(ctx, uri, f.Options)
    68  }
    69  
    70  func (f *CreatorImpl) baseURI() string {
    71  	uri := fmt.Sprintf("%s://%s", string(f.ResourceType), url.QueryEscape(f.Bucket))
    72  	if f.Prefix != "" {
    73  		uri += "/" + url.QueryEscape(f.Prefix)
    74  	}
    75  	return uri
    76  }
    77  
    78  func (f *CreatorImpl) resourceType() model.ResourceType {
    79  	return f.ResourceType
    80  }
    81  
    82  func (f *CreatorImpl) newBucketFromURI(
    83  	ctx context.Context,
    84  	uri string,
    85  ) (brStorage.ExternalStorage, error) {
    86  	return GetExternalStorageFromURI(ctx, uri, f.Options)
    87  }
    88  
    89  // GetExternalStorageFromURI creates a new brStorage.ExternalStorage from a uri.
    90  func GetExternalStorageFromURI(
    91  	ctx context.Context, uri string, opts *brStorage.BackendOptions,
    92  ) (brStorage.ExternalStorage, error) {
    93  	// Note that we may have network I/O here.
    94  	ret, err := util.GetExternalStorage(ctx, uri, opts, util.DefaultS3Retryer())
    95  	if err != nil {
    96  		retErr := errors.ErrFailToCreateExternalStorage.Wrap(errors.Trace(err))
    97  		return nil, retErr.GenWithStackByArgs("creating ExternalStorage for bucket")
    98  	}
    99  	return ret, nil
   100  }