github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/pkg/externalresource/internal/bucket/resource_controller.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package bucket
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  
    20  	"github.com/pingcap/log"
    21  	"github.com/pingcap/tiflow/engine/model"
    22  	"github.com/pingcap/tiflow/engine/pkg/externalresource/internal"
    23  	resModel "github.com/pingcap/tiflow/engine/pkg/externalresource/model"
    24  	"go.uber.org/zap"
    25  )
    26  
    27  const defaultControllerID = "leader-controller"
    28  
    29  var _ internal.ResourceController = &resourceController{}
    30  
    31  // resourceController defines operations specific to the s3 file type.
    32  type resourceController struct {
    33  	// fm is used to operate s3 storage.
    34  	fm *FileManager
    35  }
    36  
    37  // NewResourceController creates a new bucket resourceController.
    38  func NewResourceController(config *resModel.Config) *resourceController {
    39  	log.Debug("create bucket resource controller", zap.Any("config", config))
    40  	fm := NewFileManagerWithConfig(defaultControllerID, config)
    41  	return &resourceController{fm: fm}
    42  }
    43  
    44  // GCSingleResource returns a closure to the invoker to perform GC.
    45  func (r *resourceController) GCSingleResource(ctx context.Context, res *resModel.ResourceMeta) error {
    46  	tp, resName, err := resModel.ParseResourceID(res.ID)
    47  	if err != nil {
    48  		return err
    49  	}
    50  	if tp != resModel.ResourceTypeS3 && tp != resModel.ResourceTypeGCS {
    51  		log.Panic("unexpected resource type", zap.Any("resource", res))
    52  	}
    53  
    54  	return r.fm.RemoveResource(ctx, internal.ResourceIdent{
    55  		Name: resName,
    56  		ResourceScope: internal.ResourceScope{
    57  			Executor: res.Executor,
    58  			WorkerID: res.Worker,
    59  		},
    60  	})
    61  }
    62  
    63  // GCExecutor cleanups all temporary resources created by the executor.
    64  func (r *resourceController) GCExecutor(
    65  	ctx context.Context, resources []*resModel.ResourceMeta, executorID model.ExecutorID,
    66  ) error {
    67  	persistedResSet := make(map[string]struct{})
    68  	for _, res := range resources {
    69  		if res.Executor != executorID {
    70  			log.Panic("unexpected resource which is persisted by other exeuctor",
    71  				zap.Any("resource", res), zap.Any("expectedeExecutor", executorID))
    72  		}
    73  		tp, resName, err := resModel.ParseResourceID(res.ID)
    74  		if err != nil {
    75  			return err
    76  		}
    77  		if tp != resModel.ResourceTypeS3 && tp != resModel.ResourceTypeGCS {
    78  			log.Panic("unexpected resource type", zap.Any("resource", res))
    79  		}
    80  
    81  		resPath := fmt.Sprintf("%s/%s", res.Worker, resName)
    82  		persistedResSet[resPath] = struct{}{}
    83  	}
    84  
    85  	scope := internal.ResourceScope{
    86  		Executor: executorID,
    87  		WorkerID: "",
    88  	}
    89  	return r.fm.removeAllTemporaryFilesByMeta(ctx, scope, persistedResSet)
    90  }