github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/pkg/externalresource/internal/file_manager.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package internal
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"net/url"
    20  
    21  	frameModel "github.com/pingcap/tiflow/engine/framework/model"
    22  	"github.com/pingcap/tiflow/engine/model"
    23  	resModel "github.com/pingcap/tiflow/engine/pkg/externalresource/model"
    24  	"github.com/pingcap/tiflow/engine/pkg/tenant"
    25  )
    26  
    27  // ResourceScope represents the environment in which the resource
    28  // has been or is to be created.
    29  type ResourceScope struct {
    30  	tenant.ProjectInfo
    31  
    32  	// Executor denotes the executor on which the resource is created.
    33  	Executor model.ExecutorID
    34  
    35  	// WorkerID denotes the worker that is creating or has created the resource.
    36  	// Note that it is NOT necessarily that consumes or depends on the resource.
    37  	WorkerID frameModel.WorkerID
    38  }
    39  
    40  // BuildResPath builds a resource path from the given scope.
    41  func (s ResourceScope) BuildResPath() string {
    42  	if s.Executor == "" {
    43  		return ""
    44  	}
    45  	resPath := url.QueryEscape(string(s.Executor))
    46  	if s.WorkerID != "" {
    47  		resPath += fmt.Sprintf("/%s", url.QueryEscape(s.WorkerID))
    48  	}
    49  	return resPath
    50  }
    51  
    52  // ResourceIdent provides information for the file manager to
    53  // uniquely determine where and how the resource is stored.
    54  type ResourceIdent struct {
    55  	ResourceScope
    56  
    57  	// Name is the custom part of the resourceID.
    58  	// For example, the resource name of `/local/resource-1` is `resource-1`.
    59  	Name resModel.ResourceName
    60  }
    61  
    62  // Scope returns the Scope of the ResourceIdent.
    63  func (i ResourceIdent) Scope() ResourceScope {
    64  	return i.ResourceScope
    65  }
    66  
    67  // BuildResPath builds a resource path from the given ident.
    68  func (i ResourceIdent) BuildResPath() string {
    69  	return fmt.Sprintf("%s/%s", i.Scope().BuildResPath(), url.QueryEscape(i.Name))
    70  }
    71  
    72  // FileManager abstracts the operations on the underlying storage.
    73  type FileManager interface {
    74  	// CreateResource creates a new resource.
    75  	CreateResource(ctx context.Context, ident ResourceIdent) (ResourceDescriptor, error)
    76  
    77  	// GetPersistedResource returns the descriptor of an already persisted resource.
    78  	GetPersistedResource(ctx context.Context, ident ResourceIdent) (ResourceDescriptor, error)
    79  
    80  	// CleanOrRecreatePersistedResource cleans or recreates the persisted resource.
    81  	// For local filemanager, it simply removes the resource and recreates it.
    82  	// For s3 filemanager, it either cleans files or recreates resources.
    83  	CleanOrRecreatePersistedResource(ctx context.Context, ident ResourceIdent) (ResourceDescriptor, error)
    84  
    85  	// RemoveTemporaryFiles cleans up all un-persisted resource files under the scope.
    86  	RemoveTemporaryFiles(ctx context.Context, scope ResourceScope) error
    87  
    88  	// RemoveResource removes a resource's files.
    89  	RemoveResource(ctx context.Context, ident ResourceIdent) error
    90  
    91  	// SetPersisted sets a resource as persisted.
    92  	SetPersisted(ctx context.Context, ident ResourceIdent) error
    93  }