github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/pkg/externalresource/internal/bucket/file_manager_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package bucket
    15  
    16  import (
    17  	"context"
    18  	"path/filepath"
    19  	"testing"
    20  
    21  	"github.com/pingcap/tiflow/engine/pkg/externalresource/internal"
    22  	"github.com/pingcap/tiflow/pkg/errors"
    23  	"github.com/stretchr/testify/require"
    24  )
    25  
    26  func TestFileManagerCreateAndRemoveResource(t *testing.T) {
    27  	t.Parallel()
    28  
    29  	ctx := context.Background()
    30  	fm, factory := NewFileManagerForUT(t.TempDir(), MockExecutorID)
    31  
    32  	ident := internal.ResourceIdent{
    33  		ResourceScope: internal.ResourceScope{
    34  			Executor: MockExecutorID,
    35  			WorkerID: "worker-1",
    36  		},
    37  		Name: "resource-1",
    38  	}
    39  	desc, err := fm.CreateResource(ctx, ident)
    40  	require.NoError(t, err)
    41  	factory.assertFileExists(t, filepath.Join(
    42  		UtBucketName, MockExecutorID, "worker-1", "resource-1", placeholderFileName))
    43  
    44  	storage, err := desc.ExternalStorage(ctx)
    45  	require.NoError(t, err)
    46  	err = storage.WriteFile(ctx, "file-1", []byte("dummydummy"))
    47  	require.NoError(t, err)
    48  	factory.assertFileExists(t, filepath.Join(
    49  		UtBucketName, MockExecutorID, "worker-1", "resource-1", "file-1"))
    50  
    51  	err = fm.RemoveResource(ctx, ident)
    52  	require.NoError(t, err)
    53  
    54  	factory.assertFileNotExist(t, filepath.Join(
    55  		UtBucketName, MockExecutorID, "worker-1", "resource-1", placeholderFileName))
    56  	factory.assertFileNotExist(t, filepath.Join(
    57  		UtBucketName, MockExecutorID, "worker-1", "resource-1", "file-1"))
    58  }
    59  
    60  func TestFileManagerCreateDuplicate(t *testing.T) {
    61  	t.Parallel()
    62  
    63  	ctx := context.Background()
    64  	fm, _ := NewFileManagerForUT(t.TempDir(), MockExecutorID)
    65  
    66  	ident := internal.ResourceIdent{
    67  		ResourceScope: internal.ResourceScope{
    68  			Executor: MockExecutorID,
    69  			WorkerID: "worker-1",
    70  		},
    71  		Name: "resource-1",
    72  	}
    73  	_, err := fm.CreateResource(ctx, ident)
    74  	require.NoError(t, err)
    75  
    76  	_, err = fm.CreateResource(ctx, ident)
    77  	require.ErrorContains(t, err, "resource already exists")
    78  }
    79  
    80  func TestFileManagerSetAndGetPersisted(t *testing.T) {
    81  	t.Parallel()
    82  
    83  	ctx := context.Background()
    84  	fm, _ := NewFileManagerForUT(t.TempDir(), MockExecutorID)
    85  
    86  	ident := internal.ResourceIdent{
    87  		ResourceScope: internal.ResourceScope{
    88  			Executor: MockExecutorID,
    89  			WorkerID: "worker-1",
    90  		},
    91  		Name: "resource-1",
    92  	}
    93  	desc, err := fm.CreateResource(ctx, ident)
    94  	require.NoError(t, err)
    95  
    96  	storage, err := desc.ExternalStorage(ctx)
    97  	require.NoError(t, err)
    98  	err = storage.WriteFile(ctx, "file-1", []byte("dummydummy"))
    99  	require.NoError(t, err)
   100  
   101  	err = fm.SetPersisted(ctx, ident)
   102  	require.NoError(t, err)
   103  
   104  	desc, err = fm.GetPersistedResource(ctx, ident)
   105  	require.NoError(t, err)
   106  	storage, err = desc.ExternalStorage(ctx)
   107  	require.NoError(t, err)
   108  	ok, err := storage.FileExists(ctx, "file-1")
   109  	require.NoError(t, err)
   110  	require.True(t, ok)
   111  }
   112  
   113  func TestFileManagerDoublePersisted(t *testing.T) {
   114  	t.Parallel()
   115  
   116  	ctx := context.Background()
   117  	fm, _ := NewFileManagerForUT(t.TempDir(), MockExecutorID)
   118  
   119  	ident := internal.ResourceIdent{
   120  		ResourceScope: internal.ResourceScope{
   121  			Executor: MockExecutorID,
   122  			WorkerID: "worker-1",
   123  		},
   124  		Name: "resource-1",
   125  	}
   126  	desc, err := fm.CreateResource(ctx, ident)
   127  	require.NoError(t, err)
   128  
   129  	storage, err := desc.ExternalStorage(ctx)
   130  	require.NoError(t, err)
   131  	err = storage.WriteFile(ctx, "file-1", []byte("dummydummy"))
   132  	require.NoError(t, err)
   133  
   134  	err = fm.SetPersisted(ctx, ident)
   135  	require.NoError(t, err)
   136  
   137  	// Double persistence is allowed to maintain idempotency.
   138  	err = fm.SetPersisted(ctx, ident)
   139  	require.NoError(t, err)
   140  }
   141  
   142  func TestFileManagerRemoveTemporaryResources(t *testing.T) {
   143  	t.Parallel()
   144  
   145  	ctx := context.Background()
   146  	fm, factory := NewFileManagerForUT(t.TempDir(), MockExecutorID)
   147  
   148  	ident1 := internal.ResourceIdent{
   149  		ResourceScope: internal.ResourceScope{
   150  			Executor: "executor-1",
   151  			WorkerID: "worker-1",
   152  		},
   153  		Name: "resource-1",
   154  	}
   155  	_, err := fm.CreateResource(ctx, ident1)
   156  	require.NoError(t, err)
   157  
   158  	err = fm.SetPersisted(ctx, ident1)
   159  	require.NoError(t, err)
   160  
   161  	factory.assertFileExists(t, filepath.Join(
   162  		UtBucketName, MockExecutorID, "worker-1", "resource-1", ".keep"))
   163  
   164  	ident2 := internal.ResourceIdent{
   165  		ResourceScope: internal.ResourceScope{
   166  			Executor: "executor-1",
   167  			WorkerID: "worker-1",
   168  		},
   169  		Name: "resource-2",
   170  	}
   171  	_, err = fm.CreateResource(ctx, ident2)
   172  	require.NoError(t, err)
   173  
   174  	factory.assertFileExists(t, filepath.Join(
   175  		UtBucketName, MockExecutorID, "worker-1", "resource-2", ".keep"))
   176  
   177  	err = fm.RemoveTemporaryFiles(ctx, internal.ResourceScope{
   178  		Executor: MockExecutorID,
   179  		WorkerID: "worker-1",
   180  	})
   181  	require.NoError(t, err)
   182  
   183  	factory.assertFileNotExist(t, filepath.Join(
   184  		UtBucketName, MockExecutorID, "worker-1", "resource-2", ".keep"))
   185  }
   186  
   187  func TestFileManagerShareResourceAcrossExecutors(t *testing.T) {
   188  	t.Parallel()
   189  
   190  	ctx := context.Background()
   191  	factory := newMockBucketCreator(t.TempDir(), UtBucketName)
   192  	fm1 := NewFileManagerForUTFromSharedStorageFactory("executor-1", factory)
   193  	fm2 := NewFileManagerForUTFromSharedStorageFactory("executor-2", factory)
   194  
   195  	ident := internal.ResourceIdent{
   196  		ResourceScope: internal.ResourceScope{
   197  			Executor: "executor-1",
   198  			WorkerID: "worker-1",
   199  		},
   200  		Name: "resource-1",
   201  	}
   202  	desc, err := fm1.CreateResource(ctx, ident)
   203  	require.NoError(t, err)
   204  
   205  	storage, err := desc.ExternalStorage(ctx)
   206  	require.NoError(t, err)
   207  
   208  	err = storage.WriteFile(ctx, "file-1", []byte("test-content"))
   209  	require.NoError(t, err)
   210  
   211  	// TODO: Open the test here after using the contents of the placeholder
   212  	// to indicate the persistent state.
   213  	// _, err = fm2.GetPersistedResource(ctx, ident)
   214  	// require.True(t, errors.Is(err, errors.ErrResourceFilesNotFound))
   215  
   216  	err = fm1.SetPersisted(ctx, ident)
   217  	require.NoError(t, err)
   218  
   219  	desc, err = fm2.GetPersistedResource(ctx, ident)
   220  	require.NoError(t, err)
   221  
   222  	storage, err = desc.ExternalStorage(ctx)
   223  	require.NoError(t, err)
   224  
   225  	bytes, err := storage.ReadFile(ctx, "file-1")
   226  	require.NoError(t, err)
   227  	require.Equal(t, []byte("test-content"), bytes)
   228  
   229  	err = fm1.RemoveResource(ctx, ident)
   230  	require.NoError(t, err)
   231  
   232  	_, err = fm2.GetPersistedResource(ctx, ident)
   233  	require.True(t, errors.Is(err, errors.ErrResourceFilesNotFound))
   234  }
   235  
   236  func TestFileManagerCleanOrRecreatePersistedResource(t *testing.T) {
   237  	t.Parallel()
   238  
   239  	ctx := context.Background()
   240  	factory := newMockBucketCreator(t.TempDir(), UtBucketName)
   241  	fm1 := NewFileManagerForUTFromSharedStorageFactory("executor-1", factory)
   242  	fm2 := NewFileManagerForUTFromSharedStorageFactory("executor-2", factory)
   243  
   244  	ident := internal.ResourceIdent{
   245  		ResourceScope: internal.ResourceScope{
   246  			Executor: "executor-1",
   247  			WorkerID: "worker-1",
   248  		},
   249  		Name: "resource-1",
   250  	}
   251  	desc, err := fm1.CreateResource(ctx, ident)
   252  	require.NoError(t, err)
   253  	storage, err := desc.ExternalStorage(ctx)
   254  	require.NoError(t, err)
   255  	err = fm1.SetPersisted(ctx, ident)
   256  	require.NoError(t, err)
   257  
   258  	// clean from creator
   259  	err = storage.WriteFile(ctx, "file-1", []byte("test-content"))
   260  	require.NoError(t, err)
   261  	_, err = fm1.CleanOrRecreatePersistedResource(ctx, ident)
   262  	require.NoError(t, err)
   263  	ok, err := storage.FileExists(ctx, placeholderFileName)
   264  	require.NoError(t, err)
   265  	require.True(t, ok)
   266  	ok, err = storage.FileExists(ctx, "file-1")
   267  	require.NoError(t, err)
   268  	require.False(t, ok)
   269  
   270  	err = storage.WriteFile(ctx, "file-1", []byte("test-content"))
   271  	require.NoError(t, err)
   272  	ok, err = storage.FileExists(ctx, "file-1")
   273  	require.NoError(t, err)
   274  	require.True(t, ok)
   275  
   276  	// clean from other node
   277  	_, err = fm2.CleanOrRecreatePersistedResource(ctx, ident)
   278  	require.NoError(t, err)
   279  	ok, err = storage.FileExists(ctx, placeholderFileName)
   280  	require.NoError(t, err)
   281  	require.True(t, ok)
   282  	ok, err = storage.FileExists(ctx, "file-1")
   283  	require.NoError(t, err)
   284  	require.False(t, ok)
   285  
   286  	// clean non-existent resources from other nodes
   287  	err = fm1.RemoveResource(ctx, ident)
   288  	require.NoError(t, err)
   289  	ok, err = storage.FileExists(ctx, placeholderFileName)
   290  	require.NoError(t, err)
   291  	require.False(t, ok)
   292  	_, err = fm2.CleanOrRecreatePersistedResource(ctx, ident)
   293  	require.NoError(t, err)
   294  	ok, err = storage.FileExists(ctx, placeholderFileName)
   295  	require.NoError(t, err)
   296  	require.True(t, ok)
   297  }