github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/relay/purger_helper_test.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package relay
    15  
    16  import (
    17  	"os"
    18  	"path/filepath"
    19  	"time"
    20  
    21  	. "github.com/pingcap/check"
    22  	"github.com/pingcap/tiflow/dm/pkg/log"
    23  	"github.com/pingcap/tiflow/dm/pkg/streamer"
    24  	"github.com/pingcap/tiflow/dm/pkg/utils"
    25  )
    26  
    27  func (t *testPurgerSuite) TestPurgeRelayFilesBeforeFile(c *C) {
    28  	// UUID mismatch
    29  	safeRelay := &streamer.RelayLogInfo{
    30  		SubDir: "not-found-uuid",
    31  	}
    32  	files, err := getRelayFilesBeforeFile(log.L(), "", t.uuids, safeRelay)
    33  	c.Assert(err, NotNil)
    34  	c.Assert(files, IsNil)
    35  
    36  	// create relay log dir
    37  	baseDir := c.MkDir()
    38  	// empty relay log dirs
    39  	safeRelay = &streamer.RelayLogInfo{
    40  		SubDir: t.uuids[len(t.uuids)-1],
    41  	}
    42  	files, err = getRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
    43  	c.Assert(err, NotNil)
    44  	c.Assert(files, IsNil)
    45  
    46  	// create relay log files
    47  	relayDirsPath, relayFilesPath, _ := t.genRelayLogFiles(c, baseDir, -1, -1)
    48  
    49  	// no older
    50  	safeRelay = &streamer.RelayLogInfo{
    51  		SubDir:   t.uuids[0],
    52  		Filename: t.relayFiles[0][0],
    53  	}
    54  	files, err = getRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
    55  	c.Assert(err, IsNil)
    56  	c.Assert(files, DeepEquals, []*subRelayFiles{})
    57  
    58  	// only relay files in first sub dir
    59  	safeRelay = &streamer.RelayLogInfo{
    60  		SubDir:   t.uuids[1],
    61  		Filename: t.relayFiles[1][0],
    62  	}
    63  	files, err = getRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
    64  	c.Assert(err, IsNil)
    65  	c.Assert(len(files), Equals, 1)
    66  	c.Assert(files[0].dir, Equals, relayDirsPath[0])
    67  	c.Assert(files[0].files, DeepEquals, relayFilesPath[0])
    68  	c.Assert(files[0].hasAll, IsTrue)
    69  
    70  	// relay files in first sub dir, and some in second sub dir
    71  	safeRelay = &streamer.RelayLogInfo{
    72  		SubDir:   t.uuids[1],
    73  		Filename: t.relayFiles[1][len(t.relayFiles[1])-1],
    74  	}
    75  	files, err = getRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
    76  	c.Assert(err, IsNil)
    77  	c.Assert(len(files), Equals, 2)
    78  	c.Assert(files[0].dir, Equals, relayDirsPath[0])
    79  	c.Assert(files[0].files, DeepEquals, relayFilesPath[0])
    80  	c.Assert(files[0].hasAll, IsTrue)
    81  	c.Assert(files[1].dir, Equals, relayDirsPath[1])
    82  	c.Assert(files[1].files, DeepEquals, relayFilesPath[1][:len(t.relayFiles[1])-1])
    83  	c.Assert(files[1].hasAll, IsFalse)
    84  
    85  	// relay files in first and second sub dir, and some in third sub dir
    86  	safeRelay = &streamer.RelayLogInfo{
    87  		SubDir:   t.uuids[2],
    88  		Filename: t.relayFiles[2][1],
    89  	}
    90  	files, err = getRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
    91  	c.Assert(err, IsNil)
    92  	c.Assert(len(files), Equals, 3)
    93  	c.Assert(files[0].dir, Equals, relayDirsPath[0])
    94  	c.Assert(files[0].files, DeepEquals, relayFilesPath[0])
    95  	c.Assert(files[0].hasAll, IsTrue)
    96  	c.Assert(files[1].dir, Equals, relayDirsPath[1])
    97  	c.Assert(files[1].files, DeepEquals, relayFilesPath[1])
    98  	c.Assert(files[1].hasAll, IsTrue)
    99  	c.Assert(files[2].dir, Equals, relayDirsPath[2])
   100  	c.Assert(files[2].files, DeepEquals, relayFilesPath[2][:1])
   101  	c.Assert(files[2].hasAll, IsFalse)
   102  
   103  	// write a fake meta file to first sub dir
   104  	fakeMeta := filepath.Join(relayDirsPath[0], utils.MetaFilename)
   105  	c.Assert(os.WriteFile(fakeMeta, []byte{}, 0o666), IsNil)
   106  
   107  	// purge all relay log files in first and second sub dir, and some in third sub dir
   108  	err = purgeRelayFilesBeforeFile(log.L(), baseDir, t.uuids, safeRelay)
   109  	c.Assert(err, IsNil)
   110  	c.Assert(utils.IsDirExists(relayDirsPath[0]), IsFalse)
   111  	c.Assert(utils.IsDirExists(relayDirsPath[1]), IsFalse)
   112  	c.Assert(utils.IsDirExists(relayDirsPath[2]), IsTrue)
   113  	c.Assert(utils.IsFileExists(relayFilesPath[2][0]), IsFalse)
   114  	c.Assert(utils.IsFileExists(relayFilesPath[2][1]), IsTrue)
   115  	c.Assert(utils.IsFileExists(relayFilesPath[2][2]), IsTrue)
   116  }
   117  
   118  func (t *testPurgerSuite) TestPurgeRelayFilesBeforeFileAndTime(c *C) {
   119  	// create relay log dir
   120  	baseDir := c.MkDir()
   121  
   122  	// empty relay log dirs
   123  	safeRelay := &streamer.RelayLogInfo{
   124  		SubDir: t.uuids[len(t.uuids)-1],
   125  	}
   126  	files, err := getRelayFilesBeforeFileAndTime(log.L(), baseDir, t.uuids, safeRelay, time.Now())
   127  	c.Assert(err, NotNil)
   128  	c.Assert(files, IsNil)
   129  
   130  	// create relay log files
   131  	relayDirsPath, relayFilesPath, safeTime := t.genRelayLogFiles(c, baseDir, 1, 1)
   132  
   133  	// safeRelay older than safeTime
   134  	safeRelay = &streamer.RelayLogInfo{
   135  		SubDir:   t.uuids[1],
   136  		Filename: t.relayFiles[1][0],
   137  	}
   138  	files, err = getRelayFilesBeforeFileAndTime(log.L(), baseDir, t.uuids, safeRelay, safeTime)
   139  	c.Assert(err, IsNil)
   140  	c.Assert(len(files), Equals, 1)
   141  	c.Assert(files[0].dir, Equals, relayDirsPath[0])
   142  	c.Assert(files[0].files, DeepEquals, relayFilesPath[0])
   143  	c.Assert(files[0].hasAll, IsTrue)
   144  
   145  	// safeRelay newer than safeTime
   146  	safeRelay = &streamer.RelayLogInfo{
   147  		SubDir:   t.uuids[2],
   148  		Filename: t.relayFiles[2][0],
   149  	}
   150  	files, err = getRelayFilesBeforeFileAndTime(log.L(), baseDir, t.uuids, safeRelay, safeTime)
   151  	c.Assert(err, IsNil)
   152  	c.Assert(len(files), Equals, 2)
   153  	c.Assert(files[0].dir, Equals, relayDirsPath[0])
   154  	c.Assert(files[0].files, DeepEquals, relayFilesPath[0])
   155  	c.Assert(files[0].hasAll, IsTrue)
   156  	c.Assert(files[1].dir, Equals, relayDirsPath[1])
   157  	c.Assert(files[1].files, DeepEquals, relayFilesPath[1][:2])
   158  	c.Assert(files[1].hasAll, IsFalse)
   159  
   160  	// write a fake meta file to first sub dir
   161  	fakeMeta := filepath.Join(relayDirsPath[0], utils.MetaFilename)
   162  	c.Assert(os.WriteFile(fakeMeta, []byte{}, 0o666), IsNil)
   163  
   164  	// purge all relay log files in first and second sub dir, and some in third sub dir
   165  	err = purgeRelayFilesBeforeFileAndTime(log.L(), baseDir, t.uuids, safeRelay, safeTime)
   166  	c.Assert(err, IsNil)
   167  	c.Assert(utils.IsDirExists(relayDirsPath[0]), IsFalse)
   168  	c.Assert(utils.IsDirExists(relayDirsPath[1]), IsTrue)
   169  	c.Assert(utils.IsDirExists(relayDirsPath[2]), IsTrue)
   170  	c.Assert(utils.IsFileExists(relayFilesPath[1][0]), IsFalse)
   171  	c.Assert(utils.IsFileExists(relayFilesPath[1][1]), IsFalse)
   172  	c.Assert(utils.IsFileExists(relayFilesPath[1][2]), IsTrue)
   173  }