github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/pkg/binlog/reader/file_test.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package reader
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"io"
    20  	"os"
    21  	"path/filepath"
    22  	"testing"
    23  	"time"
    24  
    25  	gmysql "github.com/go-mysql-org/go-mysql/mysql"
    26  	"github.com/go-mysql-org/go-mysql/replication"
    27  	"github.com/pingcap/errors"
    28  	"github.com/pingcap/tiflow/dm/pkg/binlog/common"
    29  	"github.com/pingcap/tiflow/dm/pkg/binlog/event"
    30  	"github.com/pingcap/tiflow/dm/pkg/terror"
    31  	"github.com/stretchr/testify/require"
    32  )
    33  
    34  func TestInterfaceMethods(t *testing.T) {
    35  	t.Parallel()
    36  	var (
    37  		cfg                       = &FileReaderConfig{}
    38  		gSet                      gmysql.GTIDSet // nil GTID set
    39  		timeoutCtx, timeoutCancel = context.WithTimeout(context.Background(), 10*time.Second)
    40  	)
    41  	defer timeoutCancel()
    42  
    43  	r := NewFileReader(cfg)
    44  	require.NotNil(t, r)
    45  
    46  	// check status, stageNew
    47  	status := r.Status()
    48  	frStatus, ok := status.(*FileReaderStatus)
    49  	require.True(t, ok)
    50  	require.Equal(t, common.StageNew.String(), frStatus.Stage)
    51  	require.Equal(t, uint32(0), frStatus.ReadOffset)
    52  	require.Equal(t, uint32(0), frStatus.SendOffset)
    53  	frStatusStr := frStatus.String()
    54  	require.Regexp(t, fmt.Sprintf(`.*"stage":"%s".*`, common.StageNew), frStatusStr)
    55  
    56  	// not prepared
    57  	e, err := r.GetEvent(timeoutCtx)
    58  	require.Regexp(t, fmt.Sprintf(".*%s.*", common.StageNew), err)
    59  	require.Nil(t, e)
    60  
    61  	// by GTID, not supported yet
    62  	err = r.StartSyncByGTID(gSet)
    63  	require.Regexp(t, ".*not supported.*", err)
    64  
    65  	// by pos
    66  	err = r.StartSyncByPos(gmysql.Position{})
    67  	require.Nil(t, err)
    68  
    69  	// check status, stagePrepared
    70  	status = r.Status()
    71  	frStatus, ok = status.(*FileReaderStatus)
    72  	require.True(t, ok)
    73  	require.Equal(t, common.StagePrepared.String(), frStatus.Stage)
    74  	require.Equal(t, uint32(0), frStatus.ReadOffset)
    75  	require.Equal(t, uint32(0), frStatus.SendOffset)
    76  
    77  	// re-prepare is invalid
    78  	err = r.StartSyncByPos(gmysql.Position{})
    79  	require.NotNil(t, err)
    80  
    81  	// binlog file not exists
    82  	e, err = r.GetEvent(timeoutCtx)
    83  	require.True(t, os.IsNotExist(errors.Cause(err)))
    84  	require.Nil(t, e)
    85  
    86  	// close the reader
    87  	require.Nil(t, r.Close())
    88  
    89  	// check status, stageClosed
    90  	status = r.Status()
    91  	frStatus, ok = status.(*FileReaderStatus)
    92  	require.True(t, ok)
    93  	require.Equal(t, common.StageClosed.String(), frStatus.Stage)
    94  	require.Equal(t, uint32(0), frStatus.ReadOffset)
    95  	require.Equal(t, uint32(0), frStatus.SendOffset)
    96  
    97  	// re-close is invalid
    98  	require.NotNil(t, r.Close())
    99  }
   100  
   101  func TestGetEvent(t *testing.T) {
   102  	t.Parallel()
   103  	timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 10*time.Second)
   104  	defer timeoutCancel()
   105  
   106  	// create a empty file
   107  	dir := t.TempDir()
   108  	filename := filepath.Join(dir, "mysql-bin-test.000001")
   109  	f, err := os.Create(filename)
   110  	require.Nil(t, err)
   111  	defer f.Close()
   112  
   113  	// start from the beginning
   114  	startPos := gmysql.Position{Name: filename}
   115  
   116  	// no data can be read, EOF
   117  	r := NewFileReader(&FileReaderConfig{})
   118  	require.NotNil(t, r)
   119  	require.Nil(t, r.StartSyncByPos(startPos))
   120  	e, err := r.GetEvent(timeoutCtx)
   121  	require.Equal(t, io.EOF, errors.Cause(err))
   122  	require.Nil(t, e)
   123  	require.Nil(t, r.Close()) // close the reader
   124  
   125  	// writer a binlog file header
   126  	_, err = f.Write(replication.BinLogFileHeader)
   127  	require.Nil(t, err)
   128  	// no valid events can be read, but can cancel it by the context argument
   129  	r = NewFileReader(&FileReaderConfig{})
   130  	require.NotNil(t, r)
   131  	require.Nil(t, r.StartSyncByPos(startPos))
   132  	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
   133  	defer cancel()
   134  	e, err = r.GetEvent(ctx)
   135  	require.True(t, terror.ErrReaderReachEndOfFile.Equal(err))
   136  	require.Nil(t, e)
   137  	require.Nil(t, r.Close()) // close the reader
   138  
   139  	// writer a FormatDescriptionEvent
   140  	header := &replication.EventHeader{
   141  		Timestamp: uint32(time.Now().Unix()),
   142  		ServerID:  uint32(101),
   143  	}
   144  	latestPos := uint32(len(replication.BinLogFileHeader))
   145  	formatDescEv, err := event.GenFormatDescriptionEvent(header, latestPos)
   146  	require.Nil(t, err)
   147  	require.NotNil(t, formatDescEv)
   148  	_, err = f.Write(formatDescEv.RawData)
   149  	require.Nil(t, err)
   150  	latestPos = formatDescEv.Header.LogPos
   151  
   152  	// got a FormatDescriptionEvent
   153  	r = NewFileReader(&FileReaderConfig{})
   154  	require.NotNil(t, r)
   155  	require.Nil(t, r.StartSyncByPos(startPos))
   156  	e, err = r.GetEvent(timeoutCtx)
   157  	require.Nil(t, err)
   158  	require.Equal(t, formatDescEv, e)
   159  	require.Nil(t, r.Close()) // close the reader
   160  
   161  	// check status, stageClosed
   162  	fStat, err := f.Stat()
   163  	require.Nil(t, err)
   164  	fSize := uint32(fStat.Size())
   165  	status := r.Status()
   166  	frStatus, ok := status.(*FileReaderStatus)
   167  	require.True(t, ok)
   168  	require.Equal(t, common.StageClosed.String(), frStatus.Stage)
   169  	require.Equal(t, fSize, frStatus.ReadOffset)
   170  	require.Equal(t, fSize, frStatus.SendOffset)
   171  
   172  	// write two QueryEvent
   173  	var queryEv *replication.BinlogEvent
   174  	for i := 0; i < 2; i++ {
   175  		queryEv, err = event.GenQueryEvent(
   176  			header, latestPos, 0, 0, 0, nil,
   177  			[]byte(fmt.Sprintf("schema-%d", i)), []byte(fmt.Sprintf("query-%d", i)))
   178  		require.Nil(t, err)
   179  		require.NotNil(t, queryEv)
   180  		_, err = f.Write(queryEv.RawData)
   181  		require.Nil(t, err)
   182  		latestPos = queryEv.Header.LogPos
   183  	}
   184  
   185  	// read from the middle
   186  	startPos.Pos = latestPos - queryEv.Header.EventSize
   187  	r = NewFileReader(&FileReaderConfig{})
   188  	require.NotNil(t, r)
   189  	require.Nil(t, r.StartSyncByPos(startPos))
   190  	e, err = r.GetEvent(timeoutCtx)
   191  	require.Nil(t, err)
   192  	require.Equal(t, formatDescEv.RawData, e.RawData) // always got a FormatDescriptionEvent first
   193  	e, err = r.GetEvent(timeoutCtx)
   194  	require.Nil(t, err)
   195  	require.Equal(t, queryEv.RawData, e.RawData) // the last QueryEvent
   196  	require.Nil(t, r.Close())                    // close the reader
   197  
   198  	// read from an invalid pos
   199  	startPos.Pos--
   200  	r = NewFileReader(&FileReaderConfig{})
   201  	require.NotNil(t, r)
   202  	require.Nil(t, r.StartSyncByPos(startPos))
   203  	e, err = r.GetEvent(timeoutCtx)
   204  	require.Nil(t, err)
   205  	require.Equal(t, formatDescEv.RawData, e.RawData) // always got a FormatDescriptionEvent first
   206  	e, err = r.GetEvent(timeoutCtx)
   207  	require.Regexp(t, ".*EOF.*", err)
   208  	require.Nil(t, e)
   209  }
   210  
   211  func TestWithChannelBuffer(t *testing.T) {
   212  	t.Parallel()
   213  	var (
   214  		cfg                       = &FileReaderConfig{ChBufferSize: 10}
   215  		timeoutCtx, timeoutCancel = context.WithTimeout(context.Background(), 10*time.Second)
   216  	)
   217  	defer timeoutCancel()
   218  
   219  	// create a empty file
   220  	dir := t.TempDir()
   221  	filename := filepath.Join(dir, "mysql-bin-test.000001")
   222  	f, err := os.Create(filename)
   223  	require.Nil(t, err)
   224  	defer f.Close()
   225  
   226  	// writer a binlog file header
   227  	_, err = f.Write(replication.BinLogFileHeader)
   228  	require.Nil(t, err)
   229  
   230  	// writer a FormatDescriptionEvent
   231  	header := &replication.EventHeader{
   232  		Timestamp: uint32(time.Now().Unix()),
   233  		ServerID:  uint32(101),
   234  	}
   235  	latestPos := uint32(len(replication.BinLogFileHeader))
   236  	formatDescEv, err := event.GenFormatDescriptionEvent(header, latestPos)
   237  	require.Nil(t, err)
   238  	require.NotNil(t, formatDescEv)
   239  	_, err = f.Write(formatDescEv.RawData)
   240  	require.Nil(t, err)
   241  	latestPos = formatDescEv.Header.LogPos
   242  
   243  	// write channelBufferSize QueryEvent
   244  	var queryEv *replication.BinlogEvent
   245  	for i := 0; i < cfg.ChBufferSize; i++ {
   246  		queryEv, err = event.GenQueryEvent(
   247  			header, latestPos, 0, 0, 0, nil,
   248  			[]byte(fmt.Sprintf("schema-%d", i)), []byte(fmt.Sprintf("query-%d", i)))
   249  		require.Nil(t, err)
   250  		require.NotNil(t, queryEv)
   251  		_, err = f.Write(queryEv.RawData)
   252  		require.Nil(t, err)
   253  		latestPos = queryEv.Header.LogPos
   254  	}
   255  
   256  	r := NewFileReader(cfg)
   257  	require.NotNil(t, r)
   258  	require.Nil(t, r.StartSyncByPos(gmysql.Position{Name: filename}))
   259  	time.Sleep(time.Second) // wait events to be read
   260  
   261  	// check status, stagePrepared
   262  	readOffset := latestPos - queryEv.Header.EventSize // an FormatDescriptionEvent in the channel buffer
   263  	status := r.Status()
   264  	frStatus, ok := status.(*FileReaderStatus)
   265  	require.True(t, ok)
   266  	require.Equal(t, common.StagePrepared.String(), frStatus.Stage)
   267  	require.Equal(t, readOffset, frStatus.ReadOffset)
   268  	require.Equal(t, uint32(0), frStatus.SendOffset) // no event sent yet
   269  
   270  	// get one event
   271  	e, err := r.GetEvent(timeoutCtx)
   272  	require.Nil(t, err)
   273  	require.NotNil(t, e)
   274  	require.Equal(t, formatDescEv.RawData, e.RawData)
   275  	time.Sleep(time.Second) // wait events to be read
   276  
   277  	// check status, again
   278  	readOffset = latestPos // reach the end
   279  	status = r.Status()
   280  	frStatus, ok = status.(*FileReaderStatus)
   281  	require.True(t, ok)
   282  	require.Equal(t, common.StagePrepared.String(), frStatus.Stage)
   283  	require.Equal(t, readOffset, frStatus.ReadOffset)
   284  	require.Equal(t, formatDescEv.Header.LogPos, frStatus.SendOffset) // already get formatDescEv
   285  
   286  	require.Nil(t, r.Close())
   287  }