github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/relay/file_util_test.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package relay
    15  
    16  import (
    17  	"bytes"
    18  	"context"
    19  	"io"
    20  	"os"
    21  	"path/filepath"
    22  	"time"
    23  
    24  	gmysql "github.com/go-mysql-org/go-mysql/mysql"
    25  	"github.com/go-mysql-org/go-mysql/replication"
    26  	"github.com/pingcap/check"
    27  	"github.com/pingcap/errors"
    28  	"github.com/pingcap/tidb/pkg/parser"
    29  	"github.com/pingcap/tiflow/dm/pkg/binlog/event"
    30  	"github.com/pingcap/tiflow/dm/pkg/gtid"
    31  )
    32  
    33  var _ = check.Suite(&testFileUtilSuite{})
    34  
    35  type testFileUtilSuite struct{}
    36  
    37  func (t *testFileUtilSuite) TestCheckBinlogHeaderExist(c *check.C) {
    38  	// file not exists
    39  	filename := filepath.Join(c.MkDir(), "test-mysql-bin.000001")
    40  	exist, err := checkBinlogHeaderExist(filename)
    41  	c.Assert(err, check.ErrorMatches, ".*(no such file or directory|The system cannot find the file specified).*")
    42  	c.Assert(exist, check.IsFalse)
    43  
    44  	// empty file
    45  	err = os.WriteFile(filename, nil, 0o644)
    46  	c.Assert(err, check.IsNil)
    47  	exist, err = checkBinlogHeaderExist(filename)
    48  	c.Assert(err, check.IsNil)
    49  	c.Assert(exist, check.IsFalse)
    50  
    51  	// no enough data
    52  	err = os.WriteFile(filename, replication.BinLogFileHeader[:len(replication.BinLogFileHeader)-1], 0o644)
    53  	c.Assert(err, check.IsNil)
    54  	exist, err = checkBinlogHeaderExist(filename)
    55  	c.Assert(err, check.ErrorMatches, ".*has no enough data.*")
    56  	c.Assert(exist, check.IsFalse)
    57  
    58  	// equal
    59  	err = os.WriteFile(filename, replication.BinLogFileHeader, 0o644)
    60  	c.Assert(err, check.IsNil)
    61  	exist, err = checkBinlogHeaderExist(filename)
    62  	c.Assert(err, check.IsNil)
    63  	c.Assert(exist, check.IsTrue)
    64  
    65  	// more data
    66  	err = os.WriteFile(filename, bytes.Repeat(replication.BinLogFileHeader, 2), 0o644)
    67  	c.Assert(err, check.IsNil)
    68  	exist, err = checkBinlogHeaderExist(filename)
    69  	c.Assert(err, check.IsNil)
    70  	c.Assert(exist, check.IsTrue)
    71  
    72  	// invalid data
    73  	invalidData := make([]byte, len(replication.BinLogFileHeader))
    74  	copy(invalidData, replication.BinLogFileHeader)
    75  	invalidData[0]++
    76  	err = os.WriteFile(filename, invalidData, 0o644)
    77  	c.Assert(err, check.IsNil)
    78  	exist, err = checkBinlogHeaderExist(filename)
    79  	c.Assert(err, check.ErrorMatches, ".*header not valid.*")
    80  	c.Assert(exist, check.IsFalse)
    81  }
    82  
    83  func (t *testFileUtilSuite) TestCheckFormatDescriptionEventExist(c *check.C) {
    84  	var (
    85  		header = &replication.EventHeader{
    86  			Timestamp: uint32(time.Now().Unix()),
    87  			ServerID:  11,
    88  			Flags:     0x01,
    89  		}
    90  		latestPos uint32 = 4
    91  	)
    92  	formatDescEv, err := event.GenFormatDescriptionEvent(header, latestPos)
    93  	c.Assert(err, check.IsNil)
    94  
    95  	// file not exists
    96  	filename := filepath.Join(c.MkDir(), "test-mysql-bin.000001")
    97  	exist, err := checkFormatDescriptionEventExist(filename)
    98  	c.Assert(err, check.ErrorMatches, ".*(no such file or directory|The system cannot find the file specified).*")
    99  	c.Assert(exist, check.IsFalse)
   100  
   101  	// empty file
   102  	err = os.WriteFile(filename, nil, 0o644)
   103  	c.Assert(err, check.IsNil)
   104  	exist, err = checkFormatDescriptionEventExist(filename)
   105  	c.Assert(err, check.ErrorMatches, ".*no binlog file header at the beginning.*")
   106  	c.Assert(exist, check.IsFalse)
   107  
   108  	// only file header
   109  	err = os.WriteFile(filename, replication.BinLogFileHeader, 0o644)
   110  	c.Assert(err, check.IsNil)
   111  	exist, err = checkFormatDescriptionEventExist(filename)
   112  	c.Assert(err, check.IsNil)
   113  	c.Assert(exist, check.IsFalse)
   114  
   115  	// no enough data, < EventHeaderSize
   116  	var buff bytes.Buffer
   117  	buff.Write(replication.BinLogFileHeader)
   118  	buff.Write(formatDescEv.RawData[:replication.EventHeaderSize-1])
   119  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   120  	c.Assert(err, check.IsNil)
   121  	exist, err = checkFormatDescriptionEventExist(filename)
   122  	c.Assert(errors.Cause(err), check.Equals, io.EOF)
   123  	c.Assert(exist, check.IsFalse)
   124  
   125  	// no enough data, = EventHeaderSize
   126  	buff.Reset()
   127  	buff.Write(replication.BinLogFileHeader)
   128  	buff.Write(formatDescEv.RawData[:replication.EventHeaderSize])
   129  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   130  	c.Assert(err, check.IsNil)
   131  	exist, err = checkFormatDescriptionEventExist(filename)
   132  	c.Assert(err, check.ErrorMatches, ".*get event err EOF.*")
   133  	c.Assert(exist, check.IsFalse)
   134  
   135  	// no enough data, > EventHeaderSize, < EventSize
   136  	buff.Reset()
   137  	buff.Write(replication.BinLogFileHeader)
   138  	buff.Write(formatDescEv.RawData[:replication.EventHeaderSize+1])
   139  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   140  	c.Assert(err, check.IsNil)
   141  	exist, err = checkFormatDescriptionEventExist(filename)
   142  	c.Assert(err, check.ErrorMatches, ".*get event err EOF.*")
   143  	c.Assert(exist, check.IsFalse)
   144  
   145  	// exactly the event
   146  	buff.Reset()
   147  	buff.Write(replication.BinLogFileHeader)
   148  	buff.Write(formatDescEv.RawData)
   149  	dataCopy := make([]byte, buff.Len())
   150  	copy(dataCopy, buff.Bytes())
   151  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   152  	c.Assert(err, check.IsNil)
   153  	exist, err = checkFormatDescriptionEventExist(filename)
   154  	c.Assert(err, check.IsNil)
   155  	c.Assert(exist, check.IsTrue)
   156  
   157  	// more than the event
   158  	buff.Write([]byte("more data"))
   159  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   160  	c.Assert(err, check.IsNil)
   161  	exist, err = checkFormatDescriptionEventExist(filename)
   162  	c.Assert(err, check.IsNil)
   163  	c.Assert(exist, check.IsTrue)
   164  
   165  	// other event type
   166  	queryEv, err := event.GenQueryEvent(header, latestPos, 0, 0, 0, nil, []byte("schema"), []byte("BEGIN"))
   167  	c.Assert(err, check.IsNil)
   168  	buff.Reset()
   169  	buff.Write(replication.BinLogFileHeader)
   170  	buff.Write(queryEv.RawData)
   171  	err = os.WriteFile(filename, buff.Bytes(), 0o644)
   172  	c.Assert(err, check.IsNil)
   173  	exist, err = checkFormatDescriptionEventExist(filename)
   174  	c.Assert(err, check.ErrorMatches, ".*expect FormatDescriptionEvent.*")
   175  	c.Assert(exist, check.IsFalse)
   176  }
   177  
   178  func (t *testFileUtilSuite) TestCheckIsDuplicateEvent(c *check.C) {
   179  	// use a binlog event generator to generate some binlog events.
   180  	var (
   181  		flavor                    = gmysql.MySQLFlavor
   182  		serverID           uint32 = 11
   183  		latestPos          uint32
   184  		previousGTIDSetStr        = "3ccc475b-2343-11e7-be21-6c0b84d59f30:1-14,406a3f61-690d-11e7-87c5-6c92bf46f384:1-94321383,53bfca22-690d-11e7-8a62-18ded7a37b78:1-495,686e1ab6-c47e-11e7-a42c-6c92bf46f384:1-34981190,03fc0263-28c7-11e7-a653-6c0b84d59f30:1-7041423,05474d3c-28c7-11e7-8352-203db246dd3d:1-170,10b039fc-c843-11e7-8f6a-1866daf8d810:1-308290454"
   185  		latestGTIDStr             = "3ccc475b-2343-11e7-be21-6c0b84d59f30:14"
   186  		latestXID          uint64 = 10
   187  		allEvents                 = make([]*replication.BinlogEvent, 0, 10)
   188  		allData            bytes.Buffer
   189  	)
   190  	previousGTIDSet, err := gtid.ParserGTID(flavor, previousGTIDSetStr)
   191  	c.Assert(err, check.IsNil)
   192  	latestGTID, err := gtid.ParserGTID(flavor, latestGTIDStr)
   193  	c.Assert(err, check.IsNil)
   194  	g, err := event.NewGenerator(flavor, serverID, latestPos, latestGTID, previousGTIDSet, latestXID)
   195  	c.Assert(err, check.IsNil)
   196  	// file header with FormatDescriptionEvent and PreviousGTIDsEvent
   197  	events, data, err := g.GenFileHeader(0)
   198  	c.Assert(err, check.IsNil)
   199  	allEvents = append(allEvents, events...)
   200  	allData.Write(data)
   201  	// CREATE DATABASE/TABLE
   202  	queries := []string{
   203  		"CRATE DATABASE `db`",
   204  		"CREATE TABLE `db`.`tbl1` (c1 INT)",
   205  		"CREATE TABLE `db`.`tbl2` (c1 INT)",
   206  	}
   207  	for _, query := range queries {
   208  		events, data, err = g.GenDDLEvents("db", query, 0)
   209  		c.Assert(err, check.IsNil)
   210  		allEvents = append(allEvents, events...)
   211  		allData.Write(data)
   212  	}
   213  	// write the events to a file
   214  	filename := filepath.Join(c.MkDir(), "test-mysql-bin.000001")
   215  	err = os.WriteFile(filename, allData.Bytes(), 0o644)
   216  	c.Assert(err, check.IsNil)
   217  
   218  	// all events in the file
   219  	for _, ev := range allEvents {
   220  		duplicate, err2 := checkIsDuplicateEvent(filename, ev)
   221  		c.Assert(err2, check.IsNil)
   222  		c.Assert(duplicate, check.IsTrue)
   223  	}
   224  
   225  	// event not in the file, because its start pos > file size
   226  	events, _, err = g.GenDDLEvents("", "BEGIN", 0)
   227  	c.Assert(err, check.IsNil)
   228  	duplicate, err := checkIsDuplicateEvent(filename, events[0])
   229  	c.Assert(err, check.IsNil)
   230  	c.Assert(duplicate, check.IsFalse)
   231  
   232  	// event not in the file, because event start pos < file size < event end pos, invalid
   233  	lastEvent := allEvents[len(allEvents)-1]
   234  	header := *lastEvent.Header // clone
   235  	latestPos = lastEvent.Header.LogPos - lastEvent.Header.EventSize
   236  	eventSize := lastEvent.Header.EventSize + 1 // greater event size
   237  	dummyEv, err := event.GenDummyEvent(&header, latestPos, eventSize)
   238  	c.Assert(err, check.IsNil)
   239  	duplicate, err = checkIsDuplicateEvent(filename, dummyEv)
   240  	c.Assert(err, check.ErrorMatches, ".*file size.*is between event's start pos.*")
   241  	c.Assert(duplicate, check.IsFalse)
   242  
   243  	// event's start pos not match any event in the file, invalid
   244  	latestPos = lastEvent.Header.LogPos - lastEvent.Header.EventSize - 1 // start pos mismatch
   245  	eventSize = lastEvent.Header.EventSize
   246  	dummyEv, err = event.GenDummyEvent(&header, latestPos, eventSize)
   247  	c.Assert(err, check.IsNil)
   248  	duplicate, err = checkIsDuplicateEvent(filename, dummyEv)
   249  	c.Assert(err, check.ErrorMatches, "*diff from passed-in event.*")
   250  	c.Assert(duplicate, check.IsFalse)
   251  
   252  	// event's start/end pos matched, but content mismatched, invalid
   253  	latestPos = lastEvent.Header.LogPos - lastEvent.Header.EventSize
   254  	eventSize = lastEvent.Header.EventSize
   255  	dummyEv, err = event.GenDummyEvent(&header, latestPos, eventSize)
   256  	c.Assert(err, check.IsNil)
   257  	duplicate, err = checkIsDuplicateEvent(filename, dummyEv)
   258  	c.Assert(err, check.ErrorMatches, ".*diff from passed-in event.*")
   259  	c.Assert(duplicate, check.IsFalse)
   260  
   261  	// file not exists, invalid
   262  	filename += ".no-exist"
   263  	duplicate, err = checkIsDuplicateEvent(filename, lastEvent)
   264  	c.Assert(err, check.ErrorMatches, ".*get stat for.*")
   265  	c.Assert(duplicate, check.IsFalse)
   266  }
   267  
   268  func (t *testFileUtilSuite) TestGetTxnPosGTIDsMySQL(c *check.C) {
   269  	var (
   270  		filename           = filepath.Join(c.MkDir(), "test-mysql-bin.000001")
   271  		flavor             = gmysql.MySQLFlavor
   272  		previousGTIDSetStr = "3ccc475b-2343-11e7-be21-6c0b84d59f30:1-14,53bfca22-690d-11e7-8a62-18ded7a37b78:1-495,406a3f61-690d-11e7-87c5-6c92bf46f384:123-456,686e1ab6-c47e-11e7-a42c-6c92bf46f384:234-567"
   273  		latestGTIDStr1     = "3ccc475b-2343-11e7-be21-6c0b84d59f30:14"
   274  		latestGTIDStr2     = "53bfca22-690d-11e7-8a62-18ded7a37b78:495"
   275  		// 3 DDL + 10 DML
   276  		expectedGTIDsStr1 = "3ccc475b-2343-11e7-be21-6c0b84d59f30:1-18,53bfca22-690d-11e7-8a62-18ded7a37b78:1-505,406a3f61-690d-11e7-87c5-6c92bf46f384:123-456,686e1ab6-c47e-11e7-a42c-6c92bf46f384:234-567"
   277  		// 3 DDL + 11 DML
   278  		expectedGTIDsStr2 = "3ccc475b-2343-11e7-be21-6c0b84d59f30:1-18,53bfca22-690d-11e7-8a62-18ded7a37b78:1-506,406a3f61-690d-11e7-87c5-6c92bf46f384:123-456,686e1ab6-c47e-11e7-a42c-6c92bf46f384:234-567"
   279  	)
   280  
   281  	t.testGetTxnPosGTIDs(c, filename, flavor, previousGTIDSetStr, latestGTIDStr1, latestGTIDStr2, expectedGTIDsStr1, expectedGTIDsStr2)
   282  }
   283  
   284  func (t *testFileUtilSuite) TestGetTxnPosGTIDMariaDB(c *check.C) {
   285  	var (
   286  		filename           = filepath.Join(c.MkDir(), "test-mysql-bin.000001")
   287  		flavor             = gmysql.MariaDBFlavor
   288  		previousGTIDSetStr = "1-11-1,2-11-2"
   289  		latestGTIDStr1     = "1-11-1"
   290  		latestGTIDStr2     = "2-11-2"
   291  		// 3 DDL + 10 DML
   292  		expectedGTIDsStr1 = "1-11-5,2-11-12"
   293  		// 3 DDL + 11 DML
   294  		expectedGTIDsStr2 = "1-11-5,2-11-13"
   295  	)
   296  
   297  	t.testGetTxnPosGTIDs(c, filename, flavor, previousGTIDSetStr, latestGTIDStr1, latestGTIDStr2, expectedGTIDsStr1, expectedGTIDsStr2)
   298  }
   299  
   300  func (t *testFileUtilSuite) testGetTxnPosGTIDs(c *check.C, filename, flavor, previousGTIDSetStr,
   301  	latestGTIDStr1, latestGTIDStr2, expectedGTIDsStr1, expectedGTIDsStr2 string,
   302  ) {
   303  	parser2 := parser.New()
   304  
   305  	// different SIDs in GTID set
   306  	previousGTIDSet, err := gtid.ParserGTID(flavor, previousGTIDSetStr)
   307  	c.Assert(err, check.IsNil)
   308  	latestGTID1, err := gtid.ParserGTID(flavor, latestGTIDStr1)
   309  	c.Assert(err, check.IsNil)
   310  	latestGTID2, err := gtid.ParserGTID(flavor, latestGTIDStr2)
   311  	c.Assert(err, check.IsNil)
   312  
   313  	g, _, baseData := genBinlogEventsWithGTIDs(c, flavor, previousGTIDSet, latestGTID1, latestGTID2)
   314  
   315  	// expected latest pos/GTID set
   316  	expectedPos := int64(len(baseData))
   317  	expectedGTIDs, err := gtid.ParserGTID(flavor, expectedGTIDsStr1) // 3 DDL + 10 DML
   318  	c.Assert(err, check.IsNil)
   319  
   320  	// write the events to a file
   321  	err = os.WriteFile(filename, baseData, 0o644)
   322  	c.Assert(err, check.IsNil)
   323  
   324  	// not extra data exists
   325  	pos, gSet, err := getTxnPosGTIDs(context.Background(), filename, parser2)
   326  	c.Assert(err, check.IsNil)
   327  	c.Assert(pos, check.DeepEquals, expectedPos)
   328  	c.Assert(gSet, check.DeepEquals, expectedGTIDs)
   329  
   330  	// generate another transaction, DML
   331  	var (
   332  		tableID    uint64 = 9
   333  		columnType        = []byte{gmysql.MYSQL_TYPE_LONG}
   334  		eventType         = replication.UPDATE_ROWS_EVENTv2
   335  		schema            = "db"
   336  		table             = "tbl2"
   337  	)
   338  	updateRows := make([][]interface{}, 0, 2)
   339  	updateRows = append(updateRows, []interface{}{int32(1)}, []interface{}{int32(2)})
   340  	dmlData := []*event.DMLData{
   341  		{
   342  			TableID:    tableID,
   343  			Schema:     schema,
   344  			Table:      table,
   345  			ColumnType: columnType,
   346  			Rows:       updateRows,
   347  		},
   348  	}
   349  	extraEvents, extraData, err := g.GenDMLEvents(eventType, dmlData, 0)
   350  	c.Assert(err, check.IsNil)
   351  	c.Assert(extraEvents, check.HasLen, 5) // [GTID, BEGIN, TableMap, UPDATE, XID]
   352  
   353  	// write an incomplete event to the file
   354  	corruptData := extraEvents[0].RawData[:len(extraEvents[0].RawData)-2]
   355  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND, 0o644)
   356  	c.Assert(err, check.IsNil)
   357  	_, err = f.Write(corruptData)
   358  	c.Assert(err, check.IsNil)
   359  	c.Assert(f.Close(), check.IsNil)
   360  
   361  	// check again
   362  	pos, gSet, err = getTxnPosGTIDs(context.Background(), filename, parser2)
   363  	c.Assert(err, check.IsNil)
   364  	c.Assert(pos, check.DeepEquals, expectedPos)
   365  	c.Assert(gSet, check.DeepEquals, expectedGTIDs)
   366  
   367  	// truncate extra data
   368  	f, err = os.OpenFile(filename, os.O_WRONLY, 0o644)
   369  	c.Assert(err, check.IsNil)
   370  	err = f.Truncate(expectedPos)
   371  	c.Assert(err, check.IsNil)
   372  	c.Assert(f.Close(), check.IsNil)
   373  
   374  	// write an incomplete transaction with some completed events
   375  	for i := 0; i < len(extraEvents)-1; i++ {
   376  		f, err = os.OpenFile(filename, os.O_WRONLY|os.O_APPEND, 0o644)
   377  		c.Assert(err, check.IsNil)
   378  		_, err = f.Write(extraEvents[i].RawData) // write the event
   379  		c.Assert(err, check.IsNil)
   380  		c.Assert(f.Close(), check.IsNil)
   381  		// check again
   382  		pos, gSet, err = getTxnPosGTIDs(context.Background(), filename, parser2)
   383  		c.Assert(err, check.IsNil)
   384  		c.Assert(pos, check.DeepEquals, expectedPos)
   385  		c.Assert(gSet, check.DeepEquals, expectedGTIDs)
   386  	}
   387  
   388  	// write a completed event (and a completed transaction) to the file
   389  	f, err = os.OpenFile(filename, os.O_WRONLY|os.O_APPEND, 0o644)
   390  	c.Assert(err, check.IsNil)
   391  	_, err = f.Write(extraEvents[len(extraEvents)-1].RawData) // write the event
   392  	c.Assert(err, check.IsNil)
   393  	c.Assert(f.Close(), check.IsNil)
   394  
   395  	// check again
   396  	expectedPos += int64(len(extraData))
   397  	expectedGTIDs, err = gtid.ParserGTID(flavor, expectedGTIDsStr2) // 3 DDL + 11 DML
   398  	c.Assert(err, check.IsNil)
   399  	pos, gSet, err = getTxnPosGTIDs(context.Background(), filename, parser2)
   400  	c.Assert(err, check.IsNil)
   401  	c.Assert(pos, check.DeepEquals, expectedPos)
   402  	c.Assert(gSet, check.DeepEquals, expectedGTIDs)
   403  }
   404  
   405  func (t *testFileUtilSuite) TestGetTxnPosGTIDsNoGTID(c *check.C) {
   406  	// generate some events but without GTID enabled
   407  	var (
   408  		header = &replication.EventHeader{
   409  			Timestamp: uint32(time.Now().Unix()),
   410  			ServerID:  11,
   411  		}
   412  		latestPos uint32 = 4
   413  		filename         = filepath.Join(c.MkDir(), "test-mysql-bin.000001")
   414  	)
   415  
   416  	// FormatDescriptionEvent
   417  	formatDescEv, err := event.GenFormatDescriptionEvent(header, latestPos)
   418  	c.Assert(err, check.IsNil)
   419  	latestPos = formatDescEv.Header.LogPos
   420  
   421  	// QueryEvent, DDL
   422  	queryEv, err := event.GenQueryEvent(header, latestPos, 0, 0, 0, nil, []byte("db"), []byte("CREATE DATABASE db"))
   423  	c.Assert(err, check.IsNil)
   424  	latestPos = queryEv.Header.LogPos
   425  
   426  	// write events to the file
   427  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0o644)
   428  	c.Assert(err, check.IsNil)
   429  	_, err = f.Write(replication.BinLogFileHeader)
   430  	c.Assert(err, check.IsNil)
   431  	_, err = f.Write(formatDescEv.RawData)
   432  	c.Assert(err, check.IsNil)
   433  	_, err = f.Write(queryEv.RawData)
   434  	c.Assert(err, check.IsNil)
   435  	c.Assert(f.Close(), check.IsNil)
   436  
   437  	// check latest pos/GTID set
   438  	pos, gSet, err := getTxnPosGTIDs(context.Background(), filename, parser.New())
   439  	c.Assert(err, check.IsNil)
   440  	c.Assert(pos, check.Equals, int64(latestPos))
   441  	c.Assert(gSet, check.IsNil) // GTID not enabled
   442  }
   443  
   444  func (t *testFileUtilSuite) TestGetTxnPosGTIDsIllegalGTIDMySQL(c *check.C) {
   445  	// generate some events with GTID enabled, but without PreviousGTIDEvent
   446  	var (
   447  		header = &replication.EventHeader{
   448  			Timestamp: uint32(time.Now().Unix()),
   449  			ServerID:  11,
   450  		}
   451  		latestPos uint32 = 4
   452  	)
   453  
   454  	// GTID event
   455  	gtidEv, err := event.GenGTIDEvent(header, latestPos, 0, "3ccc475b-2343-11e7-be21-6c0b84d59f30", 14, 10, 10)
   456  	c.Assert(err, check.IsNil)
   457  
   458  	t.testGetTxnPosGTIDsIllegalGTID(c, gtidEv, ".*should have a PreviousGTIDsEvent before the GTIDEvent.*")
   459  }
   460  
   461  func (t *testFileUtilSuite) TestGetTxnPosGTIDsIllegalGTIDMairaDB(c *check.C) {
   462  	// generate some events with GTID enabled, but without MariaDBGTIDEvent
   463  	var (
   464  		header = &replication.EventHeader{
   465  			Timestamp: uint32(time.Now().Unix()),
   466  			ServerID:  11,
   467  		}
   468  		latestPos uint32 = 4
   469  	)
   470  
   471  	// GTID event
   472  	mariaDBGTIDEv, err := event.GenMariaDBGTIDEvent(header, latestPos, 10, 10)
   473  	c.Assert(err, check.IsNil)
   474  
   475  	t.testGetTxnPosGTIDsIllegalGTID(c, mariaDBGTIDEv, ".*should have a MariadbGTIDListEvent before the MariadbGTIDEvent.*")
   476  }
   477  
   478  func (t *testFileUtilSuite) testGetTxnPosGTIDsIllegalGTID(c *check.C, gtidEv *replication.BinlogEvent, errRegStr string) {
   479  	var (
   480  		header = &replication.EventHeader{
   481  			Timestamp: uint32(time.Now().Unix()),
   482  			ServerID:  11,
   483  		}
   484  		latestPos uint32 = 4
   485  		filename         = filepath.Join(c.MkDir(), "test-mysql-bin.000001")
   486  	)
   487  
   488  	// FormatDescriptionEvent
   489  	formatDescEv, err := event.GenFormatDescriptionEvent(header, latestPos)
   490  	c.Assert(err, check.IsNil)
   491  
   492  	// write events to the file
   493  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0o644)
   494  	c.Assert(err, check.IsNil)
   495  	_, err = f.Write(replication.BinLogFileHeader)
   496  	c.Assert(err, check.IsNil)
   497  	_, err = f.Write(formatDescEv.RawData)
   498  	c.Assert(err, check.IsNil)
   499  	_, err = f.Write(gtidEv.RawData)
   500  	c.Assert(err, check.IsNil)
   501  	c.Assert(f.Close(), check.IsNil)
   502  
   503  	// check latest pos/GTID set
   504  	pos, gSet, err := getTxnPosGTIDs(context.Background(), filename, parser.New())
   505  	c.Assert(err, check.ErrorMatches, errRegStr)
   506  	c.Assert(pos, check.Equals, int64(0))
   507  	c.Assert(gSet, check.IsNil)
   508  }
   509  
   510  func (t *testFileUtilSuite) TestDontTruncateOnlyHeader(c *check.C) {
   511  	var (
   512  		header = &replication.EventHeader{
   513  			Timestamp: uint32(time.Now().Unix()),
   514  			ServerID:  11,
   515  		}
   516  		latestPos              = 4
   517  		previousGSetMySQL, _   = gtid.ParserGTID("mysql", "3ccc475b-2343-11e7-be21-6c0b84d59f30:14")
   518  		previousGSetMariaDB, _ = gtid.ParserGTID("mariadb", "0-1-5")
   519  	)
   520  
   521  	formatDescEv, err := event.GenFormatDescriptionEvent(header, uint32(latestPos))
   522  	c.Assert(err, check.IsNil)
   523  	latestPos += len(formatDescEv.RawData)
   524  
   525  	mysqlGTIDev, _ := event.GenPreviousGTIDsEvent(header, uint32(latestPos), previousGSetMySQL)
   526  	mariaDBGTIDev, _ := event.GenMariaDBGTIDListEvent(header, uint32(latestPos), previousGSetMariaDB)
   527  
   528  	t.testDontTruncate(c, []*replication.BinlogEvent{formatDescEv, mysqlGTIDev})
   529  	t.testDontTruncate(c, []*replication.BinlogEvent{formatDescEv, mariaDBGTIDev})
   530  }
   531  
   532  func (t *testFileUtilSuite) testDontTruncate(c *check.C, events []*replication.BinlogEvent) {
   533  	var (
   534  		filename = filepath.Join(c.MkDir(), "dont-truncate.000001")
   535  		parser2  = parser.New()
   536  	)
   537  
   538  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0o644)
   539  	c.Assert(err, check.IsNil)
   540  
   541  	_, err = f.Write(replication.BinLogFileHeader)
   542  	c.Assert(err, check.IsNil)
   543  
   544  	for _, ev := range events {
   545  		_, err = f.Write(ev.RawData)
   546  		c.Assert(err, check.IsNil)
   547  
   548  		stat, _ := f.Stat()
   549  		pos, _, err := getTxnPosGTIDs(context.Background(), filename, parser2)
   550  		c.Assert(err, check.IsNil)
   551  		c.Assert(pos, check.Equals, stat.Size())
   552  	}
   553  }