vitess.io/vitess@v0.16.2/go/vt/vtorc/inst/instance_binlog.go (about)

     1  /*
     2     Copyright 2014 Outbrain Inc.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package inst
    18  
    19  import (
    20  	"errors"
    21  	"regexp"
    22  
    23  	"vitess.io/vitess/go/vt/log"
    24  )
    25  
    26  // Event entries may contains table IDs (can be different for same tables on different servers)
    27  // and also COMMIT transaction IDs (different values on different servers).
    28  // So these need to be removed from the event entry if we're to compare and validate matching
    29  // entries.
    30  var eventInfoTransformations = map[*regexp.Regexp]string{
    31  	regexp.MustCompile(`(.*) [/][*].*?[*][/](.*$)`):  "$1 $2",         // strip comments
    32  	regexp.MustCompile(`(COMMIT) .*$`):               "$1",            // commit number varies cross servers
    33  	regexp.MustCompile(`(table_id:) [0-9]+ (.*$)`):   "$1 ### $2",     // table ids change cross servers
    34  	regexp.MustCompile(`(table_id:) [0-9]+$`):        "$1 ###",        // table ids change cross servers
    35  	regexp.MustCompile(` X'([0-9a-fA-F]+)' COLLATE`): " 0x$1 COLLATE", // different ways to represent collate
    36  	regexp.MustCompile(`(BEGIN GTID [^ ]+) cid=.*`):  "$1",            // MariaDB GTID someimtes gets addition of "cid=...". Stripping
    37  }
    38  
    39  type BinlogEvent struct {
    40  	Coordinates  BinlogCoordinates
    41  	NextEventPos int64
    42  	EventType    string
    43  	Info         string
    44  }
    45  
    46  func (binlogEvent *BinlogEvent) NextBinlogCoordinates() BinlogCoordinates {
    47  	return BinlogCoordinates{LogFile: binlogEvent.Coordinates.LogFile, LogPos: binlogEvent.NextEventPos, Type: binlogEvent.Coordinates.Type}
    48  }
    49  
    50  func (binlogEvent *BinlogEvent) NormalizeInfo() {
    51  	for reg, replace := range eventInfoTransformations {
    52  		binlogEvent.Info = reg.ReplaceAllString(binlogEvent.Info, replace)
    53  	}
    54  }
    55  
    56  func (binlogEvent *BinlogEvent) Equals(other *BinlogEvent) bool {
    57  	return binlogEvent.Coordinates.Equals(&other.Coordinates) &&
    58  		binlogEvent.NextEventPos == other.NextEventPos &&
    59  		binlogEvent.EventType == other.EventType && binlogEvent.Info == other.Info
    60  }
    61  
    62  func (binlogEvent *BinlogEvent) EqualsIgnoreCoordinates(other *BinlogEvent) bool {
    63  	return binlogEvent.NextEventPos == other.NextEventPos &&
    64  		binlogEvent.EventType == other.EventType && binlogEvent.Info == other.Info
    65  }
    66  
    67  const maxEmptyEventsEvents int = 10
    68  
    69  type BinlogEventCursor struct {
    70  	cachedEvents      []BinlogEvent
    71  	currentEventIndex int
    72  	fetchNextEvents   func(BinlogCoordinates) ([]BinlogEvent, error)
    73  	nextCoordinates   BinlogCoordinates
    74  }
    75  
    76  // nextEvent will return the next event entry from binary logs; it will automatically skip to next
    77  // binary log if need be.
    78  // Internally, it uses the cachedEvents array, so that it does not go to the MySQL server upon each call.
    79  // Returns nil upon reaching end of binary logs.
    80  func (binlogEventCursor *BinlogEventCursor) nextEvent(numEmptyEventsEvents int) (*BinlogEvent, error) {
    81  	if numEmptyEventsEvents > maxEmptyEventsEvents {
    82  		log.Infof("End of logs. currentEventIndex: %d, nextCoordinates: %+v", binlogEventCursor.currentEventIndex, binlogEventCursor.nextCoordinates)
    83  		// End of logs
    84  		return nil, nil
    85  	}
    86  	if len(binlogEventCursor.cachedEvents) == 0 {
    87  		// Cache exhausted; get next bulk of entries and return the next entry
    88  		nextFileCoordinates, err := binlogEventCursor.nextCoordinates.NextFileCoordinates()
    89  		if err != nil {
    90  			return nil, err
    91  		}
    92  		log.Infof("zero cached events, next file: %+v", nextFileCoordinates)
    93  		binlogEventCursor.cachedEvents, err = binlogEventCursor.fetchNextEvents(nextFileCoordinates)
    94  		if err != nil {
    95  			return nil, err
    96  		}
    97  		binlogEventCursor.currentEventIndex = -1
    98  		// While this seems recursive do note that recursion level is at most 1, since we either have
    99  		// entries in the next binlog (no further recursion) or we don't (immediate termination)
   100  		return binlogEventCursor.nextEvent(numEmptyEventsEvents + 1)
   101  	}
   102  	if binlogEventCursor.currentEventIndex+1 < len(binlogEventCursor.cachedEvents) {
   103  		// We have enough cache to go by
   104  		binlogEventCursor.currentEventIndex++
   105  		event := &binlogEventCursor.cachedEvents[binlogEventCursor.currentEventIndex]
   106  		binlogEventCursor.nextCoordinates = event.NextBinlogCoordinates()
   107  		return event, nil
   108  	}
   109  	// Cache exhausted; get next bulk of entries and return the next entry
   110  	var err error
   111  	binlogEventCursor.cachedEvents, err = binlogEventCursor.fetchNextEvents(binlogEventCursor.cachedEvents[len(binlogEventCursor.cachedEvents)-1].NextBinlogCoordinates())
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  	binlogEventCursor.currentEventIndex = -1
   116  	// While this seems recursive do note that recursion level is at most 1, since we either have
   117  	// entries in the next binlog (no further recursion) or we don't (immediate termination)
   118  	return binlogEventCursor.nextEvent(numEmptyEventsEvents + 1)
   119  }
   120  
   121  // NextCoordinates return the binlog coordinates of the next entry as yet unprocessed by the cursor.
   122  // Moreover, when the cursor terminates (consumes last entry), these coordinates indicate what will be the futuristic
   123  // coordinates of the next binlog entry.
   124  // The value of this function is used by match-below to move a replica behind another, after exhausting the shared binlog
   125  // entries of both.
   126  func (binlogEventCursor *BinlogEventCursor) getNextCoordinates() (BinlogCoordinates, error) {
   127  	if binlogEventCursor.nextCoordinates.LogPos == 0 {
   128  		return binlogEventCursor.nextCoordinates, errors.New("Next coordinates unfound")
   129  	}
   130  	return binlogEventCursor.nextCoordinates, nil
   131  }