github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/cdc/puller/frontier/frontier_test.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package frontier
    15  
    16  import (
    17  	"bytes"
    18  	"math/rand"
    19  	"sort"
    20  	"testing"
    21  
    22  	"github.com/pingcap/check"
    23  	"github.com/pingcap/ticdc/pkg/regionspan"
    24  	"github.com/pingcap/ticdc/pkg/util/testleak"
    25  )
    26  
    27  type spanFrontierSuite struct{}
    28  
    29  func Test(t *testing.T) { check.TestingT(t) }
    30  
    31  var _ = check.Suite(&spanFrontierSuite{})
    32  
    33  func (s *spanFrontierSuite) TestSpanFrontier(c *check.C) {
    34  	defer testleak.AfterTest(c)()
    35  	keyA := []byte("a")
    36  	keyB := []byte("b")
    37  	keyC := []byte("c")
    38  	keyD := []byte("d")
    39  
    40  	spAB := regionspan.ComparableSpan{Start: keyA, End: keyB}
    41  	spAC := regionspan.ComparableSpan{Start: keyA, End: keyC}
    42  	spAD := regionspan.ComparableSpan{Start: keyA, End: keyD}
    43  	spBC := regionspan.ComparableSpan{Start: keyB, End: keyC}
    44  	spBD := regionspan.ComparableSpan{Start: keyB, End: keyD}
    45  	spCD := regionspan.ComparableSpan{Start: keyC, End: keyD}
    46  
    47  	f := NewFrontier(5, spAD).(*spanFrontier)
    48  
    49  	c.Assert(f.Frontier(), check.Equals, uint64(5))
    50  	c.Assert(f.String(), check.Equals, `[a @ 5] [d @ Max] `)
    51  	checkFrontier(c, f)
    52  
    53  	f.Forward(
    54  		regionspan.ComparableSpan{Start: []byte("d"), End: []byte("e")},
    55  		100,
    56  	)
    57  	c.Assert(f.Frontier(), check.Equals, uint64(5))
    58  	c.Assert(f.String(), check.Equals, `[a @ 5] [d @ 100] [e @ Max] `)
    59  	checkFrontier(c, f)
    60  
    61  	f.Forward(
    62  		regionspan.ComparableSpan{Start: []byte("g"), End: []byte("h")},
    63  		200,
    64  	)
    65  	c.Assert(f.Frontier(), check.Equals, uint64(5))
    66  	c.Assert(f.String(), check.Equals, `[a @ 5] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
    67  	checkFrontier(c, f)
    68  
    69  	// Forward the tracked span space.
    70  	f.Forward(
    71  		regionspan.ComparableSpan{Start: []byte("a"), End: []byte("d")},
    72  		1,
    73  	)
    74  	c.Assert(f.Frontier(), check.Equals, uint64(1))
    75  	c.Assert(f.String(), check.Equals, `[a @ 1] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
    76  	checkFrontier(c, f)
    77  
    78  	// // Forward it again
    79  	f.Forward(
    80  		regionspan.ComparableSpan{Start: []byte("a"), End: []byte("d")},
    81  		2,
    82  	)
    83  	c.Assert(f.Frontier(), check.Equals, uint64(2))
    84  	c.Assert(f.String(), check.Equals, `[a @ 2] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
    85  	checkFrontier(c, f)
    86  
    87  	// // Forward to smaller ts
    88  	f.Forward(
    89  		regionspan.ComparableSpan{Start: []byte("a"), End: []byte("d")},
    90  		1,
    91  	)
    92  	c.Assert(f.Frontier(), check.Equals, uint64(1))
    93  	c.Assert(f.String(), check.Equals, `[a @ 1] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
    94  	checkFrontier(c, f)
    95  
    96  	// // Forward b-c
    97  	f.Forward(spBC, 3)
    98  	c.Assert(f.Frontier(), check.Equals, uint64(1))
    99  	c.Assert(f.String(), check.Equals, `[a @ 1] [b @ 3] [c @ 1] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   100  	checkFrontier(c, f)
   101  
   102  	// Forward b-c more to be 4
   103  	f.Forward(spBC, 4)
   104  	c.Assert(f.Frontier(), check.Equals, uint64(1))
   105  	c.Assert(f.String(), check.Equals, `[a @ 1] [b @ 4] [c @ 1] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   106  	checkFrontier(c, f)
   107  
   108  	// Forward all to at least 3
   109  	f.Forward(spAD, 3)
   110  	c.Assert(f.Frontier(), check.Equals, uint64(3))
   111  	c.Assert(f.String(), check.Equals, `[a @ 3] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   112  	checkFrontier(c, f)
   113  
   114  	// Forward AB and CD to be 5, keep BC at 4
   115  	f.Forward(spAB, 5)
   116  	c.Assert(f.Frontier(), check.Equals, uint64(3))
   117  	c.Assert(f.String(), check.Equals, `[a @ 5] [b @ 3] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   118  	checkFrontier(c, f)
   119  
   120  	f.Forward(spCD, 5)
   121  	c.Assert(f.Frontier(), check.Equals, uint64(3))
   122  	c.Assert(f.String(), check.Equals, `[a @ 5] [b @ 3] [c @ 5] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   123  	checkFrontier(c, f)
   124  
   125  	// Catch BC to be 5 too
   126  	f.Forward(spBC, 5)
   127  	c.Assert(f.Frontier(), check.Equals, uint64(5))
   128  	c.Assert(f.String(), check.Equals, `[a @ 5] [b @ 5] [c @ 5] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   129  	checkFrontier(c, f)
   130  
   131  	// Forward all to be 6
   132  	f.Forward(spAD, 6)
   133  	c.Assert(f.Frontier(), check.Equals, uint64(6))
   134  	c.Assert(f.String(), check.Equals, `[a @ 6] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   135  	checkFrontier(c, f)
   136  
   137  	// Forward ac to 7
   138  	f.Forward(spAC, 7)
   139  	c.Assert(f.Frontier(), check.Equals, uint64(6))
   140  	c.Assert(f.String(), check.Equals, `[a @ 7] [c @ 6] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   141  	checkFrontier(c, f)
   142  
   143  	// Forward bd to 8
   144  	f.Forward(spBD, 8)
   145  	c.Assert(f.Frontier(), check.Equals, uint64(7))
   146  	c.Assert(f.String(), check.Equals, `[a @ 7] [b @ 8] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   147  	checkFrontier(c, f)
   148  
   149  	// Forward ab to 8
   150  	f.Forward(spAB, 8)
   151  	c.Assert(f.Frontier(), check.Equals, uint64(8))
   152  	c.Assert(f.String(), check.Equals, `[a @ 8] [b @ 8] [d @ 100] [e @ Max] [g @ 200] [h @ Max] `)
   153  	checkFrontier(c, f)
   154  
   155  	f.Forward(regionspan.ComparableSpan{Start: []byte("1"), End: []byte("g")}, 9)
   156  	c.Assert(f.Frontier(), check.Equals, uint64(9))
   157  	c.Assert(f.String(), check.Equals, `[1 @ 9] [g @ 200] [h @ Max] `)
   158  	checkFrontier(c, f)
   159  
   160  	f.Forward(regionspan.ComparableSpan{Start: []byte("g"), End: []byte("i")}, 10)
   161  	c.Assert(f.Frontier(), check.Equals, uint64(9))
   162  	c.Assert(f.String(), check.Equals, `[1 @ 9] [g @ 10] [i @ Max] `)
   163  	checkFrontier(c, f)
   164  }
   165  
   166  func (s *spanFrontierSuite) TestSpanFrontierFallback(c *check.C) {
   167  	defer testleak.AfterTest(c)()
   168  	keyA := []byte("a")
   169  	keyB := []byte("b")
   170  	keyC := []byte("c")
   171  	keyD := []byte("d")
   172  	keyE := []byte("e")
   173  
   174  	spAB := regionspan.ComparableSpan{Start: keyA, End: keyB}
   175  	spBC := regionspan.ComparableSpan{Start: keyB, End: keyC}
   176  	spCD := regionspan.ComparableSpan{Start: keyC, End: keyD}
   177  	spDE := regionspan.ComparableSpan{Start: keyD, End: keyE}
   178  
   179  	f := NewFrontier(20, spAB).(*spanFrontier)
   180  	f.Forward(spBC, 20)
   181  	f.Forward(spCD, 10)
   182  	f.Forward(spDE, 20)
   183  
   184  	// [A, B) [B, C) [C, D) [D, E)
   185  	// 20     20     10     20
   186  	c.Assert(f.Frontier(), check.Equals, uint64(10))
   187  	c.Assert(f.String(), check.Equals, `[a @ 20] [b @ 20] [c @ 10] [d @ 20] [e @ Max] `)
   188  	checkFrontier(c, f)
   189  
   190  	// [A, B) [B, D) [D, E)
   191  	// 20     10     10
   192  	// [B, D) does not forward, because of split to [B, C) and [C, D) immediately
   193  
   194  	// [A, B) [B, C) [C, D) [D, E)
   195  	// 20     10     10     20
   196  	// [B, C) does not forward, because of merge into [A, C) immediately
   197  	f.Forward(spCD, 20)
   198  	c.Assert(f.Frontier(), check.Equals, uint64(20))
   199  	// the frontier stoes [A, B) and [B, C) but they are not correct exactly
   200  	c.Assert(f.String(), check.Equals, `[a @ 20] [b @ 20] [c @ 20] [d @ 20] [e @ Max] `)
   201  	checkFrontier(c, f)
   202  
   203  	// Bump, here we meet resolved ts fall back, where 10 is less than f.Frontier()
   204  	// But there is no data loss actually.
   205  	// f.Forward(spAC, 10)
   206  }
   207  
   208  func (s *spanFrontierSuite) TestMinMax(c *check.C) {
   209  	defer testleak.AfterTest(c)()
   210  	var keyMin []byte
   211  	var keyMax []byte
   212  	keyMid := []byte("m")
   213  
   214  	spMinMid := regionspan.ComparableSpan{Start: keyMin, End: keyMid}
   215  	spMidMax := regionspan.ComparableSpan{Start: keyMid, End: keyMax}
   216  	spMinMax := regionspan.ComparableSpan{Start: keyMin, End: keyMax}
   217  
   218  	f := NewFrontier(0, spMinMax)
   219  	c.Assert(f.Frontier(), check.Equals, uint64(0))
   220  	c.Assert(f.String(), check.Equals, "[ @ 0] [\xff\xff\xff\xff\xff @ Max] ")
   221  	checkFrontier(c, f)
   222  
   223  	f.Forward(spMinMax, 1)
   224  	c.Assert(f.Frontier(), check.Equals, uint64(1))
   225  	c.Assert(f.String(), check.Equals, "[ @ 1] [\xff\xff\xff\xff\xff @ Max] ")
   226  	checkFrontier(c, f)
   227  
   228  	f.Forward(spMinMid, 2)
   229  	c.Assert(f.Frontier(), check.Equals, uint64(1))
   230  	c.Assert(f.String(), check.Equals, "[ @ 2] [m @ 1] [\xff\xff\xff\xff\xff @ Max] ")
   231  	checkFrontier(c, f)
   232  
   233  	f.Forward(spMidMax, 2)
   234  	c.Assert(f.Frontier(), check.Equals, uint64(2))
   235  	c.Assert(f.String(), check.Equals, "[ @ 2] [m @ 2] [\xff\xff\xff\xff\xff @ Max] ")
   236  	checkFrontier(c, f)
   237  
   238  	f.Forward(spMinMax, 3)
   239  	c.Assert(f.Frontier(), check.Equals, uint64(3))
   240  	c.Assert(f.String(), check.Equals, "[ @ 3] [\xff\xff\xff\xff\xff @ Max] ")
   241  	checkFrontier(c, f)
   242  }
   243  
   244  func (s *spanFrontierSuite) TestSpanFrontierDisjoinSpans(c *check.C) {
   245  	defer testleak.AfterTest(c)()
   246  	key1 := []byte("1")
   247  	key2 := []byte("2")
   248  	keyA := []byte("a")
   249  	keyB := []byte("b")
   250  	keyC := []byte("c")
   251  	keyD := []byte("d")
   252  	keyE := []byte("e")
   253  	keyF := []byte("f")
   254  
   255  	spAB := regionspan.ComparableSpan{Start: keyA, End: keyB}
   256  	spAD := regionspan.ComparableSpan{Start: keyA, End: keyD}
   257  	spAE := regionspan.ComparableSpan{Start: keyA, End: keyE}
   258  	spDE := regionspan.ComparableSpan{Start: keyD, End: keyE}
   259  	spCE := regionspan.ComparableSpan{Start: keyC, End: keyE}
   260  	sp12 := regionspan.ComparableSpan{Start: key1, End: key2}
   261  	sp1F := regionspan.ComparableSpan{Start: key1, End: keyF}
   262  
   263  	f := NewFrontier(0, spAB, spCE)
   264  	c.Assert(f.Frontier(), check.Equals, uint64(0))
   265  	c.Assert(f.String(), check.Equals, `[a @ 0] [b @ Max] [c @ 0] [e @ Max] `)
   266  	checkFrontier(c, f)
   267  
   268  	// Advance the tracked spans
   269  	f.Forward(spAB, 1)
   270  	c.Assert(f.Frontier(), check.Equals, uint64(0))
   271  	c.Assert(f.String(), check.Equals, `[a @ 1] [b @ Max] [c @ 0] [e @ Max] `)
   272  	checkFrontier(c, f)
   273  	f.Forward(spCE, 1)
   274  	c.Assert(f.Frontier(), check.Equals, uint64(1))
   275  	c.Assert(f.String(), check.Equals, `[a @ 1] [b @ Max] [c @ 1] [e @ Max] `)
   276  	checkFrontier(c, f)
   277  
   278  	// Advance d-e split c-e to c-d and d-e
   279  	f.Forward(spDE, 2)
   280  	c.Assert(f.Frontier(), check.Equals, uint64(1))
   281  	c.Assert(f.String(), check.Equals, `[a @ 1] [b @ Max] [c @ 1] [d @ 2] [e @ Max] `)
   282  	checkFrontier(c, f)
   283  
   284  	// Advance a-d cover a-b and c-d
   285  	f.Forward(spAD, 3)
   286  	c.Assert(f.Frontier(), check.Equals, uint64(2))
   287  	c.Assert(f.String(), check.Equals, `[a @ 3] [d @ 2] [e @ Max] `)
   288  	checkFrontier(c, f)
   289  
   290  	// Advance one cover all 3 span
   291  	f.Forward(spAE, 4)
   292  	c.Assert(f.Frontier(), check.Equals, uint64(4))
   293  	c.Assert(f.String(), check.Equals, `[a @ 4] [e @ Max] `)
   294  	checkFrontier(c, f)
   295  
   296  	// Advance all with a larger span
   297  	f.Forward(sp1F, 5)
   298  	c.Assert(f.Frontier(), check.Equals, uint64(5))
   299  	c.Assert(f.String(), check.Equals, `[1 @ 5] [f @ Max] `)
   300  	checkFrontier(c, f)
   301  
   302  	// Advance span smaller than all tracked spans
   303  	f.Forward(sp12, 6)
   304  	c.Assert(f.Frontier(), check.Equals, uint64(5))
   305  	c.Assert(f.String(), check.Equals, `[1 @ 6] [2 @ 5] [f @ Max] `)
   306  	checkFrontier(c, f)
   307  }
   308  
   309  func (s *spanFrontierSuite) TestSpanFrontierRandomly(c *check.C) {
   310  	defer testleak.AfterTest(c)()
   311  	var keyMin []byte
   312  	var keyMax []byte
   313  	spMinMax := regionspan.ComparableSpan{Start: keyMin, End: keyMax}
   314  	f := NewFrontier(0, spMinMax)
   315  
   316  	var spans []regionspan.ComparableSpan
   317  	for len(spans) < 500000 {
   318  		span := regionspan.ComparableSpan{
   319  			Start: make([]byte, rand.Intn(32)+1),
   320  			End:   make([]byte, rand.Intn(32)+1),
   321  		}
   322  		rand.Read(span.Start)
   323  		rand.Read(span.End)
   324  		cmp := bytes.Compare(span.Start, span.End)
   325  		if cmp == 0 {
   326  			continue
   327  		} else if cmp > 0 {
   328  			span.Start, span.End = span.End, span.Start
   329  		}
   330  
   331  		spans = append(spans, span)
   332  
   333  		ts := rand.Uint64()
   334  
   335  		f.Forward(span, ts)
   336  		checkFrontier(c, f)
   337  	}
   338  }
   339  
   340  func checkFrontier(c *check.C, f Frontier) {
   341  	sf := f.(*spanFrontier)
   342  	var tsInList, tsInHeap []uint64
   343  	sf.spanList.Entries(func(n *skipListNode) bool {
   344  		tsInList = append(tsInList, n.Value().key)
   345  		return true
   346  	})
   347  	sf.minTsHeap.Entries(func(n *fibonacciHeapNode) bool {
   348  		tsInHeap = append(tsInHeap, n.key)
   349  		return true
   350  	})
   351  	c.Assert(len(tsInList), check.Equals, len(tsInHeap))
   352  	sort.Slice(tsInList, func(i, j int) bool { return tsInList[i] < tsInList[j] })
   353  	sort.Slice(tsInHeap, func(i, j int) bool { return tsInHeap[i] < tsInHeap[j] })
   354  	c.Assert(tsInList, check.DeepEquals, tsInHeap)
   355  	c.Assert(f.Frontier(), check.Equals, tsInList[0])
   356  }