github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/cdc/processor/pipeline/cyclic_mark_test.go (about) 1 // Copyright 2020 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package pipeline 15 16 import ( 17 "context" 18 "sort" 19 "sync" 20 21 "github.com/google/go-cmp/cmp" 22 "github.com/pingcap/check" 23 "github.com/pingcap/ticdc/cdc/model" 24 "github.com/pingcap/ticdc/pkg/config" 25 cdcContext "github.com/pingcap/ticdc/pkg/context" 26 "github.com/pingcap/ticdc/pkg/cyclic/mark" 27 "github.com/pingcap/ticdc/pkg/pipeline" 28 "github.com/pingcap/ticdc/pkg/util/testleak" 29 "github.com/pingcap/tidb/tablecodec" 30 ) 31 32 type markSuite struct{} 33 34 var _ = check.Suite(&markSuite{}) 35 36 func (s *markSuite) TestCyclicMarkNode(c *check.C) { 37 defer testleak.AfterTest(c)() 38 markTableID := model.TableID(161025) 39 testCases := []struct { 40 input []*model.RowChangedEvent 41 expected []*model.RowChangedEvent 42 filterID []uint64 43 replicaID uint64 44 }{ 45 { 46 input: []*model.RowChangedEvent{}, 47 expected: []*model.RowChangedEvent{}, 48 filterID: []uint64{}, 49 replicaID: 1, 50 }, 51 { 52 input: []*model.RowChangedEvent{{Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}}, 53 expected: []*model.RowChangedEvent{{Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}}, 54 filterID: []uint64{}, 55 replicaID: 1, 56 }, 57 { 58 input: []*model.RowChangedEvent{ 59 {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, 60 }, 61 expected: []*model.RowChangedEvent{}, 62 filterID: []uint64{}, 63 replicaID: 1, 64 }, 65 { 66 input: []*model.RowChangedEvent{ 67 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, 68 {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, 69 }, 70 expected: []*model.RowChangedEvent{}, 71 filterID: []uint64{10}, 72 replicaID: 1, 73 }, 74 { 75 input: []*model.RowChangedEvent{ 76 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, 77 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, 78 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, 79 {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, 80 {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, 81 }, 82 expected: []*model.RowChangedEvent{ 83 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 11}, 84 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, 85 }, 86 filterID: []uint64{10}, 87 replicaID: 1, 88 }, 89 { 90 input: []*model.RowChangedEvent{ 91 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, 92 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, 93 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, 94 {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, 95 {StartTs: 1, CommitTs: 5, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, 96 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5}, 97 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5}, 98 }, 99 expected: []*model.RowChangedEvent{ 100 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}, 101 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, 102 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5, ReplicaID: 11}, 103 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5, ReplicaID: 1}, 104 }, 105 filterID: []uint64{10}, 106 replicaID: 1, 107 }, 108 { 109 input: []*model.RowChangedEvent{ 110 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, 111 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, 112 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, 113 {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, 114 {StartTs: 1, CommitTs: 5, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, 115 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5}, 116 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5}, 117 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 5, CommitTs: 8}, 118 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 8}, 119 {StartTs: 5, CommitTs: 8, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(12)}}}, 120 }, 121 expected: []*model.RowChangedEvent{ 122 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}, 123 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, 124 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5, ReplicaID: 1}, 125 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 8, ReplicaID: 1}, 126 {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 5, CommitTs: 8, ReplicaID: 12}, 127 }, 128 filterID: []uint64{10, 11}, 129 replicaID: 1, 130 }, 131 } 132 133 for _, tc := range testCases { 134 ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{}) 135 ctx = cdcContext.WithChangefeedVars(ctx, &cdcContext.ChangefeedVars{ 136 Info: &model.ChangeFeedInfo{ 137 Config: &config.ReplicaConfig{ 138 Cyclic: &config.CyclicConfig{ 139 Enable: true, 140 ReplicaID: tc.replicaID, 141 FilterReplicaID: tc.filterID, 142 }, 143 }, 144 }, 145 }) 146 n := newCyclicMarkNode(markTableID) 147 err := n.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil)) 148 c.Assert(err, check.IsNil) 149 outputCh := make(chan pipeline.Message) 150 var wg sync.WaitGroup 151 wg.Add(2) 152 go func() { 153 defer wg.Done() 154 defer close(outputCh) 155 var lastCommitTs model.Ts 156 for _, row := range tc.input { 157 event := model.NewPolymorphicEvent(&model.RawKVEntry{ 158 OpType: model.OpTypePut, 159 Key: tablecodec.GenTableRecordPrefix(row.Table.TableID), 160 StartTs: row.StartTs, 161 CRTs: row.CommitTs, 162 }) 163 event.Row = row 164 err := n.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(event), outputCh)) 165 c.Assert(err, check.IsNil) 166 lastCommitTs = row.CommitTs 167 } 168 err := n.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, lastCommitTs+1)), outputCh)) 169 c.Assert(err, check.IsNil) 170 }() 171 output := []*model.RowChangedEvent{} 172 go func() { 173 defer wg.Done() 174 for row := range outputCh { 175 if row.PolymorphicEvent.RawKV.OpType == model.OpTypeResolved { 176 continue 177 } 178 row.PolymorphicEvent.Row.ReplicaID = row.PolymorphicEvent.ReplicaID 179 output = append(output, row.PolymorphicEvent.Row) 180 } 181 }() 182 wg.Wait() 183 // check the commitTs is increasing 184 var lastCommitTs model.Ts 185 for _, event := range output { 186 c.Assert(event.CommitTs, check.GreaterEqual, lastCommitTs) 187 lastCommitTs = event.CommitTs 188 } 189 sort.Slice(output, func(i, j int) bool { 190 if output[i].CommitTs == output[j].CommitTs { 191 return output[i].StartTs < output[j].StartTs 192 } 193 return output[i].CommitTs < output[j].CommitTs 194 }) 195 c.Assert(output, check.DeepEquals, tc.expected, 196 check.Commentf("%s", cmp.Diff(output, tc.expected))) 197 } 198 }