github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/orchestrator/reactor_state_test.go (about) 1 // Copyright 2021 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package orchestrator 15 16 import ( 17 "encoding/json" 18 "fmt" 19 "testing" 20 "time" 21 22 "github.com/google/go-cmp/cmp" 23 "github.com/google/go-cmp/cmp/cmpopts" 24 "github.com/pingcap/tiflow/cdc/model" 25 "github.com/pingcap/tiflow/pkg/config" 26 "github.com/pingcap/tiflow/pkg/etcd" 27 "github.com/pingcap/tiflow/pkg/orchestrator/util" 28 putil "github.com/pingcap/tiflow/pkg/util" 29 "github.com/stretchr/testify/require" 30 ) 31 32 func TestCheckCaptureAlive(t *testing.T) { 33 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 34 model.DefaultChangeFeedID("test")) 35 stateTester := NewReactorStateTester(t, state, nil) 36 state.CheckCaptureAlive("6bbc01c8-0605-4f86-a0f9-b3119109b225") 37 require.Contains(t, stateTester.ApplyPatches().Error(), "[CDC:ErrLeaseExpired]") 38 err := stateTester.Update(etcd.DefaultClusterAndMetaPrefix+ 39 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 40 []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`)) 41 require.Nil(t, err) 42 state.CheckCaptureAlive("6bbc01c8-0605-4f86-a0f9-b3119109b225") 43 stateTester.MustApplyPatches() 44 } 45 46 func TestChangefeedStateUpdate(t *testing.T) { 47 changefeedInfo := ` 48 { 49 "sink-uri": "blackhole://", 50 "opts": {}, 51 "create-time": "2020-02-02T00:00:00.000000+00:00", 52 "start-ts": 421980685886554116, 53 "target-ts": 0, 54 "admin-job-type": 0, 55 "sort-engine": "memory", 56 "sort-dir": "", 57 "config": { 58 "case-sensitive": true, 59 "force-replicate": false, 60 "check-gc-safe-point": true, 61 "filter": { 62 "rules": [ 63 "*.*" 64 ], 65 "ignore-txn-start-ts": null 66 }, 67 "mounter": { 68 "worker-num": 16 69 } 70 }, 71 "state": "normal", 72 "history": null, 73 "error": null, 74 "sync-point-enabled": false, 75 "sync-point-interval": 600000000000 76 } 77 ` 78 createTime, err := time.Parse("2006-01-02", "2020-02-02") 79 require.Nil(t, err) 80 testCases := []struct { 81 changefeedID string 82 updateKey []string 83 updateValue []string 84 expected ChangefeedReactorState 85 }{ 86 { // common case 87 changefeedID: "test1", 88 updateKey: []string{ 89 etcd.DefaultClusterAndNamespacePrefix + 90 "/changefeed/info/test1", 91 etcd.DefaultClusterAndNamespacePrefix + 92 "/changefeed/status/test1", 93 etcd.DefaultClusterAndNamespacePrefix + 94 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 95 etcd.DefaultClusterAndMetaPrefix + 96 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 97 }, 98 updateValue: []string{ 99 changefeedInfo, 100 `{"checkpoint-ts":421980719742451713,"admin-job-type":0}`, 101 `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, 102 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 103 }, 104 expected: ChangefeedReactorState{ 105 ClusterID: etcd.DefaultCDCClusterID, 106 ID: model.DefaultChangeFeedID("test1"), 107 Info: &model.ChangeFeedInfo{ 108 SinkURI: "blackhole://", 109 CreateTime: createTime, 110 StartTs: 421980685886554116, 111 Engine: model.SortInMemory, 112 State: "normal", 113 Config: &config.ReplicaConfig{ 114 CaseSensitive: true, 115 CheckGCSafePoint: true, 116 Filter: &config.FilterConfig{Rules: []string{"*.*"}}, 117 Mounter: &config.MounterConfig{WorkerNum: 16}, 118 Scheduler: config.GetDefaultReplicaConfig().Scheduler, 119 Sink: &config.SinkConfig{ 120 Terminator: putil.AddressOf(config.CRLF), 121 AdvanceTimeoutInSec: putil.AddressOf(uint(150)), 122 CSVConfig: config.GetDefaultReplicaConfig().Sink.CSVConfig, 123 EncoderConcurrency: config.GetDefaultReplicaConfig().Sink.EncoderConcurrency, 124 DateSeparator: config.GetDefaultReplicaConfig().Sink.DateSeparator, 125 EnablePartitionSeparator: config.GetDefaultReplicaConfig().Sink.EnablePartitionSeparator, 126 EnableKafkaSinkV2: config.GetDefaultReplicaConfig().Sink.EnableKafkaSinkV2, 127 OnlyOutputUpdatedColumns: config.GetDefaultReplicaConfig().Sink.OnlyOutputUpdatedColumns, 128 DeleteOnlyOutputHandleKeyColumns: config.GetDefaultReplicaConfig().Sink.DeleteOnlyOutputHandleKeyColumns, 129 ContentCompatible: config.GetDefaultReplicaConfig().Sink.ContentCompatible, 130 SendBootstrapIntervalInSec: config.GetDefaultReplicaConfig().Sink.SendBootstrapIntervalInSec, 131 SendBootstrapInMsgCount: config.GetDefaultReplicaConfig().Sink.SendBootstrapInMsgCount, 132 SendBootstrapToAllPartition: config.GetDefaultReplicaConfig().Sink.SendBootstrapToAllPartition, 133 DebeziumDisableSchema: config.GetDefaultReplicaConfig().Sink.DebeziumDisableSchema, 134 Debezium: config.GetDefaultReplicaConfig().Sink.Debezium, 135 OpenProtocol: config.GetDefaultReplicaConfig().Sink.OpenProtocol, 136 }, 137 Consistent: config.GetDefaultReplicaConfig().Consistent, 138 Integrity: config.GetDefaultReplicaConfig().Integrity, 139 ChangefeedErrorStuckDuration: config. 140 GetDefaultReplicaConfig().ChangefeedErrorStuckDuration, 141 SyncedStatus: config.GetDefaultReplicaConfig().SyncedStatus, 142 }, 143 }, 144 Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713}, 145 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 146 "6bbc01c8-0605-4f86-a0f9-b3119109b225": {CheckPointTs: 421980720003809281, ResolvedTs: 421980720003809281}, 147 }, 148 }, 149 }, 150 { // test multiple capture 151 changefeedID: "test1", 152 updateKey: []string{ 153 etcd.DefaultClusterAndNamespacePrefix + 154 "/changefeed/info/test1", 155 etcd.DefaultClusterAndNamespacePrefix + 156 "/changefeed/status/test1", 157 etcd.DefaultClusterAndNamespacePrefix + 158 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 159 etcd.DefaultClusterAndMetaPrefix + 160 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 161 etcd.DefaultClusterAndNamespacePrefix + 162 "/task/position/666777888/test1", 163 etcd.DefaultClusterAndMetaPrefix + 164 "/capture/666777888", 165 }, 166 updateValue: []string{ 167 changefeedInfo, 168 `{"checkpoint-ts":421980719742451713,"admin-job-type":0}`, 169 `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, 170 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 171 `{"checkpoint-ts":11332244,"resolved-ts":312321,"count":8,"error":null}`, 172 `{"id":"666777888","address":"127.0.0.1:8300"}`, 173 }, 174 expected: ChangefeedReactorState{ 175 ClusterID: etcd.DefaultCDCClusterID, 176 ID: model.DefaultChangeFeedID("test1"), 177 Info: &model.ChangeFeedInfo{ 178 SinkURI: "blackhole://", 179 CreateTime: createTime, 180 StartTs: 421980685886554116, 181 Engine: model.SortInMemory, 182 State: "normal", 183 Config: &config.ReplicaConfig{ 184 CaseSensitive: true, 185 CheckGCSafePoint: true, 186 Filter: &config.FilterConfig{Rules: []string{"*.*"}}, 187 Mounter: &config.MounterConfig{WorkerNum: 16}, 188 Sink: &config.SinkConfig{ 189 Terminator: putil.AddressOf(config.CRLF), 190 AdvanceTimeoutInSec: putil.AddressOf(uint(150)), 191 CSVConfig: config.GetDefaultReplicaConfig().Sink.CSVConfig, 192 EncoderConcurrency: config.GetDefaultReplicaConfig().Sink.EncoderConcurrency, 193 DateSeparator: config.GetDefaultReplicaConfig().Sink.DateSeparator, 194 EnablePartitionSeparator: config.GetDefaultReplicaConfig().Sink.EnablePartitionSeparator, 195 EnableKafkaSinkV2: config.GetDefaultReplicaConfig().Sink.EnableKafkaSinkV2, 196 OnlyOutputUpdatedColumns: config.GetDefaultReplicaConfig().Sink.OnlyOutputUpdatedColumns, 197 DeleteOnlyOutputHandleKeyColumns: config.GetDefaultReplicaConfig().Sink.DeleteOnlyOutputHandleKeyColumns, 198 ContentCompatible: config.GetDefaultReplicaConfig().Sink.ContentCompatible, 199 SendBootstrapIntervalInSec: config.GetDefaultReplicaConfig().Sink.SendBootstrapIntervalInSec, 200 SendBootstrapInMsgCount: config.GetDefaultReplicaConfig().Sink.SendBootstrapInMsgCount, 201 SendBootstrapToAllPartition: config.GetDefaultReplicaConfig().Sink.SendBootstrapToAllPartition, 202 DebeziumDisableSchema: config.GetDefaultReplicaConfig().Sink.DebeziumDisableSchema, 203 Debezium: config.GetDefaultReplicaConfig().Sink.Debezium, 204 OpenProtocol: config.GetDefaultReplicaConfig().Sink.OpenProtocol, 205 }, 206 Scheduler: config.GetDefaultReplicaConfig().Scheduler, 207 Integrity: config.GetDefaultReplicaConfig().Integrity, 208 Consistent: config.GetDefaultReplicaConfig().Consistent, 209 ChangefeedErrorStuckDuration: config. 210 GetDefaultReplicaConfig().ChangefeedErrorStuckDuration, 211 SyncedStatus: config.GetDefaultReplicaConfig().SyncedStatus, 212 }, 213 }, 214 Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713}, 215 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 216 "6bbc01c8-0605-4f86-a0f9-b3119109b225": {CheckPointTs: 421980720003809281, ResolvedTs: 421980720003809281}, 217 "666777888": {CheckPointTs: 11332244, ResolvedTs: 312321, Count: 8}, 218 }, 219 }, 220 }, 221 { // testing changefeedID not match 222 changefeedID: "test1", 223 updateKey: []string{ 224 etcd.DefaultClusterAndNamespacePrefix + 225 "/changefeed/info/test1", 226 etcd.DefaultClusterAndNamespacePrefix + 227 "/changefeed/status/test1", 228 229 etcd.DefaultClusterAndNamespacePrefix + 230 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 231 etcd.DefaultClusterAndMetaPrefix + 232 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 233 etcd.DefaultClusterAndNamespacePrefix + 234 "/changefeed/info/test-fake", 235 etcd.DefaultClusterAndNamespacePrefix + 236 "/changefeed/status/test-fake", 237 etcd.DefaultClusterAndNamespacePrefix + 238 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", 239 }, 240 updateValue: []string{ 241 changefeedInfo, 242 `{"checkpoint-ts":421980719742451713,"admin-job-type":0}`, 243 `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, 244 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 245 `fake value`, 246 `fake value`, 247 `fake value`, 248 }, 249 expected: ChangefeedReactorState{ 250 ClusterID: etcd.DefaultCDCClusterID, 251 ID: model.DefaultChangeFeedID("test1"), 252 Info: &model.ChangeFeedInfo{ 253 SinkURI: "blackhole://", 254 CreateTime: createTime, 255 StartTs: 421980685886554116, 256 Engine: model.SortInMemory, 257 State: "normal", 258 Config: &config.ReplicaConfig{ 259 CaseSensitive: true, 260 CheckGCSafePoint: true, 261 Filter: &config.FilterConfig{Rules: []string{"*.*"}}, 262 Mounter: &config.MounterConfig{WorkerNum: 16}, 263 Sink: &config.SinkConfig{ 264 Terminator: putil.AddressOf(config.CRLF), 265 AdvanceTimeoutInSec: putil.AddressOf(uint(150)), 266 EncoderConcurrency: config.GetDefaultReplicaConfig().Sink.EncoderConcurrency, 267 CSVConfig: config.GetDefaultReplicaConfig().Sink.CSVConfig, 268 DateSeparator: config.GetDefaultReplicaConfig().Sink.DateSeparator, 269 EnablePartitionSeparator: config.GetDefaultReplicaConfig().Sink.EnablePartitionSeparator, 270 EnableKafkaSinkV2: config.GetDefaultReplicaConfig().Sink.EnableKafkaSinkV2, 271 OnlyOutputUpdatedColumns: config.GetDefaultReplicaConfig().Sink.OnlyOutputUpdatedColumns, 272 DeleteOnlyOutputHandleKeyColumns: config.GetDefaultReplicaConfig().Sink.DeleteOnlyOutputHandleKeyColumns, 273 ContentCompatible: config.GetDefaultReplicaConfig().Sink.ContentCompatible, 274 SendBootstrapIntervalInSec: config.GetDefaultReplicaConfig().Sink.SendBootstrapIntervalInSec, 275 SendBootstrapInMsgCount: config.GetDefaultReplicaConfig().Sink.SendBootstrapInMsgCount, 276 SendBootstrapToAllPartition: config.GetDefaultReplicaConfig().Sink.SendBootstrapToAllPartition, 277 DebeziumDisableSchema: config.GetDefaultReplicaConfig().Sink.DebeziumDisableSchema, 278 Debezium: config.GetDefaultReplicaConfig().Sink.Debezium, 279 OpenProtocol: config.GetDefaultReplicaConfig().Sink.OpenProtocol, 280 }, 281 Consistent: config.GetDefaultReplicaConfig().Consistent, 282 Scheduler: config.GetDefaultReplicaConfig().Scheduler, 283 Integrity: config.GetDefaultReplicaConfig().Integrity, 284 ChangefeedErrorStuckDuration: config. 285 GetDefaultReplicaConfig().ChangefeedErrorStuckDuration, 286 SyncedStatus: config.GetDefaultReplicaConfig().SyncedStatus, 287 }, 288 }, 289 Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713}, 290 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 291 "6bbc01c8-0605-4f86-a0f9-b3119109b225": {CheckPointTs: 421980720003809281, ResolvedTs: 421980720003809281}, 292 }, 293 }, 294 }, 295 { // testing value is nil 296 changefeedID: "test1", 297 updateKey: []string{ 298 etcd.DefaultClusterAndNamespacePrefix + 299 "/changefeed/info/test1", 300 etcd.DefaultClusterAndNamespacePrefix + 301 "/changefeed/status/test1", 302 etcd.DefaultClusterAndNamespacePrefix + 303 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 304 etcd.DefaultClusterAndMetaPrefix + 305 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 306 etcd.DefaultClusterAndNamespacePrefix + 307 "/task/position/666777888/test1", 308 etcd.DefaultClusterAndNamespacePrefix + 309 "/changefeed/info/test1", 310 etcd.DefaultClusterAndNamespacePrefix + 311 "/changefeed/status/test1", 312 etcd.DefaultClusterAndNamespacePrefix + 313 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 314 etcd.DefaultClusterAndMetaPrefix + 315 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 316 }, 317 updateValue: []string{ 318 changefeedInfo, 319 `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`, 320 `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, 321 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 322 `{"checkpoint-ts":11332244,"resolved-ts":312321,"count":8,"error":null}`, 323 ``, 324 ``, 325 ``, 326 ``, 327 ``, 328 ``, 329 }, 330 expected: ChangefeedReactorState{ 331 ClusterID: etcd.DefaultCDCClusterID, 332 ID: model.DefaultChangeFeedID("test1"), 333 Info: nil, 334 Status: nil, 335 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 336 "666777888": {CheckPointTs: 11332244, ResolvedTs: 312321, Count: 8}, 337 }, 338 }, 339 }, 340 } 341 for i, tc := range testCases { 342 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 343 model.DefaultChangeFeedID(tc.changefeedID)) 344 for i, k := range tc.updateKey { 345 value := []byte(tc.updateValue[i]) 346 if len(value) == 0 { 347 value = nil 348 } 349 err = state.Update(util.NewEtcdKey(k), value, false) 350 require.Nil(t, err) 351 } 352 require.True(t, cmp.Equal( 353 state, &tc.expected, 354 cmpopts.IgnoreUnexported(ChangefeedReactorState{}), 355 ), 356 fmt.Sprintf("%d,%s", i, cmp.Diff(state, &tc.expected, cmpopts.IgnoreUnexported(ChangefeedReactorState{})))) 357 } 358 } 359 360 func TestPatchInfo(t *testing.T) { 361 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 362 model.DefaultChangeFeedID("test1")) 363 stateTester := NewReactorStateTester(t, state, nil) 364 state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { 365 require.Nil(t, info) 366 return &model.ChangeFeedInfo{SinkURI: "123", Config: &config.ReplicaConfig{}}, true, nil 367 }) 368 stateTester.MustApplyPatches() 369 defaultConfig := config.GetDefaultReplicaConfig() 370 cfInfo := &model.ChangeFeedInfo{ 371 SinkURI: "123", 372 Engine: model.SortUnified, 373 Config: &config.ReplicaConfig{ 374 Filter: defaultConfig.Filter, 375 Mounter: defaultConfig.Mounter, 376 Sink: defaultConfig.Sink, 377 Consistent: defaultConfig.Consistent, 378 Scheduler: defaultConfig.Scheduler, 379 Integrity: defaultConfig.Integrity, 380 ChangefeedErrorStuckDuration: defaultConfig.ChangefeedErrorStuckDuration, 381 SyncedStatus: defaultConfig.SyncedStatus, 382 }, 383 } 384 cfInfo.RmUnusedFields() 385 require.Equal(t, state.Info, cfInfo) 386 387 state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { 388 info.StartTs = 6 389 return info, true, nil 390 }) 391 stateTester.MustApplyPatches() 392 cfInfo = &model.ChangeFeedInfo{ 393 SinkURI: "123", 394 StartTs: 6, 395 Engine: model.SortUnified, 396 Config: &config.ReplicaConfig{ 397 Filter: defaultConfig.Filter, 398 Mounter: defaultConfig.Mounter, 399 Sink: defaultConfig.Sink, 400 Consistent: defaultConfig.Consistent, 401 Scheduler: defaultConfig.Scheduler, 402 Integrity: defaultConfig.Integrity, 403 ChangefeedErrorStuckDuration: defaultConfig.ChangefeedErrorStuckDuration, 404 SyncedStatus: defaultConfig.SyncedStatus, 405 }, 406 } 407 cfInfo.RmUnusedFields() 408 require.Equal(t, state.Info, cfInfo) 409 410 state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { 411 return nil, true, nil 412 }) 413 stateTester.MustApplyPatches() 414 require.Nil(t, state.Info) 415 } 416 417 func TestPatchStatus(t *testing.T) { 418 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 419 model.DefaultChangeFeedID("test1")) 420 stateTester := NewReactorStateTester(t, state, nil) 421 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 422 require.Nil(t, status) 423 return &model.ChangeFeedStatus{CheckpointTs: 5}, true, nil 424 }) 425 stateTester.MustApplyPatches() 426 require.Equal(t, state.Status, &model.ChangeFeedStatus{CheckpointTs: 5}) 427 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 428 status.CheckpointTs = 6 429 return status, true, nil 430 }) 431 stateTester.MustApplyPatches() 432 require.Equal(t, state.Status, &model.ChangeFeedStatus{CheckpointTs: 6}) 433 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 434 return nil, true, nil 435 }) 436 stateTester.MustApplyPatches() 437 require.Nil(t, state.Status) 438 } 439 440 func TestPatchTaskPosition(t *testing.T) { 441 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 442 model.DefaultChangeFeedID("test1")) 443 stateTester := NewReactorStateTester(t, state, nil) 444 captureID1 := "capture1" 445 captureID2 := "capture2" 446 state.PatchTaskPosition(captureID1, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 447 require.Nil(t, position) 448 return &model.TaskPosition{ 449 CheckPointTs: 1, 450 }, true, nil 451 }) 452 state.PatchTaskPosition(captureID2, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 453 require.Nil(t, position) 454 return &model.TaskPosition{ 455 CheckPointTs: 2, 456 }, true, nil 457 }) 458 stateTester.MustApplyPatches() 459 require.Equal(t, state.TaskPositions, map[string]*model.TaskPosition{ 460 captureID1: { 461 CheckPointTs: 1, 462 }, 463 captureID2: { 464 CheckPointTs: 2, 465 }, 466 }) 467 state.PatchTaskPosition(captureID1, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 468 position.CheckPointTs = 3 469 return position, true, nil 470 }) 471 state.PatchTaskPosition(captureID2, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 472 position.ResolvedTs = 2 473 return position, true, nil 474 }) 475 stateTester.MustApplyPatches() 476 require.Equal(t, state.TaskPositions, map[string]*model.TaskPosition{ 477 captureID1: { 478 CheckPointTs: 3, 479 }, 480 captureID2: { 481 CheckPointTs: 2, 482 ResolvedTs: 2, 483 }, 484 }) 485 state.PatchTaskPosition(captureID1, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 486 return nil, false, nil 487 }) 488 state.PatchTaskPosition(captureID2, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 489 return nil, true, nil 490 }) 491 state.PatchTaskPosition(captureID1, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { 492 position.Count = 6 493 return position, true, nil 494 }) 495 stateTester.MustApplyPatches() 496 require.Equal(t, state.TaskPositions, map[string]*model.TaskPosition{ 497 captureID1: { 498 CheckPointTs: 3, 499 Count: 6, 500 }, 501 }) 502 } 503 504 func TestGlobalStateUpdate(t *testing.T) { 505 t.Parallel() 506 507 testCases := []struct { 508 updateKey []string 509 updateValue []string 510 expected GlobalReactorState 511 timeout int 512 }{ 513 { // common case 514 updateKey: []string{ 515 etcd.DefaultClusterAndMetaPrefix + 516 "/owner/22317526c4fc9a37", 517 etcd.DefaultClusterAndMetaPrefix + 518 "/owner/22317526c4fc9a38", 519 etcd.DefaultClusterAndMetaPrefix + 520 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 521 etcd.DefaultClusterAndNamespacePrefix + 522 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 523 etcd.DefaultClusterAndNamespacePrefix + 524 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", 525 etcd.DefaultClusterAndNamespacePrefix + 526 "/upstream/12345", 527 }, 528 updateValue: []string{ 529 `6bbc01c8-0605-4f86-a0f9-b3119109b225`, 530 `55551111`, 531 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 532 `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713, 533 "admin-job-type":0}`, 534 `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713, 535 "admin-job-type":0}`, 536 `{}`, 537 }, 538 expected: GlobalReactorState{ 539 ClusterID: etcd.DefaultCDCClusterID, 540 Owner: map[string]struct{}{"22317526c4fc9a37": {}, "22317526c4fc9a38": {}}, 541 Captures: map[model.CaptureID]*model.CaptureInfo{"6bbc01c8-0605-4f86-a0f9-b3119109b225": { 542 ID: "6bbc01c8-0605-4f86-a0f9-b3119109b225", 543 AdvertiseAddr: "127.0.0.1:8300", 544 }}, 545 Upstreams: map[model.UpstreamID]*model.UpstreamInfo{ 546 model.UpstreamID(12345): {}, 547 }, 548 Changefeeds: map[model.ChangeFeedID]*ChangefeedReactorState{ 549 model.DefaultChangeFeedID("test1"): { 550 ClusterID: etcd.DefaultCDCClusterID, 551 ID: model.DefaultChangeFeedID("test1"), 552 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 553 "6bbc01c8-0605-4f86-a0f9-b3119109b225": {CheckPointTs: 421980719742451713, ResolvedTs: 421980720003809281}, 554 }, 555 }, 556 model.DefaultChangeFeedID("test2"): { 557 ClusterID: etcd.DefaultCDCClusterID, 558 ID: model.DefaultChangeFeedID("test2"), 559 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 560 "6bbc01c8-0605-4f86-a0f9-b3119109b225": { 561 CheckPointTs: 421980719742451713, 562 ResolvedTs: 421980720003809281, 563 }, 564 }, 565 }, 566 }, 567 }, 568 }, 569 { // testing remove changefeed 570 updateKey: []string{ 571 etcd.DefaultClusterAndMetaPrefix + 572 "/owner/22317526c4fc9a37", 573 etcd.DefaultClusterAndMetaPrefix + 574 "/owner/22317526c4fc9a38", 575 etcd.DefaultClusterAndMetaPrefix + 576 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 577 etcd.DefaultClusterAndNamespacePrefix + 578 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 579 etcd.DefaultClusterAndNamespacePrefix + 580 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", 581 etcd.DefaultClusterAndMetaPrefix + 582 "/owner/22317526c4fc9a37", 583 etcd.DefaultClusterAndNamespacePrefix + 584 "/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", 585 etcd.DefaultClusterAndMetaPrefix + 586 "/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", 587 }, 588 updateValue: []string{ 589 `6bbc01c8-0605-4f86-a0f9-b3119109b225`, 590 `55551111`, 591 `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, 592 `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713, 593 "admin-job-type":0}`, 594 `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713, 595 "admin-job-type":0}`, 596 ``, 597 ``, 598 ``, 599 }, 600 timeout: 6, 601 expected: GlobalReactorState{ 602 ClusterID: etcd.DefaultCDCClusterID, 603 Owner: map[string]struct{}{"22317526c4fc9a38": {}}, 604 Captures: map[model.CaptureID]*model.CaptureInfo{}, 605 Upstreams: map[model.UpstreamID]*model.UpstreamInfo{}, 606 Changefeeds: map[model.ChangeFeedID]*ChangefeedReactorState{ 607 model.DefaultChangeFeedID("test2"): { 608 ClusterID: etcd.DefaultCDCClusterID, 609 ID: model.DefaultChangeFeedID("test2"), 610 TaskPositions: map[model.CaptureID]*model.TaskPosition{ 611 "6bbc01c8-0605-4f86-a0f9-b3119109b225": { 612 CheckPointTs: 421980719742451713, 613 ResolvedTs: 421980720003809281, 614 }, 615 }, 616 }, 617 }, 618 }, 619 }, 620 } 621 for _, tc := range testCases { 622 state := NewGlobalState(etcd.DefaultCDCClusterID, 10) 623 for i, k := range tc.updateKey { 624 value := []byte(tc.updateValue[i]) 625 if len(value) == 0 { 626 value = nil 627 } 628 err := state.Update(util.NewEtcdKey(k), value, false) 629 require.Nil(t, err) 630 } 631 time.Sleep(time.Duration(tc.timeout) * time.Second) 632 state.UpdatePendingChange() 633 require.True(t, cmp.Equal(state, &tc.expected, cmpopts.IgnoreUnexported(GlobalReactorState{}, ChangefeedReactorState{})), 634 cmp.Diff(state, &tc.expected, cmpopts.IgnoreUnexported(GlobalReactorState{}, ChangefeedReactorState{}))) 635 } 636 } 637 638 func TestCaptureChangeHooks(t *testing.T) { 639 t.Parallel() 640 641 state := NewGlobalState(etcd.DefaultCDCClusterID, 10) 642 643 var callCount int 644 state.onCaptureAdded = func(captureID model.CaptureID, addr string) { 645 callCount++ 646 require.Equal(t, captureID, "capture-1") 647 require.Equal(t, addr, "ip-1:8300") 648 } 649 state.onCaptureRemoved = func(captureID model.CaptureID) { 650 callCount++ 651 require.Equal(t, captureID, "capture-1") 652 } 653 654 captureInfo := &model.CaptureInfo{ 655 ID: "capture-1", 656 AdvertiseAddr: "ip-1:8300", 657 } 658 captureInfoBytes, err := json.Marshal(captureInfo) 659 require.Nil(t, err) 660 661 err = state.Update(util.NewEtcdKey( 662 etcd.CaptureInfoKeyPrefix(etcd.DefaultCDCClusterID)+"/capture-1"), 663 captureInfoBytes, false) 664 require.Nil(t, err) 665 require.Eventually(t, func() bool { 666 return callCount == 1 667 }, time.Second*3, 10*time.Millisecond) 668 669 err = state.Update(util.NewEtcdKey( 670 etcd.CaptureInfoKeyPrefix(etcd.DefaultCDCClusterID)+"/capture-1"), 671 nil /* delete */, false) 672 require.Nil(t, err) 673 require.Eventually(t, func() bool { 674 state.UpdatePendingChange() 675 return callCount == 2 676 }, time.Second*10, 10*time.Millisecond) 677 } 678 679 func TestCheckChangefeedNormal(t *testing.T) { 680 state := NewChangefeedReactorState(etcd.DefaultCDCClusterID, 681 model.DefaultChangeFeedID("test1")) 682 stateTester := NewReactorStateTester(t, state, nil) 683 state.CheckChangefeedNormal() 684 stateTester.MustApplyPatches() 685 state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { 686 return &model.ChangeFeedInfo{SinkURI: "123", AdminJobType: model.AdminNone, Config: &config.ReplicaConfig{}}, true, nil 687 }) 688 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 689 return &model.ChangeFeedStatus{CheckpointTs: 1, AdminJobType: model.AdminNone}, true, nil 690 }) 691 state.CheckChangefeedNormal() 692 stateTester.MustApplyPatches() 693 require.Equal(t, state.Status.CheckpointTs, uint64(1)) 694 695 state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { 696 info.AdminJobType = model.AdminStop 697 return info, true, nil 698 }) 699 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 700 status.CheckpointTs = 2 701 return status, true, nil 702 }) 703 state.CheckChangefeedNormal() 704 stateTester.MustApplyPatches() 705 require.Equal(t, state.Status.CheckpointTs, uint64(1)) 706 707 state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { 708 status.CheckpointTs = 2 709 return status, true, nil 710 }) 711 state.CheckChangefeedNormal() 712 stateTester.MustApplyPatches() 713 require.Equal(t, state.Status.CheckpointTs, uint64(2)) 714 }