vitess.io/vitess@v0.16.2/go/vt/mysqlctl/binlogs_gtid_test.go (about) 1 // Package mysqlctl_test is the blackbox tests for package mysqlctl. 2 // Tests that need to use fakemysqldaemon must be written as blackbox tests; 3 // since fakemysqldaemon imports mysqlctl, importing fakemysqldaemon in 4 // a `package mysqlctl` test would cause a circular import. 5 package mysqlctl 6 7 import ( 8 "context" 9 "fmt" 10 "testing" 11 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/require" 14 15 "vitess.io/vitess/go/mysql" 16 ) 17 18 func TestChooseBinlogsForIncrementalBackup(t *testing.T) { 19 binlogs := []string{ 20 "vt-bin.000001", 21 "vt-bin.000002", 22 "vt-bin.000003", 23 "vt-bin.000004", 24 "vt-bin.000005", 25 "vt-bin.000006", 26 } 27 basePreviousGTIDs := map[string]string{ 28 "vt-bin.000001": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-50", 29 "vt-bin.000002": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 30 "vt-bin.000003": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 31 "vt-bin.000004": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-78", 32 "vt-bin.000005": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-243", 33 "vt-bin.000006": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-331", 34 } 35 tt := []struct { 36 previousGTIDs map[string]string 37 backupPos string 38 expectBinlogs []string 39 expectError string 40 }{ 41 { 42 previousGTIDs: basePreviousGTIDs, 43 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-78", 44 expectBinlogs: []string{"vt-bin.000004", "vt-bin.000005"}, 45 }, 46 { 47 previousGTIDs: basePreviousGTIDs, 48 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 49 expectBinlogs: []string{"vt-bin.000003", "vt-bin.000004", "vt-bin.000005"}, 50 }, 51 { 52 previousGTIDs: basePreviousGTIDs, 53 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-63", 54 expectBinlogs: []string{"vt-bin.000003", "vt-bin.000004", "vt-bin.000005"}, 55 }, 56 { 57 previousGTIDs: basePreviousGTIDs, 58 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-243", 59 expectBinlogs: []string{"vt-bin.000005"}, 60 }, 61 { 62 previousGTIDs: basePreviousGTIDs, 63 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-331", 64 expectError: "no binary logs to backup", 65 }, 66 { 67 previousGTIDs: basePreviousGTIDs, 68 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-630000", 69 expectError: "no binary logs to backup", 70 }, 71 { 72 previousGTIDs: basePreviousGTIDs, 73 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f0000:1-63", 74 expectError: "There are GTID entries that are missing", 75 }, 76 { 77 previousGTIDs: map[string]string{ 78 "vt-bin.000001": "", 79 "vt-bin.000002": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 80 "vt-bin.000003": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 81 "vt-bin.000004": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-78", 82 "vt-bin.000005": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-243", 83 "vt-bin.000006": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-331", 84 }, 85 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f0000:1-63", 86 expectError: "neither contains requested GTID", 87 }, 88 { 89 previousGTIDs: map[string]string{ 90 "vt-bin.000001": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-50", 91 "vt-bin.000002": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 92 "vt-bin.000003": "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-60", 93 "vt-bin.000004": "16b1039f-22b6-11ed-b765-0a43f95f28a3:3-78", 94 "vt-bin.000005": "16b1039f-22b6-11ed-b765-0a43f95f28a3:20-243", 95 "vt-bin.000006": "16b1039f-22b6-11ed-b765-0a43f95f28a3:200-331", 96 }, 97 backupPos: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-63", 98 expectBinlogs: []string{"vt-bin.000003", "vt-bin.000004", "vt-bin.000005"}, 99 }, 100 } 101 for i, tc := range tt { 102 t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { 103 backupPos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, tc.backupPos) 104 require.NoError(t, err) 105 require.NoError(t, err) 106 binlogsToBackup, fromGTID, toGTID, err := ChooseBinlogsForIncrementalBackup( 107 context.Background(), 108 backupPos.GTIDSet, 109 binlogs, 110 func(ctx context.Context, binlog string) (gtids string, err error) { 111 gtids, ok := tc.previousGTIDs[binlog] 112 if !ok { 113 return "", fmt.Errorf("previous gtids not found for binary log %v", binlog) 114 } 115 return gtids, nil 116 }, 117 true, 118 ) 119 if tc.expectError != "" { 120 require.Error(t, err) 121 assert.Contains(t, err.Error(), tc.expectError) 122 return 123 } 124 require.NoError(t, err) 125 require.NotEmpty(t, binlogsToBackup) 126 assert.Equal(t, tc.expectBinlogs, binlogsToBackup) 127 assert.Equal(t, tc.previousGTIDs[binlogsToBackup[0]], fromGTID) 128 assert.Equal(t, tc.previousGTIDs[binlogs[len(binlogs)-1]], toGTID) 129 assert.NotEqual(t, fromGTID, toGTID) 130 }) 131 } 132 } 133 134 func TestIsValidIncrementalBakcup(t *testing.T) { 135 incrementalManifest := func(backupPos string, backupFromPos string) *BackupManifest { 136 return &BackupManifest{ 137 Position: mysql.MustParsePosition(mysql.Mysql56FlavorID, fmt.Sprintf("16b1039f-22b6-11ed-b765-0a43f95f28a3:%s", backupPos)), 138 FromPosition: mysql.MustParsePosition(mysql.Mysql56FlavorID, fmt.Sprintf("16b1039f-22b6-11ed-b765-0a43f95f28a3:%s", backupFromPos)), 139 Incremental: true, 140 } 141 } 142 tt := []struct { 143 baseGTID string 144 purgedGTID string 145 backupFromPos string 146 backupPos string 147 expectIsValid bool 148 }{ 149 { 150 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 151 backupFromPos: "1-58", 152 backupPos: "1-70", 153 expectIsValid: true, 154 }, 155 { 156 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 157 backupFromPos: "1-51", 158 backupPos: "1-70", 159 expectIsValid: true, 160 }, 161 { 162 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 163 backupFromPos: "1-51", 164 backupPos: "1-58", 165 expectIsValid: false, 166 }, 167 { 168 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 169 backupFromPos: "1-58", 170 backupPos: "1-58", 171 expectIsValid: false, 172 }, 173 { 174 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 175 backupFromPos: "1-51", 176 backupPos: "1-55", 177 expectIsValid: false, 178 }, 179 { 180 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 181 backupFromPos: "1-59", 182 backupPos: "1-70", 183 expectIsValid: false, 184 }, 185 { 186 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 187 backupFromPos: "1-60", 188 backupPos: "1-70", 189 expectIsValid: false, 190 }, 191 { 192 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 193 backupFromPos: "3-51", 194 backupPos: "3-70", 195 expectIsValid: false, 196 }, 197 { 198 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 199 purgedGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-2", 200 backupFromPos: "3-51", 201 backupPos: "3-70", 202 expectIsValid: true, 203 }, 204 { 205 baseGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 206 purgedGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-2", 207 backupFromPos: "4-51", 208 backupPos: "4-70", 209 expectIsValid: false, 210 }, 211 } 212 for i, tc := range tt { 213 t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { 214 basePos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, tc.baseGTID) 215 require.NoError(t, err) 216 purgedPos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, tc.purgedGTID) 217 require.NoError(t, err) 218 isValid := IsValidIncrementalBakcup(basePos.GTIDSet, purgedPos.GTIDSet, incrementalManifest(tc.backupPos, tc.backupFromPos)) 219 assert.Equal(t, tc.expectIsValid, isValid) 220 }) 221 } 222 } 223 224 func TestFindPITRPath(t *testing.T) { 225 generatePosition := func(posRange string) mysql.Position { 226 return mysql.MustParsePosition(mysql.Mysql56FlavorID, fmt.Sprintf("16b1039f-22b6-11ed-b765-0a43f95f28a3:%s", posRange)) 227 } 228 fullManifest := func(backupPos string) *BackupManifest { 229 return &BackupManifest{ 230 Position: generatePosition(backupPos), 231 } 232 } 233 incrementalManifest := func(backupPos string, backupFromPos string) *BackupManifest { 234 return &BackupManifest{ 235 Position: generatePosition(backupPos), 236 FromPosition: generatePosition(backupFromPos), 237 Incremental: true, 238 } 239 } 240 fullBackups := []*BackupManifest{ 241 fullManifest("1-50"), 242 fullManifest("1-5"), 243 fullManifest("1-80"), 244 fullManifest("1-70"), 245 fullManifest("1-70"), 246 } 247 incrementalBackups := []*BackupManifest{ 248 incrementalManifest("1-34", "1-5"), 249 incrementalManifest("1-38", "1-34"), 250 incrementalManifest("1-52", "1-35"), 251 incrementalManifest("1-60", "1-50"), 252 incrementalManifest("1-70", "1-60"), 253 incrementalManifest("1-82", "1-70"), 254 incrementalManifest("1-92", "1-79"), 255 incrementalManifest("1-95", "1-89"), 256 } 257 tt := []struct { 258 name string 259 restoreGTID string 260 purgedGTID string 261 incrementalBackups []*BackupManifest 262 expectFullManifest *BackupManifest 263 expectIncrementalManifests []*BackupManifest 264 expectError string 265 }{ 266 { 267 name: "1-58", 268 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-58", 269 expectFullManifest: fullManifest("1-50"), 270 expectIncrementalManifests: []*BackupManifest{ 271 incrementalManifest("1-52", "1-35"), 272 incrementalManifest("1-60", "1-50"), 273 }, 274 }, 275 { 276 name: "1-50", 277 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-50", 278 expectFullManifest: fullManifest("1-50"), 279 }, 280 { 281 name: "1-78", 282 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-78", 283 expectFullManifest: fullManifest("1-70"), 284 expectIncrementalManifests: []*BackupManifest{ 285 incrementalManifest("1-82", "1-70"), 286 }, 287 }, 288 { 289 name: "1-45", 290 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-45", 291 expectFullManifest: fullManifest("1-5"), 292 expectIncrementalManifests: []*BackupManifest{ 293 incrementalManifest("1-34", "1-5"), 294 incrementalManifest("1-38", "1-34"), 295 incrementalManifest("1-52", "1-35"), 296 }, 297 }, 298 { 299 name: "1-28", 300 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-28", 301 expectFullManifest: fullManifest("1-5"), 302 expectIncrementalManifests: []*BackupManifest{ 303 incrementalManifest("1-34", "1-5"), 304 }, 305 }, 306 { 307 name: "1-88", 308 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-88", 309 expectFullManifest: fullManifest("1-80"), 310 expectIncrementalManifests: []*BackupManifest{ 311 incrementalManifest("1-82", "1-70"), 312 incrementalManifest("1-92", "1-79"), 313 }, 314 }, 315 { 316 name: "fail 1-2", 317 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-2", 318 expectError: "no full backup", 319 }, 320 { 321 name: "fail unknown UUID", 322 restoreGTID: "00000000-0000-0000-0000-0a43f95f28a3:1-50", 323 expectError: "no full backup", 324 }, 325 { 326 name: "fail 1-99", 327 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-99", 328 expectError: "no path found", 329 }, 330 { 331 name: "1-94", 332 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-94", 333 expectFullManifest: fullManifest("1-80"), 334 expectIncrementalManifests: []*BackupManifest{ 335 incrementalManifest("1-82", "1-70"), 336 incrementalManifest("1-92", "1-79"), 337 incrementalManifest("1-95", "1-89"), 338 }, 339 }, 340 { 341 name: "1-95", 342 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-95", 343 expectFullManifest: fullManifest("1-80"), 344 expectIncrementalManifests: []*BackupManifest{ 345 incrementalManifest("1-82", "1-70"), 346 incrementalManifest("1-92", "1-79"), 347 incrementalManifest("1-95", "1-89"), 348 }, 349 }, 350 { 351 name: "fail 1-88 with gaps", 352 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-88", 353 incrementalBackups: []*BackupManifest{ 354 incrementalManifest("1-34", "1-5"), 355 incrementalManifest("1-38", "1-34"), 356 incrementalManifest("1-52", "1-35"), 357 incrementalManifest("1-60", "1-50"), 358 incrementalManifest("1-70", "1-60"), 359 incrementalManifest("1-82", "1-70"), 360 incrementalManifest("1-92", "1-84"), 361 incrementalManifest("1-95", "1-89"), 362 }, 363 expectError: "no path found", 364 }, 365 { 366 name: "1-45 first solution even when shorter exists", 367 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-45", 368 incrementalBackups: append( 369 incrementalBackups, 370 incrementalManifest("1-99", "1-5"), 371 ), 372 expectFullManifest: fullManifest("1-5"), 373 expectIncrementalManifests: []*BackupManifest{ 374 incrementalManifest("1-34", "1-5"), 375 incrementalManifest("1-38", "1-34"), 376 incrementalManifest("1-52", "1-35"), 377 }, 378 }, 379 { 380 name: "fail incomplete binlog previous GTIDs", 381 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-92", 382 incrementalBackups: []*BackupManifest{ 383 incrementalManifest("3-90", "3-75"), 384 incrementalManifest("3-95", "3-90"), 385 }, 386 expectFullManifest: fullManifest("1-80"), 387 expectIncrementalManifests: []*BackupManifest{ 388 incrementalManifest("3-90", "3-75"), 389 incrementalManifest("3-95", "3-90"), 390 }, 391 expectError: "no path found", 392 }, 393 { 394 name: "incomplete binlog previous GTIDs", 395 restoreGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-92", 396 purgedGTID: "16b1039f-22b6-11ed-b765-0a43f95f28a3:1-2", 397 incrementalBackups: []*BackupManifest{ 398 incrementalManifest("3-90", "3-75"), 399 incrementalManifest("3-95", "3-90"), 400 }, 401 expectFullManifest: fullManifest("1-80"), 402 expectIncrementalManifests: []*BackupManifest{ 403 incrementalManifest("3-90", "3-75"), 404 incrementalManifest("3-95", "3-90"), 405 }, 406 }, 407 } 408 for _, tc := range tt { 409 t.Run(tc.name, func(t *testing.T) { 410 if tc.incrementalBackups == nil { 411 tc.incrementalBackups = incrementalBackups 412 } 413 for i := range fullBackups { 414 var err error 415 fullBackup := fullBackups[i] 416 fullBackup.PurgedPosition, err = mysql.ParsePosition(mysql.Mysql56FlavorID, tc.purgedGTID) 417 require.NoError(t, err) 418 defer func() { 419 fullBackup.PurgedPosition = mysql.Position{} 420 }() 421 } 422 var manifests []*BackupManifest 423 manifests = append(manifests, fullBackups...) 424 manifests = append(manifests, tc.incrementalBackups...) 425 426 restorePos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, tc.restoreGTID) 427 require.NoErrorf(t, err, "%v", err) 428 path, err := FindPITRPath(restorePos.GTIDSet, manifests) 429 if tc.expectError != "" { 430 require.Error(t, err) 431 assert.Contains(t, err.Error(), tc.expectError) 432 return 433 } 434 require.NoErrorf(t, err, "%v", err) 435 require.NotEmpty(t, path) 436 // the path always consists of one full backup and zero or more incremental backups 437 fullBackup := path[0] 438 require.False(t, fullBackup.Incremental) 439 for _, manifest := range path[1:] { 440 require.True(t, manifest.Incremental) 441 } 442 assert.Equal(t, tc.expectFullManifest.Position.GTIDSet, fullBackup.Position.GTIDSet) 443 if tc.expectIncrementalManifests == nil { 444 tc.expectIncrementalManifests = []*BackupManifest{} 445 } 446 expected := BackupManifestPath(tc.expectIncrementalManifests) 447 got := BackupManifestPath(path[1:]) 448 assert.Equal(t, expected, got, "expected: %s, got: %s", expected.String(), got.String()) 449 }) 450 } 451 }