github.com/pingcap/tidb-lightning@v5.0.0-rc.0.20210428090220-84b649866577+incompatible/lightning/lightning_test.go (about) 1 // Copyright 2019 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package lightning 15 16 import ( 17 "context" 18 "encoding/json" 19 "fmt" 20 "net/http" 21 "path/filepath" 22 "runtime" 23 "strings" 24 "testing" 25 "time" 26 27 "github.com/pingcap/tidb-lightning/lightning/checkpoints" 28 "github.com/pingcap/tidb-lightning/lightning/glue" 29 "github.com/pingcap/tidb-lightning/lightning/mydump" 30 31 . "github.com/pingcap/check" 32 "github.com/pingcap/failpoint" 33 34 "github.com/pingcap/tidb-lightning/lightning/config" 35 ) 36 37 type lightningSuite struct{} 38 39 var _ = Suite(&lightningSuite{}) 40 41 func TestLightning(t *testing.T) { 42 TestingT(t) 43 } 44 45 func (s *lightningSuite) TestInitEnv(c *C) { 46 cfg := &config.GlobalConfig{ 47 App: config.GlobalLightning{StatusAddr: ":45678"}, 48 } 49 err := initEnv(cfg) 50 c.Assert(err, IsNil) 51 cfg.App.StatusAddr = "" 52 cfg.App.Config.File = "." 53 err = initEnv(cfg) 54 c.Assert(err, ErrorMatches, "can't use directory as log file name") 55 } 56 57 func (s *lightningSuite) TestRun(c *C) { 58 globalConfig := config.NewGlobalConfig() 59 globalConfig.TiDB.Host = "test.invalid" 60 globalConfig.TiDB.Port = 4000 61 globalConfig.TiDB.PdAddr = "test.invalid:2379" 62 globalConfig.Mydumper.SourceDir = "not-exists" 63 lightning := New(globalConfig) 64 cfg := config.NewConfig() 65 err := cfg.LoadFromGlobal(globalConfig) 66 c.Assert(err, IsNil) 67 err = lightning.RunOnce(context.Background(), cfg, nil, nil) 68 c.Assert(err, ErrorMatches, ".*mydumper dir does not exist") 69 70 path, _ := filepath.Abs(".") 71 ctx := context.Background() 72 invalidGlue := glue.NewExternalTiDBGlue(nil, 0) 73 err = lightning.run(ctx, &config.Config{ 74 Mydumper: config.MydumperRuntime{ 75 SourceDir: "file://" + filepath.ToSlash(path), 76 Filter: []string{"*.*"}, 77 DefaultFileRules: true, 78 }, 79 Checkpoint: config.Checkpoint{ 80 Enable: true, 81 Driver: "invalid", 82 }, 83 }, invalidGlue) 84 c.Assert(err, ErrorMatches, "open checkpoint db failed: Unknown checkpoint driver invalid") 85 86 err = lightning.run(ctx, &config.Config{ 87 Mydumper: config.MydumperRuntime{ 88 SourceDir: ".", 89 Filter: []string{"*.*"}, 90 }, 91 Checkpoint: config.Checkpoint{ 92 Enable: true, 93 Driver: "file", 94 DSN: "any-file", 95 }, 96 }, invalidGlue) 97 c.Assert(err, NotNil) 98 } 99 100 var _ = Suite(&lightningServerSuite{}) 101 102 type lightningServerSuite struct { 103 lightning *Lightning 104 taskCfgCh chan *config.Config 105 } 106 107 func (s *lightningServerSuite) SetUpTest(c *C) { 108 cfg := config.NewGlobalConfig() 109 cfg.TiDB.Host = "test.invalid" 110 cfg.TiDB.Port = 4000 111 cfg.TiDB.PdAddr = "test.invalid:2379" 112 cfg.App.ServerMode = true 113 cfg.App.StatusAddr = "127.0.0.1:0" 114 cfg.Mydumper.SourceDir = "file://." 115 116 s.lightning = New(cfg) 117 s.taskCfgCh = make(chan *config.Config) 118 s.lightning.ctx = context.WithValue(s.lightning.ctx, &taskCfgRecorderKey, s.taskCfgCh) 119 s.lightning.GoServe() 120 121 failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/SkipRunTask", "return") 122 } 123 124 func (s *lightningServerSuite) TearDownTest(c *C) { 125 failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/SkipRunTask") 126 s.lightning.Stop() 127 } 128 129 func (s *lightningServerSuite) TestRunServer(c *C) { 130 url := "http://" + s.lightning.serverAddr.String() + "/tasks" 131 132 resp, err := http.Post(url, "application/toml", strings.NewReader("????")) 133 c.Assert(err, IsNil) 134 c.Assert(resp.StatusCode, Equals, http.StatusNotImplemented) 135 var data map[string]string 136 err = json.NewDecoder(resp.Body).Decode(&data) 137 c.Assert(err, IsNil) 138 c.Assert(data, HasKey, "error") 139 c.Assert(data["error"], Equals, "server-mode not enabled") 140 resp.Body.Close() 141 142 go s.lightning.RunServer() 143 time.Sleep(100 * time.Millisecond) 144 145 req, err := http.NewRequest(http.MethodPut, url, nil) 146 c.Assert(err, IsNil) 147 resp, err = http.DefaultClient.Do(req) 148 c.Assert(err, IsNil) 149 c.Assert(resp.StatusCode, Equals, http.StatusMethodNotAllowed) 150 c.Assert(resp.Header.Get("Allow"), Matches, ".*"+http.MethodPost+".*") 151 resp.Body.Close() 152 153 resp, err = http.Post(url, "application/toml", strings.NewReader("????")) 154 c.Assert(err, IsNil) 155 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 156 err = json.NewDecoder(resp.Body).Decode(&data) 157 c.Assert(err, IsNil) 158 c.Assert(data, HasKey, "error") 159 c.Assert(data["error"], Matches, "cannot parse task.*") 160 resp.Body.Close() 161 162 resp, err = http.Post(url, "application/toml", strings.NewReader("[mydumper.csv]\nseparator = 'fooo'\ndelimiter= 'foo'")) 163 c.Assert(err, IsNil) 164 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 165 err = json.NewDecoder(resp.Body).Decode(&data) 166 c.Assert(err, IsNil) 167 c.Assert(data, HasKey, "error") 168 c.Assert(data["error"], Matches, "invalid task configuration:.*") 169 resp.Body.Close() 170 171 for i := 0; i < 20; i++ { 172 resp, err = http.Post(url, "application/toml", strings.NewReader(fmt.Sprintf(` 173 [mydumper] 174 data-source-dir = 'file://demo-path-%d' 175 [mydumper.csv] 176 separator = '/' 177 `, i))) 178 c.Assert(err, IsNil) 179 c.Assert(resp.StatusCode, Equals, http.StatusOK) 180 var result map[string]int 181 err = json.NewDecoder(resp.Body).Decode(&result) 182 resp.Body.Close() 183 c.Assert(err, IsNil) 184 c.Assert(result, HasKey, "id") 185 186 select { 187 case taskCfg := <-s.taskCfgCh: 188 c.Assert(taskCfg.TiDB.Host, Equals, "test.invalid") 189 c.Assert(taskCfg.Mydumper.SourceDir, Equals, fmt.Sprintf("file://demo-path-%d", i)) 190 c.Assert(taskCfg.Mydumper.CSV.Separator, Equals, "/") 191 case <-time.After(500 * time.Millisecond): 192 c.Fatalf("task is not queued after 500ms (i = %d)", i) 193 } 194 } 195 } 196 197 func (s *lightningServerSuite) TestGetDeleteTask(c *C) { 198 url := "http://" + s.lightning.serverAddr.String() + "/tasks" 199 200 type getAllResultType struct { 201 Current int64 202 Queue []int64 203 } 204 205 getAllTasks := func() (result getAllResultType) { 206 resp, err := http.Get(url) 207 c.Assert(err, IsNil) 208 c.Assert(resp.StatusCode, Equals, http.StatusOK) 209 err = json.NewDecoder(resp.Body).Decode(&result) 210 resp.Body.Close() 211 c.Assert(err, IsNil) 212 return 213 } 214 215 postTask := func(i int) int64 { 216 resp, err := http.Post(url, "application/toml", strings.NewReader(fmt.Sprintf(` 217 [mydumper] 218 data-source-dir = 'file://demo-path-%d' 219 `, i))) 220 c.Assert(err, IsNil) 221 c.Assert(resp.StatusCode, Equals, http.StatusOK) 222 var result struct{ ID int64 } 223 err = json.NewDecoder(resp.Body).Decode(&result) 224 resp.Body.Close() 225 c.Assert(err, IsNil) 226 return result.ID 227 } 228 229 go s.lightning.RunServer() 230 time.Sleep(100 * time.Millisecond) 231 232 // Check `GET /tasks` without any active tasks 233 234 c.Assert(getAllTasks(), DeepEquals, getAllResultType{ 235 Current: 0, 236 Queue: []int64{}, 237 }) 238 239 first := postTask(1) 240 second := postTask(2) 241 third := postTask(3) 242 243 c.Assert(first, Not(Equals), 123456) 244 c.Assert(second, Not(Equals), 123456) 245 c.Assert(third, Not(Equals), 123456) 246 247 // Check `GET /tasks` returns all tasks currently running 248 249 time.Sleep(100 * time.Millisecond) 250 c.Assert(getAllTasks(), DeepEquals, getAllResultType{ 251 Current: first, 252 Queue: []int64{second, third}, 253 }) 254 255 // Check `GET /tasks/abcdef` returns error 256 257 resp, err := http.Get(url + "/abcdef") 258 c.Assert(err, IsNil) 259 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 260 resp.Body.Close() 261 262 // Check `GET /tasks/123456` returns not found 263 264 resp, err = http.Get(url + "/123456") 265 c.Assert(err, IsNil) 266 c.Assert(resp.StatusCode, Equals, http.StatusNotFound) 267 resp.Body.Close() 268 269 // Check `GET /tasks/1` returns the desired cfg 270 271 var resCfg config.Config 272 273 resp, err = http.Get(fmt.Sprintf("%s/%d", url, second)) 274 c.Assert(err, IsNil) 275 c.Assert(resp.StatusCode, Equals, http.StatusOK) 276 err = json.NewDecoder(resp.Body).Decode(&resCfg) 277 resp.Body.Close() 278 c.Assert(err, IsNil) 279 c.Assert(resCfg.Mydumper.SourceDir, Equals, "file://demo-path-2") 280 281 resp, err = http.Get(fmt.Sprintf("%s/%d", url, first)) 282 c.Assert(err, IsNil) 283 c.Assert(resp.StatusCode, Equals, http.StatusOK) 284 err = json.NewDecoder(resp.Body).Decode(&resCfg) 285 resp.Body.Close() 286 c.Assert(err, IsNil) 287 c.Assert(resCfg.Mydumper.SourceDir, Equals, "file://demo-path-1") 288 289 // Check `DELETE /tasks` returns error. 290 291 req, err := http.NewRequest(http.MethodDelete, url, nil) 292 c.Assert(err, IsNil) 293 resp, err = http.DefaultClient.Do(req) 294 c.Assert(err, IsNil) 295 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 296 resp.Body.Close() 297 298 // Check `DELETE /tasks/` returns error. 299 300 req.URL.Path = "/tasks/" 301 resp, err = http.DefaultClient.Do(req) 302 c.Assert(err, IsNil) 303 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 304 resp.Body.Close() 305 306 // Check `DELETE /tasks/(not a number)` returns error. 307 308 req.URL.Path = "/tasks/abcdef" 309 resp, err = http.DefaultClient.Do(req) 310 c.Assert(err, IsNil) 311 c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) 312 resp.Body.Close() 313 314 // Check `DELETE /tasks/123456` returns not found 315 316 req.URL.Path = "/tasks/123456" 317 resp, err = http.DefaultClient.Do(req) 318 c.Assert(err, IsNil) 319 c.Assert(resp.StatusCode, Equals, http.StatusNotFound) 320 resp.Body.Close() 321 322 // Cancel a queued task, then verify the task list. 323 324 req.URL.Path = fmt.Sprintf("/tasks/%d", second) 325 resp, err = http.DefaultClient.Do(req) 326 c.Assert(err, IsNil) 327 c.Assert(resp.StatusCode, Equals, http.StatusOK) 328 resp.Body.Close() 329 330 c.Assert(getAllTasks(), DeepEquals, getAllResultType{ 331 Current: first, 332 Queue: []int64{third}, 333 }) 334 335 // Cancel a running task, then verify the task list. 336 337 req.URL.Path = fmt.Sprintf("/tasks/%d", first) 338 resp, err = http.DefaultClient.Do(req) 339 c.Assert(err, IsNil) 340 c.Assert(resp.StatusCode, Equals, http.StatusOK) 341 resp.Body.Close() 342 343 time.Sleep(100 * time.Millisecond) 344 c.Assert(getAllTasks(), DeepEquals, getAllResultType{ 345 Current: third, 346 Queue: []int64{}, 347 }) 348 } 349 350 func (s *lightningServerSuite) TestHTTPAPIOutsideServerMode(c *C) { 351 s.lightning.globalCfg.App.ServerMode = false 352 353 url := "http://" + s.lightning.serverAddr.String() + "/tasks" 354 355 errCh := make(chan error) 356 cfg := config.NewConfig() 357 err := cfg.LoadFromGlobal(s.lightning.globalCfg) 358 c.Assert(err, IsNil) 359 go func() { 360 errCh <- s.lightning.RunOnce(s.lightning.ctx, cfg, nil, nil) 361 }() 362 time.Sleep(100 * time.Millisecond) 363 364 var curTask struct { 365 Current int64 366 Queue []int64 367 } 368 369 // `GET /tasks` should work fine. 370 resp, err := http.Get(url) 371 c.Assert(err, IsNil) 372 c.Assert(resp.StatusCode, Equals, http.StatusOK) 373 err = json.NewDecoder(resp.Body).Decode(&curTask) 374 resp.Body.Close() 375 c.Assert(err, IsNil) 376 c.Assert(curTask.Current, Not(Equals), int64(0)) 377 c.Assert(curTask.Queue, HasLen, 0) 378 379 // `POST /tasks` should return 501 380 resp, err = http.Post(url, "application/toml", strings.NewReader("??????")) 381 c.Assert(err, IsNil) 382 c.Assert(resp.StatusCode, Equals, http.StatusNotImplemented) 383 resp.Body.Close() 384 385 // `GET /tasks/(current)` should work fine. 386 resp, err = http.Get(fmt.Sprintf("%s/%d", url, curTask.Current)) 387 c.Assert(err, IsNil) 388 c.Assert(resp.StatusCode, Equals, http.StatusOK) 389 resp.Body.Close() 390 391 // `GET /tasks/123456` should return 404 392 resp, err = http.Get(url + "/123456") 393 c.Assert(err, IsNil) 394 c.Assert(resp.StatusCode, Equals, http.StatusNotFound) 395 resp.Body.Close() 396 397 // `PATCH /tasks/(current)/front` should return 501 398 req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/%d/front", url, curTask.Current), nil) 399 c.Assert(err, IsNil) 400 resp, err = http.DefaultClient.Do(req) 401 c.Assert(err, IsNil) 402 c.Assert(resp.StatusCode, Equals, http.StatusNotImplemented) 403 resp.Body.Close() 404 405 // `DELETE /tasks/123456` should return 404 406 req.Method = http.MethodDelete 407 req.URL.Path = "/tasks/123456" 408 resp, err = http.DefaultClient.Do(req) 409 c.Assert(err, IsNil) 410 c.Assert(resp.StatusCode, Equals, http.StatusNotFound) 411 resp.Body.Close() 412 413 // `DELETE /tasks/(current)` should return 200 414 req.URL.Path = fmt.Sprintf("/tasks/%d", curTask.Current) 415 resp, err = http.DefaultClient.Do(req) 416 c.Assert(err, IsNil) 417 c.Assert(resp.StatusCode, Equals, http.StatusOK) 418 419 // ... and the task should be canceled now. 420 c.Assert(<-errCh, Equals, context.Canceled) 421 } 422 423 func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) { 424 if runtime.GOOS == "windows" { 425 c.Skip("Local-backend is not supported on Windows") 426 return 427 } 428 429 cfg := config.NewConfig() 430 cfg.App.CheckRequirements = true 431 cfg.App.TableConcurrency = 4 432 cfg.TikvImporter.Backend = config.BackendLocal 433 434 dbMetas := []*mydump.MDDatabaseMeta{ 435 { 436 Tables: []*mydump.MDTableMeta{ 437 { 438 TotalSize: 500 << 20, 439 }, 440 { 441 TotalSize: 150_000 << 20, 442 }, 443 }, 444 }, 445 { 446 Tables: []*mydump.MDTableMeta{ 447 { 448 TotalSize: 150_800 << 20, 449 }, 450 { 451 TotalSize: 35 << 20, 452 }, 453 { 454 TotalSize: 100_000 << 20, 455 }, 456 }, 457 }, 458 { 459 Tables: []*mydump.MDTableMeta{ 460 { 461 TotalSize: 240 << 20, 462 }, 463 { 464 TotalSize: 124_000 << 20, 465 }, 466 }, 467 }, 468 } 469 470 // with max open files 1024, the max table size will be: 65536MB 471 err := failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/backend/GetRlimitValue", "return(2049)") 472 c.Assert(err, IsNil) 473 err = failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/backend/SetRlimitError", "return(true)") 474 c.Assert(err, IsNil) 475 defer failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/backend/SetRlimitError") 476 // with this dbMetas, the estimated fds will be 2050, so should return error 477 err = checkSystemRequirement(cfg, dbMetas) 478 c.Assert(err, NotNil) 479 480 //disable check-requirement, should return nil 481 cfg.App.CheckRequirements = false 482 err = checkSystemRequirement(cfg, dbMetas) 483 c.Assert(err, IsNil) 484 cfg.App.CheckRequirements = true 485 486 err = failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/backend/GetRlimitValue") 487 c.Assert(err, IsNil) 488 489 // the min rlimit should be bigger than the default min value (16384) 490 err = failpoint.Enable("github.com/pingcap/tidb-lightning/lightning/backend/GetRlimitValue", "return(8200)") 491 defer failpoint.Disable("github.com/pingcap/tidb-lightning/lightning/backend/GetRlimitValue") 492 c.Assert(err, IsNil) 493 err = checkSystemRequirement(cfg, dbMetas) 494 c.Assert(err, IsNil) 495 } 496 497 func (s *lightningServerSuite) TestCheckSchemaConflict(c *C) { 498 cfg := config.NewConfig() 499 cfg.Checkpoint.Schema = "cp" 500 cfg.Checkpoint.Driver = config.CheckpointDriverMySQL 501 502 dbMetas := []*mydump.MDDatabaseMeta{ 503 { 504 Name: "test", 505 Tables: []*mydump.MDTableMeta{ 506 { 507 Name: checkpoints.CheckpointTableNameTable, 508 }, 509 { 510 Name: checkpoints.CheckpointTableNameEngine, 511 }, 512 }, 513 }, 514 { 515 Name: "cp", 516 Tables: []*mydump.MDTableMeta{ 517 { 518 Name: "test", 519 }, 520 }, 521 }, 522 } 523 err := checkSchemaConflict(cfg, dbMetas) 524 c.Assert(err, IsNil) 525 526 dbMetas = append(dbMetas, &mydump.MDDatabaseMeta{ 527 Name: "cp", 528 Tables: []*mydump.MDTableMeta{ 529 { 530 Name: checkpoints.CheckpointTableNameChunk, 531 }, 532 { 533 Name: "test123", 534 }, 535 }, 536 }) 537 err = checkSchemaConflict(cfg, dbMetas) 538 c.Assert(err, NotNil) 539 540 cfg.Checkpoint.Enable = false 541 err = checkSchemaConflict(cfg, dbMetas) 542 c.Assert(err, IsNil) 543 544 cfg.Checkpoint.Enable = true 545 cfg.Checkpoint.Driver = config.CheckpointDriverFile 546 err = checkSchemaConflict(cfg, dbMetas) 547 c.Assert(err, IsNil) 548 549 }