github.com/m3db/m3@v1.5.0/src/query/api/v1/handler/database/create_test.go (about) 1 // Copyright (c) 2018 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package database 22 23 import ( 24 "bytes" 25 "encoding/json" 26 "fmt" 27 "io/ioutil" 28 "net/http" 29 "net/http/httptest" 30 "testing" 31 "time" 32 33 "github.com/m3db/m3/src/cluster/client" 34 "github.com/m3db/m3/src/cluster/generated/proto/placementpb" 35 "github.com/m3db/m3/src/cluster/kv" 36 "github.com/m3db/m3/src/cluster/kv/fake" 37 "github.com/m3db/m3/src/cluster/placement" 38 "github.com/m3db/m3/src/cluster/placementhandler/handleroptions" 39 "github.com/m3db/m3/src/cluster/services" 40 dbconfig "github.com/m3db/m3/src/cmd/services/m3dbnode/config" 41 "github.com/m3db/m3/src/cmd/services/m3query/config" 42 "github.com/m3db/m3/src/query/api/v1/handler/namespace" 43 "github.com/m3db/m3/src/query/api/v1/validators" 44 "github.com/m3db/m3/src/x/instrument" 45 xjson "github.com/m3db/m3/src/x/json" 46 xtest "github.com/m3db/m3/src/x/test" 47 48 "github.com/golang/mock/gomock" 49 "github.com/stretchr/testify/assert" 50 "github.com/stretchr/testify/require" 51 ) 52 53 var ( 54 listenAddress = "0.0.0.0:9000" 55 testDBCfg = &dbconfig.DBConfiguration{ 56 ListenAddress: &listenAddress, 57 } 58 59 svcDefaultOptions = []handleroptions.ServiceOptionsDefault{ 60 func(o handleroptions.ServiceOptions) handleroptions.ServiceOptions { 61 return o 62 }, 63 } 64 ) 65 66 func SetupDatabaseTest( 67 t *testing.T, 68 ctrl *gomock.Controller, 69 ) (*client.MockClient, *kv.MockStore, *placement.MockService) { 70 mockClient := client.NewMockClient(ctrl) 71 require.NotNil(t, mockClient) 72 mockKV := kv.NewMockStore(ctrl) 73 require.NotNil(t, mockKV) 74 mockPlacementService := placement.NewMockService(ctrl) 75 require.NotNil(t, mockPlacementService) 76 mockServices := services.NewMockServices(ctrl) 77 require.NotNil(t, mockServices) 78 79 mockServices.EXPECT().PlacementService(gomock.Any(), gomock.Any()).Return(mockPlacementService, nil).AnyTimes() 80 mockClient.EXPECT().KV().Return(mockKV, nil).AnyTimes() 81 mockClient.EXPECT().Services(gomock.Any()).Return(mockServices, nil).AnyTimes() 82 83 return mockClient, mockKV, mockPlacementService 84 } 85 86 func TestLocalType(t *testing.T) { 87 testLocalType(t, "local", false) 88 } 89 90 func TestLocalTypePlacementAlreadyExists(t *testing.T) { 91 testLocalType(t, "local", true) 92 } 93 94 func TestLocalTypePlacementAlreadyExistsNoTypeProvided(t *testing.T) { 95 testLocalType(t, "", true) 96 } 97 98 func testLocalType(t *testing.T, providedType string, placementExists bool) { 99 ctrl := gomock.NewController(t) 100 defer ctrl.Finish() 101 102 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 103 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 104 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 105 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 106 require.NoError(t, err) 107 w := httptest.NewRecorder() 108 109 jsonInput := xjson.Map{ 110 "namespaceName": "testNamespace", 111 "type": providedType, 112 } 113 114 req := httptest.NewRequest("POST", "/database/create", 115 xjson.MustNewTestReader(t, jsonInput)) 116 require.NotNil(t, req) 117 118 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 119 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 120 121 placementProto := &placementpb.Placement{ 122 Instances: map[string]*placementpb.Instance{ 123 "localhost": &placementpb.Instance{ 124 Id: "m3db_local", 125 IsolationGroup: "local", 126 Zone: "embedded", 127 Weight: 1, 128 Endpoint: "http://localhost:9000", 129 Hostname: "localhost", 130 Port: 9000, 131 }, 132 }, 133 } 134 newPlacement, err := placement.NewPlacementFromProto(placementProto) 135 require.NoError(t, err) 136 137 if placementExists { 138 mockPlacementService.EXPECT().Placement().Return(newPlacement, nil) 139 } else { 140 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 141 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 64, 1).Return(newPlacement, nil) 142 } 143 144 createHandler.ServeHTTP(w, req) 145 146 resp := w.Result() 147 body, err := ioutil.ReadAll(resp.Body) 148 assert.NoError(t, err) 149 assert.Equal(t, http.StatusOK, resp.StatusCode) 150 151 expectedResponse := ` 152 { 153 "namespace": { 154 "registry": { 155 "namespaces": { 156 "testNamespace": { 157 "aggregationOptions": { 158 "aggregations": [ 159 { 160 "aggregated": false, 161 "attributes": null 162 } 163 ] 164 }, 165 "bootstrapEnabled": true, 166 "cacheBlocksOnRetrieve": false, 167 "flushEnabled": true, 168 "writesToCommitLog": true, 169 "cleanupEnabled": true, 170 "repairEnabled": false, 171 "retentionOptions": { 172 "retentionPeriodNanos": "86400000000000", 173 "blockSizeNanos": "3600000000000", 174 "bufferFutureNanos": "120000000000", 175 "bufferPastNanos": "600000000000", 176 "blockDataExpiry": true, 177 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 178 "futureRetentionPeriodNanos": "0" 179 }, 180 "snapshotEnabled": true, 181 "indexOptions": { 182 "enabled": true, 183 "blockSizeNanos": "3600000000000" 184 }, 185 "runtimeOptions": null, 186 "schemaOptions": null, 187 "coldWritesEnabled": false, 188 "extendedOptions": null, 189 "stagingState": { 190 "status": "UNKNOWN" 191 } 192 } 193 } 194 } 195 }, 196 "placement": { 197 "placement": { 198 "instances": { 199 "m3db_local": { 200 "id": "m3db_local", 201 "isolationGroup": "local", 202 "zone": "embedded", 203 "weight": 1, 204 "endpoint": "http://localhost:9000", 205 "shards": [], 206 "shardSetId": 0, 207 "hostname": "localhost", 208 "port": 9000, 209 "metadata": { 210 "debugPort": 0 211 } 212 } 213 }, 214 "replicaFactor": 0, 215 "numShards": 0, 216 "isSharded": false, 217 "cutoverTime": "0", 218 "isMirrored": false, 219 "maxShardSetId": 0 220 }, 221 "version": 0 222 } 223 } 224 ` 225 226 expected := xtest.MustPrettyJSONString(t, expectedResponse) 227 actual := xtest.MustPrettyJSONString(t, string(body)) 228 229 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 230 } 231 232 func TestLocalTypeClusteredPlacementAlreadyExists(t *testing.T) { 233 ctrl := gomock.NewController(t) 234 defer ctrl.Finish() 235 236 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 237 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 238 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 239 require.NoError(t, err) 240 w := httptest.NewRecorder() 241 242 jsonInput := xjson.Map{ 243 "namespaceName": "testNamespace", 244 "type": "local", 245 } 246 247 req := httptest.NewRequest("POST", "/database/create", 248 xjson.MustNewTestReader(t, jsonInput)) 249 require.NotNil(t, req) 250 251 placementProto := &placementpb.Placement{ 252 Instances: map[string]*placementpb.Instance{ 253 "localhost": &placementpb.Instance{ 254 Id: "m3db_not_local", 255 IsolationGroup: "local", 256 Zone: "embedded", 257 Weight: 1, 258 Endpoint: "http://localhost:9000", 259 Hostname: "localhost", 260 Port: 9000, 261 }, 262 }, 263 } 264 newPlacement, err := placement.NewPlacementFromProto(placementProto) 265 require.NoError(t, err) 266 267 mockPlacementService.EXPECT().Placement().Return(newPlacement, nil) 268 269 createHandler.ServeHTTP(w, req) 270 271 resp := w.Result() 272 _, err = ioutil.ReadAll(resp.Body) 273 assert.NoError(t, err) 274 assert.Equal(t, http.StatusBadRequest, resp.StatusCode) 275 } 276 277 func TestLocalTypeWithNumShards(t *testing.T) { 278 ctrl := gomock.NewController(t) 279 defer ctrl.Finish() 280 281 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 282 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 283 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 284 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 285 require.NoError(t, err) 286 287 w := httptest.NewRecorder() 288 289 jsonInput := xjson.Map{ 290 "namespaceName": "testNamespace", 291 "type": "local", 292 "numShards": 51, 293 } 294 295 req := httptest.NewRequest("POST", "/database/create", 296 xjson.MustNewTestReader(t, jsonInput)) 297 require.NotNil(t, req) 298 299 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 300 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 301 302 placementProto := &placementpb.Placement{ 303 Instances: map[string]*placementpb.Instance{ 304 "localhost": &placementpb.Instance{ 305 Id: "m3db_local", 306 IsolationGroup: "local", 307 Zone: "embedded", 308 Weight: 1, 309 Endpoint: "http://localhost:9000", 310 Hostname: "localhost", 311 Port: 9000, 312 }, 313 }, 314 } 315 newPlacement, err := placement.NewPlacementFromProto(placementProto) 316 require.NoError(t, err) 317 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 318 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 51, 1).Return(newPlacement, nil) 319 320 createHandler.ServeHTTP(w, req) 321 322 resp := w.Result() 323 body, err := ioutil.ReadAll(resp.Body) 324 assert.NoError(t, err) 325 assert.Equal(t, http.StatusOK, resp.StatusCode) 326 327 expectedResponse := ` 328 { 329 "namespace": { 330 "registry": { 331 "namespaces": { 332 "testNamespace": { 333 "aggregationOptions": { 334 "aggregations": [ 335 { 336 "aggregated": false, 337 "attributes": null 338 } 339 ] 340 }, 341 "bootstrapEnabled": true, 342 "cacheBlocksOnRetrieve": false, 343 "flushEnabled": true, 344 "writesToCommitLog": true, 345 "cleanupEnabled": true, 346 "repairEnabled": false, 347 "retentionOptions": { 348 "retentionPeriodNanos": "86400000000000", 349 "blockSizeNanos": "3600000000000", 350 "bufferFutureNanos": "120000000000", 351 "bufferPastNanos": "600000000000", 352 "blockDataExpiry": true, 353 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 354 "futureRetentionPeriodNanos": "0" 355 }, 356 "snapshotEnabled": true, 357 "indexOptions": { 358 "enabled": true, 359 "blockSizeNanos": "3600000000000" 360 }, 361 "runtimeOptions": null, 362 "schemaOptions": null, 363 "coldWritesEnabled": false, 364 "extendedOptions": null, 365 "stagingState": { 366 "status": "UNKNOWN" 367 } 368 } 369 } 370 } 371 }, 372 "placement": { 373 "placement": { 374 "instances": { 375 "m3db_local": { 376 "id": "m3db_local", 377 "isolationGroup": "local", 378 "zone": "embedded", 379 "weight": 1, 380 "endpoint": "http://localhost:9000", 381 "shards": [], 382 "shardSetId": 0, 383 "hostname": "localhost", 384 "port": 9000, 385 "metadata": { 386 "debugPort": 0 387 } 388 } 389 }, 390 "replicaFactor": 0, 391 "numShards": 0, 392 "isSharded": false, 393 "cutoverTime": "0", 394 "isMirrored": false, 395 "maxShardSetId": 0 396 }, 397 "version": 0 398 } 399 } 400 ` 401 expected := xtest.MustPrettyJSONString(t, expectedResponse) 402 actual := xtest.MustPrettyJSONString(t, string(body)) 403 404 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 405 } 406 407 func TestLocalWithBlockSizeNanos(t *testing.T) { 408 ctrl := gomock.NewController(t) 409 defer ctrl.Finish() 410 411 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 412 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 413 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 414 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 415 require.NoError(t, err) 416 w := httptest.NewRecorder() 417 418 jsonInput := xjson.Map{ 419 "namespaceName": "testNamespace", 420 "type": "local", 421 "blockSize": xjson.Map{"time": "3h"}, 422 } 423 424 req := httptest.NewRequest("POST", "/database/create", 425 xjson.MustNewTestReader(t, jsonInput)) 426 require.NotNil(t, req) 427 428 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 429 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 430 431 placementProto := &placementpb.Placement{ 432 Instances: map[string]*placementpb.Instance{ 433 "localhost": &placementpb.Instance{ 434 Id: DefaultLocalHostID, 435 IsolationGroup: "local", 436 Zone: "embedded", 437 Weight: 1, 438 Endpoint: "http://localhost:9000", 439 Hostname: "localhost", 440 Port: 9000, 441 }, 442 }, 443 } 444 newPlacement, err := placement.NewPlacementFromProto(placementProto) 445 require.NoError(t, err) 446 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 447 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 64, 1).Return(newPlacement, nil) 448 449 createHandler.ServeHTTP(w, req) 450 451 resp := w.Result() 452 body, err := ioutil.ReadAll(resp.Body) 453 assert.NoError(t, err) 454 assert.Equal(t, http.StatusOK, resp.StatusCode) 455 456 expectedResponse := ` 457 { 458 "namespace": { 459 "registry": { 460 "namespaces": { 461 "testNamespace": { 462 "aggregationOptions": { 463 "aggregations": [ 464 { 465 "aggregated": false, 466 "attributes": null 467 } 468 ] 469 }, 470 "bootstrapEnabled": true, 471 "cacheBlocksOnRetrieve": false, 472 "flushEnabled": true, 473 "writesToCommitLog": true, 474 "cleanupEnabled": true, 475 "repairEnabled": false, 476 "retentionOptions": { 477 "retentionPeriodNanos": "86400000000000", 478 "blockSizeNanos": "10800000000000", 479 "bufferFutureNanos": "120000000000", 480 "bufferPastNanos": "600000000000", 481 "blockDataExpiry": true, 482 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 483 "futureRetentionPeriodNanos": "0" 484 }, 485 "snapshotEnabled": true, 486 "indexOptions": { 487 "enabled": true, 488 "blockSizeNanos": "10800000000000" 489 }, 490 "runtimeOptions": null, 491 "schemaOptions": null, 492 "coldWritesEnabled": false, 493 "extendedOptions": null, 494 "stagingState": { 495 "status": "UNKNOWN" 496 } 497 } 498 } 499 } 500 }, 501 "placement": { 502 "placement": { 503 "instances": { 504 "m3db_local": { 505 "id": "m3db_local", 506 "isolationGroup": "local", 507 "zone": "embedded", 508 "weight": 1, 509 "endpoint": "http://localhost:9000", 510 "shards": [], 511 "shardSetId": 0, 512 "hostname": "localhost", 513 "port": 9000, 514 "metadata": { 515 "debugPort": 0 516 } 517 } 518 }, 519 "replicaFactor": 0, 520 "numShards": 0, 521 "isSharded": false, 522 "cutoverTime": "0", 523 "isMirrored": false, 524 "maxShardSetId": 0 525 }, 526 "version": 0 527 } 528 } 529 ` 530 expected := xtest.MustPrettyJSONString(t, expectedResponse) 531 actual := xtest.MustPrettyJSONString(t, string(body)) 532 533 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 534 } 535 536 func TestLocalWithBlockSizeExpectedSeriesDatapointsPerHour(t *testing.T) { 537 ctrl := gomock.NewController(t) 538 defer ctrl.Finish() 539 540 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 541 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 542 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 543 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 544 require.NoError(t, err) 545 w := httptest.NewRecorder() 546 547 min := minRecommendCalculateBlockSize 548 desiredBlockSize := min + 5*time.Minute 549 550 jsonInput := xjson.Map{ 551 "namespaceName": "testNamespace", 552 "type": "local", 553 "blockSize": xjson.Map{ 554 "expectedSeriesDatapointsPerHour": int64(float64(blockSizeFromExpectedSeriesScalar) / float64(desiredBlockSize)), 555 }, 556 } 557 558 req := httptest.NewRequest("POST", "/database/create", 559 xjson.MustNewTestReader(t, jsonInput)) 560 require.NotNil(t, req) 561 562 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 563 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 564 565 placementProto := &placementpb.Placement{ 566 Instances: map[string]*placementpb.Instance{ 567 "localhost": &placementpb.Instance{ 568 Id: DefaultLocalHostID, 569 IsolationGroup: "local", 570 Zone: "embedded", 571 Weight: 1, 572 Endpoint: "http://localhost:9000", 573 Hostname: "localhost", 574 Port: 9000, 575 }, 576 }, 577 } 578 newPlacement, err := placement.NewPlacementFromProto(placementProto) 579 require.NoError(t, err) 580 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 581 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 64, 1).Return(newPlacement, nil) 582 583 createHandler.ServeHTTP(w, req) 584 585 resp := w.Result() 586 body, err := ioutil.ReadAll(resp.Body) 587 assert.NoError(t, err) 588 assert.Equal(t, http.StatusOK, resp.StatusCode) 589 590 expectedResponse := fmt.Sprintf(` 591 { 592 "namespace": { 593 "registry": { 594 "namespaces": { 595 "testNamespace": { 596 "aggregationOptions": { 597 "aggregations": [ 598 { 599 "aggregated": false, 600 "attributes": null 601 } 602 ] 603 }, 604 "bootstrapEnabled": true, 605 "cacheBlocksOnRetrieve": false, 606 "flushEnabled": true, 607 "writesToCommitLog": true, 608 "cleanupEnabled": true, 609 "repairEnabled": false, 610 "retentionOptions": { 611 "retentionPeriodNanos": "86400000000000", 612 "blockSizeNanos": "%d", 613 "bufferFutureNanos": "120000000000", 614 "bufferPastNanos": "600000000000", 615 "blockDataExpiry": true, 616 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 617 "futureRetentionPeriodNanos": "0" 618 }, 619 "snapshotEnabled": true, 620 "indexOptions": { 621 "enabled": true, 622 "blockSizeNanos": "%d" 623 }, 624 "runtimeOptions": null, 625 "schemaOptions": null, 626 "coldWritesEnabled": false, 627 "extendedOptions": null, 628 "stagingState": { 629 "status": "UNKNOWN" 630 } 631 } 632 } 633 } 634 }, 635 "placement": { 636 "placement": { 637 "instances": { 638 "m3db_local": { 639 "id": "m3db_local", 640 "isolationGroup": "local", 641 "zone": "embedded", 642 "weight": 1, 643 "endpoint": "http://localhost:9000", 644 "shards": [], 645 "shardSetId": 0, 646 "hostname": "localhost", 647 "port": 9000, 648 "metadata": { 649 "debugPort": 0 650 } 651 } 652 }, 653 "replicaFactor": 0, 654 "numShards": 0, 655 "isSharded": false, 656 "cutoverTime": "0", 657 "isMirrored": false, 658 "maxShardSetId": 0 659 }, 660 "version": 0 661 } 662 } 663 `, desiredBlockSize, desiredBlockSize) 664 665 expected := xtest.MustPrettyJSONString(t, expectedResponse) 666 actual := xtest.MustPrettyJSONString(t, string(body)) 667 668 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 669 } 670 671 func TestClusterTypeHosts(t *testing.T) { 672 testClusterTypeHosts(t, false) 673 } 674 675 func TestClusterTypeHostsNotProvided(t *testing.T) { 676 testClusterTypeHosts(t, true) 677 } 678 679 func TestClusterTypeHostsPlacementAlreadyExistsHostsProvided(t *testing.T) { 680 ctrl := gomock.NewController(t) 681 defer ctrl.Finish() 682 683 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 684 mockClient.EXPECT().Store(gomock.Any()).Return(nil, nil).AnyTimes() 685 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 686 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 687 require.NoError(t, err) 688 w := httptest.NewRecorder() 689 690 jsonInput := xjson.Map{ 691 "namespaceName": "testNamespace", 692 "type": "cluster", 693 "hosts": xjson.Array{xjson.Map{"id": "host1"}, xjson.Map{"id": "host2"}}, 694 } 695 696 req := httptest.NewRequest("POST", "/database/create", 697 xjson.MustNewTestReader(t, jsonInput)) 698 require.NotNil(t, req) 699 700 placementProto := &placementpb.Placement{ 701 Instances: map[string]*placementpb.Instance{ 702 "host1": &placementpb.Instance{ 703 Id: "host1", 704 IsolationGroup: "cluster", 705 Zone: "embedded", 706 Weight: 1, 707 Endpoint: "http://host1:9000", 708 Hostname: "host1", 709 Port: 9000, 710 }, 711 "host2": &placementpb.Instance{ 712 Id: "host2", 713 IsolationGroup: "cluster", 714 Zone: "embedded", 715 Weight: 1, 716 Endpoint: "http://host2:9000", 717 Hostname: "host2", 718 Port: 9000, 719 }, 720 }, 721 } 722 newPlacement, err := placement.NewPlacementFromProto(placementProto) 723 require.NoError(t, err) 724 725 mockPlacementService.EXPECT().Placement().Return(newPlacement, nil) 726 727 createHandler.ServeHTTP(w, req) 728 729 resp := w.Result() 730 _, err = ioutil.ReadAll(resp.Body) 731 assert.NoError(t, err) 732 assert.Equal(t, http.StatusBadRequest, resp.StatusCode) 733 } 734 735 func TestClusterTypeHostsPlacementAlreadyExistsExistingIsLocal(t *testing.T) { 736 ctrl := gomock.NewController(t) 737 defer ctrl.Finish() 738 739 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 740 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 741 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 742 require.NoError(t, err) 743 w := httptest.NewRecorder() 744 745 jsonInput := xjson.Map{ 746 "namespaceName": "testNamespace", 747 "type": "cluster", 748 } 749 750 req := httptest.NewRequest("POST", "/database/create", 751 xjson.MustNewTestReader(t, jsonInput)) 752 require.NotNil(t, req) 753 754 placementProto := &placementpb.Placement{ 755 Instances: map[string]*placementpb.Instance{ 756 "localhost": &placementpb.Instance{ 757 Id: DefaultLocalHostID, 758 IsolationGroup: "local", 759 Zone: "embedded", 760 Weight: 1, 761 Endpoint: "http://localhost:9000", 762 Hostname: "localhost", 763 Port: 9000, 764 }, 765 }, 766 } 767 newPlacement, err := placement.NewPlacementFromProto(placementProto) 768 require.NoError(t, err) 769 770 mockPlacementService.EXPECT().Placement().Return(newPlacement, nil) 771 772 createHandler.ServeHTTP(w, req) 773 774 resp := w.Result() 775 _, err = ioutil.ReadAll(resp.Body) 776 assert.NoError(t, err) 777 assert.Equal(t, http.StatusBadRequest, resp.StatusCode) 778 } 779 780 func testClusterTypeHosts(t *testing.T, placementExists bool) { 781 ctrl := gomock.NewController(t) 782 defer ctrl.Finish() 783 784 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 785 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 786 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 787 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 788 require.NoError(t, err) 789 w := httptest.NewRecorder() 790 791 var jsonInput xjson.Map 792 793 if placementExists { 794 jsonInput = xjson.Map{ 795 "namespaceName": "testNamespace", 796 "type": "cluster", 797 } 798 } else { 799 jsonInput = xjson.Map{ 800 "namespaceName": "testNamespace", 801 "type": "cluster", 802 "hosts": xjson.Array{xjson.Map{"id": "host1"}, xjson.Map{"id": "host2"}}, 803 } 804 } 805 806 reqBody := bytes.NewBuffer(nil) 807 require.NoError(t, json.NewEncoder(reqBody).Encode(jsonInput)) 808 809 req := httptest.NewRequest("POST", "/database/create", reqBody) 810 require.NotNil(t, req) 811 812 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 813 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 814 815 placementProto := &placementpb.Placement{ 816 Instances: map[string]*placementpb.Instance{ 817 "host1": &placementpb.Instance{ 818 Id: "host1", 819 IsolationGroup: "cluster", 820 Zone: "embedded", 821 Weight: 1, 822 Endpoint: "http://host1:9000", 823 Hostname: "host1", 824 Port: 9000, 825 }, 826 "host2": &placementpb.Instance{ 827 Id: "host2", 828 IsolationGroup: "cluster", 829 Zone: "embedded", 830 Weight: 1, 831 Endpoint: "http://host2:9000", 832 Hostname: "host2", 833 Port: 9000, 834 }, 835 }, 836 } 837 newPlacement, err := placement.NewPlacementFromProto(placementProto) 838 require.NoError(t, err) 839 840 if placementExists { 841 mockPlacementService.EXPECT().Placement().Return(newPlacement, nil) 842 } else { 843 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 844 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 128, 3).Return(newPlacement, nil) 845 } 846 847 createHandler.ServeHTTP(w, req) 848 849 resp := w.Result() 850 body, err := ioutil.ReadAll(resp.Body) 851 assert.NoError(t, err) 852 assert.Equal(t, http.StatusOK, resp.StatusCode) 853 854 expectedResponse := ` 855 { 856 "namespace": { 857 "registry": { 858 "namespaces": { 859 "testNamespace": { 860 "aggregationOptions": { 861 "aggregations": [ 862 { 863 "aggregated": false, 864 "attributes": null 865 } 866 ] 867 }, 868 "bootstrapEnabled": true, 869 "cacheBlocksOnRetrieve": false, 870 "flushEnabled": true, 871 "writesToCommitLog": true, 872 "cleanupEnabled": true, 873 "repairEnabled": false, 874 "retentionOptions": { 875 "retentionPeriodNanos": "86400000000000", 876 "blockSizeNanos": "3600000000000", 877 "bufferFutureNanos": "120000000000", 878 "bufferPastNanos": "600000000000", 879 "blockDataExpiry": true, 880 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 881 "futureRetentionPeriodNanos": "0" 882 }, 883 "snapshotEnabled": true, 884 "indexOptions": { 885 "enabled": true, 886 "blockSizeNanos": "3600000000000" 887 }, 888 "runtimeOptions": null, 889 "schemaOptions": null, 890 "coldWritesEnabled": false, 891 "extendedOptions": null, 892 "stagingState": { 893 "status": "UNKNOWN" 894 } 895 } 896 } 897 } 898 }, 899 "placement": { 900 "placement": { 901 "instances": { 902 "host1": { 903 "id": "host1", 904 "isolationGroup": "cluster", 905 "zone": "embedded", 906 "weight": 1, 907 "endpoint": "http://host1:9000", 908 "shards": [], 909 "shardSetId": 0, 910 "hostname": "host1", 911 "port": 9000, 912 "metadata": { 913 "debugPort": 0 914 } 915 }, 916 "host2": { 917 "id": "host2", 918 "isolationGroup": "cluster", 919 "zone": "embedded", 920 "weight": 1, 921 "endpoint": "http://host2:9000", 922 "shards": [], 923 "shardSetId": 0, 924 "hostname": "host2", 925 "port": 9000, 926 "metadata": { 927 "debugPort": 0 928 } 929 } 930 }, 931 "replicaFactor": 0, 932 "numShards": 0, 933 "isSharded": false, 934 "cutoverTime": "0", 935 "isMirrored": false, 936 "maxShardSetId": 0 937 }, 938 "version": 0 939 } 940 } 941 ` 942 943 expected := xtest.MustPrettyJSONString(t, expectedResponse) 944 actual := xtest.MustPrettyJSONString(t, string(body)) 945 946 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 947 } 948 949 func TestClusterTypeHostsWithIsolationGroup(t *testing.T) { 950 ctrl := gomock.NewController(t) 951 defer ctrl.Finish() 952 953 mockClient, mockKV, mockPlacementService := SetupDatabaseTest(t, ctrl) 954 mockClient.EXPECT().Store(gomock.Any()).Return(mockKV, nil).AnyTimes() 955 956 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 957 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 958 require.NoError(t, err) 959 w := httptest.NewRecorder() 960 961 jsonInput := xjson.Map{ 962 "namespaceName": "testNamespace", 963 "type": "cluster", 964 "hosts": xjson.Array{ 965 xjson.Map{"id": "host1", "isolationGroup": "group1"}, 966 xjson.Map{"id": "host2", "isolationGroup": "group2"}, 967 }, 968 } 969 970 req := httptest.NewRequest("POST", "/database/create", 971 xjson.MustNewTestReader(t, jsonInput)) 972 require.NotNil(t, req) 973 974 mockKV.EXPECT().Get(namespace.M3DBNodeNamespacesKey).Return(nil, kv.ErrNotFound).Times(2) 975 mockKV.EXPECT().CheckAndSet(namespace.M3DBNodeNamespacesKey, gomock.Any(), gomock.Not(nil)).Return(1, nil) 976 977 placementProto := &placementpb.Placement{ 978 Instances: map[string]*placementpb.Instance{ 979 "host1": &placementpb.Instance{ 980 Id: "host1", 981 IsolationGroup: "group1", 982 Zone: "embedded", 983 Weight: 1, 984 Endpoint: "http://host1:9000", 985 Hostname: "host1", 986 Port: 9000, 987 }, 988 "host2": &placementpb.Instance{ 989 Id: "host2", 990 IsolationGroup: "group2", 991 Zone: "embedded", 992 Weight: 1, 993 Endpoint: "http://host2:9000", 994 Hostname: "host2", 995 Port: 9000, 996 }, 997 }, 998 } 999 newPlacement, err := placement.NewPlacementFromProto(placementProto) 1000 require.NoError(t, err) 1001 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 1002 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 128, 3).Return(newPlacement, nil) 1003 1004 createHandler.ServeHTTP(w, req) 1005 1006 resp := w.Result() 1007 body, err := ioutil.ReadAll(resp.Body) 1008 assert.NoError(t, err) 1009 assert.Equal(t, http.StatusOK, resp.StatusCode) 1010 1011 expectedResponse := ` 1012 { 1013 "namespace": { 1014 "registry": { 1015 "namespaces": { 1016 "testNamespace": { 1017 "aggregationOptions": { 1018 "aggregations": [ 1019 { 1020 "aggregated": false, 1021 "attributes": null 1022 } 1023 ] 1024 }, 1025 "bootstrapEnabled": true, 1026 "cacheBlocksOnRetrieve": false, 1027 "flushEnabled": true, 1028 "writesToCommitLog": true, 1029 "cleanupEnabled": true, 1030 "repairEnabled": false, 1031 "retentionOptions": { 1032 "retentionPeriodNanos": "86400000000000", 1033 "blockSizeNanos": "3600000000000", 1034 "bufferFutureNanos": "120000000000", 1035 "bufferPastNanos": "600000000000", 1036 "blockDataExpiry": true, 1037 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 1038 "futureRetentionPeriodNanos": "0" 1039 }, 1040 "snapshotEnabled": true, 1041 "indexOptions": { 1042 "enabled": true, 1043 "blockSizeNanos": "3600000000000" 1044 }, 1045 "runtimeOptions": null, 1046 "schemaOptions": null, 1047 "coldWritesEnabled": false, 1048 "extendedOptions": null, 1049 "stagingState": { 1050 "status": "UNKNOWN" 1051 } 1052 } 1053 } 1054 } 1055 }, 1056 "placement": { 1057 "placement": { 1058 "instances": { 1059 "host1": { 1060 "id": "host1", 1061 "isolationGroup": "group1", 1062 "zone": "embedded", 1063 "weight": 1, 1064 "endpoint": "http://host1:9000", 1065 "shards": [], 1066 "shardSetId": 0, 1067 "hostname": "host1", 1068 "port": 9000, 1069 "metadata": { 1070 "debugPort": 0 1071 } 1072 }, 1073 "host2": { 1074 "id": "host2", 1075 "isolationGroup": "group2", 1076 "zone": "embedded", 1077 "weight": 1, 1078 "endpoint": "http://host2:9000", 1079 "shards": [], 1080 "shardSetId": 0, 1081 "hostname": "host2", 1082 "port": 9000, 1083 "metadata": { 1084 "debugPort": 0 1085 } 1086 } 1087 }, 1088 "replicaFactor": 0, 1089 "numShards": 0, 1090 "isSharded": false, 1091 "cutoverTime": "0", 1092 "isMirrored": false, 1093 "maxShardSetId": 0 1094 }, 1095 "version": 0 1096 } 1097 } 1098 ` 1099 1100 expected := xtest.MustPrettyJSONString(t, expectedResponse) 1101 actual := xtest.MustPrettyJSONString(t, string(body)) 1102 1103 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 1104 } 1105 func TestClusterTypeMissingHostnames(t *testing.T) { 1106 ctrl := gomock.NewController(t) 1107 defer ctrl.Finish() 1108 1109 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 1110 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 1111 1112 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 1113 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 1114 require.NoError(t, err) 1115 w := httptest.NewRecorder() 1116 1117 jsonInput := xjson.Map{ 1118 "namespaceName": "testNamespace", 1119 "type": "cluster", 1120 } 1121 1122 req := httptest.NewRequest("POST", "/database/create", 1123 xjson.MustNewTestReader(t, jsonInput)) 1124 require.NotNil(t, req) 1125 1126 createHandler.ServeHTTP(w, req) 1127 1128 resp := w.Result() 1129 body, err := ioutil.ReadAll(resp.Body) 1130 assert.NoError(t, err) 1131 assert.Equal(t, http.StatusBadRequest, resp.StatusCode) 1132 assert.Equal(t, 1133 xtest.MustPrettyJSONMap(t, 1134 xjson.Map{ 1135 "status": "error", 1136 "error": "missing required field", 1137 }, 1138 ), 1139 xtest.MustPrettyJSONString(t, string(body))) 1140 } 1141 1142 func TestBadType(t *testing.T) { 1143 ctrl := gomock.NewController(t) 1144 defer ctrl.Finish() 1145 1146 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 1147 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 1148 1149 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 1150 nil, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 1151 require.NoError(t, err) 1152 w := httptest.NewRecorder() 1153 1154 jsonInput := xjson.Map{ 1155 "namespaceName": "testNamespace", 1156 "type": "badtype", 1157 } 1158 1159 req := httptest.NewRequest("POST", "/database/create", 1160 xjson.MustNewTestReader(t, jsonInput)) 1161 require.NotNil(t, req) 1162 createHandler.ServeHTTP(w, req) 1163 1164 resp := w.Result() 1165 body, err := ioutil.ReadAll(resp.Body) 1166 assert.NoError(t, err) 1167 assert.Equal(t, http.StatusBadRequest, resp.StatusCode) 1168 assert.Equal(t, 1169 xtest.MustPrettyJSONMap(t, 1170 xjson.Map{ 1171 "status": "error", 1172 "error": "invalid database type", 1173 }, 1174 ), 1175 xtest.MustPrettyJSONString(t, string(body))) 1176 } 1177 1178 func TestLocalTypeWithAggregatedNamespace(t *testing.T) { 1179 ctrl := gomock.NewController(t) 1180 defer ctrl.Finish() 1181 1182 mockClient, _, mockPlacementService := SetupDatabaseTest(t, ctrl) 1183 fakeKV := fake.NewStore() 1184 mockClient.EXPECT().Store(gomock.Any()).Return(fakeKV, nil).AnyTimes() 1185 createHandler, err := NewCreateHandler(mockClient, config.Configuration{}, 1186 testDBCfg, svcDefaultOptions, instrument.NewOptions(), validators.NamespaceValidator) 1187 require.NoError(t, err) 1188 1189 w := httptest.NewRecorder() 1190 1191 jsonInput := xjson.Map{ 1192 "namespaceName": "testNamespace", 1193 "type": "local", 1194 "aggregatedNamespace": xjson.Map{ 1195 "name": "testAggregatedNamespace", 1196 "resolution": "5m", 1197 "retentionTime": "2440h", 1198 }, 1199 } 1200 1201 req := httptest.NewRequest("POST", "/database/create", 1202 xjson.MustNewTestReader(t, jsonInput)) 1203 require.NotNil(t, req) 1204 1205 placementProto := &placementpb.Placement{ 1206 Instances: map[string]*placementpb.Instance{ 1207 "localhost": &placementpb.Instance{ 1208 Id: "m3db_local", 1209 IsolationGroup: "local", 1210 Zone: "embedded", 1211 Weight: 1, 1212 Endpoint: "http://localhost:9000", 1213 Hostname: "localhost", 1214 Port: 9000, 1215 }, 1216 }, 1217 } 1218 newPlacement, err := placement.NewPlacementFromProto(placementProto) 1219 require.NoError(t, err) 1220 mockPlacementService.EXPECT().Placement().Return(nil, kv.ErrNotFound) 1221 mockPlacementService.EXPECT().BuildInitialPlacement(gomock.Any(), 64, 1).Return(newPlacement, nil) 1222 1223 createHandler.ServeHTTP(w, req) 1224 1225 resp := w.Result() 1226 body, err := ioutil.ReadAll(resp.Body) 1227 assert.NoError(t, err) 1228 assert.Equal(t, http.StatusOK, resp.StatusCode) 1229 1230 expectedResponse := ` 1231 { 1232 "namespace": { 1233 "registry": { 1234 "namespaces": { 1235 "testNamespace": { 1236 "aggregationOptions": { 1237 "aggregations": [ 1238 { 1239 "aggregated": false, 1240 "attributes": null 1241 } 1242 ] 1243 }, 1244 "bootstrapEnabled": true, 1245 "cacheBlocksOnRetrieve": false, 1246 "flushEnabled": true, 1247 "writesToCommitLog": true, 1248 "cleanupEnabled": true, 1249 "repairEnabled": false, 1250 "retentionOptions": { 1251 "retentionPeriodNanos": "86400000000000", 1252 "blockSizeNanos": "3600000000000", 1253 "bufferFutureNanos": "120000000000", 1254 "bufferPastNanos": "600000000000", 1255 "blockDataExpiry": true, 1256 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 1257 "futureRetentionPeriodNanos": "0" 1258 }, 1259 "snapshotEnabled": true, 1260 "indexOptions": { 1261 "enabled": true, 1262 "blockSizeNanos": "3600000000000" 1263 }, 1264 "runtimeOptions": null, 1265 "schemaOptions": null, 1266 "coldWritesEnabled": false, 1267 "extendedOptions": null, 1268 "stagingState": { 1269 "status": "UNKNOWN" 1270 } 1271 }, 1272 "testAggregatedNamespace": { 1273 "aggregationOptions": { 1274 "aggregations": [ 1275 { 1276 "aggregated": true, 1277 "attributes": { 1278 "resolutionNanos": "300000000000", 1279 "downsampleOptions": { 1280 "all": true 1281 } 1282 } 1283 } 1284 ] 1285 }, 1286 "bootstrapEnabled": true, 1287 "cacheBlocksOnRetrieve": false, 1288 "flushEnabled": true, 1289 "writesToCommitLog": true, 1290 "cleanupEnabled": true, 1291 "repairEnabled": false, 1292 "retentionOptions": { 1293 "retentionPeriodNanos": "8784000000000000", 1294 "blockSizeNanos": "86400000000000", 1295 "bufferFutureNanos": "120000000000", 1296 "bufferPastNanos": "600000000000", 1297 "blockDataExpiry": true, 1298 "blockDataExpiryAfterNotAccessPeriodNanos": "300000000000", 1299 "futureRetentionPeriodNanos": "0" 1300 }, 1301 "snapshotEnabled": true, 1302 "indexOptions": { 1303 "enabled": true, 1304 "blockSizeNanos": "86400000000000" 1305 }, 1306 "runtimeOptions": null, 1307 "schemaOptions": null, 1308 "coldWritesEnabled": false, 1309 "extendedOptions": null, 1310 "stagingState": { 1311 "status": "UNKNOWN" 1312 } 1313 } 1314 } 1315 } 1316 }, 1317 "placement": { 1318 "placement": { 1319 "instances": { 1320 "m3db_local": { 1321 "id": "m3db_local", 1322 "isolationGroup": "local", 1323 "zone": "embedded", 1324 "weight": 1, 1325 "endpoint": "http://localhost:9000", 1326 "shards": [], 1327 "shardSetId": 0, 1328 "hostname": "localhost", 1329 "port": 9000, 1330 "metadata": { 1331 "debugPort": 0 1332 } 1333 } 1334 }, 1335 "replicaFactor": 0, 1336 "numShards": 0, 1337 "isSharded": false, 1338 "cutoverTime": "0", 1339 "isMirrored": false, 1340 "maxShardSetId": 0 1341 }, 1342 "version": 0 1343 } 1344 } 1345 ` 1346 expected := xtest.MustPrettyJSONString(t, expectedResponse) 1347 actual := xtest.MustPrettyJSONString(t, string(body)) 1348 1349 assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) 1350 }