github.com/clly/consul@v1.4.5/agent/consul/prepared_query_endpoint_test.go (about) 1 package consul 2 3 import ( 4 "bytes" 5 "fmt" 6 "log" 7 "net/rpc" 8 "os" 9 "reflect" 10 "sort" 11 "strings" 12 "testing" 13 "time" 14 15 "github.com/hashicorp/consul/acl" 16 "github.com/hashicorp/consul/agent/structs" 17 "github.com/hashicorp/consul/api" 18 "github.com/hashicorp/consul/testrpc" 19 "github.com/hashicorp/consul/testutil/retry" 20 "github.com/hashicorp/consul/types" 21 "github.com/hashicorp/net-rpc-msgpackrpc" 22 "github.com/hashicorp/serf/coordinate" 23 "github.com/stretchr/testify/require" 24 ) 25 26 func TestPreparedQuery_Apply(t *testing.T) { 27 t.Parallel() 28 dir1, s1 := testServer(t) 29 defer os.RemoveAll(dir1) 30 defer s1.Shutdown() 31 codec := rpcClient(t, s1) 32 defer codec.Close() 33 34 testrpc.WaitForLeader(t, s1.RPC, "dc1") 35 36 // Set up a bare bones query. 37 query := structs.PreparedQueryRequest{ 38 Datacenter: "dc1", 39 Op: structs.PreparedQueryCreate, 40 Query: &structs.PreparedQuery{ 41 Name: "test", 42 Service: structs.ServiceQuery{ 43 Service: "redis", 44 }, 45 }, 46 } 47 var reply string 48 49 // Set an ID which should fail the create. 50 query.Query.ID = "nope" 51 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 52 if err == nil || !strings.Contains(err.Error(), "ID must be empty") { 53 t.Fatalf("bad: %v", err) 54 } 55 56 // Change it to a bogus modify which should also fail. 57 query.Op = structs.PreparedQueryUpdate 58 query.Query.ID = generateUUID() 59 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 60 if err == nil || !strings.Contains(err.Error(), "Cannot modify non-existent prepared query") { 61 t.Fatalf("bad: %v", err) 62 } 63 64 // Fix up the ID but invalidate the query itself. This proves we call 65 // parseQuery for a create, but that function is checked in detail as 66 // part of another test so we don't have to exercise all the checks 67 // here. 68 query.Op = structs.PreparedQueryCreate 69 query.Query.ID = "" 70 query.Query.Service.Failover.NearestN = -1 71 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 72 if err == nil || !strings.Contains(err.Error(), "Bad NearestN") { 73 t.Fatalf("bad: %v", err) 74 } 75 76 // Fix that and make sure it propagates an error from the Raft apply. 77 query.Query.Service.Failover.NearestN = 0 78 query.Query.Session = "nope" 79 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 80 if err == nil || !strings.Contains(err.Error(), "failed session lookup") { 81 t.Fatalf("bad: %v", err) 82 } 83 84 // Fix that and make sure the apply goes through. 85 query.Query.Session = "" 86 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 87 t.Fatalf("err: %v", err) 88 } 89 90 // Capture the ID and read the query back to verify. 91 query.Query.ID = reply 92 { 93 req := &structs.PreparedQuerySpecificRequest{ 94 Datacenter: "dc1", 95 QueryID: query.Query.ID, 96 } 97 var resp structs.IndexedPreparedQueries 98 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 99 t.Fatalf("err: %v", err) 100 } 101 102 if len(resp.Queries) != 1 { 103 t.Fatalf("bad: %v", resp) 104 } 105 actual := resp.Queries[0] 106 if resp.Index != actual.ModifyIndex { 107 t.Fatalf("bad index: %d", resp.Index) 108 } 109 actual.CreateIndex, actual.ModifyIndex = 0, 0 110 if !reflect.DeepEqual(actual, query.Query) { 111 t.Fatalf("bad: %v", actual) 112 } 113 } 114 115 // Make the op an update. This should go through now that we have an ID. 116 query.Op = structs.PreparedQueryUpdate 117 query.Query.Service.Failover.NearestN = 2 118 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 119 t.Fatalf("err: %v", err) 120 } 121 122 // Read back again to verify the update worked. 123 { 124 req := &structs.PreparedQuerySpecificRequest{ 125 Datacenter: "dc1", 126 QueryID: query.Query.ID, 127 } 128 var resp structs.IndexedPreparedQueries 129 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 130 t.Fatalf("err: %v", err) 131 } 132 133 if len(resp.Queries) != 1 { 134 t.Fatalf("bad: %v", resp) 135 } 136 actual := resp.Queries[0] 137 if resp.Index != actual.ModifyIndex { 138 t.Fatalf("bad index: %d", resp.Index) 139 } 140 actual.CreateIndex, actual.ModifyIndex = 0, 0 141 if !reflect.DeepEqual(actual, query.Query) { 142 t.Fatalf("bad: %v", actual) 143 } 144 } 145 146 // Give a bogus op and make sure it fails. 147 query.Op = "nope" 148 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 149 if err == nil || !strings.Contains(err.Error(), "Unknown prepared query operation:") { 150 t.Fatalf("bad: %v", err) 151 } 152 153 // Prove that an update also goes through the parseQuery validation. 154 query.Op = structs.PreparedQueryUpdate 155 query.Query.Service.Failover.NearestN = -1 156 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 157 if err == nil || !strings.Contains(err.Error(), "Bad NearestN") { 158 t.Fatalf("bad: %v", err) 159 } 160 161 // Now change the op to delete; the bad query field should be ignored 162 // because all we care about for a delete op is the ID. 163 query.Op = structs.PreparedQueryDelete 164 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 165 t.Fatalf("err: %v", err) 166 } 167 168 // Verify that this query is deleted. 169 { 170 req := &structs.PreparedQuerySpecificRequest{ 171 Datacenter: "dc1", 172 QueryID: query.Query.ID, 173 } 174 var resp structs.IndexedPreparedQueries 175 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 176 if err.Error() != ErrQueryNotFound.Error() { 177 t.Fatalf("err: %v", err) 178 } 179 } 180 181 if len(resp.Queries) != 0 { 182 t.Fatalf("bad: %v", resp) 183 } 184 } 185 } 186 187 func TestPreparedQuery_Apply_ACLDeny(t *testing.T) { 188 t.Parallel() 189 dir1, s1 := testServerWithConfig(t, func(c *Config) { 190 c.ACLDatacenter = "dc1" 191 c.ACLsEnabled = true 192 c.ACLMasterToken = "root" 193 c.ACLDefaultPolicy = "deny" 194 }) 195 defer os.RemoveAll(dir1) 196 defer s1.Shutdown() 197 codec := rpcClient(t, s1) 198 defer codec.Close() 199 200 testrpc.WaitForLeader(t, s1.RPC, "dc1") 201 202 // Create an ACL with write permissions for redis queries. 203 var token string 204 { 205 var rules = ` 206 query "redis" { 207 policy = "write" 208 } 209 ` 210 211 req := structs.ACLRequest{ 212 Datacenter: "dc1", 213 Op: structs.ACLSet, 214 ACL: structs.ACL{ 215 Name: "User token", 216 Type: structs.ACLTokenTypeClient, 217 Rules: rules, 218 }, 219 WriteRequest: structs.WriteRequest{Token: "root"}, 220 } 221 if err := msgpackrpc.CallWithCodec(codec, "ACL.Apply", &req, &token); err != nil { 222 t.Fatalf("err: %v", err) 223 } 224 } 225 226 // Set up a bare bones query. 227 query := structs.PreparedQueryRequest{ 228 Datacenter: "dc1", 229 Op: structs.PreparedQueryCreate, 230 Query: &structs.PreparedQuery{ 231 Name: "redis-master", 232 Service: structs.ServiceQuery{ 233 Service: "the-redis", 234 }, 235 }, 236 } 237 var reply string 238 239 // Creating without a token should fail since the default policy is to 240 // deny. 241 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 242 if !acl.IsErrPermissionDenied(err) { 243 t.Fatalf("bad: %v", err) 244 } 245 246 // Now add the token and try again. 247 query.WriteRequest.Token = token 248 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 249 t.Fatalf("err: %v", err) 250 } 251 252 // Capture the ID and set the token, then read back the query to verify. 253 // Note that unlike previous versions of Consul, we DO NOT capture the 254 // token. We will set that here just to be explicit about it. 255 query.Query.ID = reply 256 query.Query.Token = "" 257 { 258 req := &structs.PreparedQuerySpecificRequest{ 259 Datacenter: "dc1", 260 QueryID: query.Query.ID, 261 QueryOptions: structs.QueryOptions{Token: "root"}, 262 } 263 var resp structs.IndexedPreparedQueries 264 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 265 t.Fatalf("err: %v", err) 266 } 267 268 if len(resp.Queries) != 1 { 269 t.Fatalf("bad: %v", resp) 270 } 271 actual := resp.Queries[0] 272 if resp.Index != actual.ModifyIndex { 273 t.Fatalf("bad index: %d", resp.Index) 274 } 275 actual.CreateIndex, actual.ModifyIndex = 0, 0 276 if !reflect.DeepEqual(actual, query.Query) { 277 t.Fatalf("bad: %v", actual) 278 } 279 } 280 281 // Try to do an update without a token; this should get rejected. 282 query.Op = structs.PreparedQueryUpdate 283 query.WriteRequest.Token = "" 284 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 285 if !acl.IsErrPermissionDenied(err) { 286 t.Fatalf("bad: %v", err) 287 } 288 289 // Try again with the original token; this should go through. 290 query.WriteRequest.Token = token 291 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 292 t.Fatalf("err: %v", err) 293 } 294 295 // Try to do a delete with no token; this should get rejected. 296 query.Op = structs.PreparedQueryDelete 297 query.WriteRequest.Token = "" 298 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 299 if !acl.IsErrPermissionDenied(err) { 300 t.Fatalf("bad: %v", err) 301 } 302 303 // Try again with the original token. This should go through. 304 query.WriteRequest.Token = token 305 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 306 t.Fatalf("err: %v", err) 307 } 308 309 // Make sure the query got deleted. 310 { 311 req := &structs.PreparedQuerySpecificRequest{ 312 Datacenter: "dc1", 313 QueryID: query.Query.ID, 314 QueryOptions: structs.QueryOptions{Token: "root"}, 315 } 316 var resp structs.IndexedPreparedQueries 317 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 318 if err.Error() != ErrQueryNotFound.Error() { 319 t.Fatalf("err: %v", err) 320 } 321 } 322 323 if len(resp.Queries) != 0 { 324 t.Fatalf("bad: %v", resp) 325 } 326 } 327 328 // Make the query again. 329 query.Op = structs.PreparedQueryCreate 330 query.Query.ID = "" 331 query.WriteRequest.Token = token 332 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 333 t.Fatalf("err: %v", err) 334 } 335 336 // Check that it's there, and again make sure that the token did not get 337 // captured. 338 query.Query.ID = reply 339 query.Query.Token = "" 340 { 341 req := &structs.PreparedQuerySpecificRequest{ 342 Datacenter: "dc1", 343 QueryID: query.Query.ID, 344 QueryOptions: structs.QueryOptions{Token: "root"}, 345 } 346 var resp structs.IndexedPreparedQueries 347 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 348 t.Fatalf("err: %v", err) 349 } 350 351 if len(resp.Queries) != 1 { 352 t.Fatalf("bad: %v", resp) 353 } 354 actual := resp.Queries[0] 355 if resp.Index != actual.ModifyIndex { 356 t.Fatalf("bad index: %d", resp.Index) 357 } 358 actual.CreateIndex, actual.ModifyIndex = 0, 0 359 if !reflect.DeepEqual(actual, query.Query) { 360 t.Fatalf("bad: %v", actual) 361 } 362 } 363 364 // A management token should be able to update the query no matter what. 365 query.Op = structs.PreparedQueryUpdate 366 query.WriteRequest.Token = "root" 367 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 368 t.Fatalf("err: %v", err) 369 } 370 371 // That last update should not have captured a token. 372 query.Query.Token = "" 373 { 374 req := &structs.PreparedQuerySpecificRequest{ 375 Datacenter: "dc1", 376 QueryID: query.Query.ID, 377 QueryOptions: structs.QueryOptions{Token: "root"}, 378 } 379 var resp structs.IndexedPreparedQueries 380 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 381 t.Fatalf("err: %v", err) 382 } 383 384 if len(resp.Queries) != 1 { 385 t.Fatalf("bad: %v", resp) 386 } 387 actual := resp.Queries[0] 388 if resp.Index != actual.ModifyIndex { 389 t.Fatalf("bad index: %d", resp.Index) 390 } 391 actual.CreateIndex, actual.ModifyIndex = 0, 0 392 if !reflect.DeepEqual(actual, query.Query) { 393 t.Fatalf("bad: %v", actual) 394 } 395 } 396 397 // A management token should be able to delete the query no matter what. 398 query.Op = structs.PreparedQueryDelete 399 query.WriteRequest.Token = "root" 400 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 401 t.Fatalf("err: %v", err) 402 } 403 404 // Make sure the query got deleted. 405 { 406 req := &structs.PreparedQuerySpecificRequest{ 407 Datacenter: "dc1", 408 QueryID: query.Query.ID, 409 QueryOptions: structs.QueryOptions{Token: "root"}, 410 } 411 var resp structs.IndexedPreparedQueries 412 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 413 if err.Error() != ErrQueryNotFound.Error() { 414 t.Fatalf("err: %v", err) 415 } 416 } 417 418 if len(resp.Queries) != 0 { 419 t.Fatalf("bad: %v", resp) 420 } 421 } 422 423 // Use the root token to make a query under a different name. 424 query.Op = structs.PreparedQueryCreate 425 query.Query.ID = "" 426 query.Query.Name = "cassandra" 427 query.WriteRequest.Token = "root" 428 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 429 t.Fatalf("err: %v", err) 430 } 431 432 // Check that it's there and that the token did not get captured. 433 query.Query.ID = reply 434 query.Query.Token = "" 435 { 436 req := &structs.PreparedQuerySpecificRequest{ 437 Datacenter: "dc1", 438 QueryID: query.Query.ID, 439 QueryOptions: structs.QueryOptions{Token: "root"}, 440 } 441 var resp structs.IndexedPreparedQueries 442 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 443 t.Fatalf("err: %v", err) 444 } 445 446 if len(resp.Queries) != 1 { 447 t.Fatalf("bad: %v", resp) 448 } 449 actual := resp.Queries[0] 450 if resp.Index != actual.ModifyIndex { 451 t.Fatalf("bad index: %d", resp.Index) 452 } 453 actual.CreateIndex, actual.ModifyIndex = 0, 0 454 if !reflect.DeepEqual(actual, query.Query) { 455 t.Fatalf("bad: %v", actual) 456 } 457 } 458 459 // Now try to change that to redis with the valid redis token. This will 460 // fail because that token can't change cassandra queries. 461 query.Op = structs.PreparedQueryUpdate 462 query.Query.Name = "redis" 463 query.WriteRequest.Token = token 464 err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 465 if !acl.IsErrPermissionDenied(err) { 466 t.Fatalf("bad: %v", err) 467 } 468 } 469 470 func TestPreparedQuery_Apply_ForwardLeader(t *testing.T) { 471 t.Parallel() 472 dir1, s1 := testServerWithConfig(t, func(c *Config) { 473 c.Bootstrap = false 474 }) 475 defer os.RemoveAll(dir1) 476 defer s1.Shutdown() 477 codec1 := rpcClient(t, s1) 478 defer codec1.Close() 479 480 dir2, s2 := testServer(t) 481 defer os.RemoveAll(dir2) 482 defer s2.Shutdown() 483 codec2 := rpcClient(t, s2) 484 defer codec2.Close() 485 486 // Try to join. 487 joinLAN(t, s2, s1) 488 489 testrpc.WaitForLeader(t, s1.RPC, "dc1") 490 testrpc.WaitForLeader(t, s2.RPC, "dc1") 491 492 // Use the follower as the client. 493 var codec rpc.ClientCodec 494 if !s1.IsLeader() { 495 codec = codec1 496 } else { 497 codec = codec2 498 } 499 500 // Set up a node and service in the catalog. 501 { 502 req := structs.RegisterRequest{ 503 Datacenter: "dc1", 504 Node: "foo", 505 Address: "127.0.0.1", 506 Service: &structs.NodeService{ 507 Service: "redis", 508 Tags: []string{"master"}, 509 Port: 8000, 510 }, 511 } 512 var reply struct{} 513 err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply) 514 if err != nil { 515 t.Fatalf("err: %v", err) 516 } 517 } 518 519 // Set up a bare bones query. 520 query := structs.PreparedQueryRequest{ 521 Datacenter: "dc1", 522 Op: structs.PreparedQueryCreate, 523 Query: &structs.PreparedQuery{ 524 Name: "test", 525 Service: structs.ServiceQuery{ 526 Service: "redis", 527 }, 528 }, 529 } 530 531 // Make sure the apply works even when forwarded through the non-leader. 532 var reply string 533 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 534 t.Fatalf("err: %v", err) 535 } 536 } 537 538 func TestPreparedQuery_parseQuery(t *testing.T) { 539 t.Parallel() 540 query := &structs.PreparedQuery{} 541 542 err := parseQuery(query, true) 543 if err == nil || !strings.Contains(err.Error(), "Must be bound to a session") { 544 t.Fatalf("bad: %v", err) 545 } 546 547 query.Session = "adf4238a-882b-9ddc-4a9d-5b6758e4159e" 548 err = parseQuery(query, true) 549 if err == nil || !strings.Contains(err.Error(), "Must provide a Service") { 550 t.Fatalf("bad: %v", err) 551 } 552 553 query.Session = "" 554 query.Template.Type = "some-kind-of-template" 555 err = parseQuery(query, true) 556 if err == nil || !strings.Contains(err.Error(), "Must provide a Service") { 557 t.Fatalf("bad: %v", err) 558 } 559 560 query.Template.Type = "" 561 err = parseQuery(query, false) 562 if err == nil || !strings.Contains(err.Error(), "Must provide a Service") { 563 t.Fatalf("bad: %v", err) 564 } 565 566 // None of the rest of these care about version 8 ACL enforcement. 567 for _, version8 := range []bool{true, false} { 568 query = &structs.PreparedQuery{} 569 query.Session = "adf4238a-882b-9ddc-4a9d-5b6758e4159e" 570 query.Service.Service = "foo" 571 if err := parseQuery(query, version8); err != nil { 572 t.Fatalf("err: %v", err) 573 } 574 575 query.Token = redactedToken 576 err = parseQuery(query, version8) 577 if err == nil || !strings.Contains(err.Error(), "Bad Token") { 578 t.Fatalf("bad: %v", err) 579 } 580 581 query.Token = "adf4238a-882b-9ddc-4a9d-5b6758e4159e" 582 if err := parseQuery(query, version8); err != nil { 583 t.Fatalf("err: %v", err) 584 } 585 586 query.Service.Failover.NearestN = -1 587 err = parseQuery(query, version8) 588 if err == nil || !strings.Contains(err.Error(), "Bad NearestN") { 589 t.Fatalf("bad: %v", err) 590 } 591 592 query.Service.Failover.NearestN = 3 593 if err := parseQuery(query, version8); err != nil { 594 t.Fatalf("err: %v", err) 595 } 596 597 query.DNS.TTL = "two fortnights" 598 err = parseQuery(query, version8) 599 if err == nil || !strings.Contains(err.Error(), "Bad DNS TTL") { 600 t.Fatalf("bad: %v", err) 601 } 602 603 query.DNS.TTL = "-3s" 604 err = parseQuery(query, version8) 605 if err == nil || !strings.Contains(err.Error(), "must be >=0") { 606 t.Fatalf("bad: %v", err) 607 } 608 609 query.DNS.TTL = "3s" 610 if err := parseQuery(query, version8); err != nil { 611 t.Fatalf("err: %v", err) 612 } 613 614 query.Service.NodeMeta = map[string]string{"": "somevalue"} 615 err = parseQuery(query, version8) 616 if err == nil || !strings.Contains(err.Error(), "cannot be blank") { 617 t.Fatalf("bad: %v", err) 618 } 619 620 query.Service.NodeMeta = map[string]string{"somekey": "somevalue"} 621 if err := parseQuery(query, version8); err != nil { 622 t.Fatalf("err: %v", err) 623 } 624 } 625 } 626 627 func TestPreparedQuery_ACLDeny_Catchall_Template(t *testing.T) { 628 t.Parallel() 629 dir1, s1 := testServerWithConfig(t, func(c *Config) { 630 c.ACLDatacenter = "dc1" 631 c.ACLsEnabled = true 632 c.ACLMasterToken = "root" 633 c.ACLDefaultPolicy = "deny" 634 }) 635 defer os.RemoveAll(dir1) 636 defer s1.Shutdown() 637 codec := rpcClient(t, s1) 638 defer codec.Close() 639 640 testrpc.WaitForLeader(t, s1.RPC, "dc1") 641 642 // Create an ACL with write permissions for any prefix. 643 var token string 644 { 645 var rules = ` 646 query "" { 647 policy = "write" 648 } 649 ` 650 651 req := structs.ACLRequest{ 652 Datacenter: "dc1", 653 Op: structs.ACLSet, 654 ACL: structs.ACL{ 655 Name: "User token", 656 Type: structs.ACLTokenTypeClient, 657 Rules: rules, 658 }, 659 WriteRequest: structs.WriteRequest{Token: "root"}, 660 } 661 if err := msgpackrpc.CallWithCodec(codec, "ACL.Apply", &req, &token); err != nil { 662 t.Fatalf("err: %v", err) 663 } 664 } 665 666 // Set up a catch-all template. 667 query := structs.PreparedQueryRequest{ 668 Datacenter: "dc1", 669 Op: structs.PreparedQueryCreate, 670 Query: &structs.PreparedQuery{ 671 Name: "", 672 Token: "5e1e24e5-1329-f86f-18c6-3d3734edb2cd", 673 Template: structs.QueryTemplateOptions{ 674 Type: structs.QueryTemplateTypeNamePrefixMatch, 675 }, 676 Service: structs.ServiceQuery{ 677 Service: "${name.full}", 678 }, 679 }, 680 } 681 var reply string 682 683 // Creating without a token should fail since the default policy is to 684 // deny. 685 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply) 686 if !acl.IsErrPermissionDenied(err) { 687 t.Fatalf("bad: %v", err) 688 } 689 690 // Now add the token and try again. 691 query.WriteRequest.Token = token 692 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 693 t.Fatalf("err: %v", err) 694 } 695 696 // Capture the ID and read back the query to verify. Note that the token 697 // will be redacted since this isn't a management token. 698 query.Query.ID = reply 699 query.Query.Token = redactedToken 700 { 701 req := &structs.PreparedQuerySpecificRequest{ 702 Datacenter: "dc1", 703 QueryID: query.Query.ID, 704 QueryOptions: structs.QueryOptions{Token: token}, 705 } 706 var resp structs.IndexedPreparedQueries 707 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 708 t.Fatalf("err: %v", err) 709 } 710 711 if len(resp.Queries) != 1 { 712 t.Fatalf("bad: %v", resp) 713 } 714 actual := resp.Queries[0] 715 if resp.Index != actual.ModifyIndex { 716 t.Fatalf("bad index: %d", resp.Index) 717 } 718 actual.CreateIndex, actual.ModifyIndex = 0, 0 719 if !reflect.DeepEqual(actual, query.Query) { 720 t.Fatalf("bad: %v", actual) 721 } 722 } 723 724 // Try to query by ID without a token and make sure it gets denied, even 725 // though this has an empty name and would normally be shown. 726 { 727 req := &structs.PreparedQuerySpecificRequest{ 728 Datacenter: "dc1", 729 QueryID: query.Query.ID, 730 } 731 var resp structs.IndexedPreparedQueries 732 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp) 733 if !acl.IsErrPermissionDenied(err) { 734 t.Fatalf("bad: %v", err) 735 } 736 737 if len(resp.Queries) != 0 { 738 t.Fatalf("bad: %v", resp) 739 } 740 } 741 742 // We should get the same result listing all the queries without a 743 // token. 744 { 745 req := &structs.DCSpecificRequest{ 746 Datacenter: "dc1", 747 } 748 var resp structs.IndexedPreparedQueries 749 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 750 t.Fatalf("err: %v", err) 751 } 752 753 if len(resp.Queries) != 0 { 754 t.Fatalf("bad: %v", resp) 755 } 756 } 757 758 // But a management token should be able to see it, and the real token. 759 query.Query.Token = "5e1e24e5-1329-f86f-18c6-3d3734edb2cd" 760 { 761 req := &structs.PreparedQuerySpecificRequest{ 762 Datacenter: "dc1", 763 QueryID: query.Query.ID, 764 QueryOptions: structs.QueryOptions{Token: "root"}, 765 } 766 var resp structs.IndexedPreparedQueries 767 if err = msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 768 t.Fatalf("err: %v", err) 769 } 770 771 if len(resp.Queries) != 1 { 772 t.Fatalf("bad: %v", resp) 773 } 774 actual := resp.Queries[0] 775 if resp.Index != actual.ModifyIndex { 776 t.Fatalf("bad index: %d", resp.Index) 777 } 778 actual.CreateIndex, actual.ModifyIndex = 0, 0 779 if !reflect.DeepEqual(actual, query.Query) { 780 t.Fatalf("bad: %v", actual) 781 } 782 } 783 784 // Explaining should also be denied without a token. 785 { 786 req := &structs.PreparedQueryExecuteRequest{ 787 Datacenter: "dc1", 788 QueryIDOrName: "anything", 789 } 790 var resp structs.PreparedQueryExplainResponse 791 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 792 if !acl.IsErrPermissionDenied(err) { 793 t.Fatalf("bad: %v", err) 794 } 795 } 796 797 // The user can explain and see the redacted token. 798 query.Query.Token = redactedToken 799 query.Query.Service.Service = "anything" 800 { 801 req := &structs.PreparedQueryExecuteRequest{ 802 Datacenter: "dc1", 803 QueryIDOrName: "anything", 804 QueryOptions: structs.QueryOptions{Token: token}, 805 } 806 var resp structs.PreparedQueryExplainResponse 807 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 808 if err != nil { 809 t.Fatalf("err: %v", err) 810 } 811 812 actual := &resp.Query 813 actual.CreateIndex, actual.ModifyIndex = 0, 0 814 if !reflect.DeepEqual(actual, query.Query) { 815 t.Fatalf("bad: %v", actual) 816 } 817 } 818 819 // Make sure the management token can also explain and see the token. 820 query.Query.Token = "5e1e24e5-1329-f86f-18c6-3d3734edb2cd" 821 query.Query.Service.Service = "anything" 822 { 823 req := &structs.PreparedQueryExecuteRequest{ 824 Datacenter: "dc1", 825 QueryIDOrName: "anything", 826 QueryOptions: structs.QueryOptions{Token: "root"}, 827 } 828 var resp structs.PreparedQueryExplainResponse 829 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 830 if err != nil { 831 t.Fatalf("err: %v", err) 832 } 833 834 actual := &resp.Query 835 actual.CreateIndex, actual.ModifyIndex = 0, 0 836 if !reflect.DeepEqual(actual, query.Query) { 837 t.Fatalf("bad: %v", actual) 838 } 839 } 840 } 841 842 func TestPreparedQuery_Get(t *testing.T) { 843 t.Parallel() 844 dir1, s1 := testServerWithConfig(t, func(c *Config) { 845 c.ACLDatacenter = "dc1" 846 c.ACLsEnabled = true 847 c.ACLMasterToken = "root" 848 c.ACLDefaultPolicy = "deny" 849 }) 850 defer os.RemoveAll(dir1) 851 defer s1.Shutdown() 852 codec := rpcClient(t, s1) 853 defer codec.Close() 854 855 testrpc.WaitForLeader(t, s1.RPC, "dc1") 856 857 // Create an ACL with write permissions for redis queries. 858 var token string 859 { 860 var rules = ` 861 query "redis" { 862 policy = "write" 863 } 864 ` 865 866 req := structs.ACLRequest{ 867 Datacenter: "dc1", 868 Op: structs.ACLSet, 869 ACL: structs.ACL{ 870 Name: "User token", 871 Type: structs.ACLTokenTypeClient, 872 Rules: rules, 873 }, 874 WriteRequest: structs.WriteRequest{Token: "root"}, 875 } 876 if err := msgpackrpc.CallWithCodec(codec, "ACL.Apply", &req, &token); err != nil { 877 t.Fatalf("err: %v", err) 878 } 879 } 880 881 // Set up a bare bones query. 882 query := structs.PreparedQueryRequest{ 883 Datacenter: "dc1", 884 Op: structs.PreparedQueryCreate, 885 Query: &structs.PreparedQuery{ 886 Name: "redis-master", 887 Service: structs.ServiceQuery{ 888 Service: "the-redis", 889 }, 890 }, 891 WriteRequest: structs.WriteRequest{Token: token}, 892 } 893 var reply string 894 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 895 t.Fatalf("err: %v", err) 896 } 897 898 // Capture the ID, then read back the query to verify. 899 query.Query.ID = reply 900 { 901 req := &structs.PreparedQuerySpecificRequest{ 902 Datacenter: "dc1", 903 QueryID: query.Query.ID, 904 QueryOptions: structs.QueryOptions{Token: token}, 905 } 906 var resp structs.IndexedPreparedQueries 907 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 908 t.Fatalf("err: %v", err) 909 } 910 911 if len(resp.Queries) != 1 { 912 t.Fatalf("bad: %v", resp) 913 } 914 actual := resp.Queries[0] 915 if resp.Index != actual.ModifyIndex { 916 t.Fatalf("bad index: %d", resp.Index) 917 } 918 actual.CreateIndex, actual.ModifyIndex = 0, 0 919 if !reflect.DeepEqual(actual, query.Query) { 920 t.Fatalf("bad: %v", actual) 921 } 922 } 923 924 // Try again with no token, which should return an error. 925 { 926 req := &structs.PreparedQuerySpecificRequest{ 927 Datacenter: "dc1", 928 QueryID: query.Query.ID, 929 QueryOptions: structs.QueryOptions{Token: ""}, 930 } 931 var resp structs.IndexedPreparedQueries 932 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp) 933 if !acl.IsErrPermissionDenied(err) { 934 t.Fatalf("bad: %v", err) 935 } 936 937 if len(resp.Queries) != 0 { 938 t.Fatalf("bad: %v", resp) 939 } 940 } 941 942 // A management token should be able to read no matter what. 943 { 944 req := &structs.PreparedQuerySpecificRequest{ 945 Datacenter: "dc1", 946 QueryID: query.Query.ID, 947 QueryOptions: structs.QueryOptions{Token: "root"}, 948 } 949 var resp structs.IndexedPreparedQueries 950 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 951 t.Fatalf("err: %v", err) 952 } 953 954 if len(resp.Queries) != 1 { 955 t.Fatalf("bad: %v", resp) 956 } 957 actual := resp.Queries[0] 958 if resp.Index != actual.ModifyIndex { 959 t.Fatalf("bad index: %d", resp.Index) 960 } 961 actual.CreateIndex, actual.ModifyIndex = 0, 0 962 if !reflect.DeepEqual(actual, query.Query) { 963 t.Fatalf("bad: %v", actual) 964 } 965 } 966 967 // Create a session. 968 var session string 969 { 970 req := structs.SessionRequest{ 971 Datacenter: "dc1", 972 Op: structs.SessionCreate, 973 Session: structs.Session{ 974 Node: s1.config.NodeName, 975 }, 976 } 977 if err := msgpackrpc.CallWithCodec(codec, "Session.Apply", &req, &session); err != nil { 978 t.Fatalf("err: %v", err) 979 } 980 } 981 982 // Now update the query to take away its name. 983 query.Op = structs.PreparedQueryUpdate 984 query.Query.Name = "" 985 query.Query.Session = session 986 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 987 t.Fatalf("err: %v", err) 988 } 989 990 // Try again with no token, this should work since this query is only 991 // managed by an ID (no name) so no ACLs apply to it. 992 { 993 req := &structs.PreparedQuerySpecificRequest{ 994 Datacenter: "dc1", 995 QueryID: query.Query.ID, 996 QueryOptions: structs.QueryOptions{Token: ""}, 997 } 998 var resp structs.IndexedPreparedQueries 999 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 1000 t.Fatalf("err: %v", err) 1001 } 1002 1003 if len(resp.Queries) != 1 { 1004 t.Fatalf("bad: %v", resp) 1005 } 1006 actual := resp.Queries[0] 1007 if resp.Index != actual.ModifyIndex { 1008 t.Fatalf("bad index: %d", resp.Index) 1009 } 1010 actual.CreateIndex, actual.ModifyIndex = 0, 0 1011 if !reflect.DeepEqual(actual, query.Query) { 1012 t.Fatalf("bad: %v", actual) 1013 } 1014 } 1015 1016 // Capture a token. 1017 query.Op = structs.PreparedQueryUpdate 1018 query.Query.Token = "le-token" 1019 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 1020 t.Fatalf("err: %v", err) 1021 } 1022 1023 // This should get redacted when we read it back without a token. 1024 query.Query.Token = redactedToken 1025 { 1026 req := &structs.PreparedQuerySpecificRequest{ 1027 Datacenter: "dc1", 1028 QueryID: query.Query.ID, 1029 QueryOptions: structs.QueryOptions{Token: ""}, 1030 } 1031 var resp structs.IndexedPreparedQueries 1032 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 1033 t.Fatalf("err: %v", err) 1034 } 1035 1036 if len(resp.Queries) != 1 { 1037 t.Fatalf("bad: %v", resp) 1038 } 1039 actual := resp.Queries[0] 1040 if resp.Index != actual.ModifyIndex { 1041 t.Fatalf("bad index: %d", resp.Index) 1042 } 1043 actual.CreateIndex, actual.ModifyIndex = 0, 0 1044 if !reflect.DeepEqual(actual, query.Query) { 1045 t.Fatalf("bad: %v", actual) 1046 } 1047 } 1048 1049 // But a management token should be able to see it. 1050 query.Query.Token = "le-token" 1051 { 1052 req := &structs.PreparedQuerySpecificRequest{ 1053 Datacenter: "dc1", 1054 QueryID: query.Query.ID, 1055 QueryOptions: structs.QueryOptions{Token: "root"}, 1056 } 1057 var resp structs.IndexedPreparedQueries 1058 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 1059 t.Fatalf("err: %v", err) 1060 } 1061 1062 if len(resp.Queries) != 1 { 1063 t.Fatalf("bad: %v", resp) 1064 } 1065 actual := resp.Queries[0] 1066 if resp.Index != actual.ModifyIndex { 1067 t.Fatalf("bad index: %d", resp.Index) 1068 } 1069 actual.CreateIndex, actual.ModifyIndex = 0, 0 1070 if !reflect.DeepEqual(actual, query.Query) { 1071 t.Fatalf("bad: %v", actual) 1072 } 1073 } 1074 1075 // Try to get an unknown ID. 1076 { 1077 req := &structs.PreparedQuerySpecificRequest{ 1078 Datacenter: "dc1", 1079 QueryID: generateUUID(), 1080 QueryOptions: structs.QueryOptions{Token: token}, 1081 } 1082 var resp structs.IndexedPreparedQueries 1083 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Get", req, &resp); err != nil { 1084 if err.Error() != ErrQueryNotFound.Error() { 1085 t.Fatalf("err: %v", err) 1086 } 1087 } 1088 1089 if len(resp.Queries) != 0 { 1090 t.Fatalf("bad: %v", resp) 1091 } 1092 } 1093 } 1094 1095 func TestPreparedQuery_List(t *testing.T) { 1096 t.Parallel() 1097 dir1, s1 := testServerWithConfig(t, func(c *Config) { 1098 c.ACLDatacenter = "dc1" 1099 c.ACLsEnabled = true 1100 c.ACLMasterToken = "root" 1101 c.ACLDefaultPolicy = "deny" 1102 }) 1103 defer os.RemoveAll(dir1) 1104 defer s1.Shutdown() 1105 codec := rpcClient(t, s1) 1106 defer codec.Close() 1107 1108 testrpc.WaitForLeader(t, s1.RPC, "dc1") 1109 1110 // Create an ACL with write permissions for redis queries. 1111 var token string 1112 { 1113 var rules = ` 1114 query "redis" { 1115 policy = "write" 1116 } 1117 ` 1118 1119 req := structs.ACLRequest{ 1120 Datacenter: "dc1", 1121 Op: structs.ACLSet, 1122 ACL: structs.ACL{ 1123 Name: "User token", 1124 Type: structs.ACLTokenTypeClient, 1125 Rules: rules, 1126 }, 1127 WriteRequest: structs.WriteRequest{Token: "root"}, 1128 } 1129 if err := msgpackrpc.CallWithCodec(codec, "ACL.Apply", &req, &token); err != nil { 1130 t.Fatalf("err: %v", err) 1131 } 1132 } 1133 1134 // Query with a legit token but no queries. 1135 { 1136 req := &structs.DCSpecificRequest{ 1137 Datacenter: "dc1", 1138 QueryOptions: structs.QueryOptions{Token: token}, 1139 } 1140 var resp structs.IndexedPreparedQueries 1141 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1142 t.Fatalf("err: %v", err) 1143 } 1144 1145 if len(resp.Queries) != 0 { 1146 t.Fatalf("bad: %v", resp) 1147 } 1148 } 1149 1150 // Set up a bare bones query. 1151 query := structs.PreparedQueryRequest{ 1152 Datacenter: "dc1", 1153 Op: structs.PreparedQueryCreate, 1154 Query: &structs.PreparedQuery{ 1155 Name: "redis-master", 1156 Token: "le-token", 1157 Service: structs.ServiceQuery{ 1158 Service: "the-redis", 1159 }, 1160 }, 1161 WriteRequest: structs.WriteRequest{Token: token}, 1162 } 1163 var reply string 1164 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 1165 t.Fatalf("err: %v", err) 1166 } 1167 1168 // Capture the ID and read back the query to verify. We also make sure 1169 // the captured token gets redacted. 1170 query.Query.ID = reply 1171 query.Query.Token = redactedToken 1172 { 1173 req := &structs.DCSpecificRequest{ 1174 Datacenter: "dc1", 1175 QueryOptions: structs.QueryOptions{Token: token}, 1176 } 1177 var resp structs.IndexedPreparedQueries 1178 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1179 t.Fatalf("err: %v", err) 1180 } 1181 1182 if len(resp.Queries) != 1 { 1183 t.Fatalf("bad: %v", resp) 1184 } 1185 actual := resp.Queries[0] 1186 if resp.Index != actual.ModifyIndex { 1187 t.Fatalf("bad index: %d", resp.Index) 1188 } 1189 actual.CreateIndex, actual.ModifyIndex = 0, 0 1190 if !reflect.DeepEqual(actual, query.Query) { 1191 t.Fatalf("bad: %v", actual) 1192 } 1193 } 1194 1195 // An empty token should result in an empty list because of ACL 1196 // filtering. 1197 { 1198 req := &structs.DCSpecificRequest{ 1199 Datacenter: "dc1", 1200 QueryOptions: structs.QueryOptions{Token: ""}, 1201 } 1202 var resp structs.IndexedPreparedQueries 1203 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1204 t.Fatalf("err: %v", err) 1205 } 1206 1207 if len(resp.Queries) != 0 { 1208 t.Fatalf("bad: %v", resp) 1209 } 1210 } 1211 1212 // But a management token should work, and be able to see the captured 1213 // token. 1214 query.Query.Token = "le-token" 1215 { 1216 req := &structs.DCSpecificRequest{ 1217 Datacenter: "dc1", 1218 QueryOptions: structs.QueryOptions{Token: "root"}, 1219 } 1220 var resp structs.IndexedPreparedQueries 1221 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1222 t.Fatalf("err: %v", err) 1223 } 1224 1225 if len(resp.Queries) != 1 { 1226 t.Fatalf("bad: %v", resp) 1227 } 1228 actual := resp.Queries[0] 1229 if resp.Index != actual.ModifyIndex { 1230 t.Fatalf("bad index: %d", resp.Index) 1231 } 1232 actual.CreateIndex, actual.ModifyIndex = 0, 0 1233 if !reflect.DeepEqual(actual, query.Query) { 1234 t.Fatalf("bad: %v", actual) 1235 } 1236 } 1237 1238 // Create a session. 1239 var session string 1240 { 1241 req := structs.SessionRequest{ 1242 Datacenter: "dc1", 1243 Op: structs.SessionCreate, 1244 Session: structs.Session{ 1245 Node: s1.config.NodeName, 1246 }, 1247 } 1248 if err := msgpackrpc.CallWithCodec(codec, "Session.Apply", &req, &session); err != nil { 1249 t.Fatalf("err: %v", err) 1250 } 1251 } 1252 1253 // Now take away the query name. 1254 query.Op = structs.PreparedQueryUpdate 1255 query.Query.Name = "" 1256 query.Query.Session = session 1257 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 1258 t.Fatalf("err: %v", err) 1259 } 1260 1261 // A query with the redis token shouldn't show anything since it doesn't 1262 // match any un-named queries. 1263 { 1264 req := &structs.DCSpecificRequest{ 1265 Datacenter: "dc1", 1266 QueryOptions: structs.QueryOptions{Token: token}, 1267 } 1268 var resp structs.IndexedPreparedQueries 1269 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1270 t.Fatalf("err: %v", err) 1271 } 1272 1273 if len(resp.Queries) != 0 { 1274 t.Fatalf("bad: %v", resp) 1275 } 1276 } 1277 1278 // But a management token should work. 1279 { 1280 req := &structs.DCSpecificRequest{ 1281 Datacenter: "dc1", 1282 QueryOptions: structs.QueryOptions{Token: "root"}, 1283 } 1284 var resp structs.IndexedPreparedQueries 1285 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.List", req, &resp); err != nil { 1286 t.Fatalf("err: %v", err) 1287 } 1288 1289 if len(resp.Queries) != 1 { 1290 t.Fatalf("bad: %v", resp) 1291 } 1292 actual := resp.Queries[0] 1293 if resp.Index != actual.ModifyIndex { 1294 t.Fatalf("bad index: %d", resp.Index) 1295 } 1296 actual.CreateIndex, actual.ModifyIndex = 0, 0 1297 if !reflect.DeepEqual(actual, query.Query) { 1298 t.Fatalf("bad: %v", actual) 1299 } 1300 } 1301 } 1302 1303 func TestPreparedQuery_Explain(t *testing.T) { 1304 t.Parallel() 1305 dir1, s1 := testServerWithConfig(t, func(c *Config) { 1306 c.ACLDatacenter = "dc1" 1307 c.ACLsEnabled = true 1308 c.ACLMasterToken = "root" 1309 c.ACLDefaultPolicy = "deny" 1310 }) 1311 defer os.RemoveAll(dir1) 1312 defer s1.Shutdown() 1313 codec := rpcClient(t, s1) 1314 defer codec.Close() 1315 1316 testrpc.WaitForLeader(t, s1.RPC, "dc1") 1317 1318 // Create an ACL with write permissions for prod- queries. 1319 var token string 1320 { 1321 var rules = ` 1322 query "prod-" { 1323 policy = "write" 1324 } 1325 ` 1326 1327 req := structs.ACLRequest{ 1328 Datacenter: "dc1", 1329 Op: structs.ACLSet, 1330 ACL: structs.ACL{ 1331 Name: "User token", 1332 Type: structs.ACLTokenTypeClient, 1333 Rules: rules, 1334 }, 1335 WriteRequest: structs.WriteRequest{Token: "root"}, 1336 } 1337 if err := msgpackrpc.CallWithCodec(codec, "ACL.Apply", &req, &token); err != nil { 1338 t.Fatalf("err: %v", err) 1339 } 1340 } 1341 1342 // Set up a template. 1343 query := structs.PreparedQueryRequest{ 1344 Datacenter: "dc1", 1345 Op: structs.PreparedQueryCreate, 1346 Query: &structs.PreparedQuery{ 1347 Name: "prod-", 1348 Token: "5e1e24e5-1329-f86f-18c6-3d3734edb2cd", 1349 Template: structs.QueryTemplateOptions{ 1350 Type: structs.QueryTemplateTypeNamePrefixMatch, 1351 }, 1352 Service: structs.ServiceQuery{ 1353 Service: "${name.full}", 1354 }, 1355 }, 1356 WriteRequest: structs.WriteRequest{Token: token}, 1357 } 1358 var reply string 1359 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 1360 t.Fatalf("err: %v", err) 1361 } 1362 1363 // Explain via the management token. 1364 query.Query.ID = reply 1365 query.Query.Service.Service = "prod-redis" 1366 { 1367 req := &structs.PreparedQueryExecuteRequest{ 1368 Datacenter: "dc1", 1369 QueryIDOrName: "prod-redis", 1370 QueryOptions: structs.QueryOptions{Token: "root"}, 1371 } 1372 var resp structs.PreparedQueryExplainResponse 1373 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 1374 if err != nil { 1375 t.Fatalf("err: %v", err) 1376 } 1377 1378 actual := &resp.Query 1379 actual.CreateIndex, actual.ModifyIndex = 0, 0 1380 if !reflect.DeepEqual(actual, query.Query) { 1381 t.Fatalf("bad: %v", actual) 1382 } 1383 } 1384 1385 // Explain via the user token, which will redact the captured token. 1386 query.Query.Token = redactedToken 1387 query.Query.Service.Service = "prod-redis" 1388 { 1389 req := &structs.PreparedQueryExecuteRequest{ 1390 Datacenter: "dc1", 1391 QueryIDOrName: "prod-redis", 1392 QueryOptions: structs.QueryOptions{Token: token}, 1393 } 1394 var resp structs.PreparedQueryExplainResponse 1395 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 1396 if err != nil { 1397 t.Fatalf("err: %v", err) 1398 } 1399 1400 actual := &resp.Query 1401 actual.CreateIndex, actual.ModifyIndex = 0, 0 1402 if !reflect.DeepEqual(actual, query.Query) { 1403 t.Fatalf("bad: %v", actual) 1404 } 1405 } 1406 1407 // Explaining should be denied without a token, since the user isn't 1408 // allowed to see the query. 1409 { 1410 req := &structs.PreparedQueryExecuteRequest{ 1411 Datacenter: "dc1", 1412 QueryIDOrName: "prod-redis", 1413 } 1414 var resp structs.PreparedQueryExplainResponse 1415 err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp) 1416 if !acl.IsErrPermissionDenied(err) { 1417 t.Fatalf("bad: %v", err) 1418 } 1419 } 1420 1421 // Try to explain a bogus ID. 1422 { 1423 req := &structs.PreparedQueryExecuteRequest{ 1424 Datacenter: "dc1", 1425 QueryIDOrName: generateUUID(), 1426 QueryOptions: structs.QueryOptions{Token: "root"}, 1427 } 1428 var resp structs.IndexedPreparedQueries 1429 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Explain", req, &resp); err != nil { 1430 if err.Error() != ErrQueryNotFound.Error() { 1431 t.Fatalf("err: %v", err) 1432 } 1433 } 1434 } 1435 } 1436 1437 // This is a beast of a test, but the setup is so extensive it makes sense to 1438 // walk through the different cases once we have it up. This is broken into 1439 // sections so it's still pretty easy to read. 1440 func TestPreparedQuery_Execute(t *testing.T) { 1441 t.Parallel() 1442 dir1, s1 := testServerWithConfig(t, func(c *Config) { 1443 c.ACLDatacenter = "dc1" 1444 c.ACLsEnabled = true 1445 c.ACLMasterToken = "root" 1446 c.ACLDefaultPolicy = "deny" 1447 c.ACLEnforceVersion8 = false 1448 }) 1449 defer os.RemoveAll(dir1) 1450 defer s1.Shutdown() 1451 codec1 := rpcClient(t, s1) 1452 defer codec1.Close() 1453 1454 dir2, s2 := testServerWithConfig(t, func(c *Config) { 1455 c.Datacenter = "dc2" 1456 c.ACLDatacenter = "dc1" 1457 c.ACLsEnabled = true 1458 }) 1459 defer os.RemoveAll(dir2) 1460 defer s2.Shutdown() 1461 codec2 := rpcClient(t, s2) 1462 defer codec2.Close() 1463 1464 testrpc.WaitForLeader(t, s1.RPC, "dc1") 1465 testrpc.WaitForLeader(t, s2.RPC, "dc2") 1466 1467 // Try to WAN join. 1468 joinWAN(t, s2, s1) 1469 retry.Run(t, func(r *retry.R) { 1470 if got, want := len(s1.WANMembers()), 2; got != want { 1471 r.Fatalf("got %d WAN members want %d", got, want) 1472 } 1473 }) 1474 1475 // Create an ACL with read permission to the service. 1476 var execToken string 1477 { 1478 var rules = ` 1479 service "foo" { 1480 policy = "read" 1481 } 1482 ` 1483 1484 req := structs.ACLRequest{ 1485 Datacenter: "dc1", 1486 Op: structs.ACLSet, 1487 ACL: structs.ACL{ 1488 Name: "User token", 1489 Type: structs.ACLTokenTypeClient, 1490 Rules: rules, 1491 }, 1492 WriteRequest: structs.WriteRequest{Token: "root"}, 1493 } 1494 if err := msgpackrpc.CallWithCodec(codec1, "ACL.Apply", &req, &execToken); err != nil { 1495 t.Fatalf("err: %v", err) 1496 } 1497 } 1498 1499 // Set up some nodes in each DC that host the service. 1500 { 1501 for i := 0; i < 10; i++ { 1502 for _, dc := range []string{"dc1", "dc2"} { 1503 req := structs.RegisterRequest{ 1504 Datacenter: dc, 1505 Node: fmt.Sprintf("node%d", i+1), 1506 Address: fmt.Sprintf("127.0.0.%d", i+1), 1507 NodeMeta: map[string]string{ 1508 "group": fmt.Sprintf("%d", i/5), 1509 "instance_type": "t2.micro", 1510 }, 1511 Service: &structs.NodeService{ 1512 Service: "foo", 1513 Port: 8000, 1514 Tags: []string{dc, fmt.Sprintf("tag%d", i+1)}, 1515 Meta: map[string]string{ 1516 "svc-group": fmt.Sprintf("%d", i%2), 1517 "foo": "true", 1518 }, 1519 }, 1520 WriteRequest: structs.WriteRequest{Token: "root"}, 1521 } 1522 if i == 0 { 1523 req.NodeMeta["unique"] = "true" 1524 req.Service.Meta["unique"] = "true" 1525 } 1526 1527 var codec rpc.ClientCodec 1528 if dc == "dc1" { 1529 codec = codec1 1530 } else { 1531 codec = codec2 1532 } 1533 1534 var reply struct{} 1535 if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply); err != nil { 1536 t.Fatalf("err: %v", err) 1537 } 1538 } 1539 } 1540 } 1541 1542 // Set up a service query. 1543 query := structs.PreparedQueryRequest{ 1544 Datacenter: "dc1", 1545 Op: structs.PreparedQueryCreate, 1546 Query: &structs.PreparedQuery{ 1547 Name: "test", 1548 Service: structs.ServiceQuery{ 1549 Service: "foo", 1550 }, 1551 DNS: structs.QueryDNSOptions{ 1552 TTL: "10s", 1553 }, 1554 }, 1555 WriteRequest: structs.WriteRequest{Token: "root"}, 1556 } 1557 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 1558 t.Fatalf("err: %v", err) 1559 } 1560 1561 // Run a query that doesn't exist. 1562 { 1563 req := structs.PreparedQueryExecuteRequest{ 1564 Datacenter: "dc1", 1565 QueryIDOrName: "nope", 1566 } 1567 1568 var reply structs.PreparedQueryExecuteResponse 1569 err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply) 1570 if err == nil || err.Error() != ErrQueryNotFound.Error() { 1571 t.Fatalf("bad: %v", err) 1572 } 1573 1574 if len(reply.Nodes) != 0 { 1575 t.Fatalf("bad: %v", reply) 1576 } 1577 } 1578 1579 // Run the registered query. 1580 { 1581 req := structs.PreparedQueryExecuteRequest{ 1582 Datacenter: "dc1", 1583 QueryIDOrName: query.Query.ID, 1584 QueryOptions: structs.QueryOptions{Token: execToken}, 1585 } 1586 1587 var reply structs.PreparedQueryExecuteResponse 1588 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1589 t.Fatalf("err: %v", err) 1590 } 1591 1592 if len(reply.Nodes) != 10 || 1593 reply.Datacenter != "dc1" || reply.Failovers != 0 || 1594 reply.Service != query.Query.Service.Service || 1595 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 1596 !reply.QueryMeta.KnownLeader { 1597 t.Fatalf("bad: %v", reply) 1598 } 1599 } 1600 1601 // Try with a limit. 1602 { 1603 req := structs.PreparedQueryExecuteRequest{ 1604 Datacenter: "dc1", 1605 QueryIDOrName: query.Query.ID, 1606 Limit: 3, 1607 QueryOptions: structs.QueryOptions{Token: execToken}, 1608 } 1609 1610 var reply structs.PreparedQueryExecuteResponse 1611 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1612 t.Fatalf("err: %v", err) 1613 } 1614 1615 if len(reply.Nodes) != 3 || 1616 reply.Datacenter != "dc1" || reply.Failovers != 0 || 1617 reply.Service != query.Query.Service.Service || 1618 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 1619 !reply.QueryMeta.KnownLeader { 1620 t.Fatalf("bad: %v", reply) 1621 } 1622 } 1623 1624 // Run various service queries with node metadata filters. 1625 { 1626 cases := []struct { 1627 filters map[string]string 1628 numNodes int 1629 }{ 1630 { 1631 filters: map[string]string{}, 1632 numNodes: 10, 1633 }, 1634 { 1635 filters: map[string]string{"instance_type": "t2.micro"}, 1636 numNodes: 10, 1637 }, 1638 { 1639 filters: map[string]string{"group": "1"}, 1640 numNodes: 5, 1641 }, 1642 { 1643 filters: map[string]string{"group": "0", "unique": "true"}, 1644 numNodes: 1, 1645 }, 1646 } 1647 1648 for _, tc := range cases { 1649 nodeMetaQuery := structs.PreparedQueryRequest{ 1650 Datacenter: "dc1", 1651 Op: structs.PreparedQueryCreate, 1652 Query: &structs.PreparedQuery{ 1653 Service: structs.ServiceQuery{ 1654 Service: "foo", 1655 NodeMeta: tc.filters, 1656 }, 1657 DNS: structs.QueryDNSOptions{ 1658 TTL: "10s", 1659 }, 1660 }, 1661 WriteRequest: structs.WriteRequest{Token: "root"}, 1662 } 1663 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &nodeMetaQuery, &nodeMetaQuery.Query.ID); err != nil { 1664 t.Fatalf("err: %v", err) 1665 } 1666 1667 req := structs.PreparedQueryExecuteRequest{ 1668 Datacenter: "dc1", 1669 QueryIDOrName: nodeMetaQuery.Query.ID, 1670 QueryOptions: structs.QueryOptions{Token: execToken}, 1671 } 1672 1673 var reply structs.PreparedQueryExecuteResponse 1674 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1675 t.Fatalf("err: %v", err) 1676 } 1677 1678 if len(reply.Nodes) != tc.numNodes { 1679 t.Fatalf("bad: %v, %v", len(reply.Nodes), tc.numNodes) 1680 } 1681 1682 for _, node := range reply.Nodes { 1683 if !structs.SatisfiesMetaFilters(node.Node.Meta, tc.filters) { 1684 t.Fatalf("bad: %v", node.Node.Meta) 1685 } 1686 } 1687 } 1688 } 1689 1690 // Run various service queries with service metadata filters 1691 { 1692 cases := []struct { 1693 filters map[string]string 1694 numNodes int 1695 }{ 1696 { 1697 filters: map[string]string{}, 1698 numNodes: 10, 1699 }, 1700 { 1701 filters: map[string]string{"foo": "true"}, 1702 numNodes: 10, 1703 }, 1704 { 1705 filters: map[string]string{"svc-group": "0"}, 1706 numNodes: 5, 1707 }, 1708 { 1709 filters: map[string]string{"svc-group": "1"}, 1710 numNodes: 5, 1711 }, 1712 { 1713 filters: map[string]string{"svc-group": "0", "unique": "true"}, 1714 numNodes: 1, 1715 }, 1716 } 1717 1718 for _, tc := range cases { 1719 svcMetaQuery := structs.PreparedQueryRequest{ 1720 Datacenter: "dc1", 1721 Op: structs.PreparedQueryCreate, 1722 Query: &structs.PreparedQuery{ 1723 Service: structs.ServiceQuery{ 1724 Service: "foo", 1725 ServiceMeta: tc.filters, 1726 }, 1727 DNS: structs.QueryDNSOptions{ 1728 TTL: "10s", 1729 }, 1730 }, 1731 WriteRequest: structs.WriteRequest{Token: "root"}, 1732 } 1733 1734 require.NoError(t, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &svcMetaQuery, &svcMetaQuery.Query.ID)) 1735 1736 req := structs.PreparedQueryExecuteRequest{ 1737 Datacenter: "dc1", 1738 QueryIDOrName: svcMetaQuery.Query.ID, 1739 QueryOptions: structs.QueryOptions{Token: execToken}, 1740 } 1741 1742 var reply structs.PreparedQueryExecuteResponse 1743 require.NoError(t, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply)) 1744 require.Len(t, reply.Nodes, tc.numNodes) 1745 for _, node := range reply.Nodes { 1746 require.True(t, structs.SatisfiesMetaFilters(node.Service.Meta, tc.filters)) 1747 } 1748 } 1749 } 1750 1751 // Push a coordinate for one of the nodes so we can try an RTT sort. We 1752 // have to sleep a little while for the coordinate batch to get flushed. 1753 { 1754 req := structs.CoordinateUpdateRequest{ 1755 Datacenter: "dc1", 1756 Node: "node3", 1757 Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()), 1758 } 1759 var out struct{} 1760 if err := msgpackrpc.CallWithCodec(codec1, "Coordinate.Update", &req, &out); err != nil { 1761 t.Fatalf("err: %v", err) 1762 } 1763 time.Sleep(3 * s1.config.CoordinateUpdatePeriod) 1764 } 1765 1766 // Try an RTT sort. We don't have any other coordinates in there but 1767 // showing that the node with a coordinate is always first proves we 1768 // call the RTT sorting function, which is tested elsewhere. 1769 for i := 0; i < 100; i++ { 1770 req := structs.PreparedQueryExecuteRequest{ 1771 Datacenter: "dc1", 1772 QueryIDOrName: query.Query.ID, 1773 Source: structs.QuerySource{ 1774 Datacenter: "dc1", 1775 Node: "node3", 1776 }, 1777 QueryOptions: structs.QueryOptions{Token: execToken}, 1778 } 1779 1780 var reply structs.PreparedQueryExecuteResponse 1781 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1782 t.Fatalf("err: %v", err) 1783 } 1784 1785 if len(reply.Nodes) != 10 || 1786 reply.Datacenter != "dc1" || reply.Failovers != 0 || 1787 reply.Service != query.Query.Service.Service || 1788 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 1789 !reply.QueryMeta.KnownLeader { 1790 t.Fatalf("bad: %v", reply) 1791 } 1792 if reply.Nodes[0].Node.Node != "node3" { 1793 t.Fatalf("bad: %v", reply) 1794 } 1795 } 1796 1797 // Make sure the shuffle looks like it's working. 1798 uniques := make(map[string]struct{}) 1799 for i := 0; i < 100; i++ { 1800 req := structs.PreparedQueryExecuteRequest{ 1801 Datacenter: "dc1", 1802 QueryIDOrName: query.Query.ID, 1803 QueryOptions: structs.QueryOptions{Token: execToken}, 1804 } 1805 1806 var reply structs.PreparedQueryExecuteResponse 1807 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1808 t.Fatalf("err: %v", err) 1809 } 1810 1811 if len(reply.Nodes) != 10 || 1812 reply.Datacenter != "dc1" || reply.Failovers != 0 || 1813 reply.Service != query.Query.Service.Service || 1814 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 1815 !reply.QueryMeta.KnownLeader { 1816 t.Fatalf("bad: %v", reply) 1817 } 1818 var names []string 1819 for _, node := range reply.Nodes { 1820 names = append(names, node.Node.Node) 1821 } 1822 key := strings.Join(names, "|") 1823 uniques[key] = struct{}{} 1824 } 1825 1826 // We have to allow for the fact that there won't always be a unique 1827 // shuffle each pass, so we just look for smell here without the test 1828 // being flaky. 1829 if len(uniques) < 50 { 1830 t.Fatalf("unique shuffle ratio too low: %d/100", len(uniques)) 1831 } 1832 1833 // Set the query to return results nearest to node3. This is the only 1834 // node with coordinates, and it carries the service we are asking for, 1835 // so node3 should always show up first. 1836 query.Op = structs.PreparedQueryUpdate 1837 query.Query.Service.Near = "node3" 1838 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 1839 t.Fatalf("err: %v", err) 1840 } 1841 1842 // Now run the query and make sure the sort looks right. 1843 { 1844 req := structs.PreparedQueryExecuteRequest{ 1845 Agent: structs.QuerySource{ 1846 Datacenter: "dc1", 1847 Node: "node3", 1848 }, 1849 Datacenter: "dc1", 1850 QueryIDOrName: query.Query.ID, 1851 QueryOptions: structs.QueryOptions{Token: execToken}, 1852 } 1853 1854 for i := 0; i < 10; i++ { 1855 var reply structs.PreparedQueryExecuteResponse 1856 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1857 t.Fatalf("err: %v", err) 1858 } 1859 if n := len(reply.Nodes); n != 10 { 1860 t.Fatalf("expect 10 nodes, got: %d", n) 1861 } 1862 if node := reply.Nodes[0].Node.Node; node != "node3" { 1863 t.Fatalf("expect node3 first, got: %q", node) 1864 } 1865 } 1866 } 1867 1868 // Query again, but this time set a client-supplied query source. This 1869 // proves that we allow overriding the baked-in value with ?near. 1870 { 1871 // Set up the query with a non-existent node. This will cause the 1872 // nodes to be shuffled if the passed node is respected, proving 1873 // that we allow the override to happen. 1874 req := structs.PreparedQueryExecuteRequest{ 1875 Source: structs.QuerySource{ 1876 Datacenter: "dc1", 1877 Node: "foo", 1878 }, 1879 Agent: structs.QuerySource{ 1880 Datacenter: "dc1", 1881 Node: "node3", 1882 }, 1883 Datacenter: "dc1", 1884 QueryIDOrName: query.Query.ID, 1885 QueryOptions: structs.QueryOptions{Token: execToken}, 1886 } 1887 1888 shuffled := false 1889 for i := 0; i < 10; i++ { 1890 var reply structs.PreparedQueryExecuteResponse 1891 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1892 t.Fatalf("err: %v", err) 1893 } 1894 if n := len(reply.Nodes); n != 10 { 1895 t.Fatalf("expect 10 nodes, got: %d", n) 1896 } 1897 if node := reply.Nodes[0].Node.Node; node != "node3" { 1898 shuffled = true 1899 break 1900 } 1901 } 1902 1903 if !shuffled { 1904 t.Fatalf("expect nodes to be shuffled") 1905 } 1906 } 1907 1908 // If the exact node we are sorting near appears in the list, make sure it 1909 // gets popped to the front of the result. 1910 { 1911 req := structs.PreparedQueryExecuteRequest{ 1912 Source: structs.QuerySource{ 1913 Datacenter: "dc1", 1914 Node: "node1", 1915 }, 1916 Datacenter: "dc1", 1917 QueryIDOrName: query.Query.ID, 1918 QueryOptions: structs.QueryOptions{Token: execToken}, 1919 } 1920 1921 for i := 0; i < 10; i++ { 1922 var reply structs.PreparedQueryExecuteResponse 1923 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1924 t.Fatalf("err: %v", err) 1925 } 1926 if n := len(reply.Nodes); n != 10 { 1927 t.Fatalf("expect 10 nodes, got: %d", n) 1928 } 1929 if node := reply.Nodes[0].Node.Node; node != "node1" { 1930 t.Fatalf("expect node1 first, got: %q", node) 1931 } 1932 } 1933 } 1934 1935 // Bake the magic "_agent" flag into the query. 1936 query.Query.Service.Near = "_agent" 1937 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 1938 t.Fatalf("err: %v", err) 1939 } 1940 1941 // Check that we sort the local agent first when the magic flag is set. 1942 { 1943 req := structs.PreparedQueryExecuteRequest{ 1944 Agent: structs.QuerySource{ 1945 Datacenter: "dc1", 1946 Node: "node3", 1947 }, 1948 Datacenter: "dc1", 1949 QueryIDOrName: query.Query.ID, 1950 QueryOptions: structs.QueryOptions{Token: execToken}, 1951 } 1952 1953 for i := 0; i < 10; i++ { 1954 var reply structs.PreparedQueryExecuteResponse 1955 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1956 t.Fatalf("err: %v", err) 1957 } 1958 if n := len(reply.Nodes); n != 10 { 1959 t.Fatalf("expect 10 nodes, got: %d", n) 1960 } 1961 if node := reply.Nodes[0].Node.Node; node != "node3" { 1962 t.Fatalf("expect node3 first, got: %q", node) 1963 } 1964 } 1965 } 1966 1967 // Check that the query isn't just sorting "node3" first because we 1968 // provided it in the Agent query source. Proves that we use the 1969 // Agent source when the magic "_agent" flag is passed. 1970 { 1971 req := structs.PreparedQueryExecuteRequest{ 1972 Agent: structs.QuerySource{ 1973 Datacenter: "dc1", 1974 Node: "foo", 1975 }, 1976 Datacenter: "dc1", 1977 QueryIDOrName: query.Query.ID, 1978 QueryOptions: structs.QueryOptions{Token: execToken}, 1979 } 1980 1981 // Expect the set to be shuffled since we have no coordinates 1982 // on the "foo" node. 1983 shuffled := false 1984 for i := 0; i < 10; i++ { 1985 var reply structs.PreparedQueryExecuteResponse 1986 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 1987 t.Fatalf("err: %v", err) 1988 } 1989 if n := len(reply.Nodes); n != 10 { 1990 t.Fatalf("expect 10 nodes, got: %d", n) 1991 } 1992 if node := reply.Nodes[0].Node.Node; node != "node3" { 1993 shuffled = true 1994 break 1995 } 1996 } 1997 1998 if !shuffled { 1999 t.Fatal("expect nodes to be shuffled") 2000 } 2001 } 2002 2003 // Shuffles if the response comes from a non-local DC. Proves that the 2004 // agent query source does not interfere with the order. 2005 { 2006 req := structs.PreparedQueryExecuteRequest{ 2007 Source: structs.QuerySource{ 2008 Datacenter: "dc2", 2009 Node: "node3", 2010 }, 2011 Agent: structs.QuerySource{ 2012 Datacenter: "dc1", 2013 Node: "node3", 2014 }, 2015 Datacenter: "dc1", 2016 QueryIDOrName: query.Query.ID, 2017 QueryOptions: structs.QueryOptions{Token: execToken}, 2018 } 2019 2020 shuffled := false 2021 for i := 0; i < 10; i++ { 2022 var reply structs.PreparedQueryExecuteResponse 2023 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2024 t.Fatalf("err: %v", err) 2025 } 2026 if n := len(reply.Nodes); n != 10 { 2027 t.Fatalf("expect 10 nodes, got: %d", n) 2028 } 2029 if reply.Nodes[0].Node.Node != "node3" { 2030 shuffled = true 2031 break 2032 } 2033 } 2034 2035 if !shuffled { 2036 t.Fatal("expect node shuffle for remote results") 2037 } 2038 } 2039 2040 // Un-bake the near parameter. 2041 query.Query.Service.Near = "" 2042 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2043 t.Fatalf("err: %v", err) 2044 } 2045 2046 // Update the health of a node to mark it critical. 2047 setHealth := func(node string, health string) { 2048 req := structs.RegisterRequest{ 2049 Datacenter: "dc1", 2050 Node: node, 2051 Address: "127.0.0.1", 2052 Service: &structs.NodeService{ 2053 Service: "foo", 2054 Port: 8000, 2055 Tags: []string{"dc1", "tag1"}, 2056 }, 2057 Check: &structs.HealthCheck{ 2058 Name: "failing", 2059 Status: health, 2060 ServiceID: "foo", 2061 }, 2062 WriteRequest: structs.WriteRequest{Token: "root"}, 2063 } 2064 var reply struct{} 2065 if err := msgpackrpc.CallWithCodec(codec1, "Catalog.Register", &req, &reply); err != nil { 2066 t.Fatalf("err: %v", err) 2067 } 2068 } 2069 setHealth("node1", api.HealthCritical) 2070 2071 // The failing node should be filtered. 2072 { 2073 req := structs.PreparedQueryExecuteRequest{ 2074 Datacenter: "dc1", 2075 QueryIDOrName: query.Query.ID, 2076 QueryOptions: structs.QueryOptions{Token: execToken}, 2077 } 2078 2079 var reply structs.PreparedQueryExecuteResponse 2080 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2081 t.Fatalf("err: %v", err) 2082 } 2083 2084 if len(reply.Nodes) != 9 || 2085 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2086 reply.Service != query.Query.Service.Service || 2087 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2088 !reply.QueryMeta.KnownLeader { 2089 t.Fatalf("bad: %v", reply) 2090 } 2091 for _, node := range reply.Nodes { 2092 if node.Node.Node == "node1" { 2093 t.Fatalf("bad: %v", node) 2094 } 2095 } 2096 } 2097 2098 // Upgrade it to a warning and re-query, should be 10 nodes again. 2099 setHealth("node1", api.HealthWarning) 2100 { 2101 req := structs.PreparedQueryExecuteRequest{ 2102 Datacenter: "dc1", 2103 QueryIDOrName: query.Query.ID, 2104 QueryOptions: structs.QueryOptions{Token: execToken}, 2105 } 2106 2107 var reply structs.PreparedQueryExecuteResponse 2108 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2109 t.Fatalf("err: %v", err) 2110 } 2111 2112 if len(reply.Nodes) != 10 || 2113 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2114 reply.Service != query.Query.Service.Service || 2115 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2116 !reply.QueryMeta.KnownLeader { 2117 t.Fatalf("bad: %v", reply) 2118 } 2119 } 2120 2121 // Make the query more picky so it excludes warning nodes. 2122 query.Query.Service.OnlyPassing = true 2123 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2124 t.Fatalf("err: %v", err) 2125 } 2126 2127 // The node in the warning state should be filtered. 2128 { 2129 req := structs.PreparedQueryExecuteRequest{ 2130 Datacenter: "dc1", 2131 QueryIDOrName: query.Query.ID, 2132 QueryOptions: structs.QueryOptions{Token: execToken}, 2133 } 2134 2135 var reply structs.PreparedQueryExecuteResponse 2136 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2137 t.Fatalf("err: %v", err) 2138 } 2139 2140 if len(reply.Nodes) != 9 || 2141 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2142 reply.Service != query.Query.Service.Service || 2143 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2144 !reply.QueryMeta.KnownLeader { 2145 t.Fatalf("bad: %v", reply) 2146 } 2147 for _, node := range reply.Nodes { 2148 if node.Node.Node == "node1" { 2149 t.Fatalf("bad: %v", node) 2150 } 2151 } 2152 } 2153 2154 // Make the query ignore all our health checks (which have "failing" ID 2155 // implicitly from their name). 2156 query.Query.Service.IgnoreCheckIDs = []types.CheckID{"failing"} 2157 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2158 t.Fatalf("err: %v", err) 2159 } 2160 2161 // We should end up with 10 nodes again 2162 { 2163 req := structs.PreparedQueryExecuteRequest{ 2164 Datacenter: "dc1", 2165 QueryIDOrName: query.Query.ID, 2166 QueryOptions: structs.QueryOptions{Token: execToken}, 2167 } 2168 2169 var reply structs.PreparedQueryExecuteResponse 2170 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2171 t.Fatalf("err: %v", err) 2172 } 2173 2174 if len(reply.Nodes) != 10 || 2175 reply.Datacenter != "dc1" || 2176 reply.Service != query.Query.Service.Service || 2177 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2178 !reply.QueryMeta.KnownLeader { 2179 t.Fatalf("bad: %v", reply) 2180 } 2181 } 2182 2183 // Undo that so all the following tests aren't broken! 2184 query.Query.Service.IgnoreCheckIDs = nil 2185 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2186 t.Fatalf("err: %v", err) 2187 } 2188 2189 // Make the query more picky by adding a tag filter. This just proves we 2190 // call into the tag filter, it is tested more thoroughly in a separate 2191 // test. 2192 query.Query.Service.Tags = []string{"!tag3"} 2193 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2194 t.Fatalf("err: %v", err) 2195 } 2196 2197 // The node in the warning state should be filtered as well as the node 2198 // with the filtered tag. 2199 { 2200 req := structs.PreparedQueryExecuteRequest{ 2201 Datacenter: "dc1", 2202 QueryIDOrName: query.Query.ID, 2203 QueryOptions: structs.QueryOptions{Token: execToken}, 2204 } 2205 2206 var reply structs.PreparedQueryExecuteResponse 2207 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2208 t.Fatalf("err: %v", err) 2209 } 2210 2211 if len(reply.Nodes) != 8 || 2212 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2213 reply.Service != query.Query.Service.Service || 2214 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2215 !reply.QueryMeta.KnownLeader { 2216 t.Fatalf("bad: %v", reply) 2217 } 2218 for _, node := range reply.Nodes { 2219 if node.Node.Node == "node1" || node.Node.Node == "node3" { 2220 t.Fatalf("bad: %v", node) 2221 } 2222 } 2223 } 2224 2225 // Make a new exec token that can't read the service. 2226 var denyToken string 2227 { 2228 var rules = ` 2229 service "foo" { 2230 policy = "deny" 2231 } 2232 ` 2233 2234 req := structs.ACLRequest{ 2235 Datacenter: "dc1", 2236 Op: structs.ACLSet, 2237 ACL: structs.ACL{ 2238 Name: "User token", 2239 Type: structs.ACLTokenTypeClient, 2240 Rules: rules, 2241 }, 2242 WriteRequest: structs.WriteRequest{Token: "root"}, 2243 } 2244 if err := msgpackrpc.CallWithCodec(codec1, "ACL.Apply", &req, &denyToken); err != nil { 2245 t.Fatalf("err: %v", err) 2246 } 2247 } 2248 2249 // Make sure the query gets denied with this token. 2250 { 2251 req := structs.PreparedQueryExecuteRequest{ 2252 Datacenter: "dc1", 2253 QueryIDOrName: query.Query.ID, 2254 QueryOptions: structs.QueryOptions{Token: denyToken}, 2255 } 2256 2257 var reply structs.PreparedQueryExecuteResponse 2258 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2259 t.Fatalf("err: %v", err) 2260 } 2261 2262 if len(reply.Nodes) != 0 || 2263 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2264 reply.Service != query.Query.Service.Service || 2265 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2266 !reply.QueryMeta.KnownLeader { 2267 t.Fatalf("bad: %v", reply) 2268 } 2269 } 2270 2271 // Bake the exec token into the query. 2272 query.Query.Token = execToken 2273 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2274 t.Fatalf("err: %v", err) 2275 } 2276 2277 // Now even querying with the deny token should work. 2278 { 2279 req := structs.PreparedQueryExecuteRequest{ 2280 Datacenter: "dc1", 2281 QueryIDOrName: query.Query.ID, 2282 QueryOptions: structs.QueryOptions{Token: denyToken}, 2283 } 2284 2285 var reply structs.PreparedQueryExecuteResponse 2286 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2287 t.Fatalf("err: %v", err) 2288 } 2289 2290 if len(reply.Nodes) != 8 || 2291 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2292 reply.Service != query.Query.Service.Service || 2293 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2294 !reply.QueryMeta.KnownLeader { 2295 t.Fatalf("bad: %v", reply) 2296 } 2297 for _, node := range reply.Nodes { 2298 if node.Node.Node == "node1" || node.Node.Node == "node3" { 2299 t.Fatalf("bad: %v", node) 2300 } 2301 } 2302 } 2303 2304 // Un-bake the token. 2305 query.Query.Token = "" 2306 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2307 t.Fatalf("err: %v", err) 2308 } 2309 2310 // Make sure the query gets denied again with the deny token. 2311 { 2312 req := structs.PreparedQueryExecuteRequest{ 2313 Datacenter: "dc1", 2314 QueryIDOrName: query.Query.ID, 2315 QueryOptions: structs.QueryOptions{Token: denyToken}, 2316 } 2317 2318 var reply structs.PreparedQueryExecuteResponse 2319 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2320 t.Fatalf("err: %v", err) 2321 } 2322 2323 if len(reply.Nodes) != 0 || 2324 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2325 reply.Service != query.Query.Service.Service || 2326 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2327 !reply.QueryMeta.KnownLeader { 2328 t.Fatalf("bad: %v", reply) 2329 } 2330 } 2331 2332 // Turn on version 8 ACLs, which will start to filter even with the exec 2333 // token. 2334 s1.config.ACLEnforceVersion8 = true 2335 { 2336 req := structs.PreparedQueryExecuteRequest{ 2337 Datacenter: "dc1", 2338 QueryIDOrName: query.Query.ID, 2339 QueryOptions: structs.QueryOptions{Token: execToken}, 2340 } 2341 2342 var reply structs.PreparedQueryExecuteResponse 2343 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2344 t.Fatalf("err: %v", err) 2345 } 2346 2347 if len(reply.Nodes) != 0 || 2348 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2349 reply.Service != query.Query.Service.Service || 2350 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2351 !reply.QueryMeta.KnownLeader { 2352 t.Fatalf("bad: %v", reply) 2353 } 2354 } 2355 2356 // Revert version 8 ACLs and make sure the query works again. 2357 s1.config.ACLEnforceVersion8 = false 2358 { 2359 req := structs.PreparedQueryExecuteRequest{ 2360 Datacenter: "dc1", 2361 QueryIDOrName: query.Query.ID, 2362 QueryOptions: structs.QueryOptions{Token: execToken}, 2363 } 2364 2365 var reply structs.PreparedQueryExecuteResponse 2366 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2367 t.Fatalf("err: %v", err) 2368 } 2369 2370 if len(reply.Nodes) != 8 || 2371 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2372 reply.Service != query.Query.Service.Service || 2373 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2374 !reply.QueryMeta.KnownLeader { 2375 t.Fatalf("bad: %v", reply) 2376 } 2377 for _, node := range reply.Nodes { 2378 if node.Node.Node == "node1" || node.Node.Node == "node3" { 2379 t.Fatalf("bad: %v", node) 2380 } 2381 } 2382 } 2383 2384 // Now fail everything in dc1 and we should get an empty list back. 2385 for i := 0; i < 10; i++ { 2386 setHealth(fmt.Sprintf("node%d", i+1), api.HealthCritical) 2387 } 2388 { 2389 req := structs.PreparedQueryExecuteRequest{ 2390 Datacenter: "dc1", 2391 QueryIDOrName: query.Query.ID, 2392 QueryOptions: structs.QueryOptions{Token: execToken}, 2393 } 2394 2395 var reply structs.PreparedQueryExecuteResponse 2396 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2397 t.Fatalf("err: %v", err) 2398 } 2399 2400 if len(reply.Nodes) != 0 || 2401 reply.Datacenter != "dc1" || reply.Failovers != 0 || 2402 reply.Service != query.Query.Service.Service || 2403 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2404 !reply.QueryMeta.KnownLeader { 2405 t.Fatalf("bad: %v", reply) 2406 } 2407 } 2408 2409 // Modify the query to have it fail over to a bogus DC and then dc2. 2410 query.Query.Service.Failover.Datacenters = []string{"bogus", "dc2"} 2411 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2412 t.Fatalf("err: %v", err) 2413 } 2414 2415 // Now we should see 9 nodes from dc2 (we have the tag filter still). 2416 { 2417 req := structs.PreparedQueryExecuteRequest{ 2418 Datacenter: "dc1", 2419 QueryIDOrName: query.Query.ID, 2420 QueryOptions: structs.QueryOptions{Token: execToken}, 2421 } 2422 2423 var reply structs.PreparedQueryExecuteResponse 2424 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2425 t.Fatalf("err: %v", err) 2426 } 2427 2428 if len(reply.Nodes) != 9 || 2429 reply.Datacenter != "dc2" || reply.Failovers != 1 || 2430 reply.Service != query.Query.Service.Service || 2431 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2432 !reply.QueryMeta.KnownLeader { 2433 t.Fatalf("bad: %v", reply) 2434 } 2435 for _, node := range reply.Nodes { 2436 if node.Node.Node == "node3" { 2437 t.Fatalf("bad: %v", node) 2438 } 2439 } 2440 } 2441 2442 // Make sure the limit and query options are forwarded. 2443 { 2444 req := structs.PreparedQueryExecuteRequest{ 2445 Datacenter: "dc1", 2446 QueryIDOrName: query.Query.ID, 2447 Limit: 3, 2448 QueryOptions: structs.QueryOptions{ 2449 Token: execToken, 2450 RequireConsistent: true, 2451 }, 2452 } 2453 2454 var reply structs.PreparedQueryExecuteResponse 2455 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2456 t.Fatalf("err: %v", err) 2457 } 2458 2459 if len(reply.Nodes) != 3 || 2460 reply.Datacenter != "dc2" || reply.Failovers != 1 || 2461 reply.Service != query.Query.Service.Service || 2462 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2463 !reply.QueryMeta.KnownLeader { 2464 t.Fatalf("bad: %v", reply) 2465 } 2466 for _, node := range reply.Nodes { 2467 if node.Node.Node == "node3" { 2468 t.Fatalf("bad: %v", node) 2469 } 2470 } 2471 } 2472 2473 // Make sure the remote shuffle looks like it's working. 2474 uniques = make(map[string]struct{}) 2475 for i := 0; i < 100; i++ { 2476 req := structs.PreparedQueryExecuteRequest{ 2477 Datacenter: "dc1", 2478 QueryIDOrName: query.Query.ID, 2479 QueryOptions: structs.QueryOptions{Token: execToken}, 2480 } 2481 2482 var reply structs.PreparedQueryExecuteResponse 2483 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2484 t.Fatalf("err: %v", err) 2485 } 2486 2487 if len(reply.Nodes) != 9 || 2488 reply.Datacenter != "dc2" || reply.Failovers != 1 || 2489 reply.Service != query.Query.Service.Service || 2490 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2491 !reply.QueryMeta.KnownLeader { 2492 t.Fatalf("bad: %v", reply) 2493 } 2494 var names []string 2495 for _, node := range reply.Nodes { 2496 names = append(names, node.Node.Node) 2497 } 2498 key := strings.Join(names, "|") 2499 uniques[key] = struct{}{} 2500 } 2501 2502 // We have to allow for the fact that there won't always be a unique 2503 // shuffle each pass, so we just look for smell here without the test 2504 // being flaky. 2505 if len(uniques) < 50 { 2506 t.Fatalf("unique shuffle ratio too low: %d/100", len(uniques)) 2507 } 2508 2509 // Make sure the query response from dc2 gets denied with the deny token. 2510 { 2511 req := structs.PreparedQueryExecuteRequest{ 2512 Datacenter: "dc1", 2513 QueryIDOrName: query.Query.ID, 2514 QueryOptions: structs.QueryOptions{Token: denyToken}, 2515 } 2516 2517 var reply structs.PreparedQueryExecuteResponse 2518 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2519 t.Fatalf("err: %v", err) 2520 } 2521 2522 if len(reply.Nodes) != 0 || 2523 reply.Datacenter != "dc2" || reply.Failovers != 1 || 2524 reply.Service != query.Query.Service.Service || 2525 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2526 !reply.QueryMeta.KnownLeader { 2527 t.Fatalf("bad: %v", reply) 2528 } 2529 } 2530 2531 // Bake the exec token into the query. 2532 query.Query.Token = execToken 2533 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID); err != nil { 2534 t.Fatalf("err: %v", err) 2535 } 2536 2537 // Now even querying with the deny token should work. 2538 { 2539 req := structs.PreparedQueryExecuteRequest{ 2540 Datacenter: "dc1", 2541 QueryIDOrName: query.Query.ID, 2542 QueryOptions: structs.QueryOptions{Token: denyToken}, 2543 } 2544 2545 var reply structs.PreparedQueryExecuteResponse 2546 if err := msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply); err != nil { 2547 t.Fatalf("err: %v", err) 2548 } 2549 2550 if len(reply.Nodes) != 9 || 2551 reply.Datacenter != "dc2" || reply.Failovers != 1 || 2552 reply.Service != query.Query.Service.Service || 2553 !reflect.DeepEqual(reply.DNS, query.Query.DNS) || 2554 !reply.QueryMeta.KnownLeader { 2555 t.Fatalf("bad: %v", reply) 2556 } 2557 for _, node := range reply.Nodes { 2558 if node.Node.Node == "node3" { 2559 t.Fatalf("bad: %v", node) 2560 } 2561 } 2562 } 2563 } 2564 2565 func TestPreparedQuery_Execute_ForwardLeader(t *testing.T) { 2566 t.Parallel() 2567 dir1, s1 := testServer(t) 2568 defer os.RemoveAll(dir1) 2569 defer s1.Shutdown() 2570 codec1 := rpcClient(t, s1) 2571 defer codec1.Close() 2572 2573 dir2, s2 := testServer(t) 2574 defer os.RemoveAll(dir2) 2575 defer s2.Shutdown() 2576 codec2 := rpcClient(t, s2) 2577 defer codec2.Close() 2578 2579 // Try to join. 2580 joinLAN(t, s2, s1) 2581 2582 testrpc.WaitForLeader(t, s1.RPC, "dc1") 2583 testrpc.WaitForLeader(t, s2.RPC, "dc1") 2584 2585 // Use the follower as the client. 2586 var codec rpc.ClientCodec 2587 if !s1.IsLeader() { 2588 codec = codec1 2589 } else { 2590 codec = codec2 2591 } 2592 2593 // Set up a node and service in the catalog. 2594 { 2595 req := structs.RegisterRequest{ 2596 Datacenter: "dc1", 2597 Node: "foo", 2598 Address: "127.0.0.1", 2599 Service: &structs.NodeService{ 2600 Service: "redis", 2601 Tags: []string{"master"}, 2602 Port: 8000, 2603 }, 2604 } 2605 var reply struct{} 2606 if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply); err != nil { 2607 t.Fatalf("err: %v", err) 2608 } 2609 } 2610 2611 // Set up a bare bones query. 2612 query := structs.PreparedQueryRequest{ 2613 Datacenter: "dc1", 2614 Op: structs.PreparedQueryCreate, 2615 Query: &structs.PreparedQuery{ 2616 Name: "test", 2617 Service: structs.ServiceQuery{ 2618 Service: "redis", 2619 }, 2620 }, 2621 } 2622 var reply string 2623 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Apply", &query, &reply); err != nil { 2624 t.Fatalf("err: %v", err) 2625 } 2626 2627 // Execute it through the follower. 2628 { 2629 req := structs.PreparedQueryExecuteRequest{ 2630 Datacenter: "dc1", 2631 QueryIDOrName: reply, 2632 } 2633 var reply structs.PreparedQueryExecuteResponse 2634 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Execute", &req, &reply); err != nil { 2635 t.Fatalf("err: %v", err) 2636 } 2637 2638 if len(reply.Nodes) != 1 { 2639 t.Fatalf("bad: %v", reply) 2640 } 2641 } 2642 2643 // Execute it through the follower with consistency turned on. 2644 { 2645 req := structs.PreparedQueryExecuteRequest{ 2646 Datacenter: "dc1", 2647 QueryIDOrName: reply, 2648 QueryOptions: structs.QueryOptions{RequireConsistent: true}, 2649 } 2650 var reply structs.PreparedQueryExecuteResponse 2651 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.Execute", &req, &reply); err != nil { 2652 t.Fatalf("err: %v", err) 2653 } 2654 2655 if len(reply.Nodes) != 1 { 2656 t.Fatalf("bad: %v", reply) 2657 } 2658 } 2659 2660 // Remote execute it through the follower. 2661 { 2662 req := structs.PreparedQueryExecuteRemoteRequest{ 2663 Datacenter: "dc1", 2664 Query: *query.Query, 2665 } 2666 var reply structs.PreparedQueryExecuteResponse 2667 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.ExecuteRemote", &req, &reply); err != nil { 2668 t.Fatalf("err: %v", err) 2669 } 2670 2671 if len(reply.Nodes) != 1 { 2672 t.Fatalf("bad: %v", reply) 2673 } 2674 } 2675 2676 // Remote execute it through the follower with consistency turned on. 2677 { 2678 req := structs.PreparedQueryExecuteRemoteRequest{ 2679 Datacenter: "dc1", 2680 Query: *query.Query, 2681 QueryOptions: structs.QueryOptions{RequireConsistent: true}, 2682 } 2683 var reply structs.PreparedQueryExecuteResponse 2684 if err := msgpackrpc.CallWithCodec(codec, "PreparedQuery.ExecuteRemote", &req, &reply); err != nil { 2685 t.Fatalf("err: %v", err) 2686 } 2687 2688 if len(reply.Nodes) != 1 { 2689 t.Fatalf("bad: %v", reply) 2690 } 2691 } 2692 } 2693 2694 func TestPreparedQuery_Execute_ConnectExact(t *testing.T) { 2695 t.Parallel() 2696 2697 require := require.New(t) 2698 dir1, s1 := testServer(t) 2699 defer os.RemoveAll(dir1) 2700 defer s1.Shutdown() 2701 codec := rpcClient(t, s1) 2702 defer codec.Close() 2703 2704 // Setup 3 services on 3 nodes: one is non-Connect, one is Connect native, 2705 // and one is a proxy to the non-Connect one. 2706 for i := 0; i < 3; i++ { 2707 req := structs.RegisterRequest{ 2708 Datacenter: "dc1", 2709 Node: fmt.Sprintf("node%d", i+1), 2710 Address: fmt.Sprintf("127.0.0.%d", i+1), 2711 Service: &structs.NodeService{ 2712 Service: "foo", 2713 Port: 8000, 2714 }, 2715 } 2716 2717 switch i { 2718 case 0: 2719 // Default do nothing 2720 2721 case 1: 2722 // Connect native 2723 req.Service.Connect.Native = true 2724 2725 case 2: 2726 // Connect proxy 2727 req.Service.Kind = structs.ServiceKindConnectProxy 2728 req.Service.Proxy.DestinationServiceName = req.Service.Service 2729 req.Service.Service = "proxy" 2730 } 2731 2732 var reply struct{} 2733 require.NoError(msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply)) 2734 } 2735 2736 // The query, start with connect disabled 2737 query := structs.PreparedQueryRequest{ 2738 Datacenter: "dc1", 2739 Op: structs.PreparedQueryCreate, 2740 Query: &structs.PreparedQuery{ 2741 Name: "test", 2742 Service: structs.ServiceQuery{ 2743 Service: "foo", 2744 }, 2745 DNS: structs.QueryDNSOptions{ 2746 TTL: "10s", 2747 }, 2748 }, 2749 } 2750 require.NoError(msgpackrpc.CallWithCodec( 2751 codec, "PreparedQuery.Apply", &query, &query.Query.ID)) 2752 2753 // In the future we'll run updates 2754 query.Op = structs.PreparedQueryUpdate 2755 2756 // Run the registered query. 2757 { 2758 req := structs.PreparedQueryExecuteRequest{ 2759 Datacenter: "dc1", 2760 QueryIDOrName: query.Query.ID, 2761 } 2762 2763 var reply structs.PreparedQueryExecuteResponse 2764 require.NoError(msgpackrpc.CallWithCodec( 2765 codec, "PreparedQuery.Execute", &req, &reply)) 2766 2767 // Result should have two because it omits the proxy whose name 2768 // doesn't match the query. 2769 require.Len(reply.Nodes, 2) 2770 require.Equal(query.Query.Service.Service, reply.Service) 2771 require.Equal(query.Query.DNS, reply.DNS) 2772 require.True(reply.QueryMeta.KnownLeader, "queried leader") 2773 } 2774 2775 // Run with the Connect setting specified on the request 2776 { 2777 req := structs.PreparedQueryExecuteRequest{ 2778 Datacenter: "dc1", 2779 QueryIDOrName: query.Query.ID, 2780 Connect: true, 2781 } 2782 2783 var reply structs.PreparedQueryExecuteResponse 2784 require.NoError(msgpackrpc.CallWithCodec( 2785 codec, "PreparedQuery.Execute", &req, &reply)) 2786 2787 // Result should have two because we should get the native AND 2788 // the proxy (since the destination matches our service name). 2789 require.Len(reply.Nodes, 2) 2790 require.Equal(query.Query.Service.Service, reply.Service) 2791 require.Equal(query.Query.DNS, reply.DNS) 2792 require.True(reply.QueryMeta.KnownLeader, "queried leader") 2793 2794 // Make sure the native is the first one 2795 if !reply.Nodes[0].Service.Connect.Native { 2796 reply.Nodes[0], reply.Nodes[1] = reply.Nodes[1], reply.Nodes[0] 2797 } 2798 2799 require.True(reply.Nodes[0].Service.Connect.Native, "native") 2800 require.Equal(reply.Service, reply.Nodes[0].Service.Service) 2801 2802 require.Equal(structs.ServiceKindConnectProxy, reply.Nodes[1].Service.Kind) 2803 require.Equal(reply.Service, reply.Nodes[1].Service.Proxy.DestinationServiceName) 2804 } 2805 2806 // Update the query 2807 query.Query.Service.Connect = true 2808 require.NoError(msgpackrpc.CallWithCodec( 2809 codec, "PreparedQuery.Apply", &query, &query.Query.ID)) 2810 2811 // Run the registered query. 2812 { 2813 req := structs.PreparedQueryExecuteRequest{ 2814 Datacenter: "dc1", 2815 QueryIDOrName: query.Query.ID, 2816 } 2817 2818 var reply structs.PreparedQueryExecuteResponse 2819 require.NoError(msgpackrpc.CallWithCodec( 2820 codec, "PreparedQuery.Execute", &req, &reply)) 2821 2822 // Result should have two because we should get the native AND 2823 // the proxy (since the destination matches our service name). 2824 require.Len(reply.Nodes, 2) 2825 require.Equal(query.Query.Service.Service, reply.Service) 2826 require.Equal(query.Query.DNS, reply.DNS) 2827 require.True(reply.QueryMeta.KnownLeader, "queried leader") 2828 2829 // Make sure the native is the first one 2830 if !reply.Nodes[0].Service.Connect.Native { 2831 reply.Nodes[0], reply.Nodes[1] = reply.Nodes[1], reply.Nodes[0] 2832 } 2833 2834 require.True(reply.Nodes[0].Service.Connect.Native, "native") 2835 require.Equal(reply.Service, reply.Nodes[0].Service.Service) 2836 2837 require.Equal(structs.ServiceKindConnectProxy, reply.Nodes[1].Service.Kind) 2838 require.Equal(reply.Service, reply.Nodes[1].Service.Proxy.DestinationServiceName) 2839 } 2840 2841 // Unset the query 2842 query.Query.Service.Connect = false 2843 require.NoError(msgpackrpc.CallWithCodec( 2844 codec, "PreparedQuery.Apply", &query, &query.Query.ID)) 2845 } 2846 2847 func TestPreparedQuery_tagFilter(t *testing.T) { 2848 t.Parallel() 2849 testNodes := func() structs.CheckServiceNodes { 2850 return structs.CheckServiceNodes{ 2851 structs.CheckServiceNode{ 2852 Node: &structs.Node{Node: "node1"}, 2853 Service: &structs.NodeService{Tags: []string{"foo"}}, 2854 }, 2855 structs.CheckServiceNode{ 2856 Node: &structs.Node{Node: "node2"}, 2857 Service: &structs.NodeService{Tags: []string{"foo", "BAR"}}, 2858 }, 2859 structs.CheckServiceNode{ 2860 Node: &structs.Node{Node: "node3"}, 2861 }, 2862 structs.CheckServiceNode{ 2863 Node: &structs.Node{Node: "node4"}, 2864 Service: &structs.NodeService{Tags: []string{"foo", "baz"}}, 2865 }, 2866 structs.CheckServiceNode{ 2867 Node: &structs.Node{Node: "node5"}, 2868 Service: &structs.NodeService{Tags: []string{"foo", "zoo"}}, 2869 }, 2870 structs.CheckServiceNode{ 2871 Node: &structs.Node{Node: "node6"}, 2872 Service: &structs.NodeService{Tags: []string{"bar"}}, 2873 }, 2874 } 2875 } 2876 2877 // This always sorts so that it's not annoying to compare after the swap 2878 // operations that the algorithm performs. 2879 stringify := func(nodes structs.CheckServiceNodes) string { 2880 var names []string 2881 for _, node := range nodes { 2882 names = append(names, node.Node.Node) 2883 } 2884 sort.Strings(names) 2885 return strings.Join(names, "|") 2886 } 2887 2888 ret := stringify(tagFilter([]string{}, testNodes())) 2889 if ret != "node1|node2|node3|node4|node5|node6" { 2890 t.Fatalf("bad: %s", ret) 2891 } 2892 2893 ret = stringify(tagFilter([]string{"foo"}, testNodes())) 2894 if ret != "node1|node2|node4|node5" { 2895 t.Fatalf("bad: %s", ret) 2896 } 2897 2898 ret = stringify(tagFilter([]string{"!foo"}, testNodes())) 2899 if ret != "node3|node6" { 2900 t.Fatalf("bad: %s", ret) 2901 } 2902 2903 ret = stringify(tagFilter([]string{"!foo", "bar"}, testNodes())) 2904 if ret != "node6" { 2905 t.Fatalf("bad: %s", ret) 2906 } 2907 2908 ret = stringify(tagFilter([]string{"!foo", "!bar"}, testNodes())) 2909 if ret != "node3" { 2910 t.Fatalf("bad: %s", ret) 2911 } 2912 2913 ret = stringify(tagFilter([]string{"nope"}, testNodes())) 2914 if ret != "" { 2915 t.Fatalf("bad: %s", ret) 2916 } 2917 2918 ret = stringify(tagFilter([]string{"bar"}, testNodes())) 2919 if ret != "node2|node6" { 2920 t.Fatalf("bad: %s", ret) 2921 } 2922 2923 ret = stringify(tagFilter([]string{"BAR"}, testNodes())) 2924 if ret != "node2|node6" { 2925 t.Fatalf("bad: %s", ret) 2926 } 2927 2928 ret = stringify(tagFilter([]string{"bAr"}, testNodes())) 2929 if ret != "node2|node6" { 2930 t.Fatalf("bad: %s", ret) 2931 } 2932 2933 ret = stringify(tagFilter([]string{""}, testNodes())) 2934 if ret != "" { 2935 t.Fatalf("bad: %s", ret) 2936 } 2937 } 2938 2939 func TestPreparedQuery_Wrapper(t *testing.T) { 2940 t.Parallel() 2941 dir1, s1 := testServerWithConfig(t, func(c *Config) { 2942 c.ACLDatacenter = "dc1" 2943 c.ACLsEnabled = true 2944 c.ACLMasterToken = "root" 2945 c.ACLDefaultPolicy = "deny" 2946 }) 2947 defer os.RemoveAll(dir1) 2948 defer s1.Shutdown() 2949 2950 dir2, s2 := testServerWithConfig(t, func(c *Config) { 2951 c.Datacenter = "dc2" 2952 c.ACLDatacenter = "dc1" 2953 c.ACLsEnabled = true 2954 c.ACLMasterToken = "root" 2955 c.ACLDefaultPolicy = "deny" 2956 }) 2957 defer os.RemoveAll(dir2) 2958 defer s2.Shutdown() 2959 2960 testrpc.WaitForLeader(t, s1.RPC, "dc1") 2961 testrpc.WaitForLeader(t, s2.RPC, "dc2") 2962 2963 // Try to WAN join. 2964 joinWAN(t, s2, s1) 2965 2966 // Try all the operations on a real server via the wrapper. 2967 wrapper := &queryServerWrapper{s1} 2968 wrapper.GetLogger().Printf("[DEBUG] Test") 2969 2970 ret, err := wrapper.GetOtherDatacentersByDistance() 2971 wrapper.GetLogger().Println("Returned value: ", ret) 2972 if err != nil { 2973 t.Fatalf("err: %v", err) 2974 } 2975 if len(ret) != 1 || ret[0] != "dc2" { 2976 t.Fatalf("bad: %v", ret) 2977 } 2978 // Since we have no idea when the joinWAN operation completes 2979 // we keep on querying until the the join operation completes. 2980 retry.Run(t, func(r *retry.R) { 2981 r.Check(s1.forwardDC("Status.Ping", "dc2", &struct{}{}, &struct{}{})) 2982 }) 2983 } 2984 2985 type mockQueryServer struct { 2986 Datacenters []string 2987 DatacentersError error 2988 QueryLog []string 2989 QueryFn func(dc string, args interface{}, reply interface{}) error 2990 Logger *log.Logger 2991 LogBuffer *bytes.Buffer 2992 } 2993 2994 func (m *mockQueryServer) JoinQueryLog() string { 2995 return strings.Join(m.QueryLog, "|") 2996 } 2997 2998 func (m *mockQueryServer) GetLogger() *log.Logger { 2999 if m.Logger == nil { 3000 m.LogBuffer = new(bytes.Buffer) 3001 m.Logger = log.New(m.LogBuffer, "", 0) 3002 } 3003 return m.Logger 3004 } 3005 3006 func (m *mockQueryServer) GetOtherDatacentersByDistance() ([]string, error) { 3007 return m.Datacenters, m.DatacentersError 3008 } 3009 3010 func (m *mockQueryServer) ForwardDC(method, dc string, args interface{}, reply interface{}) error { 3011 m.QueryLog = append(m.QueryLog, fmt.Sprintf("%s:%s", dc, method)) 3012 if ret, ok := reply.(*structs.PreparedQueryExecuteResponse); ok { 3013 ret.Datacenter = dc 3014 } 3015 if m.QueryFn != nil { 3016 return m.QueryFn(dc, args, reply) 3017 } 3018 return nil 3019 } 3020 3021 func TestPreparedQuery_queryFailover(t *testing.T) { 3022 t.Parallel() 3023 query := &structs.PreparedQuery{ 3024 Name: "test", 3025 Service: structs.ServiceQuery{ 3026 Failover: structs.QueryDatacenterOptions{ 3027 NearestN: 0, 3028 Datacenters: []string{""}, 3029 }, 3030 }, 3031 } 3032 3033 nodes := func() structs.CheckServiceNodes { 3034 return structs.CheckServiceNodes{ 3035 structs.CheckServiceNode{ 3036 Node: &structs.Node{Node: "node1"}, 3037 }, 3038 structs.CheckServiceNode{ 3039 Node: &structs.Node{Node: "node2"}, 3040 }, 3041 structs.CheckServiceNode{ 3042 Node: &structs.Node{Node: "node3"}, 3043 }, 3044 } 3045 } 3046 3047 // Datacenters are available but the query doesn't use them. 3048 { 3049 mock := &mockQueryServer{ 3050 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3051 } 3052 3053 var reply structs.PreparedQueryExecuteResponse 3054 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3055 t.Fatalf("err: %v", err) 3056 } 3057 if len(reply.Nodes) != 0 || reply.Datacenter != "" || reply.Failovers != 0 { 3058 t.Fatalf("bad: %v", reply) 3059 } 3060 } 3061 3062 // Make it fail to get datacenters. 3063 { 3064 mock := &mockQueryServer{ 3065 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3066 DatacentersError: fmt.Errorf("XXX"), 3067 } 3068 3069 var reply structs.PreparedQueryExecuteResponse 3070 err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply) 3071 if err == nil || !strings.Contains(err.Error(), "XXX") { 3072 t.Fatalf("bad: %v", err) 3073 } 3074 if len(reply.Nodes) != 0 || reply.Datacenter != "" || reply.Failovers != 0 { 3075 t.Fatalf("bad: %v", reply) 3076 } 3077 } 3078 3079 // The query wants to use other datacenters but none are available. 3080 query.Service.Failover.NearestN = 3 3081 { 3082 mock := &mockQueryServer{ 3083 Datacenters: []string{}, 3084 } 3085 3086 var reply structs.PreparedQueryExecuteResponse 3087 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3088 t.Fatalf("err: %v", err) 3089 } 3090 if len(reply.Nodes) != 0 || reply.Datacenter != "" || reply.Failovers != 0 { 3091 t.Fatalf("bad: %v", reply) 3092 } 3093 } 3094 3095 // Try the first three nearest datacenters, first one has the data. 3096 query.Service.Failover.NearestN = 3 3097 { 3098 mock := &mockQueryServer{ 3099 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3100 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3101 ret := reply.(*structs.PreparedQueryExecuteResponse) 3102 if dc == "dc1" { 3103 ret.Nodes = nodes() 3104 } 3105 return nil 3106 }, 3107 } 3108 3109 var reply structs.PreparedQueryExecuteResponse 3110 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3111 t.Fatalf("err: %v", err) 3112 } 3113 if len(reply.Nodes) != 3 || 3114 reply.Datacenter != "dc1" || reply.Failovers != 1 || 3115 !reflect.DeepEqual(reply.Nodes, nodes()) { 3116 t.Fatalf("bad: %v", reply) 3117 } 3118 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote" { 3119 t.Fatalf("bad: %s", queries) 3120 } 3121 } 3122 3123 // Try the first three nearest datacenters, last one has the data. 3124 query.Service.Failover.NearestN = 3 3125 { 3126 mock := &mockQueryServer{ 3127 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3128 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3129 ret := reply.(*structs.PreparedQueryExecuteResponse) 3130 if dc == "dc3" { 3131 ret.Nodes = nodes() 3132 } 3133 return nil 3134 }, 3135 } 3136 3137 var reply structs.PreparedQueryExecuteResponse 3138 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3139 t.Fatalf("err: %v", err) 3140 } 3141 if len(reply.Nodes) != 3 || 3142 reply.Datacenter != "dc3" || reply.Failovers != 3 || 3143 !reflect.DeepEqual(reply.Nodes, nodes()) { 3144 t.Fatalf("bad: %v", reply) 3145 } 3146 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc3:PreparedQuery.ExecuteRemote" { 3147 t.Fatalf("bad: %s", queries) 3148 } 3149 } 3150 3151 // Try the first four nearest datacenters, nobody has the data. 3152 query.Service.Failover.NearestN = 4 3153 { 3154 mock := &mockQueryServer{ 3155 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3156 } 3157 3158 var reply structs.PreparedQueryExecuteResponse 3159 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3160 t.Fatalf("err: %v", err) 3161 } 3162 if len(reply.Nodes) != 0 || 3163 reply.Datacenter != "xxx" || reply.Failovers != 4 { 3164 t.Fatalf("bad: %v", reply) 3165 } 3166 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc3:PreparedQuery.ExecuteRemote|xxx:PreparedQuery.ExecuteRemote" { 3167 t.Fatalf("bad: %s", queries) 3168 } 3169 } 3170 3171 // Try the first two nearest datacenters, plus a user-specified one that 3172 // has the data. 3173 query.Service.Failover.NearestN = 2 3174 query.Service.Failover.Datacenters = []string{"dc4"} 3175 { 3176 mock := &mockQueryServer{ 3177 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3178 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3179 ret := reply.(*structs.PreparedQueryExecuteResponse) 3180 if dc == "dc4" { 3181 ret.Nodes = nodes() 3182 } 3183 return nil 3184 }, 3185 } 3186 3187 var reply structs.PreparedQueryExecuteResponse 3188 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3189 t.Fatalf("err: %v", err) 3190 } 3191 if len(reply.Nodes) != 3 || 3192 reply.Datacenter != "dc4" || reply.Failovers != 3 || 3193 !reflect.DeepEqual(reply.Nodes, nodes()) { 3194 t.Fatalf("bad: %v", reply) 3195 } 3196 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc4:PreparedQuery.ExecuteRemote" { 3197 t.Fatalf("bad: %s", queries) 3198 } 3199 } 3200 3201 // Add in a hard-coded value that overlaps with the nearest list. 3202 query.Service.Failover.NearestN = 2 3203 query.Service.Failover.Datacenters = []string{"dc4", "dc1"} 3204 { 3205 mock := &mockQueryServer{ 3206 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3207 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3208 ret := reply.(*structs.PreparedQueryExecuteResponse) 3209 if dc == "dc4" { 3210 ret.Nodes = nodes() 3211 } 3212 return nil 3213 }, 3214 } 3215 3216 var reply structs.PreparedQueryExecuteResponse 3217 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3218 t.Fatalf("err: %v", err) 3219 } 3220 if len(reply.Nodes) != 3 || 3221 reply.Datacenter != "dc4" || reply.Failovers != 3 || 3222 !reflect.DeepEqual(reply.Nodes, nodes()) { 3223 t.Fatalf("bad: %v", reply) 3224 } 3225 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc4:PreparedQuery.ExecuteRemote" { 3226 t.Fatalf("bad: %s", queries) 3227 } 3228 } 3229 3230 // Now add a bogus user-defined one to the mix. 3231 query.Service.Failover.NearestN = 2 3232 query.Service.Failover.Datacenters = []string{"nope", "dc4", "dc1"} 3233 { 3234 mock := &mockQueryServer{ 3235 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3236 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3237 ret := reply.(*structs.PreparedQueryExecuteResponse) 3238 if dc == "dc4" { 3239 ret.Nodes = nodes() 3240 } 3241 return nil 3242 }, 3243 } 3244 3245 var reply structs.PreparedQueryExecuteResponse 3246 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3247 t.Fatalf("err: %v", err) 3248 } 3249 if len(reply.Nodes) != 3 || 3250 reply.Datacenter != "dc4" || reply.Failovers != 3 || 3251 !reflect.DeepEqual(reply.Nodes, nodes()) { 3252 t.Fatalf("bad: %v", reply) 3253 } 3254 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc4:PreparedQuery.ExecuteRemote" { 3255 t.Fatalf("bad: %s", queries) 3256 } 3257 if !strings.Contains(mock.LogBuffer.String(), "Skipping unknown datacenter") { 3258 t.Fatalf("bad: %s", mock.LogBuffer.String()) 3259 } 3260 } 3261 3262 // Same setup as before but dc1 is going to return an error and should 3263 // get skipped over, still yielding data from dc4 which comes later. 3264 query.Service.Failover.NearestN = 2 3265 query.Service.Failover.Datacenters = []string{"dc4", "dc1"} 3266 { 3267 mock := &mockQueryServer{ 3268 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3269 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3270 ret := reply.(*structs.PreparedQueryExecuteResponse) 3271 if dc == "dc1" { 3272 return fmt.Errorf("XXX") 3273 } else if dc == "dc4" { 3274 ret.Nodes = nodes() 3275 } 3276 return nil 3277 }, 3278 } 3279 3280 var reply structs.PreparedQueryExecuteResponse 3281 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3282 t.Fatalf("err: %v", err) 3283 } 3284 if len(reply.Nodes) != 3 || 3285 reply.Datacenter != "dc4" || reply.Failovers != 3 || 3286 !reflect.DeepEqual(reply.Nodes, nodes()) { 3287 t.Fatalf("bad: %v", reply) 3288 } 3289 if queries := mock.JoinQueryLog(); queries != "dc1:PreparedQuery.ExecuteRemote|dc2:PreparedQuery.ExecuteRemote|dc4:PreparedQuery.ExecuteRemote" { 3290 t.Fatalf("bad: %s", queries) 3291 } 3292 if !strings.Contains(mock.LogBuffer.String(), "Failed querying") { 3293 t.Fatalf("bad: %s", mock.LogBuffer.String()) 3294 } 3295 } 3296 3297 // Just use a hard-coded list and now xxx has the data. 3298 query.Service.Failover.NearestN = 0 3299 query.Service.Failover.Datacenters = []string{"dc3", "xxx"} 3300 { 3301 mock := &mockQueryServer{ 3302 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3303 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3304 ret := reply.(*structs.PreparedQueryExecuteResponse) 3305 if dc == "xxx" { 3306 ret.Nodes = nodes() 3307 } 3308 return nil 3309 }, 3310 } 3311 3312 var reply structs.PreparedQueryExecuteResponse 3313 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{}, &reply); err != nil { 3314 t.Fatalf("err: %v", err) 3315 } 3316 if len(reply.Nodes) != 3 || 3317 reply.Datacenter != "xxx" || reply.Failovers != 2 || 3318 !reflect.DeepEqual(reply.Nodes, nodes()) { 3319 t.Fatalf("bad: %v", reply) 3320 } 3321 if queries := mock.JoinQueryLog(); queries != "dc3:PreparedQuery.ExecuteRemote|xxx:PreparedQuery.ExecuteRemote" { 3322 t.Fatalf("bad: %s", queries) 3323 } 3324 } 3325 3326 // Make sure the limit and query options are plumbed through. 3327 query.Service.Failover.NearestN = 0 3328 query.Service.Failover.Datacenters = []string{"xxx"} 3329 { 3330 mock := &mockQueryServer{ 3331 Datacenters: []string{"dc1", "dc2", "dc3", "xxx", "dc4"}, 3332 QueryFn: func(dc string, args interface{}, reply interface{}) error { 3333 inp := args.(*structs.PreparedQueryExecuteRemoteRequest) 3334 ret := reply.(*structs.PreparedQueryExecuteResponse) 3335 if dc == "xxx" { 3336 if inp.Limit != 5 { 3337 t.Fatalf("bad: %d", inp.Limit) 3338 } 3339 if inp.RequireConsistent != true { 3340 t.Fatalf("bad: %v", inp.RequireConsistent) 3341 } 3342 ret.Nodes = nodes() 3343 } 3344 return nil 3345 }, 3346 } 3347 3348 var reply structs.PreparedQueryExecuteResponse 3349 if err := queryFailover(mock, query, &structs.PreparedQueryExecuteRequest{ 3350 Limit: 5, 3351 QueryOptions: structs.QueryOptions{RequireConsistent: true}, 3352 }, &reply); err != nil { 3353 t.Fatalf("err: %v", err) 3354 } 3355 if len(reply.Nodes) != 3 || 3356 reply.Datacenter != "xxx" || reply.Failovers != 1 || 3357 !reflect.DeepEqual(reply.Nodes, nodes()) { 3358 t.Fatalf("bad: %v", reply) 3359 } 3360 if queries := mock.JoinQueryLog(); queries != "xxx:PreparedQuery.ExecuteRemote" { 3361 t.Fatalf("bad: %s", queries) 3362 } 3363 } 3364 }