github.com/gravitational/teleport/api@v0.0.0-20240507183017-3110591cbafc/types/server_test.go (about) 1 /* 2 * Copyright 2022 Gravitational, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package types 18 19 import ( 20 "fmt" 21 "testing" 22 23 "github.com/gravitational/trace" 24 "github.com/stretchr/testify/assert" 25 "github.com/stretchr/testify/require" 26 27 "github.com/gravitational/teleport/api/defaults" 28 "github.com/gravitational/teleport/api/utils/aws" 29 ) 30 31 func getTestVal(isTestField bool, testVal string) string { 32 if isTestField { 33 return testVal 34 } 35 36 return "foo" 37 } 38 39 func TestServerSorter(t *testing.T) { 40 t.Parallel() 41 42 testValsUnordered := []string{"d", "b", "a", "c"} 43 44 makeServers := func(testVals []string, testField string) []Server { 45 servers := make([]Server, len(testVals)) 46 for i := 0; i < len(testVals); i++ { 47 testVal := testVals[i] 48 var err error 49 servers[i], err = NewServer( 50 getTestVal(testField == ResourceMetadataName, testVal), 51 KindNode, 52 ServerSpecV2{ 53 Hostname: getTestVal(testField == ResourceSpecHostname, testVal), 54 Addr: getTestVal(testField == ResourceSpecAddr, testVal), 55 }) 56 require.NoError(t, err) 57 } 58 return servers 59 } 60 61 cases := []struct { 62 name string 63 wantErr bool 64 fieldName string 65 }{ 66 { 67 name: "by name", 68 fieldName: ResourceMetadataName, 69 }, 70 { 71 name: "by hostname", 72 fieldName: ResourceSpecHostname, 73 }, 74 { 75 name: "by addr", 76 fieldName: ResourceSpecAddr, 77 }, 78 } 79 80 for _, c := range cases { 81 c := c 82 t.Run(fmt.Sprintf("%s desc", c.name), func(t *testing.T) { 83 sortBy := SortBy{Field: c.fieldName, IsDesc: true} 84 servers := Servers(makeServers(testValsUnordered, c.fieldName)) 85 require.NoError(t, servers.SortByCustom(sortBy)) 86 targetVals, err := servers.GetFieldVals(c.fieldName) 87 require.NoError(t, err) 88 require.IsDecreasing(t, targetVals) 89 }) 90 91 t.Run(fmt.Sprintf("%s asc", c.name), func(t *testing.T) { 92 sortBy := SortBy{Field: c.fieldName} 93 servers := Servers(makeServers(testValsUnordered, c.fieldName)) 94 require.NoError(t, servers.SortByCustom(sortBy)) 95 targetVals, err := servers.GetFieldVals(c.fieldName) 96 require.NoError(t, err) 97 require.IsIncreasing(t, targetVals) 98 }) 99 } 100 101 // Test error. 102 sortBy := SortBy{Field: "unsupported"} 103 servers := makeServers(testValsUnordered, "does-not-matter") 104 require.True(t, trace.IsNotImplemented(Servers(servers).SortByCustom(sortBy))) 105 } 106 107 func TestServerCheckAndSetDefaults(t *testing.T) { 108 t.Parallel() 109 110 makeOpenSSHEC2InstanceConnectEndpointNode := func(fn func(s *ServerV2)) *ServerV2 { 111 s := &ServerV2{ 112 Kind: KindNode, 113 SubKind: SubKindOpenSSHEICENode, 114 Version: V2, 115 Metadata: Metadata{ 116 Namespace: defaults.Namespace, 117 }, 118 Spec: ServerSpecV2{ 119 Addr: "example:22", 120 Hostname: "openssh-node", 121 CloudMetadata: &CloudMetadata{ 122 AWS: &AWSInfo{ 123 AccountID: "123456789012", 124 InstanceID: "i-123456789012", 125 Region: "us-east-1", 126 VPCID: "vpc-abcd", 127 SubnetID: "subnet-123", 128 Integration: "teleportdev", 129 }, 130 }, 131 }, 132 } 133 if fn != nil { 134 fn(s) 135 } 136 return s 137 } 138 139 tests := []struct { 140 name string 141 server *ServerV2 142 assertion func(t *testing.T, s *ServerV2, err error) 143 }{ 144 { 145 name: "Teleport node", 146 server: &ServerV2{ 147 Kind: KindNode, 148 SubKind: SubKindTeleportNode, 149 Version: V2, 150 Metadata: Metadata{ 151 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 152 Namespace: defaults.Namespace, 153 }, 154 Spec: ServerSpecV2{ 155 Addr: "1.2.3.4:3022", 156 Hostname: "teleport-node", 157 PublicAddrs: []string{"1.2.3.4:3080"}, 158 }, 159 }, 160 assertion: func(t *testing.T, s *ServerV2, err error) { 161 require.NoError(t, err) 162 expectedServer := &ServerV2{ 163 Kind: KindNode, 164 SubKind: SubKindTeleportNode, 165 Version: V2, 166 Metadata: Metadata{ 167 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 168 Namespace: defaults.Namespace, 169 }, 170 Spec: ServerSpecV2{ 171 Addr: "1.2.3.4:3022", 172 Hostname: "teleport-node", 173 PublicAddrs: []string{"1.2.3.4:3080"}, 174 }, 175 } 176 require.Equal(t, expectedServer, s) 177 }, 178 }, 179 { 180 name: "Teleport node subkind unset", 181 server: &ServerV2{ 182 Kind: KindNode, 183 Version: V2, 184 Metadata: Metadata{ 185 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 186 Namespace: defaults.Namespace, 187 }, 188 Spec: ServerSpecV2{ 189 Addr: "1.2.3.4:3022", 190 Hostname: "teleport-node", 191 PublicAddrs: []string{"1.2.3.4:3080"}, 192 }, 193 }, 194 assertion: func(t *testing.T, s *ServerV2, err error) { 195 require.NoError(t, err) 196 expectedServer := &ServerV2{ 197 Kind: KindNode, 198 Version: V2, 199 Metadata: Metadata{ 200 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 201 Namespace: defaults.Namespace, 202 }, 203 Spec: ServerSpecV2{ 204 Addr: "1.2.3.4:3022", 205 Hostname: "teleport-node", 206 PublicAddrs: []string{"1.2.3.4:3080"}, 207 }, 208 } 209 require.Equal(t, expectedServer, s) 210 require.False(t, s.IsOpenSSHNode(), "IsOpenSSHNode must be false for this node") 211 }, 212 }, 213 { 214 name: "OpenSSH node", 215 server: &ServerV2{ 216 Kind: KindNode, 217 SubKind: SubKindOpenSSHNode, 218 Version: V2, 219 Metadata: Metadata{ 220 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 221 Namespace: defaults.Namespace, 222 }, 223 Spec: ServerSpecV2{ 224 Addr: "1.2.3.4:3022", 225 Hostname: "openssh-node", 226 }, 227 }, 228 assertion: func(t *testing.T, s *ServerV2, err error) { 229 require.NoError(t, err) 230 expectedServer := &ServerV2{ 231 Kind: KindNode, 232 SubKind: SubKindOpenSSHNode, 233 Version: V2, 234 Metadata: Metadata{ 235 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 236 Namespace: defaults.Namespace, 237 }, 238 Spec: ServerSpecV2{ 239 Addr: "1.2.3.4:3022", 240 Hostname: "openssh-node", 241 }, 242 } 243 require.Equal(t, expectedServer, s) 244 }, 245 }, 246 { 247 name: "OpenSSH node with dns address", 248 server: &ServerV2{ 249 Kind: KindNode, 250 SubKind: SubKindOpenSSHNode, 251 Version: V2, 252 Metadata: Metadata{ 253 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 254 Namespace: defaults.Namespace, 255 }, 256 Spec: ServerSpecV2{ 257 Addr: "example:22", 258 Hostname: "openssh-node", 259 }, 260 }, 261 assertion: func(t *testing.T, s *ServerV2, err error) { 262 require.NoError(t, err) 263 expectedServer := &ServerV2{ 264 Kind: KindNode, 265 SubKind: SubKindOpenSSHNode, 266 Version: V2, 267 Metadata: Metadata{ 268 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 269 Namespace: defaults.Namespace, 270 }, 271 Spec: ServerSpecV2{ 272 Addr: "example:22", 273 Hostname: "openssh-node", 274 }, 275 } 276 require.Equal(t, expectedServer, s) 277 require.True(t, s.IsOpenSSHNode(), "IsOpenSSHNode must be true for this node") 278 }, 279 }, 280 { 281 name: "OpenSSH node with unset name", 282 server: &ServerV2{ 283 Kind: KindNode, 284 SubKind: SubKindOpenSSHNode, 285 Version: V2, 286 Spec: ServerSpecV2{ 287 Addr: "1.2.3.4:22", 288 Hostname: "openssh-node", 289 }, 290 }, 291 assertion: func(t *testing.T, s *ServerV2, err error) { 292 require.NoError(t, err) 293 require.NotEmpty(t, s.Metadata.Name) 294 require.True(t, s.IsOpenSSHNode(), "IsOpenSSHNode must be true for this node") 295 }, 296 }, 297 { 298 name: "OpenSSH node with unset addr", 299 server: &ServerV2{ 300 Kind: KindNode, 301 SubKind: SubKindOpenSSHNode, 302 Version: V2, 303 Metadata: Metadata{ 304 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 305 Namespace: defaults.Namespace, 306 }, 307 Spec: ServerSpecV2{ 308 Hostname: "openssh-node", 309 }, 310 }, 311 assertion: func(t *testing.T, s *ServerV2, err error) { 312 require.ErrorContains(t, err, "addr must be set") 313 }, 314 }, 315 { 316 name: "OpenSSH node with unset hostname", 317 server: &ServerV2{ 318 Kind: KindNode, 319 SubKind: SubKindOpenSSHNode, 320 Version: V2, 321 Metadata: Metadata{ 322 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 323 Namespace: defaults.Namespace, 324 }, 325 Spec: ServerSpecV2{ 326 Addr: "1.2.3.4:3022", 327 }, 328 }, 329 assertion: func(t *testing.T, s *ServerV2, err error) { 330 require.ErrorContains(t, err, "hostname must be set") 331 }, 332 }, 333 { 334 name: "OpenSSH node with public addr", 335 server: &ServerV2{ 336 Kind: KindNode, 337 SubKind: SubKindOpenSSHNode, 338 Version: V2, 339 Metadata: Metadata{ 340 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 341 Namespace: defaults.Namespace, 342 }, 343 Spec: ServerSpecV2{ 344 Addr: "1.2.3.4:3022", 345 Hostname: "openssh-node", 346 PublicAddrs: []string{"1.2.3.4:80"}, 347 }, 348 }, 349 assertion: func(t *testing.T, s *ServerV2, err error) { 350 require.ErrorContains(t, err, "publicAddrs must not be set") 351 }, 352 }, 353 { 354 name: "OpenSSH node with invalid addr", 355 server: &ServerV2{ 356 Kind: KindNode, 357 SubKind: SubKindOpenSSHNode, 358 Version: V2, 359 Metadata: Metadata{ 360 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 361 Namespace: defaults.Namespace, 362 }, 363 Spec: ServerSpecV2{ 364 Addr: "invalid-addr", 365 Hostname: "openssh-node", 366 }, 367 }, 368 assertion: func(t *testing.T, s *ServerV2, err error) { 369 require.ErrorContains(t, err, `invalid Addr "invalid-addr"`) 370 }, 371 }, 372 { 373 name: "node with invalid subkind", 374 server: &ServerV2{ 375 Kind: KindNode, 376 SubKind: "invalid-subkind", 377 Version: V2, 378 Metadata: Metadata{ 379 Name: "5da56852-2adb-4540-a37c-80790203f6a9", 380 Namespace: defaults.Namespace, 381 }, 382 Spec: ServerSpecV2{ 383 Addr: "1.2.3.4:22", 384 Hostname: "node", 385 }, 386 }, 387 assertion: func(t *testing.T, s *ServerV2, err error) { 388 require.EqualError(t, err, `invalid SubKind "invalid-subkind"`) 389 }, 390 }, 391 { 392 name: "OpenSSHEC2InstanceConnectEndpoint node without cloud metadata", 393 server: makeOpenSSHEC2InstanceConnectEndpointNode(func(s *ServerV2) { 394 s.Spec.CloudMetadata = nil 395 }), 396 assertion: func(t *testing.T, s *ServerV2, err error) { 397 require.ErrorContains(t, err, "missing account id or instance id in openssh-ec2-ice node") 398 }, 399 }, 400 { 401 name: "OpenSSHEC2InstanceConnectEndpoint node with cloud metadata but missing aws info", 402 server: makeOpenSSHEC2InstanceConnectEndpointNode(func(s *ServerV2) { 403 s.Spec.CloudMetadata.AWS = nil 404 }), 405 assertion: func(t *testing.T, s *ServerV2, err error) { 406 require.ErrorContains(t, err, "missing account id or instance id in openssh-ec2-ice node") 407 }, 408 }, 409 { 410 name: "OpenSSHEC2InstanceConnectEndpoint node with aws cloud metadata but missing accountid", 411 server: makeOpenSSHEC2InstanceConnectEndpointNode(func(s *ServerV2) { 412 s.Spec.CloudMetadata.AWS.AccountID = "" 413 }), 414 assertion: func(t *testing.T, s *ServerV2, err error) { 415 require.ErrorContains(t, err, "missing account id or instance id in openssh-ec2-ice node") 416 }, 417 }, 418 { 419 name: "OpenSSHEC2InstanceConnectEndpoint node with aws cloud metadata but missing instanceid", 420 server: makeOpenSSHEC2InstanceConnectEndpointNode(func(s *ServerV2) { 421 s.Spec.CloudMetadata.AWS.InstanceID = "" 422 }), 423 assertion: func(t *testing.T, s *ServerV2, err error) { 424 require.ErrorContains(t, err, "missing account id or instance id in openssh-ec2-ice node") 425 }, 426 }, 427 { 428 name: "OpenSSHEC2InstanceConnectEndpoint node with aws cloud metadata but missing region", 429 server: makeOpenSSHEC2InstanceConnectEndpointNode(func(s *ServerV2) { 430 s.Spec.CloudMetadata.AWS.Region = "" 431 }), 432 assertion: func(t *testing.T, s *ServerV2, err error) { 433 require.ErrorContains(t, err, "missing AWS Region") 434 }, 435 }, 436 { 437 name: "OpenSSHEC2InstanceConnectEndpoint node with aws cloud metadata but missing vpc id", 438 server: &ServerV2{ 439 Kind: KindNode, 440 SubKind: SubKindOpenSSHEICENode, 441 Version: V2, 442 Metadata: Metadata{ 443 Namespace: defaults.Namespace, 444 }, 445 Spec: ServerSpecV2{ 446 Addr: "example:22", 447 Hostname: "openssh-node", 448 CloudMetadata: &CloudMetadata{ 449 AWS: &AWSInfo{ 450 AccountID: "123456789012", 451 InstanceID: "i-123456789012", 452 Region: "us-east-1", 453 Integration: "teleportdev", 454 }, 455 }, 456 }, 457 }, 458 assertion: func(t *testing.T, s *ServerV2, err error) { 459 require.ErrorContains(t, err, "missing AWS VPC ID") 460 }, 461 }, 462 { 463 name: "OpenSSHEC2InstanceConnectEndpoint node with aws cloud metadata but missing integration", 464 server: &ServerV2{ 465 Kind: KindNode, 466 SubKind: SubKindOpenSSHEICENode, 467 Version: V2, 468 Metadata: Metadata{ 469 Namespace: defaults.Namespace, 470 }, 471 Spec: ServerSpecV2{ 472 Addr: "example:22", 473 Hostname: "openssh-node", 474 CloudMetadata: &CloudMetadata{ 475 AWS: &AWSInfo{ 476 AccountID: "123456789012", 477 InstanceID: "i-123456789012", 478 Region: "us-east-1", 479 VPCID: "vpc-abcd", 480 }, 481 }, 482 }, 483 }, 484 assertion: func(t *testing.T, s *ServerV2, err error) { 485 require.ErrorContains(t, err, "missing AWS OIDC Integration") 486 }, 487 }, 488 { 489 name: "valid OpenSSHEC2InstanceConnectEndpoint node", 490 server: makeOpenSSHEC2InstanceConnectEndpointNode(nil), 491 assertion: func(t *testing.T, s *ServerV2, err error) { 492 require.NoError(t, err) 493 expectedServer := &ServerV2{ 494 Kind: KindNode, 495 SubKind: SubKindOpenSSHEICENode, 496 Version: V2, 497 Metadata: Metadata{ 498 Name: "123456789012-i-123456789012", 499 Namespace: defaults.Namespace, 500 }, 501 Spec: ServerSpecV2{ 502 Addr: "example:22", 503 Hostname: "openssh-node", 504 CloudMetadata: &CloudMetadata{ 505 AWS: &AWSInfo{ 506 AccountID: "123456789012", 507 InstanceID: "i-123456789012", 508 Region: "us-east-1", 509 VPCID: "vpc-abcd", 510 SubnetID: "subnet-123", 511 Integration: "teleportdev", 512 }, 513 }, 514 }, 515 } 516 assert.Equal(t, expectedServer, s) 517 518 require.True(t, s.IsOpenSSHNode(), "IsOpenSSHNode must be true for this node") 519 520 require.True(t, aws.IsEC2NodeID(s.GetName()), 521 "expected an EC2 Node ID format (<accid>-<instanceid>), got %s", s.GetName(), 522 ) 523 }, 524 }, 525 { 526 name: "already existing OpenSSHEC2InstanceConnectEndpoint nodes use UUID and that must be accepted", 527 server: &ServerV2{ 528 Kind: KindNode, 529 SubKind: SubKindOpenSSHEICENode, 530 Version: V2, 531 Metadata: Metadata{ 532 Name: "f043b730-8fdd-4f9a-81e4-45f5a9ea23a7", 533 Namespace: defaults.Namespace, 534 }, 535 Spec: ServerSpecV2{ 536 Addr: "example:22", 537 Hostname: "openssh-node", 538 CloudMetadata: &CloudMetadata{ 539 AWS: &AWSInfo{ 540 AccountID: "123456789012", 541 InstanceID: "i-123456789012", 542 Region: "us-east-1", 543 Integration: "teleportdev", 544 VPCID: "some-vpc", 545 SubnetID: "some-subnet", 546 }, 547 }, 548 }, 549 }, 550 assertion: func(t *testing.T, s *ServerV2, err error) { 551 require.NoError(t, err) 552 }, 553 }, 554 { 555 name: "OpenSSHEC2InstanceConnectEndpoint nodes with invalid account id or instance id must be rejected", 556 server: &ServerV2{ 557 Kind: KindNode, 558 SubKind: SubKindOpenSSHEICENode, 559 Version: V2, 560 Metadata: Metadata{ 561 Namespace: defaults.Namespace, 562 }, 563 Spec: ServerSpecV2{ 564 Addr: "example:22", 565 Hostname: "openssh-node", 566 CloudMetadata: &CloudMetadata{ 567 AWS: &AWSInfo{ 568 AccountID: "abcd", 569 InstanceID: "i-defg", 570 Region: "us-east-1", 571 Integration: "teleportdev", 572 VPCID: "some-vpc", 573 SubnetID: "some-subnet", 574 }, 575 }, 576 }, 577 }, 578 assertion: func(t *testing.T, s *ServerV2, err error) { 579 require.ErrorContains(t, err, `invalid account "abcd" or instance id "i-defg"`) 580 }, 581 }, 582 } 583 584 for _, tt := range tests { 585 t.Run(tt.name, func(t *testing.T) { 586 err := tt.server.CheckAndSetDefaults() 587 tt.assertion(t, tt.server, err) 588 }) 589 } 590 } 591 592 func TestIsOpenSSHNodeSubKind(t *testing.T) { 593 tests := []struct { 594 name string 595 subkind string 596 want bool 597 }{ 598 { 599 name: "openssh using EC2 Instance Connect Endpoint", 600 subkind: SubKindOpenSSHEICENode, 601 want: true, 602 }, 603 { 604 name: "openssh using raw sshd server", 605 subkind: SubKindOpenSSHNode, 606 want: true, 607 }, 608 { 609 name: "regular node", 610 subkind: SubKindTeleportNode, 611 want: false, 612 }, 613 { 614 name: "another value", 615 subkind: "xyz", 616 want: false, 617 }, 618 } 619 for _, tt := range tests { 620 t.Run(tt.name, func(t *testing.T) { 621 if got := IsOpenSSHNodeSubKind(tt.subkind); got != tt.want { 622 t.Errorf("IsOpenSSHNodeSubKind() = %v, want %v", got, tt.want) 623 } 624 }) 625 } 626 } 627 628 func TestIsEICE(t *testing.T) { 629 tests := []struct { 630 name string 631 server *ServerV2 632 want bool 633 }{ 634 { 635 name: "eice node with account and instance id labels is EICE", 636 server: &ServerV2{ 637 SubKind: SubKindOpenSSHEICENode, 638 Metadata: Metadata{ 639 Labels: map[string]string{ 640 AWSAccountIDLabel: "123456789012", 641 AWSInstanceIDLabel: "i-123", 642 }, 643 }, 644 }, 645 want: true, 646 }, 647 { 648 name: "regular node not eice", 649 server: &ServerV2{ 650 SubKind: SubKindTeleportNode, 651 }, 652 want: false, 653 }, 654 { 655 name: "agentless openssh node is not eice", 656 server: &ServerV2{ 657 SubKind: SubKindOpenSSHNode, 658 }, 659 want: false, 660 }, 661 { 662 name: "eice node without account id label is not EICE", 663 server: &ServerV2{ 664 SubKind: SubKindOpenSSHEICENode, 665 Metadata: Metadata{ 666 Labels: map[string]string{ 667 AWSInstanceIDLabel: "i-123", 668 }, 669 }, 670 }, 671 want: false, 672 }, 673 { 674 name: "eice node without instance id label is not EICE", 675 server: &ServerV2{ 676 SubKind: SubKindOpenSSHEICENode, 677 Metadata: Metadata{ 678 Labels: map[string]string{ 679 AWSAccountIDLabel: "123456789012", 680 }, 681 }, 682 }, 683 want: false, 684 }, 685 } 686 for _, tt := range tests { 687 t.Run(tt.name, func(t *testing.T) { 688 if got := tt.server.IsEICE(); got != tt.want { 689 t.Errorf("IsEICE() = %v, want %v", got, tt.want) 690 } 691 }) 692 } 693 } 694 695 func TestGetCloudMetadataAWS(t *testing.T) { 696 for _, tt := range []struct { 697 name string 698 in Server 699 expected *AWSInfo 700 }{ 701 { 702 name: "no cloud metadata", 703 in: &ServerV2{ 704 Spec: ServerSpecV2{}, 705 }, 706 expected: nil, 707 }, 708 { 709 name: "cloud metadata but no AWS Information", 710 in: &ServerV2{ 711 Spec: ServerSpecV2{CloudMetadata: &CloudMetadata{}}, 712 }, 713 expected: nil, 714 }, 715 { 716 name: "cloud metadata with aws info", 717 in: &ServerV2{ 718 Spec: ServerSpecV2{CloudMetadata: &CloudMetadata{ 719 AWS: &AWSInfo{ 720 AccountID: "abcd", 721 }, 722 }}, 723 }, 724 expected: &AWSInfo{AccountID: "abcd"}, 725 }, 726 } { 727 t.Run(tt.name, func(t *testing.T) { 728 out := tt.in.GetAWSInfo() 729 require.Equal(t, tt.expected, out) 730 }) 731 } 732 }