github.com/openshift/installer@v1.4.17/pkg/asset/installconfig/openstack/validation/platform_test.go (about) 1 package validation 2 3 import ( 4 "testing" 5 6 "github.com/gophercloud/gophercloud/v2/openstack/image/v2/images" 7 "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips" 8 "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" 9 "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" 10 "github.com/stretchr/testify/assert" 11 12 "github.com/openshift/installer/pkg/ipnet" 13 "github.com/openshift/installer/pkg/types" 14 "github.com/openshift/installer/pkg/types/openstack" 15 ) 16 17 var ( 18 validCloud = "valid-cloud" 19 validExternalNetwork = "valid-external-network" 20 validFIP1 = "128.35.27.8" 21 validFIP2 = "128.35.27.13" 22 validSubnetID = "031a5b9d-5a89-4465-8d54-3517ec2bad48" 23 ) 24 25 // Returns a default install 26 func validPlatform() *openstack.Platform { 27 return &openstack.Platform{ 28 APIFloatingIP: validFIP1, 29 Cloud: validCloud, 30 ExternalNetwork: validExternalNetwork, 31 IngressFloatingIP: validFIP2, 32 } 33 } 34 35 func validControlPlanePort() *openstack.PortTarget { 36 fixedIP := openstack.FixedIP{ 37 Subnet: openstack.SubnetFilter{ID: validSubnetID, Name: "valid-subnet"}, 38 } 39 controlPlanePort := &openstack.PortTarget{ 40 FixedIPs: []openstack.FixedIP{fixedIP}, 41 } 42 return controlPlanePort 43 } 44 45 func validNetworking() *types.Networking { 46 return &types.Networking{} 47 } 48 49 func withControlPlanePortSubnets(subnetCIDR, allocationPoolStart, allocationPoolEnd string) func(*CloudInfo) { 50 return func(ci *CloudInfo) { 51 subnet := subnets.Subnet{ 52 CIDR: subnetCIDR, 53 AllocationPools: []subnets.AllocationPool{ 54 {Start: allocationPoolStart, End: allocationPoolEnd}, 55 }, 56 } 57 Allsubnets := []*subnets.Subnet{&subnet} 58 ci.ControlPlanePortSubnets = Allsubnets 59 } 60 } 61 func validPlatformCloudInfo(options ...func(*CloudInfo)) *CloudInfo { 62 ci := CloudInfo{ 63 ExternalNetwork: &networks.Network{ 64 ID: "71b97520-69af-4c35-8153-cdf827z96e60", 65 Name: validExternalNetwork, 66 AdminStateUp: true, 67 Status: "ACTIVE", 68 }, 69 APIFIP: &floatingips.FloatingIP{ 70 ID: validFIP1, 71 Status: "DOWN", 72 }, 73 IngressFIP: &floatingips.FloatingIP{ 74 ID: validFIP2, 75 Status: "DOWN", 76 }, 77 } 78 79 for _, apply := range options { 80 apply(&ci) 81 } 82 83 return &ci 84 } 85 86 func TestOpenStackPlatformValidation(t *testing.T) { 87 cases := []struct { 88 name string 89 platform *openstack.Platform 90 cloudInfo *CloudInfo 91 networking *types.Networking 92 expectedError bool 93 expectedErrMsg string // NOTE: this is a REGEXP 94 }{ 95 { 96 name: "valid platform", 97 platform: validPlatform(), 98 cloudInfo: validPlatformCloudInfo(), 99 networking: validNetworking(), 100 expectedError: false, 101 expectedErrMsg: "", 102 }, 103 { 104 name: "not found api FIP", 105 platform: validPlatform(), 106 cloudInfo: func() *CloudInfo { 107 ci := validPlatformCloudInfo() 108 ci.APIFIP = nil 109 return ci 110 }(), 111 networking: validNetworking(), 112 expectedError: true, 113 expectedErrMsg: `platform.openstack.apiFloatingIP: Not found: "128.35.27.8"`, 114 }, 115 { 116 name: "not found ingress FIP", 117 platform: validPlatform(), 118 cloudInfo: func() *CloudInfo { 119 ci := validPlatformCloudInfo() 120 ci.IngressFIP = nil 121 return ci 122 }(), 123 networking: validNetworking(), 124 expectedError: true, 125 expectedErrMsg: `platform.openstack.ingressFloatingIP: Not found: "128.35.27.13"`, 126 }, 127 { 128 name: "not found both FIPs", 129 platform: validPlatform(), 130 cloudInfo: func() *CloudInfo { 131 ci := validPlatformCloudInfo() 132 ci.IngressFIP = nil 133 ci.APIFIP = nil 134 return ci 135 }(), 136 networking: validNetworking(), 137 expectedError: true, 138 expectedErrMsg: `[platform.openstack.apiFloatingIP: Not found: "128.35.27.8", platform.openstack.ingressFloatingIP: Not found: "128.35.27.13"]`, 139 }, 140 { 141 name: "in use ingress FIP", 142 platform: validPlatform(), 143 cloudInfo: func() *CloudInfo { 144 ci := validPlatformCloudInfo() 145 ci.IngressFIP.Status = "ACTIVE" 146 return ci 147 }(), 148 networking: validNetworking(), 149 expectedError: true, 150 expectedErrMsg: `platform.openstack.ingressFloatingIP: Invalid value: "128.35.27.13": Floating IP already in use`, 151 }, 152 { 153 name: "in use api FIP", 154 platform: validPlatform(), 155 cloudInfo: func() *CloudInfo { 156 ci := validPlatformCloudInfo() 157 ci.APIFIP.Status = "ACTIVE" 158 return ci 159 }(), 160 networking: validNetworking(), 161 expectedError: true, 162 expectedErrMsg: `platform.openstack.apiFloatingIP: Invalid value: "128.35.27.8": Floating IP already in use`, 163 }, 164 { 165 name: "invalid usage both FIPs", 166 platform: func() *openstack.Platform { 167 p := validPlatform() 168 p.ExternalNetwork = "" 169 return p 170 }(), 171 cloudInfo: validPlatformCloudInfo(), 172 networking: validNetworking(), 173 expectedError: true, 174 expectedErrMsg: `[platform.openstack.ingressFloatingIP: Invalid value: "128.35.27.13": Cannot set floating ips when external network not specified, platform.openstack.apiFloatingIP: Invalid value: "128.35.27.8": Cannot set floating ips when external network not specified]`, 175 }, 176 { 177 name: "ingress and API FIPs identical", 178 platform: func() *openstack.Platform { 179 p := validPlatform() 180 p.IngressFloatingIP = p.APIFloatingIP 181 return p 182 }(), 183 cloudInfo: validPlatformCloudInfo(), 184 networking: validNetworking(), 185 expectedError: true, 186 expectedErrMsg: `platform.openstack.ingressFloatingIP: Invalid value: "128.35.27.8": ingressFloatingIP can not be the same as apiFloatingIP`, 187 }, 188 { 189 name: "no external network provided", 190 platform: func() *openstack.Platform { 191 p := validPlatform() 192 p.ExternalNetwork = "" 193 p.APIFloatingIP = "" 194 p.IngressFloatingIP = "" 195 return p 196 }(), 197 cloudInfo: func() *CloudInfo { 198 ci := validPlatformCloudInfo() 199 ci.ExternalNetwork = nil 200 ci.IngressFIP = nil 201 ci.APIFIP = nil 202 return ci 203 }(), 204 networking: validNetworking(), 205 expectedError: false, 206 expectedErrMsg: "", 207 }, 208 { 209 name: "valid external network", 210 platform: validPlatform(), 211 cloudInfo: validPlatformCloudInfo(), 212 networking: validNetworking(), 213 expectedError: false, 214 expectedErrMsg: "", 215 }, 216 { 217 name: "external network not found", 218 platform: validPlatform(), 219 cloudInfo: func() *CloudInfo { 220 ci := validPlatformCloudInfo() 221 ci.ExternalNetwork = nil 222 return ci 223 }(), 224 networking: validNetworking(), 225 expectedError: true, 226 expectedErrMsg: "platform.openstack.externalNetwork: Not found: \"valid-external-network\"", 227 }, 228 { 229 name: "APIVIP inside subnet allocation pool", 230 platform: func() *openstack.Platform { 231 p := validPlatform() 232 p.APIVIPs = []string{"10.0.128.10"} 233 return p 234 }(), 235 cloudInfo: validPlatformCloudInfo(withControlPlanePortSubnets( 236 "10.0.128.0/24", 237 "10.0.128.8", 238 "10.0.128.255", 239 )), 240 networking: validNetworking(), 241 expectedError: true, 242 expectedErrMsg: "platform.openstack.apiVIPs: Invalid value: \"10.0.128.10\": apiVIP can not fall in a MachineNetwork allocation pool", 243 }, 244 { 245 name: "ingressVIP inside subnet allocation pool", 246 platform: func() *openstack.Platform { 247 p := validPlatform() 248 p.IngressVIPs = []string{"10.0.128.42"} 249 return p 250 }(), 251 cloudInfo: validPlatformCloudInfo(withControlPlanePortSubnets( 252 "10.0.128.0/24", 253 "10.0.128.8", 254 "10.0.128.255", 255 )), 256 networking: validNetworking(), 257 expectedError: true, 258 expectedErrMsg: "platform.openstack.ingressVIPs: Invalid value: \"10.0.128.42\": ingressVIP can not fall in a MachineNetwork allocation pool", 259 }, 260 } 261 262 for _, tc := range cases { 263 t.Run(tc.name, func(t *testing.T) { 264 aggregatedErrors := ValidatePlatform(tc.platform, tc.networking, tc.cloudInfo).ToAggregate() 265 if tc.expectedError { 266 assert.Regexp(t, tc.expectedErrMsg, aggregatedErrors) 267 } else { 268 assert.NoError(t, aggregatedErrors) 269 } 270 }) 271 } 272 } 273 274 func TestClusterOSImage(t *testing.T) { 275 cases := []struct { 276 name string 277 platform *openstack.Platform 278 cloudInfo *CloudInfo 279 networking *types.Networking 280 expectedErrMsg string // NOTE: this is a REGEXP 281 }{ 282 { 283 name: "no image provided", 284 platform: validPlatform(), 285 cloudInfo: validPlatformCloudInfo(), 286 networking: validNetworking(), 287 expectedErrMsg: "", 288 }, 289 { 290 name: "HTTP address instead of the image name", 291 platform: func() *openstack.Platform { 292 p := validPlatform() 293 p.ClusterOSImage = "http://example.com/myrhcos.iso" 294 return p 295 }(), 296 cloudInfo: validPlatformCloudInfo(), 297 networking: validNetworking(), 298 expectedErrMsg: "", 299 }, 300 { 301 name: "file location instead of the image name", 302 platform: func() *openstack.Platform { 303 p := validPlatform() 304 p.ClusterOSImage = "file:///home/user/myrhcos.iso" 305 return p 306 }(), 307 cloudInfo: validPlatformCloudInfo(), 308 networking: validNetworking(), 309 expectedErrMsg: "", 310 }, 311 { 312 name: "valid image", 313 platform: func() *openstack.Platform { 314 p := validPlatform() 315 p.ClusterOSImage = "my-rhcos" 316 return p 317 }(), 318 cloudInfo: func() *CloudInfo { 319 ci := validPlatformCloudInfo() 320 ci.OSImage = &images.Image{ 321 Name: "my-rhcos", 322 Status: images.ImageStatusActive, 323 } 324 return ci 325 }(), 326 networking: validNetworking(), 327 expectedErrMsg: "", 328 }, 329 { 330 name: "image with invalid status", 331 platform: func() *openstack.Platform { 332 p := validPlatform() 333 p.ClusterOSImage = "my-rhcos" 334 return p 335 }(), 336 cloudInfo: func() *CloudInfo { 337 ci := validPlatformCloudInfo() 338 ci.OSImage = &images.Image{ 339 Name: "my-rhcos", 340 Status: images.ImageStatusSaving, 341 } 342 return ci 343 }(), 344 networking: validNetworking(), 345 expectedErrMsg: "platform.openstack.clusterOSImage: Invalid value: \"my-rhcos\": OS image must be active but its status is 'saving'", 346 }, 347 { 348 name: "image not found", 349 platform: func() *openstack.Platform { 350 p := validPlatform() 351 p.ClusterOSImage = "my-rhcos" 352 return p 353 }(), 354 cloudInfo: validPlatformCloudInfo(), 355 networking: validNetworking(), 356 expectedErrMsg: "platform.openstack.clusterOSImage: Not found: \"my-rhcos\"", 357 }, 358 { 359 name: "Unsupported image URL scheme", 360 platform: func() *openstack.Platform { 361 p := validPlatform() 362 p.ClusterOSImage = "s3://mybucket/myrhcos.iso" 363 return p 364 }(), 365 cloudInfo: validPlatformCloudInfo(), 366 networking: validNetworking(), 367 expectedErrMsg: "platform.openstack.clusterOSImage: Invalid value: \"s3://mybucket/myrhcos.iso\": URL scheme should be either http\\(s\\) or file but it is 's3'", 368 }, 369 } 370 371 for _, tc := range cases { 372 t.Run(tc.name, func(t *testing.T) { 373 aggregatedErrors := ValidatePlatform(tc.platform, tc.networking, tc.cloudInfo).ToAggregate() 374 if tc.expectedErrMsg != "" { 375 assert.Regexp(t, tc.expectedErrMsg, aggregatedErrors) 376 } else { 377 assert.NoError(t, aggregatedErrors) 378 } 379 }) 380 } 381 } 382 383 func TestMachineSubnet(t *testing.T) { 384 cases := []struct { 385 name string 386 platform *openstack.Platform 387 cloudInfo *CloudInfo 388 networking *types.Networking 389 expectedErrMsg string // NOTE: this is a REGEXP 390 }{ 391 { 392 name: "external dns is not supported", 393 platform: func() *openstack.Platform { 394 p := validPlatform() 395 p.ExternalDNS = append(p.ExternalDNS, "1.2.3.4") 396 p.ControlPlanePort = validControlPlanePort() 397 return p 398 }(), 399 cloudInfo: validPlatformCloudInfo(), 400 networking: validNetworking(), 401 expectedErrMsg: `platform.openstack.externalDNS: Invalid value: \[\]string{"1.2.3.4"}: externalDNS is set, externalDNS is not supported when ControlPlanePort is set`, 402 }, 403 { 404 name: "control plane port subnet not found", 405 platform: func() *openstack.Platform { 406 p := validPlatform() 407 p.ControlPlanePort = validControlPlanePort() 408 return p 409 }(), 410 cloudInfo: func() *CloudInfo { 411 ci := validPlatformCloudInfo() 412 subnet := subnets.Subnet{ 413 ID: "00000000-5a89-4465-8d54-3517ec2bad48", 414 } 415 Allsubnets := []*subnets.Subnet{&subnet} 416 ci.ControlPlanePortSubnets = Allsubnets 417 return ci 418 }(), 419 networking: validNetworking(), 420 expectedErrMsg: `platform.openstack.controlPlanePort.fixedIPs: Not found: "031a5b9d-5a89-4465-8d54-3517ec2bad48"`, 421 }, 422 { 423 name: "network does not contain subnets", 424 platform: func() *openstack.Platform { 425 p := validPlatform() 426 p.ControlPlanePort = validControlPlanePort() 427 p.ControlPlanePort.Network.ID = "00000000-2a22-4465-8d54-3517ec2bad48" 428 return p 429 }(), 430 cloudInfo: func() *CloudInfo { 431 ci := validPlatformCloudInfo() 432 subnet := subnets.Subnet{ 433 ID: "031a5b9d-5a89-4465-8d54-3517ec2bad48", 434 NetworkID: "00000000-1a11-4465-8d54-3517ec2bad48", 435 CIDR: "172.0.0.1/24", 436 } 437 Allsubnets := []*subnets.Subnet{&subnet} 438 ci.ControlPlanePortSubnets = Allsubnets 439 ci.ControlPlanePortNetwork = &networks.Network{ID: "00000000-2a22-4465-8d54-3517ec2bad48"} 440 return ci 441 }(), 442 networking: func() *types.Networking { 443 n := validNetworking() 444 machineNetworkEntry := &types.MachineNetworkEntry{ 445 CIDR: *ipnet.MustParseCIDR("172.0.0.1/24"), 446 } 447 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry} 448 return n 449 }(), 450 expectedErrMsg: `platform.openstack.controlPlanePort.network: Invalid value: "00000000-2a22-4465-8d54-3517ec2bad48": network must contain subnets`, 451 }, 452 { 453 name: "doesn't match the CIDR", 454 platform: func() *openstack.Platform { 455 p := validPlatform() 456 p.ControlPlanePort = validControlPlanePort() 457 return p 458 }(), 459 cloudInfo: func() *CloudInfo { 460 ci := validPlatformCloudInfo() 461 subnet := subnets.Subnet{ 462 ID: validSubnetID, 463 CIDR: "172.0.0.1/16", 464 } 465 Allsubnets := []*subnets.Subnet{&subnet} 466 ci.ControlPlanePortSubnets = Allsubnets 467 return ci 468 }(), 469 networking: func() *types.Networking { 470 n := validNetworking() 471 machineNetworkEntry := &types.MachineNetworkEntry{ 472 CIDR: *ipnet.MustParseCIDR("172.0.0.1/24"), 473 } 474 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry} 475 return n 476 }(), 477 expectedErrMsg: `platform.openstack.controlPlanePort.fixedIPs: Invalid value: "172.0.0.1/16": controlPlanePort CIDR does not match machineNetwork`, 478 }, 479 { 480 name: "control plane port subnets on different network", 481 platform: func() *openstack.Platform { 482 p := validPlatform() 483 fixedIP := openstack.FixedIP{ 484 Subnet: openstack.SubnetFilter{ID: "00000000-5a89-4465-8d54-3517ec2bad48"}, 485 } 486 fixedIPv6 := openstack.FixedIP{ 487 Subnet: openstack.SubnetFilter{ID: "00000000-1111-4465-8d54-3517ec2bad48"}, 488 } 489 p.ControlPlanePort = &openstack.PortTarget{ 490 FixedIPs: []openstack.FixedIP{fixedIP, fixedIPv6}, 491 } 492 return p 493 }(), 494 cloudInfo: func() *CloudInfo { 495 ci := validPlatformCloudInfo() 496 subnet := subnets.Subnet{ 497 ID: "00000000-5a89-4465-8d54-3517ec2bad48", 498 NetworkID: "00000000-2222-4465-8d54-3517ec2bad48", 499 CIDR: "172.0.0.1/16", 500 IPVersion: 4, 501 } 502 subnetv6 := subnets.Subnet{ 503 ID: "00000000-1111-4465-8d54-3517ec2bad48", 504 NetworkID: "00000000-3333-4465-8d54-3517ec2bad48", 505 CIDR: "2001:db8::/64", 506 IPVersion: 6, 507 } 508 Allsubnets := []*subnets.Subnet{&subnet, &subnetv6} 509 ci.ControlPlanePortSubnets = Allsubnets 510 return ci 511 }(), 512 networking: func() *types.Networking { 513 n := validNetworking() 514 machineNetworkEntry := &types.MachineNetworkEntry{ 515 CIDR: *ipnet.MustParseCIDR("172.0.0.1/16"), 516 } 517 machineNetworkEntryv6 := &types.MachineNetworkEntry{ 518 CIDR: *ipnet.MustParseCIDR("2001:db8::/64"), 519 } 520 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry, *machineNetworkEntryv6} 521 return n 522 }(), 523 expectedErrMsg: `platform.openstack.controlPlanePort.fixedIPs: Invalid value: "00000000-3333-4465-8d54-3517ec2bad48": fixedIPs subnets must be on the same Network`, 524 }, 525 { 526 name: "valid control plane port", 527 platform: func() *openstack.Platform { 528 p := validPlatform() 529 p.ControlPlanePort = validControlPlanePort() 530 return p 531 }(), 532 cloudInfo: func() *CloudInfo { 533 ci := validPlatformCloudInfo() 534 subnet := subnets.Subnet{ 535 ID: validSubnetID, 536 CIDR: "172.0.0.1/16", 537 } 538 Allsubnets := []*subnets.Subnet{&subnet} 539 ci.ControlPlanePortSubnets = Allsubnets 540 return ci 541 }(), 542 networking: func() *types.Networking { 543 n := validNetworking() 544 machineNetworkEntry := &types.MachineNetworkEntry{ 545 CIDR: *ipnet.MustParseCIDR("172.0.0.1/16"), 546 } 547 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry} 548 return n 549 }(), 550 expectedErrMsg: "", 551 }, 552 { 553 name: "control plane port multiple ipv4 subnets", 554 platform: func() *openstack.Platform { 555 p := validPlatform() 556 fixedIP := openstack.FixedIP{ 557 Subnet: openstack.SubnetFilter{ID: "00000000-5a89-4465-8d54-3517ec2bad48"}, 558 } 559 fixedIPv6 := openstack.FixedIP{ 560 Subnet: openstack.SubnetFilter{ID: "00000000-1111-4465-8d54-3517ec2bad48"}, 561 } 562 p.ControlPlanePort = &openstack.PortTarget{ 563 FixedIPs: []openstack.FixedIP{fixedIP, fixedIPv6}, 564 } 565 return p 566 }(), 567 cloudInfo: func() *CloudInfo { 568 ci := validPlatformCloudInfo() 569 subnet := subnets.Subnet{ 570 ID: "00000000-5a89-4465-8d54-3517ec2bad48", 571 CIDR: "172.0.0.1/16", 572 IPVersion: 4, 573 } 574 subnetv6 := subnets.Subnet{ 575 ID: "00000000-1111-4465-8d54-3517ec2bad48", 576 CIDR: "10.0.0.0/16", 577 IPVersion: 4, 578 } 579 Allsubnets := []*subnets.Subnet{&subnet, &subnetv6} 580 ci.ControlPlanePortSubnets = Allsubnets 581 return ci 582 }(), 583 networking: func() *types.Networking { 584 n := validNetworking() 585 machineNetworkEntry := &types.MachineNetworkEntry{ 586 CIDR: *ipnet.MustParseCIDR("172.0.0.1/16"), 587 } 588 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry} 589 return n 590 }(), 591 expectedErrMsg: `[platform.openstack.controlPlanePort.fixedIPs: Internal error: controlPlanePort CIDRs does not match machineNetwork, platform.openstack.controlPlanePort.fixedIPs: Internal error: multiple IPv4 subnets is not supported]`, 592 }, 593 { 594 name: "control plane port no ipv4 subnets", 595 platform: func() *openstack.Platform { 596 p := validPlatform() 597 fixedIPv6 := openstack.FixedIP{ 598 Subnet: openstack.SubnetFilter{ID: "00000000-1111-4465-8d54-3517ec2bad48"}, 599 } 600 p.ControlPlanePort = &openstack.PortTarget{ 601 FixedIPs: []openstack.FixedIP{fixedIPv6}, 602 } 603 return p 604 }(), 605 cloudInfo: func() *CloudInfo { 606 ci := validPlatformCloudInfo() 607 subnetv6 := subnets.Subnet{ 608 ID: "00000000-1111-4465-8d54-3517ec2bad48", 609 CIDR: "2001:db8::/64", 610 IPVersion: 6, 611 } 612 Allsubnets := []*subnets.Subnet{&subnetv6} 613 ci.ControlPlanePortSubnets = Allsubnets 614 return ci 615 }(), 616 networking: func() *types.Networking { 617 n := validNetworking() 618 machineNetworkEntry := &types.MachineNetworkEntry{ 619 CIDR: *ipnet.MustParseCIDR("2001:db8::/64"), 620 } 621 n.MachineNetwork = []types.MachineNetworkEntry{*machineNetworkEntry} 622 return n 623 }(), 624 expectedErrMsg: `platform.openstack.controlPlanePort.fixedIPs: Internal error: one IPv4 subnet must be specified`, 625 }, 626 } 627 628 for _, tc := range cases { 629 t.Run(tc.name, func(t *testing.T) { 630 aggregatedErrors := ValidatePlatform(tc.platform, tc.networking, tc.cloudInfo).ToAggregate() 631 if tc.expectedErrMsg != "" { 632 assert.Regexp(t, tc.expectedErrMsg, aggregatedErrors) 633 } else { 634 assert.NoError(t, aggregatedErrors) 635 } 636 }) 637 } 638 }