github.com/mongey/nomad@v0.5.2/command/agent/agent.go (about) 1 package agent 2 3 import ( 4 "fmt" 5 "io" 6 "log" 7 "net" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strconv" 12 "strings" 13 "sync" 14 "sync/atomic" 15 "time" 16 17 "github.com/hashicorp/nomad/client" 18 clientconfig "github.com/hashicorp/nomad/client/config" 19 "github.com/hashicorp/nomad/command/agent/consul" 20 "github.com/hashicorp/nomad/nomad" 21 "github.com/hashicorp/nomad/nomad/structs" 22 ) 23 24 const ( 25 clientHttpCheckInterval = 10 * time.Second 26 clientHttpCheckTimeout = 3 * time.Second 27 serverHttpCheckInterval = 10 * time.Second 28 serverHttpCheckTimeout = 6 * time.Second 29 serverRpcCheckInterval = 10 * time.Second 30 serverRpcCheckTimeout = 3 * time.Second 31 serverSerfCheckInterval = 10 * time.Second 32 serverSerfCheckTimeout = 3 * time.Second 33 ) 34 35 // Agent is a long running daemon that is used to run both 36 // clients and servers. Servers are responsible for managing 37 // state and making scheduling decisions. Clients can be 38 // scheduled to, and are responsible for interfacing with 39 // servers to run allocations. 40 type Agent struct { 41 config *Config 42 logger *log.Logger 43 logOutput io.Writer 44 45 // consulSyncer registers the Nomad agent with the Consul Agent 46 consulSyncer *consul.Syncer 47 48 client *client.Client 49 50 server *nomad.Server 51 52 shutdown bool 53 shutdownCh chan struct{} 54 shutdownLock sync.Mutex 55 } 56 57 // NewAgent is used to create a new agent with the given configuration 58 func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) { 59 a := &Agent{ 60 config: config, 61 logger: log.New(logOutput, "", log.LstdFlags|log.Lmicroseconds), 62 logOutput: logOutput, 63 shutdownCh: make(chan struct{}), 64 } 65 66 if err := a.setupConsulSyncer(); err != nil { 67 return nil, fmt.Errorf("Failed to initialize Consul syncer task: %v", err) 68 } 69 if err := a.setupServer(); err != nil { 70 return nil, err 71 } 72 if err := a.setupClient(); err != nil { 73 return nil, err 74 } 75 if a.client == nil && a.server == nil { 76 return nil, fmt.Errorf("must have at least client or server mode enabled") 77 } 78 79 // The Nomad Agent runs the consul.Syncer regardless of whether or not the 80 // Agent is running in Client or Server mode (or both), and regardless of 81 // the consul.auto_advertise parameter. The Client and Server both reuse the 82 // same consul.Syncer instance. This Syncer task periodically executes 83 // callbacks that update Consul. The reason the Syncer is always running is 84 // because one of the callbacks is attempts to self-bootstrap Nomad using 85 // information found in Consul. 86 go a.consulSyncer.Run() 87 88 return a, nil 89 } 90 91 // serverConfig is used to generate a new server configuration struct 92 // for initializing a nomad server. 93 func (a *Agent) serverConfig() (*nomad.Config, error) { 94 conf := a.config.NomadConfig 95 if conf == nil { 96 conf = nomad.DefaultConfig() 97 } 98 conf.LogOutput = a.logOutput 99 conf.DevMode = a.config.DevMode 100 conf.Build = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease) 101 if a.config.Region != "" { 102 conf.Region = a.config.Region 103 } 104 if a.config.Datacenter != "" { 105 conf.Datacenter = a.config.Datacenter 106 } 107 if a.config.NodeName != "" { 108 conf.NodeName = a.config.NodeName 109 } 110 if a.config.Server.BootstrapExpect > 0 { 111 if a.config.Server.BootstrapExpect == 1 { 112 conf.Bootstrap = true 113 } else { 114 atomic.StoreInt32(&conf.BootstrapExpect, int32(a.config.Server.BootstrapExpect)) 115 } 116 } 117 if a.config.DataDir != "" { 118 conf.DataDir = filepath.Join(a.config.DataDir, "server") 119 } 120 if a.config.Server.DataDir != "" { 121 conf.DataDir = a.config.Server.DataDir 122 } 123 if a.config.Server.ProtocolVersion != 0 { 124 conf.ProtocolVersion = uint8(a.config.Server.ProtocolVersion) 125 } 126 if a.config.Server.NumSchedulers != 0 { 127 conf.NumSchedulers = a.config.Server.NumSchedulers 128 } 129 if len(a.config.Server.EnabledSchedulers) != 0 { 130 conf.EnabledSchedulers = a.config.Server.EnabledSchedulers 131 } 132 133 // Set up the bind addresses 134 rpcAddr, err := net.ResolveTCPAddr("tcp", a.config.normalizedAddrs.RPC) 135 if err != nil { 136 return nil, fmt.Errorf("Failed to parse RPC address %q: %v", a.config.normalizedAddrs.RPC, err) 137 } 138 serfAddr, err := net.ResolveTCPAddr("tcp", a.config.normalizedAddrs.Serf) 139 if err != nil { 140 return nil, fmt.Errorf("Failed to parse Serf address %q: %v", a.config.normalizedAddrs.Serf, err) 141 } 142 conf.RPCAddr.Port = rpcAddr.Port 143 conf.RPCAddr.IP = rpcAddr.IP 144 conf.SerfConfig.MemberlistConfig.BindPort = serfAddr.Port 145 conf.SerfConfig.MemberlistConfig.BindAddr = serfAddr.IP.String() 146 147 // Set up the advertise addresses 148 rpcAddr, err = net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.RPC) 149 if err != nil { 150 return nil, fmt.Errorf("Failed to parse RPC advertise address %q: %v", a.config.AdvertiseAddrs.RPC, err) 151 } 152 serfAddr, err = net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.Serf) 153 if err != nil { 154 return nil, fmt.Errorf("Failed to parse Serf advertise address %q: %v", a.config.AdvertiseAddrs.Serf, err) 155 } 156 conf.RPCAdvertise = rpcAddr 157 conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String() 158 conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port 159 160 // Set up gc threshold and heartbeat grace period 161 if gcThreshold := a.config.Server.NodeGCThreshold; gcThreshold != "" { 162 dur, err := time.ParseDuration(gcThreshold) 163 if err != nil { 164 return nil, err 165 } 166 conf.NodeGCThreshold = dur 167 } 168 169 if heartbeatGrace := a.config.Server.HeartbeatGrace; heartbeatGrace != "" { 170 dur, err := time.ParseDuration(heartbeatGrace) 171 if err != nil { 172 return nil, err 173 } 174 conf.HeartbeatGrace = dur 175 } 176 177 if a.config.Consul.AutoAdvertise && a.config.Consul.ServerServiceName == "" { 178 return nil, fmt.Errorf("server_service_name must be set when auto_advertise is enabled") 179 } 180 181 // Add the Consul and Vault configs 182 conf.ConsulConfig = a.config.Consul 183 conf.VaultConfig = a.config.Vault 184 185 // Set the TLS config 186 conf.TLSConfig = a.config.TLSConfig 187 188 return conf, nil 189 } 190 191 // clientConfig is used to generate a new client configuration struct 192 // for initializing a Nomad client. 193 func (a *Agent) clientConfig() (*clientconfig.Config, error) { 194 // Setup the configuration 195 conf := a.config.ClientConfig 196 if conf == nil { 197 conf = clientconfig.DefaultConfig() 198 } 199 if a.server != nil { 200 conf.RPCHandler = a.server 201 } 202 conf.LogOutput = a.logOutput 203 conf.DevMode = a.config.DevMode 204 if a.config.Region != "" { 205 conf.Region = a.config.Region 206 } 207 if a.config.DataDir != "" { 208 conf.StateDir = filepath.Join(a.config.DataDir, "client") 209 conf.AllocDir = filepath.Join(a.config.DataDir, "alloc") 210 } 211 if a.config.Client.StateDir != "" { 212 conf.StateDir = a.config.Client.StateDir 213 } 214 if a.config.Client.AllocDir != "" { 215 conf.AllocDir = a.config.Client.AllocDir 216 } 217 conf.Servers = a.config.Client.Servers 218 if a.config.Client.NetworkInterface != "" { 219 conf.NetworkInterface = a.config.Client.NetworkInterface 220 } 221 conf.ChrootEnv = a.config.Client.ChrootEnv 222 conf.Options = a.config.Client.Options 223 // Logging deprecation messages about consul related configuration in client 224 // options 225 var invalidConsulKeys []string 226 for key := range conf.Options { 227 if strings.HasPrefix(key, "consul") { 228 invalidConsulKeys = append(invalidConsulKeys, fmt.Sprintf("options.%s", key)) 229 } 230 } 231 if len(invalidConsulKeys) > 0 { 232 a.logger.Printf("[WARN] agent: Invalid keys: %v", strings.Join(invalidConsulKeys, ",")) 233 a.logger.Printf(`Nomad client ignores consul related configuration in client options. 234 Please refer to the guide https://www.nomadproject.io/docs/agent/configuration/consul.html 235 to configure Nomad to work with Consul.`) 236 } 237 238 if a.config.Client.NetworkSpeed != 0 { 239 conf.NetworkSpeed = a.config.Client.NetworkSpeed 240 } 241 if a.config.Client.MaxKillTimeout != "" { 242 dur, err := time.ParseDuration(a.config.Client.MaxKillTimeout) 243 if err != nil { 244 return nil, fmt.Errorf("Error parsing max kill timeout: %s", err) 245 } 246 conf.MaxKillTimeout = dur 247 } 248 conf.ClientMaxPort = uint(a.config.Client.ClientMaxPort) 249 conf.ClientMinPort = uint(a.config.Client.ClientMinPort) 250 251 // Setup the node 252 conf.Node = new(structs.Node) 253 conf.Node.Datacenter = a.config.Datacenter 254 conf.Node.Name = a.config.NodeName 255 conf.Node.Meta = a.config.Client.Meta 256 conf.Node.NodeClass = a.config.Client.NodeClass 257 258 // Set up the HTTP advertise address 259 conf.Node.HTTPAddr = a.config.AdvertiseAddrs.HTTP 260 261 // Reserve resources on the node. 262 r := conf.Node.Reserved 263 if r == nil { 264 r = new(structs.Resources) 265 conf.Node.Reserved = r 266 } 267 r.CPU = a.config.Client.Reserved.CPU 268 r.MemoryMB = a.config.Client.Reserved.MemoryMB 269 r.DiskMB = a.config.Client.Reserved.DiskMB 270 r.IOPS = a.config.Client.Reserved.IOPS 271 conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts 272 273 conf.Version = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease) 274 conf.Revision = a.config.Revision 275 276 if a.config.Consul.AutoAdvertise && a.config.Consul.ClientServiceName == "" { 277 return nil, fmt.Errorf("client_service_name must be set when auto_advertise is enabled") 278 } 279 280 conf.ConsulConfig = a.config.Consul 281 conf.VaultConfig = a.config.Vault 282 conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval 283 conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics 284 conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics 285 286 // Set the TLS related configs 287 conf.TLSConfig = a.config.TLSConfig 288 conf.Node.TLSEnabled = conf.TLSConfig.EnableHTTP 289 290 return conf, nil 291 } 292 293 // setupServer is used to setup the server if enabled 294 func (a *Agent) setupServer() error { 295 if !a.config.Server.Enabled { 296 return nil 297 } 298 299 // Setup the configuration 300 conf, err := a.serverConfig() 301 if err != nil { 302 return fmt.Errorf("server config setup failed: %s", err) 303 } 304 305 // Sets up the keyring for gossip encryption 306 if err := a.setupKeyrings(conf); err != nil { 307 return fmt.Errorf("failed to configure keyring: %v", err) 308 } 309 310 // Create the server 311 server, err := nomad.NewServer(conf, a.consulSyncer, a.logger) 312 if err != nil { 313 return fmt.Errorf("server setup failed: %v", err) 314 } 315 a.server = server 316 317 // Consul check addresses default to bind but can be toggled to use advertise 318 httpCheckAddr := a.config.normalizedAddrs.HTTP 319 rpcCheckAddr := a.config.normalizedAddrs.RPC 320 serfCheckAddr := a.config.normalizedAddrs.Serf 321 if a.config.Consul.ChecksUseAdvertise { 322 httpCheckAddr = a.config.AdvertiseAddrs.HTTP 323 rpcCheckAddr = a.config.AdvertiseAddrs.RPC 324 serfCheckAddr = a.config.AdvertiseAddrs.Serf 325 } 326 327 // Create the Nomad Server services for Consul 328 // TODO re-introduce HTTP/S checks when Consul 0.7.1 comes out 329 if a.config.Consul.AutoAdvertise { 330 httpServ := &structs.Service{ 331 Name: a.config.Consul.ServerServiceName, 332 PortLabel: a.config.AdvertiseAddrs.HTTP, 333 Tags: []string{consul.ServiceTagHTTP}, 334 Checks: []*structs.ServiceCheck{ 335 &structs.ServiceCheck{ 336 Name: "Nomad Server HTTP Check", 337 Type: "http", 338 Path: "/v1/status/peers", 339 Protocol: "http", 340 Interval: serverHttpCheckInterval, 341 Timeout: serverHttpCheckTimeout, 342 PortLabel: httpCheckAddr, 343 }, 344 }, 345 } 346 rpcServ := &structs.Service{ 347 Name: a.config.Consul.ServerServiceName, 348 PortLabel: a.config.AdvertiseAddrs.RPC, 349 Tags: []string{consul.ServiceTagRPC}, 350 Checks: []*structs.ServiceCheck{ 351 &structs.ServiceCheck{ 352 Name: "Nomad Server RPC Check", 353 Type: "tcp", 354 Interval: serverRpcCheckInterval, 355 Timeout: serverRpcCheckTimeout, 356 PortLabel: rpcCheckAddr, 357 }, 358 }, 359 } 360 serfServ := &structs.Service{ 361 Name: a.config.Consul.ServerServiceName, 362 PortLabel: a.config.AdvertiseAddrs.Serf, 363 Tags: []string{consul.ServiceTagSerf}, 364 Checks: []*structs.ServiceCheck{ 365 &structs.ServiceCheck{ 366 Name: "Nomad Server Serf Check", 367 Type: "tcp", 368 Interval: serverSerfCheckInterval, 369 Timeout: serverSerfCheckTimeout, 370 PortLabel: serfCheckAddr, 371 }, 372 }, 373 } 374 375 // Add the http port check if TLS isn't enabled 376 // TODO Add TLS check when Consul 0.7.1 comes out. 377 consulServices := map[consul.ServiceKey]*structs.Service{ 378 consul.GenerateServiceKey(rpcServ): rpcServ, 379 consul.GenerateServiceKey(serfServ): serfServ, 380 } 381 if !conf.TLSConfig.EnableHTTP { 382 consulServices[consul.GenerateServiceKey(httpServ)] = httpServ 383 } 384 a.consulSyncer.SetServices(consul.ServerDomain, consulServices) 385 } 386 387 return nil 388 } 389 390 // setupKeyrings is used to initialize and load keyrings during agent startup 391 func (a *Agent) setupKeyrings(config *nomad.Config) error { 392 file := filepath.Join(a.config.DataDir, serfKeyring) 393 394 if a.config.Server.EncryptKey == "" { 395 goto LOAD 396 } 397 if _, err := os.Stat(file); err != nil { 398 if err := initKeyring(file, a.config.Server.EncryptKey); err != nil { 399 return err 400 } 401 } 402 403 LOAD: 404 if _, err := os.Stat(file); err == nil { 405 config.SerfConfig.KeyringFile = file 406 } 407 if err := loadKeyringFile(config.SerfConfig); err != nil { 408 return err 409 } 410 // Success! 411 return nil 412 } 413 414 // setupClient is used to setup the client if enabled 415 func (a *Agent) setupClient() error { 416 if !a.config.Client.Enabled { 417 return nil 418 } 419 420 // Setup the configuration 421 conf, err := a.clientConfig() 422 if err != nil { 423 return fmt.Errorf("client setup failed: %v", err) 424 } 425 426 // Reserve some ports for the plugins if we are on Windows 427 if runtime.GOOS == "windows" { 428 if err := a.reservePortsForClient(conf); err != nil { 429 return err 430 } 431 } 432 433 // Create the client 434 client, err := client.NewClient(conf, a.consulSyncer, a.logger) 435 if err != nil { 436 return fmt.Errorf("client setup failed: %v", err) 437 } 438 a.client = client 439 440 // Resolve the http check address 441 httpCheckAddr := a.config.normalizedAddrs.HTTP 442 if a.config.Consul.ChecksUseAdvertise { 443 httpCheckAddr = a.config.AdvertiseAddrs.HTTP 444 } 445 446 // Create the Nomad Client services for Consul 447 // TODO think how we can re-introduce HTTP/S checks when Consul 0.7.1 comes 448 // out 449 if a.config.Consul.AutoAdvertise { 450 httpServ := &structs.Service{ 451 Name: a.config.Consul.ClientServiceName, 452 PortLabel: a.config.AdvertiseAddrs.HTTP, 453 Tags: []string{consul.ServiceTagHTTP}, 454 Checks: []*structs.ServiceCheck{ 455 &structs.ServiceCheck{ 456 Name: "Nomad Client HTTP Check", 457 Type: "http", 458 Path: "/v1/agent/servers", 459 Protocol: "http", 460 Interval: clientHttpCheckInterval, 461 Timeout: clientHttpCheckTimeout, 462 PortLabel: httpCheckAddr, 463 }, 464 }, 465 } 466 if !conf.TLSConfig.EnableHTTP { 467 a.consulSyncer.SetServices(consul.ClientDomain, map[consul.ServiceKey]*structs.Service{ 468 consul.GenerateServiceKey(httpServ): httpServ, 469 }) 470 } 471 } 472 473 return nil 474 } 475 476 // reservePortsForClient reserves a range of ports for the client to use when 477 // it creates various plugins for log collection, executors, drivers, etc 478 func (a *Agent) reservePortsForClient(conf *clientconfig.Config) error { 479 // finding the device name for loopback 480 deviceName, addr, mask, err := a.findLoopbackDevice() 481 if err != nil { 482 return fmt.Errorf("error finding the device name for loopback: %v", err) 483 } 484 485 // seeing if the user has already reserved some resources on this device 486 var nr *structs.NetworkResource 487 if conf.Node.Reserved == nil { 488 conf.Node.Reserved = &structs.Resources{} 489 } 490 for _, n := range conf.Node.Reserved.Networks { 491 if n.Device == deviceName { 492 nr = n 493 } 494 } 495 // If the user hasn't already created the device, we create it 496 if nr == nil { 497 nr = &structs.NetworkResource{ 498 Device: deviceName, 499 IP: addr, 500 CIDR: mask, 501 ReservedPorts: make([]structs.Port, 0), 502 } 503 } 504 // appending the port ranges we want to use for the client to the list of 505 // reserved ports for this device 506 for i := conf.ClientMinPort; i <= conf.ClientMaxPort; i++ { 507 nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: int(i)}) 508 } 509 conf.Node.Reserved.Networks = append(conf.Node.Reserved.Networks, nr) 510 return nil 511 } 512 513 // findLoopbackDevice iterates through all the interfaces on a machine and 514 // returns the ip addr, mask of the loopback device 515 func (a *Agent) findLoopbackDevice() (string, string, string, error) { 516 var ifcs []net.Interface 517 var err error 518 ifcs, err = net.Interfaces() 519 if err != nil { 520 return "", "", "", err 521 } 522 for _, ifc := range ifcs { 523 addrs, err := ifc.Addrs() 524 if err != nil { 525 return "", "", "", err 526 } 527 for _, addr := range addrs { 528 var ip net.IP 529 switch v := addr.(type) { 530 case *net.IPNet: 531 ip = v.IP 532 case *net.IPAddr: 533 ip = v.IP 534 } 535 if ip.IsLoopback() { 536 if ip.To4() == nil { 537 continue 538 } 539 return ifc.Name, ip.String(), addr.String(), nil 540 } 541 } 542 } 543 544 return "", "", "", fmt.Errorf("no loopback devices with IPV4 addr found") 545 } 546 547 // Leave is used gracefully exit. Clients will inform servers 548 // of their departure so that allocations can be rescheduled. 549 func (a *Agent) Leave() error { 550 if a.client != nil { 551 if err := a.client.Leave(); err != nil { 552 a.logger.Printf("[ERR] agent: client leave failed: %v", err) 553 } 554 } 555 if a.server != nil { 556 if err := a.server.Leave(); err != nil { 557 a.logger.Printf("[ERR] agent: server leave failed: %v", err) 558 } 559 } 560 return nil 561 } 562 563 // Shutdown is used to terminate the agent. 564 func (a *Agent) Shutdown() error { 565 a.shutdownLock.Lock() 566 defer a.shutdownLock.Unlock() 567 568 if a.shutdown { 569 return nil 570 } 571 572 a.logger.Println("[INFO] agent: requesting shutdown") 573 if a.client != nil { 574 if err := a.client.Shutdown(); err != nil { 575 a.logger.Printf("[ERR] agent: client shutdown failed: %v", err) 576 } 577 } 578 if a.server != nil { 579 if err := a.server.Shutdown(); err != nil { 580 a.logger.Printf("[ERR] agent: server shutdown failed: %v", err) 581 } 582 } 583 584 if err := a.consulSyncer.Shutdown(); err != nil { 585 a.logger.Printf("[ERR] agent: shutting down consul service failed: %v", err) 586 } 587 588 a.logger.Println("[INFO] agent: shutdown complete") 589 a.shutdown = true 590 close(a.shutdownCh) 591 return nil 592 } 593 594 // RPC is used to make an RPC call to the Nomad servers 595 func (a *Agent) RPC(method string, args interface{}, reply interface{}) error { 596 if a.server != nil { 597 return a.server.RPC(method, args, reply) 598 } 599 return a.client.RPC(method, args, reply) 600 } 601 602 // Client returns the configured client or nil 603 func (a *Agent) Client() *client.Client { 604 return a.client 605 } 606 607 // Server returns the configured server or nil 608 func (a *Agent) Server() *nomad.Server { 609 return a.server 610 } 611 612 // Stats is used to return statistics for debugging and insight 613 // for various sub-systems 614 func (a *Agent) Stats() map[string]map[string]string { 615 stats := make(map[string]map[string]string) 616 if a.server != nil { 617 subStat := a.server.Stats() 618 for k, v := range subStat { 619 stats[k] = v 620 } 621 } 622 if a.client != nil { 623 subStat := a.client.Stats() 624 for k, v := range subStat { 625 stats[k] = v 626 } 627 } 628 return stats 629 } 630 631 // setupConsulSyncer creates the Consul tasks used by this Nomad Agent 632 // (either Client or Server mode). 633 func (a *Agent) setupConsulSyncer() error { 634 var err error 635 a.consulSyncer, err = consul.NewSyncer(a.config.Consul, a.shutdownCh, a.logger) 636 if err != nil { 637 return err 638 } 639 640 a.consulSyncer.SetAddrFinder(func(portLabel string) (string, int) { 641 host, port, err := net.SplitHostPort(portLabel) 642 if err != nil { 643 p, err := strconv.Atoi(port) 644 if err != nil { 645 return "", 0 646 } 647 return "", p 648 } 649 650 // If the addr for the service is ":port", then we fall back 651 // to Nomad's default address resolution protocol. 652 // 653 // TODO(sean@): This should poll Consul to figure out what 654 // its advertise address is and use that in order to handle 655 // the case where there is something funky like NAT on this 656 // host. For now we just use the BindAddr if set, otherwise 657 // we fall back to a loopback addr. 658 if host == "" { 659 if a.config.BindAddr != "" { 660 host = a.config.BindAddr 661 } else { 662 host = "127.0.0.1" 663 } 664 } 665 p, err := strconv.Atoi(port) 666 if err != nil { 667 return host, 0 668 } 669 return host, p 670 }) 671 672 return nil 673 }