github.com/StackPointCloud/packer@v0.10.2-0.20180716202532-b28098e0f79b/builder/vmware/iso/driver_esx5.go (about) 1 package iso 2 3 import ( 4 "bufio" 5 "bytes" 6 "encoding/csv" 7 "errors" 8 "fmt" 9 "io" 10 "log" 11 "net" 12 "os" 13 "path/filepath" 14 "strings" 15 "time" 16 17 vmwcommon "github.com/hashicorp/packer/builder/vmware/common" 18 commonssh "github.com/hashicorp/packer/common/ssh" 19 "github.com/hashicorp/packer/communicator/ssh" 20 "github.com/hashicorp/packer/helper/multistep" 21 "github.com/hashicorp/packer/packer" 22 gossh "golang.org/x/crypto/ssh" 23 ) 24 25 // ESX5 driver talks to an ESXi5 hypervisor remotely over SSH to build 26 // virtual machines. This driver can only manage one machine at a time. 27 type ESX5Driver struct { 28 base vmwcommon.VmwareDriver 29 30 Host string 31 Port uint 32 Username string 33 Password string 34 PrivateKey string 35 Datastore string 36 CacheDatastore string 37 CacheDirectory string 38 39 comm packer.Communicator 40 outputDir string 41 vmId string 42 } 43 44 func (d *ESX5Driver) Clone(dst, src string, linked bool) error { 45 return errors.New("Cloning is not supported with the ESX driver.") 46 } 47 48 func (d *ESX5Driver) CompactDisk(diskPathLocal string) error { 49 return nil 50 } 51 52 func (d *ESX5Driver) CreateDisk(diskPathLocal string, size string, adapter_type string, typeId string) error { 53 diskPath := d.datastorePath(diskPathLocal) 54 return d.sh("vmkfstools", "-c", size, "-d", typeId, "-a", adapter_type, diskPath) 55 } 56 57 func (d *ESX5Driver) IsRunning(string) (bool, error) { 58 state, err := d.run(nil, "vim-cmd", "vmsvc/power.getstate", d.vmId) 59 if err != nil { 60 return false, err 61 } 62 return strings.Contains(state, "Powered on"), nil 63 } 64 65 func (d *ESX5Driver) ReloadVM() error { 66 return d.sh("vim-cmd", "vmsvc/reload", d.vmId) 67 } 68 69 func (d *ESX5Driver) Start(vmxPathLocal string, headless bool) error { 70 for i := 0; i < 20; i++ { 71 //intentionally not checking for error since poweron may fail specially after initial VM registration 72 d.sh("vim-cmd", "vmsvc/power.on", d.vmId) 73 time.Sleep((time.Duration(i) * time.Second) + 1) 74 running, err := d.IsRunning(vmxPathLocal) 75 if err != nil { 76 return err 77 } 78 if running { 79 return nil 80 } 81 } 82 return errors.New("Retry limit exceeded") 83 } 84 85 func (d *ESX5Driver) Stop(vmxPathLocal string) error { 86 return d.sh("vim-cmd", "vmsvc/power.off", d.vmId) 87 } 88 89 func (d *ESX5Driver) Register(vmxPathLocal string) error { 90 vmxPath := filepath.ToSlash(filepath.Join(d.outputDir, filepath.Base(vmxPathLocal))) 91 if err := d.upload(vmxPath, vmxPathLocal); err != nil { 92 return err 93 } 94 r, err := d.run(nil, "vim-cmd", "solo/registervm", vmxPath) 95 if err != nil { 96 return err 97 } 98 d.vmId = strings.TrimRight(r, "\n") 99 return nil 100 } 101 102 func (d *ESX5Driver) SuppressMessages(vmxPath string) error { 103 return nil 104 } 105 106 func (d *ESX5Driver) Unregister(vmxPathLocal string) error { 107 return d.sh("vim-cmd", "vmsvc/unregister", d.vmId) 108 } 109 110 func (d *ESX5Driver) Destroy() error { 111 return d.sh("vim-cmd", "vmsvc/destroy", d.vmId) 112 } 113 114 func (d *ESX5Driver) IsDestroyed() (bool, error) { 115 err := d.sh("test", "!", "-e", d.outputDir) 116 if err != nil { 117 return false, err 118 } 119 return true, err 120 } 121 122 func (d *ESX5Driver) UploadISO(localPath string, checksum string, checksumType string) (string, error) { 123 finalPath := d.cachePath(localPath) 124 if err := d.mkdir(filepath.ToSlash(filepath.Dir(finalPath))); err != nil { 125 return "", err 126 } 127 128 log.Printf("Verifying checksum of %s", finalPath) 129 if d.verifyChecksum(checksumType, checksum, finalPath) { 130 log.Println("Initial checksum matched, no upload needed.") 131 return finalPath, nil 132 } 133 134 if err := d.upload(finalPath, localPath); err != nil { 135 return "", err 136 } 137 138 return finalPath, nil 139 } 140 141 func (d *ESX5Driver) RemoveCache(localPath string) error { 142 finalPath := d.cachePath(localPath) 143 log.Printf("Removing remote cache path %s (local %s)", finalPath, localPath) 144 return d.sh("rm", "-f", finalPath) 145 } 146 147 func (d *ESX5Driver) ToolsIsoPath(string) string { 148 return "" 149 } 150 151 func (d *ESX5Driver) ToolsInstall() error { 152 return d.sh("vim-cmd", "vmsvc/tools.install", d.vmId) 153 } 154 155 func (d *ESX5Driver) Verify() error { 156 // Ensure that NetworkMapper is nil, since the mapping of device<->network 157 // is handled by ESX and thus can't be performed by packer unless we 158 // query things. 159 160 // FIXME: If we want to expose the network devices to the user, then we can 161 // probably use esxcli to enumerate the portgroup and switchId 162 d.base.NetworkMapper = nil 163 164 // Be safe/friendly and overwrite the rest of the utility functions with 165 // log functions despite the fact that these shouldn't be called anyways. 166 d.base.DhcpLeasesPath = func(device string) string { 167 log.Printf("Unexpected error, ESX5 driver attempted to call DhcpLeasesPath(%#v)\n", device) 168 return "" 169 } 170 d.base.DhcpConfPath = func(device string) string { 171 log.Printf("Unexpected error, ESX5 driver attempted to call DhcpConfPath(%#v)\n", device) 172 return "" 173 } 174 d.base.VmnetnatConfPath = func(device string) string { 175 log.Printf("Unexpected error, ESX5 driver attempted to call VmnetnatConfPath(%#v)\n", device) 176 return "" 177 } 178 179 checks := []func() error{ 180 d.connect, 181 d.checkSystemVersion, 182 d.checkGuestIPHackEnabled, 183 } 184 185 for _, check := range checks { 186 if err := check(); err != nil { 187 return err 188 } 189 } 190 return nil 191 } 192 193 func (d *ESX5Driver) HostIP(multistep.StateBag) (string, error) { 194 conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port)) 195 if err != nil { 196 return "", err 197 } 198 defer conn.Close() 199 200 host, _, err := net.SplitHostPort(conn.LocalAddr().String()) 201 return host, err 202 } 203 204 func (d *ESX5Driver) GuestIP(multistep.StateBag) (string, error) { 205 // GuestIP is defined by the user as d.Host..but let's validate it just to be sure 206 conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port)) 207 defer conn.Close() 208 if err != nil { 209 return "", err 210 } 211 212 host, _, err := net.SplitHostPort(conn.RemoteAddr().String()) 213 return host, err 214 } 215 216 func (d *ESX5Driver) HostAddress(multistep.StateBag) (string, error) { 217 // make a connection 218 conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", d.Host, d.Port)) 219 defer conn.Close() 220 if err != nil { 221 return "", err 222 } 223 224 // get the local address (the host) 225 host, _, err := net.SplitHostPort(conn.LocalAddr().String()) 226 if err != nil { 227 return "", fmt.Errorf("Unable to determine host address for ESXi: %v", err) 228 } 229 230 // iterate through all the interfaces.. 231 interfaces, err := net.Interfaces() 232 if err != nil { 233 return "", fmt.Errorf("Unable to enumerate host interfaces : %v", err) 234 } 235 236 for _, intf := range interfaces { 237 addrs, err := intf.Addrs() 238 if err != nil { 239 continue 240 } 241 242 // ..checking to see if any if it's addrs match the host address 243 for _, addr := range addrs { 244 if addr.String() == host { // FIXME: Is this the proper way to compare two HardwareAddrs? 245 return intf.HardwareAddr.String(), nil 246 } 247 } 248 } 249 250 // ..unfortunately nothing was found 251 return "", fmt.Errorf("Unable to locate interface matching host address in ESXi: %v", host) 252 } 253 254 func (d *ESX5Driver) GuestAddress(multistep.StateBag) (string, error) { 255 // list all the interfaces on the esx host 256 r, err := d.esxcli("network", "ip", "interface", "list") 257 if err != nil { 258 return "", fmt.Errorf("Could not retrieve network interfaces for ESXi: %v", err) 259 } 260 261 // rip out the interface name and the MAC address from the csv output 262 addrs := make(map[string]string) 263 for record, err := r.read(); record != nil && err == nil; record, err = r.read() { 264 if strings.ToUpper(record["Enabled"]) != "TRUE" { 265 continue 266 } 267 addrs[record["Name"]] = record["MAC Address"] 268 } 269 270 // list all the addresses on the esx host 271 r, err = d.esxcli("network", "ip", "interface", "ipv4", "get") 272 if err != nil { 273 return "", fmt.Errorf("Could not retrieve network addresses for ESXi: %v", err) 274 } 275 276 // figure out the interface name that matches the specified d.Host address 277 var intf string 278 intf = "" 279 for record, err := r.read(); record != nil && err == nil; record, err = r.read() { 280 if record["IPv4 Address"] == d.Host && record["Name"] != "" { 281 intf = record["Name"] 282 break 283 } 284 } 285 if intf == "" { 286 return "", fmt.Errorf("Unable to find matching address for ESXi guest") 287 } 288 289 // find the MAC address according to the interface name 290 result, ok := addrs[intf] 291 if !ok { 292 return "", fmt.Errorf("Unable to find address for ESXi guest interface") 293 } 294 295 // ..and we're good 296 return result, nil 297 } 298 299 func (d *ESX5Driver) VNCAddress(_ string, portMin, portMax uint) (string, uint, error) { 300 var vncPort uint 301 302 //Process ports ESXi is listening on to determine which are available 303 //This process does best effort to detect ports that are unavailable, 304 //it will ignore any ports listened to by only localhost 305 r, err := d.esxcli("network", "ip", "connection", "list") 306 if err != nil { 307 err = fmt.Errorf("Could not retrieve network information for ESXi: %v", err) 308 return "", 0, err 309 } 310 311 listenPorts := make(map[string]bool) 312 for record, err := r.read(); record != nil && err == nil; record, err = r.read() { 313 if record["State"] == "LISTEN" { 314 splitAddress := strings.Split(record["LocalAddress"], ":") 315 if splitAddress[0] != "127.0.0.1" { 316 port := splitAddress[len(splitAddress)-1] 317 log.Printf("ESXi listening on address %s, port %s unavailable for VNC", record["LocalAddress"], port) 318 listenPorts[port] = true 319 } 320 } 321 } 322 323 vncTimeout := time.Duration(15 * time.Second) 324 envTimeout := os.Getenv("PACKER_ESXI_VNC_PROBE_TIMEOUT") 325 if envTimeout != "" { 326 if parsedTimeout, err := time.ParseDuration(envTimeout); err != nil { 327 log.Printf("Error parsing PACKER_ESXI_VNC_PROBE_TIMEOUT. Falling back to default (15s). %s", err) 328 } else { 329 vncTimeout = parsedTimeout 330 } 331 } 332 333 for port := portMin; port <= portMax; port++ { 334 if _, ok := listenPorts[fmt.Sprintf("%d", port)]; ok { 335 log.Printf("Port %d in use", port) 336 continue 337 } 338 address := fmt.Sprintf("%s:%d", d.Host, port) 339 log.Printf("Trying address: %s...", address) 340 l, err := net.DialTimeout("tcp", address, vncTimeout) 341 342 if err != nil { 343 if e, ok := err.(*net.OpError); ok { 344 if e.Timeout() { 345 log.Printf("Timeout connecting to: %s (check firewall rules)", address) 346 } else { 347 vncPort = port 348 break 349 } 350 } 351 } else { 352 defer l.Close() 353 } 354 } 355 356 if vncPort == 0 { 357 err := fmt.Errorf("Unable to find available VNC port between %d and %d", 358 portMin, portMax) 359 return d.Host, vncPort, err 360 } 361 362 return d.Host, vncPort, nil 363 } 364 365 // UpdateVMX, adds the VNC port to the VMX data. 366 func (ESX5Driver) UpdateVMX(_, password string, port uint, data map[string]string) { 367 // Do not set remotedisplay.vnc.ip - this breaks ESXi. 368 data["remotedisplay.vnc.enabled"] = "TRUE" 369 data["remotedisplay.vnc.port"] = fmt.Sprintf("%d", port) 370 if len(password) > 0 { 371 data["remotedisplay.vnc.password"] = password 372 } 373 } 374 375 func (d *ESX5Driver) CommHost(state multistep.StateBag) (string, error) { 376 config := state.Get("config").(*Config) 377 sshc := config.SSHConfig.Comm 378 port := sshc.SSHPort 379 if sshc.Type == "winrm" { 380 port = sshc.WinRMPort 381 } 382 383 if address := config.CommConfig.Host(); address != "" { 384 return address, nil 385 } 386 387 r, err := d.esxcli("network", "vm", "list") 388 if err != nil { 389 return "", err 390 } 391 392 // The value in the Name field returned by 'esxcli network vm list' 393 // corresponds directly to the value of displayName set in the VMX file 394 var displayName string 395 if v, ok := state.GetOk("display_name"); ok { 396 displayName = v.(string) 397 } 398 record, err := r.find("Name", displayName) 399 if err != nil { 400 return "", err 401 } 402 wid := record["WorldID"] 403 if wid == "" { 404 return "", errors.New("VM WorldID not found") 405 } 406 407 r, err = d.esxcli("network", "vm", "port", "list", "-w", wid) 408 if err != nil { 409 return "", err 410 } 411 412 // Loop through interfaces 413 for { 414 record, err = r.read() 415 if err == io.EOF { 416 break 417 } 418 if err != nil { 419 return "", err 420 } 421 422 if record["IPAddress"] == "0.0.0.0" { 423 continue 424 } 425 // When multiple NICs are connected to the same network, choose 426 // one that has a route back. This Dial should ensure that. 427 conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", record["IPAddress"], port), 2*time.Second) 428 if err != nil { 429 if e, ok := err.(*net.OpError); ok { 430 if e.Timeout() { 431 log.Printf("Timeout connecting to %s", record["IPAddress"]) 432 continue 433 } else if strings.Contains(e.Error(), "connection refused") { 434 log.Printf("Connection refused when connecting to: %s", record["IPAddress"]) 435 continue 436 } 437 } 438 } else { 439 defer conn.Close() 440 address := record["IPAddress"] 441 return address, nil 442 } 443 } 444 return "", errors.New("No interface on the VM has an IP address ready") 445 } 446 447 //------------------------------------------------------------------- 448 // OutputDir implementation 449 //------------------------------------------------------------------- 450 451 func (d *ESX5Driver) DirExists() (bool, error) { 452 err := d.sh("test", "-e", d.outputDir) 453 return err == nil, nil 454 } 455 456 func (d *ESX5Driver) ListFiles() ([]string, error) { 457 stdout, err := d.ssh("ls -1p "+d.outputDir, nil) 458 if err != nil { 459 return nil, err 460 } 461 462 files := make([]string, 0, 10) 463 reader := bufio.NewReader(stdout) 464 for { 465 line, _, err := reader.ReadLine() 466 if err == io.EOF { 467 break 468 } 469 if line[len(line)-1] == '/' { 470 continue 471 } 472 473 files = append(files, filepath.ToSlash(filepath.Join(d.outputDir, string(line)))) 474 } 475 476 return files, nil 477 } 478 479 func (d *ESX5Driver) MkdirAll() error { 480 return d.mkdir(d.outputDir) 481 } 482 483 func (d *ESX5Driver) Remove(path string) error { 484 return d.sh("rm", path) 485 } 486 487 func (d *ESX5Driver) RemoveAll() error { 488 return d.sh("rm", "-rf", d.outputDir) 489 } 490 491 func (d *ESX5Driver) SetOutputDir(path string) { 492 d.outputDir = d.datastorePath(path) 493 } 494 495 func (d *ESX5Driver) String() string { 496 return d.outputDir 497 } 498 499 func (d *ESX5Driver) datastorePath(path string) string { 500 dirPath := filepath.Dir(path) 501 return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, dirPath, filepath.Base(path))) 502 } 503 504 func (d *ESX5Driver) cachePath(path string) string { 505 return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.CacheDatastore, d.CacheDirectory, filepath.Base(path))) 506 } 507 508 func (d *ESX5Driver) connect() error { 509 address := fmt.Sprintf("%s:%d", d.Host, d.Port) 510 511 auth := []gossh.AuthMethod{ 512 gossh.Password(d.Password), 513 gossh.KeyboardInteractive( 514 ssh.PasswordKeyboardInteractive(d.Password)), 515 } 516 517 if d.PrivateKey != "" { 518 signer, err := commonssh.FileSigner(d.PrivateKey) 519 if err != nil { 520 return err 521 } 522 523 auth = append(auth, gossh.PublicKeys(signer)) 524 } 525 526 sshConfig := &ssh.Config{ 527 Connection: ssh.ConnectFunc("tcp", address), 528 SSHConfig: &gossh.ClientConfig{ 529 User: d.Username, 530 Auth: auth, 531 HostKeyCallback: gossh.InsecureIgnoreHostKey(), 532 }, 533 } 534 535 comm, err := ssh.New(address, sshConfig) 536 if err != nil { 537 return err 538 } 539 540 d.comm = comm 541 return nil 542 } 543 544 func (d *ESX5Driver) checkSystemVersion() error { 545 r, err := d.esxcli("system", "version", "get") 546 if err != nil { 547 return err 548 } 549 550 record, err := r.read() 551 if err != nil { 552 return err 553 } 554 555 log.Printf("Connected to %s %s %s", record["Product"], 556 record["Version"], record["Build"]) 557 return nil 558 } 559 560 func (d *ESX5Driver) checkGuestIPHackEnabled() error { 561 r, err := d.esxcli("system", "settings", "advanced", "list", "-o", "/Net/GuestIPHack") 562 if err != nil { 563 return err 564 } 565 566 record, err := r.read() 567 if err != nil { 568 return err 569 } 570 571 if record["IntValue"] != "1" { 572 return errors.New( 573 "GuestIPHack is required, enable by running this on the ESX machine:\n" + 574 "esxcli system settings advanced set -o /Net/GuestIPHack -i 1") 575 } 576 577 return nil 578 } 579 580 func (d *ESX5Driver) mkdir(path string) error { 581 return d.sh("mkdir", "-p", path) 582 } 583 584 func (d *ESX5Driver) upload(dst, src string) error { 585 f, err := os.Open(src) 586 if err != nil { 587 return err 588 } 589 defer f.Close() 590 return d.comm.Upload(dst, f, nil) 591 } 592 593 func (d *ESX5Driver) verifyChecksum(ctype string, hash string, file string) bool { 594 if ctype == "none" { 595 if err := d.sh("stat", file); err != nil { 596 return false 597 } 598 } else { 599 stdin := bytes.NewBufferString(fmt.Sprintf("%s %s", hash, file)) 600 _, err := d.run(stdin, fmt.Sprintf("%ssum", ctype), "-c") 601 if err != nil { 602 return false 603 } 604 } 605 606 return true 607 } 608 609 func (d *ESX5Driver) ssh(command string, stdin io.Reader) (*bytes.Buffer, error) { 610 var stdout, stderr bytes.Buffer 611 612 cmd := &packer.RemoteCmd{ 613 Command: command, 614 Stdout: &stdout, 615 Stderr: &stderr, 616 Stdin: stdin, 617 } 618 619 err := d.comm.Start(cmd) 620 if err != nil { 621 return nil, err 622 } 623 624 cmd.Wait() 625 626 if cmd.ExitStatus != 0 { 627 err = fmt.Errorf("'%s'\n\nStdout: %s\n\nStderr: %s", 628 cmd.Command, stdout.String(), stderr.String()) 629 return nil, err 630 } 631 632 return &stdout, nil 633 } 634 635 func (d *ESX5Driver) run(stdin io.Reader, args ...string) (string, error) { 636 stdout, err := d.ssh(strings.Join(args, " "), stdin) 637 if err != nil { 638 return "", err 639 } 640 return stdout.String(), nil 641 } 642 643 func (d *ESX5Driver) sh(args ...string) error { 644 _, err := d.run(nil, args...) 645 return err 646 } 647 648 func (d *ESX5Driver) esxcli(args ...string) (*esxcliReader, error) { 649 stdout, err := d.ssh("esxcli --formatter csv "+strings.Join(args, " "), nil) 650 if err != nil { 651 return nil, err 652 } 653 r := csv.NewReader(bytes.NewReader(stdout.Bytes())) 654 r.TrailingComma = true 655 header, err := r.Read() 656 if err != nil { 657 return nil, err 658 } 659 return &esxcliReader{r, header}, nil 660 } 661 662 func (d *ESX5Driver) GetVmwareDriver() vmwcommon.VmwareDriver { 663 return d.base 664 } 665 666 type esxcliReader struct { 667 cr *csv.Reader 668 header []string 669 } 670 671 func (r *esxcliReader) read() (map[string]string, error) { 672 fields, err := r.cr.Read() 673 674 if err != nil { 675 return nil, err 676 } 677 678 record := map[string]string{} 679 for i, v := range fields { 680 record[r.header[i]] = v 681 } 682 683 return record, nil 684 } 685 686 func (r *esxcliReader) find(key, val string) (map[string]string, error) { 687 for { 688 record, err := r.read() 689 if err != nil { 690 return nil, err 691 } 692 if record[key] == val { 693 return record, nil 694 } 695 } 696 }