github.com/openshift/installer@v1.4.17/upi/vsphere/powercli/upi.ps1 (about)

     1  #!/usr/bin/pwsh
     2  
     3  $MYINV = $MyInvocation
     4  $SCRIPTDIR = split-path $MYINV.MyCommand.Path
     5  
     6  Write-Output "SCRIPT DIR: $($SCRIPTDIR)"
     7  
     8  . .\variables.ps1
     9  . ${SCRIPTDIR}\upi-functions.ps1
    10  
    11  $ErrorActionPreference = "Stop"
    12  
    13  # since we do not have ca for vsphere certs, we'll just set insecure
    14  Set-PowerCLIConfiguration -InvalidCertificateAction:Ignore -Confirm:$false | Out-Null
    15  $Env:GOVC_INSECURE = 1
    16  
    17  # Connect to vCenter
    18  Connect-VIServer -Server $vcenter -Credential (Import-Clixml $vcentercredpath)
    19  $cliContext = Get-PowerCLIContext
    20  
    21  if ($downloadInstaller) {
    22      Write-Output "Downloading the most recent $($version) installer"
    23  
    24      $releaseApiUri = "https://api.github.com/repos/openshift/okd/releases"
    25      $progressPreference = 'silentlyContinue'
    26      $webrequest = Invoke-WebRequest -uri $releaseApiUri
    27      $progressPreference = 'Continue'
    28      $releases = ConvertFrom-Json $webrequest.Content -AsHashtable
    29      $publishedDate = (Get-Date).AddDays(-365)
    30      $currentRelease = $null
    31  
    32      foreach($r in $releases) {
    33          if($r['name'] -like "*$($version)*") {
    34              if ($publishedDate -lt $r['published_at'] ) {
    35                  $publishedDate = $r['published_at']
    36                  $currentRelease = $r
    37              }
    38          }
    39      }
    40  
    41      foreach($asset in $currentRelease['assets']) {
    42          if($asset['name'] -like "openshift-install-linux*") {
    43              $installerUrl = $asset['browser_download_url']
    44          }
    45      }
    46  
    47      # If openshift-install doesn't exist on the path, download it and extract
    48      if (-Not (Test-Path -Path "openshift-install")) {
    49  
    50          $progressPreference = 'silentlyContinue'
    51          Invoke-WebRequest -uri $installerUrl -OutFile "installer.tar.gz"
    52          tar -xvf "installer.tar.gz"
    53          $progressPreference = 'Continue'
    54      }
    55  }
    56  
    57  if ($uploadTemplateOva) {
    58      Write-Output "Checking for RHCOS OVA"
    59  
    60      # If the OVA doesn't exist on the path, determine the url from openshift-install and download it.
    61      if (-Not (Test-Path -Path "template-$($Version).ova")) {
    62          Write-Output "Downloading RHCOS OVA"
    63          Start-Process -Wait -Path ./openshift-install -ArgumentList @("coreos", "print-stream-json") -RedirectStandardOutput coreos.json
    64  
    65          $coreosData = Get-Content -Path ./coreos.json | ConvertFrom-Json -AsHashtable
    66          $ovaUri = $coreosData.architectures.x86_64.artifacts.vmware.formats.ova.disk.location
    67          $progressPreference = 'silentlyContinue'
    68          Invoke-WebRequest -uri $ovaUri -OutFile "template-$($Version).ova"
    69          $progressPreference = 'Continue'
    70      }
    71  }
    72  
    73  $sshKey = [string](Get-Content -Path $sshkeypath -Raw:$true) -Replace '\n',''
    74  
    75  if ($createInstallConfig) {
    76      # Without having to add additional powershell modules yaml is difficult to deal
    77      # with. There is a supplied install-config.json which is converted to a powershell
    78      # object
    79      $config = ConvertFrom-Json -InputObject $installconfig
    80  
    81      # Set the install-config.json from upi-variables
    82      $config.metadata.name = $clustername
    83      $config.baseDomain = $basedomain
    84      $config.sshKey = $sshKey
    85      $config.platform.vsphere.vcenter = $vcenter
    86      $config.platform.vsphere.username = $username
    87      $config.platform.vsphere.password = $password
    88      $config.platform.vsphere.datacenter = $datacenter
    89      $config.platform.vsphere.defaultDatastore = $datastore
    90      $config.platform.vsphere.cluster = $cluster
    91      $config.platform.vsphere.network = $portgroup
    92      # $config.platform.vsphere.apiVIP = $apivip
    93      # $config.platform.vsphere.ingressVIP = $ingressvip
    94  
    95      $config.pullSecret = $pullsecret -replace "`n", "" -replace " ", ""
    96  
    97      # Write out the install-config.yaml (really json)
    98      $config | ConvertTo-Json -Depth 8 | Out-File -FilePath install-config.yaml -Force:$true
    99  }
   100  
   101  if ($generateIgnitions) {
   102      # openshift-install create manifests
   103      start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "manifests")
   104  
   105      # Remove master machines and the worker machinesets
   106      rm -f openshift/99_openshift-cluster-api_master-machines-*.yaml openshift/99_openshift-cluster-api_worker-machineset-*.yaml
   107  
   108      # openshift-install create ignition-configs
   109      start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "ignition-configs")
   110  }
   111  
   112  # Check failure domains.  If not set, create a default single failure domain from settings
   113  if ($null -eq $failure_domains) {
   114      Write-Output "Generating Failure Domain..."
   115      $failure_domains = @"
   116  [
   117    {
   118          "datacenter": "$($datacenter)",
   119          "cluster": "$($cluster)",
   120          "datastore": "$($datastore)",
   121          "network": "$($portgroup)"
   122    }
   123  ]
   124  "@
   125  }
   126  $fds = $failure_domains | ConvertFrom-Json
   127  
   128  # Convert the installer metadata to a powershell object
   129  $metadata = Get-Content -Path ./metadata.json | ConvertFrom-Json
   130  
   131  # Since we are using MachineSets for the workers make sure we set the
   132  # template name to what is expected to be generated by the installer.
   133  if ($null -eq $vm_template) {
   134      $vm_template = "$( $metadata.infraID )-rhcos"
   135  }
   136  
   137  # Create tag for all resources we create
   138  $tagCategory = Get-TagCategory -Name "openshift-$($metadata.infraID)" -ErrorAction continue 2>$null
   139  if (-Not $?) {
   140      Write-Output "Creating Tag Category openshift-$($metadata.infraID)"
   141      $tagCategory = New-TagCategory -Name "openshift-$($metadata.infraID)" -EntityType "urn:vim25:VirtualMachine","urn:vim25:ResourcePool","urn:vim25:Folder","urn:vim25:Datastore","urn:vim25:StoragePod"
   142  }
   143  $tag = Get-Tag -Category $tagCategory -Name "$($metadata.infraID)" -ErrorAction continue 2>$null
   144  if (-Not $?) {
   145      Write-Output "Creating Tag $($metadata.infraID)"
   146      $tag = New-Tag -Category $tagCategory -Name "$($metadata.infraID)"
   147  }
   148  
   149  $jobs = @()
   150  $templateInProgress = @()
   151  
   152  # Check each failure domain for ova template
   153  foreach ($fd in $fds)
   154  {
   155      $datastoreInfo = Get-Datastore -Name $fd.datastore -Location $fd.datacenter
   156  
   157      # If the folder already exists
   158      Write-Output "Checking for folder in failure domain $($fd.datacenter)/$($fd.cluster)"
   159      $folder = Get-Folder -Name $clustername -Location $fd.datacenter -ErrorAction continue 2>$null
   160  
   161      # Otherwise create the folder within the datacenter as defined in the upi-variables
   162      if (-Not $?) {
   163          Write-Output "Creating folder $($clustername) in datacenter $($fd.datacenter)"
   164          (get-view (Get-Datacenter -Name $fd.datacenter).ExtensionData.vmfolder).CreateFolder($clustername)
   165          $folder = Get-Folder -Name $clustername -Location $fd.datacenter
   166          New-TagAssignment -Entity $folder -Tag $tag > $null
   167      }
   168  
   169      # Create resource pool for all future VMs
   170      Write-Output "Checking for resource pool in failure domain $($fd.datacenter)/$($fd.cluster)"
   171      $rp = Get-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fd.cluster)) -ErrorAction continue 2>$null
   172  
   173      if (-Not $?) {
   174          Write-Output "Creating resource pool $($metadata.infraID) in datacenter $($fd.datacenter)"
   175          $rp = New-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fd.cluster))
   176          New-TagAssignment -Entity $rp -Tag $tag > $null
   177      }
   178  
   179      # If the rhcos virtual machine already exists
   180      Write-Output "Checking for vm template in failure domain $($fd.datacenter)/$($fd.cluster)"
   181      $template = Get-VM -Name $vm_template -Location $fd.datacenter -ErrorAction continue
   182  
   183      # Otherwise import the ova to a random host on the vSphere cluster
   184      if (-Not $? -And -Not $templateInProgress.Contains($fd.datacenter))
   185      {
   186          $templateInProgress += $fd.datacenter
   187          $vmhost = Get-Random -InputObject (Get-VMHost -Location (Get-Cluster $fd.cluster))
   188          $ovfConfig = Get-OvfConfiguration -Ovf "template-$($Version).ova"
   189          $ovfConfig.NetworkMapping.VM_Network.Value = $fd.network
   190          Write-Output "OVF: $($ovfConfig)"
   191          $jobs += Start-ThreadJob -n "upload-template-$($fd.cluster)" -ScriptBlock {
   192              param($Version,$vm_template,$ovfConfig,$vmhost,$datastoreInfo,$folder,$tag,$scriptdir,$cliContext)
   193              . .\variables.ps1
   194              . ${scriptdir}\upi-functions.ps1
   195              Write-Output "Version: $($Version)"
   196              Write-Output "VM Template: $($vm_template)"
   197              Write-Output "OVF Config: $($ovfConfig)"
   198              Write-Output "VM Host: $($vmhost)"
   199              Use-PowerCLIContext -PowerCLIContext $cliContext
   200              $template = Import-Vapp -Source "template-$($Version).ova" -Name $vm_template -OvfConfiguration $ovfConfig -VMHost $vmhost -Datastore $datastoreInfo -InventoryLocation $folder -Force:$true
   201  
   202              $templateVIObj = Get-View -VIObject $template.Name
   203              # Need to look into upgrading hardware.  For me it keeps throwing exception.
   204              <# try {
   205              $templateVIObj.UpgradeVM($hardwareVersion)
   206          }
   207          catch {
   208              Write-Output "Something happened setting VM hardware version"
   209              Write-Output $_
   210          } #>
   211  
   212              New-TagAssignment -Entity $template -Tag $tag
   213              Set-VM -VM $template -MemoryGB 16 -NumCpu 4 -CoresPerSocket 4 -Confirm:$false > $null
   214              #Get-HardDisk -VM $template | Select-Object -First 1 | Set-HardDisk -CapacityGB 120 -Confirm:$false > $null
   215              updateDisk -VM $template -CapacityGB 120
   216              New-AdvancedSetting -Entity $template -name "disk.EnableUUID" -value 'TRUE' -confirm:$false -Force > $null
   217              New-AdvancedSetting -Entity $template -name "guestinfo.ignition.config.data.encoding" -value "base64" -confirm:$false -Force > $null
   218              #$snapshot = New-Snapshot -VM $template -Name "linked-clone" -Description "linked-clone" -Memory -Quiesce
   219          } -ArgumentList @($Version,$vm_template,$ovfConfig,$vmhost,$datastoreInfo,$folder,$tag,$SCRIPTDIR,$cliContext)
   220      }
   221  }
   222  
   223  # If jobs were started, lets wait till they are done
   224  if ($jobs.count -gt 0)
   225  {
   226      Wait-Job -Job $jobs
   227      foreach ($job in $jobs) {
   228          Receive-Job -Job $job
   229      }
   230  }
   231  
   232  Write-Output "Creating LB"
   233  
   234  # Data needed for LB VM creation
   235  $rp = Get-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fds[0].cluster)) -Server $vcenter
   236  $datastoreInfo = Get-Datastore -Name $fds[0].datastore -Server $vcenter -Location $fds[0].datacenter
   237  $folder = Get-Folder -Name $clustername -Location $fds[0].datacenter
   238  $template = Get-VM -Name $vm_template -Location $fds[0].datacenter
   239  
   240  # Create LB for Cluster
   241  $ignition = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((New-LoadBalancerIgnition $sshKey)))
   242  $network = New-VMNetworkConfig -Hostname "$($metadata.infraID)-lb" -IPAddress $lb_ip_address -Netmask $netmask -Gateway $gateway -DNS $dns -Network $failure_domains[0].network
   243  $vm = New-OpenShiftVM -IgnitionData $ignition -Name "$($metadata.infraID)-lb" -Template $template -ResourcePool $rp -Datastore $datastoreInfo -Location $folder -Tag $tag -Networking $network -Network $($fds[0].network) -SecureBoot $secureboot -StoragePolicy $storagepolicy -MemoryMB 8192 -NumCpu 4
   244  $vm | Start-VM
   245  
   246  # Take the $virtualmachines defined in upi-variables and convert to a powershell object
   247  if ($null -eq $virtualmachines)
   248  {
   249      $virtualmachines = New-VMConfigs
   250  }
   251  $vmHash = ConvertFrom-Json -InputObject $virtualmachines -AsHashtable
   252  
   253  Write-Progress -id 222 -Activity "Creating virtual machines" -PercentComplete 0
   254  
   255  New-OpenshiftVMs "bootstrap"
   256  New-OpenshiftVMs "master"
   257  New-OpenshiftVMs "worker"
   258  Write-Progress -id 222 -Activity "Completed virtual machines" -PercentComplete 100 -Completed
   259  
   260  ## This is nice to have to clear screen when doing things manually.  Maybe i'll
   261  # make this configurable.
   262  # Clear-Host
   263  
   264  # Instead of restarting openshift-install to wait for bootstrap, monitor
   265  # the bootstrap configmap in the kube-system namespace
   266  
   267  # Extract the Client Certificate Data from auth/kubeconfig
   268  $match = Select-String "client-certificate-data: (.*)" -Path ./auth/kubeconfig
   269  [Byte[]]$bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value)
   270  $clientCertData = [System.Text.Encoding]::ASCII.GetString($bytes)
   271  
   272  # Extract the Client Key Data from auth/kubeconfig
   273  $match = Select-String "client-key-data: (.*)" -Path ./auth/kubeconfig
   274  $bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value)
   275  $clientKeyData = [System.Text.Encoding]::ASCII.GetString($bytes)
   276  
   277  # Create a X509Certificate2 object for Invoke-WebRequest
   278  $cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::CreateFromPem($clientCertData, $clientKeyData)
   279  
   280  # Extract the kubernetes endpoint uri
   281  $match = Select-String "server: (.*)" -Path ./auth/kubeconfig
   282  $kubeurl = $match.Matches.Groups[1].Value
   283  
   284  if ($waitForComplete)
   285  {
   286      $apiTimeout = (20*60)
   287      $apiCount = 1
   288      $apiSleep = 30
   289      Write-Progress -Id 444 -Status "1% Complete" -Activity "API" -PercentComplete 1
   290      :api while ($true) {
   291          Start-Sleep -Seconds $apiSleep
   292          try {
   293              $webrequest = Invoke-WebRequest -Uri "$($kubeurl)/version" -SkipCertificateCheck
   294              $version = (ConvertFrom-Json $webrequest.Content).gitVersion
   295  
   296              if ($version -ne "" ) {
   297                  Write-Debug "API Version: $($version)"
   298                  Write-Progress -Id 444 -Status "Completed" -Activity "API" -PercentComplete 100
   299                  break api
   300              }
   301          }
   302          catch {}
   303  
   304          $percentage = ((($apiCount*$apiSleep)/$apiTimeout)*100)
   305          if ($percentage -le 100) {
   306              Write-Progress -Id 444 -Status "$percentage% Complete" -Activity "API" -PercentComplete $percentage
   307          }
   308          $apiCount++
   309      }
   310  
   311      $bootstrapTimeout = (30*60)
   312      $bootstrapCount = 1
   313      $bootstrapSleep = 30
   314      Write-Progress -Id 333 -Status "1% Complete" -Activity "Bootstrap" -PercentComplete 1
   315      :bootstrap while ($true)
   316      {
   317          Start-Sleep -Seconds $bootstrapSleep
   318  
   319          try
   320          {
   321              $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/namespaces/kube-system/configmaps/bootstrap" -SkipCertificateCheck
   322  
   323              $bootstrapStatus = (ConvertFrom-Json $webrequest.Content).data.status
   324  
   325              if ($bootstrapStatus -eq "complete")
   326              {
   327                  Get-VM "$( $metadata.infraID )-bootstrap" | Stop-VM -Confirm:$false | Remove-VM -DeletePermanently -Confirm:$false
   328                  Write-Progress -Id 333 -Status "Completed" -Activity "Bootstrap" -PercentComplete 100
   329                  break bootstrap
   330              }
   331          }
   332          catch
   333          {
   334          }
   335  
   336          $percentage = ((($bootstrapCount*$bootstrapSleep)/$bootstrapTimeout)*100)
   337          if ($percentage -le 100)
   338          {
   339              Write-Progress -Id 333 -Status "$percentage% Complete" -Activity "Bootstrap" -PercentComplete $percentage
   340          }
   341          else
   342          {
   343              Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow
   344          }
   345  
   346          $bootstrapCount++
   347      }
   348  
   349      # Now that bootstrap is complete, we should be getting worker node CSRs that need to be approved before being
   350      # able to finish installation.  Lets monitor for CSRs, approve them and verify the number of worker nodes have
   351      # now appeared and are Ready before moving on.
   352  
   353      # [ngirard@fedora ibm7-installs]$ oc get csr | grep Pending
   354      #csr-2hgbd                                        2m52s   kubernetes.io/kube-apiserver-client-kubelet   system:serviceaccount:openshift-machine-config-operator:node-bootstrapper         <none>              Pending
   355      #csr-lwmgf                                        2m19s   kubernetes.io/kube-apiserver-client-kubelet   system:serviceaccount:openshift-machine-config-operator:node-bootstrapper         <none>              Pending
   356      #csr-scvk6                                        2m30s   kubernetes.io/kube-apiserver-client-kubelet   system:serviceaccount:openshift-machine-config-operator:node-bootstrapper         <none>              Pending
   357  
   358      # apis/certificates.k8s.io/v1/certificatesigningrequests
   359      $csrTimeout = (600/5)
   360      $csrCount = 1
   361      $csrSleep = 5
   362      Write-Progress -Id 222 -Status "1% Complete" -Activity "Worker Ready" -PercentComplete 0
   363      :csrLoop while ($true)
   364      {
   365          Start-Sleep -Seconds $csrSleep
   366  
   367          try
   368          {
   369              $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests" -SkipCertificateCheck
   370  
   371              $csrs = (ConvertFrom-Json $webrequest.Content).items
   372  
   373              foreach ($csr in $csrs)
   374              {
   375                  # Check if no status (Pending) and if its a type we are looking for (kubernetes.io/kubelet-serving) (kubernetes.io/kube-apiserver-client-kubelet)
   376                  $bootstrapper = "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper"
   377                  $nodeUser = "system:node:"
   378  
   379                  $csrUser = $csr.spec.username
   380                  if ($csr.status.conditions -eq $null -And ($csrUser -eq $bootstrapper -Or $csrUser.Contains($nodeUser)))
   381                  {
   382                      $conditions = New-Object System.Collections.ArrayList
   383                      $condition = @{
   384                          type = "Approved"
   385                          status = "True"
   386                          reason = "PowershellApproved"
   387                          message = "This CSR was approved by script in PowerShell."
   388                      }
   389                      $conditions.Add($condition) > $null
   390                      $csr.status | add-member -Name "conditions" -value $conditions  -MemberType NoteProperty
   391                      Write-Output "Accepting CSR: $( $csr.metadata.name )"
   392                      $csrResponse = Invoke-RestMethod -Method "Put" -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests/$( $csr.metadata.name )/approval" -SkipCertificateCheck -Body (ConvertTo-Json $csr -Depth 6)
   393                  }
   394              }
   395          }
   396          catch
   397          {
   398              #Write-Output $_
   399          }
   400  
   401          # Check number of worker nodes with NotReady/Ready.  NotReady will be 1 pt where Ready will be 2.
   402          $currentComputePoints = 0
   403  
   404          try
   405          {
   406              $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/nodes" -SkipCertificateCheck
   407  
   408              $nodes = (ConvertFrom-Json $webrequest.Content).items
   409  
   410              foreach ($node in $nodes)
   411              {
   412                  if ($node.metadata.labels.psobject.properties.name -Contains "node-role.kubernetes.io/worker")
   413                  {
   414                      #Write-Output "Checking node $($node.metadata.name)"
   415                      foreach ($condition in $node.status.conditions)
   416                      {
   417                          if ($condition.type -eq "Ready")
   418                          {
   419                              #Write-Output "Is node ready? $($condition.status)"
   420                              if ($condition.status -eq "True")
   421                              {
   422                                  $currentComputePoints = $currentComputePoints + 2
   423                              }
   424                              else
   425                              {
   426                                  $currentComputePoints++
   427                              }
   428                          }
   429                      }
   430                  }
   431              }
   432          }
   433          catch
   434          {
   435              #Write-Output $_
   436          }
   437  
   438          $maxComputePoints = $compute_count * 2
   439          $percentage = ((($currentComputePoints)/$maxComputePoints)*100)
   440          if ($percentage -eq 100)
   441          {
   442              Write-Progress -Id 222 -Status "Completed" -Activity "Worker Ready" -PercentComplete 100
   443              break csrLoop
   444          }
   445          elseif ($percentage -le 100)
   446          {
   447              Write-Progress -Id 222 -Status "$percentage% Complete" -Activity "Worker Ready" -PercentComplete $percentage
   448          }
   449  
   450          if ($csrCount -ge $csrTimeout)
   451          {
   452              Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow
   453              break csrLoop
   454          }
   455  
   456          $csrCount++
   457      }
   458  
   459      $progressMsg = ""
   460      Write-Progress -Id 111 -Status "1% Complete" -Activity "Install" -PercentComplete 1
   461      :installcomplete while ($true)
   462      {
   463          Start-Sleep -Seconds 30
   464          try
   465          {
   466              $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/config.openshift.io/v1/clusterversions" -SkipCertificateCheck
   467  
   468              $clusterversions = ConvertFrom-Json $webrequest.Content -AsHashtable
   469  
   470              # just like the installer check the status conditions of the clusterversions config
   471              foreach ($condition in $clusterversions['items'][0]['status']['conditions'])
   472              {
   473                  switch ($condition['type'])
   474                  {
   475                      "Progressing" {
   476                          if ($condition['status'] -eq "True")
   477                          {
   478  
   479                              $matchper = ($condition['message'] | Select-String "^Working.*\(([0-9]{1,3})\%.*\)")
   480                              $matchmsg = ($condition['message'] | Select-String -AllMatches -Pattern "^(Working.*)\:.*")
   481  
   482                              # During install, the status of CVO will / may go degraded due to operators going
   483                              # degraded from taking a while to install.  It seems this is the new norm as control
   484                              # plane takes a while to roll out and certain operators go degraded until the control
   485                              # plane is stable.
   486                              if ($matchmsg.Matches.Groups -ne $null)
   487                              {
   488                                  $progressMsg = $matchmsg.Matches.Groups[1].Value
   489                                  $progressPercent = $matchper.Matches.Groups[1].Value
   490  
   491                                  Write-Progress -Id 111 -Status "$progressPercent% Complete - $( $progressMsg )" -Activity "Install" -PercentComplete $progressPercent
   492                              }
   493                              continue
   494                          }
   495                      }
   496                      "Available" {
   497                          if ($condition['status'] -eq "True")
   498                          {
   499                              Write-Progress -Id 111 -Activity "Install" -Status "Completed" -PercentComplete 100
   500                              break installcomplete
   501                          }
   502                          continue
   503                      }
   504                      Default {
   505                          continue
   506                      }
   507                  }
   508              }
   509          }
   510          catch
   511          {
   512              Write-Output "Unable to check operators"
   513              Write-Output $_
   514          }
   515      }
   516  }
   517  
   518  Get-Job | Remove-Job
   519  
   520  Disconnect-VIServer -Server $vcenter -Force:$true -Confirm:$false
   521  
   522  Write-Output "Install Complete!"