github.com/openshift/installer@v1.4.17/upi/vsphere/powercli/upi.ps1 (about) 1 #!/usr/bin/pwsh 2 3 $MYINV = $MyInvocation 4 $SCRIPTDIR = split-path $MYINV.MyCommand.Path 5 6 Write-Output "SCRIPT DIR: $($SCRIPTDIR)" 7 8 . .\variables.ps1 9 . ${SCRIPTDIR}\upi-functions.ps1 10 11 $ErrorActionPreference = "Stop" 12 13 # since we do not have ca for vsphere certs, we'll just set insecure 14 Set-PowerCLIConfiguration -InvalidCertificateAction:Ignore -Confirm:$false | Out-Null 15 $Env:GOVC_INSECURE = 1 16 17 # Connect to vCenter 18 Connect-VIServer -Server $vcenter -Credential (Import-Clixml $vcentercredpath) 19 $cliContext = Get-PowerCLIContext 20 21 if ($downloadInstaller) { 22 Write-Output "Downloading the most recent $($version) installer" 23 24 $releaseApiUri = "https://api.github.com/repos/openshift/okd/releases" 25 $progressPreference = 'silentlyContinue' 26 $webrequest = Invoke-WebRequest -uri $releaseApiUri 27 $progressPreference = 'Continue' 28 $releases = ConvertFrom-Json $webrequest.Content -AsHashtable 29 $publishedDate = (Get-Date).AddDays(-365) 30 $currentRelease = $null 31 32 foreach($r in $releases) { 33 if($r['name'] -like "*$($version)*") { 34 if ($publishedDate -lt $r['published_at'] ) { 35 $publishedDate = $r['published_at'] 36 $currentRelease = $r 37 } 38 } 39 } 40 41 foreach($asset in $currentRelease['assets']) { 42 if($asset['name'] -like "openshift-install-linux*") { 43 $installerUrl = $asset['browser_download_url'] 44 } 45 } 46 47 # If openshift-install doesn't exist on the path, download it and extract 48 if (-Not (Test-Path -Path "openshift-install")) { 49 50 $progressPreference = 'silentlyContinue' 51 Invoke-WebRequest -uri $installerUrl -OutFile "installer.tar.gz" 52 tar -xvf "installer.tar.gz" 53 $progressPreference = 'Continue' 54 } 55 } 56 57 if ($uploadTemplateOva) { 58 Write-Output "Checking for RHCOS OVA" 59 60 # If the OVA doesn't exist on the path, determine the url from openshift-install and download it. 61 if (-Not (Test-Path -Path "template-$($Version).ova")) { 62 Write-Output "Downloading RHCOS OVA" 63 Start-Process -Wait -Path ./openshift-install -ArgumentList @("coreos", "print-stream-json") -RedirectStandardOutput coreos.json 64 65 $coreosData = Get-Content -Path ./coreos.json | ConvertFrom-Json -AsHashtable 66 $ovaUri = $coreosData.architectures.x86_64.artifacts.vmware.formats.ova.disk.location 67 $progressPreference = 'silentlyContinue' 68 Invoke-WebRequest -uri $ovaUri -OutFile "template-$($Version).ova" 69 $progressPreference = 'Continue' 70 } 71 } 72 73 $sshKey = [string](Get-Content -Path $sshkeypath -Raw:$true) -Replace '\n','' 74 75 if ($createInstallConfig) { 76 # Without having to add additional powershell modules yaml is difficult to deal 77 # with. There is a supplied install-config.json which is converted to a powershell 78 # object 79 $config = ConvertFrom-Json -InputObject $installconfig 80 81 # Set the install-config.json from upi-variables 82 $config.metadata.name = $clustername 83 $config.baseDomain = $basedomain 84 $config.sshKey = $sshKey 85 $config.platform.vsphere.vcenter = $vcenter 86 $config.platform.vsphere.username = $username 87 $config.platform.vsphere.password = $password 88 $config.platform.vsphere.datacenter = $datacenter 89 $config.platform.vsphere.defaultDatastore = $datastore 90 $config.platform.vsphere.cluster = $cluster 91 $config.platform.vsphere.network = $portgroup 92 # $config.platform.vsphere.apiVIP = $apivip 93 # $config.platform.vsphere.ingressVIP = $ingressvip 94 95 $config.pullSecret = $pullsecret -replace "`n", "" -replace " ", "" 96 97 # Write out the install-config.yaml (really json) 98 $config | ConvertTo-Json -Depth 8 | Out-File -FilePath install-config.yaml -Force:$true 99 } 100 101 if ($generateIgnitions) { 102 # openshift-install create manifests 103 start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "manifests") 104 105 # Remove master machines and the worker machinesets 106 rm -f openshift/99_openshift-cluster-api_master-machines-*.yaml openshift/99_openshift-cluster-api_worker-machineset-*.yaml 107 108 # openshift-install create ignition-configs 109 start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "ignition-configs") 110 } 111 112 # Check failure domains. If not set, create a default single failure domain from settings 113 if ($null -eq $failure_domains) { 114 Write-Output "Generating Failure Domain..." 115 $failure_domains = @" 116 [ 117 { 118 "datacenter": "$($datacenter)", 119 "cluster": "$($cluster)", 120 "datastore": "$($datastore)", 121 "network": "$($portgroup)" 122 } 123 ] 124 "@ 125 } 126 $fds = $failure_domains | ConvertFrom-Json 127 128 # Convert the installer metadata to a powershell object 129 $metadata = Get-Content -Path ./metadata.json | ConvertFrom-Json 130 131 # Since we are using MachineSets for the workers make sure we set the 132 # template name to what is expected to be generated by the installer. 133 if ($null -eq $vm_template) { 134 $vm_template = "$( $metadata.infraID )-rhcos" 135 } 136 137 # Create tag for all resources we create 138 $tagCategory = Get-TagCategory -Name "openshift-$($metadata.infraID)" -ErrorAction continue 2>$null 139 if (-Not $?) { 140 Write-Output "Creating Tag Category openshift-$($metadata.infraID)" 141 $tagCategory = New-TagCategory -Name "openshift-$($metadata.infraID)" -EntityType "urn:vim25:VirtualMachine","urn:vim25:ResourcePool","urn:vim25:Folder","urn:vim25:Datastore","urn:vim25:StoragePod" 142 } 143 $tag = Get-Tag -Category $tagCategory -Name "$($metadata.infraID)" -ErrorAction continue 2>$null 144 if (-Not $?) { 145 Write-Output "Creating Tag $($metadata.infraID)" 146 $tag = New-Tag -Category $tagCategory -Name "$($metadata.infraID)" 147 } 148 149 $jobs = @() 150 $templateInProgress = @() 151 152 # Check each failure domain for ova template 153 foreach ($fd in $fds) 154 { 155 $datastoreInfo = Get-Datastore -Name $fd.datastore -Location $fd.datacenter 156 157 # If the folder already exists 158 Write-Output "Checking for folder in failure domain $($fd.datacenter)/$($fd.cluster)" 159 $folder = Get-Folder -Name $clustername -Location $fd.datacenter -ErrorAction continue 2>$null 160 161 # Otherwise create the folder within the datacenter as defined in the upi-variables 162 if (-Not $?) { 163 Write-Output "Creating folder $($clustername) in datacenter $($fd.datacenter)" 164 (get-view (Get-Datacenter -Name $fd.datacenter).ExtensionData.vmfolder).CreateFolder($clustername) 165 $folder = Get-Folder -Name $clustername -Location $fd.datacenter 166 New-TagAssignment -Entity $folder -Tag $tag > $null 167 } 168 169 # Create resource pool for all future VMs 170 Write-Output "Checking for resource pool in failure domain $($fd.datacenter)/$($fd.cluster)" 171 $rp = Get-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fd.cluster)) -ErrorAction continue 2>$null 172 173 if (-Not $?) { 174 Write-Output "Creating resource pool $($metadata.infraID) in datacenter $($fd.datacenter)" 175 $rp = New-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fd.cluster)) 176 New-TagAssignment -Entity $rp -Tag $tag > $null 177 } 178 179 # If the rhcos virtual machine already exists 180 Write-Output "Checking for vm template in failure domain $($fd.datacenter)/$($fd.cluster)" 181 $template = Get-VM -Name $vm_template -Location $fd.datacenter -ErrorAction continue 182 183 # Otherwise import the ova to a random host on the vSphere cluster 184 if (-Not $? -And -Not $templateInProgress.Contains($fd.datacenter)) 185 { 186 $templateInProgress += $fd.datacenter 187 $vmhost = Get-Random -InputObject (Get-VMHost -Location (Get-Cluster $fd.cluster)) 188 $ovfConfig = Get-OvfConfiguration -Ovf "template-$($Version).ova" 189 $ovfConfig.NetworkMapping.VM_Network.Value = $fd.network 190 Write-Output "OVF: $($ovfConfig)" 191 $jobs += Start-ThreadJob -n "upload-template-$($fd.cluster)" -ScriptBlock { 192 param($Version,$vm_template,$ovfConfig,$vmhost,$datastoreInfo,$folder,$tag,$scriptdir,$cliContext) 193 . .\variables.ps1 194 . ${scriptdir}\upi-functions.ps1 195 Write-Output "Version: $($Version)" 196 Write-Output "VM Template: $($vm_template)" 197 Write-Output "OVF Config: $($ovfConfig)" 198 Write-Output "VM Host: $($vmhost)" 199 Use-PowerCLIContext -PowerCLIContext $cliContext 200 $template = Import-Vapp -Source "template-$($Version).ova" -Name $vm_template -OvfConfiguration $ovfConfig -VMHost $vmhost -Datastore $datastoreInfo -InventoryLocation $folder -Force:$true 201 202 $templateVIObj = Get-View -VIObject $template.Name 203 # Need to look into upgrading hardware. For me it keeps throwing exception. 204 <# try { 205 $templateVIObj.UpgradeVM($hardwareVersion) 206 } 207 catch { 208 Write-Output "Something happened setting VM hardware version" 209 Write-Output $_ 210 } #> 211 212 New-TagAssignment -Entity $template -Tag $tag 213 Set-VM -VM $template -MemoryGB 16 -NumCpu 4 -CoresPerSocket 4 -Confirm:$false > $null 214 #Get-HardDisk -VM $template | Select-Object -First 1 | Set-HardDisk -CapacityGB 120 -Confirm:$false > $null 215 updateDisk -VM $template -CapacityGB 120 216 New-AdvancedSetting -Entity $template -name "disk.EnableUUID" -value 'TRUE' -confirm:$false -Force > $null 217 New-AdvancedSetting -Entity $template -name "guestinfo.ignition.config.data.encoding" -value "base64" -confirm:$false -Force > $null 218 #$snapshot = New-Snapshot -VM $template -Name "linked-clone" -Description "linked-clone" -Memory -Quiesce 219 } -ArgumentList @($Version,$vm_template,$ovfConfig,$vmhost,$datastoreInfo,$folder,$tag,$SCRIPTDIR,$cliContext) 220 } 221 } 222 223 # If jobs were started, lets wait till they are done 224 if ($jobs.count -gt 0) 225 { 226 Wait-Job -Job $jobs 227 foreach ($job in $jobs) { 228 Receive-Job -Job $job 229 } 230 } 231 232 Write-Output "Creating LB" 233 234 # Data needed for LB VM creation 235 $rp = Get-ResourcePool -Name $($metadata.infraID) -Location $(Get-Cluster -Name $($fds[0].cluster)) -Server $vcenter 236 $datastoreInfo = Get-Datastore -Name $fds[0].datastore -Server $vcenter -Location $fds[0].datacenter 237 $folder = Get-Folder -Name $clustername -Location $fds[0].datacenter 238 $template = Get-VM -Name $vm_template -Location $fds[0].datacenter 239 240 # Create LB for Cluster 241 $ignition = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((New-LoadBalancerIgnition $sshKey))) 242 $network = New-VMNetworkConfig -Hostname "$($metadata.infraID)-lb" -IPAddress $lb_ip_address -Netmask $netmask -Gateway $gateway -DNS $dns -Network $failure_domains[0].network 243 $vm = New-OpenShiftVM -IgnitionData $ignition -Name "$($metadata.infraID)-lb" -Template $template -ResourcePool $rp -Datastore $datastoreInfo -Location $folder -Tag $tag -Networking $network -Network $($fds[0].network) -SecureBoot $secureboot -StoragePolicy $storagepolicy -MemoryMB 8192 -NumCpu 4 244 $vm | Start-VM 245 246 # Take the $virtualmachines defined in upi-variables and convert to a powershell object 247 if ($null -eq $virtualmachines) 248 { 249 $virtualmachines = New-VMConfigs 250 } 251 $vmHash = ConvertFrom-Json -InputObject $virtualmachines -AsHashtable 252 253 Write-Progress -id 222 -Activity "Creating virtual machines" -PercentComplete 0 254 255 New-OpenshiftVMs "bootstrap" 256 New-OpenshiftVMs "master" 257 New-OpenshiftVMs "worker" 258 Write-Progress -id 222 -Activity "Completed virtual machines" -PercentComplete 100 -Completed 259 260 ## This is nice to have to clear screen when doing things manually. Maybe i'll 261 # make this configurable. 262 # Clear-Host 263 264 # Instead of restarting openshift-install to wait for bootstrap, monitor 265 # the bootstrap configmap in the kube-system namespace 266 267 # Extract the Client Certificate Data from auth/kubeconfig 268 $match = Select-String "client-certificate-data: (.*)" -Path ./auth/kubeconfig 269 [Byte[]]$bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value) 270 $clientCertData = [System.Text.Encoding]::ASCII.GetString($bytes) 271 272 # Extract the Client Key Data from auth/kubeconfig 273 $match = Select-String "client-key-data: (.*)" -Path ./auth/kubeconfig 274 $bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value) 275 $clientKeyData = [System.Text.Encoding]::ASCII.GetString($bytes) 276 277 # Create a X509Certificate2 object for Invoke-WebRequest 278 $cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::CreateFromPem($clientCertData, $clientKeyData) 279 280 # Extract the kubernetes endpoint uri 281 $match = Select-String "server: (.*)" -Path ./auth/kubeconfig 282 $kubeurl = $match.Matches.Groups[1].Value 283 284 if ($waitForComplete) 285 { 286 $apiTimeout = (20*60) 287 $apiCount = 1 288 $apiSleep = 30 289 Write-Progress -Id 444 -Status "1% Complete" -Activity "API" -PercentComplete 1 290 :api while ($true) { 291 Start-Sleep -Seconds $apiSleep 292 try { 293 $webrequest = Invoke-WebRequest -Uri "$($kubeurl)/version" -SkipCertificateCheck 294 $version = (ConvertFrom-Json $webrequest.Content).gitVersion 295 296 if ($version -ne "" ) { 297 Write-Debug "API Version: $($version)" 298 Write-Progress -Id 444 -Status "Completed" -Activity "API" -PercentComplete 100 299 break api 300 } 301 } 302 catch {} 303 304 $percentage = ((($apiCount*$apiSleep)/$apiTimeout)*100) 305 if ($percentage -le 100) { 306 Write-Progress -Id 444 -Status "$percentage% Complete" -Activity "API" -PercentComplete $percentage 307 } 308 $apiCount++ 309 } 310 311 $bootstrapTimeout = (30*60) 312 $bootstrapCount = 1 313 $bootstrapSleep = 30 314 Write-Progress -Id 333 -Status "1% Complete" -Activity "Bootstrap" -PercentComplete 1 315 :bootstrap while ($true) 316 { 317 Start-Sleep -Seconds $bootstrapSleep 318 319 try 320 { 321 $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/namespaces/kube-system/configmaps/bootstrap" -SkipCertificateCheck 322 323 $bootstrapStatus = (ConvertFrom-Json $webrequest.Content).data.status 324 325 if ($bootstrapStatus -eq "complete") 326 { 327 Get-VM "$( $metadata.infraID )-bootstrap" | Stop-VM -Confirm:$false | Remove-VM -DeletePermanently -Confirm:$false 328 Write-Progress -Id 333 -Status "Completed" -Activity "Bootstrap" -PercentComplete 100 329 break bootstrap 330 } 331 } 332 catch 333 { 334 } 335 336 $percentage = ((($bootstrapCount*$bootstrapSleep)/$bootstrapTimeout)*100) 337 if ($percentage -le 100) 338 { 339 Write-Progress -Id 333 -Status "$percentage% Complete" -Activity "Bootstrap" -PercentComplete $percentage 340 } 341 else 342 { 343 Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow 344 } 345 346 $bootstrapCount++ 347 } 348 349 # Now that bootstrap is complete, we should be getting worker node CSRs that need to be approved before being 350 # able to finish installation. Lets monitor for CSRs, approve them and verify the number of worker nodes have 351 # now appeared and are Ready before moving on. 352 353 # [ngirard@fedora ibm7-installs]$ oc get csr | grep Pending 354 #csr-2hgbd 2m52s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending 355 #csr-lwmgf 2m19s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending 356 #csr-scvk6 2m30s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending 357 358 # apis/certificates.k8s.io/v1/certificatesigningrequests 359 $csrTimeout = (600/5) 360 $csrCount = 1 361 $csrSleep = 5 362 Write-Progress -Id 222 -Status "1% Complete" -Activity "Worker Ready" -PercentComplete 0 363 :csrLoop while ($true) 364 { 365 Start-Sleep -Seconds $csrSleep 366 367 try 368 { 369 $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests" -SkipCertificateCheck 370 371 $csrs = (ConvertFrom-Json $webrequest.Content).items 372 373 foreach ($csr in $csrs) 374 { 375 # Check if no status (Pending) and if its a type we are looking for (kubernetes.io/kubelet-serving) (kubernetes.io/kube-apiserver-client-kubelet) 376 $bootstrapper = "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper" 377 $nodeUser = "system:node:" 378 379 $csrUser = $csr.spec.username 380 if ($csr.status.conditions -eq $null -And ($csrUser -eq $bootstrapper -Or $csrUser.Contains($nodeUser))) 381 { 382 $conditions = New-Object System.Collections.ArrayList 383 $condition = @{ 384 type = "Approved" 385 status = "True" 386 reason = "PowershellApproved" 387 message = "This CSR was approved by script in PowerShell." 388 } 389 $conditions.Add($condition) > $null 390 $csr.status | add-member -Name "conditions" -value $conditions -MemberType NoteProperty 391 Write-Output "Accepting CSR: $( $csr.metadata.name )" 392 $csrResponse = Invoke-RestMethod -Method "Put" -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests/$( $csr.metadata.name )/approval" -SkipCertificateCheck -Body (ConvertTo-Json $csr -Depth 6) 393 } 394 } 395 } 396 catch 397 { 398 #Write-Output $_ 399 } 400 401 # Check number of worker nodes with NotReady/Ready. NotReady will be 1 pt where Ready will be 2. 402 $currentComputePoints = 0 403 404 try 405 { 406 $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/nodes" -SkipCertificateCheck 407 408 $nodes = (ConvertFrom-Json $webrequest.Content).items 409 410 foreach ($node in $nodes) 411 { 412 if ($node.metadata.labels.psobject.properties.name -Contains "node-role.kubernetes.io/worker") 413 { 414 #Write-Output "Checking node $($node.metadata.name)" 415 foreach ($condition in $node.status.conditions) 416 { 417 if ($condition.type -eq "Ready") 418 { 419 #Write-Output "Is node ready? $($condition.status)" 420 if ($condition.status -eq "True") 421 { 422 $currentComputePoints = $currentComputePoints + 2 423 } 424 else 425 { 426 $currentComputePoints++ 427 } 428 } 429 } 430 } 431 } 432 } 433 catch 434 { 435 #Write-Output $_ 436 } 437 438 $maxComputePoints = $compute_count * 2 439 $percentage = ((($currentComputePoints)/$maxComputePoints)*100) 440 if ($percentage -eq 100) 441 { 442 Write-Progress -Id 222 -Status "Completed" -Activity "Worker Ready" -PercentComplete 100 443 break csrLoop 444 } 445 elseif ($percentage -le 100) 446 { 447 Write-Progress -Id 222 -Status "$percentage% Complete" -Activity "Worker Ready" -PercentComplete $percentage 448 } 449 450 if ($csrCount -ge $csrTimeout) 451 { 452 Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow 453 break csrLoop 454 } 455 456 $csrCount++ 457 } 458 459 $progressMsg = "" 460 Write-Progress -Id 111 -Status "1% Complete" -Activity "Install" -PercentComplete 1 461 :installcomplete while ($true) 462 { 463 Start-Sleep -Seconds 30 464 try 465 { 466 $webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/config.openshift.io/v1/clusterversions" -SkipCertificateCheck 467 468 $clusterversions = ConvertFrom-Json $webrequest.Content -AsHashtable 469 470 # just like the installer check the status conditions of the clusterversions config 471 foreach ($condition in $clusterversions['items'][0]['status']['conditions']) 472 { 473 switch ($condition['type']) 474 { 475 "Progressing" { 476 if ($condition['status'] -eq "True") 477 { 478 479 $matchper = ($condition['message'] | Select-String "^Working.*\(([0-9]{1,3})\%.*\)") 480 $matchmsg = ($condition['message'] | Select-String -AllMatches -Pattern "^(Working.*)\:.*") 481 482 # During install, the status of CVO will / may go degraded due to operators going 483 # degraded from taking a while to install. It seems this is the new norm as control 484 # plane takes a while to roll out and certain operators go degraded until the control 485 # plane is stable. 486 if ($matchmsg.Matches.Groups -ne $null) 487 { 488 $progressMsg = $matchmsg.Matches.Groups[1].Value 489 $progressPercent = $matchper.Matches.Groups[1].Value 490 491 Write-Progress -Id 111 -Status "$progressPercent% Complete - $( $progressMsg )" -Activity "Install" -PercentComplete $progressPercent 492 } 493 continue 494 } 495 } 496 "Available" { 497 if ($condition['status'] -eq "True") 498 { 499 Write-Progress -Id 111 -Activity "Install" -Status "Completed" -PercentComplete 100 500 break installcomplete 501 } 502 continue 503 } 504 Default { 505 continue 506 } 507 } 508 } 509 } 510 catch 511 { 512 Write-Output "Unable to check operators" 513 Write-Output $_ 514 } 515 } 516 } 517 518 Get-Job | Remove-Job 519 520 Disconnect-VIServer -Server $vcenter -Force:$true -Confirm:$false 521 522 Write-Output "Install Complete!"