github.com/afbjorklund/moby@v20.10.5+incompatible/hack/ci/windows.ps1 (about)

     1  # WARNING: When editing this file, consider submitting a PR to
     2  # https://github.com/kevpar/docker-w2wCIScripts/blob/master/runCI/executeCI.ps1, and make sure that
     3  # https://github.com/kevpar/docker-w2wCIScripts/blob/master/runCI/Invoke-DockerCI.ps1 isn't broken.
     4  # Validate using a test context in Jenkins, then copy/paste into Jenkins production.
     5  #
     6  # Jenkins CI scripts for Windows to Windows CI (Powershell Version)
     7  # By John Howard (@jhowardmsft) January 2016 - bash version; July 2016 Ported to PowerShell
     8  
     9  $ErrorActionPreference = 'Stop'
    10  $StartTime=Get-Date
    11  
    12  # Put up top to be blindingly obvious. The production jenkins.dockerproject.org Linux-container 
    13  # CI job is "Docker-PRs-LoW-RS3". Force into LCOW mode for this run, or not.
    14  if ($env:BUILD_TAG -match "-LoW") { $env:LCOW_MODE=1 }
    15  if ($env:BUILD_TAG -match "-WoW") { $env:LCOW_MODE="" }
    16  
    17  
    18  Write-Host -ForegroundColor Red "DEBUG: print all environment variables to check how Jenkins runs this script"
    19  $allArgs = [Environment]::GetCommandLineArgs()
    20  Write-Host -ForegroundColor Red $allArgs
    21  Write-Host -ForegroundColor Red "----------------------------------------------------------------------------"
    22  
    23  # -------------------------------------------------------------------------------------------
    24  # When executed, we rely on four variables being set in the environment:
    25  #
    26  # [The reason for being environment variables rather than parameters is historical. No reason
    27  # why it couldn't be updated.]
    28  #
    29  #    SOURCES_DRIVE       is the drive on which the sources being tested are cloned from.
    30  #                        This should be a straight drive letter, no platform semantics.
    31  #                        For example 'c'
    32  #
    33  #    SOURCES_SUBDIR      is the top level directory under SOURCES_DRIVE where the
    34  #                        sources are cloned to. There are no platform semantics in this
    35  #                        as it does not include slashes. 
    36  #                        For example 'gopath'
    37  #
    38  #                        Based on the above examples, it would be expected that Jenkins
    39  #                        would clone the sources being tested to
    40  #                        SOURCES_DRIVE\SOURCES_SUBDIR\src\github.com\docker\docker, or
    41  #                        c:\gopath\src\github.com\docker\docker
    42  #
    43  #    TESTRUN_DRIVE       is the drive where we build the binary on and redirect everything
    44  #                        to for the daemon under test. On an Azure D2 type host which has
    45  #                        an SSD temporary storage D: drive, this is ideal for performance.
    46  #                        For example 'd'
    47  #
    48  #    TESTRUN_SUBDIR      is the top level directory under TESTRUN_DRIVE where we redirect
    49  #                        everything to for the daemon under test. For example 'CI'.
    50  #                        Hence, the daemon under test is run under
    51  #                        TESTRUN_DRIVE\TESTRUN_SUBDIR\CI-<CommitID> or
    52  #                        d:\CI\CI-<CommitID>
    53  #
    54  # Optional environment variables help in CI:
    55  #
    56  #    BUILD_NUMBER + BRANCH_NAME   are optional variables to be added to the directory below TESTRUN_SUBDIR
    57  #                        to have individual folder per CI build. If some files couldn't be
    58  #                        cleaned up and we want to re-run the build in CI.
    59  #                        Hence, the daemon under test is run under
    60  #                        TESTRUN_DRIVE\TESTRUN_SUBDIR\PR-<PR-Number>\<BuildNumber> or
    61  #                        d:\CI\PR-<PR-Number>\<BuildNumber>
    62  #
    63  # In addition, the following variables can control the run configuration:
    64  #
    65  #    DOCKER_DUT_DEBUG         if defined starts the daemon under test in debug mode.
    66  #
    67  #   DOCKER_STORAGE_OPTS       comma-separated list of optional storage driver options for the daemon under test
    68  #                             examples:
    69  #                             DOCKER_STORAGE_OPTS="size=40G"
    70  #                             DOCKER_STORAGE_OPTS="lcow.globalmode=false,lcow.kernel=kernel.efi"
    71  #
    72  #    SKIP_VALIDATION_TESTS    if defined skips the validation tests
    73  #
    74  #    SKIP_UNIT_TESTS          if defined skips the unit tests
    75  #
    76  #    SKIP_INTEGRATION_TESTS   if defined skips the integration tests
    77  #
    78  #    SKIP_COPY_GO             if defined skips copy the go installer from the image
    79  #
    80  #    DOCKER_DUT_HYPERV        if default daemon under test default isolation is hyperv
    81  #
    82  #    INTEGRATION_TEST_NAME    to only run partial tests eg "TestInfo*" will only run
    83  #                             any tests starting "TestInfo"
    84  #
    85  #    SKIP_BINARY_BUILD        if defined skips building the binary
    86  #
    87  #    SKIP_ZAP_DUT             if defined doesn't zap the daemon under test directory
    88  #
    89  #    SKIP_IMAGE_BUILD         if defined doesn't build the 'docker' image
    90  #
    91  #    INTEGRATION_IN_CONTAINER if defined, runs the integration tests from inside a container.
    92  #                             As of July 2016, there are known issues with this. 
    93  #
    94  #    SKIP_ALL_CLEANUP         if defined, skips any cleanup at the start or end of the run
    95  #
    96  #    WINDOWS_BASE_IMAGE       if defined, uses that as the base image. Note that the
    97  #                             docker integration tests are also coded to use the same
    98  #                             environment variable, and if no set, defaults to microsoft/windowsservercore
    99  #
   100  #    WINDOWS_BASE_IMAGE_TAG   if defined, uses that as the tag name for the base image.
   101  #                             if no set, defaults to latest
   102  #
   103  #    LCOW_BASIC_MODE          if defined, does very basic LCOW verification. Ultimately we 
   104  #                             want to run the entire CI suite from docker, but that's a way off.
   105  #                            
   106  #    LCOW_MODE                if defined, runs the entire CI suite
   107  #                            
   108  # -------------------------------------------------------------------------------------------
   109  #
   110  # Jenkins Integration. Add a Windows Powershell build step as follows:
   111  #
   112  #    Write-Host -ForegroundColor green "INFO: Jenkins build step starting"
   113  #    $CISCRIPT_DEFAULT_LOCATION = "https://raw.githubusercontent.com/moby/moby/master/hack/ci/windows.ps1"
   114  #    $CISCRIPT_LOCAL_LOCATION = "$env:TEMP\executeCI.ps1"
   115  #    Write-Host -ForegroundColor green "INFO: Removing cached execution script"
   116  #    Remove-Item $CISCRIPT_LOCAL_LOCATION -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   117  #    $wc = New-Object net.webclient
   118  #    try {
   119  #        Write-Host -ForegroundColor green "INFO: Downloading latest execution script..."
   120  #        $wc.Downloadfile($CISCRIPT_DEFAULT_LOCATION, $CISCRIPT_LOCAL_LOCATION)
   121  #    } 
   122  #    catch [System.Net.WebException]
   123  #    {
   124  #        Throw ("Failed to download: $_")
   125  #    }
   126  #    & $CISCRIPT_LOCAL_LOCATION
   127  # -------------------------------------------------------------------------------------------
   128  
   129  
   130  $SCRIPT_VER="05-Feb-2019 09:03 PDT" 
   131  $FinallyColour="Cyan"
   132  
   133  #$env:DOCKER_DUT_DEBUG="yes" # Comment out to not be in debug mode
   134  #$env:SKIP_UNIT_TESTS="yes"
   135  #$env:SKIP_VALIDATION_TESTS="yes"
   136  #$env:SKIP_ZAP_DUT=""
   137  #$env:SKIP_BINARY_BUILD="yes"
   138  #$env:INTEGRATION_TEST_NAME=""
   139  #$env:SKIP_IMAGE_BUILD="yes"
   140  #$env:SKIP_ALL_CLEANUP="yes"
   141  #$env:INTEGRATION_IN_CONTAINER="yes"
   142  #$env:WINDOWS_BASE_IMAGE=""
   143  #$env:SKIP_COPY_GO="yes"
   144  #$env:INTEGRATION_TESTFLAGS="-test.v"
   145  
   146  Function Nuke-Everything {
   147      $ErrorActionPreference = 'SilentlyContinue'
   148  
   149      try {
   150  
   151          if ($null -eq $env:SKIP_ALL_CLEANUP) {
   152              Write-Host -ForegroundColor green "INFO: Nuke-Everything..."
   153              $containerCount = ($(docker ps -aq | Measure-Object -line).Lines) 
   154              if (-not $LastExitCode -eq 0) {
   155                  Throw "ERROR: Failed to get container count from control daemon while nuking"
   156              }
   157  
   158              Write-Host -ForegroundColor green "INFO: Container count on control daemon to delete is $containerCount"
   159              if ($(docker ps -aq | Measure-Object -line).Lines -gt 0) {
   160                  docker rm -f $(docker ps -aq)
   161              }
   162  
   163              $allImages  = $(docker images --format "{{.Repository}}#{{.ID}}")
   164              $toRemove   = ($allImages | Select-String -NotMatch "servercore","nanoserver","docker","busybox")
   165              $imageCount = ($toRemove | Measure-Object -line).Lines
   166  
   167              if ($imageCount -gt 0) {
   168                  Write-Host -Foregroundcolor green "INFO: Non-base image count on control daemon to delete is $imageCount"
   169                  docker rmi -f ($toRemove | Foreach-Object { $_.ToString().Split("#")[1] })
   170              }
   171          } else {
   172              Write-Host -ForegroundColor Magenta "WARN: Skipping cleanup of images and containers"
   173          }
   174  
   175          # Kill any spurious daemons. The '-' is IMPORTANT otherwise will kill the control daemon!
   176          $pids=$(get-process | where-object {$_.ProcessName -like 'dockerd-*'}).id
   177          foreach ($p in $pids) {
   178              Write-Host "INFO: Killing daemon with PID $p"
   179              Stop-Process -Id $p -Force -ErrorAction SilentlyContinue
   180          }
   181  
   182          if ($null -ne $pidFile) {
   183              Write-Host "INFO: Tidying pidfile $pidfile"
   184               if (Test-Path $pidFile) {
   185                  $p=Get-Content $pidFile -raw
   186                  if ($null -ne $p){
   187                      Write-Host -ForegroundColor green "INFO: Stopping possible daemon pid $p"
   188                      taskkill -f -t -pid $p
   189                  }
   190                  Remove-Item "$env:TEMP\docker.pid" -force -ErrorAction SilentlyContinue
   191              }
   192          }
   193  
   194          Stop-Process -name "cc1" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   195          Stop-Process -name "link" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   196          Stop-Process -name "compile" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   197          Stop-Process -name "ld" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   198          Stop-Process -name "go" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   199          Stop-Process -name "git" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   200          Stop-Process -name "git-remote-https" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   201          Stop-Process -name "integration-cli.test" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   202          Stop-Process -name "tail" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null
   203  
   204          # Detach any VHDs
   205          gwmi msvm_mountedstorageimage -namespace root/virtualization/v2 -ErrorAction SilentlyContinue | foreach-object {$_.DetachVirtualHardDisk() }
   206  
   207          # Stop any compute processes
   208          Get-ComputeProcess | Stop-ComputeProcess -Force
   209  
   210          # Delete the directory using our dangerous utility unless told not to
   211          if (Test-Path "$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR") {
   212              if (($null -ne $env:SKIP_ZAP_DUT) -or ($null -eq $env:SKIP_ALL_CLEANUP)) {
   213                  Write-Host -ForegroundColor Green "INFO: Nuking $env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR"
   214                  docker-ci-zap "-folder=$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR"
   215              } else {
   216                  Write-Host -ForegroundColor Magenta "WARN: Skip nuking $env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR"
   217              }
   218          }
   219  
   220          # TODO: This should be able to be removed in August 2017 update. Only needed for RS1  Production Server workaround - Psched
   221          $reg = "HKLM:\System\CurrentControlSet\Services\Psched\Parameters\NdisAdapters"
   222          $count=(Get-ChildItem $reg | Measure-Object).Count
   223          if ($count -gt 0) {
   224              Write-Warning "There are $count NdisAdapters leaked under Psched\Parameters"
   225              Write-Warning "Cleaning Psched..."
   226              Get-ChildItem $reg | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue | Out-Null
   227          }
   228  
   229          # TODO: This should be able to be removed in August 2017 update. Only needed for RS1
   230          $reg = "HKLM:\System\CurrentControlSet\Services\WFPLWFS\Parameters\NdisAdapters"
   231          $count=(Get-ChildItem $reg | Measure-Object).Count
   232          if ($count -gt 0) {
   233              Write-Warning "There are $count NdisAdapters leaked under WFPLWFS\Parameters"
   234              Write-Warning "Cleaning WFPLWFS..."
   235              Get-ChildItem $reg | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue | Out-Null
   236          }
   237      } catch {
   238          # Don't throw any errors onwards Throw $_
   239      }
   240  }
   241  
   242  Try {
   243      Write-Host -ForegroundColor Cyan "`nINFO: executeCI.ps1 starting at $(date)`n"
   244      Write-Host  -ForegroundColor Green "INFO: Script version $SCRIPT_VER"
   245      Set-PSDebug -Trace 0  # 1 to turn on
   246      $origPath="$env:PATH"            # so we can restore it at the end
   247      $origDOCKER_HOST="$DOCKER_HOST"  # So we can restore it at the end
   248      $origGOROOT="$env:GOROOT"        # So we can restore it at the end
   249      $origGOPATH="$env:GOPATH"        # So we can restore it at the end
   250  
   251      # Turn off progress bars
   252  	$origProgressPreference=$global:ProgressPreference
   253  	$global:ProgressPreference='SilentlyContinue'
   254  
   255      # Git version
   256      Write-Host  -ForegroundColor Green "INFO: Running $(git version)"
   257  
   258      # OS Version
   259      $bl=(Get-ItemProperty -Path "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion"  -Name BuildLabEx).BuildLabEx
   260      $a=$bl.ToString().Split(".")
   261      $Branch=$a[3]
   262      $WindowsBuild=$a[0]+"."+$a[1]+"."+$a[4]
   263      Write-Host -ForegroundColor green "INFO: Branch:$Branch Build:$WindowsBuild"
   264  
   265      # List the environment variables
   266      Write-Host -ForegroundColor green "INFO: Environment variables:"
   267      Get-ChildItem Env: | Out-String
   268  
   269      # PR
   270      if (-not ($null -eq $env:PR)) { Write-Output "INFO: PR#$env:PR (https://github.com/docker/docker/pull/$env:PR)" }
   271  
   272      # Make sure docker is installed
   273      if ($null -eq (Get-Command "docker" -ErrorAction SilentlyContinue)) { Throw "ERROR: docker is not installed or not found on path" }
   274  
   275      # Make sure docker-ci-zap is installed
   276      if ($null -eq (Get-Command "docker-ci-zap" -ErrorAction SilentlyContinue)) { Throw "ERROR: docker-ci-zap is not installed or not found on path" }
   277  
   278      # Make sure Windows Defender is disabled
   279      $defender = $false
   280      Try {
   281        $status = Get-MpComputerStatus
   282        if ($status) {
   283          if ($status.RealTimeProtectionEnabled) {
   284            $defender = $true
   285          }
   286        }
   287      } Catch {}
   288      if ($defender) { Write-Host -ForegroundColor Magenta "WARN: Windows Defender real time protection is enabled, which may cause some integration tests to fail" }
   289  
   290      # Make sure SOURCES_DRIVE is set
   291      if ($null -eq $env:SOURCES_DRIVE) { Throw "ERROR: Environment variable SOURCES_DRIVE is not set" }
   292  
   293      # Make sure TESTRUN_DRIVE is set
   294      if ($null -eq $env:TESTRUN_DRIVE) { Throw "ERROR: Environment variable TESTRUN_DRIVE is not set" }
   295  
   296      # Make sure SOURCES_SUBDIR is set
   297      if ($null -eq $env:SOURCES_SUBDIR) { Throw "ERROR: Environment variable SOURCES_SUBDIR is not set" }
   298  
   299      # Make sure TESTRUN_SUBDIR is set
   300      if ($null -eq $env:TESTRUN_SUBDIR) { Throw "ERROR: Environment variable TESTRUN_SUBDIR is not set" }
   301  
   302      # SOURCES_DRIVE\SOURCES_SUBDIR must be a directory and exist
   303      if (-not (Test-Path -PathType Container "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR")) { Throw "ERROR: $env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR must be an existing directory" }
   304  
   305      # Create the TESTRUN_DRIVE\TESTRUN_SUBDIR if it does not already exist
   306      New-Item -ItemType Directory -Force -Path "$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR" -ErrorAction SilentlyContinue | Out-Null
   307  
   308      Write-Host  -ForegroundColor Green "INFO: Sources under $env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\..."
   309      Write-Host  -ForegroundColor Green "INFO: Test run under $env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR\..."
   310  
   311      # Check the intended source location is a directory
   312      if (-not (Test-Path -PathType Container "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker" -ErrorAction SilentlyContinue)) {
   313          Throw "ERROR: $env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker is not a directory!"
   314      }
   315  
   316      # Make sure we start at the root of the sources
   317      Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
   318      Write-Host  -ForegroundColor Green "INFO: Running in $(Get-Location)"
   319  
   320      # Make sure we are in repo
   321      if (-not (Test-Path -PathType Leaf -Path ".\Dockerfile.windows")) {
   322          Throw "$(Get-Location) does not contain Dockerfile.windows!"
   323      }
   324      Write-Host  -ForegroundColor Green "INFO: docker/docker repository was found"
   325  
   326      # Make sure microsoft/windowsservercore:latest image is installed in the control daemon. On public CI machines, windowsservercore.tar and nanoserver.tar
   327      # are pre-baked and tagged appropriately in the c:\baseimages directory, and can be directly loaded. 
   328      # Note - this script will only work on 10B (Oct 2016) or later machines! Not 9D or previous due to image tagging assumptions.
   329      #
   330      # On machines not on Microsoft corpnet, or those which have not been pre-baked, we have to docker pull the image in which case it will
   331      # will come in directly as microsoft/windowsservercore:latest. The ultimate goal of all this code is to ensure that whatever,
   332      # we have microsoft/windowsservercore:latest
   333      #
   334      # Note we cannot use (as at Oct 2016) nanoserver as the control daemons base image, even if nanoserver is used in the tests themselves.
   335  
   336      $ErrorActionPreference = "SilentlyContinue"
   337      $ControlDaemonBaseImage="windowsservercore"
   338  
   339      $readBaseFrom="c"
   340      if ($((docker images --format "{{.Repository}}:{{.Tag}}" | Select-String $("microsoft/"+$ControlDaemonBaseImage+":latest") | Measure-Object -Line).Lines) -eq 0) {
   341          # Try the internal azure CI image version or Microsoft internal corpnet where the base image is already pre-prepared on the disk,
   342          # either through Invoke-DockerCI or, in the case of Azure CI servers, baked into the VHD at the same location.
   343          if (Test-Path $("$env:SOURCES_DRIVE`:\baseimages\"+$ControlDaemonBaseImage+".tar")) {
   344              # An optimization for CI servers to copy it to the D: drive which is an SSD.
   345              if ($env:SOURCES_DRIVE -ne $env:TESTRUN_DRIVE) {
   346                  $readBaseFrom=$env:TESTRUN_DRIVE
   347                  if (!(Test-Path "$env:TESTRUN_DRIVE`:\baseimages")) {
   348                      New-Item "$env:TESTRUN_DRIVE`:\baseimages" -type directory | Out-Null
   349                  }
   350                  if (!(Test-Path "$env:TESTRUN_DRIVE`:\baseimages\windowsservercore.tar")) {
   351                      if (Test-Path "$env:SOURCES_DRIVE`:\baseimages\windowsservercore.tar") {
   352                          Write-Host -ForegroundColor Green "INFO: Optimisation - copying $env:SOURCES_DRIVE`:\baseimages\windowsservercore.tar to $env:TESTRUN_DRIVE`:\baseimages"
   353                          Copy-Item "$env:SOURCES_DRIVE`:\baseimages\windowsservercore.tar" "$env:TESTRUN_DRIVE`:\baseimages"
   354                      }
   355                  }
   356                  if (!(Test-Path "$env:TESTRUN_DRIVE`:\baseimages\nanoserver.tar")) {
   357                      if (Test-Path "$env:SOURCES_DRIVE`:\baseimages\nanoserver.tar") {
   358                          Write-Host -ForegroundColor Green "INFO: Optimisation - copying $env:SOURCES_DRIVE`:\baseimages\nanoserver.tar to $env:TESTRUN_DRIVE`:\baseimages"
   359                          Copy-Item "$env:SOURCES_DRIVE`:\baseimages\nanoserver.tar" "$env:TESTRUN_DRIVE`:\baseimages"
   360                      }
   361                  }
   362                  $readBaseFrom=$env:TESTRUN_DRIVE
   363              }
   364              Write-Host  -ForegroundColor Green "INFO: Loading"$ControlDaemonBaseImage".tar from disk. This may take some time..."
   365              $ErrorActionPreference = "SilentlyContinue"
   366              docker load -i $("$readBaseFrom`:\baseimages\"+$ControlDaemonBaseImage+".tar")
   367              $ErrorActionPreference = "Stop"
   368              if (-not $LastExitCode -eq 0) {
   369                  Throw $("ERROR: Failed to load $readBaseFrom`:\baseimages\"+$ControlDaemonBaseImage+".tar")
   370              }
   371              Write-Host -ForegroundColor Green "INFO: docker load of"$ControlDaemonBaseImage" completed successfully"
   372          } else {
   373              # We need to docker pull it instead. It will come in directly as microsoft/imagename:latest
   374              Write-Host -ForegroundColor Green $("INFO: Pulling $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG from docker hub. This may take some time...")
   375              $ErrorActionPreference = "SilentlyContinue"
   376              docker pull "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG"
   377              $ErrorActionPreference = "Stop"
   378              if (-not $LastExitCode -eq 0) {
   379                  Throw $("ERROR: Failed to docker pull $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG.")
   380              }
   381              Write-Host -ForegroundColor Green $("INFO: docker pull of $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG completed successfully")
   382              Write-Host -ForegroundColor Green $("INFO: Tagging $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG as microsoft/$ControlDaemonBaseImage")
   383              docker tag "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG" microsoft/$ControlDaemonBaseImage
   384          }
   385      } else {
   386          Write-Host -ForegroundColor Green "INFO: Image"$("microsoft/"+$ControlDaemonBaseImage+":latest")"is already loaded in the control daemon"
   387      }
   388  
   389      # Inspect the pulled image to get the version directly
   390      $ErrorActionPreference = "SilentlyContinue"
   391      $imgVersion = $(docker inspect  $("microsoft/"+$ControlDaemonBaseImage) --format "{{.OsVersion}}")
   392      $ErrorActionPreference = "Stop"
   393      Write-Host -ForegroundColor Green $("INFO: Version of microsoft/"+$ControlDaemonBaseImage+":latest is '"+$imgVersion+"'")
   394  
   395      # Provide the docker version for debugging purposes.
   396      Write-Host  -ForegroundColor Green "INFO: Docker version of control daemon"
   397      Write-Host
   398      $ErrorActionPreference = "SilentlyContinue"
   399      docker version
   400      $ErrorActionPreference = "Stop"
   401      if (-not($LastExitCode -eq 0)) {
   402          Write-Host 
   403          Write-Host  -ForegroundColor Green "---------------------------------------------------------------------------"
   404          Write-Host  -ForegroundColor Green " Failed to get a response from the control daemon. It may be down."
   405          Write-Host  -ForegroundColor Green " Try re-running this CI job, or ask on #docker-maintainers on docker slack"
   406          Write-Host  -ForegroundColor Green " to see if the daemon is running. Also check the service configuration."
   407          Write-Host  -ForegroundColor Green " DOCKER_HOST is set to $DOCKER_HOST."
   408          Write-Host  -ForegroundColor Green "---------------------------------------------------------------------------"
   409          Write-Host 
   410          Throw "ERROR: The control daemon does not appear to be running."
   411      }
   412      Write-Host
   413  
   414      # Same as above, but docker info
   415      Write-Host  -ForegroundColor Green "INFO: Docker info of control daemon"
   416      Write-Host
   417      $ErrorActionPreference = "SilentlyContinue"
   418      docker info
   419      $ErrorActionPreference = "Stop"
   420      if (-not($LastExitCode -eq 0)) {
   421          Throw "ERROR: The control daemon does not appear to be running."
   422      }
   423      Write-Host
   424  
   425      # Get the commit has and verify we have something
   426      $ErrorActionPreference = "SilentlyContinue"
   427      $COMMITHASH=$(git rev-parse --short HEAD)
   428      $ErrorActionPreference = "Stop"
   429      if (-not($LastExitCode -eq 0)) {
   430          Throw "ERROR: Failed to get commit hash. Are you sure this is a docker repository?"
   431      }
   432      Write-Host  -ForegroundColor Green "INFO: Commit hash is $COMMITHASH"
   433  
   434      # Nuke everything and go back to our sources after
   435      Nuke-Everything
   436      cd "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
   437  
   438      # Redirect to a temporary location. 
   439      $TEMPORIG=$env:TEMP
   440      if ($null -eq $env:BUILD_NUMBER) {
   441        $env:TEMP="$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR\CI-$COMMITHASH"
   442      } else {
   443        # individual temporary location per CI build that better matches the BUILD_URL
   444        $env:TEMP="$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR\$env:BRANCH_NAME\$env:BUILD_NUMBER"
   445      }
   446      $env:LOCALAPPDATA="$env:TEMP\localappdata"
   447      $errorActionPreference='Stop'
   448      New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
   449      New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue  | Out-Null
   450      New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue  | Out-Null
   451      New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue  | Out-Null
   452      New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null
   453      New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null
   454      New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null
   455      if ($null -eq $env:SKIP_COPY_GO) {
   456          # Wipe the previous version of GO - we're going to get it out of the image
   457          if (Test-Path "$env:TEMP\go") { Remove-Item "$env:TEMP\go" -Recurse -Force -ErrorAction SilentlyContinue | Out-Null }
   458          New-Item -ItemType Directory "$env:TEMP\go" -ErrorAction SilentlyContinue | Out-Null
   459      }
   460  
   461      Write-Host -ForegroundColor Green "INFO: Location for testing is $env:TEMP"
   462  
   463      # CI Integrity check - ensure Dockerfile.windows and Dockerfile go versions match
   464      $goVersionDockerfileWindows=(Select-String -Path ".\Dockerfile.windows" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value
   465      $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value
   466  
   467      if ($null -eq $goVersionDockerfile) {
   468          Throw "ERROR: Failed to extract golang version from Dockerfile"
   469      }
   470      Write-Host  -ForegroundColor Green "INFO: Validating GOLang consistency in Dockerfile.windows..."
   471      if (-not ($goVersionDockerfile -eq $goVersionDockerfileWindows)) {
   472          Throw "ERROR: Mismatched GO versions between Dockerfile and Dockerfile.windows. Update your PR to ensure that both files are updated and in sync. $goVersionDockerfile $goVersionDockerfileWindows"
   473      }
   474  
   475      # Build the image
   476      if ($null -eq $env:SKIP_IMAGE_BUILD) {
   477          Write-Host  -ForegroundColor Cyan "`n`nINFO: Building the image from Dockerfile.windows at $(Get-Date)..."
   478          Write-Host
   479          $ErrorActionPreference = "SilentlyContinue"
   480          $Duration=$(Measure-Command { docker build --build-arg=GO_VERSION -t docker -f Dockerfile.windows . | Out-Host })
   481          $ErrorActionPreference = "Stop"
   482          if (-not($LastExitCode -eq 0)) {
   483             Throw "ERROR: Failed to build image from Dockerfile.windows"
   484          }
   485          Write-Host  -ForegroundColor Green "INFO: Image build ended at $(Get-Date). Duration`:$Duration"
   486      } else {
   487          Write-Host -ForegroundColor Magenta "WARN: Skipping building the docker image"
   488      }
   489  
   490      # Following at the moment must be docker\docker as it's dictated by dockerfile.Windows
   491      $contPath="$COMMITHASH`:c`:\gopath\src\github.com\docker\docker\bundles"
   492  
   493      # After https://github.com/docker/docker/pull/30290, .git was added to .dockerignore. Therefore
   494      # we have to calculate unsupported outside of the container, and pass the commit ID in through
   495      # an environment variable for the binary build
   496      $CommitUnsupported=""
   497      if ($(git status --porcelain --untracked-files=no).Length -ne 0) {
   498          $CommitUnsupported="-unsupported"
   499      }
   500  
   501      # Build the binary in a container unless asked to skip it.
   502      if ($null -eq $env:SKIP_BINARY_BUILD) {
   503          Write-Host  -ForegroundColor Cyan "`n`nINFO: Building the test binaries at $(Get-Date)..."
   504          $ErrorActionPreference = "SilentlyContinue"
   505          docker rm -f $COMMITHASH 2>&1 | Out-Null
   506          if ($CommitUnsupported -ne "") {
   507              Write-Host ""
   508              Write-Warning "This version is unsupported because there are uncommitted file(s)."
   509              Write-Warning "Either commit these changes, or add them to .gitignore."
   510              git status --porcelain --untracked-files=no | Write-Warning
   511               Write-Host ""
   512          }
   513          $Duration=$(Measure-Command {docker run --name $COMMITHASH -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -Daemon -Client | Out-Host })
   514          $ErrorActionPreference = "Stop"
   515          if (-not($LastExitCode -eq 0)) {
   516              Throw "ERROR: Failed to build binary"
   517          }
   518          Write-Host  -ForegroundColor Green "INFO: Binaries build ended at $(Get-Date). Duration`:$Duration"
   519  
   520          # Copy the binaries and the generated version_autogen.go out of the container
   521          $ErrorActionPreference = "SilentlyContinue"
   522          docker cp "$contPath\docker.exe" $env:TEMP\binary\
   523          if (-not($LastExitCode -eq 0)) {
   524              Throw "ERROR: Failed to docker cp the client binary (docker.exe) to $env:TEMP\binary"
   525          }
   526          docker cp "$contPath\dockerd.exe" $env:TEMP\binary\
   527          if (-not($LastExitCode -eq 0)) {
   528              Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary"
   529          }
   530  
   531          docker cp "$COMMITHASH`:c`:\gopath\bin\gotestsum.exe" $env:TEMP\binary\
   532          if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) {
   533              Throw "ERROR: gotestsum.exe not found...." `
   534          }
   535  
   536          $ErrorActionPreference = "Stop"
   537  
   538          # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
   539          Write-Host -ForegroundColor Green "INFO: Copying the built daemon binary to $env:TEMP\binary\dockerd-$COMMITHASH.exe..."
   540          Copy-Item $env:TEMP\binary\dockerd.exe $env:TEMP\binary\dockerd-$COMMITHASH.exe -Force -ErrorAction SilentlyContinue
   541  
   542          # Copy the built docker.exe to docker-$COMMITHASH.exe
   543          Write-Host -ForegroundColor Green "INFO: Copying the built client binary to $env:TEMP\binary\docker-$COMMITHASH.exe..."
   544          Copy-Item $env:TEMP\binary\docker.exe $env:TEMP\binary\docker-$COMMITHASH.exe -Force -ErrorAction SilentlyContinue
   545  
   546      } else {
   547          Write-Host -ForegroundColor Magenta "WARN: Skipping building the binaries"
   548      }
   549  
   550      Write-Host -ForegroundColor Green "INFO: Copying dockerversion from the container..."
   551      $ErrorActionPreference = "SilentlyContinue"
   552      docker cp "$contPath\..\dockerversion\version_autogen.go" "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\dockerversion"
   553      if (-not($LastExitCode -eq 0)) {
   554           Throw "ERROR: Failed to docker cp the generated version_autogen.go to $env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\dockerversion"
   555      }
   556      $ErrorActionPreference = "Stop"
   557  
   558      # Grab the golang installer out of the built image. That way, we know we are consistent once extracted and paths set,
   559      # so there's no need to re-deploy on account of an upgrade to the version of GO being used in docker.
   560      if ($null -eq $env:SKIP_COPY_GO) {
   561          Write-Host -ForegroundColor Green "INFO: Copying the golang package from the container to $env:TEMP\installer\go.zip..."
   562          docker cp "$COMMITHASH`:c`:\go.zip" $env:TEMP\installer\
   563          if (-not($LastExitCode -eq 0)) {
   564              Throw "ERROR: Failed to docker cp the golang installer 'go.zip' from container:c:\go.zip to $env:TEMP\installer"
   565          }
   566          $ErrorActionPreference = "Stop"
   567  
   568          # Extract the golang installer
   569          Write-Host -ForegroundColor Green "INFO: Extracting go.zip to $env:TEMP\go"
   570          $Duration=$(Measure-Command { Expand-Archive $env:TEMP\installer\go.zip $env:TEMP -Force | Out-Null})
   571          Write-Host  -ForegroundColor Green "INFO: Extraction ended at $(Get-Date). Duration`:$Duration"    
   572      } else {
   573          Write-Host -ForegroundColor Magenta "WARN: Skipping copying and extracting golang from the image"
   574      }
   575  
   576      # Set the GOPATH
   577      Write-Host -ForegroundColor Green "INFO: Updating the golang and path environment variables"
   578      $env:GOPATH="$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR"
   579      Write-Host -ForegroundColor Green "INFO: GOPATH=$env:GOPATH"
   580  
   581      # Set the path to have the version of go from the image at the front
   582      $env:PATH="$env:TEMP\go\bin;$env:PATH"
   583  
   584      # Set the GOROOT to be our copy of go from the image
   585      $env:GOROOT="$env:TEMP\go"
   586      Write-Host -ForegroundColor Green "INFO: $(go version)"
   587      
   588      # Work out the -H parameter for the daemon under test (DASHH_DUT) and client under test (DASHH_CUT)
   589      #$DASHH_DUT="npipe:////./pipe/$COMMITHASH" # Can't do remote named pipe
   590      #$ip = (resolve-dnsname $env:COMPUTERNAME -type A -NoHostsFile -LlmnrNetbiosOnly).IPAddress # Useful to tie down
   591      $DASHH_CUT="tcp://127.0.0.1`:2357"    # Not a typo for 2375!
   592      $DASHH_DUT="tcp://0.0.0.0:2357"       # Not a typo for 2375!
   593  
   594      # Arguments for the daemon under test
   595      $dutArgs=@()
   596      $dutArgs += "-H $DASHH_DUT"
   597      $dutArgs += "--data-root $env:TEMP\daemon"
   598      $dutArgs += "--pidfile $env:TEMP\docker.pid"
   599  
   600      # Save the PID file so we can nuke it if set
   601      $pidFile="$env:TEMP\docker.pid"
   602  
   603      # Arguments: Are we starting the daemon under test in debug mode?
   604      if (-not ("$env:DOCKER_DUT_DEBUG" -eq "")) {
   605          Write-Host -ForegroundColor Green "INFO: Running the daemon under test in debug mode"
   606          $dutArgs += "-D"
   607      }
   608  
   609      # Arguments: Are we starting the daemon under test with Hyper-V containers as the default isolation?
   610      if (-not ("$env:DOCKER_DUT_HYPERV" -eq "")) {
   611          Write-Host -ForegroundColor Green "INFO: Running the daemon under test with Hyper-V containers as the default"
   612          $dutArgs += "--exec-opt isolation=hyperv"
   613      }
   614  
   615      # Arguments: Allow setting optional storage-driver options
   616      # example usage: DOCKER_STORAGE_OPTS="lcow.globalmode=false,lcow.kernel=kernel.efi"
   617      if (-not ("$env:DOCKER_STORAGE_OPTS" -eq "")) {
   618          Write-Host -ForegroundColor Green "INFO: Running the daemon under test with storage-driver options ${env:DOCKER_STORAGE_OPTS}"
   619          $env:DOCKER_STORAGE_OPTS.Split(",") | ForEach {
   620              $dutArgs += "--storage-opt $_"
   621          }
   622      }
   623  
   624      # Start the daemon under test, ensuring everything is redirected to folders under $TEMP.
   625      # Important - we launch the -$COMMITHASH version so that we can kill it without
   626      # killing the control daemon. 
   627      Write-Host -ForegroundColor Green "INFO: Starting a daemon under test..."
   628      Write-Host -ForegroundColor Green "INFO: Args: $dutArgs"
   629      New-Item -ItemType Directory $env:TEMP\daemon -ErrorAction SilentlyContinue  | Out-Null
   630  
   631      # In LCOW mode, for now we need to set an environment variable before starting the daemon under test
   632      if (($null -ne $env:LCOW_MODE) -or ($null -ne $env:LCOW_BASIC_MODE)) {
   633          $env:LCOW_SUPPORTED=1
   634      }
   635  
   636      # Cannot fathom why, but always writes to stderr....
   637      Start-Process "$env:TEMP\binary\dockerd-$COMMITHASH" `
   638                    -ArgumentList $dutArgs `
   639                    -RedirectStandardOutput "$env:TEMP\dut.out" `
   640                    -RedirectStandardError "$env:TEMP\dut.err" 
   641      Write-Host -ForegroundColor Green "INFO: Process started successfully."
   642      $daemonStarted=1
   643  
   644      # In LCOW mode, turn off that variable
   645      if (($null -ne $env:LCOW_MODE) -or ($null -ne $env:LCOW_BASIC_MODE)) {
   646          $env:LCOW_SUPPORTED=""
   647      }
   648  
   649  
   650      # Start tailing the daemon under test if the command is installed
   651      if ($null -ne (Get-Command "tail" -ErrorAction SilentlyContinue)) {
   652          Write-Host -ForegroundColor green "INFO: Start tailing logs of the daemon under tests"
   653          $tail = Start-Process "tail" -ArgumentList "-f $env:TEMP\dut.out" -PassThru -ErrorAction SilentlyContinue
   654      }
   655  
   656      # Verify we can get the daemon under test to respond 
   657      $tries=20
   658      Write-Host -ForegroundColor Green "INFO: Waiting for the daemon under test to start..."
   659      while ($true) {
   660          $ErrorActionPreference = "SilentlyContinue"
   661          & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" version 2>&1 | Out-Null
   662          $ErrorActionPreference = "Stop"
   663          if ($LastExitCode -eq 0) {
   664              break
   665          }
   666  
   667          $tries--
   668          if ($tries -le 0) {
   669              Throw "ERROR: Failed to get a response from the daemon under test"
   670          }
   671          Write-Host -NoNewline "."
   672          sleep 1
   673      }
   674      Write-Host -ForegroundColor Green "INFO: Daemon under test started and replied!"
   675  
   676      # Provide the docker version of the daemon under test for debugging purposes.
   677      Write-Host -ForegroundColor Green "INFO: Docker version of the daemon under test"
   678      Write-Host 
   679      $ErrorActionPreference = "SilentlyContinue"
   680      & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" version
   681      $ErrorActionPreference = "Stop"
   682      if ($LastExitCode -ne 0) {
   683          Throw "ERROR: The daemon under test does not appear to be running."
   684      }
   685      Write-Host
   686  
   687      # Same as above but docker info
   688      Write-Host -ForegroundColor Green "INFO: Docker info of the daemon under test"
   689      Write-Host 
   690      $ErrorActionPreference = "SilentlyContinue"
   691      & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" info
   692      $ErrorActionPreference = "Stop"
   693      if ($LastExitCode -ne 0) {
   694          Throw "ERROR: The daemon under test does not appear to be running."
   695      }
   696      Write-Host
   697  
   698      # Same as above but docker images
   699      Write-Host -ForegroundColor Green "INFO: Docker images of the daemon under test"
   700      Write-Host 
   701      $ErrorActionPreference = "SilentlyContinue"
   702      & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" images
   703      $ErrorActionPreference = "Stop"
   704      if ($LastExitCode -ne 0) {
   705          Throw "ERROR: The daemon under test does not appear to be running."
   706      }
   707      Write-Host
   708  
   709      # Don't need Windows images when in LCOW mode.
   710      if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
   711  
   712          # Default to windowsservercore for the base image used for the tests. The "docker" image
   713          # and the control daemon use microsoft/windowsservercore regardless. This is *JUST* for the tests.
   714          if ($null -eq $env:WINDOWS_BASE_IMAGE) {
   715              $env:WINDOWS_BASE_IMAGE="microsoft/windowsservercore"
   716          }
   717          if ($null -eq $env:WINDOWS_BASE_IMAGE_TAG) {
   718              $env:WINDOWS_BASE_IMAGE_TAG="latest"
   719          }
   720  
   721          # Lowercase and make sure it has a microsoft/ prefix
   722          $env:WINDOWS_BASE_IMAGE = $env:WINDOWS_BASE_IMAGE.ToLower()
   723          if (! $($env:WINDOWS_BASE_IMAGE -Split "/")[0] -match "microsoft") {
   724              Throw "ERROR: WINDOWS_BASE_IMAGE should start microsoft/ or mcr.microsoft.com/"
   725          }
   726  
   727          Write-Host -ForegroundColor Green "INFO: Base image for tests is $env:WINDOWS_BASE_IMAGE"
   728  
   729          $ErrorActionPreference = "SilentlyContinue"
   730          if ($((& "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" images --format "{{.Repository}}:{{.Tag}}" | Select-String "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG" | Measure-Object -Line).Lines) -eq 0) {
   731              # Try the internal azure CI image version or Microsoft internal corpnet where the base image is already pre-prepared on the disk,
   732              # either through Invoke-DockerCI or, in the case of Azure CI servers, baked into the VHD at the same location.
   733              if (Test-Path $("c:\baseimages\"+$($env:WINDOWS_BASE_IMAGE -Split "/")[1]+".tar")) {
   734                  Write-Host  -ForegroundColor Green "INFO: Loading"$($env:WINDOWS_BASE_IMAGE -Split "/")[1]".tar from disk into the daemon under test. This may take some time..."
   735                  $ErrorActionPreference = "SilentlyContinue"
   736                  & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" load -i $("$readBaseFrom`:\baseimages\"+$($env:WINDOWS_BASE_IMAGE -Split "/")[1]+".tar")
   737                  $ErrorActionPreference = "Stop"
   738                  if (-not $LastExitCode -eq 0) {
   739                      Throw $("ERROR: Failed to load $readBaseFrom`:\baseimages\"+$($env:WINDOWS_BASE_IMAGE -Split "/")[1]+".tar into daemon under test")
   740                  }
   741                  Write-Host -ForegroundColor Green "INFO: docker load of"$($env:WINDOWS_BASE_IMAGE -Split "/")[1]" into daemon under test completed successfully"
   742              } else {
   743                  # We need to docker pull it instead. It will come in directly as microsoft/imagename:tagname
   744                  Write-Host -ForegroundColor Green $("INFO: Pulling "+$env:WINDOWS_BASE_IMAGE+":"+$env:WINDOWS_BASE_IMAGE_TAG+" from docker hub into daemon under test. This may take some time...")
   745                  $ErrorActionPreference = "SilentlyContinue"
   746                  & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" pull "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG"
   747                  $ErrorActionPreference = "Stop"
   748                  if (-not $LastExitCode -eq 0) {
   749                      Throw $("ERROR: Failed to docker pull $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG into daemon under test.")
   750                  }
   751                  Write-Host -ForegroundColor Green $("INFO: docker pull of $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG into daemon under test completed successfully")
   752                  Write-Host -ForegroundColor Green $("INFO: Tagging $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG as microsoft/$ControlDaemonBaseImage in daemon under test")
   753                  & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" tag "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG" microsoft/$ControlDaemonBaseImage
   754              }
   755          } else {
   756              Write-Host -ForegroundColor Green "INFO: Image $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG is already loaded in the daemon under test"
   757          }
   758      
   759      
   760          # Inspect the pulled or loaded image to get the version directly
   761          $ErrorActionPreference = "SilentlyContinue"
   762          $dutimgVersion = $(&"$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" inspect "$($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG" --format "{{.OsVersion}}")
   763          $ErrorActionPreference = "Stop"
   764          Write-Host -ForegroundColor Green $("INFO: Version of $($env:WINDOWS_BASE_IMAGE):$env:WINDOWS_BASE_IMAGE_TAG is '"+$dutimgVersion+"'")
   765      }
   766  
   767      # Run the validation tests unless SKIP_VALIDATION_TESTS is defined.
   768      if ($null -eq $env:SKIP_VALIDATION_TESTS) {
   769          Write-Host -ForegroundColor Cyan "INFO: Running validation tests at $(Get-Date)..."
   770          $ErrorActionPreference = "SilentlyContinue"
   771          $Duration=$(Measure-Command { hack\make.ps1 -DCO -GoFormat -PkgImports | Out-Host })
   772          $ErrorActionPreference = "Stop"
   773          if (-not($LastExitCode -eq 0)) {
   774              Throw "ERROR: Validation tests failed"
   775          }
   776          Write-Host  -ForegroundColor Green "INFO: Validation tests ended at $(Get-Date). Duration`:$Duration"
   777      } else {
   778          Write-Host -ForegroundColor Magenta "WARN: Skipping validation tests"
   779      }
   780  
   781      # Note the unit tests won't work in LCOW mode as I turned off loading the base images above.
   782      # Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined
   783      if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
   784          if ($null -eq $env:SKIP_UNIT_TESTS) {
   785              $ContainerNameForUnitTests = $COMMITHASH + "_UnitTests"
   786              Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..."
   787              $ErrorActionPreference = "SilentlyContinue"
   788              $Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
   789              $TestRunExitCode = $LastExitCode
   790              $ErrorActionPreference = "Stop"
   791  
   792              # Saving where jenkins will take a look at.....
   793              New-Item -Force -ItemType Directory bundles | Out-Null
   794              $unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
   795              $JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
   796              docker cp $JunitExpectedContFilePath "bundles"
   797              if (-not($LastExitCode -eq 0)) {
   798                  Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to bundles"
   799              }
   800  
   801              if (Test-Path "bundles\junit-report-unit-tests.xml") {
   802                  Write-Host -ForegroundColor Magenta "INFO: Unit tests results(bundles\junit-report-unit-tests.xml) exist. pwd=$pwd"
   803              } else {
   804                  Write-Host -ForegroundColor Magenta "ERROR: Unit tests results(bundles\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
   805              }
   806  
   807              if (-not($TestRunExitCode -eq 0)) {
   808                  Throw "ERROR: Unit tests failed"
   809              }
   810              Write-Host  -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration"
   811          } else {
   812              Write-Host -ForegroundColor Magenta "WARN: Skipping unit tests"
   813          }
   814      }
   815  
   816      # Add the Windows busybox image. Needed for WCOW integration tests
   817      if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
   818          if ($null -eq $env:SKIP_INTEGRATION_TESTS) {
   819              Write-Host -ForegroundColor Green "INFO: Building busybox"
   820              $ErrorActionPreference = "SilentlyContinue"
   821              $(& "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" build  -t busybox --build-arg WINDOWS_BASE_IMAGE --build-arg WINDOWS_BASE_IMAGE_TAG "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\contrib\busybox\" | Out-Host)
   822              $ErrorActionPreference = "Stop"
   823              if (-not($LastExitCode -eq 0)) {
   824                  Throw "ERROR: Failed to build busybox image"
   825              }
   826  
   827              Write-Host -ForegroundColor Green "INFO: Docker images of the daemon under test"
   828              Write-Host 
   829              $ErrorActionPreference = "SilentlyContinue"
   830              & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" images
   831              $ErrorActionPreference = "Stop"
   832              if ($LastExitCode -ne 0) {
   833                  Throw "ERROR: The daemon under test does not appear to be running."
   834              }
   835              Write-Host
   836          }
   837      }
   838  
   839      # Run the WCOW integration tests unless SKIP_INTEGRATION_TESTS is defined
   840      if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
   841          if ($null -eq $env:SKIP_INTEGRATION_TESTS) {
   842              Write-Host -ForegroundColor Cyan "INFO: Running integration tests at $(Get-Date)..."
   843              $ErrorActionPreference = "SilentlyContinue"
   844      
   845              # Location of the daemon under test.
   846              $env:OrigDOCKER_HOST="$env:DOCKER_HOST"
   847      
   848              #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
   849              $jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
   850              $xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
   851              $c = "gotestsum --format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
   852              if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
   853                  $c += "`"-test.run`" "
   854                  $c += "`"$env:INTEGRATION_TEST_NAME`" "
   855                  Write-Host -ForegroundColor Magenta "WARN: Only running integration tests matching $env:INTEGRATION_TEST_NAME"
   856              }
   857              $c += "`"-tags`" " + "`"autogen`" "
   858              $c += "`"-test.timeout`" " + "`"200m`" "
   859      
   860              if ($null -ne $env:INTEGRATION_IN_CONTAINER) {
   861                  Write-Host -ForegroundColor Green "INFO: Integration tests being run inside a container"
   862                  # Note we talk back through the containers gateway address
   863                  # And the ridiculous lengths we have to go to get the default gateway address... (GetNetIPConfiguration doesn't work in nanoserver)
   864                  # I just could not get the escaping to work in a single command, so output $c to a file and run that in the container instead...
   865                  # Not the prettiest, but it works.
   866                  $c | Out-File -Force "$env:TEMP\binary\runIntegrationCLI.ps1"
   867                  $Duration= $(Measure-Command { & docker run `
   868                                                          --rm `
   869                                                          -e c=$c `
   870                                                          --workdir "c`:\gopath\src\github.com\docker\docker\integration-cli" `
   871                                                          -v "$env:TEMP\binary`:c:\target" `
   872                                                          docker `
   873                                                          "`$env`:PATH`='c`:\target;'+`$env:PATH`;  `$env:DOCKER_HOST`='tcp`://'+(ipconfig | select -last 1).Substring(39)+'`:2357'; c:\target\runIntegrationCLI.ps1" | Out-Host } )
   874              } else  {
   875                  $env:DOCKER_HOST=$DASHH_CUT  
   876                  $env:PATH="$env:TEMP\binary;$env:PATH;"  # Force to use the test binaries, not the host ones.
   877                  Write-Host -ForegroundColor Green "INFO: DOCKER_HOST at $DASHH_CUT"
   878  
   879                  $ErrorActionPreference = "SilentlyContinue"
   880                  Write-Host -ForegroundColor Cyan "INFO: Integration API tests being run from the host:"
   881                  $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
   882                  $IntTestsRunResult = $LastExitCode
   883                  $ErrorActionPreference = "Stop"
   884                  if (-not($IntTestsRunResult -eq 0)) {
   885                      Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
   886                  }
   887  
   888                  $ErrorActionPreference = "SilentlyContinue"
   889                  Write-Host -ForegroundColor Green "INFO: Integration CLI tests being run from the host:"
   890                  Write-Host -ForegroundColor Green "INFO: $c"
   891                  Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
   892                  # Explicit to not use measure-command otherwise don't get output as it goes
   893                  $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
   894              }
   895              $ErrorActionPreference = "Stop"
   896              if (-not($LastExitCode -eq 0)) {
   897                  Throw "ERROR: Integration CLI tests failed at $(Get-Date). Duration`:$Duration"
   898              }
   899              Write-Host  -ForegroundColor Green "INFO: Integration tests ended at $(Get-Date). Duration`:$Duration"
   900          } else {
   901              Write-Host -ForegroundColor Magenta "WARN: Skipping integration tests"
   902          }
   903      } else {
   904          # The LCOW version of the tests here
   905          if ($null -eq $env:SKIP_INTEGRATION_TESTS) {
   906              Write-Host -ForegroundColor Cyan "INFO: Running LCOW tests at $(Get-Date)..."
   907  
   908              $ErrorActionPreference = "SilentlyContinue"
   909      
   910              # Location of the daemon under test.
   911              $env:OrigDOCKER_HOST="$env:DOCKER_HOST"
   912  
   913              # Make sure we are pointing at the DUT
   914              $env:DOCKER_HOST=$DASHH_CUT  
   915              Write-Host -ForegroundColor Green "INFO: DOCKER_HOST at $DASHH_CUT"
   916  
   917              # Force to use the test binaries, not the host ones.
   918              $env:PATH="$env:TEMP\binary;$env:PATH;"  
   919  
   920              if ($null -ne $env:LCOW_BASIC_MODE) {
   921                  $wc = New-Object net.webclient
   922                  try {
   923                      Write-Host -ForegroundColor green "INFO: Downloading latest execution script..."
   924                      $wc.Downloadfile("https://raw.githubusercontent.com/kevpar/docker-w2wCIScripts/master/runCI/lcowbasicvalidation.ps1", "$env:TEMP\binary\lcowbasicvalidation.ps1")
   925                  } 
   926                  catch [System.Net.WebException]
   927                  {
   928                      Throw ("Failed to download: $_")
   929                  }
   930  
   931                  # Explicit to not use measure-command otherwise don't get output as it goes
   932                  $ErrorActionPreference = "Stop"
   933                  $start=(Get-Date); Invoke-Expression "powershell $env:TEMP\binary\lcowbasicvalidation.ps1"; $lec=$lastExitCode; $Duration=New-Timespan -Start $start -End (Get-Date)
   934                  $Duration=New-Timespan -Start $start -End (Get-Date)
   935                  Write-Host  -ForegroundColor Green "INFO: LCOW tests ended at $(Get-Date). Duration`:$Duration"
   936                  if ($lec -ne 0) {
   937                      Throw "LCOW validation tests failed"
   938                  }
   939              } else {
   940                  #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
   941                  $c = "go test "
   942                  $c += "`"-test.v`" "
   943                  if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
   944                      $c += "`"-test.run`" "
   945                      $c += "`"$env:INTEGRATION_TEST_NAME`" "
   946                      Write-Host -ForegroundColor Magenta "WARN: Only running LCOW integration tests matching $env:INTEGRATION_TEST_NAME"
   947                  }
   948                  $c += "`"-tags`" " + "`"autogen`" "
   949                  $c += "`"-test.timeout`" " + "`"200m`" "
   950  
   951                  Write-Host -ForegroundColor Green "INFO: LCOW Integration tests being run from the host:"
   952                  Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
   953                  Write-Host -ForegroundColor Green "INFO: $c"
   954                  Write-Host -ForegroundColor Green "INFO: DOCKER_HOST at $DASHH_CUT"
   955                  # Explicit to not use measure-command otherwise don't get output as it goes
   956                  $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
   957  
   958              }
   959              $ErrorActionPreference = "Stop"
   960              if (-not($LastExitCode -eq 0)) {
   961                  Throw "ERROR: Integration tests failed at $(Get-Date). Duration`:$Duration"
   962              }
   963              Write-Host  -ForegroundColor Green "INFO: Integration tests ended at $(Get-Date). Duration`:$Duration"
   964          } else {
   965              Write-Host -ForegroundColor Magenta "WARN: Skipping LCOW tests"
   966          }
   967      }
   968  
   969      # Docker info now to get counts (after or if jjh/containercounts is merged)
   970      if ($daemonStarted -eq 1) {
   971          Write-Host -ForegroundColor Green "INFO: Docker info of the daemon under test at end of run"
   972          Write-Host 
   973          $ErrorActionPreference = "SilentlyContinue"
   974          & "$env:TEMP\binary\docker-$COMMITHASH" "-H=$($DASHH_CUT)" info
   975          $ErrorActionPreference = "Stop"
   976          if ($LastExitCode -ne 0) {
   977              Throw "ERROR: The daemon under test does not appear to be running."
   978          }
   979          Write-Host
   980      }
   981  
   982      # Stop the daemon under test
   983      if (Test-Path "$env:TEMP\docker.pid") {
   984          $p=Get-Content "$env:TEMP\docker.pid" -raw
   985          if (($null -ne $p) -and ($daemonStarted -eq 1)) {
   986              Write-Host -ForegroundColor green "INFO: Stopping daemon under test"
   987              taskkill -f -t -pid $p
   988              #sleep 5
   989          }
   990          Remove-Item "$env:TEMP\docker.pid" -force -ErrorAction SilentlyContinue
   991      }
   992  
   993      # Stop the tail process (if started)
   994      if ($null -ne $tail) {
   995          Write-Host -ForegroundColor green "INFO: Stop tailing logs of the daemon under tests"
   996          Stop-Process -InputObject $tail -Force
   997      }
   998  
   999      Write-Host -ForegroundColor Green "INFO: executeCI.ps1 Completed successfully at $(Get-Date)."
  1000  }
  1001  Catch [Exception] {
  1002      $FinallyColour="Red"
  1003      Write-Host -ForegroundColor Red ("`r`n`r`nERROR: Failed '$_' at $(Get-Date)")
  1004      Write-Host -ForegroundColor Red ($_.InvocationInfo.PositionMessage)
  1005      Write-Host "`n`n"
  1006  
  1007      # Exit to ensure Jenkins captures it. Don't do this in the ISE or interactive Powershell - they will catch the Throw onwards.
  1008      if ( ([bool]([Environment]::GetCommandLineArgs() -Like '*-NonInteractive*')) -and `
  1009           ([bool]([Environment]::GetCommandLineArgs() -NotLike "*Powershell_ISE.exe*"))) {
  1010          exit 1
  1011      }
  1012      Throw $_
  1013  }
  1014  Finally {
  1015      $ErrorActionPreference="SilentlyContinue"
  1016      $global:ProgressPreference=$origProgressPreference
  1017      Write-Host  -ForegroundColor Green "INFO: Tidying up at end of run"
  1018  
  1019      # Restore the path
  1020      if ($null -ne $origPath) { $env:PATH=$origPath }
  1021  
  1022      # Restore the DOCKER_HOST
  1023      if ($null -ne $origDOCKER_HOST) { $env:DOCKER_HOST=$origDOCKER_HOST }
  1024  
  1025      # Restore the GOROOT and GOPATH variables
  1026      if ($null -ne $origGOROOT) { $env:GOROOT=$origGOROOT }
  1027      if ($null -ne $origGOPATH) { $env:GOPATH=$origGOPATH }
  1028  
  1029      # Dump the daemon log. This will include any possible panic stack in the .err.
  1030      if (($daemonStarted -eq 1) -and  ($(Get-Item "$env:TEMP\dut.err").Length -gt 0)) {
  1031          Write-Host -ForegroundColor Cyan "----------- DAEMON LOG ------------"
  1032          Get-Content "$env:TEMP\dut.err" -ErrorAction SilentlyContinue | Write-Host -ForegroundColor Cyan
  1033          Write-Host -ForegroundColor Cyan "----------- END DAEMON LOG --------"
  1034      }
  1035  
  1036      # Save the daemon under test log
  1037      if ($daemonStarted -eq 1) {
  1038          Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
  1039          Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.out) to bundles\CIDUT.out"
  1040          Copy-Item  "$env:TEMP\dut.out" "bundles\CIDUT.out" -Force -ErrorAction SilentlyContinue
  1041          Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.err) to bundles\CIDUT.err"
  1042          Copy-Item  "$env:TEMP\dut.err" "bundles\CIDUT.err" -Force -ErrorAction SilentlyContinue
  1043      }
  1044  
  1045      Set-Location "$env:SOURCES_DRIVE\$env:SOURCES_SUBDIR" -ErrorAction SilentlyContinue
  1046      Nuke-Everything
  1047  
  1048      # Restore the TEMP path
  1049      if ($null -ne $TEMPORIG) { $env:TEMP="$TEMPORIG" }
  1050  
  1051      $Dur=New-TimeSpan -Start $StartTime -End $(Get-Date)
  1052      Write-Host -ForegroundColor $FinallyColour "`nINFO: executeCI.ps1 exiting at $(date). Duration $dur`n"
  1053  }