github.com/devdivbcp/moby@v17.12.0-ce-rc1.0.20200726071732-2d4bfdc789ad+incompatible/hack/make.ps1 (about) 1 <# 2 .NOTES 3 Author: @jhowardmsft 4 5 Summary: Windows native build script. This is similar to functionality provided 6 by hack\make.sh, but uses native Windows PowerShell semantics. It does 7 not support the full set of options provided by the Linux counterpart. 8 For example: 9 10 - You can't cross-build Linux docker binaries on Windows 11 - Hashes aren't generated on binaries 12 - 'Releasing' isn't supported. 13 - Integration tests. This is because they currently cannot run inside a container, 14 and require significant external setup. 15 16 It does however provided the minimum necessary to support parts of local Windows 17 development and Windows to Windows CI. 18 19 Usage Examples (run from repo root): 20 "hack\make.ps1 -Client" to build docker.exe client 64-bit binary (remote repo) 21 "hack\make.ps1 -TestUnit" to run unit tests 22 "hack\make.ps1 -Daemon -TestUnit" to build the daemon and run unit tests 23 "hack\make.ps1 -All" to run everything this script knows about that can run in a container 24 "hack\make.ps1" to build the daemon binary (same as -Daemon) 25 "hack\make.ps1 -Binary" shortcut to -Client and -Daemon 26 27 .PARAMETER Client 28 Builds the client binaries. 29 30 .PARAMETER Daemon 31 Builds the daemon binary. 32 33 .PARAMETER Binary 34 Builds the client and daemon binaries. A convenient shortcut to `make.ps1 -Client -Daemon`. 35 36 .PARAMETER Race 37 Use -race in go build and go test. 38 39 .PARAMETER Noisy 40 Use -v in go build. 41 42 .PARAMETER ForceBuildAll 43 Use -a in go build. 44 45 .PARAMETER NoOpt 46 Use -gcflags -N -l in go build to disable optimisation (can aide debugging). 47 48 .PARAMETER CommitSuffix 49 Adds a custom string to be appended to the commit ID (spaces are stripped). 50 51 .PARAMETER DCO 52 Runs the DCO (Developer Certificate Of Origin) test (must be run outside a container). 53 54 .PARAMETER PkgImports 55 Runs the pkg\ directory imports test (must be run outside a container). 56 57 .PARAMETER GoFormat 58 Runs the Go formatting test (must be run outside a container). 59 60 .PARAMETER TestUnit 61 Runs unit tests. 62 63 .PARAMETER TestIntegration 64 Runs integration tests. 65 66 .PARAMETER All 67 Runs everything this script knows about that can run in a container. 68 69 70 TODO 71 - Unify the head commit 72 - Add golint and other checks (swagger maybe?) 73 74 #> 75 76 77 param( 78 [Parameter(Mandatory=$False)][switch]$Client, 79 [Parameter(Mandatory=$False)][switch]$Daemon, 80 [Parameter(Mandatory=$False)][switch]$Binary, 81 [Parameter(Mandatory=$False)][switch]$Race, 82 [Parameter(Mandatory=$False)][switch]$Noisy, 83 [Parameter(Mandatory=$False)][switch]$ForceBuildAll, 84 [Parameter(Mandatory=$False)][switch]$NoOpt, 85 [Parameter(Mandatory=$False)][string]$CommitSuffix="", 86 [Parameter(Mandatory=$False)][switch]$DCO, 87 [Parameter(Mandatory=$False)][switch]$PkgImports, 88 [Parameter(Mandatory=$False)][switch]$GoFormat, 89 [Parameter(Mandatory=$False)][switch]$TestUnit, 90 [Parameter(Mandatory=$False)][switch]$TestIntegration, 91 [Parameter(Mandatory=$False)][switch]$All 92 ) 93 94 $ErrorActionPreference = "Stop" 95 $ProgressPreference = "SilentlyContinue" 96 $pushed=$False # To restore the directory if we have temporarily pushed to one. 97 98 # Utility function to get the commit ID of the repository 99 Function Get-GitCommit() { 100 if (-not (Test-Path ".\.git")) { 101 # If we don't have a .git directory, but we do have the environment 102 # variable DOCKER_GITCOMMIT set, that can override it. 103 if ($env:DOCKER_GITCOMMIT.Length -eq 0) { 104 Throw ".git directory missing and DOCKER_GITCOMMIT environment variable not specified." 105 } 106 Write-Host "INFO: Git commit ($env:DOCKER_GITCOMMIT) assumed from DOCKER_GITCOMMIT environment variable" 107 return $env:DOCKER_GITCOMMIT 108 } 109 $gitCommit=$(git rev-parse --short HEAD) 110 if ($(git status --porcelain --untracked-files=no).Length -ne 0) { 111 $gitCommit="$gitCommit-unsupported" 112 Write-Host "" 113 Write-Warning "This version is unsupported because there are uncommitted file(s)." 114 Write-Warning "Either commit these changes, or add them to .gitignore." 115 git status --porcelain --untracked-files=no | Write-Warning 116 Write-Host "" 117 } 118 return $gitCommit 119 } 120 121 # Utility function to determine if we are running in a container or not. 122 # In Windows, we get this through an environment variable set in `Dockerfile.Windows` 123 Function Check-InContainer() { 124 if ($env:FROM_DOCKERFILE.Length -eq 0) { 125 Write-Host "" 126 Write-Warning "Not running in a container. The result might be an incorrect build." 127 Write-Host "" 128 return $False 129 } 130 return $True 131 } 132 133 # Utility function to warn if the version of go is correct. Used for local builds 134 # outside of a container where it may be out of date with master. 135 Function Verify-GoVersion() { 136 Try { 137 $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value -replace '\.0$','' 138 $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2) 139 } 140 Catch [Exception] { 141 Throw "Failed to validate go version correctness: $_" 142 } 143 if (-not($goVersionInstalled -eq $goVersionDockerfile)) { 144 Write-Host "" 145 Write-Warning "Building with golang version $goVersionInstalled. You should update to $goVersionDockerfile" 146 Write-Host "" 147 } 148 } 149 150 # Utility function to get the commit for HEAD 151 Function Get-HeadCommit() { 152 $head = Invoke-Expression "git rev-parse --verify HEAD" 153 if ($LASTEXITCODE -ne 0) { Throw "Failed getting HEAD commit" } 154 155 return $head 156 } 157 158 # Utility function to get the commit for upstream 159 Function Get-UpstreamCommit() { 160 Invoke-Expression "git fetch -q https://github.com/docker/docker.git refs/heads/master" 161 if ($LASTEXITCODE -ne 0) { Throw "Failed fetching" } 162 163 $upstream = Invoke-Expression "git rev-parse --verify FETCH_HEAD" 164 if ($LASTEXITCODE -ne 0) { Throw "Failed getting upstream commit" } 165 166 return $upstream 167 } 168 169 # Build a binary (client or daemon) 170 Function Execute-Build($type, $additionalBuildTags, $directory) { 171 # Generate the build flags 172 $buildTags = "autogen" 173 if ($Noisy) { $verboseParm=" -v" } 174 if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"} 175 if ($ForceBuildAll) { $allParm=" -a" } 176 if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" } 177 if ($additionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) } 178 179 # Do the go build in the appropriate directory 180 # Note -linkmode=internal is required to be able to debug on Windows. 181 # https://github.com/golang/go/issues/14319#issuecomment-189576638 182 Write-Host "INFO: Building $type..." 183 Push-Location $root\cmd\$directory; $global:pushed=$True 184 $buildCommand = "go build" + ` 185 $raceParm + ` 186 $verboseParm + ` 187 $allParm + ` 188 $optParm + ` 189 " -tags """ + $buildTags + """" + ` 190 " -ldflags """ + "-linkmode=internal" + """" + ` 191 " -o $root\bundles\"+$directory+".exe" 192 Invoke-Expression $buildCommand 193 if ($LASTEXITCODE -ne 0) { Throw "Failed to compile $type" } 194 Pop-Location; $global:pushed=$False 195 } 196 197 198 # Validates the DCO marker is present on each commit 199 Function Validate-DCO($headCommit, $upstreamCommit) { 200 Write-Host "INFO: Validating Developer Certificate of Origin..." 201 # Username may only contain alphanumeric characters or dashes and cannot begin with a dash 202 $usernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+' 203 204 $dcoPrefix="Signed-off-by:" 205 $dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \(github: ($usernameRegex)\))?$" 206 207 $counts = Invoke-Expression "git diff --numstat $upstreamCommit...$headCommit" 208 if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" } 209 210 # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :( 211 $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{ 212 $a=$_.Split(" "); 213 if ($a[0] -ne "-") { $adds+=[int]$a[0] } 214 if ($a[1] -ne "-") { $dels+=[int]$a[1] } 215 } 216 if (($adds -eq 0) -and ($dels -eq 0)) { 217 Write-Warning "DCO validation - nothing to validate!" 218 return 219 } 220 221 $commits = Invoke-Expression "git log $upstreamCommit..$headCommit --format=format:%H%n" 222 if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" } 223 $commits = $($commits -split '\s+' -match '\S') 224 $badCommits=@() 225 $commits | ForEach-Object{ 226 # Skip commits with no content such as merge commits etc 227 if ($(git log -1 --format=format: --name-status $_).Length -gt 0) { 228 # Ignore exit code on next call - always process regardless 229 $commitMessage = Invoke-Expression "git log -1 --format=format:%B --name-status $_" 230 if (($commitMessage -match $dcoRegex).Length -eq 0) { $badCommits+=$_ } 231 } 232 } 233 if ($badCommits.Length -eq 0) { 234 Write-Host "Congratulations! All commits are properly signed with the DCO!" 235 } else { 236 $e = "`nThese commits do not have a proper '$dcoPrefix' marker:`n" 237 $badCommits | %{ $e+=" - $_`n"} 238 $e += "`nPlease amend each commit to include a properly formatted DCO marker.`n`n" 239 $e += "Visit the following URL for information about the Docker DCO:`n" 240 $e += "https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work`n" 241 Throw $e 242 } 243 } 244 245 # Validates that .\pkg\... is safely isolated from internal code 246 Function Validate-PkgImports($headCommit, $upstreamCommit) { 247 Write-Host "INFO: Validating pkg import isolation..." 248 249 # Get a list of go source-code files which have changed under pkg\. Ignore exit code on next call - always process regardless 250 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'" 251 $badFiles=@(); $files | ForEach-Object{ 252 $file=$_ 253 # For the current changed file, get its list of dependencies, sorted and uniqued. 254 $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file" 255 if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" } 256 $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique 257 # Filter out what we are looking for 258 $imports = @() + $imports -NotMatch "^github.com/docker/docker/pkg/" ` 259 -NotMatch "^github.com/docker/docker/vendor" ` 260 -Match "^github.com/docker/docker" ` 261 -Replace "`n", "" 262 $imports | ForEach-Object{ $badFiles+="$file imports $_`n" } 263 } 264 if ($badFiles.Length -eq 0) { 265 Write-Host 'Congratulations! ".\pkg\*.go" is safely isolated from internal code.' 266 } else { 267 $e = "`nThese files import internal code: (either directly or indirectly)`n" 268 $badFiles | ForEach-Object{ $e+=" - $_"} 269 Throw $e 270 } 271 } 272 273 # Validates that changed files are correctly go-formatted 274 Function Validate-GoFormat($headCommit, $upstreamCommit) { 275 Write-Host "INFO: Validating go formatting on changed files..." 276 277 # Verify gofmt is installed 278 if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" } 279 280 # Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless 281 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'" 282 $files = $files | Select-String -NotMatch "^vendor/" 283 $badFiles=@(); $files | %{ 284 # Deliberately ignore error on next line - treat as failed 285 $content=Invoke-Expression "git show $headCommit`:$_" 286 287 # Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed. 288 # Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file. 289 $content=$content -join "`n" 290 $content+="`n" 291 $outputFile=[System.IO.Path]::GetTempFileName() 292 if (Test-Path $outputFile) { Remove-Item $outputFile } 293 [System.IO.File]::WriteAllText($outputFile, $content, (New-Object System.Text.UTF8Encoding($False))) 294 $currentFile = $_ -Replace("/","\") 295 Write-Host Checking $currentFile 296 Invoke-Expression "gofmt -s -l $outputFile" 297 if ($LASTEXITCODE -ne 0) { $badFiles+=$currentFile } 298 if (Test-Path $outputFile) { Remove-Item $outputFile } 299 } 300 if ($badFiles.Length -eq 0) { 301 Write-Host 'Congratulations! All Go source files are properly formatted.' 302 } else { 303 $e = "`nThese files are not properly gofmt`'d:`n" 304 $badFiles | ForEach-Object{ $e+=" - $_`n"} 305 $e+= "`nPlease reformat the above files using `"gofmt -s -w`" and commit the result." 306 Throw $e 307 } 308 } 309 310 # Run the unit tests 311 Function Run-UnitTests() { 312 Write-Host "INFO: Running unit tests..." 313 $testPath="./..." 314 $goListCommand = "go list -e -f '{{if ne .Name """ + '\"github.com/docker/docker\"' + """}}{{.ImportPath}}{{end}}' $testPath" 315 $pkgList = $(Invoke-Expression $goListCommand) 316 if ($LASTEXITCODE -ne 0) { Throw "go list for unit tests failed" } 317 $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker" 318 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor" 319 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man" 320 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" 321 $pkgList = $pkgList -replace "`r`n", " " 322 $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList" 323 Invoke-Expression $goTestCommand 324 if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" } 325 } 326 327 # Run the integration tests 328 Function Run-IntegrationTests() { 329 $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp" 330 $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\... 331 $integration_api_dirs = @() 332 ForEach($dir in $dirs) { 333 $integration_api_dirs += $dir 334 Write-Host "Building test suite binary $dir" 335 go test -c -o "$dir\test.exe" $dir 336 } 337 338 ForEach($dir in $integration_api_dirs) { 339 Set-Location $dir 340 Write-Host "Running $($PWD.Path)" 341 $pinfo = New-Object System.Diagnostics.ProcessStartInfo 342 $pinfo.FileName = "$($PWD.Path)\test.exe" 343 $pinfo.WorkingDirectory = "$($PWD.Path)" 344 $pinfo.RedirectStandardError = $true 345 $pinfo.UseShellExecute = $false 346 $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS 347 $p = New-Object System.Diagnostics.Process 348 $p.StartInfo = $pinfo 349 $p.Start() | Out-Null 350 $p.WaitForExit() 351 $err = $p.StandardError.ReadToEnd() 352 if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) { 353 Throw "Integration tests failed: $err" 354 } 355 } 356 } 357 358 # Start of main code. 359 Try { 360 Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)" 361 362 # Get to the root of the repo 363 $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent) 364 Push-Location $root 365 366 # Handle the "-All" shortcut to turn on all things we can handle. 367 # Note we expressly only include the items which can run in a container - the validations tests cannot 368 # as they require the .git directory which is excluded from the image by .dockerignore 369 if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True; } 370 371 # Handle the "-Binary" shortcut to build both client and daemon. 372 if ($Binary) { $Client = $True; $Daemon = $True } 373 374 # Default to building the daemon if not asked for anything explicitly. 375 if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit) -and -not($TestIntegration)) { $Daemon=$True } 376 377 # Verify git is installed 378 if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" } 379 380 # Verify go is installed 381 if ($(Get-Command go -ErrorAction SilentlyContinue) -eq $nil) { Throw "GoLang does not appear to be installed" } 382 383 # Get the git commit. This will also verify if we are in a repo or not. Then add a custom string if supplied. 384 $gitCommit=Get-GitCommit 385 if ($CommitSuffix -ne "") { $gitCommit += "-"+$CommitSuffix -Replace ' ', '' } 386 387 # Get the version of docker (eg 17.04.0-dev) 388 $dockerVersion="0.0.0-dev" 389 # Overwrite dockerVersion if VERSION Environment variable is available 390 if (Test-Path Env:\VERSION) { $dockerVersion=$env:VERSION } 391 392 # Give a warning if we are not running in a container and are building binaries or running unit tests. 393 # Not relevant for validation tests as these are fine to run outside of a container. 394 if ($Client -or $Daemon -or $TestUnit) { $inContainer=Check-InContainer } 395 396 # If we are not in a container, validate the version of GO that is installed. 397 if (-not $inContainer) { Verify-GoVersion } 398 399 # Verify GOPATH is set 400 if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" } 401 402 # Run autogen if building binaries or running unit tests. 403 if ($Client -or $Daemon -or $TestUnit) { 404 Write-Host "INFO: Invoking autogen..." 405 Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" -Product "$env:PRODUCT" } 406 Catch [Exception] { Throw $_ } 407 } 408 409 # DCO, Package import and Go formatting tests. 410 if ($DCO -or $PkgImports -or $GoFormat) { 411 # We need the head and upstream commits for these 412 $headCommit=Get-HeadCommit 413 $upstreamCommit=Get-UpstreamCommit 414 415 # Run DCO validation 416 if ($DCO) { Validate-DCO $headCommit $upstreamCommit } 417 418 # Run `gofmt` validation 419 if ($GoFormat) { Validate-GoFormat $headCommit $upstreamCommit } 420 421 # Run pkg isolation validation 422 if ($PkgImports) { Validate-PkgImports $headCommit $upstreamCommit } 423 } 424 425 # Build the binaries 426 if ($Client -or $Daemon) { 427 # Create the bundles directory if it doesn't exist 428 if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null } 429 430 # Perform the actual build 431 if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" } 432 if ($Client) { 433 # Get the Docker channel and version from the environment, or use the defaults. 434 if (-not ($channel = $env:DOCKERCLI_CHANNEL)) { $channel = "stable" } 435 if (-not ($version = $env:DOCKERCLI_VERSION)) { $version = "17.06.2-ce" } 436 437 # Download the zip file and extract the client executable. 438 Write-Host "INFO: Downloading docker/cli version $version from $channel..." 439 $url = "https://download.docker.com/win/static/$channel/x86_64/docker-$version.zip" 440 Invoke-WebRequest $url -OutFile "docker.zip" 441 Try { 442 Add-Type -AssemblyName System.IO.Compression.FileSystem 443 $zip = [System.IO.Compression.ZipFile]::OpenRead("$PWD\docker.zip") 444 Try { 445 if (-not ($entry = $zip.Entries | Where-Object { $_.Name -eq "docker.exe" })) { 446 Throw "Cannot find docker.exe in $url" 447 } 448 [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, "$PWD\bundles\docker.exe", $true) 449 } 450 Finally { 451 $zip.Dispose() 452 } 453 } 454 Finally { 455 Remove-Item -Force "docker.zip" 456 } 457 } 458 } 459 460 # Run unit tests 461 if ($TestUnit) { Run-UnitTests } 462 463 # Run integration tests 464 if ($TestIntegration) { Run-IntegrationTests } 465 466 # Gratuitous ASCII art. 467 if ($Daemon -or $Client) { 468 Write-Host 469 Write-Host -ForegroundColor Green " ________ ____ __." 470 Write-Host -ForegroundColor Green " \_____ \ `| `|/ _`|" 471 Write-Host -ForegroundColor Green " / `| \`| `<" 472 Write-Host -ForegroundColor Green " / `| \ `| \" 473 Write-Host -ForegroundColor Green " \_______ /____`|__ \" 474 Write-Host -ForegroundColor Green " \/ \/" 475 Write-Host 476 } 477 } 478 Catch [Exception] { 479 Write-Host -ForegroundColor Red ("`nERROR: make.ps1 failed:`n$_") 480 Write-Host -ForegroundColor Red ($_.InvocationInfo.PositionMessage) 481 482 # More gratuitous ASCII art. 483 Write-Host 484 Write-Host -ForegroundColor Red "___________ .__.__ .___" 485 Write-Host -ForegroundColor Red "\_ _____/____ `|__`| `| ____ __`| _/" 486 Write-Host -ForegroundColor Red " `| __) \__ \ `| `| `| _/ __ \ / __ `| " 487 Write-Host -ForegroundColor Red " `| \ / __ \`| `| `|_\ ___// /_/ `| " 488 Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| " 489 Write-Host -ForegroundColor Red " \/ \/ \/ \/ " 490 Write-Host 491 492 Throw $_ 493 } 494 Finally { 495 Pop-Location # As we pushed to the root of the repo as the very first thing 496 if ($global:pushed) { Pop-Location } 497 Write-Host -ForegroundColor Cyan "INFO: make.ps1 ended at $(Get-Date)" 498 }