github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/hack/make.ps1 (about) 1 <# 2 .NOTES 3 Author: @jhowardmsft 4 5 Summary: Windows native build script. This is similar to functionality provided 6 by hack\make.sh, but uses native Windows PowerShell semantics. It does 7 not support the full set of options provided by the Linux counterpart. 8 For example: 9 10 - You can't cross-build Linux docker binaries on Windows 11 - Hashes aren't generated on binaries 12 - 'Releasing' isn't supported. 13 - Integration tests. This is because they currently cannot run inside a container, 14 and require significant external setup. 15 16 It does however provided the minimum necessary to support parts of local Windows 17 development and Windows to Windows CI. 18 19 Usage Examples (run from repo root): 20 "hack\make.ps1 -Client" to build docker.exe client 64-bit binary (remote repo) 21 "hack\make.ps1 -TestUnit" to run unit tests 22 "hack\make.ps1 -Daemon -TestUnit" to build the daemon and run unit tests 23 "hack\make.ps1 -All" to run everything this script knows about that can run in a container 24 "hack\make.ps1" to build the daemon binary (same as -Daemon) 25 "hack\make.ps1 -Binary" shortcut to -Client and -Daemon 26 27 .PARAMETER Client 28 Builds the client binaries. 29 30 .PARAMETER Daemon 31 Builds the daemon binary. 32 33 .PARAMETER Binary 34 Builds the client and daemon binaries. A convenient shortcut to `make.ps1 -Client -Daemon`. 35 36 .PARAMETER Race 37 Use -race in go build and go test. 38 39 .PARAMETER Noisy 40 Use -v in go build. 41 42 .PARAMETER ForceBuildAll 43 Use -a in go build. 44 45 .PARAMETER NoOpt 46 Use -gcflags -N -l in go build to disable optimisation (can aide debugging). 47 48 .PARAMETER CommitSuffix 49 Adds a custom string to be appended to the commit ID (spaces are stripped). 50 51 .PARAMETER DCO 52 Runs the DCO (Developer Certificate Of Origin) test (must be run outside a container). 53 54 .PARAMETER PkgImports 55 Runs the pkg\ directory imports test (must be run outside a container). 56 57 .PARAMETER GoFormat 58 Runs the Go formatting test (must be run outside a container). 59 60 .PARAMETER TestUnit 61 Runs unit tests. 62 63 .PARAMETER TestIntegration 64 Runs integration tests. 65 66 .PARAMETER All 67 Runs everything this script knows about that can run in a container. 68 69 70 TODO 71 - Unify the head commit 72 - Add golint and other checks (swagger maybe?) 73 74 #> 75 76 77 param( 78 [Parameter(Mandatory=$False)][switch]$Client, 79 [Parameter(Mandatory=$False)][switch]$Daemon, 80 [Parameter(Mandatory=$False)][switch]$Binary, 81 [Parameter(Mandatory=$False)][switch]$Race, 82 [Parameter(Mandatory=$False)][switch]$Noisy, 83 [Parameter(Mandatory=$False)][switch]$ForceBuildAll, 84 [Parameter(Mandatory=$False)][switch]$NoOpt, 85 [Parameter(Mandatory=$False)][string]$CommitSuffix="", 86 [Parameter(Mandatory=$False)][switch]$DCO, 87 [Parameter(Mandatory=$False)][switch]$PkgImports, 88 [Parameter(Mandatory=$False)][switch]$GoFormat, 89 [Parameter(Mandatory=$False)][switch]$TestUnit, 90 [Parameter(Mandatory=$False)][switch]$TestIntegration, 91 [Parameter(Mandatory=$False)][switch]$All 92 ) 93 94 $ErrorActionPreference = "Stop" 95 $ProgressPreference = "SilentlyContinue" 96 $pushed=$False # To restore the directory if we have temporarily pushed to one. 97 98 # Utility function to get the commit ID of the repository 99 Function Get-GitCommit() { 100 if (-not (Test-Path ".\.git")) { 101 # If we don't have a .git directory, but we do have the environment 102 # variable DOCKER_GITCOMMIT set, that can override it. 103 if ($env:DOCKER_GITCOMMIT.Length -eq 0) { 104 Throw ".git directory missing and DOCKER_GITCOMMIT environment variable not specified." 105 } 106 Write-Host "INFO: Git commit ($env:DOCKER_GITCOMMIT) assumed from DOCKER_GITCOMMIT environment variable" 107 return $env:DOCKER_GITCOMMIT 108 } 109 $gitCommit=$(git rev-parse --short HEAD) 110 if ($(git status --porcelain --untracked-files=no).Length -ne 0) { 111 $gitCommit="$gitCommit-unsupported" 112 Write-Host "" 113 Write-Warning "This version is unsupported because there are uncommitted file(s)." 114 Write-Warning "Either commit these changes, or add them to .gitignore." 115 git status --porcelain --untracked-files=no | Write-Warning 116 Write-Host "" 117 } 118 return $gitCommit 119 } 120 121 # Utility function to determine if we are running in a container or not. 122 # In Windows, we get this through an environment variable set in `Dockerfile.Windows` 123 Function Check-InContainer() { 124 if ($env:FROM_DOCKERFILE.Length -eq 0) { 125 Write-Host "" 126 Write-Warning "Not running in a container. The result might be an incorrect build." 127 Write-Host "" 128 return $False 129 } 130 return $True 131 } 132 133 # Utility function to warn if the version of go is correct. Used for local builds 134 # outside of a container where it may be out of date with master. 135 Function Verify-GoVersion() { 136 Try { 137 $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^FROM golang:").ToString().Split(" ")[1].SubString(7) 138 $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2) 139 } 140 Catch [Exception] { 141 Throw "Failed to validate go version correctness: $_" 142 } 143 if (-not($goVersionInstalled -eq $goVersionDockerfile)) { 144 Write-Host "" 145 Write-Warning "Building with golang version $goVersionInstalled. You should update to $goVersionDockerfile" 146 Write-Host "" 147 } 148 } 149 150 # Utility function to get the commit for HEAD 151 Function Get-HeadCommit() { 152 $head = Invoke-Expression "git rev-parse --verify HEAD" 153 if ($LASTEXITCODE -ne 0) { Throw "Failed getting HEAD commit" } 154 155 return $head 156 } 157 158 # Utility function to get the commit for upstream 159 Function Get-UpstreamCommit() { 160 Invoke-Expression "git fetch -q https://github.com/docker/docker.git refs/heads/master" 161 if ($LASTEXITCODE -ne 0) { Throw "Failed fetching" } 162 163 $upstream = Invoke-Expression "git rev-parse --verify FETCH_HEAD" 164 if ($LASTEXITCODE -ne 0) { Throw "Failed getting upstream commit" } 165 166 return $upstream 167 } 168 169 # Build a binary (client or daemon) 170 Function Execute-Build($type, $additionalBuildTags, $directory) { 171 # Generate the build flags 172 $buildTags = "autogen" 173 if ($Noisy) { $verboseParm=" -v" } 174 if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"} 175 if ($ForceBuildAll) { $allParm=" -a" } 176 if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" } 177 if ($additionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) } 178 179 # Do the go build in the appropriate directory 180 # Note -linkmode=internal is required to be able to debug on Windows. 181 # https://github.com/golang/go/issues/14319#issuecomment-189576638 182 Write-Host "INFO: Building $type..." 183 Push-Location $root\cmd\$directory; $global:pushed=$True 184 $buildCommand = "go build" + ` 185 $raceParm + ` 186 $verboseParm + ` 187 $allParm + ` 188 $optParm + ` 189 " -tags """ + $buildTags + """" + ` 190 " -ldflags """ + "-linkmode=internal" + """" + ` 191 " -o $root\bundles\"+$directory+".exe" 192 Invoke-Expression $buildCommand 193 if ($LASTEXITCODE -ne 0) { Throw "Failed to compile $type" } 194 Pop-Location; $global:pushed=$False 195 } 196 197 198 # Validates the DCO marker is present on each commit 199 Function Validate-DCO($headCommit, $upstreamCommit) { 200 Write-Host "INFO: Validating Developer Certificate of Origin..." 201 # Username may only contain alphanumeric characters or dashes and cannot begin with a dash 202 $usernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+' 203 204 $dcoPrefix="Signed-off-by:" 205 $dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \(github: ($usernameRegex)\))?$" 206 207 $counts = Invoke-Expression "git diff --numstat $upstreamCommit...$headCommit" 208 if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" } 209 210 # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :( 211 $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{ 212 $a=$_.Split(" "); 213 if ($a[0] -ne "-") { $adds+=[int]$a[0] } 214 if ($a[1] -ne "-") { $dels+=[int]$a[1] } 215 } 216 if (($adds -eq 0) -and ($dels -eq 0)) { 217 Write-Warning "DCO validation - nothing to validate!" 218 return 219 } 220 221 $commits = Invoke-Expression "git log $upstreamCommit..$headCommit --format=format:%H%n" 222 if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" } 223 $commits = $($commits -split '\s+' -match '\S') 224 $badCommits=@() 225 $commits | ForEach-Object{ 226 # Skip commits with no content such as merge commits etc 227 if ($(git log -1 --format=format: --name-status $_).Length -gt 0) { 228 # Ignore exit code on next call - always process regardless 229 $commitMessage = Invoke-Expression "git log -1 --format=format:%B --name-status $_" 230 if (($commitMessage -match $dcoRegex).Length -eq 0) { $badCommits+=$_ } 231 } 232 } 233 if ($badCommits.Length -eq 0) { 234 Write-Host "Congratulations! All commits are properly signed with the DCO!" 235 } else { 236 $e = "`nThese commits do not have a proper '$dcoPrefix' marker:`n" 237 $badCommits | %{ $e+=" - $_`n"} 238 $e += "`nPlease amend each commit to include a properly formatted DCO marker.`n`n" 239 $e += "Visit the following URL for information about the Docker DCO:`n" 240 $e += "https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work`n" 241 Throw $e 242 } 243 } 244 245 # Validates that .\pkg\... is safely isolated from internal code 246 Function Validate-PkgImports($headCommit, $upstreamCommit) { 247 Write-Host "INFO: Validating pkg import isolation..." 248 249 # Get a list of go source-code files which have changed under pkg\. Ignore exit code on next call - always process regardless 250 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'" 251 $badFiles=@(); $files | ForEach-Object{ 252 $file=$_ 253 # For the current changed file, get its list of dependencies, sorted and uniqued. 254 $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file" 255 if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" } 256 $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique 257 # Filter out what we are looking for 258 $imports = @() + $imports -NotMatch "^github.com/docker/docker/pkg/" ` 259 -NotMatch "^github.com/docker/docker/vendor" ` 260 -Match "^github.com/docker/docker" ` 261 -Replace "`n", "" 262 $imports | ForEach-Object{ $badFiles+="$file imports $_`n" } 263 } 264 if ($badFiles.Length -eq 0) { 265 Write-Host 'Congratulations! ".\pkg\*.go" is safely isolated from internal code.' 266 } else { 267 $e = "`nThese files import internal code: (either directly or indirectly)`n" 268 $badFiles | ForEach-Object{ $e+=" - $_"} 269 Throw $e 270 } 271 } 272 273 # Validates that changed files are correctly go-formatted 274 Function Validate-GoFormat($headCommit, $upstreamCommit) { 275 Write-Host "INFO: Validating go formatting on changed files..." 276 277 # Verify gofmt is installed 278 if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" } 279 280 # Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless 281 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'" 282 $files = $files | Select-String -NotMatch "^vendor/" 283 $badFiles=@(); $files | %{ 284 # Deliberately ignore error on next line - treat as failed 285 $content=Invoke-Expression "git show $headCommit`:$_" 286 287 # Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed. 288 # Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file. 289 $content=$content -join "`n" 290 $content+="`n" 291 $outputFile=[System.IO.Path]::GetTempFileName() 292 if (Test-Path $outputFile) { Remove-Item $outputFile } 293 [System.IO.File]::WriteAllText($outputFile, $content, (New-Object System.Text.UTF8Encoding($False))) 294 $currentFile = $_ -Replace("/","\") 295 Write-Host Checking $currentFile 296 Invoke-Expression "gofmt -s -l $outputFile" 297 if ($LASTEXITCODE -ne 0) { $badFiles+=$currentFile } 298 if (Test-Path $outputFile) { Remove-Item $outputFile } 299 } 300 if ($badFiles.Length -eq 0) { 301 Write-Host 'Congratulations! All Go source files are properly formatted.' 302 } else { 303 $e = "`nThese files are not properly gofmt`'d:`n" 304 $badFiles | ForEach-Object{ $e+=" - $_`n"} 305 $e+= "`nPlease reformat the above files using `"gofmt -s -w`" and commit the result." 306 Throw $e 307 } 308 } 309 310 # Run the unit tests 311 Function Run-UnitTests() { 312 Write-Host "INFO: Running unit tests..." 313 $testPath="./..." 314 $goListCommand = "go list -e -f '{{if ne .Name """ + '\"github.com/docker/docker\"' + """}}{{.ImportPath}}{{end}}' $testPath" 315 $pkgList = $(Invoke-Expression $goListCommand) 316 if ($LASTEXITCODE -ne 0) { Throw "go list for unit tests failed" } 317 $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker" 318 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor" 319 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man" 320 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" 321 $pkgList = $pkgList -replace "`r`n", " " 322 $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList" 323 Invoke-Expression $goTestCommand 324 if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" } 325 } 326 327 # Run the integration tests 328 Function Run-IntegrationTests() { 329 $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp" 330 $dirs = Get-ChildItem -Path integration -Directory -Recurse 331 $integration_api_dirs = @() 332 ForEach($dir in $dirs) { 333 $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")","" 334 If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') { 335 $integration_api_dirs += $dir 336 Write-Host "Building test suite binary $RelativePath" 337 go test -c -o "$RelativePath\test.exe" $RelativePath 338 } 339 } 340 341 ForEach($dir in $integration_api_dirs) { 342 Set-Location $dir.FullName 343 Write-Host "Running $($PWD.Path)" 344 $pinfo = New-Object System.Diagnostics.ProcessStartInfo 345 $pinfo.FileName = "$($PWD.Path)\test.exe" 346 $pinfo.RedirectStandardError = $true 347 $pinfo.UseShellExecute = $false 348 $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS 349 $p = New-Object System.Diagnostics.Process 350 $p.StartInfo = $pinfo 351 $p.Start() | Out-Null 352 $p.WaitForExit() 353 $err = $p.StandardError.ReadToEnd() 354 if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) { 355 Throw "Integration tests failed: $err" 356 } 357 } 358 } 359 360 # Start of main code. 361 Try { 362 Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)" 363 364 # Get to the root of the repo 365 $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent) 366 Push-Location $root 367 368 # Handle the "-All" shortcut to turn on all things we can handle. 369 # Note we expressly only include the items which can run in a container - the validations tests cannot 370 # as they require the .git directory which is excluded from the image by .dockerignore 371 if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True; } 372 373 # Handle the "-Binary" shortcut to build both client and daemon. 374 if ($Binary) { $Client = $True; $Daemon = $True } 375 376 # Default to building the daemon if not asked for anything explicitly. 377 if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit) -and -not($TestIntegration)) { $Daemon=$True } 378 379 # Verify git is installed 380 if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" } 381 382 # Verify go is installed 383 if ($(Get-Command go -ErrorAction SilentlyContinue) -eq $nil) { Throw "GoLang does not appear to be installed" } 384 385 # Get the git commit. This will also verify if we are in a repo or not. Then add a custom string if supplied. 386 $gitCommit=Get-GitCommit 387 if ($CommitSuffix -ne "") { $gitCommit += "-"+$CommitSuffix -Replace ' ', '' } 388 389 # Get the version of docker (eg 17.04.0-dev) 390 $dockerVersion="0.0.0-dev" 391 # Overwrite dockerVersion if VERSION Environment variable is available 392 if (Test-Path Env:\VERSION) { $dockerVersion=$env:VERSION } 393 394 # Give a warning if we are not running in a container and are building binaries or running unit tests. 395 # Not relevant for validation tests as these are fine to run outside of a container. 396 if ($Client -or $Daemon -or $TestUnit) { $inContainer=Check-InContainer } 397 398 # If we are not in a container, validate the version of GO that is installed. 399 if (-not $inContainer) { Verify-GoVersion } 400 401 # Verify GOPATH is set 402 if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" } 403 404 # Run autogen if building binaries or running unit tests. 405 if ($Client -or $Daemon -or $TestUnit) { 406 Write-Host "INFO: Invoking autogen..." 407 Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" -Product "$env:PRODUCT" } 408 Catch [Exception] { Throw $_ } 409 } 410 411 # DCO, Package import and Go formatting tests. 412 if ($DCO -or $PkgImports -or $GoFormat) { 413 # We need the head and upstream commits for these 414 $headCommit=Get-HeadCommit 415 $upstreamCommit=Get-UpstreamCommit 416 417 # Run DCO validation 418 if ($DCO) { Validate-DCO $headCommit $upstreamCommit } 419 420 # Run `gofmt` validation 421 if ($GoFormat) { Validate-GoFormat $headCommit $upstreamCommit } 422 423 # Run pkg isolation validation 424 if ($PkgImports) { Validate-PkgImports $headCommit $upstreamCommit } 425 } 426 427 # Build the binaries 428 if ($Client -or $Daemon) { 429 # Create the bundles directory if it doesn't exist 430 if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null } 431 432 # Perform the actual build 433 if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" } 434 if ($Client) { 435 # Get the Docker channel and version from the environment, or use the defaults. 436 if (-not ($channel = $env:DOCKERCLI_CHANNEL)) { $channel = "edge" } 437 if (-not ($version = $env:DOCKERCLI_VERSION)) { $version = "17.06.0-ce" } 438 439 # Download the zip file and extract the client executable. 440 Write-Host "INFO: Downloading docker/cli version $version from $channel..." 441 $url = "https://download.docker.com/win/static/$channel/x86_64/docker-$version.zip" 442 Invoke-WebRequest $url -OutFile "docker.zip" 443 Try { 444 Add-Type -AssemblyName System.IO.Compression.FileSystem 445 $zip = [System.IO.Compression.ZipFile]::OpenRead("$PWD\docker.zip") 446 Try { 447 if (-not ($entry = $zip.Entries | Where-Object { $_.Name -eq "docker.exe" })) { 448 Throw "Cannot find docker.exe in $url" 449 } 450 [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, "$PWD\bundles\docker.exe", $true) 451 } 452 Finally { 453 $zip.Dispose() 454 } 455 } 456 Finally { 457 Remove-Item -Force "docker.zip" 458 } 459 } 460 } 461 462 # Run unit tests 463 if ($TestUnit) { Run-UnitTests } 464 465 # Run integration tests 466 if ($TestIntegration) { Run-IntegrationTests } 467 468 # Gratuitous ASCII art. 469 if ($Daemon -or $Client) { 470 Write-Host 471 Write-Host -ForegroundColor Green " ________ ____ __." 472 Write-Host -ForegroundColor Green " \_____ \ `| `|/ _`|" 473 Write-Host -ForegroundColor Green " / `| \`| `<" 474 Write-Host -ForegroundColor Green " / `| \ `| \" 475 Write-Host -ForegroundColor Green " \_______ /____`|__ \" 476 Write-Host -ForegroundColor Green " \/ \/" 477 Write-Host 478 } 479 } 480 Catch [Exception] { 481 Write-Host -ForegroundColor Red ("`nERROR: make.ps1 failed:`n$_") 482 483 # More gratuitous ASCII art. 484 Write-Host 485 Write-Host -ForegroundColor Red "___________ .__.__ .___" 486 Write-Host -ForegroundColor Red "\_ _____/____ `|__`| `| ____ __`| _/" 487 Write-Host -ForegroundColor Red " `| __) \__ \ `| `| `| _/ __ \ / __ `| " 488 Write-Host -ForegroundColor Red " `| \ / __ \`| `| `|_\ ___// /_/ `| " 489 Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| " 490 Write-Host -ForegroundColor Red " \/ \/ \/ \/ " 491 Write-Host 492 493 Throw $_ 494 } 495 Finally { 496 Pop-Location # As we pushed to the root of the repo as the very first thing 497 if ($global:pushed) { Pop-Location } 498 Write-Host -ForegroundColor Cyan "INFO: make.ps1 ended at $(Get-Date)" 499 }