github.com/Cleverse/go-ethereum@v0.0.0-20220927095127-45113064e7f2/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 //go:build none 18 // +build none 19 20 /* 21 The ci command is called from Continuous Integration scripts. 22 23 Usage: go run build/ci.go <command> <command flags/arguments> 24 25 Available commands are: 26 27 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 28 test [ -coverage ] [ packages... ] -- runs the tests 29 lint -- runs certain pre-selected linters 30 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts 31 importkeys -- imports signing keys from env 32 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 33 nsis -- creates a Windows NSIS installer 34 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 35 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "log" 50 "os" 51 "os/exec" 52 "path" 53 "path/filepath" 54 "regexp" 55 "runtime" 56 "strconv" 57 "strings" 58 "time" 59 60 "github.com/cespare/cp" 61 "github.com/ethereum/go-ethereum/common" 62 "github.com/ethereum/go-ethereum/crypto/signify" 63 "github.com/ethereum/go-ethereum/internal/build" 64 "github.com/ethereum/go-ethereum/params" 65 ) 66 67 var ( 68 // Files that end up in the geth*.zip archive. 69 gethArchiveFiles = []string{ 70 "COPYING", 71 executablePath("geth"), 72 } 73 74 // Files that end up in the geth-alltools*.zip archive. 75 allToolsArchiveFiles = []string{ 76 "COPYING", 77 executablePath("abigen"), 78 executablePath("bootnode"), 79 executablePath("evm"), 80 executablePath("geth"), 81 executablePath("puppeth"), 82 executablePath("rlpdump"), 83 executablePath("clef"), 84 } 85 86 // A debian package is created for all executables listed here. 87 debExecutables = []debExecutable{ 88 { 89 BinaryName: "abigen", 90 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 91 }, 92 { 93 BinaryName: "bootnode", 94 Description: "Ethereum bootnode.", 95 }, 96 { 97 BinaryName: "evm", 98 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 99 }, 100 { 101 BinaryName: "geth", 102 Description: "Ethereum CLI client.", 103 }, 104 { 105 BinaryName: "puppeth", 106 Description: "Ethereum private network manager.", 107 }, 108 { 109 BinaryName: "rlpdump", 110 Description: "Developer utility tool that prints RLP structures.", 111 }, 112 { 113 BinaryName: "clef", 114 Description: "Ethereum account management tool.", 115 }, 116 } 117 118 // A debian package is created for all executables listed here. 119 debEthereum = debPackage{ 120 Name: "ethereum", 121 Version: params.Version, 122 Executables: debExecutables, 123 } 124 125 // Debian meta packages to build and push to Ubuntu PPA 126 debPackages = []debPackage{ 127 debEthereum, 128 } 129 130 // Distros for which packages are created. 131 // Note: vivid is unsupported because there is no golang-1.6 package for it. 132 // Note: the following Ubuntu releases have been officially deprecated on Launchpad: 133 // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite 134 debDistroGoBoots = map[string]string{ 135 "trusty": "golang-1.11", // EOL: 04/2024 136 "xenial": "golang-go", // EOL: 04/2026 137 "bionic": "golang-go", // EOL: 04/2028 138 "focal": "golang-go", // EOL: 04/2030 139 "impish": "golang-go", // EOL: 07/2022 140 "jammy": "golang-go", // EOL: 04/2032 141 //"kinetic": "golang-go", // EOL: 07/2023 142 } 143 144 debGoBootPaths = map[string]string{ 145 "golang-1.11": "/usr/lib/go-1.11", 146 "golang-go": "/usr/lib/go", 147 } 148 149 // This is the version of go that will be downloaded by 150 // 151 // go run ci.go install -dlgo 152 dlgoVersion = "1.18.4" 153 ) 154 155 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 156 157 func executablePath(name string) string { 158 if runtime.GOOS == "windows" { 159 name += ".exe" 160 } 161 return filepath.Join(GOBIN, name) 162 } 163 164 func main() { 165 log.SetFlags(log.Lshortfile) 166 167 if !common.FileExist(filepath.Join("build", "ci.go")) { 168 log.Fatal("this script must be run from the root of the repository") 169 } 170 if len(os.Args) < 2 { 171 log.Fatal("need subcommand as first argument") 172 } 173 switch os.Args[1] { 174 case "install": 175 doInstall(os.Args[2:]) 176 case "test": 177 doTest(os.Args[2:]) 178 case "lint": 179 doLint(os.Args[2:]) 180 case "archive": 181 doArchive(os.Args[2:]) 182 case "docker": 183 doDocker(os.Args[2:]) 184 case "debsrc": 185 doDebianSource(os.Args[2:]) 186 case "nsis": 187 doWindowsInstaller(os.Args[2:]) 188 case "aar": 189 doAndroidArchive(os.Args[2:]) 190 case "xcode": 191 doXCodeFramework(os.Args[2:]) 192 case "purge": 193 doPurge(os.Args[2:]) 194 default: 195 log.Fatal("unknown command ", os.Args[1]) 196 } 197 } 198 199 // Compiling 200 201 func doInstall(cmdline []string) { 202 var ( 203 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 204 arch = flag.String("arch", "", "Architecture to cross build for") 205 cc = flag.String("cc", "", "C compiler to cross build with") 206 ) 207 flag.CommandLine.Parse(cmdline) 208 209 // Configure the toolchain. 210 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 211 if *dlgo { 212 csdb := build.MustLoadChecksums("build/checksums.txt") 213 tc.Root = build.DownloadGo(csdb, dlgoVersion) 214 } 215 216 // Configure the build. 217 env := build.Env() 218 gobuild := tc.Go("build", buildFlags(env)...) 219 220 // arm64 CI builders are memory-constrained and can't handle concurrent builds, 221 // better disable it. This check isn't the best, it should probably 222 // check for something in env instead. 223 if env.CI && runtime.GOARCH == "arm64" { 224 gobuild.Args = append(gobuild.Args, "-p", "1") 225 } 226 227 // Disable CLI markdown doc generation in release builds. 228 gobuild.Args = append(gobuild.Args, "-tags", "urfave_cli_no_docs") 229 230 // We use -trimpath to avoid leaking local paths into the built executables. 231 gobuild.Args = append(gobuild.Args, "-trimpath") 232 233 // Show packages during build. 234 gobuild.Args = append(gobuild.Args, "-v") 235 236 // Now we choose what we're even building. 237 // Default: collect all 'main' packages in cmd/ and build those. 238 packages := flag.Args() 239 if len(packages) == 0 { 240 packages = build.FindMainPackages("./cmd") 241 } 242 243 // Do the build! 244 for _, pkg := range packages { 245 args := make([]string, len(gobuild.Args)) 246 copy(args, gobuild.Args) 247 args = append(args, "-o", executablePath(path.Base(pkg))) 248 args = append(args, pkg) 249 build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) 250 } 251 } 252 253 // buildFlags returns the go tool flags for building. 254 func buildFlags(env build.Environment) (flags []string) { 255 var ld []string 256 if env.Commit != "" { 257 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 258 ld = append(ld, "-X", "main.gitDate="+env.Date) 259 } 260 // Strip DWARF on darwin. This used to be required for certain things, 261 // and there is no downside to this, so we just keep doing it. 262 if runtime.GOOS == "darwin" { 263 ld = append(ld, "-s") 264 } 265 // Enforce the stacksize to 8M, which is the case on most platforms apart from 266 // alpine Linux. 267 if runtime.GOOS == "linux" { 268 ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") 269 } 270 if len(ld) > 0 { 271 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 272 } 273 return flags 274 } 275 276 // Running The Tests 277 // 278 // "tests" also includes static analysis tools such as vet. 279 280 func doTest(cmdline []string) { 281 var ( 282 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 283 arch = flag.String("arch", "", "Run tests for given architecture") 284 cc = flag.String("cc", "", "Sets C compiler binary") 285 coverage = flag.Bool("coverage", false, "Whether to record code coverage") 286 verbose = flag.Bool("v", false, "Whether to log verbosely") 287 race = flag.Bool("race", false, "Execute the race detector") 288 ) 289 flag.CommandLine.Parse(cmdline) 290 291 // Configure the toolchain. 292 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 293 if *dlgo { 294 csdb := build.MustLoadChecksums("build/checksums.txt") 295 tc.Root = build.DownloadGo(csdb, dlgoVersion) 296 } 297 gotest := tc.Go("test") 298 299 // Test a single package at a time. CI builders are slow 300 // and some tests run into timeouts under load. 301 gotest.Args = append(gotest.Args, "-p", "1") 302 if *coverage { 303 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 304 } 305 if *verbose { 306 gotest.Args = append(gotest.Args, "-v") 307 } 308 if *race { 309 gotest.Args = append(gotest.Args, "-race") 310 } 311 312 packages := []string{"./..."} 313 if len(flag.CommandLine.Args()) > 0 { 314 packages = flag.CommandLine.Args() 315 } 316 gotest.Args = append(gotest.Args, packages...) 317 build.MustRun(gotest) 318 } 319 320 // doLint runs golangci-lint on requested packages. 321 func doLint(cmdline []string) { 322 var ( 323 cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") 324 ) 325 flag.CommandLine.Parse(cmdline) 326 packages := []string{"./..."} 327 if len(flag.CommandLine.Args()) > 0 { 328 packages = flag.CommandLine.Args() 329 } 330 331 linter := downloadLinter(*cachedir) 332 lflags := []string{"run", "--config", ".golangci.yml"} 333 build.MustRunCommand(linter, append(lflags, packages...)...) 334 fmt.Println("You have achieved perfection.") 335 } 336 337 // downloadLinter downloads and unpacks golangci-lint. 338 func downloadLinter(cachedir string) string { 339 const version = "1.46.2" 340 341 csdb := build.MustLoadChecksums("build/checksums.txt") 342 arch := runtime.GOARCH 343 ext := ".tar.gz" 344 345 if runtime.GOOS == "windows" { 346 ext = ".zip" 347 } 348 if arch == "arm" { 349 arch += "v" + os.Getenv("GOARM") 350 } 351 base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) 352 url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) 353 archivePath := filepath.Join(cachedir, base+ext) 354 if err := csdb.DownloadFile(url, archivePath); err != nil { 355 log.Fatal(err) 356 } 357 if err := build.ExtractArchive(archivePath, cachedir); err != nil { 358 log.Fatal(err) 359 } 360 return filepath.Join(cachedir, base, "golangci-lint") 361 } 362 363 // Release Packaging 364 func doArchive(cmdline []string) { 365 var ( 366 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 367 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 368 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 369 signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`) 370 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 371 ext string 372 ) 373 flag.CommandLine.Parse(cmdline) 374 switch *atype { 375 case "zip": 376 ext = ".zip" 377 case "tar": 378 ext = ".tar.gz" 379 default: 380 log.Fatal("unknown archive type: ", atype) 381 } 382 383 var ( 384 env = build.Env() 385 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 386 geth = "geth-" + basegeth + ext 387 alltools = "geth-alltools-" + basegeth + ext 388 ) 389 maybeSkipArchive(env) 390 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 391 log.Fatal(err) 392 } 393 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 394 log.Fatal(err) 395 } 396 for _, archive := range []string{geth, alltools} { 397 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 398 log.Fatal(err) 399 } 400 } 401 } 402 403 func archiveBasename(arch string, archiveVersion string) string { 404 platform := runtime.GOOS + "-" + arch 405 if arch == "arm" { 406 platform += os.Getenv("GOARM") 407 } 408 if arch == "android" { 409 platform = "android-all" 410 } 411 if arch == "ios" { 412 platform = "ios-all" 413 } 414 return platform + "-" + archiveVersion 415 } 416 417 func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error { 418 // If signing was requested, generate the signature files 419 if signer != "" { 420 key := getenvBase64(signer) 421 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 422 return err 423 } 424 } 425 if signifyVar != "" { 426 key := os.Getenv(signifyVar) 427 untrustedComment := "verify with geth-release.pub" 428 trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123)) 429 if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil { 430 return err 431 } 432 } 433 // If uploading to Azure was requested, push the archive possibly with its signature 434 if blobstore != "" { 435 auth := build.AzureBlobstoreConfig{ 436 Account: strings.Split(blobstore, "/")[0], 437 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 438 Container: strings.SplitN(blobstore, "/", 2)[1], 439 } 440 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 441 return err 442 } 443 if signer != "" { 444 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 445 return err 446 } 447 } 448 if signifyVar != "" { 449 if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { 450 return err 451 } 452 } 453 } 454 return nil 455 } 456 457 // skips archiving for some build configurations. 458 func maybeSkipArchive(env build.Environment) { 459 if env.IsPullRequest { 460 log.Printf("skipping archive creation because this is a PR build") 461 os.Exit(0) 462 } 463 if env.IsCronJob { 464 log.Printf("skipping archive creation because this is a cron job") 465 os.Exit(0) 466 } 467 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 468 log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) 469 os.Exit(0) 470 } 471 } 472 473 // Builds the docker images and optionally uploads them to Docker Hub. 474 func doDocker(cmdline []string) { 475 var ( 476 image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) 477 manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) 478 upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) 479 ) 480 flag.CommandLine.Parse(cmdline) 481 482 // Skip building and pushing docker images for PR builds 483 env := build.Env() 484 maybeSkipArchive(env) 485 486 // Retrieve the upload credentials and authenticate 487 user := getenvBase64("DOCKER_HUB_USERNAME") 488 pass := getenvBase64("DOCKER_HUB_PASSWORD") 489 490 if len(user) > 0 && len(pass) > 0 { 491 auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") 492 auther.Stdin = bytes.NewReader(pass) 493 build.MustRun(auther) 494 } 495 // Retrieve the version infos to build and push to the following paths: 496 // - ethereum/client-go:latest - Pushes to the master branch, Geth only 497 // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only 498 // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools 499 // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools 500 // - ethereum/client-go:release-<major>.<minor> - Version tag publish on GitHub, Geth only 501 // - ethereum/client-go:alltools-release-<major>.<minor> - Version tag publish on GitHub, Geth & tools 502 // - ethereum/client-go:v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth only 503 // - ethereum/client-go:alltools-v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth & tools 504 var tags []string 505 506 switch { 507 case env.Branch == "master": 508 tags = []string{"latest"} 509 case strings.HasPrefix(env.Tag, "v1."): 510 tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} 511 } 512 // If architecture specific image builds are requested, build and push them 513 if *image { 514 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") 515 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") 516 517 // Tag and upload the images to Docker Hub 518 for _, tag := range tags { 519 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) 520 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) 521 522 // If the image already exists (non version tag), check the build 523 // number to prevent overwriting a newer commit if concurrent builds 524 // are running. This is still a tiny bit racey if two published are 525 // done at the same time, but that's extremely unlikely even on the 526 // master branch. 527 for _, img := range []string{gethImage, toolImage} { 528 if exec.Command("docker", "pull", img).Run() != nil { 529 continue // Generally the only failure is a missing image, which is good 530 } 531 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 532 if err != nil { 533 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 534 } 535 buildnum = bytes.TrimSpace(buildnum) 536 537 if len(buildnum) > 0 && len(env.Buildnum) > 0 { 538 oldnum, err := strconv.Atoi(string(buildnum)) 539 if err != nil { 540 log.Fatalf("Failed to parse old image build number: %v", err) 541 } 542 newnum, err := strconv.Atoi(env.Buildnum) 543 if err != nil { 544 log.Fatalf("Failed to parse current build number: %v", err) 545 } 546 if oldnum > newnum { 547 log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) 548 } else { 549 log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) 550 } 551 } 552 } 553 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) 554 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) 555 build.MustRunCommand("docker", "push", gethImage) 556 build.MustRunCommand("docker", "push", toolImage) 557 } 558 } 559 // If multi-arch image manifest push is requested, assemble it 560 if len(*manifest) != 0 { 561 // Since different architectures are pushed by different builders, wait 562 // until all required images are updated. 563 var mismatch bool 564 for i := 0; i < 2; i++ { // 2 attempts, second is race check 565 mismatch = false // hope there's no mismatch now 566 567 for _, tag := range tags { 568 for _, arch := range strings.Split(*manifest, ",") { 569 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) 570 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) 571 572 for _, img := range []string{gethImage, toolImage} { 573 if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { 574 log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) 575 mismatch = true 576 break 577 } 578 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 579 if err != nil { 580 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 581 } 582 buildnum = bytes.TrimSpace(buildnum) 583 584 if string(buildnum) != env.Buildnum { 585 log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) 586 mismatch = true 587 break 588 } 589 } 590 if mismatch { 591 break 592 } 593 } 594 if mismatch { 595 break 596 } 597 } 598 if mismatch { 599 // Build numbers mismatching, retry in a short time to 600 // avoid concurrent failes in both publisher images. If 601 // however the retry failed too, it means the concurrent 602 // builder is still crunching, let that do the publish. 603 if i == 0 { 604 time.Sleep(30 * time.Second) 605 } 606 continue 607 } 608 break 609 } 610 if mismatch { 611 log.Println("Relinquishing publish to other builder") 612 return 613 } 614 // Assemble and push the Geth manifest image 615 for _, tag := range tags { 616 gethImage := fmt.Sprintf("%s:%s", *upload, tag) 617 618 var gethSubImages []string 619 for _, arch := range strings.Split(*manifest, ",") { 620 gethSubImages = append(gethSubImages, gethImage+"-"+arch) 621 } 622 build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) 623 build.MustRunCommand("docker", "manifest", "push", gethImage) 624 } 625 // Assemble and push the alltools manifest image 626 for _, tag := range tags { 627 toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) 628 629 var toolSubImages []string 630 for _, arch := range strings.Split(*manifest, ",") { 631 toolSubImages = append(toolSubImages, toolImage+"-"+arch) 632 } 633 build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) 634 build.MustRunCommand("docker", "manifest", "push", toolImage) 635 } 636 } 637 } 638 639 // Debian Packaging 640 func doDebianSource(cmdline []string) { 641 var ( 642 cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) 643 signer = flag.String("signer", "", `Signing key name, also used as package author`) 644 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 645 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 646 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 647 now = time.Now() 648 ) 649 flag.CommandLine.Parse(cmdline) 650 *workdir = makeWorkdir(*workdir) 651 env := build.Env() 652 tc := new(build.GoToolchain) 653 maybeSkipArchive(env) 654 655 // Import the signing key. 656 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 657 gpg := exec.Command("gpg", "--import") 658 gpg.Stdin = bytes.NewReader(key) 659 build.MustRun(gpg) 660 } 661 662 // Download and verify the Go source package. 663 gobundle := downloadGoSources(*cachedir) 664 665 // Download all the dependencies needed to build the sources and run the ci script 666 srcdepfetch := tc.Go("mod", "download") 667 srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 668 build.MustRun(srcdepfetch) 669 670 cidepfetch := tc.Go("run", "./build/ci.go") 671 cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 672 cidepfetch.Run() // Command fails, don't care, we only need the deps to start it 673 674 // Create Debian packages and upload them. 675 for _, pkg := range debPackages { 676 for distro, goboot := range debDistroGoBoots { 677 // Prepare the debian package with the go-ethereum sources. 678 meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 679 pkgdir := stageDebianSource(*workdir, meta) 680 681 // Add Go source code 682 if err := build.ExtractArchive(gobundle, pkgdir); err != nil { 683 log.Fatalf("Failed to extract Go sources: %v", err) 684 } 685 if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { 686 log.Fatalf("Failed to rename Go source folder: %v", err) 687 } 688 // Add all dependency modules in compressed form 689 os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) 690 if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { 691 log.Fatalf("Failed to copy Go module dependencies: %v", err) 692 } 693 // Run the packaging and upload to the PPA 694 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") 695 debuild.Dir = pkgdir 696 build.MustRun(debuild) 697 698 var ( 699 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 700 source = filepath.Join(*workdir, basename+".tar.xz") 701 dsc = filepath.Join(*workdir, basename+".dsc") 702 changes = filepath.Join(*workdir, basename+"_source.changes") 703 buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo") 704 ) 705 if *signer != "" { 706 build.MustRunCommand("debsign", changes) 707 } 708 if *upload != "" { 709 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo}) 710 } 711 } 712 } 713 } 714 715 // downloadGoSources downloads the Go source tarball. 716 func downloadGoSources(cachedir string) string { 717 csdb := build.MustLoadChecksums("build/checksums.txt") 718 file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) 719 url := "https://dl.google.com/go/" + file 720 dst := filepath.Join(cachedir, file) 721 if err := csdb.DownloadFile(url, dst); err != nil { 722 log.Fatal(err) 723 } 724 return dst 725 } 726 727 func ppaUpload(workdir, ppa, sshUser string, files []string) { 728 p := strings.Split(ppa, "/") 729 if len(p) != 2 { 730 log.Fatal("-upload PPA name must contain single /") 731 } 732 if sshUser == "" { 733 sshUser = p[0] 734 } 735 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 736 // Create the SSH identity file if it doesn't exist. 737 var idfile string 738 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 739 idfile = filepath.Join(workdir, "sshkey") 740 if !common.FileExist(idfile) { 741 os.WriteFile(idfile, sshkey, 0600) 742 } 743 } 744 // Upload 745 dest := sshUser + "@ppa.launchpad.net" 746 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 747 log.Fatal(err) 748 } 749 } 750 751 func getenvBase64(variable string) []byte { 752 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 753 if err != nil { 754 log.Fatal("invalid base64 " + variable) 755 } 756 return []byte(dec) 757 } 758 759 func makeWorkdir(wdflag string) string { 760 var err error 761 if wdflag != "" { 762 err = os.MkdirAll(wdflag, 0744) 763 } else { 764 wdflag, err = os.MkdirTemp("", "geth-build-") 765 } 766 if err != nil { 767 log.Fatal(err) 768 } 769 return wdflag 770 } 771 772 func isUnstableBuild(env build.Environment) bool { 773 if env.Tag != "" { 774 return false 775 } 776 return true 777 } 778 779 type debPackage struct { 780 Name string // the name of the Debian package to produce, e.g. "ethereum" 781 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 782 Executables []debExecutable // executables to be included in the package 783 } 784 785 type debMetadata struct { 786 Env build.Environment 787 GoBootPackage string 788 GoBootPath string 789 790 PackageName string 791 792 // go-ethereum version being built. Note that this 793 // is not the debian package version. The package version 794 // is constructed by VersionString. 795 Version string 796 797 Author string // "name <email>", also selects signing key 798 Distro, Time string 799 Executables []debExecutable 800 } 801 802 type debExecutable struct { 803 PackageName string 804 BinaryName string 805 Description string 806 } 807 808 // Package returns the name of the package if present, or 809 // fallbacks to BinaryName 810 func (d debExecutable) Package() string { 811 if d.PackageName != "" { 812 return d.PackageName 813 } 814 return d.BinaryName 815 } 816 817 func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 818 if author == "" { 819 // No signing key, use default author. 820 author = "Ethereum Builds <fjl@ethereum.org>" 821 } 822 return debMetadata{ 823 GoBootPackage: goboot, 824 GoBootPath: debGoBootPaths[goboot], 825 PackageName: name, 826 Env: env, 827 Author: author, 828 Distro: distro, 829 Version: version, 830 Time: t.Format(time.RFC1123Z), 831 Executables: exes, 832 } 833 } 834 835 // Name returns the name of the metapackage that depends 836 // on all executable packages. 837 func (meta debMetadata) Name() string { 838 if isUnstableBuild(meta.Env) { 839 return meta.PackageName + "-unstable" 840 } 841 return meta.PackageName 842 } 843 844 // VersionString returns the debian version of the packages. 845 func (meta debMetadata) VersionString() string { 846 vsn := meta.Version 847 if meta.Env.Buildnum != "" { 848 vsn += "+build" + meta.Env.Buildnum 849 } 850 if meta.Distro != "" { 851 vsn += "+" + meta.Distro 852 } 853 return vsn 854 } 855 856 // ExeList returns the list of all executable packages. 857 func (meta debMetadata) ExeList() string { 858 names := make([]string, len(meta.Executables)) 859 for i, e := range meta.Executables { 860 names[i] = meta.ExeName(e) 861 } 862 return strings.Join(names, ", ") 863 } 864 865 // ExeName returns the package name of an executable package. 866 func (meta debMetadata) ExeName(exe debExecutable) string { 867 if isUnstableBuild(meta.Env) { 868 return exe.Package() + "-unstable" 869 } 870 return exe.Package() 871 } 872 873 // ExeConflicts returns the content of the Conflicts field 874 // for executable packages. 875 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 876 if isUnstableBuild(meta.Env) { 877 // Set up the conflicts list so that the *-unstable packages 878 // cannot be installed alongside the regular version. 879 // 880 // https://www.debian.org/doc/debian-policy/ch-relationships.html 881 // is very explicit about Conflicts: and says that Breaks: should 882 // be preferred and the conflicting files should be handled via 883 // alternates. We might do this eventually but using a conflict is 884 // easier now. 885 return "ethereum, " + exe.Package() 886 } 887 return "" 888 } 889 890 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 891 pkg := meta.Name() + "-" + meta.VersionString() 892 pkgdir = filepath.Join(tmpdir, pkg) 893 if err := os.Mkdir(pkgdir, 0755); err != nil { 894 log.Fatal(err) 895 } 896 // Copy the source code. 897 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 898 899 // Put the debian build files in place. 900 debian := filepath.Join(pkgdir, "debian") 901 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 902 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 903 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 904 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 905 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 906 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 907 for _, exe := range meta.Executables { 908 install := filepath.Join(debian, meta.ExeName(exe)+".install") 909 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 910 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 911 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 912 } 913 return pkgdir 914 } 915 916 // Windows installer 917 func doWindowsInstaller(cmdline []string) { 918 // Parse the flags and make skip installer generation on PRs 919 var ( 920 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 921 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 922 signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) 923 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 924 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 925 ) 926 flag.CommandLine.Parse(cmdline) 927 *workdir = makeWorkdir(*workdir) 928 env := build.Env() 929 maybeSkipArchive(env) 930 931 // Aggregate binaries that are included in the installer 932 var ( 933 devTools []string 934 allTools []string 935 gethTool string 936 ) 937 for _, file := range allToolsArchiveFiles { 938 if file == "COPYING" { // license, copied later 939 continue 940 } 941 allTools = append(allTools, filepath.Base(file)) 942 if filepath.Base(file) == "geth.exe" { 943 gethTool = file 944 } else { 945 devTools = append(devTools, file) 946 } 947 } 948 949 // Render NSIS scripts: Installer NSIS contains two installer sections, 950 // first section contains the geth binary, second section holds the dev tools. 951 templateData := map[string]interface{}{ 952 "License": "COPYING", 953 "Geth": gethTool, 954 "DevTools": devTools, 955 } 956 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 957 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 958 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 959 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 960 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 961 if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { 962 log.Fatalf("Failed to copy SimpleFC.dll: %v", err) 963 } 964 if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { 965 log.Fatalf("Failed to copy copyright note: %v", err) 966 } 967 // Build the installer. This assumes that all the needed files have been previously 968 // built (don't mix building and packaging to keep cross compilation complexity to a 969 // minimum). 970 version := strings.Split(params.Version, ".") 971 if env.Commit != "" { 972 version[2] += "-" + env.Commit[:8] 973 } 974 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 975 build.MustRunCommand("makensis.exe", 976 "/DOUTPUTFILE="+installer, 977 "/DMAJORVERSION="+version[0], 978 "/DMINORVERSION="+version[1], 979 "/DBUILDVERSION="+version[2], 980 "/DARCH="+*arch, 981 filepath.Join(*workdir, "geth.nsi"), 982 ) 983 // Sign and publish installer. 984 if err := archiveUpload(installer, *upload, *signer, *signify); err != nil { 985 log.Fatal(err) 986 } 987 } 988 989 // Android archives 990 991 func doAndroidArchive(cmdline []string) { 992 var ( 993 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 994 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 995 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`) 996 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 997 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 998 ) 999 flag.CommandLine.Parse(cmdline) 1000 env := build.Env() 1001 tc := new(build.GoToolchain) 1002 1003 // Sanity check that the SDK and NDK are installed and set 1004 if os.Getenv("ANDROID_HOME") == "" { 1005 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 1006 } 1007 1008 // Build gomobile. 1009 install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest") 1010 install.Env = append(install.Env) 1011 build.MustRun(install) 1012 1013 // Ensure all dependencies are available. This is required to make 1014 // gomobile bind work because it expects go.sum to contain all checksums. 1015 build.MustRun(tc.Go("mod", "download")) 1016 1017 // Build the Android archive and Maven resources 1018 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) 1019 1020 if *local { 1021 // If we're building locally, copy bundle to build dir and skip Maven 1022 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 1023 os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar")) 1024 return 1025 } 1026 meta := newMavenMetadata(env) 1027 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 1028 1029 // Skip Maven deploy and Azure upload for PR builds 1030 maybeSkipArchive(env) 1031 1032 // Sign and upload the archive to Azure 1033 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 1034 os.Rename("geth.aar", archive) 1035 1036 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 1037 log.Fatal(err) 1038 } 1039 // Sign and upload all the artifacts to Maven Central 1040 os.Rename(archive, meta.Package+".aar") 1041 if *signer != "" && *deploy != "" { 1042 // Import the signing key into the local GPG instance 1043 key := getenvBase64(*signer) 1044 gpg := exec.Command("gpg", "--import") 1045 gpg.Stdin = bytes.NewReader(key) 1046 build.MustRun(gpg) 1047 keyID, err := build.PGPKeyID(string(key)) 1048 if err != nil { 1049 log.Fatal(err) 1050 } 1051 // Upload the artifacts to Sonatype and/or Maven Central 1052 repo := *deploy + "/service/local/staging/deploy/maven2" 1053 if meta.Develop { 1054 repo = *deploy + "/content/repositories/snapshots" 1055 } 1056 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 1057 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 1058 "-Dgpg.keyname="+keyID, 1059 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 1060 } 1061 } 1062 1063 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 1064 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 1065 cmd.Args = append(cmd.Args, args...) 1066 cmd.Env = []string{ 1067 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 1068 } 1069 for _, e := range os.Environ() { 1070 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") { 1071 continue 1072 } 1073 cmd.Env = append(cmd.Env, e) 1074 } 1075 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 1076 return cmd 1077 } 1078 1079 type mavenMetadata struct { 1080 Version string 1081 Package string 1082 Develop bool 1083 Contributors []mavenContributor 1084 } 1085 1086 type mavenContributor struct { 1087 Name string 1088 Email string 1089 } 1090 1091 func newMavenMetadata(env build.Environment) mavenMetadata { 1092 // Collect the list of authors from the repo root 1093 contribs := []mavenContributor{} 1094 if authors, err := os.Open("AUTHORS"); err == nil { 1095 defer authors.Close() 1096 1097 scanner := bufio.NewScanner(authors) 1098 for scanner.Scan() { 1099 // Skip any whitespace from the authors list 1100 line := strings.TrimSpace(scanner.Text()) 1101 if line == "" || line[0] == '#' { 1102 continue 1103 } 1104 // Split the author and insert as a contributor 1105 re := regexp.MustCompile("([^<]+) <(.+)>") 1106 parts := re.FindStringSubmatch(line) 1107 if len(parts) == 3 { 1108 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 1109 } 1110 } 1111 } 1112 // Render the version and package strings 1113 version := params.Version 1114 if isUnstableBuild(env) { 1115 version += "-SNAPSHOT" 1116 } 1117 return mavenMetadata{ 1118 Version: version, 1119 Package: "geth-" + version, 1120 Develop: isUnstableBuild(env), 1121 Contributors: contribs, 1122 } 1123 } 1124 1125 // XCode frameworks 1126 1127 func doXCodeFramework(cmdline []string) { 1128 var ( 1129 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1130 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 1131 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`) 1132 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 1133 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 1134 ) 1135 flag.CommandLine.Parse(cmdline) 1136 env := build.Env() 1137 tc := new(build.GoToolchain) 1138 1139 // Build gomobile. 1140 build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 1141 1142 // Build the iOS XCode framework 1143 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") 1144 1145 if *local { 1146 // If we're building locally, use the build folder and stop afterwards 1147 bind.Dir = GOBIN 1148 build.MustRun(bind) 1149 return 1150 } 1151 1152 // Create the archive. 1153 maybeSkipArchive(env) 1154 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 1155 if err := os.MkdirAll(archive, 0755); err != nil { 1156 log.Fatal(err) 1157 } 1158 bind.Dir, _ = filepath.Abs(archive) 1159 build.MustRun(bind) 1160 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 1161 1162 // Sign and upload the framework to Azure 1163 if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { 1164 log.Fatal(err) 1165 } 1166 // Prepare and upload a PodSpec to CocoaPods 1167 if *deploy != "" { 1168 meta := newPodMetadata(env, archive) 1169 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 1170 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings") 1171 } 1172 } 1173 1174 type podMetadata struct { 1175 Version string 1176 Commit string 1177 Archive string 1178 Contributors []podContributor 1179 } 1180 1181 type podContributor struct { 1182 Name string 1183 Email string 1184 } 1185 1186 func newPodMetadata(env build.Environment, archive string) podMetadata { 1187 // Collect the list of authors from the repo root 1188 contribs := []podContributor{} 1189 if authors, err := os.Open("AUTHORS"); err == nil { 1190 defer authors.Close() 1191 1192 scanner := bufio.NewScanner(authors) 1193 for scanner.Scan() { 1194 // Skip any whitespace from the authors list 1195 line := strings.TrimSpace(scanner.Text()) 1196 if line == "" || line[0] == '#' { 1197 continue 1198 } 1199 // Split the author and insert as a contributor 1200 re := regexp.MustCompile("([^<]+) <(.+)>") 1201 parts := re.FindStringSubmatch(line) 1202 if len(parts) == 3 { 1203 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 1204 } 1205 } 1206 } 1207 version := params.Version 1208 if isUnstableBuild(env) { 1209 version += "-unstable." + env.Buildnum 1210 } 1211 return podMetadata{ 1212 Archive: archive, 1213 Version: version, 1214 Commit: env.Commit, 1215 Contributors: contribs, 1216 } 1217 } 1218 1219 // Binary distribution cleanups 1220 1221 func doPurge(cmdline []string) { 1222 var ( 1223 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1224 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1225 ) 1226 flag.CommandLine.Parse(cmdline) 1227 1228 if env := build.Env(); !env.IsCronJob { 1229 log.Printf("skipping because not a cron job") 1230 os.Exit(0) 1231 } 1232 // Create the azure authentication and list the current archives 1233 auth := build.AzureBlobstoreConfig{ 1234 Account: strings.Split(*store, "/")[0], 1235 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1236 Container: strings.SplitN(*store, "/", 2)[1], 1237 } 1238 blobs, err := build.AzureBlobstoreList(auth) 1239 if err != nil { 1240 log.Fatal(err) 1241 } 1242 fmt.Printf("Found %d blobs\n", len(blobs)) 1243 1244 // Iterate over the blobs, collect and sort all unstable builds 1245 for i := 0; i < len(blobs); i++ { 1246 if !strings.Contains(*blobs[i].Name, "unstable") { 1247 blobs = append(blobs[:i], blobs[i+1:]...) 1248 i-- 1249 } 1250 } 1251 for i := 0; i < len(blobs); i++ { 1252 for j := i + 1; j < len(blobs); j++ { 1253 if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) { 1254 blobs[i], blobs[j] = blobs[j], blobs[i] 1255 } 1256 } 1257 } 1258 // Filter out all archives more recent that the given threshold 1259 for i, blob := range blobs { 1260 if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1261 blobs = blobs[:i] 1262 break 1263 } 1264 } 1265 fmt.Printf("Deleting %d blobs\n", len(blobs)) 1266 // Delete all marked as such and return 1267 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1268 log.Fatal(err) 1269 } 1270 }