github.com/codysnider/go-ethereum@v1.10.18-0.20220420071915-14f4ae99222a/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 //go:build none 18 // +build none 19 20 /* 21 The ci command is called from Continuous Integration scripts. 22 23 Usage: go run build/ci.go <command> <command flags/arguments> 24 25 Available commands are: 26 27 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 28 test [ -coverage ] [ packages... ] -- runs the tests 29 lint -- runs certain pre-selected linters 30 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts 31 importkeys -- imports signing keys from env 32 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 33 nsis -- creates a Windows NSIS installer 34 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 35 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "io/ioutil" 50 "log" 51 "os" 52 "os/exec" 53 "path" 54 "path/filepath" 55 "regexp" 56 "runtime" 57 "strconv" 58 "strings" 59 "time" 60 61 "github.com/cespare/cp" 62 "github.com/ethereum/go-ethereum/crypto/signify" 63 "github.com/ethereum/go-ethereum/internal/build" 64 "github.com/ethereum/go-ethereum/params" 65 ) 66 67 var ( 68 // Files that end up in the geth*.zip archive. 69 gethArchiveFiles = []string{ 70 "COPYING", 71 executablePath("geth"), 72 } 73 74 // Files that end up in the geth-alltools*.zip archive. 75 allToolsArchiveFiles = []string{ 76 "COPYING", 77 executablePath("abigen"), 78 executablePath("bootnode"), 79 executablePath("evm"), 80 executablePath("geth"), 81 executablePath("puppeth"), 82 executablePath("rlpdump"), 83 executablePath("clef"), 84 } 85 86 // A debian package is created for all executables listed here. 87 debExecutables = []debExecutable{ 88 { 89 BinaryName: "abigen", 90 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 91 }, 92 { 93 BinaryName: "bootnode", 94 Description: "Ethereum bootnode.", 95 }, 96 { 97 BinaryName: "evm", 98 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 99 }, 100 { 101 BinaryName: "geth", 102 Description: "Ethereum CLI client.", 103 }, 104 { 105 BinaryName: "puppeth", 106 Description: "Ethereum private network manager.", 107 }, 108 { 109 BinaryName: "rlpdump", 110 Description: "Developer utility tool that prints RLP structures.", 111 }, 112 { 113 BinaryName: "clef", 114 Description: "Ethereum account management tool.", 115 }, 116 } 117 118 // A debian package is created for all executables listed here. 119 debEthereum = debPackage{ 120 Name: "ethereum", 121 Version: params.Version, 122 Executables: debExecutables, 123 } 124 125 // Debian meta packages to build and push to Ubuntu PPA 126 debPackages = []debPackage{ 127 debEthereum, 128 } 129 130 // Distros for which packages are created. 131 // Note: vivid is unsupported because there is no golang-1.6 package for it. 132 // Note: the following Ubuntu releases have been officially deprecated on Launchpad: 133 // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite 134 debDistroGoBoots = map[string]string{ 135 "trusty": "golang-1.11", // EOL: 04/2024 136 "xenial": "golang-go", // EOL: 04/2026 137 "bionic": "golang-go", // EOL: 04/2028 138 "focal": "golang-go", // EOL: 04/2030 139 "impish": "golang-go", // EOL: 07/2022 140 // "jammy": "golang-go", // EOL: 04/2027 141 } 142 143 debGoBootPaths = map[string]string{ 144 "golang-1.11": "/usr/lib/go-1.11", 145 "golang-go": "/usr/lib/go", 146 } 147 148 // This is the version of go that will be downloaded by 149 // 150 // go run ci.go install -dlgo 151 dlgoVersion = "1.18.1" 152 ) 153 154 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 155 156 func executablePath(name string) string { 157 if runtime.GOOS == "windows" { 158 name += ".exe" 159 } 160 return filepath.Join(GOBIN, name) 161 } 162 163 func main() { 164 log.SetFlags(log.Lshortfile) 165 166 if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) { 167 log.Fatal("this script must be run from the root of the repository") 168 } 169 if len(os.Args) < 2 { 170 log.Fatal("need subcommand as first argument") 171 } 172 switch os.Args[1] { 173 case "install": 174 doInstall(os.Args[2:]) 175 case "test": 176 doTest(os.Args[2:]) 177 case "lint": 178 doLint(os.Args[2:]) 179 case "archive": 180 doArchive(os.Args[2:]) 181 case "docker": 182 doDocker(os.Args[2:]) 183 case "debsrc": 184 doDebianSource(os.Args[2:]) 185 case "nsis": 186 doWindowsInstaller(os.Args[2:]) 187 case "aar": 188 doAndroidArchive(os.Args[2:]) 189 case "xcode": 190 doXCodeFramework(os.Args[2:]) 191 case "purge": 192 doPurge(os.Args[2:]) 193 default: 194 log.Fatal("unknown command ", os.Args[1]) 195 } 196 } 197 198 // Compiling 199 200 func doInstall(cmdline []string) { 201 var ( 202 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 203 arch = flag.String("arch", "", "Architecture to cross build for") 204 cc = flag.String("cc", "", "C compiler to cross build with") 205 ) 206 flag.CommandLine.Parse(cmdline) 207 208 // Configure the toolchain. 209 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 210 if *dlgo { 211 csdb := build.MustLoadChecksums("build/checksums.txt") 212 tc.Root = build.DownloadGo(csdb, dlgoVersion) 213 } 214 215 // Configure the build. 216 env := build.Env() 217 gobuild := tc.Go("build", buildFlags(env)...) 218 219 // arm64 CI builders are memory-constrained and can't handle concurrent builds, 220 // better disable it. This check isn't the best, it should probably 221 // check for something in env instead. 222 if env.CI && runtime.GOARCH == "arm64" { 223 gobuild.Args = append(gobuild.Args, "-p", "1") 224 } 225 226 // We use -trimpath to avoid leaking local paths into the built executables. 227 gobuild.Args = append(gobuild.Args, "-trimpath") 228 229 // Show packages during build. 230 gobuild.Args = append(gobuild.Args, "-v") 231 232 // Now we choose what we're even building. 233 // Default: collect all 'main' packages in cmd/ and build those. 234 packages := flag.Args() 235 if len(packages) == 0 { 236 packages = build.FindMainPackages("./cmd") 237 } 238 239 // Do the build! 240 for _, pkg := range packages { 241 args := make([]string, len(gobuild.Args)) 242 copy(args, gobuild.Args) 243 args = append(args, "-o", executablePath(path.Base(pkg))) 244 args = append(args, pkg) 245 build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) 246 } 247 } 248 249 // buildFlags returns the go tool flags for building. 250 func buildFlags(env build.Environment) (flags []string) { 251 var ld []string 252 if env.Commit != "" { 253 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 254 ld = append(ld, "-X", "main.gitDate="+env.Date) 255 } 256 // Strip DWARF on darwin. This used to be required for certain things, 257 // and there is no downside to this, so we just keep doing it. 258 if runtime.GOOS == "darwin" { 259 ld = append(ld, "-s") 260 } 261 // Enforce the stacksize to 8M, which is the case on most platforms apart from 262 // alpine Linux. 263 if runtime.GOOS == "linux" { 264 ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") 265 } 266 if len(ld) > 0 { 267 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 268 } 269 return flags 270 } 271 272 // Running The Tests 273 // 274 // "tests" also includes static analysis tools such as vet. 275 276 func doTest(cmdline []string) { 277 var ( 278 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 279 arch = flag.String("arch", "", "Run tests for given architecture") 280 cc = flag.String("cc", "", "Sets C compiler binary") 281 coverage = flag.Bool("coverage", false, "Whether to record code coverage") 282 verbose = flag.Bool("v", false, "Whether to log verbosely") 283 race = flag.Bool("race", false, "Execute the race detector") 284 ) 285 flag.CommandLine.Parse(cmdline) 286 287 // Configure the toolchain. 288 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 289 if *dlgo { 290 csdb := build.MustLoadChecksums("build/checksums.txt") 291 tc.Root = build.DownloadGo(csdb, dlgoVersion) 292 } 293 gotest := tc.Go("test") 294 295 // Test a single package at a time. CI builders are slow 296 // and some tests run into timeouts under load. 297 gotest.Args = append(gotest.Args, "-p", "1") 298 if *coverage { 299 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 300 } 301 if *verbose { 302 gotest.Args = append(gotest.Args, "-v") 303 } 304 if *race { 305 gotest.Args = append(gotest.Args, "-race") 306 } 307 308 packages := []string{"./..."} 309 if len(flag.CommandLine.Args()) > 0 { 310 packages = flag.CommandLine.Args() 311 } 312 gotest.Args = append(gotest.Args, packages...) 313 build.MustRun(gotest) 314 } 315 316 // doLint runs golangci-lint on requested packages. 317 func doLint(cmdline []string) { 318 var ( 319 cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") 320 ) 321 flag.CommandLine.Parse(cmdline) 322 packages := []string{"./..."} 323 if len(flag.CommandLine.Args()) > 0 { 324 packages = flag.CommandLine.Args() 325 } 326 327 linter := downloadLinter(*cachedir) 328 lflags := []string{"run", "--config", ".golangci.yml"} 329 build.MustRunCommand(linter, append(lflags, packages...)...) 330 fmt.Println("You have achieved perfection.") 331 } 332 333 // downloadLinter downloads and unpacks golangci-lint. 334 func downloadLinter(cachedir string) string { 335 const version = "1.45.2" 336 337 csdb := build.MustLoadChecksums("build/checksums.txt") 338 arch := runtime.GOARCH 339 ext := ".tar.gz" 340 341 if runtime.GOOS == "windows" { 342 ext = ".zip" 343 } 344 if arch == "arm" { 345 arch += "v" + os.Getenv("GOARM") 346 } 347 base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) 348 url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) 349 archivePath := filepath.Join(cachedir, base+ext) 350 if err := csdb.DownloadFile(url, archivePath); err != nil { 351 log.Fatal(err) 352 } 353 if err := build.ExtractArchive(archivePath, cachedir); err != nil { 354 log.Fatal(err) 355 } 356 return filepath.Join(cachedir, base, "golangci-lint") 357 } 358 359 // Release Packaging 360 func doArchive(cmdline []string) { 361 var ( 362 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 363 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 364 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 365 signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`) 366 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 367 ext string 368 ) 369 flag.CommandLine.Parse(cmdline) 370 switch *atype { 371 case "zip": 372 ext = ".zip" 373 case "tar": 374 ext = ".tar.gz" 375 default: 376 log.Fatal("unknown archive type: ", atype) 377 } 378 379 var ( 380 env = build.Env() 381 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 382 geth = "geth-" + basegeth + ext 383 alltools = "geth-alltools-" + basegeth + ext 384 ) 385 maybeSkipArchive(env) 386 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 387 log.Fatal(err) 388 } 389 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 390 log.Fatal(err) 391 } 392 for _, archive := range []string{geth, alltools} { 393 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 394 log.Fatal(err) 395 } 396 } 397 } 398 399 func archiveBasename(arch string, archiveVersion string) string { 400 platform := runtime.GOOS + "-" + arch 401 if arch == "arm" { 402 platform += os.Getenv("GOARM") 403 } 404 if arch == "android" { 405 platform = "android-all" 406 } 407 if arch == "ios" { 408 platform = "ios-all" 409 } 410 return platform + "-" + archiveVersion 411 } 412 413 func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error { 414 // If signing was requested, generate the signature files 415 if signer != "" { 416 key := getenvBase64(signer) 417 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 418 return err 419 } 420 } 421 if signifyVar != "" { 422 key := os.Getenv(signifyVar) 423 untrustedComment := "verify with geth-release.pub" 424 trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123)) 425 if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil { 426 return err 427 } 428 } 429 // If uploading to Azure was requested, push the archive possibly with its signature 430 if blobstore != "" { 431 auth := build.AzureBlobstoreConfig{ 432 Account: strings.Split(blobstore, "/")[0], 433 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 434 Container: strings.SplitN(blobstore, "/", 2)[1], 435 } 436 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 437 return err 438 } 439 if signer != "" { 440 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 441 return err 442 } 443 } 444 if signifyVar != "" { 445 if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { 446 return err 447 } 448 } 449 } 450 return nil 451 } 452 453 // skips archiving for some build configurations. 454 func maybeSkipArchive(env build.Environment) { 455 if env.IsPullRequest { 456 log.Printf("skipping archive creation because this is a PR build") 457 os.Exit(0) 458 } 459 if env.IsCronJob { 460 log.Printf("skipping archive creation because this is a cron job") 461 os.Exit(0) 462 } 463 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 464 log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) 465 os.Exit(0) 466 } 467 } 468 469 // Builds the docker images and optionally uploads them to Docker Hub. 470 func doDocker(cmdline []string) { 471 var ( 472 image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) 473 manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) 474 upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) 475 ) 476 flag.CommandLine.Parse(cmdline) 477 478 // Skip building and pushing docker images for PR builds 479 env := build.Env() 480 maybeSkipArchive(env) 481 482 // Retrieve the upload credentials and authenticate 483 user := getenvBase64("DOCKER_HUB_USERNAME") 484 pass := getenvBase64("DOCKER_HUB_PASSWORD") 485 486 if len(user) > 0 && len(pass) > 0 { 487 auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") 488 auther.Stdin = bytes.NewReader(pass) 489 build.MustRun(auther) 490 } 491 // Retrieve the version infos to build and push to the following paths: 492 // - ethereum/client-go:latest - Pushes to the master branch, Geth only 493 // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only 494 // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools 495 // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools 496 // - ethereum/client-go:release-<major>.<minor> - Version tag publish on GitHub, Geth only 497 // - ethereum/client-go:alltools-release-<major>.<minor> - Version tag publish on GitHub, Geth & tools 498 // - ethereum/client-go:v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth only 499 // - ethereum/client-go:alltools-v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth & tools 500 var tags []string 501 502 switch { 503 case env.Branch == "master": 504 tags = []string{"latest"} 505 case strings.HasPrefix(env.Tag, "v1."): 506 tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} 507 } 508 // If architecture specific image builds are requested, build and push them 509 if *image { 510 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") 511 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") 512 513 // Tag and upload the images to Docker Hub 514 for _, tag := range tags { 515 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) 516 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) 517 518 // If the image already exists (non version tag), check the build 519 // number to prevent overwriting a newer commit if concurrent builds 520 // are running. This is still a tiny bit racey if two published are 521 // done at the same time, but that's extremely unlikely even on the 522 // master branch. 523 for _, img := range []string{gethImage, toolImage} { 524 if exec.Command("docker", "pull", img).Run() != nil { 525 continue // Generally the only failure is a missing image, which is good 526 } 527 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 528 if err != nil { 529 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 530 } 531 buildnum = bytes.TrimSpace(buildnum) 532 533 if len(buildnum) > 0 && len(env.Buildnum) > 0 { 534 oldnum, err := strconv.Atoi(string(buildnum)) 535 if err != nil { 536 log.Fatalf("Failed to parse old image build number: %v", err) 537 } 538 newnum, err := strconv.Atoi(env.Buildnum) 539 if err != nil { 540 log.Fatalf("Failed to parse current build number: %v", err) 541 } 542 if oldnum > newnum { 543 log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) 544 } else { 545 log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) 546 } 547 } 548 } 549 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) 550 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) 551 build.MustRunCommand("docker", "push", gethImage) 552 build.MustRunCommand("docker", "push", toolImage) 553 } 554 } 555 // If multi-arch image manifest push is requested, assemble it 556 if len(*manifest) != 0 { 557 // Since different architectures are pushed by different builders, wait 558 // until all required images are updated. 559 var mismatch bool 560 for i := 0; i < 2; i++ { // 2 attempts, second is race check 561 mismatch = false // hope there's no mismatch now 562 563 for _, tag := range tags { 564 for _, arch := range strings.Split(*manifest, ",") { 565 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) 566 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) 567 568 for _, img := range []string{gethImage, toolImage} { 569 if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { 570 log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) 571 mismatch = true 572 break 573 } 574 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 575 if err != nil { 576 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 577 } 578 buildnum = bytes.TrimSpace(buildnum) 579 580 if string(buildnum) != env.Buildnum { 581 log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) 582 mismatch = true 583 break 584 } 585 } 586 if mismatch { 587 break 588 } 589 } 590 if mismatch { 591 break 592 } 593 } 594 if mismatch { 595 // Build numbers mismatching, retry in a short time to 596 // avoid concurrent failes in both publisher images. If 597 // however the retry failed too, it means the concurrent 598 // builder is still crunching, let that do the publish. 599 if i == 0 { 600 time.Sleep(30 * time.Second) 601 } 602 continue 603 } 604 break 605 } 606 if mismatch { 607 log.Println("Relinquishing publish to other builder") 608 return 609 } 610 // Assemble and push the Geth manifest image 611 for _, tag := range tags { 612 gethImage := fmt.Sprintf("%s:%s", *upload, tag) 613 614 var gethSubImages []string 615 for _, arch := range strings.Split(*manifest, ",") { 616 gethSubImages = append(gethSubImages, gethImage+"-"+arch) 617 } 618 build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) 619 build.MustRunCommand("docker", "manifest", "push", gethImage) 620 } 621 // Assemble and push the alltools manifest image 622 for _, tag := range tags { 623 toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) 624 625 var toolSubImages []string 626 for _, arch := range strings.Split(*manifest, ",") { 627 toolSubImages = append(toolSubImages, toolImage+"-"+arch) 628 } 629 build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) 630 build.MustRunCommand("docker", "manifest", "push", toolImage) 631 } 632 } 633 } 634 635 // Debian Packaging 636 func doDebianSource(cmdline []string) { 637 var ( 638 cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) 639 signer = flag.String("signer", "", `Signing key name, also used as package author`) 640 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 641 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 642 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 643 now = time.Now() 644 ) 645 flag.CommandLine.Parse(cmdline) 646 *workdir = makeWorkdir(*workdir) 647 env := build.Env() 648 tc := new(build.GoToolchain) 649 maybeSkipArchive(env) 650 651 // Import the signing key. 652 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 653 gpg := exec.Command("gpg", "--import") 654 gpg.Stdin = bytes.NewReader(key) 655 build.MustRun(gpg) 656 } 657 658 // Download and verify the Go source package. 659 gobundle := downloadGoSources(*cachedir) 660 661 // Download all the dependencies needed to build the sources and run the ci script 662 srcdepfetch := tc.Go("mod", "download") 663 srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 664 build.MustRun(srcdepfetch) 665 666 cidepfetch := tc.Go("run", "./build/ci.go") 667 cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 668 cidepfetch.Run() // Command fails, don't care, we only need the deps to start it 669 670 // Create Debian packages and upload them. 671 for _, pkg := range debPackages { 672 for distro, goboot := range debDistroGoBoots { 673 // Prepare the debian package with the go-ethereum sources. 674 meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 675 pkgdir := stageDebianSource(*workdir, meta) 676 677 // Add Go source code 678 if err := build.ExtractArchive(gobundle, pkgdir); err != nil { 679 log.Fatalf("Failed to extract Go sources: %v", err) 680 } 681 if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { 682 log.Fatalf("Failed to rename Go source folder: %v", err) 683 } 684 // Add all dependency modules in compressed form 685 os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) 686 if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { 687 log.Fatalf("Failed to copy Go module dependencies: %v", err) 688 } 689 // Run the packaging and upload to the PPA 690 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") 691 debuild.Dir = pkgdir 692 build.MustRun(debuild) 693 694 var ( 695 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 696 source = filepath.Join(*workdir, basename+".tar.xz") 697 dsc = filepath.Join(*workdir, basename+".dsc") 698 changes = filepath.Join(*workdir, basename+"_source.changes") 699 buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo") 700 ) 701 if *signer != "" { 702 build.MustRunCommand("debsign", changes) 703 } 704 if *upload != "" { 705 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo}) 706 } 707 } 708 } 709 } 710 711 // downloadGoSources downloads the Go source tarball. 712 func downloadGoSources(cachedir string) string { 713 csdb := build.MustLoadChecksums("build/checksums.txt") 714 file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) 715 url := "https://dl.google.com/go/" + file 716 dst := filepath.Join(cachedir, file) 717 if err := csdb.DownloadFile(url, dst); err != nil { 718 log.Fatal(err) 719 } 720 return dst 721 } 722 723 func ppaUpload(workdir, ppa, sshUser string, files []string) { 724 p := strings.Split(ppa, "/") 725 if len(p) != 2 { 726 log.Fatal("-upload PPA name must contain single /") 727 } 728 if sshUser == "" { 729 sshUser = p[0] 730 } 731 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 732 // Create the SSH identity file if it doesn't exist. 733 var idfile string 734 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 735 idfile = filepath.Join(workdir, "sshkey") 736 if _, err := os.Stat(idfile); os.IsNotExist(err) { 737 ioutil.WriteFile(idfile, sshkey, 0600) 738 } 739 } 740 // Upload 741 dest := sshUser + "@ppa.launchpad.net" 742 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 743 log.Fatal(err) 744 } 745 } 746 747 func getenvBase64(variable string) []byte { 748 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 749 if err != nil { 750 log.Fatal("invalid base64 " + variable) 751 } 752 return []byte(dec) 753 } 754 755 func makeWorkdir(wdflag string) string { 756 var err error 757 if wdflag != "" { 758 err = os.MkdirAll(wdflag, 0744) 759 } else { 760 wdflag, err = ioutil.TempDir("", "geth-build-") 761 } 762 if err != nil { 763 log.Fatal(err) 764 } 765 return wdflag 766 } 767 768 func isUnstableBuild(env build.Environment) bool { 769 if env.Tag != "" { 770 return false 771 } 772 return true 773 } 774 775 type debPackage struct { 776 Name string // the name of the Debian package to produce, e.g. "ethereum" 777 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 778 Executables []debExecutable // executables to be included in the package 779 } 780 781 type debMetadata struct { 782 Env build.Environment 783 GoBootPackage string 784 GoBootPath string 785 786 PackageName string 787 788 // go-ethereum version being built. Note that this 789 // is not the debian package version. The package version 790 // is constructed by VersionString. 791 Version string 792 793 Author string // "name <email>", also selects signing key 794 Distro, Time string 795 Executables []debExecutable 796 } 797 798 type debExecutable struct { 799 PackageName string 800 BinaryName string 801 Description string 802 } 803 804 // Package returns the name of the package if present, or 805 // fallbacks to BinaryName 806 func (d debExecutable) Package() string { 807 if d.PackageName != "" { 808 return d.PackageName 809 } 810 return d.BinaryName 811 } 812 813 func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 814 if author == "" { 815 // No signing key, use default author. 816 author = "Ethereum Builds <fjl@ethereum.org>" 817 } 818 return debMetadata{ 819 GoBootPackage: goboot, 820 GoBootPath: debGoBootPaths[goboot], 821 PackageName: name, 822 Env: env, 823 Author: author, 824 Distro: distro, 825 Version: version, 826 Time: t.Format(time.RFC1123Z), 827 Executables: exes, 828 } 829 } 830 831 // Name returns the name of the metapackage that depends 832 // on all executable packages. 833 func (meta debMetadata) Name() string { 834 if isUnstableBuild(meta.Env) { 835 return meta.PackageName + "-unstable" 836 } 837 return meta.PackageName 838 } 839 840 // VersionString returns the debian version of the packages. 841 func (meta debMetadata) VersionString() string { 842 vsn := meta.Version 843 if meta.Env.Buildnum != "" { 844 vsn += "+build" + meta.Env.Buildnum 845 } 846 if meta.Distro != "" { 847 vsn += "+" + meta.Distro 848 } 849 return vsn 850 } 851 852 // ExeList returns the list of all executable packages. 853 func (meta debMetadata) ExeList() string { 854 names := make([]string, len(meta.Executables)) 855 for i, e := range meta.Executables { 856 names[i] = meta.ExeName(e) 857 } 858 return strings.Join(names, ", ") 859 } 860 861 // ExeName returns the package name of an executable package. 862 func (meta debMetadata) ExeName(exe debExecutable) string { 863 if isUnstableBuild(meta.Env) { 864 return exe.Package() + "-unstable" 865 } 866 return exe.Package() 867 } 868 869 // ExeConflicts returns the content of the Conflicts field 870 // for executable packages. 871 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 872 if isUnstableBuild(meta.Env) { 873 // Set up the conflicts list so that the *-unstable packages 874 // cannot be installed alongside the regular version. 875 // 876 // https://www.debian.org/doc/debian-policy/ch-relationships.html 877 // is very explicit about Conflicts: and says that Breaks: should 878 // be preferred and the conflicting files should be handled via 879 // alternates. We might do this eventually but using a conflict is 880 // easier now. 881 return "ethereum, " + exe.Package() 882 } 883 return "" 884 } 885 886 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 887 pkg := meta.Name() + "-" + meta.VersionString() 888 pkgdir = filepath.Join(tmpdir, pkg) 889 if err := os.Mkdir(pkgdir, 0755); err != nil { 890 log.Fatal(err) 891 } 892 // Copy the source code. 893 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 894 895 // Put the debian build files in place. 896 debian := filepath.Join(pkgdir, "debian") 897 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 898 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 899 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 900 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 901 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 902 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 903 for _, exe := range meta.Executables { 904 install := filepath.Join(debian, meta.ExeName(exe)+".install") 905 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 906 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 907 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 908 } 909 return pkgdir 910 } 911 912 // Windows installer 913 func doWindowsInstaller(cmdline []string) { 914 // Parse the flags and make skip installer generation on PRs 915 var ( 916 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 917 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 918 signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) 919 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 920 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 921 ) 922 flag.CommandLine.Parse(cmdline) 923 *workdir = makeWorkdir(*workdir) 924 env := build.Env() 925 maybeSkipArchive(env) 926 927 // Aggregate binaries that are included in the installer 928 var ( 929 devTools []string 930 allTools []string 931 gethTool string 932 ) 933 for _, file := range allToolsArchiveFiles { 934 if file == "COPYING" { // license, copied later 935 continue 936 } 937 allTools = append(allTools, filepath.Base(file)) 938 if filepath.Base(file) == "geth.exe" { 939 gethTool = file 940 } else { 941 devTools = append(devTools, file) 942 } 943 } 944 945 // Render NSIS scripts: Installer NSIS contains two installer sections, 946 // first section contains the geth binary, second section holds the dev tools. 947 templateData := map[string]interface{}{ 948 "License": "COPYING", 949 "Geth": gethTool, 950 "DevTools": devTools, 951 } 952 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 953 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 954 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 955 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 956 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 957 if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { 958 log.Fatal("Failed to copy SimpleFC.dll: %v", err) 959 } 960 if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { 961 log.Fatal("Failed to copy copyright note: %v", err) 962 } 963 // Build the installer. This assumes that all the needed files have been previously 964 // built (don't mix building and packaging to keep cross compilation complexity to a 965 // minimum). 966 version := strings.Split(params.Version, ".") 967 if env.Commit != "" { 968 version[2] += "-" + env.Commit[:8] 969 } 970 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 971 build.MustRunCommand("makensis.exe", 972 "/DOUTPUTFILE="+installer, 973 "/DMAJORVERSION="+version[0], 974 "/DMINORVERSION="+version[1], 975 "/DBUILDVERSION="+version[2], 976 "/DARCH="+*arch, 977 filepath.Join(*workdir, "geth.nsi"), 978 ) 979 // Sign and publish installer. 980 if err := archiveUpload(installer, *upload, *signer, *signify); err != nil { 981 log.Fatal(err) 982 } 983 } 984 985 // Android archives 986 987 func doAndroidArchive(cmdline []string) { 988 var ( 989 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 990 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 991 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`) 992 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 993 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 994 ) 995 flag.CommandLine.Parse(cmdline) 996 env := build.Env() 997 tc := new(build.GoToolchain) 998 999 // Sanity check that the SDK and NDK are installed and set 1000 if os.Getenv("ANDROID_HOME") == "" { 1001 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 1002 } 1003 1004 // Build gomobile. 1005 install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest") 1006 install.Env = append(install.Env) 1007 build.MustRun(install) 1008 1009 // Ensure all dependencies are available. This is required to make 1010 // gomobile bind work because it expects go.sum to contain all checksums. 1011 build.MustRun(tc.Go("mod", "download")) 1012 1013 // Build the Android archive and Maven resources 1014 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) 1015 1016 if *local { 1017 // If we're building locally, copy bundle to build dir and skip Maven 1018 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 1019 os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar")) 1020 return 1021 } 1022 meta := newMavenMetadata(env) 1023 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 1024 1025 // Skip Maven deploy and Azure upload for PR builds 1026 maybeSkipArchive(env) 1027 1028 // Sign and upload the archive to Azure 1029 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 1030 os.Rename("geth.aar", archive) 1031 1032 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 1033 log.Fatal(err) 1034 } 1035 // Sign and upload all the artifacts to Maven Central 1036 os.Rename(archive, meta.Package+".aar") 1037 if *signer != "" && *deploy != "" { 1038 // Import the signing key into the local GPG instance 1039 key := getenvBase64(*signer) 1040 gpg := exec.Command("gpg", "--import") 1041 gpg.Stdin = bytes.NewReader(key) 1042 build.MustRun(gpg) 1043 keyID, err := build.PGPKeyID(string(key)) 1044 if err != nil { 1045 log.Fatal(err) 1046 } 1047 // Upload the artifacts to Sonatype and/or Maven Central 1048 repo := *deploy + "/service/local/staging/deploy/maven2" 1049 if meta.Develop { 1050 repo = *deploy + "/content/repositories/snapshots" 1051 } 1052 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 1053 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 1054 "-Dgpg.keyname="+keyID, 1055 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 1056 } 1057 } 1058 1059 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 1060 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 1061 cmd.Args = append(cmd.Args, args...) 1062 cmd.Env = []string{ 1063 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 1064 } 1065 for _, e := range os.Environ() { 1066 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") { 1067 continue 1068 } 1069 cmd.Env = append(cmd.Env, e) 1070 } 1071 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 1072 return cmd 1073 } 1074 1075 type mavenMetadata struct { 1076 Version string 1077 Package string 1078 Develop bool 1079 Contributors []mavenContributor 1080 } 1081 1082 type mavenContributor struct { 1083 Name string 1084 Email string 1085 } 1086 1087 func newMavenMetadata(env build.Environment) mavenMetadata { 1088 // Collect the list of authors from the repo root 1089 contribs := []mavenContributor{} 1090 if authors, err := os.Open("AUTHORS"); err == nil { 1091 defer authors.Close() 1092 1093 scanner := bufio.NewScanner(authors) 1094 for scanner.Scan() { 1095 // Skip any whitespace from the authors list 1096 line := strings.TrimSpace(scanner.Text()) 1097 if line == "" || line[0] == '#' { 1098 continue 1099 } 1100 // Split the author and insert as a contributor 1101 re := regexp.MustCompile("([^<]+) <(.+)>") 1102 parts := re.FindStringSubmatch(line) 1103 if len(parts) == 3 { 1104 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 1105 } 1106 } 1107 } 1108 // Render the version and package strings 1109 version := params.Version 1110 if isUnstableBuild(env) { 1111 version += "-SNAPSHOT" 1112 } 1113 return mavenMetadata{ 1114 Version: version, 1115 Package: "geth-" + version, 1116 Develop: isUnstableBuild(env), 1117 Contributors: contribs, 1118 } 1119 } 1120 1121 // XCode frameworks 1122 1123 func doXCodeFramework(cmdline []string) { 1124 var ( 1125 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1126 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 1127 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`) 1128 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 1129 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 1130 ) 1131 flag.CommandLine.Parse(cmdline) 1132 env := build.Env() 1133 tc := new(build.GoToolchain) 1134 1135 // Build gomobile. 1136 build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 1137 1138 // Build the iOS XCode framework 1139 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") 1140 1141 if *local { 1142 // If we're building locally, use the build folder and stop afterwards 1143 bind.Dir = GOBIN 1144 build.MustRun(bind) 1145 return 1146 } 1147 1148 // Create the archive. 1149 maybeSkipArchive(env) 1150 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 1151 if err := os.MkdirAll(archive, 0755); err != nil { 1152 log.Fatal(err) 1153 } 1154 bind.Dir, _ = filepath.Abs(archive) 1155 build.MustRun(bind) 1156 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 1157 1158 // Sign and upload the framework to Azure 1159 if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { 1160 log.Fatal(err) 1161 } 1162 // Prepare and upload a PodSpec to CocoaPods 1163 if *deploy != "" { 1164 meta := newPodMetadata(env, archive) 1165 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 1166 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings") 1167 } 1168 } 1169 1170 type podMetadata struct { 1171 Version string 1172 Commit string 1173 Archive string 1174 Contributors []podContributor 1175 } 1176 1177 type podContributor struct { 1178 Name string 1179 Email string 1180 } 1181 1182 func newPodMetadata(env build.Environment, archive string) podMetadata { 1183 // Collect the list of authors from the repo root 1184 contribs := []podContributor{} 1185 if authors, err := os.Open("AUTHORS"); err == nil { 1186 defer authors.Close() 1187 1188 scanner := bufio.NewScanner(authors) 1189 for scanner.Scan() { 1190 // Skip any whitespace from the authors list 1191 line := strings.TrimSpace(scanner.Text()) 1192 if line == "" || line[0] == '#' { 1193 continue 1194 } 1195 // Split the author and insert as a contributor 1196 re := regexp.MustCompile("([^<]+) <(.+)>") 1197 parts := re.FindStringSubmatch(line) 1198 if len(parts) == 3 { 1199 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 1200 } 1201 } 1202 } 1203 version := params.Version 1204 if isUnstableBuild(env) { 1205 version += "-unstable." + env.Buildnum 1206 } 1207 return podMetadata{ 1208 Archive: archive, 1209 Version: version, 1210 Commit: env.Commit, 1211 Contributors: contribs, 1212 } 1213 } 1214 1215 // Binary distribution cleanups 1216 1217 func doPurge(cmdline []string) { 1218 var ( 1219 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1220 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1221 ) 1222 flag.CommandLine.Parse(cmdline) 1223 1224 if env := build.Env(); !env.IsCronJob { 1225 log.Printf("skipping because not a cron job") 1226 os.Exit(0) 1227 } 1228 // Create the azure authentication and list the current archives 1229 auth := build.AzureBlobstoreConfig{ 1230 Account: strings.Split(*store, "/")[0], 1231 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1232 Container: strings.SplitN(*store, "/", 2)[1], 1233 } 1234 blobs, err := build.AzureBlobstoreList(auth) 1235 if err != nil { 1236 log.Fatal(err) 1237 } 1238 fmt.Printf("Found %d blobs\n", len(blobs)) 1239 1240 // Iterate over the blobs, collect and sort all unstable builds 1241 for i := 0; i < len(blobs); i++ { 1242 if !strings.Contains(*blobs[i].Name, "unstable") { 1243 blobs = append(blobs[:i], blobs[i+1:]...) 1244 i-- 1245 } 1246 } 1247 for i := 0; i < len(blobs); i++ { 1248 for j := i + 1; j < len(blobs); j++ { 1249 if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) { 1250 blobs[i], blobs[j] = blobs[j], blobs[i] 1251 } 1252 } 1253 } 1254 // Filter out all archives more recent that the given threshold 1255 for i, blob := range blobs { 1256 if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1257 blobs = blobs[:i] 1258 break 1259 } 1260 } 1261 fmt.Printf("Deleting %d blobs\n", len(blobs)) 1262 // Delete all marked as such and return 1263 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1264 log.Fatal(err) 1265 } 1266 }