github.com/MikyChow/arbitrum-go-ethereum@v0.0.0-20230306102812-078da49636de/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 //go:build none 18 // +build none 19 20 /* 21 The ci command is called from Continuous Integration scripts. 22 23 Usage: go run build/ci.go <command> <command flags/arguments> 24 25 Available commands are: 26 27 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 28 test [ -coverage ] [ packages... ] -- runs the tests 29 lint -- runs certain pre-selected linters 30 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts 31 importkeys -- imports signing keys from env 32 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 33 nsis -- creates a Windows NSIS installer 34 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 35 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 */ 40 package main 41 42 import ( 43 "bufio" 44 "bytes" 45 "encoding/base64" 46 "flag" 47 "fmt" 48 "log" 49 "os" 50 "os/exec" 51 "path" 52 "path/filepath" 53 "regexp" 54 "runtime" 55 "strconv" 56 "strings" 57 "time" 58 59 "github.com/MikyChow/arbitrum-go-ethereum/common" 60 "github.com/MikyChow/arbitrum-go-ethereum/crypto/signify" 61 "github.com/MikyChow/arbitrum-go-ethereum/internal/build" 62 "github.com/MikyChow/arbitrum-go-ethereum/params" 63 "github.com/cespare/cp" 64 ) 65 66 var ( 67 // Files that end up in the geth*.zip archive. 68 gethArchiveFiles = []string{ 69 "COPYING", 70 executablePath("geth"), 71 } 72 73 // Files that end up in the geth-alltools*.zip archive. 74 allToolsArchiveFiles = []string{ 75 "COPYING", 76 executablePath("abigen"), 77 executablePath("bootnode"), 78 executablePath("evm"), 79 executablePath("geth"), 80 executablePath("puppeth"), 81 executablePath("rlpdump"), 82 executablePath("clef"), 83 } 84 85 // A debian package is created for all executables listed here. 86 debExecutables = []debExecutable{ 87 { 88 BinaryName: "abigen", 89 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 90 }, 91 { 92 BinaryName: "bootnode", 93 Description: "Ethereum bootnode.", 94 }, 95 { 96 BinaryName: "evm", 97 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 98 }, 99 { 100 BinaryName: "geth", 101 Description: "Ethereum CLI client.", 102 }, 103 { 104 BinaryName: "puppeth", 105 Description: "Ethereum private network manager.", 106 }, 107 { 108 BinaryName: "rlpdump", 109 Description: "Developer utility tool that prints RLP structures.", 110 }, 111 { 112 BinaryName: "clef", 113 Description: "Ethereum account management tool.", 114 }, 115 } 116 117 // A debian package is created for all executables listed here. 118 debEthereum = debPackage{ 119 Name: "ethereum", 120 Version: params.Version, 121 Executables: debExecutables, 122 } 123 124 // Debian meta packages to build and push to Ubuntu PPA 125 debPackages = []debPackage{ 126 debEthereum, 127 } 128 129 // Distros for which packages are created. 130 // Note: vivid is unsupported because there is no golang-1.6 package for it. 131 // Note: the following Ubuntu releases have been officially deprecated on Launchpad: 132 // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite 133 debDistroGoBoots = map[string]string{ 134 "trusty": "golang-1.11", // EOL: 04/2024 135 "xenial": "golang-go", // EOL: 04/2026 136 "bionic": "golang-go", // EOL: 04/2028 137 "focal": "golang-go", // EOL: 04/2030 138 "impish": "golang-go", // EOL: 07/2022 139 "jammy": "golang-go", // EOL: 04/2032 140 //"kinetic": "golang-go", // EOL: 07/2023 141 } 142 143 debGoBootPaths = map[string]string{ 144 "golang-1.11": "/usr/lib/go-1.11", 145 "golang-go": "/usr/lib/go", 146 } 147 148 // This is the version of go that will be downloaded by 149 // 150 // go run ci.go install -dlgo 151 dlgoVersion = "1.18.5" 152 ) 153 154 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 155 156 func executablePath(name string) string { 157 if runtime.GOOS == "windows" { 158 name += ".exe" 159 } 160 return filepath.Join(GOBIN, name) 161 } 162 163 func main() { 164 log.SetFlags(log.Lshortfile) 165 166 if !common.FileExist(filepath.Join("build", "ci.go")) { 167 log.Fatal("this script must be run from the root of the repository") 168 } 169 if len(os.Args) < 2 { 170 log.Fatal("need subcommand as first argument") 171 } 172 switch os.Args[1] { 173 case "install": 174 doInstall(os.Args[2:]) 175 case "test": 176 doTest(os.Args[2:]) 177 case "lint": 178 doLint(os.Args[2:]) 179 case "archive": 180 doArchive(os.Args[2:]) 181 case "docker": 182 doDocker(os.Args[2:]) 183 case "debsrc": 184 doDebianSource(os.Args[2:]) 185 case "nsis": 186 doWindowsInstaller(os.Args[2:]) 187 case "aar": 188 doAndroidArchive(os.Args[2:]) 189 case "xcode": 190 doXCodeFramework(os.Args[2:]) 191 case "purge": 192 doPurge(os.Args[2:]) 193 default: 194 log.Fatal("unknown command ", os.Args[1]) 195 } 196 } 197 198 // Compiling 199 200 func doInstall(cmdline []string) { 201 var ( 202 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 203 arch = flag.String("arch", "", "Architecture to cross build for") 204 cc = flag.String("cc", "", "C compiler to cross build with") 205 staticlink = flag.Bool("static", false, "Create statically-linked executable") 206 ) 207 flag.CommandLine.Parse(cmdline) 208 209 // Configure the toolchain. 210 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 211 if *dlgo { 212 csdb := build.MustLoadChecksums("build/checksums.txt") 213 tc.Root = build.DownloadGo(csdb, dlgoVersion) 214 } 215 216 // Disable CLI markdown doc generation in release builds. 217 buildTags := []string{"urfave_cli_no_docs"} 218 219 // Configure the build. 220 env := build.Env() 221 gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) 222 223 // arm64 CI builders are memory-constrained and can't handle concurrent builds, 224 // better disable it. This check isn't the best, it should probably 225 // check for something in env instead. 226 if env.CI && runtime.GOARCH == "arm64" { 227 gobuild.Args = append(gobuild.Args, "-p", "1") 228 } 229 230 // We use -trimpath to avoid leaking local paths into the built executables. 231 gobuild.Args = append(gobuild.Args, "-trimpath") 232 233 // Show packages during build. 234 gobuild.Args = append(gobuild.Args, "-v") 235 236 // Now we choose what we're even building. 237 // Default: collect all 'main' packages in cmd/ and build those. 238 packages := flag.Args() 239 if len(packages) == 0 { 240 packages = build.FindMainPackages("./cmd") 241 } 242 243 // Do the build! 244 for _, pkg := range packages { 245 args := make([]string, len(gobuild.Args)) 246 copy(args, gobuild.Args) 247 args = append(args, "-o", executablePath(path.Base(pkg))) 248 args = append(args, pkg) 249 build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) 250 } 251 } 252 253 // buildFlags returns the go tool flags for building. 254 func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { 255 var ld []string 256 if env.Commit != "" { 257 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 258 ld = append(ld, "-X", "main.gitDate="+env.Date) 259 } 260 // Strip DWARF on darwin. This used to be required for certain things, 261 // and there is no downside to this, so we just keep doing it. 262 if runtime.GOOS == "darwin" { 263 ld = append(ld, "-s") 264 } 265 if runtime.GOOS == "linux" { 266 // Enforce the stacksize to 8M, which is the case on most platforms apart from 267 // alpine Linux. 268 extld := []string{"-Wl,-z,stack-size=0x800000"} 269 if staticLinking { 270 extld = append(extld, "-static") 271 // Under static linking, use of certain glibc features must be 272 // disabled to avoid shared library dependencies. 273 buildTags = append(buildTags, "osusergo", "netgo") 274 } 275 ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") 276 } 277 if len(ld) > 0 { 278 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 279 } 280 if len(buildTags) > 0 { 281 flags = append(flags, "-tags", strings.Join(buildTags, ",")) 282 } 283 return flags 284 } 285 286 // Running The Tests 287 // 288 // "tests" also includes static analysis tools such as vet. 289 290 func doTest(cmdline []string) { 291 var ( 292 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 293 arch = flag.String("arch", "", "Run tests for given architecture") 294 cc = flag.String("cc", "", "Sets C compiler binary") 295 coverage = flag.Bool("coverage", false, "Whether to record code coverage") 296 verbose = flag.Bool("v", false, "Whether to log verbosely") 297 race = flag.Bool("race", false, "Execute the race detector") 298 ) 299 flag.CommandLine.Parse(cmdline) 300 301 // Configure the toolchain. 302 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 303 if *dlgo { 304 csdb := build.MustLoadChecksums("build/checksums.txt") 305 tc.Root = build.DownloadGo(csdb, dlgoVersion) 306 } 307 gotest := tc.Go("test") 308 309 // Test a single package at a time. CI builders are slow 310 // and some tests run into timeouts under load. 311 gotest.Args = append(gotest.Args, "-p", "1") 312 if *coverage { 313 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 314 } 315 if *verbose { 316 gotest.Args = append(gotest.Args, "-v") 317 } 318 if *race { 319 gotest.Args = append(gotest.Args, "-race") 320 } 321 322 packages := []string{"./..."} 323 if len(flag.CommandLine.Args()) > 0 { 324 packages = flag.CommandLine.Args() 325 } 326 gotest.Args = append(gotest.Args, packages...) 327 build.MustRun(gotest) 328 } 329 330 // doLint runs golangci-lint on requested packages. 331 func doLint(cmdline []string) { 332 var ( 333 cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") 334 ) 335 flag.CommandLine.Parse(cmdline) 336 packages := []string{"./..."} 337 if len(flag.CommandLine.Args()) > 0 { 338 packages = flag.CommandLine.Args() 339 } 340 341 linter := downloadLinter(*cachedir) 342 lflags := []string{"run", "--config", ".golangci.yml"} 343 build.MustRunCommand(linter, append(lflags, packages...)...) 344 fmt.Println("You have achieved perfection.") 345 } 346 347 // downloadLinter downloads and unpacks golangci-lint. 348 func downloadLinter(cachedir string) string { 349 const version = "1.46.2" 350 351 csdb := build.MustLoadChecksums("build/checksums.txt") 352 arch := runtime.GOARCH 353 ext := ".tar.gz" 354 355 if runtime.GOOS == "windows" { 356 ext = ".zip" 357 } 358 if arch == "arm" { 359 arch += "v" + os.Getenv("GOARM") 360 } 361 base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) 362 url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) 363 archivePath := filepath.Join(cachedir, base+ext) 364 if err := csdb.DownloadFile(url, archivePath); err != nil { 365 log.Fatal(err) 366 } 367 if err := build.ExtractArchive(archivePath, cachedir); err != nil { 368 log.Fatal(err) 369 } 370 return filepath.Join(cachedir, base, "golangci-lint") 371 } 372 373 // Release Packaging 374 func doArchive(cmdline []string) { 375 var ( 376 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 377 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 378 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 379 signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`) 380 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 381 ext string 382 ) 383 flag.CommandLine.Parse(cmdline) 384 switch *atype { 385 case "zip": 386 ext = ".zip" 387 case "tar": 388 ext = ".tar.gz" 389 default: 390 log.Fatal("unknown archive type: ", atype) 391 } 392 393 var ( 394 env = build.Env() 395 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 396 geth = "geth-" + basegeth + ext 397 alltools = "geth-alltools-" + basegeth + ext 398 ) 399 maybeSkipArchive(env) 400 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 401 log.Fatal(err) 402 } 403 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 404 log.Fatal(err) 405 } 406 for _, archive := range []string{geth, alltools} { 407 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 408 log.Fatal(err) 409 } 410 } 411 } 412 413 func archiveBasename(arch string, archiveVersion string) string { 414 platform := runtime.GOOS + "-" + arch 415 if arch == "arm" { 416 platform += os.Getenv("GOARM") 417 } 418 if arch == "android" { 419 platform = "android-all" 420 } 421 if arch == "ios" { 422 platform = "ios-all" 423 } 424 return platform + "-" + archiveVersion 425 } 426 427 func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error { 428 // If signing was requested, generate the signature files 429 if signer != "" { 430 key := getenvBase64(signer) 431 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 432 return err 433 } 434 } 435 if signifyVar != "" { 436 key := os.Getenv(signifyVar) 437 untrustedComment := "verify with geth-release.pub" 438 trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123)) 439 if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil { 440 return err 441 } 442 } 443 // If uploading to Azure was requested, push the archive possibly with its signature 444 if blobstore != "" { 445 auth := build.AzureBlobstoreConfig{ 446 Account: strings.Split(blobstore, "/")[0], 447 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 448 Container: strings.SplitN(blobstore, "/", 2)[1], 449 } 450 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 451 return err 452 } 453 if signer != "" { 454 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 455 return err 456 } 457 } 458 if signifyVar != "" { 459 if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { 460 return err 461 } 462 } 463 } 464 return nil 465 } 466 467 // skips archiving for some build configurations. 468 func maybeSkipArchive(env build.Environment) { 469 if env.IsPullRequest { 470 log.Printf("skipping archive creation because this is a PR build") 471 os.Exit(0) 472 } 473 if env.IsCronJob { 474 log.Printf("skipping archive creation because this is a cron job") 475 os.Exit(0) 476 } 477 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 478 log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) 479 os.Exit(0) 480 } 481 } 482 483 // Builds the docker images and optionally uploads them to Docker Hub. 484 func doDocker(cmdline []string) { 485 var ( 486 image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) 487 manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) 488 upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) 489 ) 490 flag.CommandLine.Parse(cmdline) 491 492 // Skip building and pushing docker images for PR builds 493 env := build.Env() 494 maybeSkipArchive(env) 495 496 // Retrieve the upload credentials and authenticate 497 user := getenvBase64("DOCKER_HUB_USERNAME") 498 pass := getenvBase64("DOCKER_HUB_PASSWORD") 499 500 if len(user) > 0 && len(pass) > 0 { 501 auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") 502 auther.Stdin = bytes.NewReader(pass) 503 build.MustRun(auther) 504 } 505 // Retrieve the version infos to build and push to the following paths: 506 // - ethereum/client-go:latest - Pushes to the master branch, Geth only 507 // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only 508 // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools 509 // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools 510 // - ethereum/client-go:release-<major>.<minor> - Version tag publish on GitHub, Geth only 511 // - ethereum/client-go:alltools-release-<major>.<minor> - Version tag publish on GitHub, Geth & tools 512 // - ethereum/client-go:v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth only 513 // - ethereum/client-go:alltools-v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth & tools 514 var tags []string 515 516 switch { 517 case env.Branch == "master": 518 tags = []string{"latest"} 519 case strings.HasPrefix(env.Tag, "v1."): 520 tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} 521 } 522 // If architecture specific image builds are requested, build and push them 523 if *image { 524 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") 525 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") 526 527 // Tag and upload the images to Docker Hub 528 for _, tag := range tags { 529 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) 530 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) 531 532 // If the image already exists (non version tag), check the build 533 // number to prevent overwriting a newer commit if concurrent builds 534 // are running. This is still a tiny bit racey if two published are 535 // done at the same time, but that's extremely unlikely even on the 536 // master branch. 537 for _, img := range []string{gethImage, toolImage} { 538 if exec.Command("docker", "pull", img).Run() != nil { 539 continue // Generally the only failure is a missing image, which is good 540 } 541 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 542 if err != nil { 543 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 544 } 545 buildnum = bytes.TrimSpace(buildnum) 546 547 if len(buildnum) > 0 && len(env.Buildnum) > 0 { 548 oldnum, err := strconv.Atoi(string(buildnum)) 549 if err != nil { 550 log.Fatalf("Failed to parse old image build number: %v", err) 551 } 552 newnum, err := strconv.Atoi(env.Buildnum) 553 if err != nil { 554 log.Fatalf("Failed to parse current build number: %v", err) 555 } 556 if oldnum > newnum { 557 log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) 558 } else { 559 log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) 560 } 561 } 562 } 563 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) 564 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) 565 build.MustRunCommand("docker", "push", gethImage) 566 build.MustRunCommand("docker", "push", toolImage) 567 } 568 } 569 // If multi-arch image manifest push is requested, assemble it 570 if len(*manifest) != 0 { 571 // Since different architectures are pushed by different builders, wait 572 // until all required images are updated. 573 var mismatch bool 574 for i := 0; i < 2; i++ { // 2 attempts, second is race check 575 mismatch = false // hope there's no mismatch now 576 577 for _, tag := range tags { 578 for _, arch := range strings.Split(*manifest, ",") { 579 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) 580 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) 581 582 for _, img := range []string{gethImage, toolImage} { 583 if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { 584 log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) 585 mismatch = true 586 break 587 } 588 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 589 if err != nil { 590 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 591 } 592 buildnum = bytes.TrimSpace(buildnum) 593 594 if string(buildnum) != env.Buildnum { 595 log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) 596 mismatch = true 597 break 598 } 599 } 600 if mismatch { 601 break 602 } 603 } 604 if mismatch { 605 break 606 } 607 } 608 if mismatch { 609 // Build numbers mismatching, retry in a short time to 610 // avoid concurrent fails in both publisher images. If 611 // however the retry failed too, it means the concurrent 612 // builder is still crunching, let that do the publish. 613 if i == 0 { 614 time.Sleep(30 * time.Second) 615 } 616 continue 617 } 618 break 619 } 620 if mismatch { 621 log.Println("Relinquishing publish to other builder") 622 return 623 } 624 // Assemble and push the Geth manifest image 625 for _, tag := range tags { 626 gethImage := fmt.Sprintf("%s:%s", *upload, tag) 627 628 var gethSubImages []string 629 for _, arch := range strings.Split(*manifest, ",") { 630 gethSubImages = append(gethSubImages, gethImage+"-"+arch) 631 } 632 build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) 633 build.MustRunCommand("docker", "manifest", "push", gethImage) 634 } 635 // Assemble and push the alltools manifest image 636 for _, tag := range tags { 637 toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) 638 639 var toolSubImages []string 640 for _, arch := range strings.Split(*manifest, ",") { 641 toolSubImages = append(toolSubImages, toolImage+"-"+arch) 642 } 643 build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) 644 build.MustRunCommand("docker", "manifest", "push", toolImage) 645 } 646 } 647 } 648 649 // Debian Packaging 650 func doDebianSource(cmdline []string) { 651 var ( 652 cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) 653 signer = flag.String("signer", "", `Signing key name, also used as package author`) 654 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 655 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 656 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 657 now = time.Now() 658 ) 659 flag.CommandLine.Parse(cmdline) 660 *workdir = makeWorkdir(*workdir) 661 env := build.Env() 662 tc := new(build.GoToolchain) 663 maybeSkipArchive(env) 664 665 // Import the signing key. 666 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 667 gpg := exec.Command("gpg", "--import") 668 gpg.Stdin = bytes.NewReader(key) 669 build.MustRun(gpg) 670 } 671 672 // Download and verify the Go source package. 673 gobundle := downloadGoSources(*cachedir) 674 675 // Download all the dependencies needed to build the sources and run the ci script 676 srcdepfetch := tc.Go("mod", "download") 677 srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 678 build.MustRun(srcdepfetch) 679 680 cidepfetch := tc.Go("run", "./build/ci.go") 681 cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 682 cidepfetch.Run() // Command fails, don't care, we only need the deps to start it 683 684 // Create Debian packages and upload them. 685 for _, pkg := range debPackages { 686 for distro, goboot := range debDistroGoBoots { 687 // Prepare the debian package with the go-ethereum sources. 688 meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 689 pkgdir := stageDebianSource(*workdir, meta) 690 691 // Add Go source code 692 if err := build.ExtractArchive(gobundle, pkgdir); err != nil { 693 log.Fatalf("Failed to extract Go sources: %v", err) 694 } 695 if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { 696 log.Fatalf("Failed to rename Go source folder: %v", err) 697 } 698 // Add all dependency modules in compressed form 699 os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) 700 if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { 701 log.Fatalf("Failed to copy Go module dependencies: %v", err) 702 } 703 // Run the packaging and upload to the PPA 704 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") 705 debuild.Dir = pkgdir 706 build.MustRun(debuild) 707 708 var ( 709 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 710 source = filepath.Join(*workdir, basename+".tar.xz") 711 dsc = filepath.Join(*workdir, basename+".dsc") 712 changes = filepath.Join(*workdir, basename+"_source.changes") 713 buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo") 714 ) 715 if *signer != "" { 716 build.MustRunCommand("debsign", changes) 717 } 718 if *upload != "" { 719 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo}) 720 } 721 } 722 } 723 } 724 725 // downloadGoSources downloads the Go source tarball. 726 func downloadGoSources(cachedir string) string { 727 csdb := build.MustLoadChecksums("build/checksums.txt") 728 file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) 729 url := "https://dl.google.com/go/" + file 730 dst := filepath.Join(cachedir, file) 731 if err := csdb.DownloadFile(url, dst); err != nil { 732 log.Fatal(err) 733 } 734 return dst 735 } 736 737 func ppaUpload(workdir, ppa, sshUser string, files []string) { 738 p := strings.Split(ppa, "/") 739 if len(p) != 2 { 740 log.Fatal("-upload PPA name must contain single /") 741 } 742 if sshUser == "" { 743 sshUser = p[0] 744 } 745 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 746 // Create the SSH identity file if it doesn't exist. 747 var idfile string 748 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 749 idfile = filepath.Join(workdir, "sshkey") 750 if !common.FileExist(idfile) { 751 os.WriteFile(idfile, sshkey, 0600) 752 } 753 } 754 // Upload 755 dest := sshUser + "@ppa.launchpad.net" 756 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 757 log.Fatal(err) 758 } 759 } 760 761 func getenvBase64(variable string) []byte { 762 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 763 if err != nil { 764 log.Fatal("invalid base64 " + variable) 765 } 766 return []byte(dec) 767 } 768 769 func makeWorkdir(wdflag string) string { 770 var err error 771 if wdflag != "" { 772 err = os.MkdirAll(wdflag, 0744) 773 } else { 774 wdflag, err = os.MkdirTemp("", "geth-build-") 775 } 776 if err != nil { 777 log.Fatal(err) 778 } 779 return wdflag 780 } 781 782 func isUnstableBuild(env build.Environment) bool { 783 if env.Tag != "" { 784 return false 785 } 786 return true 787 } 788 789 type debPackage struct { 790 Name string // the name of the Debian package to produce, e.g. "ethereum" 791 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 792 Executables []debExecutable // executables to be included in the package 793 } 794 795 type debMetadata struct { 796 Env build.Environment 797 GoBootPackage string 798 GoBootPath string 799 800 PackageName string 801 802 // go-ethereum version being built. Note that this 803 // is not the debian package version. The package version 804 // is constructed by VersionString. 805 Version string 806 807 Author string // "name <email>", also selects signing key 808 Distro, Time string 809 Executables []debExecutable 810 } 811 812 type debExecutable struct { 813 PackageName string 814 BinaryName string 815 Description string 816 } 817 818 // Package returns the name of the package if present, or 819 // fallbacks to BinaryName 820 func (d debExecutable) Package() string { 821 if d.PackageName != "" { 822 return d.PackageName 823 } 824 return d.BinaryName 825 } 826 827 func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 828 if author == "" { 829 // No signing key, use default author. 830 author = "Ethereum Builds <fjl@ethereum.org>" 831 } 832 return debMetadata{ 833 GoBootPackage: goboot, 834 GoBootPath: debGoBootPaths[goboot], 835 PackageName: name, 836 Env: env, 837 Author: author, 838 Distro: distro, 839 Version: version, 840 Time: t.Format(time.RFC1123Z), 841 Executables: exes, 842 } 843 } 844 845 // Name returns the name of the metapackage that depends 846 // on all executable packages. 847 func (meta debMetadata) Name() string { 848 if isUnstableBuild(meta.Env) { 849 return meta.PackageName + "-unstable" 850 } 851 return meta.PackageName 852 } 853 854 // VersionString returns the debian version of the packages. 855 func (meta debMetadata) VersionString() string { 856 vsn := meta.Version 857 if meta.Env.Buildnum != "" { 858 vsn += "+build" + meta.Env.Buildnum 859 } 860 if meta.Distro != "" { 861 vsn += "+" + meta.Distro 862 } 863 return vsn 864 } 865 866 // ExeList returns the list of all executable packages. 867 func (meta debMetadata) ExeList() string { 868 names := make([]string, len(meta.Executables)) 869 for i, e := range meta.Executables { 870 names[i] = meta.ExeName(e) 871 } 872 return strings.Join(names, ", ") 873 } 874 875 // ExeName returns the package name of an executable package. 876 func (meta debMetadata) ExeName(exe debExecutable) string { 877 if isUnstableBuild(meta.Env) { 878 return exe.Package() + "-unstable" 879 } 880 return exe.Package() 881 } 882 883 // ExeConflicts returns the content of the Conflicts field 884 // for executable packages. 885 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 886 if isUnstableBuild(meta.Env) { 887 // Set up the conflicts list so that the *-unstable packages 888 // cannot be installed alongside the regular version. 889 // 890 // https://www.debian.org/doc/debian-policy/ch-relationships.html 891 // is very explicit about Conflicts: and says that Breaks: should 892 // be preferred and the conflicting files should be handled via 893 // alternates. We might do this eventually but using a conflict is 894 // easier now. 895 return "ethereum, " + exe.Package() 896 } 897 return "" 898 } 899 900 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 901 pkg := meta.Name() + "-" + meta.VersionString() 902 pkgdir = filepath.Join(tmpdir, pkg) 903 if err := os.Mkdir(pkgdir, 0755); err != nil { 904 log.Fatal(err) 905 } 906 // Copy the source code. 907 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 908 909 // Put the debian build files in place. 910 debian := filepath.Join(pkgdir, "debian") 911 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 912 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 913 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 914 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 915 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 916 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 917 for _, exe := range meta.Executables { 918 install := filepath.Join(debian, meta.ExeName(exe)+".install") 919 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 920 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 921 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 922 } 923 return pkgdir 924 } 925 926 // Windows installer 927 func doWindowsInstaller(cmdline []string) { 928 // Parse the flags and make skip installer generation on PRs 929 var ( 930 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 931 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 932 signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) 933 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 934 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 935 ) 936 flag.CommandLine.Parse(cmdline) 937 *workdir = makeWorkdir(*workdir) 938 env := build.Env() 939 maybeSkipArchive(env) 940 941 // Aggregate binaries that are included in the installer 942 var ( 943 devTools []string 944 allTools []string 945 gethTool string 946 ) 947 for _, file := range allToolsArchiveFiles { 948 if file == "COPYING" { // license, copied later 949 continue 950 } 951 allTools = append(allTools, filepath.Base(file)) 952 if filepath.Base(file) == "geth.exe" { 953 gethTool = file 954 } else { 955 devTools = append(devTools, file) 956 } 957 } 958 959 // Render NSIS scripts: Installer NSIS contains two installer sections, 960 // first section contains the geth binary, second section holds the dev tools. 961 templateData := map[string]interface{}{ 962 "License": "COPYING", 963 "Geth": gethTool, 964 "DevTools": devTools, 965 } 966 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 967 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 968 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 969 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 970 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 971 if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { 972 log.Fatalf("Failed to copy SimpleFC.dll: %v", err) 973 } 974 if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { 975 log.Fatalf("Failed to copy copyright note: %v", err) 976 } 977 // Build the installer. This assumes that all the needed files have been previously 978 // built (don't mix building and packaging to keep cross compilation complexity to a 979 // minimum). 980 version := strings.Split(params.Version, ".") 981 if env.Commit != "" { 982 version[2] += "-" + env.Commit[:8] 983 } 984 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 985 build.MustRunCommand("makensis.exe", 986 "/DOUTPUTFILE="+installer, 987 "/DMAJORVERSION="+version[0], 988 "/DMINORVERSION="+version[1], 989 "/DBUILDVERSION="+version[2], 990 "/DARCH="+*arch, 991 filepath.Join(*workdir, "geth.nsi"), 992 ) 993 // Sign and publish installer. 994 if err := archiveUpload(installer, *upload, *signer, *signify); err != nil { 995 log.Fatal(err) 996 } 997 } 998 999 // Android archives 1000 1001 func doAndroidArchive(cmdline []string) { 1002 var ( 1003 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1004 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 1005 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`) 1006 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 1007 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 1008 ) 1009 flag.CommandLine.Parse(cmdline) 1010 env := build.Env() 1011 tc := new(build.GoToolchain) 1012 1013 // Sanity check that the SDK and NDK are installed and set 1014 if os.Getenv("ANDROID_HOME") == "" { 1015 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 1016 } 1017 1018 // Build gomobile. 1019 install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest") 1020 install.Env = append(install.Env) 1021 build.MustRun(install) 1022 1023 // Ensure all dependencies are available. This is required to make 1024 // gomobile bind work because it expects go.sum to contain all checksums. 1025 build.MustRun(tc.Go("mod", "download")) 1026 1027 // Build the Android archive and Maven resources 1028 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) 1029 1030 if *local { 1031 // If we're building locally, copy bundle to build dir and skip Maven 1032 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 1033 os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar")) 1034 return 1035 } 1036 meta := newMavenMetadata(env) 1037 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 1038 1039 // Skip Maven deploy and Azure upload for PR builds 1040 maybeSkipArchive(env) 1041 1042 // Sign and upload the archive to Azure 1043 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 1044 os.Rename("geth.aar", archive) 1045 1046 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 1047 log.Fatal(err) 1048 } 1049 // Sign and upload all the artifacts to Maven Central 1050 os.Rename(archive, meta.Package+".aar") 1051 if *signer != "" && *deploy != "" { 1052 // Import the signing key into the local GPG instance 1053 key := getenvBase64(*signer) 1054 gpg := exec.Command("gpg", "--import") 1055 gpg.Stdin = bytes.NewReader(key) 1056 build.MustRun(gpg) 1057 keyID, err := build.PGPKeyID(string(key)) 1058 if err != nil { 1059 log.Fatal(err) 1060 } 1061 // Upload the artifacts to Sonatype and/or Maven Central 1062 repo := *deploy + "/service/local/staging/deploy/maven2" 1063 if meta.Develop { 1064 repo = *deploy + "/content/repositories/snapshots" 1065 } 1066 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 1067 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 1068 "-Dgpg.keyname="+keyID, 1069 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 1070 } 1071 } 1072 1073 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 1074 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 1075 cmd.Args = append(cmd.Args, args...) 1076 cmd.Env = []string{ 1077 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 1078 } 1079 for _, e := range os.Environ() { 1080 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") { 1081 continue 1082 } 1083 cmd.Env = append(cmd.Env, e) 1084 } 1085 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 1086 return cmd 1087 } 1088 1089 type mavenMetadata struct { 1090 Version string 1091 Package string 1092 Develop bool 1093 Contributors []mavenContributor 1094 } 1095 1096 type mavenContributor struct { 1097 Name string 1098 Email string 1099 } 1100 1101 func newMavenMetadata(env build.Environment) mavenMetadata { 1102 // Collect the list of authors from the repo root 1103 contribs := []mavenContributor{} 1104 if authors, err := os.Open("AUTHORS"); err == nil { 1105 defer authors.Close() 1106 1107 scanner := bufio.NewScanner(authors) 1108 for scanner.Scan() { 1109 // Skip any whitespace from the authors list 1110 line := strings.TrimSpace(scanner.Text()) 1111 if line == "" || line[0] == '#' { 1112 continue 1113 } 1114 // Split the author and insert as a contributor 1115 re := regexp.MustCompile("([^<]+) <(.+)>") 1116 parts := re.FindStringSubmatch(line) 1117 if len(parts) == 3 { 1118 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 1119 } 1120 } 1121 } 1122 // Render the version and package strings 1123 version := params.Version 1124 if isUnstableBuild(env) { 1125 version += "-SNAPSHOT" 1126 } 1127 return mavenMetadata{ 1128 Version: version, 1129 Package: "geth-" + version, 1130 Develop: isUnstableBuild(env), 1131 Contributors: contribs, 1132 } 1133 } 1134 1135 // XCode frameworks 1136 1137 func doXCodeFramework(cmdline []string) { 1138 var ( 1139 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1140 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 1141 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`) 1142 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 1143 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 1144 ) 1145 flag.CommandLine.Parse(cmdline) 1146 env := build.Env() 1147 tc := new(build.GoToolchain) 1148 1149 // Build gomobile. 1150 build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 1151 1152 // Build the iOS XCode framework 1153 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") 1154 1155 if *local { 1156 // If we're building locally, use the build folder and stop afterwards 1157 bind.Dir = GOBIN 1158 build.MustRun(bind) 1159 return 1160 } 1161 1162 // Create the archive. 1163 maybeSkipArchive(env) 1164 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 1165 if err := os.MkdirAll(archive, 0755); err != nil { 1166 log.Fatal(err) 1167 } 1168 bind.Dir, _ = filepath.Abs(archive) 1169 build.MustRun(bind) 1170 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 1171 1172 // Sign and upload the framework to Azure 1173 if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { 1174 log.Fatal(err) 1175 } 1176 // Prepare and upload a PodSpec to CocoaPods 1177 if *deploy != "" { 1178 meta := newPodMetadata(env, archive) 1179 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 1180 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings") 1181 } 1182 } 1183 1184 type podMetadata struct { 1185 Version string 1186 Commit string 1187 Archive string 1188 Contributors []podContributor 1189 } 1190 1191 type podContributor struct { 1192 Name string 1193 Email string 1194 } 1195 1196 func newPodMetadata(env build.Environment, archive string) podMetadata { 1197 // Collect the list of authors from the repo root 1198 contribs := []podContributor{} 1199 if authors, err := os.Open("AUTHORS"); err == nil { 1200 defer authors.Close() 1201 1202 scanner := bufio.NewScanner(authors) 1203 for scanner.Scan() { 1204 // Skip any whitespace from the authors list 1205 line := strings.TrimSpace(scanner.Text()) 1206 if line == "" || line[0] == '#' { 1207 continue 1208 } 1209 // Split the author and insert as a contributor 1210 re := regexp.MustCompile("([^<]+) <(.+)>") 1211 parts := re.FindStringSubmatch(line) 1212 if len(parts) == 3 { 1213 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 1214 } 1215 } 1216 } 1217 version := params.Version 1218 if isUnstableBuild(env) { 1219 version += "-unstable." + env.Buildnum 1220 } 1221 return podMetadata{ 1222 Archive: archive, 1223 Version: version, 1224 Commit: env.Commit, 1225 Contributors: contribs, 1226 } 1227 } 1228 1229 // Binary distribution cleanups 1230 1231 func doPurge(cmdline []string) { 1232 var ( 1233 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1234 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1235 ) 1236 flag.CommandLine.Parse(cmdline) 1237 1238 if env := build.Env(); !env.IsCronJob { 1239 log.Printf("skipping because not a cron job") 1240 os.Exit(0) 1241 } 1242 // Create the azure authentication and list the current archives 1243 auth := build.AzureBlobstoreConfig{ 1244 Account: strings.Split(*store, "/")[0], 1245 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1246 Container: strings.SplitN(*store, "/", 2)[1], 1247 } 1248 blobs, err := build.AzureBlobstoreList(auth) 1249 if err != nil { 1250 log.Fatal(err) 1251 } 1252 fmt.Printf("Found %d blobs\n", len(blobs)) 1253 1254 // Iterate over the blobs, collect and sort all unstable builds 1255 for i := 0; i < len(blobs); i++ { 1256 if !strings.Contains(*blobs[i].Name, "unstable") { 1257 blobs = append(blobs[:i], blobs[i+1:]...) 1258 i-- 1259 } 1260 } 1261 for i := 0; i < len(blobs); i++ { 1262 for j := i + 1; j < len(blobs); j++ { 1263 if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) { 1264 blobs[i], blobs[j] = blobs[j], blobs[i] 1265 } 1266 } 1267 } 1268 // Filter out all archives more recent that the given threshold 1269 for i, blob := range blobs { 1270 if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1271 blobs = blobs[:i] 1272 break 1273 } 1274 } 1275 fmt.Printf("Deleting %d blobs\n", len(blobs)) 1276 // Delete all marked as such and return 1277 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1278 log.Fatal(err) 1279 } 1280 }