github.com/sexdefi/go-ethereum@v0.0.0-20230807164010-b4cd42fe399f/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 //go:build none 18 // +build none 19 20 /* 21 The ci command is called from Continuous Integration scripts. 22 23 Usage: go run build/ci.go <command> <command flags/arguments> 24 25 Available commands are: 26 27 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 28 test [ -coverage ] [ packages... ] -- runs the tests 29 lint -- runs certain pre-selected linters 30 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts 31 importkeys -- imports signing keys from env 32 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 33 nsis -- creates a Windows NSIS installer 34 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 35 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "log" 50 "os" 51 "os/exec" 52 "path" 53 "path/filepath" 54 "regexp" 55 "runtime" 56 "strconv" 57 "strings" 58 "time" 59 60 "github.com/cespare/cp" 61 "github.com/sexdefi/go-ethereum/common" 62 "github.com/sexdefi/go-ethereum/crypto/signify" 63 "github.com/sexdefi/go-ethereum/internal/build" 64 "github.com/sexdefi/go-ethereum/params" 65 ) 66 67 var ( 68 // Files that end up in the geth*.zip archive. 69 gethArchiveFiles = []string{ 70 "COPYING", 71 executablePath("geth"), 72 } 73 74 // Files that end up in the geth-alltools*.zip archive. 75 allToolsArchiveFiles = []string{ 76 "COPYING", 77 executablePath("abigen"), 78 executablePath("bootnode"), 79 executablePath("evm"), 80 executablePath("geth"), 81 executablePath("puppeth"), 82 executablePath("rlpdump"), 83 executablePath("clef"), 84 } 85 86 // A debian package is created for all executables listed here. 87 debExecutables = []debExecutable{ 88 { 89 BinaryName: "abigen", 90 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 91 }, 92 { 93 BinaryName: "bootnode", 94 Description: "Ethereum bootnode.", 95 }, 96 { 97 BinaryName: "evm", 98 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 99 }, 100 { 101 BinaryName: "geth", 102 Description: "Ethereum CLI client.", 103 }, 104 { 105 BinaryName: "puppeth", 106 Description: "Ethereum private network manager.", 107 }, 108 { 109 BinaryName: "rlpdump", 110 Description: "Developer utility tool that prints RLP structures.", 111 }, 112 { 113 BinaryName: "clef", 114 Description: "Ethereum account management tool.", 115 }, 116 } 117 118 // A debian package is created for all executables listed here. 119 debEthereum = debPackage{ 120 Name: "ethereum", 121 Version: params.Version, 122 Executables: debExecutables, 123 } 124 125 // Debian meta packages to build and push to Ubuntu PPA 126 debPackages = []debPackage{ 127 debEthereum, 128 } 129 130 // Distros for which packages are created. 131 // Note: vivid is unsupported because there is no golang-1.6 package for it. 132 // Note: the following Ubuntu releases have been officially deprecated on Launchpad: 133 // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite 134 debDistroGoBoots = map[string]string{ 135 "trusty": "golang-1.11", // EOL: 04/2024 136 "xenial": "golang-go", // EOL: 04/2026 137 "bionic": "golang-go", // EOL: 04/2028 138 "focal": "golang-go", // EOL: 04/2030 139 "impish": "golang-go", // EOL: 07/2022 140 "jammy": "golang-go", // EOL: 04/2032 141 //"kinetic": "golang-go", // EOL: 07/2023 142 } 143 144 debGoBootPaths = map[string]string{ 145 "golang-1.11": "/usr/lib/go-1.11", 146 "golang-go": "/usr/lib/go", 147 } 148 149 // This is the version of go that will be downloaded by 150 // 151 // go run ci.go install -dlgo 152 dlgoVersion = "1.18.5" 153 ) 154 155 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 156 157 func executablePath(name string) string { 158 if runtime.GOOS == "windows" { 159 name += ".exe" 160 } 161 return filepath.Join(GOBIN, name) 162 } 163 164 func main() { 165 log.SetFlags(log.Lshortfile) 166 167 if !common.FileExist(filepath.Join("build", "ci.go")) { 168 log.Fatal("this script must be run from the root of the repository") 169 } 170 if len(os.Args) < 2 { 171 log.Fatal("need subcommand as first argument") 172 } 173 switch os.Args[1] { 174 case "install": 175 doInstall(os.Args[2:]) 176 case "test": 177 doTest(os.Args[2:]) 178 case "lint": 179 doLint(os.Args[2:]) 180 case "archive": 181 doArchive(os.Args[2:]) 182 case "docker": 183 doDocker(os.Args[2:]) 184 case "debsrc": 185 doDebianSource(os.Args[2:]) 186 case "nsis": 187 doWindowsInstaller(os.Args[2:]) 188 case "aar": 189 doAndroidArchive(os.Args[2:]) 190 case "xcode": 191 doXCodeFramework(os.Args[2:]) 192 case "purge": 193 doPurge(os.Args[2:]) 194 default: 195 log.Fatal("unknown command ", os.Args[1]) 196 } 197 } 198 199 // Compiling 200 201 func doInstall(cmdline []string) { 202 var ( 203 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 204 arch = flag.String("arch", "", "Architecture to cross build for") 205 cc = flag.String("cc", "", "C compiler to cross build with") 206 staticlink = flag.Bool("static", false, "Create statically-linked executable") 207 ) 208 flag.CommandLine.Parse(cmdline) 209 210 // Configure the toolchain. 211 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 212 if *dlgo { 213 csdb := build.MustLoadChecksums("build/checksums.txt") 214 tc.Root = build.DownloadGo(csdb, dlgoVersion) 215 } 216 217 // Disable CLI markdown doc generation in release builds. 218 buildTags := []string{"urfave_cli_no_docs"} 219 220 // Configure the build. 221 env := build.Env() 222 gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) 223 224 // arm64 CI builders are memory-constrained and can't handle concurrent builds, 225 // better disable it. This check isn't the best, it should probably 226 // check for something in env instead. 227 if env.CI && runtime.GOARCH == "arm64" { 228 gobuild.Args = append(gobuild.Args, "-p", "1") 229 } 230 231 // We use -trimpath to avoid leaking local paths into the built executables. 232 gobuild.Args = append(gobuild.Args, "-trimpath") 233 234 // Show packages during build. 235 gobuild.Args = append(gobuild.Args, "-v") 236 237 // Now we choose what we're even building. 238 // Default: collect all 'main' packages in cmd/ and build those. 239 packages := flag.Args() 240 if len(packages) == 0 { 241 packages = build.FindMainPackages("./cmd") 242 } 243 244 // Do the build! 245 for _, pkg := range packages { 246 args := make([]string, len(gobuild.Args)) 247 copy(args, gobuild.Args) 248 args = append(args, "-o", executablePath(path.Base(pkg))) 249 args = append(args, pkg) 250 build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) 251 } 252 } 253 254 // buildFlags returns the go tool flags for building. 255 func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { 256 var ld []string 257 if env.Commit != "" { 258 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 259 ld = append(ld, "-X", "main.gitDate="+env.Date) 260 } 261 // Strip DWARF on darwin. This used to be required for certain things, 262 // and there is no downside to this, so we just keep doing it. 263 if runtime.GOOS == "darwin" { 264 ld = append(ld, "-s") 265 } 266 if runtime.GOOS == "linux" { 267 // Enforce the stacksize to 8M, which is the case on most platforms apart from 268 // alpine Linux. 269 extld := []string{"-Wl,-z,stack-size=0x800000"} 270 if staticLinking { 271 extld = append(extld, "-static") 272 // Under static linking, use of certain glibc features must be 273 // disabled to avoid shared library dependencies. 274 buildTags = append(buildTags, "osusergo", "netgo") 275 } 276 ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") 277 } 278 if len(ld) > 0 { 279 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 280 } 281 if len(buildTags) > 0 { 282 flags = append(flags, "-tags", strings.Join(buildTags, ",")) 283 } 284 return flags 285 } 286 287 // Running The Tests 288 // 289 // "tests" also includes static analysis tools such as vet. 290 291 func doTest(cmdline []string) { 292 var ( 293 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 294 arch = flag.String("arch", "", "Run tests for given architecture") 295 cc = flag.String("cc", "", "Sets C compiler binary") 296 coverage = flag.Bool("coverage", false, "Whether to record code coverage") 297 verbose = flag.Bool("v", false, "Whether to log verbosely") 298 race = flag.Bool("race", false, "Execute the race detector") 299 ) 300 flag.CommandLine.Parse(cmdline) 301 302 // Configure the toolchain. 303 tc := build.GoToolchain{GOARCH: *arch, CC: *cc} 304 if *dlgo { 305 csdb := build.MustLoadChecksums("build/checksums.txt") 306 tc.Root = build.DownloadGo(csdb, dlgoVersion) 307 } 308 gotest := tc.Go("test") 309 310 // Test a single package at a time. CI builders are slow 311 // and some tests run into timeouts under load. 312 gotest.Args = append(gotest.Args, "-p", "1") 313 if *coverage { 314 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 315 } 316 if *verbose { 317 gotest.Args = append(gotest.Args, "-v") 318 } 319 if *race { 320 gotest.Args = append(gotest.Args, "-race") 321 } 322 323 packages := []string{"./..."} 324 if len(flag.CommandLine.Args()) > 0 { 325 packages = flag.CommandLine.Args() 326 } 327 gotest.Args = append(gotest.Args, packages...) 328 build.MustRun(gotest) 329 } 330 331 // doLint runs golangci-lint on requested packages. 332 func doLint(cmdline []string) { 333 var ( 334 cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") 335 ) 336 flag.CommandLine.Parse(cmdline) 337 packages := []string{"./..."} 338 if len(flag.CommandLine.Args()) > 0 { 339 packages = flag.CommandLine.Args() 340 } 341 342 linter := downloadLinter(*cachedir) 343 lflags := []string{"run", "--config", ".golangci.yml"} 344 build.MustRunCommand(linter, append(lflags, packages...)...) 345 fmt.Println("You have achieved perfection.") 346 } 347 348 // downloadLinter downloads and unpacks golangci-lint. 349 func downloadLinter(cachedir string) string { 350 const version = "1.46.2" 351 352 csdb := build.MustLoadChecksums("build/checksums.txt") 353 arch := runtime.GOARCH 354 ext := ".tar.gz" 355 356 if runtime.GOOS == "windows" { 357 ext = ".zip" 358 } 359 if arch == "arm" { 360 arch += "v" + os.Getenv("GOARM") 361 } 362 base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) 363 url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) 364 archivePath := filepath.Join(cachedir, base+ext) 365 if err := csdb.DownloadFile(url, archivePath); err != nil { 366 log.Fatal(err) 367 } 368 if err := build.ExtractArchive(archivePath, cachedir); err != nil { 369 log.Fatal(err) 370 } 371 return filepath.Join(cachedir, base, "golangci-lint") 372 } 373 374 // Release Packaging 375 func doArchive(cmdline []string) { 376 var ( 377 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 378 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 379 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 380 signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`) 381 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 382 ext string 383 ) 384 flag.CommandLine.Parse(cmdline) 385 switch *atype { 386 case "zip": 387 ext = ".zip" 388 case "tar": 389 ext = ".tar.gz" 390 default: 391 log.Fatal("unknown archive type: ", atype) 392 } 393 394 var ( 395 env = build.Env() 396 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 397 geth = "geth-" + basegeth + ext 398 alltools = "geth-alltools-" + basegeth + ext 399 ) 400 maybeSkipArchive(env) 401 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 402 log.Fatal(err) 403 } 404 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 405 log.Fatal(err) 406 } 407 for _, archive := range []string{geth, alltools} { 408 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 409 log.Fatal(err) 410 } 411 } 412 } 413 414 func archiveBasename(arch string, archiveVersion string) string { 415 platform := runtime.GOOS + "-" + arch 416 if arch == "arm" { 417 platform += os.Getenv("GOARM") 418 } 419 if arch == "android" { 420 platform = "android-all" 421 } 422 if arch == "ios" { 423 platform = "ios-all" 424 } 425 return platform + "-" + archiveVersion 426 } 427 428 func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error { 429 // If signing was requested, generate the signature files 430 if signer != "" { 431 key := getenvBase64(signer) 432 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 433 return err 434 } 435 } 436 if signifyVar != "" { 437 key := os.Getenv(signifyVar) 438 untrustedComment := "verify with geth-release.pub" 439 trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123)) 440 if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil { 441 return err 442 } 443 } 444 // If uploading to Azure was requested, push the archive possibly with its signature 445 if blobstore != "" { 446 auth := build.AzureBlobstoreConfig{ 447 Account: strings.Split(blobstore, "/")[0], 448 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 449 Container: strings.SplitN(blobstore, "/", 2)[1], 450 } 451 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 452 return err 453 } 454 if signer != "" { 455 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 456 return err 457 } 458 } 459 if signifyVar != "" { 460 if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { 461 return err 462 } 463 } 464 } 465 return nil 466 } 467 468 // skips archiving for some build configurations. 469 func maybeSkipArchive(env build.Environment) { 470 if env.IsPullRequest { 471 log.Printf("skipping archive creation because this is a PR build") 472 os.Exit(0) 473 } 474 if env.IsCronJob { 475 log.Printf("skipping archive creation because this is a cron job") 476 os.Exit(0) 477 } 478 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 479 log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) 480 os.Exit(0) 481 } 482 } 483 484 // Builds the docker images and optionally uploads them to Docker Hub. 485 func doDocker(cmdline []string) { 486 var ( 487 image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) 488 manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) 489 upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) 490 ) 491 flag.CommandLine.Parse(cmdline) 492 493 // Skip building and pushing docker images for PR builds 494 env := build.Env() 495 maybeSkipArchive(env) 496 497 // Retrieve the upload credentials and authenticate 498 user := getenvBase64("DOCKER_HUB_USERNAME") 499 pass := getenvBase64("DOCKER_HUB_PASSWORD") 500 501 if len(user) > 0 && len(pass) > 0 { 502 auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") 503 auther.Stdin = bytes.NewReader(pass) 504 build.MustRun(auther) 505 } 506 // Retrieve the version infos to build and push to the following paths: 507 // - ethereum/client-go:latest - Pushes to the master branch, Geth only 508 // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only 509 // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools 510 // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools 511 // - ethereum/client-go:release-<major>.<minor> - Version tag publish on GitHub, Geth only 512 // - ethereum/client-go:alltools-release-<major>.<minor> - Version tag publish on GitHub, Geth & tools 513 // - ethereum/client-go:v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth only 514 // - ethereum/client-go:alltools-v<major>.<minor>.<patch> - Version tag publish on GitHub, Geth & tools 515 var tags []string 516 517 switch { 518 case env.Branch == "master": 519 tags = []string{"latest"} 520 case strings.HasPrefix(env.Tag, "v1."): 521 tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} 522 } 523 // If architecture specific image builds are requested, build and push them 524 if *image { 525 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") 526 build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") 527 528 // Tag and upload the images to Docker Hub 529 for _, tag := range tags { 530 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) 531 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) 532 533 // If the image already exists (non version tag), check the build 534 // number to prevent overwriting a newer commit if concurrent builds 535 // are running. This is still a tiny bit racey if two published are 536 // done at the same time, but that's extremely unlikely even on the 537 // master branch. 538 for _, img := range []string{gethImage, toolImage} { 539 if exec.Command("docker", "pull", img).Run() != nil { 540 continue // Generally the only failure is a missing image, which is good 541 } 542 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 543 if err != nil { 544 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 545 } 546 buildnum = bytes.TrimSpace(buildnum) 547 548 if len(buildnum) > 0 && len(env.Buildnum) > 0 { 549 oldnum, err := strconv.Atoi(string(buildnum)) 550 if err != nil { 551 log.Fatalf("Failed to parse old image build number: %v", err) 552 } 553 newnum, err := strconv.Atoi(env.Buildnum) 554 if err != nil { 555 log.Fatalf("Failed to parse current build number: %v", err) 556 } 557 if oldnum > newnum { 558 log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) 559 } else { 560 log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) 561 } 562 } 563 } 564 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) 565 build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) 566 build.MustRunCommand("docker", "push", gethImage) 567 build.MustRunCommand("docker", "push", toolImage) 568 } 569 } 570 // If multi-arch image manifest push is requested, assemble it 571 if len(*manifest) != 0 { 572 // Since different architectures are pushed by different builders, wait 573 // until all required images are updated. 574 var mismatch bool 575 for i := 0; i < 2; i++ { // 2 attempts, second is race check 576 mismatch = false // hope there's no mismatch now 577 578 for _, tag := range tags { 579 for _, arch := range strings.Split(*manifest, ",") { 580 gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) 581 toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) 582 583 for _, img := range []string{gethImage, toolImage} { 584 if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { 585 log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) 586 mismatch = true 587 break 588 } 589 buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() 590 if err != nil { 591 log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) 592 } 593 buildnum = bytes.TrimSpace(buildnum) 594 595 if string(buildnum) != env.Buildnum { 596 log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) 597 mismatch = true 598 break 599 } 600 } 601 if mismatch { 602 break 603 } 604 } 605 if mismatch { 606 break 607 } 608 } 609 if mismatch { 610 // Build numbers mismatching, retry in a short time to 611 // avoid concurrent fails in both publisher images. If 612 // however the retry failed too, it means the concurrent 613 // builder is still crunching, let that do the publish. 614 if i == 0 { 615 time.Sleep(30 * time.Second) 616 } 617 continue 618 } 619 break 620 } 621 if mismatch { 622 log.Println("Relinquishing publish to other builder") 623 return 624 } 625 // Assemble and push the Geth manifest image 626 for _, tag := range tags { 627 gethImage := fmt.Sprintf("%s:%s", *upload, tag) 628 629 var gethSubImages []string 630 for _, arch := range strings.Split(*manifest, ",") { 631 gethSubImages = append(gethSubImages, gethImage+"-"+arch) 632 } 633 build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) 634 build.MustRunCommand("docker", "manifest", "push", gethImage) 635 } 636 // Assemble and push the alltools manifest image 637 for _, tag := range tags { 638 toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) 639 640 var toolSubImages []string 641 for _, arch := range strings.Split(*manifest, ",") { 642 toolSubImages = append(toolSubImages, toolImage+"-"+arch) 643 } 644 build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) 645 build.MustRunCommand("docker", "manifest", "push", toolImage) 646 } 647 } 648 } 649 650 // Debian Packaging 651 func doDebianSource(cmdline []string) { 652 var ( 653 cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) 654 signer = flag.String("signer", "", `Signing key name, also used as package author`) 655 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 656 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 657 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 658 now = time.Now() 659 ) 660 flag.CommandLine.Parse(cmdline) 661 *workdir = makeWorkdir(*workdir) 662 env := build.Env() 663 tc := new(build.GoToolchain) 664 maybeSkipArchive(env) 665 666 // Import the signing key. 667 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 668 gpg := exec.Command("gpg", "--import") 669 gpg.Stdin = bytes.NewReader(key) 670 build.MustRun(gpg) 671 } 672 673 // Download and verify the Go source package. 674 gobundle := downloadGoSources(*cachedir) 675 676 // Download all the dependencies needed to build the sources and run the ci script 677 srcdepfetch := tc.Go("mod", "download") 678 srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 679 build.MustRun(srcdepfetch) 680 681 cidepfetch := tc.Go("run", "./build/ci.go") 682 cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath")) 683 cidepfetch.Run() // Command fails, don't care, we only need the deps to start it 684 685 // Create Debian packages and upload them. 686 for _, pkg := range debPackages { 687 for distro, goboot := range debDistroGoBoots { 688 // Prepare the debian package with the go-ethereum sources. 689 meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 690 pkgdir := stageDebianSource(*workdir, meta) 691 692 // Add Go source code 693 if err := build.ExtractArchive(gobundle, pkgdir); err != nil { 694 log.Fatalf("Failed to extract Go sources: %v", err) 695 } 696 if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { 697 log.Fatalf("Failed to rename Go source folder: %v", err) 698 } 699 // Add all dependency modules in compressed form 700 os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) 701 if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { 702 log.Fatalf("Failed to copy Go module dependencies: %v", err) 703 } 704 // Run the packaging and upload to the PPA 705 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") 706 debuild.Dir = pkgdir 707 build.MustRun(debuild) 708 709 var ( 710 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 711 source = filepath.Join(*workdir, basename+".tar.xz") 712 dsc = filepath.Join(*workdir, basename+".dsc") 713 changes = filepath.Join(*workdir, basename+"_source.changes") 714 buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo") 715 ) 716 if *signer != "" { 717 build.MustRunCommand("debsign", changes) 718 } 719 if *upload != "" { 720 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo}) 721 } 722 } 723 } 724 } 725 726 // downloadGoSources downloads the Go source tarball. 727 func downloadGoSources(cachedir string) string { 728 csdb := build.MustLoadChecksums("build/checksums.txt") 729 file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) 730 url := "https://dl.google.com/go/" + file 731 dst := filepath.Join(cachedir, file) 732 if err := csdb.DownloadFile(url, dst); err != nil { 733 log.Fatal(err) 734 } 735 return dst 736 } 737 738 func ppaUpload(workdir, ppa, sshUser string, files []string) { 739 p := strings.Split(ppa, "/") 740 if len(p) != 2 { 741 log.Fatal("-upload PPA name must contain single /") 742 } 743 if sshUser == "" { 744 sshUser = p[0] 745 } 746 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 747 // Create the SSH identity file if it doesn't exist. 748 var idfile string 749 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 750 idfile = filepath.Join(workdir, "sshkey") 751 if !common.FileExist(idfile) { 752 os.WriteFile(idfile, sshkey, 0600) 753 } 754 } 755 // Upload 756 dest := sshUser + "@ppa.launchpad.net" 757 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 758 log.Fatal(err) 759 } 760 } 761 762 func getenvBase64(variable string) []byte { 763 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 764 if err != nil { 765 log.Fatal("invalid base64 " + variable) 766 } 767 return []byte(dec) 768 } 769 770 func makeWorkdir(wdflag string) string { 771 var err error 772 if wdflag != "" { 773 err = os.MkdirAll(wdflag, 0744) 774 } else { 775 wdflag, err = os.MkdirTemp("", "geth-build-") 776 } 777 if err != nil { 778 log.Fatal(err) 779 } 780 return wdflag 781 } 782 783 func isUnstableBuild(env build.Environment) bool { 784 if env.Tag != "" { 785 return false 786 } 787 return true 788 } 789 790 type debPackage struct { 791 Name string // the name of the Debian package to produce, e.g. "ethereum" 792 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 793 Executables []debExecutable // executables to be included in the package 794 } 795 796 type debMetadata struct { 797 Env build.Environment 798 GoBootPackage string 799 GoBootPath string 800 801 PackageName string 802 803 // go-ethereum version being built. Note that this 804 // is not the debian package version. The package version 805 // is constructed by VersionString. 806 Version string 807 808 Author string // "name <email>", also selects signing key 809 Distro, Time string 810 Executables []debExecutable 811 } 812 813 type debExecutable struct { 814 PackageName string 815 BinaryName string 816 Description string 817 } 818 819 // Package returns the name of the package if present, or 820 // fallbacks to BinaryName 821 func (d debExecutable) Package() string { 822 if d.PackageName != "" { 823 return d.PackageName 824 } 825 return d.BinaryName 826 } 827 828 func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 829 if author == "" { 830 // No signing key, use default author. 831 author = "Ethereum Builds <fjl@ethereum.org>" 832 } 833 return debMetadata{ 834 GoBootPackage: goboot, 835 GoBootPath: debGoBootPaths[goboot], 836 PackageName: name, 837 Env: env, 838 Author: author, 839 Distro: distro, 840 Version: version, 841 Time: t.Format(time.RFC1123Z), 842 Executables: exes, 843 } 844 } 845 846 // Name returns the name of the metapackage that depends 847 // on all executable packages. 848 func (meta debMetadata) Name() string { 849 if isUnstableBuild(meta.Env) { 850 return meta.PackageName + "-unstable" 851 } 852 return meta.PackageName 853 } 854 855 // VersionString returns the debian version of the packages. 856 func (meta debMetadata) VersionString() string { 857 vsn := meta.Version 858 if meta.Env.Buildnum != "" { 859 vsn += "+build" + meta.Env.Buildnum 860 } 861 if meta.Distro != "" { 862 vsn += "+" + meta.Distro 863 } 864 return vsn 865 } 866 867 // ExeList returns the list of all executable packages. 868 func (meta debMetadata) ExeList() string { 869 names := make([]string, len(meta.Executables)) 870 for i, e := range meta.Executables { 871 names[i] = meta.ExeName(e) 872 } 873 return strings.Join(names, ", ") 874 } 875 876 // ExeName returns the package name of an executable package. 877 func (meta debMetadata) ExeName(exe debExecutable) string { 878 if isUnstableBuild(meta.Env) { 879 return exe.Package() + "-unstable" 880 } 881 return exe.Package() 882 } 883 884 // ExeConflicts returns the content of the Conflicts field 885 // for executable packages. 886 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 887 if isUnstableBuild(meta.Env) { 888 // Set up the conflicts list so that the *-unstable packages 889 // cannot be installed alongside the regular version. 890 // 891 // https://www.debian.org/doc/debian-policy/ch-relationships.html 892 // is very explicit about Conflicts: and says that Breaks: should 893 // be preferred and the conflicting files should be handled via 894 // alternates. We might do this eventually but using a conflict is 895 // easier now. 896 return "ethereum, " + exe.Package() 897 } 898 return "" 899 } 900 901 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 902 pkg := meta.Name() + "-" + meta.VersionString() 903 pkgdir = filepath.Join(tmpdir, pkg) 904 if err := os.Mkdir(pkgdir, 0755); err != nil { 905 log.Fatal(err) 906 } 907 // Copy the source code. 908 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 909 910 // Put the debian build files in place. 911 debian := filepath.Join(pkgdir, "debian") 912 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 913 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 914 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 915 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 916 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 917 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 918 for _, exe := range meta.Executables { 919 install := filepath.Join(debian, meta.ExeName(exe)+".install") 920 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 921 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 922 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 923 } 924 return pkgdir 925 } 926 927 // Windows installer 928 func doWindowsInstaller(cmdline []string) { 929 // Parse the flags and make skip installer generation on PRs 930 var ( 931 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 932 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 933 signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) 934 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 935 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 936 ) 937 flag.CommandLine.Parse(cmdline) 938 *workdir = makeWorkdir(*workdir) 939 env := build.Env() 940 maybeSkipArchive(env) 941 942 // Aggregate binaries that are included in the installer 943 var ( 944 devTools []string 945 allTools []string 946 gethTool string 947 ) 948 for _, file := range allToolsArchiveFiles { 949 if file == "COPYING" { // license, copied later 950 continue 951 } 952 allTools = append(allTools, filepath.Base(file)) 953 if filepath.Base(file) == "geth.exe" { 954 gethTool = file 955 } else { 956 devTools = append(devTools, file) 957 } 958 } 959 960 // Render NSIS scripts: Installer NSIS contains two installer sections, 961 // first section contains the geth binary, second section holds the dev tools. 962 templateData := map[string]interface{}{ 963 "License": "COPYING", 964 "Geth": gethTool, 965 "DevTools": devTools, 966 } 967 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 968 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 969 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 970 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 971 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 972 if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { 973 log.Fatalf("Failed to copy SimpleFC.dll: %v", err) 974 } 975 if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { 976 log.Fatalf("Failed to copy copyright note: %v", err) 977 } 978 // Build the installer. This assumes that all the needed files have been previously 979 // built (don't mix building and packaging to keep cross compilation complexity to a 980 // minimum). 981 version := strings.Split(params.Version, ".") 982 if env.Commit != "" { 983 version[2] += "-" + env.Commit[:8] 984 } 985 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 986 build.MustRunCommand("makensis.exe", 987 "/DOUTPUTFILE="+installer, 988 "/DMAJORVERSION="+version[0], 989 "/DMINORVERSION="+version[1], 990 "/DBUILDVERSION="+version[2], 991 "/DARCH="+*arch, 992 filepath.Join(*workdir, "geth.nsi"), 993 ) 994 // Sign and publish installer. 995 if err := archiveUpload(installer, *upload, *signer, *signify); err != nil { 996 log.Fatal(err) 997 } 998 } 999 1000 // Android archives 1001 1002 func doAndroidArchive(cmdline []string) { 1003 var ( 1004 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1005 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 1006 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`) 1007 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 1008 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 1009 ) 1010 flag.CommandLine.Parse(cmdline) 1011 env := build.Env() 1012 tc := new(build.GoToolchain) 1013 1014 // Sanity check that the SDK and NDK are installed and set 1015 if os.Getenv("ANDROID_HOME") == "" { 1016 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 1017 } 1018 1019 // Build gomobile. 1020 install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest") 1021 install.Env = append(install.Env) 1022 build.MustRun(install) 1023 1024 // Ensure all dependencies are available. This is required to make 1025 // gomobile bind work because it expects go.sum to contain all checksums. 1026 build.MustRun(tc.Go("mod", "download")) 1027 1028 // Build the Android archive and Maven resources 1029 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/sexdefi/go-ethereum/mobile")) 1030 1031 if *local { 1032 // If we're building locally, copy bundle to build dir and skip Maven 1033 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 1034 os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar")) 1035 return 1036 } 1037 meta := newMavenMetadata(env) 1038 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 1039 1040 // Skip Maven deploy and Azure upload for PR builds 1041 maybeSkipArchive(env) 1042 1043 // Sign and upload the archive to Azure 1044 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 1045 os.Rename("geth.aar", archive) 1046 1047 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 1048 log.Fatal(err) 1049 } 1050 // Sign and upload all the artifacts to Maven Central 1051 os.Rename(archive, meta.Package+".aar") 1052 if *signer != "" && *deploy != "" { 1053 // Import the signing key into the local GPG instance 1054 key := getenvBase64(*signer) 1055 gpg := exec.Command("gpg", "--import") 1056 gpg.Stdin = bytes.NewReader(key) 1057 build.MustRun(gpg) 1058 keyID, err := build.PGPKeyID(string(key)) 1059 if err != nil { 1060 log.Fatal(err) 1061 } 1062 // Upload the artifacts to Sonatype and/or Maven Central 1063 repo := *deploy + "/service/local/staging/deploy/maven2" 1064 if meta.Develop { 1065 repo = *deploy + "/content/repositories/snapshots" 1066 } 1067 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 1068 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 1069 "-Dgpg.keyname="+keyID, 1070 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 1071 } 1072 } 1073 1074 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 1075 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 1076 cmd.Args = append(cmd.Args, args...) 1077 cmd.Env = []string{ 1078 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 1079 } 1080 for _, e := range os.Environ() { 1081 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") { 1082 continue 1083 } 1084 cmd.Env = append(cmd.Env, e) 1085 } 1086 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 1087 return cmd 1088 } 1089 1090 type mavenMetadata struct { 1091 Version string 1092 Package string 1093 Develop bool 1094 Contributors []mavenContributor 1095 } 1096 1097 type mavenContributor struct { 1098 Name string 1099 Email string 1100 } 1101 1102 func newMavenMetadata(env build.Environment) mavenMetadata { 1103 // Collect the list of authors from the repo root 1104 contribs := []mavenContributor{} 1105 if authors, err := os.Open("AUTHORS"); err == nil { 1106 defer authors.Close() 1107 1108 scanner := bufio.NewScanner(authors) 1109 for scanner.Scan() { 1110 // Skip any whitespace from the authors list 1111 line := strings.TrimSpace(scanner.Text()) 1112 if line == "" || line[0] == '#' { 1113 continue 1114 } 1115 // Split the author and insert as a contributor 1116 re := regexp.MustCompile("([^<]+) <(.+)>") 1117 parts := re.FindStringSubmatch(line) 1118 if len(parts) == 3 { 1119 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 1120 } 1121 } 1122 } 1123 // Render the version and package strings 1124 version := params.Version 1125 if isUnstableBuild(env) { 1126 version += "-SNAPSHOT" 1127 } 1128 return mavenMetadata{ 1129 Version: version, 1130 Package: "geth-" + version, 1131 Develop: isUnstableBuild(env), 1132 Contributors: contribs, 1133 } 1134 } 1135 1136 // XCode frameworks 1137 1138 func doXCodeFramework(cmdline []string) { 1139 var ( 1140 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1141 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 1142 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`) 1143 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 1144 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 1145 ) 1146 flag.CommandLine.Parse(cmdline) 1147 env := build.Env() 1148 tc := new(build.GoToolchain) 1149 1150 // Build gomobile. 1151 build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 1152 1153 // Build the iOS XCode framework 1154 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/sexdefi/go-ethereum/mobile") 1155 1156 if *local { 1157 // If we're building locally, use the build folder and stop afterwards 1158 bind.Dir = GOBIN 1159 build.MustRun(bind) 1160 return 1161 } 1162 1163 // Create the archive. 1164 maybeSkipArchive(env) 1165 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 1166 if err := os.MkdirAll(archive, 0755); err != nil { 1167 log.Fatal(err) 1168 } 1169 bind.Dir, _ = filepath.Abs(archive) 1170 build.MustRun(bind) 1171 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 1172 1173 // Sign and upload the framework to Azure 1174 if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { 1175 log.Fatal(err) 1176 } 1177 // Prepare and upload a PodSpec to CocoaPods 1178 if *deploy != "" { 1179 meta := newPodMetadata(env, archive) 1180 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 1181 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings") 1182 } 1183 } 1184 1185 type podMetadata struct { 1186 Version string 1187 Commit string 1188 Archive string 1189 Contributors []podContributor 1190 } 1191 1192 type podContributor struct { 1193 Name string 1194 Email string 1195 } 1196 1197 func newPodMetadata(env build.Environment, archive string) podMetadata { 1198 // Collect the list of authors from the repo root 1199 contribs := []podContributor{} 1200 if authors, err := os.Open("AUTHORS"); err == nil { 1201 defer authors.Close() 1202 1203 scanner := bufio.NewScanner(authors) 1204 for scanner.Scan() { 1205 // Skip any whitespace from the authors list 1206 line := strings.TrimSpace(scanner.Text()) 1207 if line == "" || line[0] == '#' { 1208 continue 1209 } 1210 // Split the author and insert as a contributor 1211 re := regexp.MustCompile("([^<]+) <(.+)>") 1212 parts := re.FindStringSubmatch(line) 1213 if len(parts) == 3 { 1214 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 1215 } 1216 } 1217 } 1218 version := params.Version 1219 if isUnstableBuild(env) { 1220 version += "-unstable." + env.Buildnum 1221 } 1222 return podMetadata{ 1223 Archive: archive, 1224 Version: version, 1225 Commit: env.Commit, 1226 Contributors: contribs, 1227 } 1228 } 1229 1230 // Binary distribution cleanups 1231 1232 func doPurge(cmdline []string) { 1233 var ( 1234 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1235 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1236 ) 1237 flag.CommandLine.Parse(cmdline) 1238 1239 if env := build.Env(); !env.IsCronJob { 1240 log.Printf("skipping because not a cron job") 1241 os.Exit(0) 1242 } 1243 // Create the azure authentication and list the current archives 1244 auth := build.AzureBlobstoreConfig{ 1245 Account: strings.Split(*store, "/")[0], 1246 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1247 Container: strings.SplitN(*store, "/", 2)[1], 1248 } 1249 blobs, err := build.AzureBlobstoreList(auth) 1250 if err != nil { 1251 log.Fatal(err) 1252 } 1253 fmt.Printf("Found %d blobs\n", len(blobs)) 1254 1255 // Iterate over the blobs, collect and sort all unstable builds 1256 for i := 0; i < len(blobs); i++ { 1257 if !strings.Contains(*blobs[i].Name, "unstable") { 1258 blobs = append(blobs[:i], blobs[i+1:]...) 1259 i-- 1260 } 1261 } 1262 for i := 0; i < len(blobs); i++ { 1263 for j := i + 1; j < len(blobs); j++ { 1264 if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) { 1265 blobs[i], blobs[j] = blobs[j], blobs[i] 1266 } 1267 } 1268 } 1269 // Filter out all archives more recent that the given threshold 1270 for i, blob := range blobs { 1271 if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1272 blobs = blobs[:i] 1273 break 1274 } 1275 } 1276 fmt.Printf("Deleting %d blobs\n", len(blobs)) 1277 // Delete all marked as such and return 1278 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1279 log.Fatal(err) 1280 } 1281 }