github.com/axiomzen/go-ethereum@v1.8.27/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // +build none 18 19 /* 20 The ci command is called from Continuous Integration scripts. 21 22 Usage: go run build/ci.go <command> <command flags/arguments> 23 24 Available commands are: 25 26 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 27 test [ -coverage ] [ packages... ] -- runs the tests 28 lint -- runs certain pre-selected linters 29 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts 30 importkeys -- imports signing keys from env 31 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 32 nsis -- creates a Windows NSIS installer 33 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 34 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 35 xgo [ -alltools ] [ options ] -- cross builds according to options 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "go/parser" 50 "go/token" 51 "io/ioutil" 52 "log" 53 "os" 54 "os/exec" 55 "path/filepath" 56 "regexp" 57 "runtime" 58 "strings" 59 "time" 60 61 "github.com/ethereum/go-ethereum/internal/build" 62 "github.com/ethereum/go-ethereum/params" 63 sv "github.com/ethereum/go-ethereum/swarm/version" 64 ) 65 66 var ( 67 // Files that end up in the geth*.zip archive. 68 gethArchiveFiles = []string{ 69 "COPYING", 70 executablePath("geth"), 71 } 72 73 // Files that end up in the geth-alltools*.zip archive. 74 allToolsArchiveFiles = []string{ 75 "COPYING", 76 executablePath("abigen"), 77 executablePath("bootnode"), 78 executablePath("evm"), 79 executablePath("geth"), 80 executablePath("puppeth"), 81 executablePath("rlpdump"), 82 executablePath("wnode"), 83 } 84 85 // Files that end up in the swarm*.zip archive. 86 swarmArchiveFiles = []string{ 87 "COPYING", 88 executablePath("swarm"), 89 } 90 91 // A debian package is created for all executables listed here. 92 debExecutables = []debExecutable{ 93 { 94 BinaryName: "abigen", 95 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 96 }, 97 { 98 BinaryName: "bootnode", 99 Description: "Ethereum bootnode.", 100 }, 101 { 102 BinaryName: "evm", 103 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 104 }, 105 { 106 BinaryName: "geth", 107 Description: "Ethereum CLI client.", 108 }, 109 { 110 BinaryName: "puppeth", 111 Description: "Ethereum private network manager.", 112 }, 113 { 114 BinaryName: "rlpdump", 115 Description: "Developer utility tool that prints RLP structures.", 116 }, 117 { 118 BinaryName: "wnode", 119 Description: "Ethereum Whisper diagnostic tool", 120 }, 121 } 122 123 // A debian package is created for all executables listed here. 124 debSwarmExecutables = []debExecutable{ 125 { 126 BinaryName: "swarm", 127 PackageName: "ethereum-swarm", 128 Description: "Ethereum Swarm daemon and tools", 129 }, 130 } 131 132 debEthereum = debPackage{ 133 Name: "ethereum", 134 Version: params.Version, 135 Executables: debExecutables, 136 } 137 138 debSwarm = debPackage{ 139 Name: "ethereum-swarm", 140 Version: sv.Version, 141 Executables: debSwarmExecutables, 142 } 143 144 // Debian meta packages to build and push to Ubuntu PPA 145 debPackages = []debPackage{ 146 debSwarm, 147 debEthereum, 148 } 149 150 // Packages to be cross-compiled by the xgo command 151 allCrossCompiledArchiveFiles = append(allToolsArchiveFiles, swarmArchiveFiles...) 152 153 // Distros for which packages are created. 154 // Note: vivid is unsupported because there is no golang-1.6 package for it. 155 // Note: wily is unsupported because it was officially deprecated on lanchpad. 156 // Note: yakkety is unsupported because it was officially deprecated on lanchpad. 157 // Note: zesty is unsupported because it was officially deprecated on lanchpad. 158 // Note: artful is unsupported because it was officially deprecated on lanchpad. 159 debDistros = []string{"trusty", "xenial", "bionic", "cosmic"} 160 ) 161 162 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 163 164 func executablePath(name string) string { 165 if runtime.GOOS == "windows" { 166 name += ".exe" 167 } 168 return filepath.Join(GOBIN, name) 169 } 170 171 func main() { 172 log.SetFlags(log.Lshortfile) 173 174 if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) { 175 log.Fatal("this script must be run from the root of the repository") 176 } 177 if len(os.Args) < 2 { 178 log.Fatal("need subcommand as first argument") 179 } 180 switch os.Args[1] { 181 case "install": 182 doInstall(os.Args[2:]) 183 case "test": 184 doTest(os.Args[2:]) 185 case "lint": 186 doLint(os.Args[2:]) 187 case "archive": 188 doArchive(os.Args[2:]) 189 case "debsrc": 190 doDebianSource(os.Args[2:]) 191 case "nsis": 192 doWindowsInstaller(os.Args[2:]) 193 case "aar": 194 doAndroidArchive(os.Args[2:]) 195 case "xcode": 196 doXCodeFramework(os.Args[2:]) 197 case "xgo": 198 doXgo(os.Args[2:]) 199 case "purge": 200 doPurge(os.Args[2:]) 201 default: 202 log.Fatal("unknown command ", os.Args[1]) 203 } 204 } 205 206 // Compiling 207 208 func doInstall(cmdline []string) { 209 var ( 210 arch = flag.String("arch", "", "Architecture to cross build for") 211 cc = flag.String("cc", "", "C compiler to cross build with") 212 ) 213 flag.CommandLine.Parse(cmdline) 214 env := build.Env() 215 216 // Check Go version. People regularly open issues about compilation 217 // failure with outdated Go. This should save them the trouble. 218 if !strings.Contains(runtime.Version(), "devel") { 219 // Figure out the minor version number since we can't textually compare (1.10 < 1.9) 220 var minor int 221 fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) 222 223 if minor < 9 { 224 log.Println("You have Go version", runtime.Version()) 225 log.Println("go-ethereum requires at least Go version 1.9 and cannot") 226 log.Println("be compiled with an earlier version. Please upgrade your Go installation.") 227 os.Exit(1) 228 } 229 } 230 // Compile packages given as arguments, or everything if there are no arguments. 231 packages := []string{"./..."} 232 if flag.NArg() > 0 { 233 packages = flag.Args() 234 } 235 packages = build.ExpandPackagesNoVendor(packages) 236 237 if *arch == "" || *arch == runtime.GOARCH { 238 goinstall := goTool("install", buildFlags(env)...) 239 goinstall.Args = append(goinstall.Args, "-v") 240 goinstall.Args = append(goinstall.Args, packages...) 241 build.MustRun(goinstall) 242 return 243 } 244 // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds 245 if *arch == "arm" { 246 os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) 247 for _, path := range filepath.SplitList(build.GOPATH()) { 248 os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) 249 } 250 } 251 // Seems we are cross compiling, work around forbidden GOBIN 252 goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) 253 goinstall.Args = append(goinstall.Args, "-v") 254 goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...) 255 goinstall.Args = append(goinstall.Args, packages...) 256 build.MustRun(goinstall) 257 258 if cmds, err := ioutil.ReadDir("cmd"); err == nil { 259 for _, cmd := range cmds { 260 pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly) 261 if err != nil { 262 log.Fatal(err) 263 } 264 for name := range pkgs { 265 if name == "main" { 266 gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...) 267 gobuild.Args = append(gobuild.Args, "-v") 268 gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...) 269 gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name())) 270 build.MustRun(gobuild) 271 break 272 } 273 } 274 } 275 } 276 } 277 278 func buildFlags(env build.Environment) (flags []string) { 279 var ld []string 280 if env.Commit != "" { 281 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 282 } 283 if runtime.GOOS == "darwin" { 284 ld = append(ld, "-s") 285 } 286 287 if len(ld) > 0 { 288 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 289 } 290 return flags 291 } 292 293 func goTool(subcmd string, args ...string) *exec.Cmd { 294 return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...) 295 } 296 297 func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { 298 cmd := build.GoTool(subcmd, args...) 299 cmd.Env = []string{"GOPATH=" + build.GOPATH()} 300 if arch == "" || arch == runtime.GOARCH { 301 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 302 } else { 303 cmd.Env = append(cmd.Env, "CGO_ENABLED=1") 304 cmd.Env = append(cmd.Env, "GOARCH="+arch) 305 } 306 if cc != "" { 307 cmd.Env = append(cmd.Env, "CC="+cc) 308 } 309 for _, e := range os.Environ() { 310 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 311 continue 312 } 313 cmd.Env = append(cmd.Env, e) 314 } 315 return cmd 316 } 317 318 // Running The Tests 319 // 320 // "tests" also includes static analysis tools such as vet. 321 322 func doTest(cmdline []string) { 323 coverage := flag.Bool("coverage", false, "Whether to record code coverage") 324 flag.CommandLine.Parse(cmdline) 325 env := build.Env() 326 327 packages := []string{"./..."} 328 if len(flag.CommandLine.Args()) > 0 { 329 packages = flag.CommandLine.Args() 330 } 331 packages = build.ExpandPackagesNoVendor(packages) 332 333 // Run the actual tests. 334 // Test a single package at a time. CI builders are slow 335 // and some tests run into timeouts under load. 336 gotest := goTool("test", buildFlags(env)...) 337 gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m") 338 if *coverage { 339 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 340 } 341 342 gotest.Args = append(gotest.Args, packages...) 343 build.MustRun(gotest) 344 } 345 346 // runs gometalinter on requested packages 347 func doLint(cmdline []string) { 348 flag.CommandLine.Parse(cmdline) 349 350 packages := []string{"./..."} 351 if len(flag.CommandLine.Args()) > 0 { 352 packages = flag.CommandLine.Args() 353 } 354 // Get metalinter and install all supported linters 355 build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) 356 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") 357 358 // Run fast linters batched together 359 configs := []string{ 360 "--vendor", 361 "--tests", 362 "--deadline=2m", 363 "--disable-all", 364 "--enable=goimports", 365 "--enable=varcheck", 366 "--enable=vet", 367 "--enable=gofmt", 368 "--enable=misspell", 369 "--enable=goconst", 370 "--min-occurrences=6", // for goconst 371 } 372 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 373 374 // Run slow linters one by one 375 for _, linter := range []string{"unconvert", "gosimple"} { 376 configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} 377 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 378 } 379 } 380 381 // Release Packaging 382 func doArchive(cmdline []string) { 383 var ( 384 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 385 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 386 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 387 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 388 ext string 389 ) 390 flag.CommandLine.Parse(cmdline) 391 switch *atype { 392 case "zip": 393 ext = ".zip" 394 case "tar": 395 ext = ".tar.gz" 396 default: 397 log.Fatal("unknown archive type: ", atype) 398 } 399 400 var ( 401 env = build.Env() 402 403 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 404 geth = "geth-" + basegeth + ext 405 alltools = "geth-alltools-" + basegeth + ext 406 407 baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit)) 408 swarm = "swarm-" + baseswarm + ext 409 ) 410 maybeSkipArchive(env) 411 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 412 log.Fatal(err) 413 } 414 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 415 log.Fatal(err) 416 } 417 if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil { 418 log.Fatal(err) 419 } 420 for _, archive := range []string{geth, alltools, swarm} { 421 if err := archiveUpload(archive, *upload, *signer); err != nil { 422 log.Fatal(err) 423 } 424 } 425 } 426 427 func archiveBasename(arch string, archiveVersion string) string { 428 platform := runtime.GOOS + "-" + arch 429 if arch == "arm" { 430 platform += os.Getenv("GOARM") 431 } 432 if arch == "android" { 433 platform = "android-all" 434 } 435 if arch == "ios" { 436 platform = "ios-all" 437 } 438 return platform + "-" + archiveVersion 439 } 440 441 func archiveUpload(archive string, blobstore string, signer string) error { 442 // If signing was requested, generate the signature files 443 if signer != "" { 444 key := getenvBase64(signer) 445 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 446 return err 447 } 448 } 449 // If uploading to Azure was requested, push the archive possibly with its signature 450 if blobstore != "" { 451 auth := build.AzureBlobstoreConfig{ 452 Account: strings.Split(blobstore, "/")[0], 453 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 454 Container: strings.SplitN(blobstore, "/", 2)[1], 455 } 456 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 457 return err 458 } 459 if signer != "" { 460 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 461 return err 462 } 463 } 464 } 465 return nil 466 } 467 468 // skips archiving for some build configurations. 469 func maybeSkipArchive(env build.Environment) { 470 if env.IsPullRequest { 471 log.Printf("skipping because this is a PR build") 472 os.Exit(0) 473 } 474 if env.IsCronJob { 475 log.Printf("skipping because this is a cron job") 476 os.Exit(0) 477 } 478 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 479 log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) 480 os.Exit(0) 481 } 482 } 483 484 // Debian Packaging 485 func doDebianSource(cmdline []string) { 486 var ( 487 signer = flag.String("signer", "", `Signing key name, also used as package author`) 488 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 489 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 490 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 491 now = time.Now() 492 ) 493 flag.CommandLine.Parse(cmdline) 494 *workdir = makeWorkdir(*workdir) 495 env := build.Env() 496 maybeSkipArchive(env) 497 498 // Import the signing key. 499 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 500 gpg := exec.Command("gpg", "--import") 501 gpg.Stdin = bytes.NewReader(key) 502 build.MustRun(gpg) 503 } 504 505 // Create Debian packages and upload them 506 for _, pkg := range debPackages { 507 for _, distro := range debDistros { 508 meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 509 pkgdir := stageDebianSource(*workdir, meta) 510 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") 511 debuild.Dir = pkgdir 512 build.MustRun(debuild) 513 514 var ( 515 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 516 source = filepath.Join(*workdir, basename+".tar.xz") 517 dsc = filepath.Join(*workdir, basename+".dsc") 518 changes = filepath.Join(*workdir, basename+"_source.changes") 519 ) 520 if *signer != "" { 521 build.MustRunCommand("debsign", changes) 522 } 523 if *upload != "" { 524 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) 525 } 526 } 527 } 528 } 529 530 func ppaUpload(workdir, ppa, sshUser string, files []string) { 531 p := strings.Split(ppa, "/") 532 if len(p) != 2 { 533 log.Fatal("-upload PPA name must contain single /") 534 } 535 if sshUser == "" { 536 sshUser = p[0] 537 } 538 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 539 // Create the SSH identity file if it doesn't exist. 540 var idfile string 541 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 542 idfile = filepath.Join(workdir, "sshkey") 543 if _, err := os.Stat(idfile); os.IsNotExist(err) { 544 ioutil.WriteFile(idfile, sshkey, 0600) 545 } 546 } 547 // Upload 548 dest := sshUser + "@ppa.launchpad.net" 549 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 550 log.Fatal(err) 551 } 552 } 553 554 func getenvBase64(variable string) []byte { 555 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 556 if err != nil { 557 log.Fatal("invalid base64 " + variable) 558 } 559 return []byte(dec) 560 } 561 562 func makeWorkdir(wdflag string) string { 563 var err error 564 if wdflag != "" { 565 err = os.MkdirAll(wdflag, 0744) 566 } else { 567 wdflag, err = ioutil.TempDir("", "geth-build-") 568 } 569 if err != nil { 570 log.Fatal(err) 571 } 572 return wdflag 573 } 574 575 func isUnstableBuild(env build.Environment) bool { 576 if env.Tag != "" { 577 return false 578 } 579 return true 580 } 581 582 type debPackage struct { 583 Name string // the name of the Debian package to produce, e.g. "ethereum", or "ethereum-swarm" 584 Version string // the clean version of the debPackage, e.g. 1.8.12 or 0.3.0, without any metadata 585 Executables []debExecutable // executables to be included in the package 586 } 587 588 type debMetadata struct { 589 Env build.Environment 590 591 PackageName string 592 593 // go-ethereum version being built. Note that this 594 // is not the debian package version. The package version 595 // is constructed by VersionString. 596 Version string 597 598 Author string // "name <email>", also selects signing key 599 Distro, Time string 600 Executables []debExecutable 601 } 602 603 type debExecutable struct { 604 PackageName string 605 BinaryName string 606 Description string 607 } 608 609 // Package returns the name of the package if present, or 610 // fallbacks to BinaryName 611 func (d debExecutable) Package() string { 612 if d.PackageName != "" { 613 return d.PackageName 614 } 615 return d.BinaryName 616 } 617 618 func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 619 if author == "" { 620 // No signing key, use default author. 621 author = "Ethereum Builds <fjl@ethereum.org>" 622 } 623 return debMetadata{ 624 PackageName: name, 625 Env: env, 626 Author: author, 627 Distro: distro, 628 Version: version, 629 Time: t.Format(time.RFC1123Z), 630 Executables: exes, 631 } 632 } 633 634 // Name returns the name of the metapackage that depends 635 // on all executable packages. 636 func (meta debMetadata) Name() string { 637 if isUnstableBuild(meta.Env) { 638 return meta.PackageName + "-unstable" 639 } 640 return meta.PackageName 641 } 642 643 // VersionString returns the debian version of the packages. 644 func (meta debMetadata) VersionString() string { 645 vsn := meta.Version 646 if meta.Env.Buildnum != "" { 647 vsn += "+build" + meta.Env.Buildnum 648 } 649 if meta.Distro != "" { 650 vsn += "+" + meta.Distro 651 } 652 return vsn 653 } 654 655 // ExeList returns the list of all executable packages. 656 func (meta debMetadata) ExeList() string { 657 names := make([]string, len(meta.Executables)) 658 for i, e := range meta.Executables { 659 names[i] = meta.ExeName(e) 660 } 661 return strings.Join(names, ", ") 662 } 663 664 // ExeName returns the package name of an executable package. 665 func (meta debMetadata) ExeName(exe debExecutable) string { 666 if isUnstableBuild(meta.Env) { 667 return exe.Package() + "-unstable" 668 } 669 return exe.Package() 670 } 671 672 // ExeConflicts returns the content of the Conflicts field 673 // for executable packages. 674 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 675 if isUnstableBuild(meta.Env) { 676 // Set up the conflicts list so that the *-unstable packages 677 // cannot be installed alongside the regular version. 678 // 679 // https://www.debian.org/doc/debian-policy/ch-relationships.html 680 // is very explicit about Conflicts: and says that Breaks: should 681 // be preferred and the conflicting files should be handled via 682 // alternates. We might do this eventually but using a conflict is 683 // easier now. 684 return "ethereum, " + exe.Package() 685 } 686 return "" 687 } 688 689 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 690 pkg := meta.Name() + "-" + meta.VersionString() 691 pkgdir = filepath.Join(tmpdir, pkg) 692 if err := os.Mkdir(pkgdir, 0755); err != nil { 693 log.Fatal(err) 694 } 695 696 // Copy the source code. 697 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 698 699 // Put the debian build files in place. 700 debian := filepath.Join(pkgdir, "debian") 701 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 702 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 703 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 704 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 705 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 706 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 707 for _, exe := range meta.Executables { 708 install := filepath.Join(debian, meta.ExeName(exe)+".install") 709 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 710 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 711 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 712 } 713 714 return pkgdir 715 } 716 717 // Windows installer 718 func doWindowsInstaller(cmdline []string) { 719 // Parse the flags and make skip installer generation on PRs 720 var ( 721 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 722 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 723 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 724 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 725 ) 726 flag.CommandLine.Parse(cmdline) 727 *workdir = makeWorkdir(*workdir) 728 env := build.Env() 729 maybeSkipArchive(env) 730 731 // Aggregate binaries that are included in the installer 732 var ( 733 devTools []string 734 allTools []string 735 gethTool string 736 ) 737 for _, file := range allToolsArchiveFiles { 738 if file == "COPYING" { // license, copied later 739 continue 740 } 741 allTools = append(allTools, filepath.Base(file)) 742 if filepath.Base(file) == "geth.exe" { 743 gethTool = file 744 } else { 745 devTools = append(devTools, file) 746 } 747 } 748 749 // Render NSIS scripts: Installer NSIS contains two installer sections, 750 // first section contains the geth binary, second section holds the dev tools. 751 templateData := map[string]interface{}{ 752 "License": "COPYING", 753 "Geth": gethTool, 754 "DevTools": devTools, 755 } 756 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 757 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 758 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 759 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 760 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 761 build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755) 762 build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755) 763 764 // Build the installer. This assumes that all the needed files have been previously 765 // built (don't mix building and packaging to keep cross compilation complexity to a 766 // minimum). 767 version := strings.Split(params.Version, ".") 768 if env.Commit != "" { 769 version[2] += "-" + env.Commit[:8] 770 } 771 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 772 build.MustRunCommand("makensis.exe", 773 "/DOUTPUTFILE="+installer, 774 "/DMAJORVERSION="+version[0], 775 "/DMINORVERSION="+version[1], 776 "/DBUILDVERSION="+version[2], 777 "/DARCH="+*arch, 778 filepath.Join(*workdir, "geth.nsi"), 779 ) 780 781 // Sign and publish installer. 782 if err := archiveUpload(installer, *upload, *signer); err != nil { 783 log.Fatal(err) 784 } 785 } 786 787 // Android archives 788 789 func doAndroidArchive(cmdline []string) { 790 var ( 791 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 792 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 793 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 794 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 795 ) 796 flag.CommandLine.Parse(cmdline) 797 env := build.Env() 798 799 // Sanity check that the SDK and NDK are installed and set 800 if os.Getenv("ANDROID_HOME") == "" { 801 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 802 } 803 if os.Getenv("ANDROID_NDK") == "" { 804 log.Fatal("Please ensure ANDROID_NDK points to your Android NDK") 805 } 806 // Build the Android archive and Maven resources 807 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 808 build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) 809 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) 810 811 if *local { 812 // If we're building locally, copy bundle to build dir and skip Maven 813 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 814 return 815 } 816 meta := newMavenMetadata(env) 817 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 818 819 // Skip Maven deploy and Azure upload for PR builds 820 maybeSkipArchive(env) 821 822 // Sign and upload the archive to Azure 823 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 824 os.Rename("geth.aar", archive) 825 826 if err := archiveUpload(archive, *upload, *signer); err != nil { 827 log.Fatal(err) 828 } 829 // Sign and upload all the artifacts to Maven Central 830 os.Rename(archive, meta.Package+".aar") 831 if *signer != "" && *deploy != "" { 832 // Import the signing key into the local GPG instance 833 key := getenvBase64(*signer) 834 gpg := exec.Command("gpg", "--import") 835 gpg.Stdin = bytes.NewReader(key) 836 build.MustRun(gpg) 837 keyID, err := build.PGPKeyID(string(key)) 838 if err != nil { 839 log.Fatal(err) 840 } 841 // Upload the artifacts to Sonatype and/or Maven Central 842 repo := *deploy + "/service/local/staging/deploy/maven2" 843 if meta.Develop { 844 repo = *deploy + "/content/repositories/snapshots" 845 } 846 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 847 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 848 "-Dgpg.keyname="+keyID, 849 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 850 } 851 } 852 853 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 854 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 855 cmd.Args = append(cmd.Args, args...) 856 cmd.Env = []string{ 857 "GOPATH=" + build.GOPATH(), 858 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 859 } 860 for _, e := range os.Environ() { 861 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") { 862 continue 863 } 864 cmd.Env = append(cmd.Env, e) 865 } 866 return cmd 867 } 868 869 type mavenMetadata struct { 870 Version string 871 Package string 872 Develop bool 873 Contributors []mavenContributor 874 } 875 876 type mavenContributor struct { 877 Name string 878 Email string 879 } 880 881 func newMavenMetadata(env build.Environment) mavenMetadata { 882 // Collect the list of authors from the repo root 883 contribs := []mavenContributor{} 884 if authors, err := os.Open("AUTHORS"); err == nil { 885 defer authors.Close() 886 887 scanner := bufio.NewScanner(authors) 888 for scanner.Scan() { 889 // Skip any whitespace from the authors list 890 line := strings.TrimSpace(scanner.Text()) 891 if line == "" || line[0] == '#' { 892 continue 893 } 894 // Split the author and insert as a contributor 895 re := regexp.MustCompile("([^<]+) <(.+)>") 896 parts := re.FindStringSubmatch(line) 897 if len(parts) == 3 { 898 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 899 } 900 } 901 } 902 // Render the version and package strings 903 version := params.Version 904 if isUnstableBuild(env) { 905 version += "-SNAPSHOT" 906 } 907 return mavenMetadata{ 908 Version: version, 909 Package: "geth-" + version, 910 Develop: isUnstableBuild(env), 911 Contributors: contribs, 912 } 913 } 914 915 // XCode frameworks 916 917 func doXCodeFramework(cmdline []string) { 918 var ( 919 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 920 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 921 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 922 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 923 ) 924 flag.CommandLine.Parse(cmdline) 925 env := build.Env() 926 927 // Build the iOS XCode framework 928 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 929 build.MustRun(gomobileTool("init")) 930 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") 931 932 if *local { 933 // If we're building locally, use the build folder and stop afterwards 934 bind.Dir, _ = filepath.Abs(GOBIN) 935 build.MustRun(bind) 936 return 937 } 938 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 939 if err := os.Mkdir(archive, os.ModePerm); err != nil { 940 log.Fatal(err) 941 } 942 bind.Dir, _ = filepath.Abs(archive) 943 build.MustRun(bind) 944 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 945 946 // Skip CocoaPods deploy and Azure upload for PR builds 947 maybeSkipArchive(env) 948 949 // Sign and upload the framework to Azure 950 if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil { 951 log.Fatal(err) 952 } 953 // Prepare and upload a PodSpec to CocoaPods 954 if *deploy != "" { 955 meta := newPodMetadata(env, archive) 956 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 957 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose") 958 } 959 } 960 961 type podMetadata struct { 962 Version string 963 Commit string 964 Archive string 965 Contributors []podContributor 966 } 967 968 type podContributor struct { 969 Name string 970 Email string 971 } 972 973 func newPodMetadata(env build.Environment, archive string) podMetadata { 974 // Collect the list of authors from the repo root 975 contribs := []podContributor{} 976 if authors, err := os.Open("AUTHORS"); err == nil { 977 defer authors.Close() 978 979 scanner := bufio.NewScanner(authors) 980 for scanner.Scan() { 981 // Skip any whitespace from the authors list 982 line := strings.TrimSpace(scanner.Text()) 983 if line == "" || line[0] == '#' { 984 continue 985 } 986 // Split the author and insert as a contributor 987 re := regexp.MustCompile("([^<]+) <(.+)>") 988 parts := re.FindStringSubmatch(line) 989 if len(parts) == 3 { 990 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 991 } 992 } 993 } 994 version := params.Version 995 if isUnstableBuild(env) { 996 version += "-unstable." + env.Buildnum 997 } 998 return podMetadata{ 999 Archive: archive, 1000 Version: version, 1001 Commit: env.Commit, 1002 Contributors: contribs, 1003 } 1004 } 1005 1006 // Cross compilation 1007 1008 func doXgo(cmdline []string) { 1009 var ( 1010 alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) 1011 ) 1012 flag.CommandLine.Parse(cmdline) 1013 env := build.Env() 1014 1015 // Make sure xgo is available for cross compilation 1016 gogetxgo := goTool("get", "github.com/karalabe/xgo") 1017 build.MustRun(gogetxgo) 1018 1019 // If all tools building is requested, build everything the builder wants 1020 args := append(buildFlags(env), flag.Args()...) 1021 1022 if *alltools { 1023 args = append(args, []string{"--dest", GOBIN}...) 1024 for _, res := range allCrossCompiledArchiveFiles { 1025 if strings.HasPrefix(res, GOBIN) { 1026 // Binary tool found, cross build it explicitly 1027 args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) 1028 xgo := xgoTool(args) 1029 build.MustRun(xgo) 1030 args = args[:len(args)-1] 1031 } 1032 } 1033 return 1034 } 1035 // Otherwise xxecute the explicit cross compilation 1036 path := args[len(args)-1] 1037 args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) 1038 1039 xgo := xgoTool(args) 1040 build.MustRun(xgo) 1041 } 1042 1043 func xgoTool(args []string) *exec.Cmd { 1044 cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) 1045 cmd.Env = []string{ 1046 "GOPATH=" + build.GOPATH(), 1047 "GOBIN=" + GOBIN, 1048 } 1049 for _, e := range os.Environ() { 1050 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 1051 continue 1052 } 1053 cmd.Env = append(cmd.Env, e) 1054 } 1055 return cmd 1056 } 1057 1058 // Binary distribution cleanups 1059 1060 func doPurge(cmdline []string) { 1061 var ( 1062 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1063 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1064 ) 1065 flag.CommandLine.Parse(cmdline) 1066 1067 if env := build.Env(); !env.IsCronJob { 1068 log.Printf("skipping because not a cron job") 1069 os.Exit(0) 1070 } 1071 // Create the azure authentication and list the current archives 1072 auth := build.AzureBlobstoreConfig{ 1073 Account: strings.Split(*store, "/")[0], 1074 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1075 Container: strings.SplitN(*store, "/", 2)[1], 1076 } 1077 blobs, err := build.AzureBlobstoreList(auth) 1078 if err != nil { 1079 log.Fatal(err) 1080 } 1081 // Iterate over the blobs, collect and sort all unstable builds 1082 for i := 0; i < len(blobs); i++ { 1083 if !strings.Contains(blobs[i].Name, "unstable") { 1084 blobs = append(blobs[:i], blobs[i+1:]...) 1085 i-- 1086 } 1087 } 1088 for i := 0; i < len(blobs); i++ { 1089 for j := i + 1; j < len(blobs); j++ { 1090 if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { 1091 blobs[i], blobs[j] = blobs[j], blobs[i] 1092 } 1093 } 1094 } 1095 // Filter out all archives more recent that the given threshold 1096 for i, blob := range blobs { 1097 if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1098 blobs = blobs[:i] 1099 break 1100 } 1101 } 1102 // Delete all marked as such and return 1103 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1104 log.Fatal(err) 1105 } 1106 }