github.com/nova-foundation/go-ethereum@v1.0.1/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 //go:build none 18 // +build none 19 20 /* 21 The ci command is called from Continuous Integration scripts. 22 23 Usage: go run build/ci.go <command> <command flags/arguments> 24 25 Available commands are: 26 27 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 28 test [ -coverage ] [ packages... ] -- runs the tests 29 lint -- runs certain pre-selected linters 30 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts 31 importkeys -- imports signing keys from env 32 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 33 nsis -- creates a Windows NSIS installer 34 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 35 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 36 xgo [ -alltools ] [ options ] -- cross builds according to options 37 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 38 39 For all commands, -n prevents execution of external programs (dry run mode). 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "go/parser" 50 "go/token" 51 "io/ioutil" 52 "log" 53 "os" 54 "os/exec" 55 "path/filepath" 56 "regexp" 57 "runtime" 58 "strings" 59 "time" 60 61 "github.com/Nova-foundation/go-ethereum/internal/build" 62 "github.com/Nova-foundation/go-ethereum/params" 63 ) 64 65 var ( 66 // Files that end up in the geth*.zip archive. 67 gethArchiveFiles = []string{ 68 "COPYING", 69 executablePath("geth"), 70 } 71 72 // Files that end up in the geth-alltools*.zip archive. 73 allToolsArchiveFiles = []string{ 74 "COPYING", 75 executablePath("abigen"), 76 executablePath("bootnode"), 77 executablePath("evm"), 78 executablePath("geth"), 79 executablePath("puppeth"), 80 executablePath("rlpdump"), 81 executablePath("wnode"), 82 executablePath("clef"), 83 } 84 85 // A debian package is created for all executables listed here. 86 debExecutables = []debExecutable{ 87 { 88 BinaryName: "abigen", 89 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 90 }, 91 { 92 BinaryName: "bootnode", 93 Description: "Ethereum bootnode.", 94 }, 95 { 96 BinaryName: "evm", 97 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 98 }, 99 { 100 BinaryName: "geth", 101 Description: "Ethereum CLI client.", 102 }, 103 { 104 BinaryName: "puppeth", 105 Description: "Ethereum private network manager.", 106 }, 107 { 108 BinaryName: "rlpdump", 109 Description: "Developer utility tool that prints RLP structures.", 110 }, 111 { 112 BinaryName: "wnode", 113 Description: "Ethereum Whisper diagnostic tool", 114 }, 115 { 116 BinaryName: "clef", 117 Description: "Ethereum account management tool.", 118 }, 119 } 120 121 // A debian package is created for all executables listed here. 122 123 debEthereum = debPackage{ 124 Name: "ethereum", 125 Version: params.Version, 126 Executables: debExecutables, 127 } 128 129 // Debian meta packages to build and push to Ubuntu PPA 130 debPackages = []debPackage{ 131 debEthereum, 132 } 133 134 // Distros for which packages are created. 135 // Note: vivid is unsupported because there is no golang-1.6 package for it. 136 // Note: wily is unsupported because it was officially deprecated on Launchpad. 137 // Note: yakkety is unsupported because it was officially deprecated on Launchpad. 138 // Note: zesty is unsupported because it was officially deprecated on Launchpad. 139 // Note: artful is unsupported because it was officially deprecated on Launchpad. 140 // Note: cosmic is unsupported because it was officially deprecated on Launchpad. 141 debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"} 142 ) 143 144 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 145 146 func executablePath(name string) string { 147 if runtime.GOOS == "windows" { 148 name += ".exe" 149 } 150 return filepath.Join(GOBIN, name) 151 } 152 153 func main() { 154 log.SetFlags(log.Lshortfile) 155 156 if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) { 157 log.Fatal("this script must be run from the root of the repository") 158 } 159 if len(os.Args) < 2 { 160 log.Fatal("need subcommand as first argument") 161 } 162 switch os.Args[1] { 163 case "install": 164 doInstall(os.Args[2:]) 165 case "test": 166 doTest(os.Args[2:]) 167 case "lint": 168 doLint(os.Args[2:]) 169 case "archive": 170 doArchive(os.Args[2:]) 171 case "debsrc": 172 doDebianSource(os.Args[2:]) 173 case "nsis": 174 doWindowsInstaller(os.Args[2:]) 175 case "aar": 176 doAndroidArchive(os.Args[2:]) 177 case "xcode": 178 doXCodeFramework(os.Args[2:]) 179 case "xgo": 180 doXgo(os.Args[2:]) 181 case "purge": 182 doPurge(os.Args[2:]) 183 default: 184 log.Fatal("unknown command ", os.Args[1]) 185 } 186 } 187 188 // Compiling 189 190 func doInstall(cmdline []string) { 191 var ( 192 arch = flag.String("arch", "", "Architecture to cross build for") 193 cc = flag.String("cc", "", "C compiler to cross build with") 194 ) 195 flag.CommandLine.Parse(cmdline) 196 env := build.Env() 197 198 // Check Go version. People regularly open issues about compilation 199 // failure with outdated Go. This should save them the trouble. 200 if !strings.Contains(runtime.Version(), "devel") { 201 // Figure out the minor version number since we can't textually compare (1.10 < 1.9) 202 var minor int 203 fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) 204 205 if minor < 9 { 206 log.Println("You have Go version", runtime.Version()) 207 log.Println("go-ethereum requires at least Go version 1.9 and cannot") 208 log.Println("be compiled with an earlier version. Please upgrade your Go installation.") 209 os.Exit(1) 210 } 211 } 212 // Compile packages given as arguments, or everything if there are no arguments. 213 packages := []string{"./..."} 214 if flag.NArg() > 0 { 215 packages = flag.Args() 216 } 217 218 if *arch == "" || *arch == runtime.GOARCH { 219 goinstall := goTool("install", buildFlags(env)...) 220 goinstall.Args = append(goinstall.Args, "-v") 221 goinstall.Args = append(goinstall.Args, packages...) 222 build.MustRun(goinstall) 223 return 224 } 225 // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds 226 if *arch == "arm" { 227 os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) 228 for _, path := range filepath.SplitList(build.GOPATH()) { 229 os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) 230 } 231 } 232 // Seems we are cross compiling, work around forbidden GOBIN 233 goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) 234 goinstall.Args = append(goinstall.Args, "-v") 235 goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...) 236 goinstall.Args = append(goinstall.Args, packages...) 237 build.MustRun(goinstall) 238 239 if cmds, err := ioutil.ReadDir("cmd"); err == nil { 240 for _, cmd := range cmds { 241 pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly) 242 if err != nil { 243 log.Fatal(err) 244 } 245 for name := range pkgs { 246 if name == "main" { 247 gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...) 248 gobuild.Args = append(gobuild.Args, "-v") 249 gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...) 250 gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name())) 251 build.MustRun(gobuild) 252 break 253 } 254 } 255 } 256 } 257 } 258 259 func buildFlags(env build.Environment) (flags []string) { 260 var ld []string 261 if env.Commit != "" { 262 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 263 ld = append(ld, "-X", "main.gitDate="+env.Date) 264 } 265 if runtime.GOOS == "darwin" { 266 ld = append(ld, "-s") 267 } 268 269 if len(ld) > 0 { 270 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 271 } 272 return flags 273 } 274 275 func goTool(subcmd string, args ...string) *exec.Cmd { 276 return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...) 277 } 278 279 func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { 280 cmd := build.GoTool(subcmd, args...) 281 cmd.Env = []string{"GOPATH=" + build.GOPATH()} 282 if arch == "" || arch == runtime.GOARCH { 283 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 284 } else { 285 cmd.Env = append(cmd.Env, "CGO_ENABLED=1") 286 cmd.Env = append(cmd.Env, "GOARCH="+arch) 287 } 288 if cc != "" { 289 cmd.Env = append(cmd.Env, "CC="+cc) 290 } 291 for _, e := range os.Environ() { 292 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 293 continue 294 } 295 cmd.Env = append(cmd.Env, e) 296 } 297 return cmd 298 } 299 300 // Running The Tests 301 // 302 // "tests" also includes static analysis tools such as vet. 303 304 func doTest(cmdline []string) { 305 coverage := flag.Bool("coverage", false, "Whether to record code coverage") 306 flag.CommandLine.Parse(cmdline) 307 env := build.Env() 308 309 packages := []string{"./..."} 310 if len(flag.CommandLine.Args()) > 0 { 311 packages = flag.CommandLine.Args() 312 } 313 314 // Run the actual tests. 315 // Test a single package at a time. CI builders are slow 316 // and some tests run into timeouts under load. 317 gotest := goTool("test", buildFlags(env)...) 318 gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m", "--short") 319 if *coverage { 320 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 321 } 322 323 gotest.Args = append(gotest.Args, packages...) 324 build.MustRun(gotest) 325 } 326 327 // runs gometalinter on requested packages 328 func doLint(cmdline []string) { 329 flag.CommandLine.Parse(cmdline) 330 331 packages := []string{"./..."} 332 if len(flag.CommandLine.Args()) > 0 { 333 packages = flag.CommandLine.Args() 334 } 335 // Get metalinter and install all supported linters 336 build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) 337 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") 338 339 // Run fast linters batched together 340 configs := []string{ 341 "--vendor", 342 "--tests", 343 "--deadline=2m", 344 "--disable-all", 345 "--enable=goimports", 346 "--enable=varcheck", 347 "--enable=vet", 348 "--enable=gofmt", 349 "--enable=misspell", 350 "--enable=goconst", 351 "--min-occurrences=6", // for goconst 352 } 353 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 354 355 // Run slow linters one by one 356 for _, linter := range []string{"unconvert", "gosimple"} { 357 configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} 358 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 359 } 360 } 361 362 // Release Packaging 363 func doArchive(cmdline []string) { 364 var ( 365 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 366 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 367 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 368 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 369 ext string 370 ) 371 flag.CommandLine.Parse(cmdline) 372 switch *atype { 373 case "zip": 374 ext = ".zip" 375 case "tar": 376 ext = ".tar.gz" 377 default: 378 log.Fatal("unknown archive type: ", atype) 379 } 380 381 var ( 382 env = build.Env() 383 384 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 385 geth = "geth-" + basegeth + ext 386 alltools = "geth-alltools-" + basegeth + ext 387 ) 388 maybeSkipArchive(env) 389 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 390 log.Fatal(err) 391 } 392 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 393 log.Fatal(err) 394 } 395 for _, archive := range []string{geth, alltools} { 396 if err := archiveUpload(archive, *upload, *signer); err != nil { 397 log.Fatal(err) 398 } 399 } 400 } 401 402 func archiveBasename(arch string, archiveVersion string) string { 403 platform := runtime.GOOS + "-" + arch 404 if arch == "arm" { 405 platform += os.Getenv("GOARM") 406 } 407 if arch == "android" { 408 platform = "android-all" 409 } 410 if arch == "ios" { 411 platform = "ios-all" 412 } 413 return platform + "-" + archiveVersion 414 } 415 416 func archiveUpload(archive string, blobstore string, signer string) error { 417 // If signing was requested, generate the signature files 418 if signer != "" { 419 key := getenvBase64(signer) 420 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 421 return err 422 } 423 } 424 // If uploading to Azure was requested, push the archive possibly with its signature 425 if blobstore != "" { 426 auth := build.AzureBlobstoreConfig{ 427 Account: strings.Split(blobstore, "/")[0], 428 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 429 Container: strings.SplitN(blobstore, "/", 2)[1], 430 } 431 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 432 return err 433 } 434 if signer != "" { 435 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 436 return err 437 } 438 } 439 } 440 return nil 441 } 442 443 // skips archiving for some build configurations. 444 func maybeSkipArchive(env build.Environment) { 445 if env.IsPullRequest { 446 log.Printf("skipping because this is a PR build") 447 os.Exit(0) 448 } 449 if env.IsCronJob { 450 log.Printf("skipping because this is a cron job") 451 os.Exit(0) 452 } 453 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 454 log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) 455 os.Exit(0) 456 } 457 } 458 459 // Debian Packaging 460 func doDebianSource(cmdline []string) { 461 var ( 462 signer = flag.String("signer", "", `Signing key name, also used as package author`) 463 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 464 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 465 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 466 now = time.Now() 467 ) 468 flag.CommandLine.Parse(cmdline) 469 *workdir = makeWorkdir(*workdir) 470 env := build.Env() 471 maybeSkipArchive(env) 472 473 // Import the signing key. 474 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 475 gpg := exec.Command("gpg", "--import") 476 gpg.Stdin = bytes.NewReader(key) 477 build.MustRun(gpg) 478 } 479 480 // Create Debian packages and upload them 481 for _, pkg := range debPackages { 482 for _, distro := range debDistros { 483 meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 484 pkgdir := stageDebianSource(*workdir, meta) 485 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") 486 debuild.Dir = pkgdir 487 build.MustRun(debuild) 488 489 var ( 490 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 491 source = filepath.Join(*workdir, basename+".tar.xz") 492 dsc = filepath.Join(*workdir, basename+".dsc") 493 changes = filepath.Join(*workdir, basename+"_source.changes") 494 ) 495 if *signer != "" { 496 build.MustRunCommand("debsign", changes) 497 } 498 if *upload != "" { 499 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) 500 } 501 } 502 } 503 } 504 505 func ppaUpload(workdir, ppa, sshUser string, files []string) { 506 p := strings.Split(ppa, "/") 507 if len(p) != 2 { 508 log.Fatal("-upload PPA name must contain single /") 509 } 510 if sshUser == "" { 511 sshUser = p[0] 512 } 513 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 514 // Create the SSH identity file if it doesn't exist. 515 var idfile string 516 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 517 idfile = filepath.Join(workdir, "sshkey") 518 if _, err := os.Stat(idfile); os.IsNotExist(err) { 519 ioutil.WriteFile(idfile, sshkey, 0600) 520 } 521 } 522 // Upload 523 dest := sshUser + "@ppa.launchpad.net" 524 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 525 log.Fatal(err) 526 } 527 } 528 529 func getenvBase64(variable string) []byte { 530 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 531 if err != nil { 532 log.Fatal("invalid base64 " + variable) 533 } 534 return []byte(dec) 535 } 536 537 func makeWorkdir(wdflag string) string { 538 var err error 539 if wdflag != "" { 540 err = os.MkdirAll(wdflag, 0744) 541 } else { 542 wdflag, err = ioutil.TempDir("", "geth-build-") 543 } 544 if err != nil { 545 log.Fatal(err) 546 } 547 return wdflag 548 } 549 550 func isUnstableBuild(env build.Environment) bool { 551 if env.Tag != "" { 552 return false 553 } 554 return true 555 } 556 557 type debPackage struct { 558 Name string // the name of the Debian package to produce, e.g. "ethereum" 559 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 560 Executables []debExecutable // executables to be included in the package 561 } 562 563 type debMetadata struct { 564 Env build.Environment 565 566 PackageName string 567 568 // go-ethereum version being built. Note that this 569 // is not the debian package version. The package version 570 // is constructed by VersionString. 571 Version string 572 573 Author string // "name <email>", also selects signing key 574 Distro, Time string 575 Executables []debExecutable 576 } 577 578 type debExecutable struct { 579 PackageName string 580 BinaryName string 581 Description string 582 } 583 584 // Package returns the name of the package if present, or 585 // fallbacks to BinaryName 586 func (d debExecutable) Package() string { 587 if d.PackageName != "" { 588 return d.PackageName 589 } 590 return d.BinaryName 591 } 592 593 func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 594 if author == "" { 595 // No signing key, use default author. 596 author = "Ethereum Builds <fjl@ethereum.org>" 597 } 598 return debMetadata{ 599 PackageName: name, 600 Env: env, 601 Author: author, 602 Distro: distro, 603 Version: version, 604 Time: t.Format(time.RFC1123Z), 605 Executables: exes, 606 } 607 } 608 609 // Name returns the name of the metapackage that depends 610 // on all executable packages. 611 func (meta debMetadata) Name() string { 612 if isUnstableBuild(meta.Env) { 613 return meta.PackageName + "-unstable" 614 } 615 return meta.PackageName 616 } 617 618 // VersionString returns the debian version of the packages. 619 func (meta debMetadata) VersionString() string { 620 vsn := meta.Version 621 if meta.Env.Buildnum != "" { 622 vsn += "+build" + meta.Env.Buildnum 623 } 624 if meta.Distro != "" { 625 vsn += "+" + meta.Distro 626 } 627 return vsn 628 } 629 630 // ExeList returns the list of all executable packages. 631 func (meta debMetadata) ExeList() string { 632 names := make([]string, len(meta.Executables)) 633 for i, e := range meta.Executables { 634 names[i] = meta.ExeName(e) 635 } 636 return strings.Join(names, ", ") 637 } 638 639 // ExeName returns the package name of an executable package. 640 func (meta debMetadata) ExeName(exe debExecutable) string { 641 if isUnstableBuild(meta.Env) { 642 return exe.Package() + "-unstable" 643 } 644 return exe.Package() 645 } 646 647 // ExeConflicts returns the content of the Conflicts field 648 // for executable packages. 649 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 650 if isUnstableBuild(meta.Env) { 651 // Set up the conflicts list so that the *-unstable packages 652 // cannot be installed alongside the regular version. 653 // 654 // https://www.debian.org/doc/debian-policy/ch-relationships.html 655 // is very explicit about Conflicts: and says that Breaks: should 656 // be preferred and the conflicting files should be handled via 657 // alternates. We might do this eventually but using a conflict is 658 // easier now. 659 return "ethereum, " + exe.Package() 660 } 661 return "" 662 } 663 664 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 665 pkg := meta.Name() + "-" + meta.VersionString() 666 pkgdir = filepath.Join(tmpdir, pkg) 667 if err := os.Mkdir(pkgdir, 0755); err != nil { 668 log.Fatal(err) 669 } 670 671 // Copy the source code. 672 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 673 674 // Put the debian build files in place. 675 debian := filepath.Join(pkgdir, "debian") 676 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 677 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 678 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 679 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 680 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 681 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 682 for _, exe := range meta.Executables { 683 install := filepath.Join(debian, meta.ExeName(exe)+".install") 684 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 685 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 686 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 687 } 688 689 return pkgdir 690 } 691 692 // Windows installer 693 func doWindowsInstaller(cmdline []string) { 694 // Parse the flags and make skip installer generation on PRs 695 var ( 696 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 697 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 698 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 699 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 700 ) 701 flag.CommandLine.Parse(cmdline) 702 *workdir = makeWorkdir(*workdir) 703 env := build.Env() 704 maybeSkipArchive(env) 705 706 // Aggregate binaries that are included in the installer 707 var ( 708 devTools []string 709 allTools []string 710 gethTool string 711 ) 712 for _, file := range allToolsArchiveFiles { 713 if file == "COPYING" { // license, copied later 714 continue 715 } 716 allTools = append(allTools, filepath.Base(file)) 717 if filepath.Base(file) == "geth.exe" { 718 gethTool = file 719 } else { 720 devTools = append(devTools, file) 721 } 722 } 723 724 // Render NSIS scripts: Installer NSIS contains two installer sections, 725 // first section contains the geth binary, second section holds the dev tools. 726 templateData := map[string]interface{}{ 727 "License": "COPYING", 728 "Geth": gethTool, 729 "DevTools": devTools, 730 } 731 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 732 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 733 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 734 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 735 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 736 build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755) 737 build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755) 738 739 // Build the installer. This assumes that all the needed files have been previously 740 // built (don't mix building and packaging to keep cross compilation complexity to a 741 // minimum). 742 version := strings.Split(params.Version, ".") 743 if env.Commit != "" { 744 version[2] += "-" + env.Commit[:8] 745 } 746 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 747 build.MustRunCommand("makensis.exe", 748 "/DOUTPUTFILE="+installer, 749 "/DMAJORVERSION="+version[0], 750 "/DMINORVERSION="+version[1], 751 "/DBUILDVERSION="+version[2], 752 "/DARCH="+*arch, 753 filepath.Join(*workdir, "geth.nsi"), 754 ) 755 756 // Sign and publish installer. 757 if err := archiveUpload(installer, *upload, *signer); err != nil { 758 log.Fatal(err) 759 } 760 } 761 762 // Android archives 763 764 func doAndroidArchive(cmdline []string) { 765 var ( 766 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 767 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 768 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 769 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 770 ) 771 flag.CommandLine.Parse(cmdline) 772 env := build.Env() 773 774 // Sanity check that the SDK and NDK are installed and set 775 if os.Getenv("ANDROID_HOME") == "" { 776 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 777 } 778 // Build the Android archive and Maven resources 779 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 780 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/Nova-foundation/go-ethereum/mobile")) 781 782 if *local { 783 // If we're building locally, copy bundle to build dir and skip Maven 784 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 785 return 786 } 787 meta := newMavenMetadata(env) 788 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 789 790 // Skip Maven deploy and Azure upload for PR builds 791 maybeSkipArchive(env) 792 793 // Sign and upload the archive to Azure 794 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 795 os.Rename("geth.aar", archive) 796 797 if err := archiveUpload(archive, *upload, *signer); err != nil { 798 log.Fatal(err) 799 } 800 // Sign and upload all the artifacts to Maven Central 801 os.Rename(archive, meta.Package+".aar") 802 if *signer != "" && *deploy != "" { 803 // Import the signing key into the local GPG instance 804 key := getenvBase64(*signer) 805 gpg := exec.Command("gpg", "--import") 806 gpg.Stdin = bytes.NewReader(key) 807 build.MustRun(gpg) 808 keyID, err := build.PGPKeyID(string(key)) 809 if err != nil { 810 log.Fatal(err) 811 } 812 // Upload the artifacts to Sonatype and/or Maven Central 813 repo := *deploy + "/service/local/staging/deploy/maven2" 814 if meta.Develop { 815 repo = *deploy + "/content/repositories/snapshots" 816 } 817 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 818 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 819 "-Dgpg.keyname="+keyID, 820 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 821 } 822 } 823 824 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 825 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 826 cmd.Args = append(cmd.Args, args...) 827 cmd.Env = []string{ 828 "GOPATH=" + build.GOPATH(), 829 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 830 } 831 for _, e := range os.Environ() { 832 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") { 833 continue 834 } 835 cmd.Env = append(cmd.Env, e) 836 } 837 return cmd 838 } 839 840 type mavenMetadata struct { 841 Version string 842 Package string 843 Develop bool 844 Contributors []mavenContributor 845 } 846 847 type mavenContributor struct { 848 Name string 849 Email string 850 } 851 852 func newMavenMetadata(env build.Environment) mavenMetadata { 853 // Collect the list of authors from the repo root 854 contribs := []mavenContributor{} 855 if authors, err := os.Open("AUTHORS"); err == nil { 856 defer authors.Close() 857 858 scanner := bufio.NewScanner(authors) 859 for scanner.Scan() { 860 // Skip any whitespace from the authors list 861 line := strings.TrimSpace(scanner.Text()) 862 if line == "" || line[0] == '#' { 863 continue 864 } 865 // Split the author and insert as a contributor 866 re := regexp.MustCompile("([^<]+) <(.+)>") 867 parts := re.FindStringSubmatch(line) 868 if len(parts) == 3 { 869 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 870 } 871 } 872 } 873 // Render the version and package strings 874 version := params.Version 875 if isUnstableBuild(env) { 876 version += "-SNAPSHOT" 877 } 878 return mavenMetadata{ 879 Version: version, 880 Package: "geth-" + version, 881 Develop: isUnstableBuild(env), 882 Contributors: contribs, 883 } 884 } 885 886 // XCode frameworks 887 888 func doXCodeFramework(cmdline []string) { 889 var ( 890 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 891 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 892 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 893 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 894 ) 895 flag.CommandLine.Parse(cmdline) 896 env := build.Env() 897 898 // Build the iOS XCode framework 899 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 900 build.MustRun(gomobileTool("init")) 901 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/Nova-foundation/go-ethereum/mobile") 902 903 if *local { 904 // If we're building locally, use the build folder and stop afterwards 905 bind.Dir, _ = filepath.Abs(GOBIN) 906 build.MustRun(bind) 907 return 908 } 909 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 910 if err := os.Mkdir(archive, os.ModePerm); err != nil { 911 log.Fatal(err) 912 } 913 bind.Dir, _ = filepath.Abs(archive) 914 build.MustRun(bind) 915 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 916 917 // Skip CocoaPods deploy and Azure upload for PR builds 918 maybeSkipArchive(env) 919 920 // Sign and upload the framework to Azure 921 if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil { 922 log.Fatal(err) 923 } 924 // Prepare and upload a PodSpec to CocoaPods 925 if *deploy != "" { 926 meta := newPodMetadata(env, archive) 927 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 928 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose") 929 } 930 } 931 932 type podMetadata struct { 933 Version string 934 Commit string 935 Archive string 936 Contributors []podContributor 937 } 938 939 type podContributor struct { 940 Name string 941 Email string 942 } 943 944 func newPodMetadata(env build.Environment, archive string) podMetadata { 945 // Collect the list of authors from the repo root 946 contribs := []podContributor{} 947 if authors, err := os.Open("AUTHORS"); err == nil { 948 defer authors.Close() 949 950 scanner := bufio.NewScanner(authors) 951 for scanner.Scan() { 952 // Skip any whitespace from the authors list 953 line := strings.TrimSpace(scanner.Text()) 954 if line == "" || line[0] == '#' { 955 continue 956 } 957 // Split the author and insert as a contributor 958 re := regexp.MustCompile("([^<]+) <(.+)>") 959 parts := re.FindStringSubmatch(line) 960 if len(parts) == 3 { 961 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 962 } 963 } 964 } 965 version := params.Version 966 if isUnstableBuild(env) { 967 version += "-unstable." + env.Buildnum 968 } 969 return podMetadata{ 970 Archive: archive, 971 Version: version, 972 Commit: env.Commit, 973 Contributors: contribs, 974 } 975 } 976 977 // Cross compilation 978 979 func doXgo(cmdline []string) { 980 var ( 981 alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) 982 ) 983 flag.CommandLine.Parse(cmdline) 984 env := build.Env() 985 986 // Make sure xgo is available for cross compilation 987 gogetxgo := goTool("get", "github.com/karalabe/xgo") 988 build.MustRun(gogetxgo) 989 990 // If all tools building is requested, build everything the builder wants 991 args := append(buildFlags(env), flag.Args()...) 992 993 if *alltools { 994 args = append(args, []string{"--dest", GOBIN}...) 995 for _, res := range allToolsArchiveFiles { 996 if strings.HasPrefix(res, GOBIN) { 997 // Binary tool found, cross build it explicitly 998 args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) 999 xgo := xgoTool(args) 1000 build.MustRun(xgo) 1001 args = args[:len(args)-1] 1002 } 1003 } 1004 return 1005 } 1006 // Otherwise xxecute the explicit cross compilation 1007 path := args[len(args)-1] 1008 args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) 1009 1010 xgo := xgoTool(args) 1011 build.MustRun(xgo) 1012 } 1013 1014 func xgoTool(args []string) *exec.Cmd { 1015 cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) 1016 cmd.Env = []string{ 1017 "GOPATH=" + build.GOPATH(), 1018 "GOBIN=" + GOBIN, 1019 } 1020 for _, e := range os.Environ() { 1021 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 1022 continue 1023 } 1024 cmd.Env = append(cmd.Env, e) 1025 } 1026 return cmd 1027 } 1028 1029 // Binary distribution cleanups 1030 1031 func doPurge(cmdline []string) { 1032 var ( 1033 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1034 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1035 ) 1036 flag.CommandLine.Parse(cmdline) 1037 1038 if env := build.Env(); !env.IsCronJob { 1039 log.Printf("skipping because not a cron job") 1040 os.Exit(0) 1041 } 1042 // Create the azure authentication and list the current archives 1043 auth := build.AzureBlobstoreConfig{ 1044 Account: strings.Split(*store, "/")[0], 1045 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1046 Container: strings.SplitN(*store, "/", 2)[1], 1047 } 1048 blobs, err := build.AzureBlobstoreList(auth) 1049 if err != nil { 1050 log.Fatal(err) 1051 } 1052 // Iterate over the blobs, collect and sort all unstable builds 1053 for i := 0; i < len(blobs); i++ { 1054 if !strings.Contains(blobs[i].Name, "unstable") { 1055 blobs = append(blobs[:i], blobs[i+1:]...) 1056 i-- 1057 } 1058 } 1059 for i := 0; i < len(blobs); i++ { 1060 for j := i + 1; j < len(blobs); j++ { 1061 if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { 1062 blobs[i], blobs[j] = blobs[j], blobs[i] 1063 } 1064 } 1065 } 1066 // Filter out all archives more recent that the given threshold 1067 for i, blob := range blobs { 1068 if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1069 blobs = blobs[:i] 1070 break 1071 } 1072 } 1073 // Delete all marked as such and return 1074 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1075 log.Fatal(err) 1076 } 1077 }