github.com/bloxroute-labs/bor@v0.1.4/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // +build none 18 19 /* 20 The ci command is called from Continuous Integration scripts. 21 22 Usage: go run build/ci.go <command> <command flags/arguments> 23 24 Available commands are: 25 26 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 27 test [ -coverage ] [ packages... ] -- runs the tests 28 lint -- runs certain pre-selected linters 29 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts 30 importkeys -- imports signing keys from env 31 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 32 nsis -- creates a Windows NSIS installer 33 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 34 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 35 xgo [ -alltools ] [ options ] -- cross builds according to options 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "go/parser" 50 "go/token" 51 "io/ioutil" 52 "log" 53 "os" 54 "os/exec" 55 "path/filepath" 56 "regexp" 57 "runtime" 58 "strings" 59 "time" 60 61 "github.com/maticnetwork/bor/internal/build" 62 "github.com/maticnetwork/bor/params" 63 ) 64 65 var ( 66 // Files that end up in the geth*.zip archive. 67 gethArchiveFiles = []string{ 68 "COPYING", 69 executablePath("bor"), 70 } 71 72 // Files that end up in the geth-alltools*.zip archive. 73 allToolsArchiveFiles = []string{ 74 "COPYING", 75 executablePath("abigen"), 76 executablePath("bootnode"), 77 executablePath("evm"), 78 executablePath("bor"), 79 executablePath("puppeth"), 80 executablePath("rlpdump"), 81 executablePath("wnode"), 82 executablePath("clef"), 83 } 84 85 // A debian package is created for all executables listed here. 86 debExecutables = []debExecutable{ 87 { 88 BinaryName: "abigen", 89 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 90 }, 91 { 92 BinaryName: "bootnode", 93 Description: "Ethereum bootnode.", 94 }, 95 { 96 BinaryName: "evm", 97 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 98 }, 99 { 100 BinaryName: "bor", 101 Description: "Ethereum CLI client.", 102 }, 103 { 104 BinaryName: "puppeth", 105 Description: "Ethereum private network manager.", 106 }, 107 { 108 BinaryName: "rlpdump", 109 Description: "Developer utility tool that prints RLP structures.", 110 }, 111 { 112 BinaryName: "wnode", 113 Description: "Ethereum Whisper diagnostic tool", 114 }, 115 { 116 BinaryName: "clef", 117 Description: "Ethereum account management tool.", 118 }, 119 } 120 121 // A debian package is created for all executables listed here. 122 123 debEthereum = debPackage{ 124 Name: "ethereum", 125 Version: params.Version, 126 Executables: debExecutables, 127 } 128 129 // Debian meta packages to build and push to Ubuntu PPA 130 debPackages = []debPackage{ 131 debEthereum, 132 } 133 134 // Distros for which packages are created. 135 // Note: vivid is unsupported because there is no golang-1.6 package for it. 136 // Note: wily is unsupported because it was officially deprecated on Launchpad. 137 // Note: yakkety is unsupported because it was officially deprecated on Launchpad. 138 // Note: zesty is unsupported because it was officially deprecated on Launchpad. 139 // Note: artful is unsupported because it was officially deprecated on Launchpad. 140 debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"} 141 ) 142 143 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 144 145 func executablePath(name string) string { 146 if runtime.GOOS == "windows" { 147 name += ".exe" 148 } 149 return filepath.Join(GOBIN, name) 150 } 151 152 func main() { 153 log.SetFlags(log.Lshortfile) 154 155 if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) { 156 log.Fatal("this script must be run from the root of the repository") 157 } 158 if len(os.Args) < 2 { 159 log.Fatal("need subcommand as first argument") 160 } 161 switch os.Args[1] { 162 case "install": 163 doInstall(os.Args[2:]) 164 case "test": 165 doTest(os.Args[2:]) 166 case "lint": 167 doLint(os.Args[2:]) 168 case "archive": 169 doArchive(os.Args[2:]) 170 case "debsrc": 171 doDebianSource(os.Args[2:]) 172 case "nsis": 173 doWindowsInstaller(os.Args[2:]) 174 case "aar": 175 doAndroidArchive(os.Args[2:]) 176 case "xcode": 177 doXCodeFramework(os.Args[2:]) 178 case "xgo": 179 doXgo(os.Args[2:]) 180 case "purge": 181 doPurge(os.Args[2:]) 182 default: 183 log.Fatal("unknown command ", os.Args[1]) 184 } 185 } 186 187 // Compiling 188 189 func doInstall(cmdline []string) { 190 var ( 191 arch = flag.String("arch", "", "Architecture to cross build for") 192 cc = flag.String("cc", "", "C compiler to cross build with") 193 ) 194 flag.CommandLine.Parse(cmdline) 195 env := build.Env() 196 197 // Check Go version. People regularly open issues about compilation 198 // failure with outdated Go. This should save them the trouble. 199 if !strings.Contains(runtime.Version(), "devel") { 200 // Figure out the minor version number since we can't textually compare (1.10 < 1.9) 201 var minor int 202 fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) 203 204 if minor < 9 { 205 log.Println("You have Go version", runtime.Version()) 206 log.Println("go-ethereum requires at least Go version 1.9 and cannot") 207 log.Println("be compiled with an earlier version. Please upgrade your Go installation.") 208 os.Exit(1) 209 } 210 } 211 // Compile packages given as arguments, or everything if there are no arguments. 212 packages := []string{"./..."} 213 if flag.NArg() > 0 { 214 packages = flag.Args() 215 } 216 packages = build.ExpandPackagesNoVendor(packages) 217 218 if *arch == "" || *arch == runtime.GOARCH { 219 goinstall := goTool("install", buildFlags(env)...) 220 goinstall.Args = append(goinstall.Args, "-v") 221 goinstall.Args = append(goinstall.Args, packages...) 222 build.MustRun(goinstall) 223 return 224 } 225 // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds 226 if *arch == "arm" { 227 os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) 228 for _, path := range filepath.SplitList(build.GOPATH()) { 229 os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) 230 } 231 } 232 // Seems we are cross compiling, work around forbidden GOBIN 233 goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) 234 goinstall.Args = append(goinstall.Args, "-v") 235 goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...) 236 goinstall.Args = append(goinstall.Args, packages...) 237 build.MustRun(goinstall) 238 239 if cmds, err := ioutil.ReadDir("cmd"); err == nil { 240 for _, cmd := range cmds { 241 pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly) 242 if err != nil { 243 log.Fatal(err) 244 } 245 for name := range pkgs { 246 if name == "main" { 247 gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...) 248 gobuild.Args = append(gobuild.Args, "-v") 249 gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...) 250 gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name())) 251 build.MustRun(gobuild) 252 break 253 } 254 } 255 } 256 } 257 } 258 259 func buildFlags(env build.Environment) (flags []string) { 260 var ld []string 261 if env.Commit != "" { 262 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 263 ld = append(ld, "-X", "main.gitDate="+env.Date) 264 } 265 if runtime.GOOS == "darwin" { 266 ld = append(ld, "-s") 267 } 268 269 if len(ld) > 0 { 270 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 271 } 272 return flags 273 } 274 275 func goTool(subcmd string, args ...string) *exec.Cmd { 276 return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...) 277 } 278 279 func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { 280 cmd := build.GoTool(subcmd, args...) 281 cmd.Env = []string{"GOPATH=" + build.GOPATH()} 282 if arch == "" || arch == runtime.GOARCH { 283 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 284 } else { 285 cmd.Env = append(cmd.Env, "CGO_ENABLED=1") 286 cmd.Env = append(cmd.Env, "GOARCH="+arch) 287 } 288 if cc != "" { 289 cmd.Env = append(cmd.Env, "CC="+cc) 290 } 291 for _, e := range os.Environ() { 292 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 293 continue 294 } 295 cmd.Env = append(cmd.Env, e) 296 } 297 return cmd 298 } 299 300 // Running The Tests 301 // 302 // "tests" also includes static analysis tools such as vet. 303 304 func doTest(cmdline []string) { 305 coverage := flag.Bool("coverage", false, "Whether to record code coverage") 306 flag.CommandLine.Parse(cmdline) 307 env := build.Env() 308 309 packages := []string{"./..."} 310 if len(flag.CommandLine.Args()) > 0 { 311 packages = flag.CommandLine.Args() 312 } 313 packages = build.ExpandPackagesNoVendor(packages) 314 315 // Run the actual tests. 316 // Test a single package at a time. CI builders are slow 317 // and some tests run into timeouts under load. 318 gotest := goTool("test", buildFlags(env)...) 319 gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m") 320 if *coverage { 321 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 322 } 323 324 gotest.Args = append(gotest.Args, packages...) 325 build.MustRun(gotest) 326 } 327 328 // runs gometalinter on requested packages 329 func doLint(cmdline []string) { 330 flag.CommandLine.Parse(cmdline) 331 332 packages := []string{"./..."} 333 if len(flag.CommandLine.Args()) > 0 { 334 packages = flag.CommandLine.Args() 335 } 336 // Get metalinter and install all supported linters 337 build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) 338 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") 339 340 // Run fast linters batched together 341 configs := []string{ 342 "--vendor", 343 "--tests", 344 "--deadline=2m", 345 "--disable-all", 346 "--enable=goimports", 347 "--enable=varcheck", 348 "--enable=vet", 349 "--enable=gofmt", 350 "--enable=misspell", 351 "--enable=goconst", 352 "--min-occurrences=6", // for goconst 353 } 354 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 355 356 // Run slow linters one by one 357 for _, linter := range []string{"unconvert", "gosimple"} { 358 configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} 359 build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) 360 } 361 } 362 363 // Release Packaging 364 func doArchive(cmdline []string) { 365 var ( 366 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 367 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 368 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 369 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 370 ext string 371 ) 372 flag.CommandLine.Parse(cmdline) 373 switch *atype { 374 case "zip": 375 ext = ".zip" 376 case "tar": 377 ext = ".tar.gz" 378 default: 379 log.Fatal("unknown archive type: ", atype) 380 } 381 382 var ( 383 env = build.Env() 384 385 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 386 geth = "geth-" + basegeth + ext 387 alltools = "geth-alltools-" + basegeth + ext 388 ) 389 maybeSkipArchive(env) 390 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 391 log.Fatal(err) 392 } 393 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 394 log.Fatal(err) 395 } 396 for _, archive := range []string{geth, alltools} { 397 if err := archiveUpload(archive, *upload, *signer); err != nil { 398 log.Fatal(err) 399 } 400 } 401 } 402 403 func archiveBasename(arch string, archiveVersion string) string { 404 platform := runtime.GOOS + "-" + arch 405 if arch == "arm" { 406 platform += os.Getenv("GOARM") 407 } 408 if arch == "android" { 409 platform = "android-all" 410 } 411 if arch == "ios" { 412 platform = "ios-all" 413 } 414 return platform + "-" + archiveVersion 415 } 416 417 func archiveUpload(archive string, blobstore string, signer string) error { 418 // If signing was requested, generate the signature files 419 if signer != "" { 420 key := getenvBase64(signer) 421 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 422 return err 423 } 424 } 425 // If uploading to Azure was requested, push the archive possibly with its signature 426 if blobstore != "" { 427 auth := build.AzureBlobstoreConfig{ 428 Account: strings.Split(blobstore, "/")[0], 429 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 430 Container: strings.SplitN(blobstore, "/", 2)[1], 431 } 432 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 433 return err 434 } 435 if signer != "" { 436 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 437 return err 438 } 439 } 440 } 441 return nil 442 } 443 444 // skips archiving for some build configurations. 445 func maybeSkipArchive(env build.Environment) { 446 if env.IsPullRequest { 447 log.Printf("skipping because this is a PR build") 448 os.Exit(0) 449 } 450 if env.IsCronJob { 451 log.Printf("skipping because this is a cron job") 452 os.Exit(0) 453 } 454 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 455 log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) 456 os.Exit(0) 457 } 458 } 459 460 // Debian Packaging 461 func doDebianSource(cmdline []string) { 462 var ( 463 signer = flag.String("signer", "", `Signing key name, also used as package author`) 464 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 465 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 466 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 467 now = time.Now() 468 ) 469 flag.CommandLine.Parse(cmdline) 470 *workdir = makeWorkdir(*workdir) 471 env := build.Env() 472 maybeSkipArchive(env) 473 474 // Import the signing key. 475 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 476 gpg := exec.Command("gpg", "--import") 477 gpg.Stdin = bytes.NewReader(key) 478 build.MustRun(gpg) 479 } 480 481 // Create Debian packages and upload them 482 for _, pkg := range debPackages { 483 for _, distro := range debDistros { 484 meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 485 pkgdir := stageDebianSource(*workdir, meta) 486 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") 487 debuild.Dir = pkgdir 488 build.MustRun(debuild) 489 490 var ( 491 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 492 source = filepath.Join(*workdir, basename+".tar.xz") 493 dsc = filepath.Join(*workdir, basename+".dsc") 494 changes = filepath.Join(*workdir, basename+"_source.changes") 495 ) 496 if *signer != "" { 497 build.MustRunCommand("debsign", changes) 498 } 499 if *upload != "" { 500 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) 501 } 502 } 503 } 504 } 505 506 func ppaUpload(workdir, ppa, sshUser string, files []string) { 507 p := strings.Split(ppa, "/") 508 if len(p) != 2 { 509 log.Fatal("-upload PPA name must contain single /") 510 } 511 if sshUser == "" { 512 sshUser = p[0] 513 } 514 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 515 // Create the SSH identity file if it doesn't exist. 516 var idfile string 517 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 518 idfile = filepath.Join(workdir, "sshkey") 519 if _, err := os.Stat(idfile); os.IsNotExist(err) { 520 ioutil.WriteFile(idfile, sshkey, 0600) 521 } 522 } 523 // Upload 524 dest := sshUser + "@ppa.launchpad.net" 525 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 526 log.Fatal(err) 527 } 528 } 529 530 func getenvBase64(variable string) []byte { 531 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 532 if err != nil { 533 log.Fatal("invalid base64 " + variable) 534 } 535 return []byte(dec) 536 } 537 538 func makeWorkdir(wdflag string) string { 539 var err error 540 if wdflag != "" { 541 err = os.MkdirAll(wdflag, 0744) 542 } else { 543 wdflag, err = ioutil.TempDir("", "geth-build-") 544 } 545 if err != nil { 546 log.Fatal(err) 547 } 548 return wdflag 549 } 550 551 func isUnstableBuild(env build.Environment) bool { 552 if env.Tag != "" { 553 return false 554 } 555 return true 556 } 557 558 type debPackage struct { 559 Name string // the name of the Debian package to produce, e.g. "ethereum" 560 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 561 Executables []debExecutable // executables to be included in the package 562 } 563 564 type debMetadata struct { 565 Env build.Environment 566 567 PackageName string 568 569 // go-ethereum version being built. Note that this 570 // is not the debian package version. The package version 571 // is constructed by VersionString. 572 Version string 573 574 Author string // "name <email>", also selects signing key 575 Distro, Time string 576 Executables []debExecutable 577 } 578 579 type debExecutable struct { 580 PackageName string 581 BinaryName string 582 Description string 583 } 584 585 // Package returns the name of the package if present, or 586 // fallbacks to BinaryName 587 func (d debExecutable) Package() string { 588 if d.PackageName != "" { 589 return d.PackageName 590 } 591 return d.BinaryName 592 } 593 594 func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 595 if author == "" { 596 // No signing key, use default author. 597 author = "Ethereum Builds <fjl@ethereum.org>" 598 } 599 return debMetadata{ 600 PackageName: name, 601 Env: env, 602 Author: author, 603 Distro: distro, 604 Version: version, 605 Time: t.Format(time.RFC1123Z), 606 Executables: exes, 607 } 608 } 609 610 // Name returns the name of the metapackage that depends 611 // on all executable packages. 612 func (meta debMetadata) Name() string { 613 if isUnstableBuild(meta.Env) { 614 return meta.PackageName + "-unstable" 615 } 616 return meta.PackageName 617 } 618 619 // VersionString returns the debian version of the packages. 620 func (meta debMetadata) VersionString() string { 621 vsn := meta.Version 622 if meta.Env.Buildnum != "" { 623 vsn += "+build" + meta.Env.Buildnum 624 } 625 if meta.Distro != "" { 626 vsn += "+" + meta.Distro 627 } 628 return vsn 629 } 630 631 // ExeList returns the list of all executable packages. 632 func (meta debMetadata) ExeList() string { 633 names := make([]string, len(meta.Executables)) 634 for i, e := range meta.Executables { 635 names[i] = meta.ExeName(e) 636 } 637 return strings.Join(names, ", ") 638 } 639 640 // ExeName returns the package name of an executable package. 641 func (meta debMetadata) ExeName(exe debExecutable) string { 642 if isUnstableBuild(meta.Env) { 643 return exe.Package() + "-unstable" 644 } 645 return exe.Package() 646 } 647 648 // ExeConflicts returns the content of the Conflicts field 649 // for executable packages. 650 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 651 if isUnstableBuild(meta.Env) { 652 // Set up the conflicts list so that the *-unstable packages 653 // cannot be installed alongside the regular version. 654 // 655 // https://www.debian.org/doc/debian-policy/ch-relationships.html 656 // is very explicit about Conflicts: and says that Breaks: should 657 // be preferred and the conflicting files should be handled via 658 // alternates. We might do this eventually but using a conflict is 659 // easier now. 660 return "ethereum, " + exe.Package() 661 } 662 return "" 663 } 664 665 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 666 pkg := meta.Name() + "-" + meta.VersionString() 667 pkgdir = filepath.Join(tmpdir, pkg) 668 if err := os.Mkdir(pkgdir, 0755); err != nil { 669 log.Fatal(err) 670 } 671 672 // Copy the source code. 673 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 674 675 // Put the debian build files in place. 676 debian := filepath.Join(pkgdir, "debian") 677 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 678 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 679 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 680 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 681 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 682 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 683 for _, exe := range meta.Executables { 684 install := filepath.Join(debian, meta.ExeName(exe)+".install") 685 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 686 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 687 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 688 } 689 690 return pkgdir 691 } 692 693 // Windows installer 694 func doWindowsInstaller(cmdline []string) { 695 // Parse the flags and make skip installer generation on PRs 696 var ( 697 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 698 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 699 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 700 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 701 ) 702 flag.CommandLine.Parse(cmdline) 703 *workdir = makeWorkdir(*workdir) 704 env := build.Env() 705 maybeSkipArchive(env) 706 707 // Aggregate binaries that are included in the installer 708 var ( 709 devTools []string 710 allTools []string 711 gethTool string 712 ) 713 for _, file := range allToolsArchiveFiles { 714 if file == "COPYING" { // license, copied later 715 continue 716 } 717 allTools = append(allTools, filepath.Base(file)) 718 if filepath.Base(file) == "geth.exe" { 719 gethTool = file 720 } else { 721 devTools = append(devTools, file) 722 } 723 } 724 725 // Render NSIS scripts: Installer NSIS contains two installer sections, 726 // first section contains the geth binary, second section holds the dev tools. 727 templateData := map[string]interface{}{ 728 "License": "COPYING", 729 "Geth": gethTool, 730 "DevTools": devTools, 731 } 732 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 733 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 734 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 735 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 736 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 737 build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755) 738 build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755) 739 740 // Build the installer. This assumes that all the needed files have been previously 741 // built (don't mix building and packaging to keep cross compilation complexity to a 742 // minimum). 743 version := strings.Split(params.Version, ".") 744 if env.Commit != "" { 745 version[2] += "-" + env.Commit[:8] 746 } 747 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 748 build.MustRunCommand("makensis.exe", 749 "/DOUTPUTFILE="+installer, 750 "/DMAJORVERSION="+version[0], 751 "/DMINORVERSION="+version[1], 752 "/DBUILDVERSION="+version[2], 753 "/DARCH="+*arch, 754 filepath.Join(*workdir, "geth.nsi"), 755 ) 756 757 // Sign and publish installer. 758 if err := archiveUpload(installer, *upload, *signer); err != nil { 759 log.Fatal(err) 760 } 761 } 762 763 // Android archives 764 765 func doAndroidArchive(cmdline []string) { 766 var ( 767 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 768 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 769 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 770 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 771 ) 772 flag.CommandLine.Parse(cmdline) 773 env := build.Env() 774 775 // Sanity check that the SDK and NDK are installed and set 776 if os.Getenv("ANDROID_HOME") == "" { 777 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 778 } 779 // Build the Android archive and Maven resources 780 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 781 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/maticnetwork/bor/mobile")) 782 783 if *local { 784 // If we're building locally, copy bundle to build dir and skip Maven 785 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 786 return 787 } 788 meta := newMavenMetadata(env) 789 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 790 791 // Skip Maven deploy and Azure upload for PR builds 792 maybeSkipArchive(env) 793 794 // Sign and upload the archive to Azure 795 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 796 os.Rename("geth.aar", archive) 797 798 if err := archiveUpload(archive, *upload, *signer); err != nil { 799 log.Fatal(err) 800 } 801 // Sign and upload all the artifacts to Maven Central 802 os.Rename(archive, meta.Package+".aar") 803 if *signer != "" && *deploy != "" { 804 // Import the signing key into the local GPG instance 805 key := getenvBase64(*signer) 806 gpg := exec.Command("gpg", "--import") 807 gpg.Stdin = bytes.NewReader(key) 808 build.MustRun(gpg) 809 keyID, err := build.PGPKeyID(string(key)) 810 if err != nil { 811 log.Fatal(err) 812 } 813 // Upload the artifacts to Sonatype and/or Maven Central 814 repo := *deploy + "/service/local/staging/deploy/maven2" 815 if meta.Develop { 816 repo = *deploy + "/content/repositories/snapshots" 817 } 818 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 819 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 820 "-Dgpg.keyname="+keyID, 821 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 822 } 823 } 824 825 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 826 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 827 cmd.Args = append(cmd.Args, args...) 828 cmd.Env = []string{ 829 "GOPATH=" + build.GOPATH(), 830 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 831 } 832 for _, e := range os.Environ() { 833 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") { 834 continue 835 } 836 cmd.Env = append(cmd.Env, e) 837 } 838 return cmd 839 } 840 841 type mavenMetadata struct { 842 Version string 843 Package string 844 Develop bool 845 Contributors []mavenContributor 846 } 847 848 type mavenContributor struct { 849 Name string 850 Email string 851 } 852 853 func newMavenMetadata(env build.Environment) mavenMetadata { 854 // Collect the list of authors from the repo root 855 contribs := []mavenContributor{} 856 if authors, err := os.Open("AUTHORS"); err == nil { 857 defer authors.Close() 858 859 scanner := bufio.NewScanner(authors) 860 for scanner.Scan() { 861 // Skip any whitespace from the authors list 862 line := strings.TrimSpace(scanner.Text()) 863 if line == "" || line[0] == '#' { 864 continue 865 } 866 // Split the author and insert as a contributor 867 re := regexp.MustCompile("([^<]+) <(.+)>") 868 parts := re.FindStringSubmatch(line) 869 if len(parts) == 3 { 870 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 871 } 872 } 873 } 874 // Render the version and package strings 875 version := params.Version 876 if isUnstableBuild(env) { 877 version += "-SNAPSHOT" 878 } 879 return mavenMetadata{ 880 Version: version, 881 Package: "geth-" + version, 882 Develop: isUnstableBuild(env), 883 Contributors: contribs, 884 } 885 } 886 887 // XCode frameworks 888 889 func doXCodeFramework(cmdline []string) { 890 var ( 891 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 892 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 893 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 894 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 895 ) 896 flag.CommandLine.Parse(cmdline) 897 env := build.Env() 898 899 // Build the iOS XCode framework 900 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 901 build.MustRun(gomobileTool("init")) 902 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/maticnetwork/bor/mobile") 903 904 if *local { 905 // If we're building locally, use the build folder and stop afterwards 906 bind.Dir, _ = filepath.Abs(GOBIN) 907 build.MustRun(bind) 908 return 909 } 910 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 911 if err := os.Mkdir(archive, os.ModePerm); err != nil { 912 log.Fatal(err) 913 } 914 bind.Dir, _ = filepath.Abs(archive) 915 build.MustRun(bind) 916 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 917 918 // Skip CocoaPods deploy and Azure upload for PR builds 919 maybeSkipArchive(env) 920 921 // Sign and upload the framework to Azure 922 if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil { 923 log.Fatal(err) 924 } 925 // Prepare and upload a PodSpec to CocoaPods 926 if *deploy != "" { 927 meta := newPodMetadata(env, archive) 928 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 929 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose") 930 } 931 } 932 933 type podMetadata struct { 934 Version string 935 Commit string 936 Archive string 937 Contributors []podContributor 938 } 939 940 type podContributor struct { 941 Name string 942 Email string 943 } 944 945 func newPodMetadata(env build.Environment, archive string) podMetadata { 946 // Collect the list of authors from the repo root 947 contribs := []podContributor{} 948 if authors, err := os.Open("AUTHORS"); err == nil { 949 defer authors.Close() 950 951 scanner := bufio.NewScanner(authors) 952 for scanner.Scan() { 953 // Skip any whitespace from the authors list 954 line := strings.TrimSpace(scanner.Text()) 955 if line == "" || line[0] == '#' { 956 continue 957 } 958 // Split the author and insert as a contributor 959 re := regexp.MustCompile("([^<]+) <(.+)>") 960 parts := re.FindStringSubmatch(line) 961 if len(parts) == 3 { 962 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 963 } 964 } 965 } 966 version := params.Version 967 if isUnstableBuild(env) { 968 version += "-unstable." + env.Buildnum 969 } 970 return podMetadata{ 971 Archive: archive, 972 Version: version, 973 Commit: env.Commit, 974 Contributors: contribs, 975 } 976 } 977 978 // Cross compilation 979 980 func doXgo(cmdline []string) { 981 var ( 982 alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) 983 ) 984 flag.CommandLine.Parse(cmdline) 985 env := build.Env() 986 987 // Make sure xgo is available for cross compilation 988 gogetxgo := goTool("get", "github.com/karalabe/xgo") 989 build.MustRun(gogetxgo) 990 991 // If all tools building is requested, build everything the builder wants 992 args := append(buildFlags(env), flag.Args()...) 993 994 if *alltools { 995 args = append(args, []string{"--dest", GOBIN}...) 996 for _, res := range allToolsArchiveFiles { 997 if strings.HasPrefix(res, GOBIN) { 998 // Binary tool found, cross build it explicitly 999 args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) 1000 xgo := xgoTool(args) 1001 build.MustRun(xgo) 1002 args = args[:len(args)-1] 1003 } 1004 } 1005 return 1006 } 1007 // Otherwise xxecute the explicit cross compilation 1008 path := args[len(args)-1] 1009 args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) 1010 1011 xgo := xgoTool(args) 1012 build.MustRun(xgo) 1013 } 1014 1015 func xgoTool(args []string) *exec.Cmd { 1016 cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) 1017 cmd.Env = []string{ 1018 "GOPATH=" + build.GOPATH(), 1019 "GOBIN=" + GOBIN, 1020 } 1021 for _, e := range os.Environ() { 1022 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 1023 continue 1024 } 1025 cmd.Env = append(cmd.Env, e) 1026 } 1027 return cmd 1028 } 1029 1030 // Binary distribution cleanups 1031 1032 func doPurge(cmdline []string) { 1033 var ( 1034 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1035 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1036 ) 1037 flag.CommandLine.Parse(cmdline) 1038 1039 if env := build.Env(); !env.IsCronJob { 1040 log.Printf("skipping because not a cron job") 1041 os.Exit(0) 1042 } 1043 // Create the azure authentication and list the current archives 1044 auth := build.AzureBlobstoreConfig{ 1045 Account: strings.Split(*store, "/")[0], 1046 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1047 Container: strings.SplitN(*store, "/", 2)[1], 1048 } 1049 blobs, err := build.AzureBlobstoreList(auth) 1050 if err != nil { 1051 log.Fatal(err) 1052 } 1053 // Iterate over the blobs, collect and sort all unstable builds 1054 for i := 0; i < len(blobs); i++ { 1055 if !strings.Contains(blobs[i].Name, "unstable") { 1056 blobs = append(blobs[:i], blobs[i+1:]...) 1057 i-- 1058 } 1059 } 1060 for i := 0; i < len(blobs); i++ { 1061 for j := i + 1; j < len(blobs); j++ { 1062 if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { 1063 blobs[i], blobs[j] = blobs[j], blobs[i] 1064 } 1065 } 1066 } 1067 // Filter out all archives more recent that the given threshold 1068 for i, blob := range blobs { 1069 if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1070 blobs = blobs[:i] 1071 break 1072 } 1073 } 1074 // Delete all marked as such and return 1075 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1076 log.Fatal(err) 1077 } 1078 }