github.com/CommerciumBlockchain/go-commercium@v0.0.0-20220709212705-b46438a77516/build/ci.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // +build none 18 19 /* 20 The ci command is called from Continuous Integration scripts. 21 22 Usage: go run build/ci.go <command> <command flags/arguments> 23 24 Available commands are: 25 26 install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables 27 test [ -coverage ] [ packages... ] -- runs the tests 28 lint -- runs certain pre-selected linters 29 archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts 30 importkeys -- imports signing keys from env 31 debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package 32 nsis -- creates a Windows NSIS installer 33 aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive 34 xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework 35 xgo [ -alltools ] [ options ] -- cross builds according to options 36 purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore 37 38 For all commands, -n prevents execution of external programs (dry run mode). 39 40 */ 41 package main 42 43 import ( 44 "bufio" 45 "bytes" 46 "encoding/base64" 47 "flag" 48 "fmt" 49 "io/ioutil" 50 "log" 51 "os" 52 "os/exec" 53 "path" 54 "path/filepath" 55 "regexp" 56 "runtime" 57 "strings" 58 "time" 59 60 "github.com/cespare/cp" 61 "github.com/CommerciumBlockchain/go-commercium/crypto/signify" 62 "github.com/CommerciumBlockchain/go-commercium/internal/build" 63 "github.com/CommerciumBlockchain/go-commercium/params" 64 ) 65 66 var ( 67 // Files that end up in the geth*.zip archive. 68 gethArchiveFiles = []string{ 69 "COPYING", 70 executablePath("geth"), 71 } 72 73 // Files that end up in the geth-alltools*.zip archive. 74 allToolsArchiveFiles = []string{ 75 "COPYING", 76 executablePath("abigen"), 77 executablePath("bootnode"), 78 executablePath("evm"), 79 executablePath("geth"), 80 executablePath("puppeth"), 81 executablePath("rlpdump"), 82 executablePath("clef"), 83 } 84 85 // A debian package is created for all executables listed here. 86 debExecutables = []debExecutable{ 87 { 88 BinaryName: "abigen", 89 Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", 90 }, 91 { 92 BinaryName: "bootnode", 93 Description: "Ethereum bootnode.", 94 }, 95 { 96 BinaryName: "evm", 97 Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", 98 }, 99 { 100 BinaryName: "geth", 101 Description: "Ethereum CLI client.", 102 }, 103 { 104 BinaryName: "puppeth", 105 Description: "Ethereum private network manager.", 106 }, 107 { 108 BinaryName: "rlpdump", 109 Description: "Developer utility tool that prints RLP structures.", 110 }, 111 { 112 BinaryName: "clef", 113 Description: "Ethereum account management tool.", 114 }, 115 } 116 117 // A debian package is created for all executables listed here. 118 119 debEthereum = debPackage{ 120 Name: "ethereum", 121 Version: params.Version, 122 Executables: debExecutables, 123 } 124 125 // Debian meta packages to build and push to Ubuntu PPA 126 debPackages = []debPackage{ 127 debEthereum, 128 } 129 130 // Distros for which packages are created. 131 // Note: vivid is unsupported because there is no golang-1.6 package for it. 132 // Note: wily is unsupported because it was officially deprecated on Launchpad. 133 // Note: yakkety is unsupported because it was officially deprecated on Launchpad. 134 // Note: zesty is unsupported because it was officially deprecated on Launchpad. 135 // Note: artful is unsupported because it was officially deprecated on Launchpad. 136 // Note: cosmic is unsupported because it was officially deprecated on Launchpad. 137 // Note: disco is unsupported because it was officially deprecated on Launchpad. 138 // Note: eoan is unsupported because it was officially deprecated on Launchpad. 139 debDistroGoBoots = map[string]string{ 140 "trusty": "golang-1.11", 141 "xenial": "golang-go", 142 "bionic": "golang-go", 143 "focal": "golang-go", 144 "groovy": "golang-go", 145 } 146 147 debGoBootPaths = map[string]string{ 148 "golang-1.11": "/usr/lib/go-1.11", 149 "golang-go": "/usr/lib/go", 150 } 151 152 // This is the version of go that will be downloaded by 153 // 154 // go run ci.go install -dlgo 155 dlgoVersion = "1.15.6" 156 ) 157 158 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) 159 160 func executablePath(name string) string { 161 if runtime.GOOS == "windows" { 162 name += ".exe" 163 } 164 return filepath.Join(GOBIN, name) 165 } 166 167 func main() { 168 log.SetFlags(log.Lshortfile) 169 170 if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) { 171 log.Fatal("this script must be run from the root of the repository") 172 } 173 if len(os.Args) < 2 { 174 log.Fatal("need subcommand as first argument") 175 } 176 switch os.Args[1] { 177 case "install": 178 doInstall(os.Args[2:]) 179 case "test": 180 doTest(os.Args[2:]) 181 case "lint": 182 doLint(os.Args[2:]) 183 case "archive": 184 doArchive(os.Args[2:]) 185 case "debsrc": 186 doDebianSource(os.Args[2:]) 187 case "nsis": 188 doWindowsInstaller(os.Args[2:]) 189 case "aar": 190 doAndroidArchive(os.Args[2:]) 191 case "xcode": 192 doXCodeFramework(os.Args[2:]) 193 case "xgo": 194 doXgo(os.Args[2:]) 195 case "purge": 196 doPurge(os.Args[2:]) 197 default: 198 log.Fatal("unknown command ", os.Args[1]) 199 } 200 } 201 202 // Compiling 203 204 func doInstall(cmdline []string) { 205 var ( 206 dlgo = flag.Bool("dlgo", false, "Download Go and build with it") 207 arch = flag.String("arch", "", "Architecture to cross build for") 208 cc = flag.String("cc", "", "C compiler to cross build with") 209 ) 210 flag.CommandLine.Parse(cmdline) 211 env := build.Env() 212 213 // Check local Go version. People regularly open issues about compilation 214 // failure with outdated Go. This should save them the trouble. 215 if !strings.Contains(runtime.Version(), "devel") { 216 // Figure out the minor version number since we can't textually compare (1.10 < 1.9) 217 var minor int 218 fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) 219 if minor < 13 { 220 log.Println("You have Go version", runtime.Version()) 221 log.Println("go-ethereum requires at least Go version 1.13 and cannot") 222 log.Println("be compiled with an earlier version. Please upgrade your Go installation.") 223 os.Exit(1) 224 } 225 } 226 227 // Choose which go command we're going to use. 228 var gobuild *exec.Cmd 229 if !*dlgo { 230 // Default behavior: use the go version which runs ci.go right now. 231 gobuild = goTool("build") 232 } else { 233 // Download of Go requested. This is for build environments where the 234 // installed version is too old and cannot be upgraded easily. 235 cachedir := filepath.Join("build", "cache") 236 goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir) 237 gobuild = localGoTool(goroot, "build") 238 } 239 240 // Configure environment for cross build. 241 if *arch != "" || *arch != runtime.GOARCH { 242 gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1") 243 gobuild.Env = append(gobuild.Env, "GOARCH="+*arch) 244 } 245 246 // Configure C compiler. 247 if *cc != "" { 248 gobuild.Env = append(gobuild.Env, "CC="+*cc) 249 } else if os.Getenv("CC") != "" { 250 gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC")) 251 } 252 253 // arm64 CI builders are memory-constrained and can't handle concurrent builds, 254 // better disable it. This check isn't the best, it should probably 255 // check for something in env instead. 256 if runtime.GOARCH == "arm64" { 257 gobuild.Args = append(gobuild.Args, "-p", "1") 258 } 259 260 // Put the default settings in. 261 gobuild.Args = append(gobuild.Args, buildFlags(env)...) 262 263 // We use -trimpath to avoid leaking local paths into the built executables. 264 gobuild.Args = append(gobuild.Args, "-trimpath") 265 266 // Show packages during build. 267 gobuild.Args = append(gobuild.Args, "-v") 268 269 // Now we choose what we're even building. 270 // Default: collect all 'main' packages in cmd/ and build those. 271 packages := flag.Args() 272 if len(packages) == 0 { 273 packages = build.FindMainPackages("./cmd") 274 } 275 276 // Do the build! 277 for _, pkg := range packages { 278 args := make([]string, len(gobuild.Args)) 279 copy(args, gobuild.Args) 280 args = append(args, "-o", executablePath(path.Base(pkg))) 281 args = append(args, pkg) 282 build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) 283 } 284 } 285 286 // buildFlags returns the go tool flags for building. 287 func buildFlags(env build.Environment) (flags []string) { 288 var ld []string 289 if env.Commit != "" { 290 ld = append(ld, "-X", "main.gitCommit="+env.Commit) 291 ld = append(ld, "-X", "main.gitDate="+env.Date) 292 } 293 // Strip DWARF on darwin. This used to be required for certain things, 294 // and there is no downside to this, so we just keep doing it. 295 if runtime.GOOS == "darwin" { 296 ld = append(ld, "-s") 297 } 298 if len(ld) > 0 { 299 flags = append(flags, "-ldflags", strings.Join(ld, " ")) 300 } 301 return flags 302 } 303 304 // goTool returns the go tool. This uses the Go version which runs ci.go. 305 func goTool(subcmd string, args ...string) *exec.Cmd { 306 cmd := build.GoTool(subcmd, args...) 307 goToolSetEnv(cmd) 308 return cmd 309 } 310 311 // localGoTool returns the go tool from the given GOROOT. 312 func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd { 313 gotool := filepath.Join(goroot, "bin", "go") 314 cmd := exec.Command(gotool, subcmd) 315 goToolSetEnv(cmd) 316 cmd.Env = append(cmd.Env, "GOROOT="+goroot) 317 cmd.Args = append(cmd.Args, args...) 318 return cmd 319 } 320 321 // goToolSetEnv forwards the build environment to the go tool. 322 func goToolSetEnv(cmd *exec.Cmd) { 323 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 324 for _, e := range os.Environ() { 325 if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") { 326 continue 327 } 328 cmd.Env = append(cmd.Env, e) 329 } 330 } 331 332 // Running The Tests 333 // 334 // "tests" also includes static analysis tools such as vet. 335 336 func doTest(cmdline []string) { 337 coverage := flag.Bool("coverage", false, "Whether to record code coverage") 338 verbose := flag.Bool("v", false, "Whether to log verbosely") 339 flag.CommandLine.Parse(cmdline) 340 env := build.Env() 341 342 packages := []string{"./..."} 343 if len(flag.CommandLine.Args()) > 0 { 344 packages = flag.CommandLine.Args() 345 } 346 347 // Run the actual tests. 348 // Test a single package at a time. CI builders are slow 349 // and some tests run into timeouts under load. 350 gotest := goTool("test", buildFlags(env)...) 351 gotest.Args = append(gotest.Args, "-p", "1") 352 if *coverage { 353 gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") 354 } 355 if *verbose { 356 gotest.Args = append(gotest.Args, "-v") 357 } 358 359 gotest.Args = append(gotest.Args, packages...) 360 build.MustRun(gotest) 361 } 362 363 // doLint runs golangci-lint on requested packages. 364 func doLint(cmdline []string) { 365 var ( 366 cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") 367 ) 368 flag.CommandLine.Parse(cmdline) 369 packages := []string{"./..."} 370 if len(flag.CommandLine.Args()) > 0 { 371 packages = flag.CommandLine.Args() 372 } 373 374 linter := downloadLinter(*cachedir) 375 lflags := []string{"run", "--config", ".golangci.yml"} 376 build.MustRunCommand(linter, append(lflags, packages...)...) 377 fmt.Println("You have achieved perfection.") 378 } 379 380 // downloadLinter downloads and unpacks golangci-lint. 381 func downloadLinter(cachedir string) string { 382 const version = "1.27.0" 383 384 csdb := build.MustLoadChecksums("build/checksums.txt") 385 base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) 386 url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) 387 archivePath := filepath.Join(cachedir, base+".tar.gz") 388 if err := csdb.DownloadFile(url, archivePath); err != nil { 389 log.Fatal(err) 390 } 391 if err := build.ExtractArchive(archivePath, cachedir); err != nil { 392 log.Fatal(err) 393 } 394 return filepath.Join(cachedir, base, "golangci-lint") 395 } 396 397 // Release Packaging 398 func doArchive(cmdline []string) { 399 var ( 400 arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") 401 atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") 402 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) 403 signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`) 404 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 405 ext string 406 ) 407 flag.CommandLine.Parse(cmdline) 408 switch *atype { 409 case "zip": 410 ext = ".zip" 411 case "tar": 412 ext = ".tar.gz" 413 default: 414 log.Fatal("unknown archive type: ", atype) 415 } 416 417 var ( 418 env = build.Env() 419 420 basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) 421 geth = "geth-" + basegeth + ext 422 alltools = "geth-alltools-" + basegeth + ext 423 ) 424 maybeSkipArchive(env) 425 if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { 426 log.Fatal(err) 427 } 428 if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { 429 log.Fatal(err) 430 } 431 for _, archive := range []string{geth, alltools} { 432 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 433 log.Fatal(err) 434 } 435 } 436 } 437 438 func archiveBasename(arch string, archiveVersion string) string { 439 platform := runtime.GOOS + "-" + arch 440 if arch == "arm" { 441 platform += os.Getenv("GOARM") 442 } 443 if arch == "android" { 444 platform = "android-all" 445 } 446 if arch == "ios" { 447 platform = "ios-all" 448 } 449 return platform + "-" + archiveVersion 450 } 451 452 func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error { 453 // If signing was requested, generate the signature files 454 if signer != "" { 455 key := getenvBase64(signer) 456 if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { 457 return err 458 } 459 } 460 if signifyVar != "" { 461 key := os.Getenv(signifyVar) 462 untrustedComment := "verify with geth-release.pub" 463 trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123)) 464 if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil { 465 return err 466 } 467 } 468 // If uploading to Azure was requested, push the archive possibly with its signature 469 if blobstore != "" { 470 auth := build.AzureBlobstoreConfig{ 471 Account: strings.Split(blobstore, "/")[0], 472 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 473 Container: strings.SplitN(blobstore, "/", 2)[1], 474 } 475 if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { 476 return err 477 } 478 if signer != "" { 479 if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { 480 return err 481 } 482 } 483 if signifyVar != "" { 484 if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil { 485 return err 486 } 487 } 488 } 489 return nil 490 } 491 492 // skips archiving for some build configurations. 493 func maybeSkipArchive(env build.Environment) { 494 if env.IsPullRequest { 495 log.Printf("skipping because this is a PR build") 496 os.Exit(0) 497 } 498 if env.IsCronJob { 499 log.Printf("skipping because this is a cron job") 500 os.Exit(0) 501 } 502 if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { 503 log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) 504 os.Exit(0) 505 } 506 } 507 508 // Debian Packaging 509 func doDebianSource(cmdline []string) { 510 var ( 511 cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) 512 signer = flag.String("signer", "", `Signing key name, also used as package author`) 513 upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) 514 sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) 515 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 516 now = time.Now() 517 ) 518 flag.CommandLine.Parse(cmdline) 519 *workdir = makeWorkdir(*workdir) 520 env := build.Env() 521 maybeSkipArchive(env) 522 523 // Import the signing key. 524 if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { 525 gpg := exec.Command("gpg", "--import") 526 gpg.Stdin = bytes.NewReader(key) 527 build.MustRun(gpg) 528 } 529 530 // Download and verify the Go source package. 531 gobundle := downloadGoSources(*cachedir) 532 533 // Download all the dependencies needed to build the sources and run the ci script 534 srcdepfetch := goTool("mod", "download") 535 srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) 536 build.MustRun(srcdepfetch) 537 538 cidepfetch := goTool("run", "./build/ci.go") 539 cidepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) 540 cidepfetch.Run() // Command fails, don't care, we only need the deps to start it 541 542 // Create Debian packages and upload them. 543 for _, pkg := range debPackages { 544 for distro, goboot := range debDistroGoBoots { 545 // Prepare the debian package with the go-ethereum sources. 546 meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) 547 pkgdir := stageDebianSource(*workdir, meta) 548 549 // Add Go source code 550 if err := build.ExtractArchive(gobundle, pkgdir); err != nil { 551 log.Fatalf("Failed to extract Go sources: %v", err) 552 } 553 if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { 554 log.Fatalf("Failed to rename Go source folder: %v", err) 555 } 556 // Add all dependency modules in compressed form 557 os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) 558 if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { 559 log.Fatalf("Failed to copy Go module dependencies: %v", err) 560 } 561 // Run the packaging and upload to the PPA 562 debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") 563 debuild.Dir = pkgdir 564 build.MustRun(debuild) 565 566 var ( 567 basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) 568 source = filepath.Join(*workdir, basename+".tar.xz") 569 dsc = filepath.Join(*workdir, basename+".dsc") 570 changes = filepath.Join(*workdir, basename+"_source.changes") 571 ) 572 if *signer != "" { 573 build.MustRunCommand("debsign", changes) 574 } 575 if *upload != "" { 576 ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) 577 } 578 } 579 } 580 } 581 582 // downloadGoSources downloads the Go source tarball. 583 func downloadGoSources(cachedir string) string { 584 csdb := build.MustLoadChecksums("build/checksums.txt") 585 file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) 586 url := "https://dl.google.com/go/" + file 587 dst := filepath.Join(cachedir, file) 588 if err := csdb.DownloadFile(url, dst); err != nil { 589 log.Fatal(err) 590 } 591 return dst 592 } 593 594 // downloadGo downloads the Go binary distribution and unpacks it into a temporary 595 // directory. It returns the GOROOT of the unpacked toolchain. 596 func downloadGo(goarch, goos, cachedir string) string { 597 if goarch == "arm" { 598 goarch = "armv6l" 599 } 600 601 csdb := build.MustLoadChecksums("build/checksums.txt") 602 file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch) 603 if goos == "windows" { 604 file += ".zip" 605 } else { 606 file += ".tar.gz" 607 } 608 url := "https://golang.org/dl/" + file 609 dst := filepath.Join(cachedir, file) 610 if err := csdb.DownloadFile(url, dst); err != nil { 611 log.Fatal(err) 612 } 613 614 ucache, err := os.UserCacheDir() 615 if err != nil { 616 log.Fatal(err) 617 } 618 godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch)) 619 if err := build.ExtractArchive(dst, godir); err != nil { 620 log.Fatal(err) 621 } 622 goroot, err := filepath.Abs(filepath.Join(godir, "go")) 623 if err != nil { 624 log.Fatal(err) 625 } 626 return goroot 627 } 628 629 func ppaUpload(workdir, ppa, sshUser string, files []string) { 630 p := strings.Split(ppa, "/") 631 if len(p) != 2 { 632 log.Fatal("-upload PPA name must contain single /") 633 } 634 if sshUser == "" { 635 sshUser = p[0] 636 } 637 incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) 638 // Create the SSH identity file if it doesn't exist. 639 var idfile string 640 if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { 641 idfile = filepath.Join(workdir, "sshkey") 642 if _, err := os.Stat(idfile); os.IsNotExist(err) { 643 ioutil.WriteFile(idfile, sshkey, 0600) 644 } 645 } 646 // Upload 647 dest := sshUser + "@ppa.launchpad.net" 648 if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { 649 log.Fatal(err) 650 } 651 } 652 653 func getenvBase64(variable string) []byte { 654 dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) 655 if err != nil { 656 log.Fatal("invalid base64 " + variable) 657 } 658 return []byte(dec) 659 } 660 661 func makeWorkdir(wdflag string) string { 662 var err error 663 if wdflag != "" { 664 err = os.MkdirAll(wdflag, 0744) 665 } else { 666 wdflag, err = ioutil.TempDir("", "geth-build-") 667 } 668 if err != nil { 669 log.Fatal(err) 670 } 671 return wdflag 672 } 673 674 func isUnstableBuild(env build.Environment) bool { 675 if env.Tag != "" { 676 return false 677 } 678 return true 679 } 680 681 type debPackage struct { 682 Name string // the name of the Debian package to produce, e.g. "ethereum" 683 Version string // the clean version of the debPackage, e.g. 1.8.12, without any metadata 684 Executables []debExecutable // executables to be included in the package 685 } 686 687 type debMetadata struct { 688 Env build.Environment 689 GoBootPackage string 690 GoBootPath string 691 692 PackageName string 693 694 // go-ethereum version being built. Note that this 695 // is not the debian package version. The package version 696 // is constructed by VersionString. 697 Version string 698 699 Author string // "name <email>", also selects signing key 700 Distro, Time string 701 Executables []debExecutable 702 } 703 704 type debExecutable struct { 705 PackageName string 706 BinaryName string 707 Description string 708 } 709 710 // Package returns the name of the package if present, or 711 // fallbacks to BinaryName 712 func (d debExecutable) Package() string { 713 if d.PackageName != "" { 714 return d.PackageName 715 } 716 return d.BinaryName 717 } 718 719 func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { 720 if author == "" { 721 // No signing key, use default author. 722 author = "Ethereum Builds <fjl@ethereum.org>" 723 } 724 return debMetadata{ 725 GoBootPackage: goboot, 726 GoBootPath: debGoBootPaths[goboot], 727 PackageName: name, 728 Env: env, 729 Author: author, 730 Distro: distro, 731 Version: version, 732 Time: t.Format(time.RFC1123Z), 733 Executables: exes, 734 } 735 } 736 737 // Name returns the name of the metapackage that depends 738 // on all executable packages. 739 func (meta debMetadata) Name() string { 740 if isUnstableBuild(meta.Env) { 741 return meta.PackageName + "-unstable" 742 } 743 return meta.PackageName 744 } 745 746 // VersionString returns the debian version of the packages. 747 func (meta debMetadata) VersionString() string { 748 vsn := meta.Version 749 if meta.Env.Buildnum != "" { 750 vsn += "+build" + meta.Env.Buildnum 751 } 752 if meta.Distro != "" { 753 vsn += "+" + meta.Distro 754 } 755 return vsn 756 } 757 758 // ExeList returns the list of all executable packages. 759 func (meta debMetadata) ExeList() string { 760 names := make([]string, len(meta.Executables)) 761 for i, e := range meta.Executables { 762 names[i] = meta.ExeName(e) 763 } 764 return strings.Join(names, ", ") 765 } 766 767 // ExeName returns the package name of an executable package. 768 func (meta debMetadata) ExeName(exe debExecutable) string { 769 if isUnstableBuild(meta.Env) { 770 return exe.Package() + "-unstable" 771 } 772 return exe.Package() 773 } 774 775 // ExeConflicts returns the content of the Conflicts field 776 // for executable packages. 777 func (meta debMetadata) ExeConflicts(exe debExecutable) string { 778 if isUnstableBuild(meta.Env) { 779 // Set up the conflicts list so that the *-unstable packages 780 // cannot be installed alongside the regular version. 781 // 782 // https://www.debian.org/doc/debian-policy/ch-relationships.html 783 // is very explicit about Conflicts: and says that Breaks: should 784 // be preferred and the conflicting files should be handled via 785 // alternates. We might do this eventually but using a conflict is 786 // easier now. 787 return "ethereum, " + exe.Package() 788 } 789 return "" 790 } 791 792 func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { 793 pkg := meta.Name() + "-" + meta.VersionString() 794 pkgdir = filepath.Join(tmpdir, pkg) 795 if err := os.Mkdir(pkgdir, 0755); err != nil { 796 log.Fatal(err) 797 } 798 // Copy the source code. 799 build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) 800 801 // Put the debian build files in place. 802 debian := filepath.Join(pkgdir, "debian") 803 build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta) 804 build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) 805 build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta) 806 build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) 807 build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) 808 build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) 809 for _, exe := range meta.Executables { 810 install := filepath.Join(debian, meta.ExeName(exe)+".install") 811 docs := filepath.Join(debian, meta.ExeName(exe)+".docs") 812 build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) 813 build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) 814 } 815 return pkgdir 816 } 817 818 // Windows installer 819 func doWindowsInstaller(cmdline []string) { 820 // Parse the flags and make skip installer generation on PRs 821 var ( 822 arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") 823 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) 824 signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) 825 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 826 workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) 827 ) 828 flag.CommandLine.Parse(cmdline) 829 *workdir = makeWorkdir(*workdir) 830 env := build.Env() 831 maybeSkipArchive(env) 832 833 // Aggregate binaries that are included in the installer 834 var ( 835 devTools []string 836 allTools []string 837 gethTool string 838 ) 839 for _, file := range allToolsArchiveFiles { 840 if file == "COPYING" { // license, copied later 841 continue 842 } 843 allTools = append(allTools, filepath.Base(file)) 844 if filepath.Base(file) == "geth.exe" { 845 gethTool = file 846 } else { 847 devTools = append(devTools, file) 848 } 849 } 850 851 // Render NSIS scripts: Installer NSIS contains two installer sections, 852 // first section contains the geth binary, second section holds the dev tools. 853 templateData := map[string]interface{}{ 854 "License": "COPYING", 855 "Geth": gethTool, 856 "DevTools": devTools, 857 } 858 build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) 859 build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) 860 build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) 861 build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) 862 build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) 863 if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { 864 log.Fatal("Failed to copy SimpleFC.dll: %v", err) 865 } 866 if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { 867 log.Fatal("Failed to copy copyright note: %v", err) 868 } 869 // Build the installer. This assumes that all the needed files have been previously 870 // built (don't mix building and packaging to keep cross compilation complexity to a 871 // minimum). 872 version := strings.Split(params.Version, ".") 873 if env.Commit != "" { 874 version[2] += "-" + env.Commit[:8] 875 } 876 installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") 877 build.MustRunCommand("makensis.exe", 878 "/DOUTPUTFILE="+installer, 879 "/DMAJORVERSION="+version[0], 880 "/DMINORVERSION="+version[1], 881 "/DBUILDVERSION="+version[2], 882 "/DARCH="+*arch, 883 filepath.Join(*workdir, "geth.nsi"), 884 ) 885 // Sign and publish installer. 886 if err := archiveUpload(installer, *upload, *signer, *signify); err != nil { 887 log.Fatal(err) 888 } 889 } 890 891 // Android archives 892 893 func doAndroidArchive(cmdline []string) { 894 var ( 895 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 896 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) 897 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`) 898 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) 899 upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) 900 ) 901 flag.CommandLine.Parse(cmdline) 902 env := build.Env() 903 904 // Sanity check that the SDK and NDK are installed and set 905 if os.Getenv("ANDROID_HOME") == "" { 906 log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") 907 } 908 // Build the Android archive and Maven resources 909 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 910 build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/CommerciumBlockchain/go-commercium/mobile")) 911 912 if *local { 913 // If we're building locally, copy bundle to build dir and skip Maven 914 os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) 915 os.Rename("geth-sources.jar", filepath.Join(GOBIN, "geth-sources.jar")) 916 return 917 } 918 meta := newMavenMetadata(env) 919 build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) 920 921 // Skip Maven deploy and Azure upload for PR builds 922 maybeSkipArchive(env) 923 924 // Sign and upload the archive to Azure 925 archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" 926 os.Rename("geth.aar", archive) 927 928 if err := archiveUpload(archive, *upload, *signer, *signify); err != nil { 929 log.Fatal(err) 930 } 931 // Sign and upload all the artifacts to Maven Central 932 os.Rename(archive, meta.Package+".aar") 933 if *signer != "" && *deploy != "" { 934 // Import the signing key into the local GPG instance 935 key := getenvBase64(*signer) 936 gpg := exec.Command("gpg", "--import") 937 gpg.Stdin = bytes.NewReader(key) 938 build.MustRun(gpg) 939 keyID, err := build.PGPKeyID(string(key)) 940 if err != nil { 941 log.Fatal(err) 942 } 943 // Upload the artifacts to Sonatype and/or Maven Central 944 repo := *deploy + "/service/local/staging/deploy/maven2" 945 if meta.Develop { 946 repo = *deploy + "/content/repositories/snapshots" 947 } 948 build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", 949 "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", 950 "-Dgpg.keyname="+keyID, 951 "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") 952 } 953 } 954 955 func gomobileTool(subcmd string, args ...string) *exec.Cmd { 956 cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) 957 cmd.Args = append(cmd.Args, args...) 958 cmd.Env = []string{ 959 "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), 960 } 961 for _, e := range os.Environ() { 962 if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") || strings.HasPrefix(e, "GOBIN=") { 963 continue 964 } 965 cmd.Env = append(cmd.Env, e) 966 } 967 cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) 968 return cmd 969 } 970 971 type mavenMetadata struct { 972 Version string 973 Package string 974 Develop bool 975 Contributors []mavenContributor 976 } 977 978 type mavenContributor struct { 979 Name string 980 Email string 981 } 982 983 func newMavenMetadata(env build.Environment) mavenMetadata { 984 // Collect the list of authors from the repo root 985 contribs := []mavenContributor{} 986 if authors, err := os.Open("AUTHORS"); err == nil { 987 defer authors.Close() 988 989 scanner := bufio.NewScanner(authors) 990 for scanner.Scan() { 991 // Skip any whitespace from the authors list 992 line := strings.TrimSpace(scanner.Text()) 993 if line == "" || line[0] == '#' { 994 continue 995 } 996 // Split the author and insert as a contributor 997 re := regexp.MustCompile("([^<]+) <(.+)>") 998 parts := re.FindStringSubmatch(line) 999 if len(parts) == 3 { 1000 contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) 1001 } 1002 } 1003 } 1004 // Render the version and package strings 1005 version := params.Version 1006 if isUnstableBuild(env) { 1007 version += "-SNAPSHOT" 1008 } 1009 return mavenMetadata{ 1010 Version: version, 1011 Package: "geth-" + version, 1012 Develop: isUnstableBuild(env), 1013 Contributors: contribs, 1014 } 1015 } 1016 1017 // XCode frameworks 1018 1019 func doXCodeFramework(cmdline []string) { 1020 var ( 1021 local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) 1022 signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) 1023 signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`) 1024 deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) 1025 upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) 1026 ) 1027 flag.CommandLine.Parse(cmdline) 1028 env := build.Env() 1029 1030 // Build the iOS XCode framework 1031 build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) 1032 build.MustRun(gomobileTool("init")) 1033 bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/CommerciumBlockchain/go-commercium/mobile") 1034 1035 if *local { 1036 // If we're building locally, use the build folder and stop afterwards 1037 bind.Dir = GOBIN 1038 build.MustRun(bind) 1039 return 1040 } 1041 archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) 1042 if err := os.Mkdir(archive, os.ModePerm); err != nil { 1043 log.Fatal(err) 1044 } 1045 bind.Dir, _ = filepath.Abs(archive) 1046 build.MustRun(bind) 1047 build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) 1048 1049 // Skip CocoaPods deploy and Azure upload for PR builds 1050 maybeSkipArchive(env) 1051 1052 // Sign and upload the framework to Azure 1053 if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil { 1054 log.Fatal(err) 1055 } 1056 // Prepare and upload a PodSpec to CocoaPods 1057 if *deploy != "" { 1058 meta := newPodMetadata(env, archive) 1059 build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) 1060 build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings") 1061 } 1062 } 1063 1064 type podMetadata struct { 1065 Version string 1066 Commit string 1067 Archive string 1068 Contributors []podContributor 1069 } 1070 1071 type podContributor struct { 1072 Name string 1073 Email string 1074 } 1075 1076 func newPodMetadata(env build.Environment, archive string) podMetadata { 1077 // Collect the list of authors from the repo root 1078 contribs := []podContributor{} 1079 if authors, err := os.Open("AUTHORS"); err == nil { 1080 defer authors.Close() 1081 1082 scanner := bufio.NewScanner(authors) 1083 for scanner.Scan() { 1084 // Skip any whitespace from the authors list 1085 line := strings.TrimSpace(scanner.Text()) 1086 if line == "" || line[0] == '#' { 1087 continue 1088 } 1089 // Split the author and insert as a contributor 1090 re := regexp.MustCompile("([^<]+) <(.+)>") 1091 parts := re.FindStringSubmatch(line) 1092 if len(parts) == 3 { 1093 contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) 1094 } 1095 } 1096 } 1097 version := params.Version 1098 if isUnstableBuild(env) { 1099 version += "-unstable." + env.Buildnum 1100 } 1101 return podMetadata{ 1102 Archive: archive, 1103 Version: version, 1104 Commit: env.Commit, 1105 Contributors: contribs, 1106 } 1107 } 1108 1109 // Cross compilation 1110 1111 func doXgo(cmdline []string) { 1112 var ( 1113 alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) 1114 ) 1115 flag.CommandLine.Parse(cmdline) 1116 env := build.Env() 1117 1118 // Make sure xgo is available for cross compilation 1119 gogetxgo := goTool("get", "github.com/karalabe/xgo") 1120 build.MustRun(gogetxgo) 1121 1122 // If all tools building is requested, build everything the builder wants 1123 args := append(buildFlags(env), flag.Args()...) 1124 1125 if *alltools { 1126 args = append(args, []string{"--dest", GOBIN}...) 1127 for _, res := range allToolsArchiveFiles { 1128 if strings.HasPrefix(res, GOBIN) { 1129 // Binary tool found, cross build it explicitly 1130 args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) 1131 xgo := xgoTool(args) 1132 build.MustRun(xgo) 1133 args = args[:len(args)-1] 1134 } 1135 } 1136 return 1137 } 1138 // Otherwise xxecute the explicit cross compilation 1139 path := args[len(args)-1] 1140 args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...) 1141 1142 xgo := xgoTool(args) 1143 build.MustRun(xgo) 1144 } 1145 1146 func xgoTool(args []string) *exec.Cmd { 1147 cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) 1148 cmd.Env = os.Environ() 1149 cmd.Env = append(cmd.Env, []string{ 1150 "GOBIN=" + GOBIN, 1151 }...) 1152 return cmd 1153 } 1154 1155 // Binary distribution cleanups 1156 1157 func doPurge(cmdline []string) { 1158 var ( 1159 store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) 1160 limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) 1161 ) 1162 flag.CommandLine.Parse(cmdline) 1163 1164 if env := build.Env(); !env.IsCronJob { 1165 log.Printf("skipping because not a cron job") 1166 os.Exit(0) 1167 } 1168 // Create the azure authentication and list the current archives 1169 auth := build.AzureBlobstoreConfig{ 1170 Account: strings.Split(*store, "/")[0], 1171 Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), 1172 Container: strings.SplitN(*store, "/", 2)[1], 1173 } 1174 blobs, err := build.AzureBlobstoreList(auth) 1175 if err != nil { 1176 log.Fatal(err) 1177 } 1178 fmt.Printf("Found %d blobs\n", len(blobs)) 1179 1180 // Iterate over the blobs, collect and sort all unstable builds 1181 for i := 0; i < len(blobs); i++ { 1182 if !strings.Contains(blobs[i].Name, "unstable") { 1183 blobs = append(blobs[:i], blobs[i+1:]...) 1184 i-- 1185 } 1186 } 1187 for i := 0; i < len(blobs); i++ { 1188 for j := i + 1; j < len(blobs); j++ { 1189 if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { 1190 blobs[i], blobs[j] = blobs[j], blobs[i] 1191 } 1192 } 1193 } 1194 // Filter out all archives more recent that the given threshold 1195 for i, blob := range blobs { 1196 if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { 1197 blobs = blobs[:i] 1198 break 1199 } 1200 } 1201 fmt.Printf("Deleting %d blobs\n", len(blobs)) 1202 // Delete all marked as such and return 1203 if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { 1204 log.Fatal(err) 1205 } 1206 }