github.com/tuotoo/go-ethereum@v1.7.4-0.20171121184211-049797d40a24/build/ci.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // +build none
    18  
    19  /*
    20  The ci command is called from Continuous Integration scripts.
    21  
    22  Usage: go run ci.go <command> <command flags/arguments>
    23  
    24  Available commands are:
    25  
    26     install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
    27     test       [ -coverage ] [ packages... ]                                                    -- runs the tests
    28     lint                                                                                        -- runs certain pre-selected linters
    29     archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    30     importkeys                                                                                  -- imports signing keys from env
    31     debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
    32     nsis                                                                                        -- creates a Windows NSIS installer
    33     aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
    34     xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
    35     xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
    36     purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
    37  
    38  For all commands, -n prevents execution of external programs (dry run mode).
    39  
    40  */
    41  package main
    42  
    43  import (
    44  	"bufio"
    45  	"bytes"
    46  	"encoding/base64"
    47  	"flag"
    48  	"fmt"
    49  	"go/parser"
    50  	"go/token"
    51  	"io/ioutil"
    52  	"log"
    53  	"os"
    54  	"os/exec"
    55  	"path/filepath"
    56  	"regexp"
    57  	"runtime"
    58  	"strings"
    59  	"time"
    60  
    61  	"github.com/ethereum/go-ethereum/internal/build"
    62  )
    63  
    64  var (
    65  	// Files that end up in the geth*.zip archive.
    66  	gethArchiveFiles = []string{
    67  		"COPYING",
    68  		executablePath("geth"),
    69  	}
    70  
    71  	// Files that end up in the geth-alltools*.zip archive.
    72  	allToolsArchiveFiles = []string{
    73  		"COPYING",
    74  		executablePath("abigen"),
    75  		executablePath("bootnode"),
    76  		executablePath("evm"),
    77  		executablePath("geth"),
    78  		executablePath("puppeth"),
    79  		executablePath("rlpdump"),
    80  		executablePath("swarm"),
    81  		executablePath("wnode"),
    82  	}
    83  
    84  	// A debian package is created for all executables listed here.
    85  	debExecutables = []debExecutable{
    86  		{
    87  			Name:        "abigen",
    88  			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
    89  		},
    90  		{
    91  			Name:        "bootnode",
    92  			Description: "Ethereum bootnode.",
    93  		},
    94  		{
    95  			Name:        "evm",
    96  			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
    97  		},
    98  		{
    99  			Name:        "geth",
   100  			Description: "Ethereum CLI client.",
   101  		},
   102  		{
   103  			Name:        "puppeth",
   104  			Description: "Ethereum private network manager.",
   105  		},
   106  		{
   107  			Name:        "rlpdump",
   108  			Description: "Developer utility tool that prints RLP structures.",
   109  		},
   110  		{
   111  			Name:        "swarm",
   112  			Description: "Ethereum Swarm daemon and tools",
   113  		},
   114  		{
   115  			Name:        "wnode",
   116  			Description: "Ethereum Whisper diagnostic tool",
   117  		},
   118  	}
   119  
   120  	// Distros for which packages are created.
   121  	// Note: vivid is unsupported because there is no golang-1.6 package for it.
   122  	// Note: wily is unsupported because it was officially deprecated on lanchpad.
   123  	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
   124  	debDistros = []string{"trusty", "xenial", "zesty", "artful"}
   125  )
   126  
   127  var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
   128  
   129  func executablePath(name string) string {
   130  	if runtime.GOOS == "windows" {
   131  		name += ".exe"
   132  	}
   133  	return filepath.Join(GOBIN, name)
   134  }
   135  
   136  func main() {
   137  	log.SetFlags(log.Lshortfile)
   138  
   139  	if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
   140  		log.Fatal("this script must be run from the root of the repository")
   141  	}
   142  	if len(os.Args) < 2 {
   143  		log.Fatal("need subcommand as first argument")
   144  	}
   145  	switch os.Args[1] {
   146  	case "install":
   147  		doInstall(os.Args[2:])
   148  	case "test":
   149  		doTest(os.Args[2:])
   150  	case "lint":
   151  		doLint(os.Args[2:])
   152  	case "archive":
   153  		doArchive(os.Args[2:])
   154  	case "debsrc":
   155  		doDebianSource(os.Args[2:])
   156  	case "nsis":
   157  		doWindowsInstaller(os.Args[2:])
   158  	case "aar":
   159  		doAndroidArchive(os.Args[2:])
   160  	case "xcode":
   161  		doXCodeFramework(os.Args[2:])
   162  	case "xgo":
   163  		doXgo(os.Args[2:])
   164  	case "purge":
   165  		doPurge(os.Args[2:])
   166  	default:
   167  		log.Fatal("unknown command ", os.Args[1])
   168  	}
   169  }
   170  
   171  // Compiling
   172  
   173  func doInstall(cmdline []string) {
   174  	var (
   175  		arch = flag.String("arch", "", "Architecture to cross build for")
   176  	)
   177  	flag.CommandLine.Parse(cmdline)
   178  	env := build.Env()
   179  
   180  	// Check Go version. People regularly open issues about compilation
   181  	// failure with outdated Go. This should save them the trouble.
   182  	if runtime.Version() < "go1.7" && !strings.Contains(runtime.Version(), "devel") {
   183  		log.Println("You have Go version", runtime.Version())
   184  		log.Println("go-ethereum requires at least Go version 1.7 and cannot")
   185  		log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
   186  		os.Exit(1)
   187  	}
   188  	// Compile packages given as arguments, or everything if there are no arguments.
   189  	packages := []string{"./..."}
   190  	if flag.NArg() > 0 {
   191  		packages = flag.Args()
   192  	}
   193  	packages = build.ExpandPackagesNoVendor(packages)
   194  
   195  	if *arch == "" || *arch == runtime.GOARCH {
   196  		goinstall := goTool("install", buildFlags(env)...)
   197  		goinstall.Args = append(goinstall.Args, "-v")
   198  		goinstall.Args = append(goinstall.Args, packages...)
   199  		build.MustRun(goinstall)
   200  		return
   201  	}
   202  	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any prvious builds
   203  	if *arch == "arm" {
   204  		os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
   205  		for _, path := range filepath.SplitList(build.GOPATH()) {
   206  			os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
   207  		}
   208  	}
   209  	// Seems we are cross compiling, work around forbidden GOBIN
   210  	goinstall := goToolArch(*arch, "install", buildFlags(env)...)
   211  	goinstall.Args = append(goinstall.Args, "-v")
   212  	goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
   213  	goinstall.Args = append(goinstall.Args, packages...)
   214  	build.MustRun(goinstall)
   215  
   216  	if cmds, err := ioutil.ReadDir("cmd"); err == nil {
   217  		for _, cmd := range cmds {
   218  			pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
   219  			if err != nil {
   220  				log.Fatal(err)
   221  			}
   222  			for name := range pkgs {
   223  				if name == "main" {
   224  					gobuild := goToolArch(*arch, "build", buildFlags(env)...)
   225  					gobuild.Args = append(gobuild.Args, "-v")
   226  					gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
   227  					gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
   228  					build.MustRun(gobuild)
   229  					break
   230  				}
   231  			}
   232  		}
   233  	}
   234  }
   235  
   236  func buildFlags(env build.Environment) (flags []string) {
   237  	var ld []string
   238  	if env.Commit != "" {
   239  		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
   240  	}
   241  	if runtime.GOOS == "darwin" {
   242  		ld = append(ld, "-s")
   243  	}
   244  
   245  	if len(ld) > 0 {
   246  		flags = append(flags, "-ldflags", strings.Join(ld, " "))
   247  	}
   248  	return flags
   249  }
   250  
   251  func goTool(subcmd string, args ...string) *exec.Cmd {
   252  	return goToolArch(runtime.GOARCH, subcmd, args...)
   253  }
   254  
   255  func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
   256  	cmd := build.GoTool(subcmd, args...)
   257  	if subcmd == "build" || subcmd == "install" || subcmd == "test" {
   258  		// Go CGO has a Windows linker error prior to 1.8 (https://github.com/golang/go/issues/8756).
   259  		// Work around issue by allowing multiple definitions for <1.8 builds.
   260  		if runtime.GOOS == "windows" && runtime.Version() < "go1.8" {
   261  			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
   262  		}
   263  	}
   264  	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
   265  	if arch == "" || arch == runtime.GOARCH {
   266  		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
   267  	} else {
   268  		cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
   269  		cmd.Env = append(cmd.Env, "GOARCH="+arch)
   270  	}
   271  	for _, e := range os.Environ() {
   272  		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
   273  			continue
   274  		}
   275  		cmd.Env = append(cmd.Env, e)
   276  	}
   277  	return cmd
   278  }
   279  
   280  // Running The Tests
   281  //
   282  // "tests" also includes static analysis tools such as vet.
   283  
   284  func doTest(cmdline []string) {
   285  	var (
   286  		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
   287  	)
   288  	flag.CommandLine.Parse(cmdline)
   289  	env := build.Env()
   290  
   291  	packages := []string{"./..."}
   292  	if len(flag.CommandLine.Args()) > 0 {
   293  		packages = flag.CommandLine.Args()
   294  	}
   295  	packages = build.ExpandPackagesNoVendor(packages)
   296  
   297  	// Run analysis tools before the tests.
   298  	build.MustRun(goTool("vet", packages...))
   299  
   300  	// Run the actual tests.
   301  	gotest := goTool("test", buildFlags(env)...)
   302  	// Test a single package at a time. CI builders are slow
   303  	// and some tests run into timeouts under load.
   304  	gotest.Args = append(gotest.Args, "-p", "1")
   305  	if *coverage {
   306  		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
   307  	}
   308  
   309  	gotest.Args = append(gotest.Args, packages...)
   310  	build.MustRun(gotest)
   311  }
   312  
   313  // runs gometalinter on requested packages
   314  func doLint(cmdline []string) {
   315  	flag.CommandLine.Parse(cmdline)
   316  
   317  	packages := []string{"./..."}
   318  	if len(flag.CommandLine.Args()) > 0 {
   319  		packages = flag.CommandLine.Args()
   320  	}
   321  	// Get metalinter and install all supported linters
   322  	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v1"))
   323  	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), "--install")
   324  
   325  	// Run fast linters batched together
   326  	configs := []string{"--vendor", "--disable-all", "--enable=vet", "--enable=gofmt", "--enable=misspell"}
   327  	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
   328  
   329  	// Run slow linters one by one
   330  	for _, linter := range []string{"unconvert"} {
   331  		configs = []string{"--vendor", "--deadline=10m", "--disable-all", "--enable=" + linter}
   332  		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
   333  	}
   334  }
   335  
   336  // Release Packaging
   337  
   338  func doArchive(cmdline []string) {
   339  	var (
   340  		arch   = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
   341  		atype  = flag.String("type", "zip", "Type of archive to write (zip|tar)")
   342  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
   343  		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
   344  		ext    string
   345  	)
   346  	flag.CommandLine.Parse(cmdline)
   347  	switch *atype {
   348  	case "zip":
   349  		ext = ".zip"
   350  	case "tar":
   351  		ext = ".tar.gz"
   352  	default:
   353  		log.Fatal("unknown archive type: ", atype)
   354  	}
   355  
   356  	var (
   357  		env      = build.Env()
   358  		base     = archiveBasename(*arch, env)
   359  		geth     = "geth-" + base + ext
   360  		alltools = "geth-alltools-" + base + ext
   361  	)
   362  	maybeSkipArchive(env)
   363  	if err := build.WriteArchive(geth, gethArchiveFiles); err != nil {
   364  		log.Fatal(err)
   365  	}
   366  	if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
   367  		log.Fatal(err)
   368  	}
   369  	for _, archive := range []string{geth, alltools} {
   370  		if err := archiveUpload(archive, *upload, *signer); err != nil {
   371  			log.Fatal(err)
   372  		}
   373  	}
   374  }
   375  
   376  func archiveBasename(arch string, env build.Environment) string {
   377  	platform := runtime.GOOS + "-" + arch
   378  	if arch == "arm" {
   379  		platform += os.Getenv("GOARM")
   380  	}
   381  	if arch == "android" {
   382  		platform = "android-all"
   383  	}
   384  	if arch == "ios" {
   385  		platform = "ios-all"
   386  	}
   387  	return platform + "-" + archiveVersion(env)
   388  }
   389  
   390  func archiveVersion(env build.Environment) string {
   391  	version := build.VERSION()
   392  	if isUnstableBuild(env) {
   393  		version += "-unstable"
   394  	}
   395  	if env.Commit != "" {
   396  		version += "-" + env.Commit[:8]
   397  	}
   398  	return version
   399  }
   400  
   401  func archiveUpload(archive string, blobstore string, signer string) error {
   402  	// If signing was requested, generate the signature files
   403  	if signer != "" {
   404  		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
   405  		if err != nil {
   406  			return fmt.Errorf("invalid base64 %s", signer)
   407  		}
   408  		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
   409  			return err
   410  		}
   411  	}
   412  	// If uploading to Azure was requested, push the archive possibly with its signature
   413  	if blobstore != "" {
   414  		auth := build.AzureBlobstoreConfig{
   415  			Account:   strings.Split(blobstore, "/")[0],
   416  			Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
   417  			Container: strings.SplitN(blobstore, "/", 2)[1],
   418  		}
   419  		if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil {
   420  			return err
   421  		}
   422  		if signer != "" {
   423  			if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil {
   424  				return err
   425  			}
   426  		}
   427  	}
   428  	return nil
   429  }
   430  
   431  // skips archiving for some build configurations.
   432  func maybeSkipArchive(env build.Environment) {
   433  	if env.IsPullRequest {
   434  		log.Printf("skipping because this is a PR build")
   435  		os.Exit(0)
   436  	}
   437  	if env.IsCronJob {
   438  		log.Printf("skipping because this is a cron job")
   439  		os.Exit(0)
   440  	}
   441  	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
   442  		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
   443  		os.Exit(0)
   444  	}
   445  }
   446  
   447  // Debian Packaging
   448  
   449  func doDebianSource(cmdline []string) {
   450  	var (
   451  		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
   452  		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
   453  		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
   454  		now     = time.Now()
   455  	)
   456  	flag.CommandLine.Parse(cmdline)
   457  	*workdir = makeWorkdir(*workdir)
   458  	env := build.Env()
   459  	maybeSkipArchive(env)
   460  
   461  	// Import the signing key.
   462  	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
   463  		key, err := base64.StdEncoding.DecodeString(b64key)
   464  		if err != nil {
   465  			log.Fatal("invalid base64 PPA_SIGNING_KEY")
   466  		}
   467  		gpg := exec.Command("gpg", "--import")
   468  		gpg.Stdin = bytes.NewReader(key)
   469  		build.MustRun(gpg)
   470  	}
   471  
   472  	// Create the packages.
   473  	for _, distro := range debDistros {
   474  		meta := newDebMetadata(distro, *signer, env, now)
   475  		pkgdir := stageDebianSource(*workdir, meta)
   476  		debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
   477  		debuild.Dir = pkgdir
   478  		build.MustRun(debuild)
   479  
   480  		changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
   481  		changes = filepath.Join(*workdir, changes)
   482  		if *signer != "" {
   483  			build.MustRunCommand("debsign", changes)
   484  		}
   485  		if *upload != "" {
   486  			build.MustRunCommand("dput", *upload, changes)
   487  		}
   488  	}
   489  }
   490  
   491  func makeWorkdir(wdflag string) string {
   492  	var err error
   493  	if wdflag != "" {
   494  		err = os.MkdirAll(wdflag, 0744)
   495  	} else {
   496  		wdflag, err = ioutil.TempDir("", "geth-build-")
   497  	}
   498  	if err != nil {
   499  		log.Fatal(err)
   500  	}
   501  	return wdflag
   502  }
   503  
   504  func isUnstableBuild(env build.Environment) bool {
   505  	if env.Tag != "" {
   506  		return false
   507  	}
   508  	return true
   509  }
   510  
   511  type debMetadata struct {
   512  	Env build.Environment
   513  
   514  	// go-ethereum version being built. Note that this
   515  	// is not the debian package version. The package version
   516  	// is constructed by VersionString.
   517  	Version string
   518  
   519  	Author       string // "name <email>", also selects signing key
   520  	Distro, Time string
   521  	Executables  []debExecutable
   522  }
   523  
   524  type debExecutable struct {
   525  	Name, Description string
   526  }
   527  
   528  func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
   529  	if author == "" {
   530  		// No signing key, use default author.
   531  		author = "Ethereum Builds <fjl@ethereum.org>"
   532  	}
   533  	return debMetadata{
   534  		Env:         env,
   535  		Author:      author,
   536  		Distro:      distro,
   537  		Version:     build.VERSION(),
   538  		Time:        t.Format(time.RFC1123Z),
   539  		Executables: debExecutables,
   540  	}
   541  }
   542  
   543  // Name returns the name of the metapackage that depends
   544  // on all executable packages.
   545  func (meta debMetadata) Name() string {
   546  	if isUnstableBuild(meta.Env) {
   547  		return "ethereum-unstable"
   548  	}
   549  	return "ethereum"
   550  }
   551  
   552  // VersionString returns the debian version of the packages.
   553  func (meta debMetadata) VersionString() string {
   554  	vsn := meta.Version
   555  	if meta.Env.Buildnum != "" {
   556  		vsn += "+build" + meta.Env.Buildnum
   557  	}
   558  	if meta.Distro != "" {
   559  		vsn += "+" + meta.Distro
   560  	}
   561  	return vsn
   562  }
   563  
   564  // ExeList returns the list of all executable packages.
   565  func (meta debMetadata) ExeList() string {
   566  	names := make([]string, len(meta.Executables))
   567  	for i, e := range meta.Executables {
   568  		names[i] = meta.ExeName(e)
   569  	}
   570  	return strings.Join(names, ", ")
   571  }
   572  
   573  // ExeName returns the package name of an executable package.
   574  func (meta debMetadata) ExeName(exe debExecutable) string {
   575  	if isUnstableBuild(meta.Env) {
   576  		return exe.Name + "-unstable"
   577  	}
   578  	return exe.Name
   579  }
   580  
   581  // ExeConflicts returns the content of the Conflicts field
   582  // for executable packages.
   583  func (meta debMetadata) ExeConflicts(exe debExecutable) string {
   584  	if isUnstableBuild(meta.Env) {
   585  		// Set up the conflicts list so that the *-unstable packages
   586  		// cannot be installed alongside the regular version.
   587  		//
   588  		// https://www.debian.org/doc/debian-policy/ch-relationships.html
   589  		// is very explicit about Conflicts: and says that Breaks: should
   590  		// be preferred and the conflicting files should be handled via
   591  		// alternates. We might do this eventually but using a conflict is
   592  		// easier now.
   593  		return "ethereum, " + exe.Name
   594  	}
   595  	return ""
   596  }
   597  
   598  func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
   599  	pkg := meta.Name() + "-" + meta.VersionString()
   600  	pkgdir = filepath.Join(tmpdir, pkg)
   601  	if err := os.Mkdir(pkgdir, 0755); err != nil {
   602  		log.Fatal(err)
   603  	}
   604  
   605  	// Copy the source code.
   606  	build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
   607  
   608  	// Put the debian build files in place.
   609  	debian := filepath.Join(pkgdir, "debian")
   610  	build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
   611  	build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
   612  	build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
   613  	build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
   614  	build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
   615  	build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
   616  	for _, exe := range meta.Executables {
   617  		install := filepath.Join(debian, meta.ExeName(exe)+".install")
   618  		docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
   619  		build.Render("build/deb.install", install, 0644, exe)
   620  		build.Render("build/deb.docs", docs, 0644, exe)
   621  	}
   622  
   623  	return pkgdir
   624  }
   625  
   626  // Windows installer
   627  
   628  func doWindowsInstaller(cmdline []string) {
   629  	// Parse the flags and make skip installer generation on PRs
   630  	var (
   631  		arch    = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging")
   632  		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`)
   633  		upload  = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
   634  		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
   635  	)
   636  	flag.CommandLine.Parse(cmdline)
   637  	*workdir = makeWorkdir(*workdir)
   638  	env := build.Env()
   639  	maybeSkipArchive(env)
   640  
   641  	// Aggregate binaries that are included in the installer
   642  	var (
   643  		devTools []string
   644  		allTools []string
   645  		gethTool string
   646  	)
   647  	for _, file := range allToolsArchiveFiles {
   648  		if file == "COPYING" { // license, copied later
   649  			continue
   650  		}
   651  		allTools = append(allTools, filepath.Base(file))
   652  		if filepath.Base(file) == "geth.exe" {
   653  			gethTool = file
   654  		} else {
   655  			devTools = append(devTools, file)
   656  		}
   657  	}
   658  
   659  	// Render NSIS scripts: Installer NSIS contains two installer sections,
   660  	// first section contains the geth binary, second section holds the dev tools.
   661  	templateData := map[string]interface{}{
   662  		"License":  "COPYING",
   663  		"Geth":     gethTool,
   664  		"DevTools": devTools,
   665  	}
   666  	build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil)
   667  	build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData)
   668  	build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools)
   669  	build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil)
   670  	build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil)
   671  	build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755)
   672  	build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755)
   673  
   674  	// Build the installer. This assumes that all the needed files have been previously
   675  	// built (don't mix building and packaging to keep cross compilation complexity to a
   676  	// minimum).
   677  	version := strings.Split(build.VERSION(), ".")
   678  	if env.Commit != "" {
   679  		version[2] += "-" + env.Commit[:8]
   680  	}
   681  	installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, env) + ".exe")
   682  	build.MustRunCommand("makensis.exe",
   683  		"/DOUTPUTFILE="+installer,
   684  		"/DMAJORVERSION="+version[0],
   685  		"/DMINORVERSION="+version[1],
   686  		"/DBUILDVERSION="+version[2],
   687  		"/DARCH="+*arch,
   688  		filepath.Join(*workdir, "geth.nsi"),
   689  	)
   690  
   691  	// Sign and publish installer.
   692  	if err := archiveUpload(installer, *upload, *signer); err != nil {
   693  		log.Fatal(err)
   694  	}
   695  }
   696  
   697  // Android archives
   698  
   699  func doAndroidArchive(cmdline []string) {
   700  	var (
   701  		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
   702  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
   703  		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
   704  		upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`)
   705  	)
   706  	flag.CommandLine.Parse(cmdline)
   707  	env := build.Env()
   708  
   709  	// Sanity check that the SDK and NDK are installed and set
   710  	if os.Getenv("ANDROID_HOME") == "" {
   711  		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
   712  	}
   713  	if os.Getenv("ANDROID_NDK") == "" {
   714  		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
   715  	}
   716  	// Build the Android archive and Maven resources
   717  	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
   718  	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
   719  	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
   720  
   721  	if *local {
   722  		// If we're building locally, copy bundle to build dir and skip Maven
   723  		os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar"))
   724  		return
   725  	}
   726  	meta := newMavenMetadata(env)
   727  	build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta)
   728  
   729  	// Skip Maven deploy and Azure upload for PR builds
   730  	maybeSkipArchive(env)
   731  
   732  	// Sign and upload the archive to Azure
   733  	archive := "geth-" + archiveBasename("android", env) + ".aar"
   734  	os.Rename("geth.aar", archive)
   735  
   736  	if err := archiveUpload(archive, *upload, *signer); err != nil {
   737  		log.Fatal(err)
   738  	}
   739  	// Sign and upload all the artifacts to Maven Central
   740  	os.Rename(archive, meta.Package+".aar")
   741  	if *signer != "" && *deploy != "" {
   742  		// Import the signing key into the local GPG instance
   743  		if b64key := os.Getenv(*signer); b64key != "" {
   744  			key, err := base64.StdEncoding.DecodeString(b64key)
   745  			if err != nil {
   746  				log.Fatalf("invalid base64 %s", *signer)
   747  			}
   748  			gpg := exec.Command("gpg", "--import")
   749  			gpg.Stdin = bytes.NewReader(key)
   750  			build.MustRun(gpg)
   751  		}
   752  		// Upload the artifacts to Sonatype and/or Maven Central
   753  		repo := *deploy + "/service/local/staging/deploy/maven2"
   754  		if meta.Develop {
   755  			repo = *deploy + "/content/repositories/snapshots"
   756  		}
   757  		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
   758  			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
   759  			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
   760  	}
   761  }
   762  
   763  func gomobileTool(subcmd string, args ...string) *exec.Cmd {
   764  	cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd)
   765  	cmd.Args = append(cmd.Args, args...)
   766  	cmd.Env = []string{
   767  		"GOPATH=" + build.GOPATH(),
   768  	}
   769  	for _, e := range os.Environ() {
   770  		if strings.HasPrefix(e, "GOPATH=") {
   771  			continue
   772  		}
   773  		cmd.Env = append(cmd.Env, e)
   774  	}
   775  	return cmd
   776  }
   777  
   778  type mavenMetadata struct {
   779  	Version      string
   780  	Package      string
   781  	Develop      bool
   782  	Contributors []mavenContributor
   783  }
   784  
   785  type mavenContributor struct {
   786  	Name  string
   787  	Email string
   788  }
   789  
   790  func newMavenMetadata(env build.Environment) mavenMetadata {
   791  	// Collect the list of authors from the repo root
   792  	contribs := []mavenContributor{}
   793  	if authors, err := os.Open("AUTHORS"); err == nil {
   794  		defer authors.Close()
   795  
   796  		scanner := bufio.NewScanner(authors)
   797  		for scanner.Scan() {
   798  			// Skip any whitespace from the authors list
   799  			line := strings.TrimSpace(scanner.Text())
   800  			if line == "" || line[0] == '#' {
   801  				continue
   802  			}
   803  			// Split the author and insert as a contributor
   804  			re := regexp.MustCompile("([^<]+) <(.+)>")
   805  			parts := re.FindStringSubmatch(line)
   806  			if len(parts) == 3 {
   807  				contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]})
   808  			}
   809  		}
   810  	}
   811  	// Render the version and package strings
   812  	version := build.VERSION()
   813  	if isUnstableBuild(env) {
   814  		version += "-SNAPSHOT"
   815  	}
   816  	return mavenMetadata{
   817  		Version:      version,
   818  		Package:      "geth-" + version,
   819  		Develop:      isUnstableBuild(env),
   820  		Contributors: contribs,
   821  	}
   822  }
   823  
   824  // XCode frameworks
   825  
   826  func doXCodeFramework(cmdline []string) {
   827  	var (
   828  		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
   829  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
   830  		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
   831  		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
   832  	)
   833  	flag.CommandLine.Parse(cmdline)
   834  	env := build.Env()
   835  
   836  	// Build the iOS XCode framework
   837  	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
   838  	build.MustRun(gomobileTool("init"))
   839  	bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
   840  
   841  	if *local {
   842  		// If we're building locally, use the build folder and stop afterwards
   843  		bind.Dir, _ = filepath.Abs(GOBIN)
   844  		build.MustRun(bind)
   845  		return
   846  	}
   847  	archive := "geth-" + archiveBasename("ios", env)
   848  	if err := os.Mkdir(archive, os.ModePerm); err != nil {
   849  		log.Fatal(err)
   850  	}
   851  	bind.Dir, _ = filepath.Abs(archive)
   852  	build.MustRun(bind)
   853  	build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive)
   854  
   855  	// Skip CocoaPods deploy and Azure upload for PR builds
   856  	maybeSkipArchive(env)
   857  
   858  	// Sign and upload the framework to Azure
   859  	if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil {
   860  		log.Fatal(err)
   861  	}
   862  	// Prepare and upload a PodSpec to CocoaPods
   863  	if *deploy != "" {
   864  		meta := newPodMetadata(env, archive)
   865  		build.Render("build/pod.podspec", "Geth.podspec", 0755, meta)
   866  		build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose")
   867  	}
   868  }
   869  
   870  type podMetadata struct {
   871  	Version      string
   872  	Commit       string
   873  	Archive      string
   874  	Contributors []podContributor
   875  }
   876  
   877  type podContributor struct {
   878  	Name  string
   879  	Email string
   880  }
   881  
   882  func newPodMetadata(env build.Environment, archive string) podMetadata {
   883  	// Collect the list of authors from the repo root
   884  	contribs := []podContributor{}
   885  	if authors, err := os.Open("AUTHORS"); err == nil {
   886  		defer authors.Close()
   887  
   888  		scanner := bufio.NewScanner(authors)
   889  		for scanner.Scan() {
   890  			// Skip any whitespace from the authors list
   891  			line := strings.TrimSpace(scanner.Text())
   892  			if line == "" || line[0] == '#' {
   893  				continue
   894  			}
   895  			// Split the author and insert as a contributor
   896  			re := regexp.MustCompile("([^<]+) <(.+)>")
   897  			parts := re.FindStringSubmatch(line)
   898  			if len(parts) == 3 {
   899  				contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]})
   900  			}
   901  		}
   902  	}
   903  	version := build.VERSION()
   904  	if isUnstableBuild(env) {
   905  		version += "-unstable." + env.Buildnum
   906  	}
   907  	return podMetadata{
   908  		Archive:      archive,
   909  		Version:      version,
   910  		Commit:       env.Commit,
   911  		Contributors: contribs,
   912  	}
   913  }
   914  
   915  // Cross compilation
   916  
   917  func doXgo(cmdline []string) {
   918  	var (
   919  		alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
   920  	)
   921  	flag.CommandLine.Parse(cmdline)
   922  	env := build.Env()
   923  
   924  	// Make sure xgo is available for cross compilation
   925  	gogetxgo := goTool("get", "github.com/karalabe/xgo")
   926  	build.MustRun(gogetxgo)
   927  
   928  	// If all tools building is requested, build everything the builder wants
   929  	args := append(buildFlags(env), flag.Args()...)
   930  
   931  	if *alltools {
   932  		args = append(args, []string{"--dest", GOBIN}...)
   933  		for _, res := range allToolsArchiveFiles {
   934  			if strings.HasPrefix(res, GOBIN) {
   935  				// Binary tool found, cross build it explicitly
   936  				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
   937  				xgo := xgoTool(args)
   938  				build.MustRun(xgo)
   939  				args = args[:len(args)-1]
   940  			}
   941  		}
   942  		return
   943  	}
   944  	// Otherwise xxecute the explicit cross compilation
   945  	path := args[len(args)-1]
   946  	args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
   947  
   948  	xgo := xgoTool(args)
   949  	build.MustRun(xgo)
   950  }
   951  
   952  func xgoTool(args []string) *exec.Cmd {
   953  	cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
   954  	cmd.Env = []string{
   955  		"GOPATH=" + build.GOPATH(),
   956  		"GOBIN=" + GOBIN,
   957  	}
   958  	for _, e := range os.Environ() {
   959  		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
   960  			continue
   961  		}
   962  		cmd.Env = append(cmd.Env, e)
   963  	}
   964  	return cmd
   965  }
   966  
   967  // Binary distribution cleanups
   968  
   969  func doPurge(cmdline []string) {
   970  	var (
   971  		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
   972  		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
   973  	)
   974  	flag.CommandLine.Parse(cmdline)
   975  
   976  	if env := build.Env(); !env.IsCronJob {
   977  		log.Printf("skipping because not a cron job")
   978  		os.Exit(0)
   979  	}
   980  	// Create the azure authentication and list the current archives
   981  	auth := build.AzureBlobstoreConfig{
   982  		Account:   strings.Split(*store, "/")[0],
   983  		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
   984  		Container: strings.SplitN(*store, "/", 2)[1],
   985  	}
   986  	blobs, err := build.AzureBlobstoreList(auth)
   987  	if err != nil {
   988  		log.Fatal(err)
   989  	}
   990  	// Iterate over the blobs, collect and sort all unstable builds
   991  	for i := 0; i < len(blobs); i++ {
   992  		if !strings.Contains(blobs[i].Name, "unstable") {
   993  			blobs = append(blobs[:i], blobs[i+1:]...)
   994  			i--
   995  		}
   996  	}
   997  	for i := 0; i < len(blobs); i++ {
   998  		for j := i + 1; j < len(blobs); j++ {
   999  			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
  1000  			if err != nil {
  1001  				log.Fatal(err)
  1002  			}
  1003  			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
  1004  			if err != nil {
  1005  				log.Fatal(err)
  1006  			}
  1007  			if iTime.After(jTime) {
  1008  				blobs[i], blobs[j] = blobs[j], blobs[i]
  1009  			}
  1010  		}
  1011  	}
  1012  	// Filter out all archives more recent that the given threshold
  1013  	for i, blob := range blobs {
  1014  		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
  1015  		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
  1016  			blobs = blobs[:i]
  1017  			break
  1018  		}
  1019  	}
  1020  	// Delete all marked as such and return
  1021  	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
  1022  		log.Fatal(err)
  1023  	}
  1024  }