github.com/klaytn/klaytn@v1.10.2/build/ci.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  //go:build none
    18  // +build none
    19  
    20  package main
    21  
    22  import (
    23  	"bufio"
    24  	"bytes"
    25  	"encoding/base64"
    26  	"flag"
    27  	"fmt"
    28  	"go/parser"
    29  	"go/token"
    30  	"io/ioutil"
    31  	"log"
    32  	"os"
    33  	"os/exec"
    34  	"path/filepath"
    35  	"regexp"
    36  	"runtime"
    37  	"strconv"
    38  	"strings"
    39  	"time"
    40  
    41  	"github.com/klaytn/klaytn/utils/build"
    42  )
    43  
    44  var (
    45  
    46  	// Files that end up in the klay*.zip archive.
    47  	klayArchiveFiles = []string{
    48  		"COPYING",
    49  		executablePath("klay"),
    50  	}
    51  
    52  	// Files that end up in the klay-alltools*.zip archive.
    53  	allToolsArchiveFiles = []string{
    54  		"COPYING",
    55  		executablePath("klay"),
    56  	}
    57  
    58  	// A debian package is created for all executables listed here.
    59  	debExecutables = []debExecutable{
    60  		{
    61  			Name:        "klay",
    62  			Description: "Klaytn CLI client.",
    63  		},
    64  	}
    65  
    66  	// Distros for which packages are created.
    67  	// Note: vivid is unsupported because there is no golang-1.6 package for it.
    68  	// Note: wily is unsupported because it was officially deprecated on lanchpad.
    69  	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
    70  	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
    71  	debDistros = []string{"trusty", "xenial", "artful", "bionic"}
    72  )
    73  
    74  var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
    75  
    76  func executablePath(name string) string {
    77  	if runtime.GOOS == "windows" {
    78  		name += ".exe"
    79  	}
    80  	return filepath.Join(GOBIN, name)
    81  }
    82  
    83  func main() {
    84  	log.SetFlags(log.Lshortfile)
    85  
    86  	if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
    87  		log.Fatal("this script must be run from the root of the repository")
    88  	}
    89  	if len(os.Args) < 2 {
    90  		log.Fatal("need subcommand as first argument")
    91  	}
    92  	switch os.Args[1] {
    93  	case "install":
    94  		doInstall(os.Args[2:])
    95  	case "test":
    96  		doTest(os.Args[2:])
    97  	case "cover":
    98  		doCover(os.Args[2:])
    99  	case "fmt":
   100  		doFmt(os.Args[2:])
   101  	case "lint":
   102  		doLint(os.Args[2:], true)
   103  	case "lint-try":
   104  		doLint(os.Args[2:], false)
   105  	case "archive":
   106  		doArchive(os.Args[2:])
   107  	case "debsrc":
   108  		doDebianSource(os.Args[2:])
   109  	case "nsis":
   110  		doWindowsInstaller(os.Args[2:])
   111  	case "aar":
   112  		doAndroidArchive(os.Args[2:])
   113  	case "xcode":
   114  		doXCodeFramework(os.Args[2:])
   115  	case "xgo":
   116  		doXgo(os.Args[2:])
   117  	default:
   118  		log.Fatal("unknown command ", os.Args[1])
   119  	}
   120  }
   121  
   122  // Compiling
   123  
   124  func doInstall(cmdline []string) {
   125  	var (
   126  		arch = flag.String("arch", "", "Architecture to cross build for")
   127  		cc   = flag.String("cc", "", "C compiler to cross build with")
   128  	)
   129  	flag.CommandLine.Parse(cmdline)
   130  	env := build.Env()
   131  
   132  	// Check Go version. People regularly open issues about compilation
   133  	// failure with outdated Go. This should save them the trouble.
   134  	if !strings.Contains(runtime.Version(), "devel") {
   135  		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
   136  		var minor int
   137  		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
   138  
   139  		if minor < 9 {
   140  			log.Println("You have Go version", runtime.Version())
   141  			log.Println("klaytn requires at least Go version 1.9 and cannot")
   142  			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
   143  			os.Exit(1)
   144  		}
   145  	}
   146  	// Compile packages given as arguments, or everything if there are no arguments.
   147  	packages := []string{"./..."}
   148  	if flag.NArg() > 0 {
   149  		packages = flag.Args()
   150  	}
   151  
   152  	if *arch == "" || *arch == runtime.GOARCH {
   153  		goinstall := goTool("install", buildFlags(env)...)
   154  		goinstall.Args = append(goinstall.Args, "-v")
   155  		// goinstall.Args = append(goinstall.Args, "-race")
   156  		goinstall.Args = append(goinstall.Args, packages...)
   157  		build.MustRun(goinstall)
   158  		return
   159  	}
   160  	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
   161  	if *arch == "arm" {
   162  		os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
   163  		for _, path := range filepath.SplitList(build.GOPATH()) {
   164  			os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
   165  		}
   166  	}
   167  	// Seems we are cross compiling, work around forbidden GOBIN
   168  	goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
   169  	goinstall.Args = append(goinstall.Args, "-v")
   170  	goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
   171  	goinstall.Args = append(goinstall.Args, packages...)
   172  	build.MustRun(goinstall)
   173  
   174  	if cmds, err := ioutil.ReadDir("cmd"); err == nil {
   175  		for _, cmd := range cmds {
   176  			pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
   177  			if err != nil {
   178  				log.Fatal(err)
   179  			}
   180  			for name := range pkgs {
   181  				if name == "main" {
   182  					gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
   183  					gobuild.Args = append(gobuild.Args, "-v")
   184  					gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
   185  					gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
   186  					build.MustRun(gobuild)
   187  					break
   188  				}
   189  			}
   190  		}
   191  	}
   192  }
   193  
   194  func buildFlags(env build.Environment) (flags []string) {
   195  	var ld []string
   196  	if env.Commit != "" {
   197  		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
   198  		ld = append(ld, "-X", "github.com/klaytn/klaytn/cmd/utils/nodecmd.gitCommit="+env.Commit)
   199  	}
   200  	if env.Tag != "" {
   201  		ld = append(ld, "-X", "github.com/klaytn/klaytn/cmd/utils/nodecmd.gitTag="+env.Tag)
   202  	}
   203  	if runtime.GOOS == "darwin" {
   204  		ld = append(ld, "-s")
   205  	}
   206  
   207  	if env.IsDisabledSymbolTable {
   208  		ld = append(ld, "-s")
   209  	}
   210  	if env.IsStaticLink {
   211  		// Pass the static link flag to the external linker.
   212  		// By default, cmd/link will use external linking mode when non-standard cgo packages are involved.
   213  		ld = append(ld, "-linkmode", "external", "-extldflags", "-static")
   214  	}
   215  	if env.IsKlaytnRaceDetectionOn {
   216  		flags = append(flags, "-race")
   217  	}
   218  	if len(ld) > 0 {
   219  		flags = append(flags, "-ldflags", strings.Join(ld, " "))
   220  	}
   221  	return flags
   222  }
   223  
   224  func goTool(subcmd string, args ...string) *exec.Cmd {
   225  	return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
   226  }
   227  
   228  func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
   229  	cmd := build.GoTool(subcmd, args...)
   230  	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
   231  	if arch == "" || arch == runtime.GOARCH {
   232  		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
   233  	} else {
   234  		cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
   235  		cmd.Env = append(cmd.Env, "GOARCH="+arch)
   236  	}
   237  	if cc != "" {
   238  		cmd.Env = append(cmd.Env, "CC="+cc)
   239  	}
   240  	for _, e := range os.Environ() {
   241  		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
   242  			continue
   243  		}
   244  		cmd.Env = append(cmd.Env, e)
   245  	}
   246  	return cmd
   247  }
   248  
   249  // Running The Tests
   250  //
   251  // "tests" also includes static analysis tools such as vet.
   252  
   253  func doTest(cmdline []string) {
   254  	var (
   255  		parallel = flag.Int("p", 0, "The number of parallel test executions (default: the number of CPUs available)")
   256  		excludes = flag.String("exclude", "", "Comma-separated top-level directories to be excluded in test")
   257  	)
   258  	flag.CommandLine.Parse(cmdline)
   259  	env := build.Env()
   260  
   261  	packages := []string{"./..."}
   262  	if len(flag.CommandLine.Args()) > 0 {
   263  		packages = flag.CommandLine.Args()
   264  	}
   265  
   266  	if *excludes != "" {
   267  		packages = build.ExcludePackages(packages, strings.Split(*excludes, ","))
   268  	}
   269  
   270  	// Run analysis tools before the tests.
   271  	build.MustRun(goTool("vet", packages...))
   272  
   273  	// Run the actual tests.
   274  	gotest := goTool("test", buildFlags(env)...)
   275  	if *parallel != 0 {
   276  		gotest.Args = append(gotest.Args, "-p", strconv.Itoa(*parallel))
   277  	}
   278  	gotest.Args = append(gotest.Args, "--timeout=30m")
   279  	gotest.Args = append(gotest.Args, packages...)
   280  	build.MustRun(gotest)
   281  }
   282  
   283  func doCover(cmdline []string) {
   284  	var (
   285  		parallel   = flag.Int("p", 0, "The number of parallel coverage test executions (default: the number of CPUs available)")
   286  		excludes   = flag.String("exclude", "", "Comma-separated top-level directories to be excluded in coverage test")
   287  		outputFile = flag.String("coverprofile", "coverage.out", "The coverage profile file will be generated by coverage test")
   288  	)
   289  	flag.CommandLine.Parse(cmdline)
   290  	env := build.Env()
   291  
   292  	packages := []string{"./..."}
   293  	if len(flag.CommandLine.Args()) > 0 {
   294  		packages = flag.CommandLine.Args()
   295  	}
   296  
   297  	if *excludes != "" {
   298  		packages = build.ExcludePackages(packages, strings.Split(*excludes, ","))
   299  	}
   300  
   301  	coverPackages := []string{"./..."}
   302  	coverExcludes := []string{
   303  		"/tests",
   304  		"/metric",
   305  		"/build",
   306  		"/client",
   307  		"/contracts",
   308  		"/simulations",
   309  		"/api",
   310  		"/fork",
   311  		"/mocks",
   312  	}
   313  
   314  	coverPackages = build.ExcludePackages(coverPackages, coverExcludes)
   315  	coverPackagesString := strings.Join(coverPackages, ",")
   316  
   317  	// Run analysis tools before the tests.
   318  	build.MustRun(goTool("vet", packages...))
   319  
   320  	// Generate a coverage output file.
   321  	build.MustRunCommand("sh", "-c", "echo 'mode: atomic' > "+*outputFile)
   322  
   323  	// Run the actual tests.
   324  	gotest := goTool("test", buildFlags(env)...)
   325  	if *parallel != 0 {
   326  		gotest.Args = append(gotest.Args, "-p", strconv.Itoa(*parallel))
   327  	}
   328  
   329  	gotest.Args = append(gotest.Args, "-cover", "-covermode=atomic", "-coverprofile="+*outputFile)
   330  	gotest.Args = append(gotest.Args, "-coverpkg", coverPackagesString)
   331  	gotest.Args = append(gotest.Args, "--timeout=30m")
   332  	gotest.Args = append(gotest.Args, packages...)
   333  	build.MustRun(gotest)
   334  }
   335  
   336  func doFmt(cmdline []string) {
   337  	// runs gometalinter on requested packages
   338  	flag.CommandLine.Parse(cmdline)
   339  
   340  	packages := []string{"./..."}
   341  	if len(flag.CommandLine.Args()) > 0 {
   342  		packages = flag.CommandLine.Args()
   343  	}
   344  
   345  	lintBin := installLinter()
   346  
   347  	// Run fast linters batched together
   348  	configs := []string{
   349  		"run",
   350  		"--tests",
   351  		"--disable-all",
   352  		"--enable=gofmt",
   353  		"--timeout=2m",
   354  	}
   355  	build.MustRunCommand(lintBin, append(configs, packages...)...)
   356  }
   357  
   358  // runs gometalinter on requested packages and exits immediately when linter warning observed if exitOnError is true
   359  func doLint(cmdline []string, exitOnError bool) {
   360  	flag.CommandLine.Parse(cmdline)
   361  
   362  	packages := []string{"./..."}
   363  	if len(flag.CommandLine.Args()) > 0 {
   364  		packages = flag.CommandLine.Args()
   365  	}
   366  
   367  	lintBin := installLinter()
   368  
   369  	// Prepare a report file for linters
   370  	fname := "linter_report.txt"
   371  	fileOut, err := os.Create(fname)
   372  	defer fileOut.Close()
   373  	if err != nil {
   374  		log.Fatal(err)
   375  	}
   376  	fmt.Printf("Generating a linter report %s using above linters.\n", fname)
   377  
   378  	oldStdout := os.Stdout
   379  	os.Stdout = fileOut
   380  
   381  	// Run fast linters batched together
   382  	configs := []string{
   383  		"run",
   384  		"--tests",
   385  		"--disable-all",
   386  		"--enable=varcheck",
   387  		"--enable=misspell",
   388  		"--enable=goconst",
   389  	}
   390  	args := append(configs, packages...)
   391  	if exitOnError {
   392  		build.MustRunCommand(lintBin, args...)
   393  	} else {
   394  		build.TryRunCommand(lintBin, args...)
   395  	}
   396  
   397  	// Run fast linters batched together
   398  	configs = []string{
   399  		"run",
   400  		"--tests",
   401  		"--disable-all",
   402  		"--enable=deadcode",
   403  		"--enable=dupl",
   404  		"--enable=errcheck",
   405  		"--enable=ineffassign",
   406  		"--enable=interfacer",
   407  		"--enable=unparam",
   408  		"--enable=unused",
   409  	}
   410  	args = append(configs, packages...)
   411  	if exitOnError {
   412  		build.MustRunCommand(lintBin, args...)
   413  	} else {
   414  		build.TryRunCommand(lintBin, args...)
   415  	}
   416  
   417  	// Run slow linters one by one
   418  	for _, linter := range []string{"unconvert", "gosimple", "staticcheck", "gocyclo"} {
   419  		configs = []string{"run", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
   420  		args = append(configs, packages...)
   421  		if exitOnError {
   422  			build.MustRunCommand(lintBin, args...)
   423  		} else {
   424  			build.TryRunCommand(lintBin, args...)
   425  		}
   426  	}
   427  
   428  	// Restore stdout
   429  	os.Stdout = oldStdout
   430  
   431  	fmt.Printf("Succefully generating %s.\n", fname)
   432  }
   433  
   434  // Release Packaging
   435  
   436  func doArchive(cmdline []string) {
   437  	var (
   438  		arch   = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
   439  		atype  = flag.String("type", "zip", "Type of archive to write (zip|tar)")
   440  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
   441  		upload = flag.String("upload", "", `Destination to upload the archives`)
   442  		ext    string
   443  	)
   444  	flag.CommandLine.Parse(cmdline)
   445  	switch *atype {
   446  	case "zip":
   447  		ext = ".zip"
   448  	case "tar":
   449  		ext = ".tar.gz"
   450  	default:
   451  		log.Fatal("unknown archive type: ", atype)
   452  	}
   453  
   454  	var (
   455  		env      = build.Env()
   456  		base     = archiveBasename(*arch, env)
   457  		klaybin  = "klay-" + base + ext
   458  		alltools = "klay-alltools-" + base + ext
   459  	)
   460  	maybeSkipArchive(env)
   461  	if err := build.WriteArchive(klaybin, klayArchiveFiles); err != nil {
   462  		log.Fatal(err)
   463  	}
   464  	if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
   465  		log.Fatal(err)
   466  	}
   467  	for _, archive := range []string{klaybin, alltools} {
   468  		if err := archiveUpload(archive, *upload, *signer); err != nil {
   469  			log.Fatal(err)
   470  		}
   471  	}
   472  }
   473  
   474  func archiveBasename(arch string, env build.Environment) string {
   475  	platform := runtime.GOOS + "-" + arch
   476  	if arch == "arm" {
   477  		platform += os.Getenv("GOARM")
   478  	}
   479  	if arch == "android" {
   480  		platform = "android-all"
   481  	}
   482  	if arch == "ios" {
   483  		platform = "ios-all"
   484  	}
   485  	return platform + "-" + archiveVersion(env)
   486  }
   487  
   488  func archiveVersion(env build.Environment) string {
   489  	version := build.VERSION()
   490  	if isUnstableBuild(env) {
   491  		version += "-unstable"
   492  	}
   493  	if env.Commit != "" {
   494  		version += "-" + env.Commit[:8]
   495  	}
   496  	return version
   497  }
   498  
   499  func archiveUpload(archive string, blobstore string, signer string) error {
   500  	// If signing was requested, generate the signature files
   501  	if signer != "" {
   502  		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
   503  		if err != nil {
   504  			return fmt.Errorf("invalid base64 %s", signer)
   505  		}
   506  		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
   507  			return err
   508  		}
   509  	}
   510  	return nil
   511  }
   512  
   513  // skips archiving for some build configurations.
   514  func maybeSkipArchive(env build.Environment) {
   515  	if env.IsPullRequest {
   516  		log.Printf("skipping because this is a PR build")
   517  		os.Exit(0)
   518  	}
   519  	if env.IsCronJob {
   520  		log.Printf("skipping because this is a cron job")
   521  		os.Exit(0)
   522  	}
   523  	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
   524  		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
   525  		os.Exit(0)
   526  	}
   527  }
   528  
   529  // Debian Packaging
   530  
   531  func doDebianSource(cmdline []string) {
   532  	var (
   533  		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
   534  		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:klaytn/klaytn")`)
   535  		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
   536  		now     = time.Now()
   537  	)
   538  	flag.CommandLine.Parse(cmdline)
   539  	*workdir = makeWorkdir(*workdir)
   540  	env := build.Env()
   541  	maybeSkipArchive(env)
   542  
   543  	// Import the signing key.
   544  	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
   545  		key, err := base64.StdEncoding.DecodeString(b64key)
   546  		if err != nil {
   547  			log.Fatal("invalid base64 PPA_SIGNING_KEY")
   548  		}
   549  		gpg := exec.Command("gpg", "--import")
   550  		gpg.Stdin = bytes.NewReader(key)
   551  		build.MustRun(gpg)
   552  	}
   553  
   554  	// Create the packages.
   555  	for _, distro := range debDistros {
   556  		meta := newDebMetadata(distro, *signer, env, now)
   557  		pkgdir := stageDebianSource(*workdir, meta)
   558  		debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
   559  		debuild.Dir = pkgdir
   560  		build.MustRun(debuild)
   561  
   562  		changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
   563  		changes = filepath.Join(*workdir, changes)
   564  		if *signer != "" {
   565  			build.MustRunCommand("debsign", changes)
   566  		}
   567  		if *upload != "" {
   568  			build.MustRunCommand("dput", *upload, changes)
   569  		}
   570  	}
   571  }
   572  
   573  func makeWorkdir(wdflag string) string {
   574  	var err error
   575  	if wdflag != "" {
   576  		err = os.MkdirAll(wdflag, 0o744)
   577  	} else {
   578  		wdflag, err = ioutil.TempDir("", "klay-build-")
   579  	}
   580  	if err != nil {
   581  		log.Fatal(err)
   582  	}
   583  	return wdflag
   584  }
   585  
   586  func isUnstableBuild(env build.Environment) bool {
   587  	if env.Tag != "" {
   588  		return false
   589  	}
   590  	return true
   591  }
   592  
   593  type debMetadata struct {
   594  	Env build.Environment
   595  
   596  	// klaytn version being built. Note that this
   597  	// is not the debian package version. The package version
   598  	// is constructed by VersionString.
   599  	Version string
   600  
   601  	Author       string // "name <email>", also selects signing key
   602  	Distro, Time string
   603  	Executables  []debExecutable
   604  }
   605  
   606  type debExecutable struct {
   607  	Name, Description string
   608  }
   609  
   610  func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
   611  	if author == "" {
   612  		// No signing key, use default author.
   613  		author = "Klaytn Builds <infra@groundx.xyz>"
   614  	}
   615  	return debMetadata{
   616  		Env:         env,
   617  		Author:      author,
   618  		Distro:      distro,
   619  		Version:     build.VERSION(),
   620  		Time:        t.Format(time.RFC1123Z),
   621  		Executables: debExecutables,
   622  	}
   623  }
   624  
   625  // Name returns the name of the metapackage that depends
   626  // on all executable packages.
   627  func (meta debMetadata) Name() string {
   628  	if isUnstableBuild(meta.Env) {
   629  		return "klaytn-unstable"
   630  	}
   631  	return "klaytn"
   632  }
   633  
   634  // VersionString returns the debian version of the packages.
   635  func (meta debMetadata) VersionString() string {
   636  	vsn := meta.Version
   637  	if meta.Env.Buildnum != "" {
   638  		vsn += "+build" + meta.Env.Buildnum
   639  	}
   640  	if meta.Distro != "" {
   641  		vsn += "+" + meta.Distro
   642  	}
   643  	return vsn
   644  }
   645  
   646  // ExeList returns the list of all executable packages.
   647  func (meta debMetadata) ExeList() string {
   648  	names := make([]string, len(meta.Executables))
   649  	for i, e := range meta.Executables {
   650  		names[i] = meta.ExeName(e)
   651  	}
   652  	return strings.Join(names, ", ")
   653  }
   654  
   655  // ExeName returns the package name of an executable package.
   656  func (meta debMetadata) ExeName(exe debExecutable) string {
   657  	if isUnstableBuild(meta.Env) {
   658  		return exe.Name + "-unstable"
   659  	}
   660  	return exe.Name
   661  }
   662  
   663  // ExeConflicts returns the content of the Conflicts field
   664  // for executable packages.
   665  func (meta debMetadata) ExeConflicts(exe debExecutable) string {
   666  	if isUnstableBuild(meta.Env) {
   667  		// Set up the conflicts list so that the *-unstable packages
   668  		// cannot be installed alongside the regular version.
   669  		//
   670  		// https://www.debian.org/doc/debian-policy/ch-relationships.html
   671  		// is very explicit about Conflicts: and says that Breaks: should
   672  		// be preferred and the conflicting files should be handled via
   673  		// alternates. We might do this eventually but using a conflict is
   674  		// easier now.
   675  		return "klaytn, " + exe.Name
   676  	}
   677  	return ""
   678  }
   679  
   680  func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
   681  	pkg := meta.Name() + "-" + meta.VersionString()
   682  	pkgdir = filepath.Join(tmpdir, pkg)
   683  	if err := os.Mkdir(pkgdir, 0o755); err != nil {
   684  		log.Fatal(err)
   685  	}
   686  
   687  	// Copy the source code.
   688  	build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
   689  
   690  	// Put the debian build files in place.
   691  	debian := filepath.Join(pkgdir, "debian")
   692  	build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0o755, meta)
   693  	build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0o644, meta)
   694  	build.Render("build/deb.control", filepath.Join(debian, "control"), 0o644, meta)
   695  	build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0o644, meta)
   696  	build.RenderString("8\n", filepath.Join(debian, "compat"), 0o644, meta)
   697  	build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0o644, meta)
   698  	for _, exe := range meta.Executables {
   699  		install := filepath.Join(debian, meta.ExeName(exe)+".install")
   700  		docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
   701  		build.Render("build/deb.install", install, 0o644, exe)
   702  		build.Render("build/deb.docs", docs, 0o644, exe)
   703  	}
   704  
   705  	return pkgdir
   706  }
   707  
   708  // Windows installer
   709  
   710  func doWindowsInstaller(cmdline []string) {
   711  	// Parse the flags and make skip installer generation on PRs
   712  	var (
   713  		arch    = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging")
   714  		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`)
   715  		upload  = flag.String("upload", "", `Destination to upload the archives`)
   716  		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
   717  	)
   718  	flag.CommandLine.Parse(cmdline)
   719  	*workdir = makeWorkdir(*workdir)
   720  	env := build.Env()
   721  	maybeSkipArchive(env)
   722  
   723  	// Aggregate binaries that are included in the installer
   724  	var (
   725  		devTools []string
   726  		allTools []string
   727  		klayTool string
   728  	)
   729  	for _, file := range allToolsArchiveFiles {
   730  		if file == "COPYING" { // license, copied later
   731  			continue
   732  		}
   733  		allTools = append(allTools, filepath.Base(file))
   734  		if filepath.Base(file) == "klay.exe" {
   735  			klayTool = file
   736  		} else {
   737  			devTools = append(devTools, file)
   738  		}
   739  	}
   740  
   741  	// Render NSIS scripts: Installer NSIS contains two installer sections,
   742  	// first section contains the klaytn binary, second section holds the dev tools.
   743  	templateData := map[string]interface{}{
   744  		"License":  "COPYING",
   745  		"Klay":     klayTool,
   746  		"DevTools": devTools,
   747  	}
   748  	build.Render("build/nsis.klay.nsi", filepath.Join(*workdir, "klay.nsi"), 0o644, nil)
   749  	build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0o644, templateData)
   750  	build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0o644, allTools)
   751  	build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0o644, nil)
   752  	build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0o644, nil)
   753  	build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0o755)
   754  	build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0o755)
   755  
   756  	// Build the installer. This assumes that all the needed files have been previously
   757  	// built (don't mix building and packaging to keep cross compilation complexity to a
   758  	// minimum).
   759  	version := strings.Split(build.VERSION(), ".")
   760  	if env.Commit != "" {
   761  		version[2] += "-" + env.Commit[:8]
   762  	}
   763  	installer, _ := filepath.Abs("klay-" + archiveBasename(*arch, env) + ".exe")
   764  	build.MustRunCommand("makensis.exe",
   765  		"/DOUTPUTFILE="+installer,
   766  		"/DMAJORVERSION="+version[0],
   767  		"/DMINORVERSION="+version[1],
   768  		"/DBUILDVERSION="+version[2],
   769  		"/DARCH="+*arch,
   770  		filepath.Join(*workdir, "klay.nsi"),
   771  	)
   772  
   773  	// Sign and publish installer.
   774  	if err := archiveUpload(installer, *upload, *signer); err != nil {
   775  		log.Fatal(err)
   776  	}
   777  }
   778  
   779  // Android archives
   780  
   781  func doAndroidArchive(cmdline []string) {
   782  	var (
   783  		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
   784  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
   785  		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
   786  		upload = flag.String("upload", "", `Destination to upload the archive`)
   787  	)
   788  	flag.CommandLine.Parse(cmdline)
   789  	env := build.Env()
   790  
   791  	// Sanity check that the SDK and NDK are installed and set
   792  	if os.Getenv("ANDROID_HOME") == "" {
   793  		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
   794  	}
   795  	if os.Getenv("ANDROID_NDK") == "" {
   796  		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
   797  	}
   798  	// Build the Android archive and Maven resources
   799  	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
   800  	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
   801  	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.klaytn", "-v", "github.com/klaytn/klaytn/mobile"))
   802  
   803  	if *local {
   804  		// If we're building locally, copy bundle to build dir and skip Maven
   805  		os.Rename("klay.aar", filepath.Join(GOBIN, "klay.aar"))
   806  		return
   807  	}
   808  	meta := newMavenMetadata(env)
   809  	build.Render("build/mvn.pom", meta.Package+".pom", 0o755, meta)
   810  
   811  	// Skip Maven deploy and Azure upload for PR builds
   812  	maybeSkipArchive(env)
   813  
   814  	// Sign and upload the archive to Azure
   815  	archive := "klay-" + archiveBasename("android", env) + ".aar"
   816  	os.Rename("klay.aar", archive)
   817  
   818  	if err := archiveUpload(archive, *upload, *signer); err != nil {
   819  		log.Fatal(err)
   820  	}
   821  	// Sign and upload all the artifacts to Maven Central
   822  	os.Rename(archive, meta.Package+".aar")
   823  	if *signer != "" && *deploy != "" {
   824  		// Import the signing key into the local GPG instance
   825  		b64key := os.Getenv(*signer)
   826  		key, err := base64.StdEncoding.DecodeString(b64key)
   827  		if err != nil {
   828  			log.Fatalf("invalid base64 %s", *signer)
   829  		}
   830  		gpg := exec.Command("gpg", "--import")
   831  		gpg.Stdin = bytes.NewReader(key)
   832  		build.MustRun(gpg)
   833  
   834  		keyID, err := build.PGPKeyID(string(key))
   835  		if err != nil {
   836  			log.Fatal(err)
   837  		}
   838  		// Upload the artifacts to Sonatype and/or Maven Central
   839  		repo := *deploy + "/service/local/staging/deploy/maven2"
   840  		if meta.Develop {
   841  			repo = *deploy + "/content/repositories/snapshots"
   842  		}
   843  		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
   844  			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
   845  			"-Dgpg.keyname="+keyID,
   846  			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
   847  	}
   848  }
   849  
   850  func gomobileTool(subcmd string, args ...string) *exec.Cmd {
   851  	cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd)
   852  	cmd.Args = append(cmd.Args, args...)
   853  	cmd.Env = []string{
   854  		"GOPATH=" + build.GOPATH(),
   855  		"PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"),
   856  	}
   857  	for _, e := range os.Environ() {
   858  		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") {
   859  			continue
   860  		}
   861  		cmd.Env = append(cmd.Env, e)
   862  	}
   863  	return cmd
   864  }
   865  
   866  type mavenMetadata struct {
   867  	Version      string
   868  	Package      string
   869  	Develop      bool
   870  	Contributors []mavenContributor
   871  }
   872  
   873  type mavenContributor struct {
   874  	Name  string
   875  	Email string
   876  }
   877  
   878  func newMavenMetadata(env build.Environment) mavenMetadata {
   879  	// Collect the list of authors from the repo root
   880  	contribs := []mavenContributor{}
   881  	if authors, err := os.Open("AUTHORS"); err == nil {
   882  		defer authors.Close()
   883  
   884  		scanner := bufio.NewScanner(authors)
   885  		for scanner.Scan() {
   886  			// Skip any whitespace from the authors list
   887  			line := strings.TrimSpace(scanner.Text())
   888  			if line == "" || line[0] == '#' {
   889  				continue
   890  			}
   891  			// Split the author and insert as a contributor
   892  			re := regexp.MustCompile("([^<]+) <(.+)>")
   893  			parts := re.FindStringSubmatch(line)
   894  			if len(parts) == 3 {
   895  				contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]})
   896  			}
   897  		}
   898  	}
   899  	// Render the version and package strings
   900  	version := build.VERSION()
   901  	if isUnstableBuild(env) {
   902  		version += "-SNAPSHOT"
   903  	}
   904  	return mavenMetadata{
   905  		Version:      version,
   906  		Package:      "klay-" + version,
   907  		Develop:      isUnstableBuild(env),
   908  		Contributors: contribs,
   909  	}
   910  }
   911  
   912  // XCode frameworks
   913  
   914  func doXCodeFramework(cmdline []string) {
   915  	var (
   916  		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
   917  		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
   918  		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
   919  		upload = flag.String("upload", "", `Destination to upload the archives`)
   920  	)
   921  	flag.CommandLine.Parse(cmdline)
   922  	env := build.Env()
   923  
   924  	// Build the iOS XCode framework
   925  	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
   926  	build.MustRun(gomobileTool("init"))
   927  	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/klaytn/klaytn/mobile")
   928  
   929  	if *local {
   930  		// If we're building locally, use the build folder and stop afterwards
   931  		bind.Dir, _ = filepath.Abs(GOBIN)
   932  		build.MustRun(bind)
   933  		return
   934  	}
   935  	archive := "klay-" + archiveBasename("ios", env)
   936  	if err := os.Mkdir(archive, os.ModePerm); err != nil {
   937  		log.Fatal(err)
   938  	}
   939  	bind.Dir, _ = filepath.Abs(archive)
   940  	build.MustRun(bind)
   941  	build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive)
   942  
   943  	// Skip CocoaPods deploy and Azure upload for PR builds
   944  	maybeSkipArchive(env)
   945  
   946  	// Sign and upload the framework to Azure
   947  	if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil {
   948  		log.Fatal(err)
   949  	}
   950  	// Prepare and upload a PodSpec to CocoaPods
   951  	if *deploy != "" {
   952  		meta := newPodMetadata(env, archive)
   953  		build.Render("build/pod.podspec", "Klaytn.podspec", 0o755, meta)
   954  		build.MustRunCommand("pod", *deploy, "push", "Klaytn.podspec", "--allow-warnings", "--verbose")
   955  	}
   956  }
   957  
   958  type podMetadata struct {
   959  	Version      string
   960  	Commit       string
   961  	Archive      string
   962  	Contributors []podContributor
   963  }
   964  
   965  type podContributor struct {
   966  	Name  string
   967  	Email string
   968  }
   969  
   970  func newPodMetadata(env build.Environment, archive string) podMetadata {
   971  	// Collect the list of authors from the repo root
   972  	contribs := []podContributor{}
   973  	if authors, err := os.Open("AUTHORS"); err == nil {
   974  		defer authors.Close()
   975  
   976  		scanner := bufio.NewScanner(authors)
   977  		for scanner.Scan() {
   978  			// Skip any whitespace from the authors list
   979  			line := strings.TrimSpace(scanner.Text())
   980  			if line == "" || line[0] == '#' {
   981  				continue
   982  			}
   983  			// Split the author and insert as a contributor
   984  			re := regexp.MustCompile("([^<]+) <(.+)>")
   985  			parts := re.FindStringSubmatch(line)
   986  			if len(parts) == 3 {
   987  				contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]})
   988  			}
   989  		}
   990  	}
   991  	version := build.VERSION()
   992  	if isUnstableBuild(env) {
   993  		version += "-unstable." + env.Buildnum
   994  	}
   995  	return podMetadata{
   996  		Archive:      archive,
   997  		Version:      version,
   998  		Commit:       env.Commit,
   999  		Contributors: contribs,
  1000  	}
  1001  }
  1002  
  1003  // Cross compilation
  1004  
  1005  func doXgo(cmdline []string) {
  1006  	alltools := flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
  1007  	flag.CommandLine.Parse(cmdline)
  1008  	env := build.Env()
  1009  
  1010  	// Make sure xgo is available for cross compilation
  1011  	build.MustRun(goTool("get", "github.com/klaytn/xgo"))
  1012  
  1013  	// From go1.18, golang requires 'go install' to install a package binary
  1014  	if strings.Compare(runtime.Version(), "go1.18") >= 0 {
  1015  		build.MustRun(goTool("install", "github.com/klaytn/xgo"))
  1016  	}
  1017  
  1018  	// If all tools building is requested, build everything the builder wants
  1019  	args := append(buildFlags(env), flag.Args()...)
  1020  
  1021  	if *alltools {
  1022  		args = append(args, []string{"--dest", GOBIN}...)
  1023  		for _, res := range allToolsArchiveFiles {
  1024  			if strings.HasPrefix(res, GOBIN) {
  1025  				// Binary tool found, cross build it explicitly
  1026  				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
  1027  				xgo := xgoTool(args)
  1028  				build.MustRun(xgo)
  1029  				args = args[:len(args)-1]
  1030  			}
  1031  		}
  1032  		return
  1033  	}
  1034  	// Otherwise xxecute the explicit cross compilation
  1035  	path := args[len(args)-1]
  1036  	args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
  1037  
  1038  	xgo := xgoTool(args)
  1039  	build.MustRun(xgo)
  1040  }
  1041  
  1042  func xgoTool(args []string) *exec.Cmd {
  1043  	cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
  1044  	cmd.Env = []string{
  1045  		"GOPATH=" + build.GOPATH(),
  1046  		"GOBIN=" + GOBIN,
  1047  	}
  1048  	for _, e := range os.Environ() {
  1049  		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
  1050  			continue
  1051  		}
  1052  		cmd.Env = append(cmd.Env, e)
  1053  	}
  1054  	return cmd
  1055  }
  1056  
  1057  func installLinter() string {
  1058  	lintBin := filepath.Join(build.GOPATH(), "bin", "golangci-lint")
  1059  
  1060  	_, err := exec.LookPath(lintBin)
  1061  	if err != nil {
  1062  		fmt.Println("Installing golangci-lint.")
  1063  
  1064  		cmdCurl := exec.Command("curl", "-sSfL", "https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh")
  1065  		cmdSh := exec.Command("sh", "-s", "--", "-b", filepath.Join(build.GOPATH(), "bin"), "v1.24.0")
  1066  		cmdSh.Stdin, err = cmdCurl.StdoutPipe()
  1067  		if err != nil {
  1068  			log.Fatal(err)
  1069  		}
  1070  
  1071  		fmt.Println(">>>", strings.Join(cmdCurl.Args, " "))
  1072  		if err := cmdCurl.Start(); err != nil {
  1073  			log.Fatal(err)
  1074  		}
  1075  
  1076  		build.MustRun(cmdSh)
  1077  
  1078  		if err := cmdCurl.Wait(); err != nil {
  1079  			log.Fatal(err)
  1080  		}
  1081  	}
  1082  	return lintBin
  1083  }