github.phpd.cn/thought-machine/please@v12.2.0+incompatible/src/parse/asp/main/main.go (about)

     1  // Package main implements a standalone parser binary,
     2  // which is simply a benchmark for how fast we can read a large number
     3  // of BUILD files.
     4  package main
     5  
     6  import (
     7  	"io/ioutil"
     8  	"os"
     9  	"path"
    10  	"regexp"
    11  	"sort"
    12  	"strings"
    13  	"sync"
    14  	"sync/atomic"
    15  	"time"
    16  
    17  	"github.com/davecgh/go-spew/spew"
    18  	"gopkg.in/op/go-logging.v1"
    19  
    20  	"cli"
    21  	"core"
    22  	"parse/asp"
    23  	"parse/rules"
    24  )
    25  
    26  var log = logging.MustGetLogger("parser")
    27  
    28  var opts = struct {
    29  	Usage        string
    30  	Verbosity    int    `short:"v" long:"verbose" default:"2" description:"Verbosity of output (higher number = more output)"`
    31  	NumThreads   int    `short:"n" long:"num_threads" default:"10" description:"Number of concurrent parse threads to run"`
    32  	ParseOnly    bool   `short:"p" long:"parse_only" description:"Only parse input files, do not interpret them."`
    33  	DumpAst      bool   `short:"d" long:"dump_ast" description:"Prints AST to stdout. Implies --parse_only."`
    34  	NoConfig     bool   `long:"no_config" description:"Don't look for or load a .plzconfig file"`
    35  	BuildDefsDir string `short:"b" long:"build_defs_dir" description:"Load build_defs files from this directory. This assumes that they are all produced by trivial build rules with obvious names. They will need to be built first."`
    36  	Args         struct {
    37  		BuildFiles []string `positional-arg-name:"files" required:"true" description:"BUILD files to parse"`
    38  	} `positional-args:"true"`
    39  }{
    40  	Usage: `Test parser for BUILD files using our standalone parser.`,
    41  }
    42  
    43  func parseFile(pkg *core.Package, p *asp.Parser, filename string) error {
    44  	if opts.ParseOnly || opts.DumpAst {
    45  		stmts, err := p.ParseFileOnly(filename)
    46  		if opts.DumpAst {
    47  			config := spew.NewDefaultConfig()
    48  			config.DisablePointerAddresses = true
    49  			config.DisableLengths = true
    50  			config.DisableTypes = true
    51  			config.OmitEmpty = true
    52  			config.Indent = "  "
    53  			os.Stdout.Write([]byte(cleanup(config.Sdump(stmts))))
    54  		}
    55  		return err
    56  	}
    57  	return p.ParseFile(pkg, filename)
    58  }
    59  
    60  // cleanup runs a few arbitrary cleanup steps on the given AST dump.
    61  // We do our best to do it analytically but one or two parts are a bit hard to alter.
    62  func cleanup(ast string) string {
    63  	r := regexp.MustCompile(`\n *Pos: .*\n`)
    64  	ast = r.ReplaceAllString(ast, "\n")
    65  	r = regexp.MustCompile(`String: "\\"(.*)\\"",`)
    66  	return r.ReplaceAllString(ast, `String: "$1",`)
    67  }
    68  
    69  func mustLoadBuildDefsDir(state *core.BuildState, dirname string) {
    70  	dir, err := ioutil.ReadDir(dirname)
    71  	if err != nil {
    72  		log.Fatalf("%s", err)
    73  	}
    74  	for _, fi := range dir {
    75  		if strings.HasSuffix(fi.Name(), ".build_defs") {
    76  			t := core.NewBuildTarget(core.NewBuildLabel(dirname, strings.TrimSuffix(fi.Name(), ".build_defs")))
    77  			t.AddOutput(fi.Name())
    78  			t.SetState(core.Built)
    79  			state.Graph.AddTarget(t)
    80  		}
    81  	}
    82  }
    83  
    84  func main() {
    85  	cli.ParseFlagsOrDie("parser", "11.0.0", &opts)
    86  	cli.InitLogging(opts.Verbosity)
    87  
    88  	config := core.DefaultConfiguration()
    89  	if !opts.NoConfig {
    90  		var err error
    91  		config, err = core.ReadConfigFiles([]string{
    92  			path.Join(core.MustFindRepoRoot(), core.ConfigFileName),
    93  		}, "")
    94  		if err != nil {
    95  			log.Fatalf("%s", err)
    96  		}
    97  	}
    98  
    99  	state := core.NewBuildState(opts.NumThreads, nil, opts.Verbosity, config)
   100  	if opts.BuildDefsDir != "" {
   101  		mustLoadBuildDefsDir(state, opts.BuildDefsDir)
   102  	}
   103  
   104  	ch := make(chan string, 100)
   105  	var wg sync.WaitGroup
   106  	wg.Add(opts.NumThreads)
   107  	total := len(opts.Args.BuildFiles)
   108  	p := asp.NewParser(state)
   109  
   110  	log.Debug("Loading built-in build rules...")
   111  	dir, _ := rules.AssetDir("")
   112  	sort.Strings(dir)
   113  	for _, filename := range dir {
   114  		if strings.HasSuffix(filename, ".gob") {
   115  			srcFile := strings.TrimSuffix(filename, ".gob")
   116  			src, _ := rules.Asset(srcFile)
   117  			p.MustLoadBuiltins("src/parse/rules/"+srcFile, src, rules.MustAsset(filename))
   118  		}
   119  	}
   120  
   121  	start := time.Now()
   122  	var errors int64
   123  	for i := 0; i < opts.NumThreads; i++ {
   124  		go func() {
   125  			for file := range ch {
   126  				pkg := core.NewPackage(file)
   127  				pkg.Filename = file
   128  				if err := parseFile(pkg, p, file); err != nil {
   129  					atomic.AddInt64(&errors, 1)
   130  					log.Error("Error parsing %s: %s", file, err)
   131  				}
   132  			}
   133  			wg.Done()
   134  		}()
   135  	}
   136  
   137  	for _, file := range opts.Args.BuildFiles {
   138  		ch <- file
   139  	}
   140  	close(ch)
   141  	wg.Wait()
   142  
   143  	log.Notice("Parsed %d files in %s", total, time.Since(start))
   144  	log.Notice("Success: %d / %d (%0.2f%%)", total-int(errors), total, 100.0*float64(total-int(errors))/float64(total))
   145  }