github.com/google/syzkaller@v0.0.0-20240517125934-c0f1611a36d6/pkg/repro/repro.go (about)

     1  // Copyright 2016 syzkaller project authors. All rights reserved.
     2  // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
     3  
     4  package repro
     5  
     6  import (
     7  	"bytes"
     8  	"errors"
     9  	"fmt"
    10  	"sort"
    11  	"sync"
    12  	"time"
    13  
    14  	"github.com/google/syzkaller/pkg/bisect/minimize"
    15  	"github.com/google/syzkaller/pkg/csource"
    16  	"github.com/google/syzkaller/pkg/flatrpc"
    17  	"github.com/google/syzkaller/pkg/instance"
    18  	"github.com/google/syzkaller/pkg/log"
    19  	"github.com/google/syzkaller/pkg/mgrconfig"
    20  	"github.com/google/syzkaller/pkg/report"
    21  	"github.com/google/syzkaller/pkg/report/crash"
    22  	"github.com/google/syzkaller/prog"
    23  	"github.com/google/syzkaller/sys/targets"
    24  	"github.com/google/syzkaller/vm"
    25  )
    26  
    27  type Result struct {
    28  	Prog     *prog.Prog
    29  	Duration time.Duration
    30  	Opts     csource.Options
    31  	CRepro   bool
    32  	// Information about the final (non-symbolized) crash that we reproduced.
    33  	// Can be different from what we started reproducing.
    34  	Report *report.Report
    35  }
    36  
    37  type Stats struct {
    38  	Log              []byte
    39  	ExtractProgTime  time.Duration
    40  	MinimizeProgTime time.Duration
    41  	SimplifyProgTime time.Duration
    42  	ExtractCTime     time.Duration
    43  	SimplifyCTime    time.Duration
    44  }
    45  
    46  type reproInstance struct {
    47  	index    int
    48  	execProg execInterface
    49  }
    50  
    51  type context struct {
    52  	logf         func(string, ...interface{})
    53  	target       *targets.Target
    54  	reporter     *report.Reporter
    55  	crashTitle   string
    56  	crashType    crash.Type
    57  	crashStart   int
    58  	entries      []*prog.LogEntry
    59  	instances    chan *reproInstance
    60  	bootRequests chan int
    61  	testTimeouts []time.Duration
    62  	startOpts    csource.Options
    63  	stats        *Stats
    64  	report       *report.Report
    65  	timeouts     targets.Timeouts
    66  }
    67  
    68  // execInterface describes what's needed from a VM by a pkg/repro.
    69  type execInterface interface {
    70  	Close()
    71  	RunCProg(p *prog.Prog, duration time.Duration, opts csource.Options) (*instance.RunResult, error)
    72  	RunSyzProg(syzProg []byte, duration time.Duration, opts csource.Options) (*instance.RunResult, error)
    73  }
    74  
    75  var ErrNoPrograms = errors.New("crash log does not contain any programs")
    76  
    77  func Run(crashLog []byte, cfg *mgrconfig.Config, features flatrpc.Feature, reporter *report.Reporter,
    78  	vmPool *vm.Pool, vmIndexes []int) (*Result, *Stats, error) {
    79  	ctx, err := prepareCtx(crashLog, cfg, features, reporter, len(vmIndexes))
    80  	if err != nil {
    81  		return nil, nil, err
    82  	}
    83  	var wg sync.WaitGroup
    84  	wg.Add(1)
    85  	go func() {
    86  		defer wg.Done()
    87  		ctx.createInstances(cfg, vmPool)
    88  	}()
    89  	// Prepare VMs in advance.
    90  	for _, idx := range vmIndexes {
    91  		ctx.bootRequests <- idx
    92  	}
    93  	// Wait until all VMs are really released.
    94  	defer wg.Wait()
    95  	return ctx.run()
    96  }
    97  
    98  func prepareCtx(crashLog []byte, cfg *mgrconfig.Config, features flatrpc.Feature, reporter *report.Reporter,
    99  	VMs int) (*context, error) {
   100  	if VMs == 0 {
   101  		return nil, fmt.Errorf("no VMs provided")
   102  	}
   103  	entries := cfg.Target.ParseLog(crashLog)
   104  	if len(entries) == 0 {
   105  		return nil, ErrNoPrograms
   106  	}
   107  	crashStart := len(crashLog)
   108  	crashTitle, crashType := "", crash.UnknownType
   109  	if rep := reporter.Parse(crashLog); rep != nil {
   110  		crashStart = rep.StartPos
   111  		crashTitle = rep.Title
   112  		crashType = rep.Type
   113  	}
   114  	testTimeouts := []time.Duration{
   115  		3 * cfg.Timeouts.Program, // to catch simpler crashes (i.e. no races and no hangs)
   116  		20 * cfg.Timeouts.Program,
   117  		cfg.Timeouts.NoOutputRunningTime, // to catch "no output", races and hangs
   118  	}
   119  	switch {
   120  	case crashTitle == "":
   121  		crashTitle = "no output/lost connection"
   122  		// Lost connection can be detected faster,
   123  		// but theoretically if it's caused by a race it may need the largest timeout.
   124  		// No output can only be reproduced with the max timeout.
   125  		// As a compromise we use the smallest and the largest timeouts.
   126  		testTimeouts = []time.Duration{testTimeouts[0], testTimeouts[2]}
   127  	case crashType == crash.MemoryLeak:
   128  		// Memory leaks can't be detected quickly because of expensive setup and scanning.
   129  		testTimeouts = testTimeouts[1:]
   130  	case crashType == crash.Hang:
   131  		testTimeouts = testTimeouts[2:]
   132  	}
   133  	ctx := &context{
   134  		target:       cfg.SysTarget,
   135  		reporter:     reporter,
   136  		crashTitle:   crashTitle,
   137  		crashType:    crashType,
   138  		crashStart:   crashStart,
   139  		entries:      entries,
   140  		instances:    make(chan *reproInstance, VMs),
   141  		bootRequests: make(chan int, VMs),
   142  		testTimeouts: testTimeouts,
   143  		startOpts:    createStartOptions(cfg, features, crashType),
   144  		stats:        new(Stats),
   145  		timeouts:     cfg.Timeouts,
   146  	}
   147  	ctx.reproLogf(0, "%v programs, %v VMs, timeouts %v", len(entries), VMs, testTimeouts)
   148  	return ctx, nil
   149  }
   150  
   151  func (ctx *context) run() (*Result, *Stats, error) {
   152  	// Indicate that we no longer need VMs.
   153  	defer close(ctx.bootRequests)
   154  
   155  	res, err := ctx.repro()
   156  	if err != nil {
   157  		return nil, nil, err
   158  	}
   159  	if res != nil {
   160  		ctx.reproLogf(3, "repro crashed as (corrupted=%v):\n%s",
   161  			ctx.report.Corrupted, ctx.report.Report)
   162  		// Try to rerun the repro if the report is corrupted.
   163  		for attempts := 0; ctx.report.Corrupted && attempts < 3; attempts++ {
   164  			ctx.reproLogf(3, "report is corrupted, running repro again")
   165  			if res.CRepro {
   166  				_, err = ctx.testCProg(res.Prog, res.Duration, res.Opts)
   167  			} else {
   168  				_, err = ctx.testProg(res.Prog, res.Duration, res.Opts)
   169  			}
   170  			if err != nil {
   171  				return nil, nil, err
   172  			}
   173  		}
   174  		ctx.reproLogf(3, "final repro crashed as (corrupted=%v):\n%s",
   175  			ctx.report.Corrupted, ctx.report.Report)
   176  		res.Report = ctx.report
   177  	}
   178  	return res, ctx.stats, nil
   179  }
   180  
   181  func createStartOptions(cfg *mgrconfig.Config, features flatrpc.Feature,
   182  	crashType crash.Type) csource.Options {
   183  	opts := csource.DefaultOpts(cfg)
   184  	if crashType == crash.MemoryLeak {
   185  		opts.Leak = true
   186  	}
   187  	if features&flatrpc.FeatureNetInjection == 0 {
   188  		opts.NetInjection = false
   189  	}
   190  	if features&flatrpc.FeatureNetDevices == 0 {
   191  		opts.NetDevices = false
   192  	}
   193  	if features&flatrpc.FeatureDevlinkPCI == 0 {
   194  		opts.DevlinkPCI = false
   195  	}
   196  	if features&flatrpc.FeatureNicVF == 0 {
   197  		opts.NicVF = false
   198  	}
   199  	if features&flatrpc.FeatureUSBEmulation == 0 {
   200  		opts.USB = false
   201  	}
   202  	if features&flatrpc.FeatureVhciInjection == 0 {
   203  		opts.VhciInjection = false
   204  	}
   205  	if features&flatrpc.FeatureWifiEmulation == 0 {
   206  		opts.Wifi = false
   207  	}
   208  	if features&flatrpc.FeatureLRWPANEmulation == 0 {
   209  		opts.IEEE802154 = false
   210  	}
   211  	if features&flatrpc.FeatureSwap == 0 {
   212  		opts.Swap = false
   213  	}
   214  	return opts
   215  }
   216  
   217  func (ctx *context) repro() (*Result, error) {
   218  	// Cut programs that were executed after crash.
   219  	for i, ent := range ctx.entries {
   220  		if ent.Start > ctx.crashStart {
   221  			ctx.entries = ctx.entries[:i]
   222  			break
   223  		}
   224  	}
   225  
   226  	reproStart := time.Now()
   227  	defer func() {
   228  		ctx.reproLogf(3, "reproducing took %s", time.Since(reproStart))
   229  	}()
   230  
   231  	res, err := ctx.extractProg(ctx.entries)
   232  	if err != nil {
   233  		return nil, err
   234  	}
   235  	if res == nil {
   236  		return nil, nil
   237  	}
   238  	res, err = ctx.minimizeProg(res)
   239  	if err != nil {
   240  		return nil, err
   241  	}
   242  
   243  	// Try extracting C repro without simplifying options first.
   244  	res, err = ctx.extractC(res)
   245  	if err != nil {
   246  		return nil, err
   247  	}
   248  
   249  	// Simplify options and try extracting C repro.
   250  	if !res.CRepro {
   251  		res, err = ctx.simplifyProg(res)
   252  		if err != nil {
   253  			return nil, err
   254  		}
   255  	}
   256  
   257  	// Simplify C related options.
   258  	if res.CRepro {
   259  		res, err = ctx.simplifyC(res)
   260  		if err != nil {
   261  			return nil, err
   262  		}
   263  	}
   264  
   265  	return res, nil
   266  }
   267  
   268  func (ctx *context) extractProg(entries []*prog.LogEntry) (*Result, error) {
   269  	ctx.reproLogf(2, "extracting reproducer from %v programs", len(entries))
   270  	start := time.Now()
   271  	defer func() {
   272  		ctx.stats.ExtractProgTime = time.Since(start)
   273  	}()
   274  
   275  	// Extract last program on every proc.
   276  	procs := make(map[int]int)
   277  	for i, ent := range entries {
   278  		procs[ent.Proc] = i
   279  	}
   280  	var indices []int
   281  	for _, idx := range procs {
   282  		indices = append(indices, idx)
   283  	}
   284  	sort.Ints(indices)
   285  	var lastEntries []*prog.LogEntry
   286  	for i := len(indices) - 1; i >= 0; i-- {
   287  		lastEntries = append(lastEntries, entries[indices[i]])
   288  	}
   289  	for _, timeout := range ctx.testTimeouts {
   290  		// Execute each program separately to detect simple crashes caused by a single program.
   291  		// Programs are executed in reverse order, usually the last program is the guilty one.
   292  		res, err := ctx.extractProgSingle(lastEntries, timeout)
   293  		if err != nil {
   294  			return nil, err
   295  		}
   296  		if res != nil {
   297  			ctx.reproLogf(3, "found reproducer with %d syscalls", len(res.Prog.Calls))
   298  			return res, nil
   299  		}
   300  
   301  		// Don't try bisecting if there's only one entry.
   302  		if len(entries) == 1 {
   303  			continue
   304  		}
   305  
   306  		// Execute all programs and bisect the log to find multiple guilty programs.
   307  		res, err = ctx.extractProgBisect(entries, timeout)
   308  		if err != nil {
   309  			return nil, err
   310  		}
   311  		if res != nil {
   312  			ctx.reproLogf(3, "found reproducer with %d syscalls", len(res.Prog.Calls))
   313  			return res, nil
   314  		}
   315  	}
   316  
   317  	ctx.reproLogf(0, "failed to extract reproducer")
   318  	return nil, nil
   319  }
   320  
   321  func (ctx *context) extractProgSingle(entries []*prog.LogEntry, duration time.Duration) (*Result, error) {
   322  	ctx.reproLogf(3, "single: executing %d programs separately with timeout %s", len(entries), duration)
   323  
   324  	opts := ctx.startOpts
   325  	for _, ent := range entries {
   326  		crashed, err := ctx.testProg(ent.P, duration, opts)
   327  		if err != nil {
   328  			return nil, err
   329  		}
   330  		if crashed {
   331  			res := &Result{
   332  				Prog:     ent.P,
   333  				Duration: duration * 3 / 2,
   334  				Opts:     opts,
   335  			}
   336  			ctx.reproLogf(3, "single: successfully extracted reproducer")
   337  			return res, nil
   338  		}
   339  	}
   340  
   341  	ctx.reproLogf(3, "single: failed to extract reproducer")
   342  	return nil, nil
   343  }
   344  
   345  func (ctx *context) extractProgBisect(entries []*prog.LogEntry, baseDuration time.Duration) (*Result, error) {
   346  	ctx.reproLogf(3, "bisect: bisecting %d programs with base timeout %s", len(entries), baseDuration)
   347  
   348  	opts := ctx.startOpts
   349  	duration := func(entries int) time.Duration {
   350  		return baseDuration + time.Duration(entries/4)*time.Second
   351  	}
   352  
   353  	// First check if replaying the log may crash the kernel at all.
   354  	ret, err := ctx.testProgs(entries, duration(len(entries)), opts)
   355  	if !ret {
   356  		ctx.reproLogf(3, "replaying the whole log did not cause a kernel crash")
   357  		return nil, nil
   358  	}
   359  	if err != nil {
   360  		return nil, err
   361  	}
   362  
   363  	// Bisect the log to find multiple guilty programs.
   364  	entries, err = ctx.bisectProgs(entries, func(progs []*prog.LogEntry) (bool, error) {
   365  		return ctx.testProgs(progs, duration(len(progs)), opts)
   366  	})
   367  	if err != nil {
   368  		return nil, err
   369  	}
   370  	if len(entries) == 0 {
   371  		return nil, nil
   372  	}
   373  
   374  	// TODO: Minimize each program before concatenation.
   375  	// TODO: Return multiple programs if concatenation fails.
   376  
   377  	ctx.reproLogf(3, "bisect: %d programs left: \n\n%s\n", len(entries), encodeEntries(entries))
   378  	ctx.reproLogf(3, "bisect: trying to concatenate")
   379  
   380  	// Concatenate all programs into one.
   381  	prog := &prog.Prog{
   382  		Target: entries[0].P.Target,
   383  	}
   384  	for _, entry := range entries {
   385  		prog.Calls = append(prog.Calls, entry.P.Calls...)
   386  	}
   387  	dur := duration(len(entries)) * 3 / 2
   388  	crashed, err := ctx.testProg(prog, dur, opts)
   389  	if err != nil {
   390  		return nil, err
   391  	}
   392  	if crashed {
   393  		res := &Result{
   394  			Prog:     prog,
   395  			Duration: dur,
   396  			Opts:     opts,
   397  		}
   398  		ctx.reproLogf(3, "bisect: concatenation succeeded")
   399  		return res, nil
   400  	}
   401  
   402  	ctx.reproLogf(3, "bisect: concatenation failed")
   403  	return nil, nil
   404  }
   405  
   406  // Minimize calls and arguments.
   407  func (ctx *context) minimizeProg(res *Result) (*Result, error) {
   408  	ctx.reproLogf(2, "minimizing guilty program")
   409  	start := time.Now()
   410  	defer func() {
   411  		ctx.stats.MinimizeProgTime = time.Since(start)
   412  	}()
   413  
   414  	res.Prog, _ = prog.Minimize(res.Prog, -1, true,
   415  		func(p1 *prog.Prog, callIndex int) bool {
   416  			crashed, err := ctx.testProg(p1, res.Duration, res.Opts)
   417  			if err != nil {
   418  				ctx.reproLogf(0, "minimization failed with %v", err)
   419  				return false
   420  			}
   421  			return crashed
   422  		})
   423  
   424  	return res, nil
   425  }
   426  
   427  // Simplify repro options (threaded, sandbox, etc).
   428  func (ctx *context) simplifyProg(res *Result) (*Result, error) {
   429  	ctx.reproLogf(2, "simplifying guilty program options")
   430  	start := time.Now()
   431  	defer func() {
   432  		ctx.stats.SimplifyProgTime = time.Since(start)
   433  	}()
   434  
   435  	// Do further simplifications.
   436  	for _, simplify := range progSimplifies {
   437  		opts := res.Opts
   438  		if !simplify(&opts) || !checkOpts(&opts, ctx.timeouts, res.Duration) {
   439  			continue
   440  		}
   441  		crashed, err := ctx.testProg(res.Prog, res.Duration, opts)
   442  		if err != nil {
   443  			return nil, err
   444  		}
   445  		if !crashed {
   446  			continue
   447  		}
   448  		res.Opts = opts
   449  		// Simplification successful, try extracting C repro.
   450  		res, err = ctx.extractC(res)
   451  		if err != nil {
   452  			return nil, err
   453  		}
   454  		if res.CRepro {
   455  			return res, nil
   456  		}
   457  	}
   458  
   459  	return res, nil
   460  }
   461  
   462  // Try triggering crash with a C reproducer.
   463  func (ctx *context) extractC(res *Result) (*Result, error) {
   464  	ctx.reproLogf(2, "extracting C reproducer")
   465  	start := time.Now()
   466  	defer func() {
   467  		ctx.stats.ExtractCTime = time.Since(start)
   468  	}()
   469  
   470  	crashed, err := ctx.testCProg(res.Prog, res.Duration, res.Opts)
   471  	if err != nil {
   472  		return nil, err
   473  	}
   474  	res.CRepro = crashed
   475  	return res, nil
   476  }
   477  
   478  // Try to simplify the C reproducer.
   479  func (ctx *context) simplifyC(res *Result) (*Result, error) {
   480  	ctx.reproLogf(2, "simplifying C reproducer")
   481  	start := time.Now()
   482  	defer func() {
   483  		ctx.stats.SimplifyCTime = time.Since(start)
   484  	}()
   485  
   486  	for _, simplify := range cSimplifies {
   487  		opts := res.Opts
   488  		if !simplify(&opts) || !checkOpts(&opts, ctx.timeouts, res.Duration) {
   489  			continue
   490  		}
   491  		crashed, err := ctx.testCProg(res.Prog, res.Duration, opts)
   492  		if err != nil {
   493  			return nil, err
   494  		}
   495  		if !crashed {
   496  			continue
   497  		}
   498  		res.Opts = opts
   499  	}
   500  	return res, nil
   501  }
   502  
   503  func checkOpts(opts *csource.Options, timeouts targets.Timeouts, timeout time.Duration) bool {
   504  	if !opts.Repeat && timeout >= time.Minute {
   505  		// If we have a non-repeating C reproducer with timeout > vm.NoOutputTimeout and it hangs
   506  		// (the reproducer itself does not terminate on its own, note: it does not have builtin timeout),
   507  		// then we will falsely detect "not output from test machine" kernel bug.
   508  		// We could fix it by adding a builtin timeout to such reproducers (like we have in all other cases).
   509  		// However, then it will exit within few seconds and we will finish the test without actually waiting
   510  		// for full vm.NoOutputTimeout, which breaks the whole reason of using vm.NoOutputTimeout in the first
   511  		// place. So we would need something more elaborate: let the program exist after few seconds, but
   512  		// continue waiting for kernel hang errors for minutes, but at the same time somehow ignore "no output"
   513  		// error because it will be false in this case.
   514  		// Instead we simply prohibit !Repeat with long timeouts.
   515  		// It makes sense on its own to some degree: if we are chasing an elusive bug, repeating the test
   516  		// will increase chances of reproducing it and can make the reproducer less flaky.
   517  		// Syz repros does not have this problem because they always have internal timeout, however
   518  		// (1) it makes sense on its own, (2) we will either not use the whole timeout or waste the remaining
   519  		// time as mentioned above, (3) if we remove repeat for syz repro, we won't be able to handle it
   520  		// when/if we switch to C repro (we can simplify options, but we can't "complicate" them back).
   521  		return false
   522  	}
   523  	return true
   524  }
   525  
   526  func (ctx *context) testProg(p *prog.Prog, duration time.Duration, opts csource.Options) (crashed bool, err error) {
   527  	entry := prog.LogEntry{P: p}
   528  	return ctx.testProgs([]*prog.LogEntry{&entry}, duration, opts)
   529  }
   530  
   531  func (ctx *context) testWithInstance(callback func(execInterface) (rep *instance.RunResult,
   532  	err error)) (bool, error) {
   533  	var result *instance.RunResult
   534  	var err error
   535  
   536  	const attempts = 3
   537  	for i := 0; i < attempts; i++ {
   538  		// It's hard to classify all kinds of errors into the one worth repeating
   539  		// and not. So let's just retry runs for all errors.
   540  		// If the problem is transient, it will likely go away.
   541  		// If the problem is permanent, it will just be the same.
   542  		result, err = ctx.runOnInstance(callback)
   543  		if err == nil {
   544  			break
   545  		}
   546  	}
   547  	if err != nil {
   548  		return false, err
   549  	}
   550  	rep := result.Report
   551  	if rep == nil {
   552  		return false, nil
   553  	}
   554  	if rep.Suppressed {
   555  		ctx.reproLogf(2, "suppressed program crash: %v", rep.Title)
   556  		return false, nil
   557  	}
   558  	if ctx.crashType == crash.MemoryLeak && rep.Type != crash.MemoryLeak {
   559  		ctx.reproLogf(2, "not a leak crash: %v", rep.Title)
   560  		return false, nil
   561  	}
   562  	ctx.report = rep
   563  	return true, nil
   564  }
   565  
   566  var ErrNoVMs = errors.New("all VMs failed to boot")
   567  
   568  // A helper method for testWithInstance.
   569  func (ctx *context) runOnInstance(callback func(execInterface) (rep *instance.RunResult,
   570  	err error)) (*instance.RunResult, error) {
   571  	inst := <-ctx.instances
   572  	if inst == nil {
   573  		return nil, ErrNoVMs
   574  	}
   575  	defer ctx.returnInstance(inst)
   576  	return callback(inst.execProg)
   577  }
   578  
   579  func encodeEntries(entries []*prog.LogEntry) []byte {
   580  	buf := new(bytes.Buffer)
   581  	for _, ent := range entries {
   582  		fmt.Fprintf(buf, "executing program %v:\n%v", ent.Proc, string(ent.P.Serialize()))
   583  	}
   584  	return buf.Bytes()
   585  }
   586  
   587  func (ctx *context) testProgs(entries []*prog.LogEntry, duration time.Duration, opts csource.Options) (
   588  	crashed bool, err error) {
   589  	if len(entries) == 0 {
   590  		return false, fmt.Errorf("no programs to execute")
   591  	}
   592  	pstr := encodeEntries(entries)
   593  	program := entries[0].P.String()
   594  	if len(entries) > 1 {
   595  		program = "["
   596  		for i, entry := range entries {
   597  			program += fmt.Sprintf("%v", len(entry.P.Calls))
   598  			if i != len(entries)-1 {
   599  				program += ", "
   600  			}
   601  		}
   602  		program += "]"
   603  	}
   604  	ctx.reproLogf(2, "testing program (duration=%v, %+v): %s", duration, opts, program)
   605  	ctx.reproLogf(3, "detailed listing:\n%s", pstr)
   606  	return ctx.testWithInstance(func(exec execInterface) (*instance.RunResult, error) {
   607  		return exec.RunSyzProg(pstr, duration, opts)
   608  	})
   609  }
   610  
   611  func (ctx *context) testCProg(p *prog.Prog, duration time.Duration, opts csource.Options) (crashed bool, err error) {
   612  	return ctx.testWithInstance(func(exec execInterface) (*instance.RunResult, error) {
   613  		return exec.RunCProg(p, duration, opts)
   614  	})
   615  }
   616  
   617  func (ctx *context) returnInstance(inst *reproInstance) {
   618  	inst.execProg.Close()
   619  	ctx.bootRequests <- inst.index
   620  }
   621  
   622  func (ctx *context) reproLogf(level int, format string, args ...interface{}) {
   623  	if ctx.logf != nil {
   624  		ctx.logf(format, args...)
   625  	}
   626  	prefix := fmt.Sprintf("reproducing crash '%v': ", ctx.crashTitle)
   627  	log.Logf(level, prefix+format, args...)
   628  	ctx.stats.Log = append(ctx.stats.Log, []byte(fmt.Sprintf(format, args...)+"\n")...)
   629  }
   630  
   631  func (ctx *context) bisectProgs(progs []*prog.LogEntry, pred func([]*prog.LogEntry) (bool, error)) (
   632  	[]*prog.LogEntry, error) {
   633  	// Set up progs bisection.
   634  	ctx.reproLogf(3, "bisect: bisecting %d programs", len(progs))
   635  	minimizePred := func(progs []*prog.LogEntry) (bool, error) {
   636  		// Don't waste time testing empty crash log.
   637  		if len(progs) == 0 {
   638  			return false, nil
   639  		}
   640  		return pred(progs)
   641  	}
   642  	ret, err := minimize.Slice(minimize.Config[*prog.LogEntry]{
   643  		Pred: minimizePred,
   644  		// For flaky crashes we usually end up with too many chunks.
   645  		// Continuing bisection would just take a lot of time and likely produce no result.
   646  		MaxChunks: 6,
   647  		Logf: func(msg string, args ...interface{}) {
   648  			ctx.reproLogf(3, "bisect: "+msg, args...)
   649  		},
   650  	}, progs)
   651  	if err == minimize.ErrTooManyChunks {
   652  		ctx.reproLogf(3, "bisect: too many guilty chunks, aborting")
   653  		return nil, nil
   654  	}
   655  	return ret, err
   656  }
   657  
   658  func (ctx *context) createInstances(cfg *mgrconfig.Config, vmPool *vm.Pool) {
   659  	var wg sync.WaitGroup
   660  	for vmIndex := range ctx.bootRequests {
   661  		wg.Add(1)
   662  		vmIndex := vmIndex
   663  		go func() {
   664  			defer wg.Done()
   665  
   666  			for try := 0; ; try++ {
   667  				select {
   668  				case <-vm.Shutdown:
   669  					return
   670  				default:
   671  				}
   672  				inst, err := instance.CreateExecProgInstance(vmPool, vmIndex, cfg,
   673  					ctx.reporter, &instance.OptionalConfig{Logf: ctx.reproLogf})
   674  				if err != nil {
   675  					ctx.reproLogf(0, "failed to boot instance (try %v): %v", try+1, err)
   676  					time.Sleep(10 * time.Second)
   677  					continue
   678  				}
   679  				ctx.instances <- &reproInstance{execProg: inst, index: vmIndex}
   680  				break
   681  			}
   682  		}()
   683  	}
   684  	wg.Wait()
   685  	// Clean up.
   686  	close(ctx.instances)
   687  	for inst := range ctx.instances {
   688  		inst.execProg.Close()
   689  	}
   690  }
   691  
   692  type Simplify func(opts *csource.Options) bool
   693  
   694  var progSimplifies = []Simplify{
   695  	func(opts *csource.Options) bool {
   696  		if opts.Collide || !opts.Threaded {
   697  			return false
   698  		}
   699  		opts.Threaded = false
   700  		return true
   701  	},
   702  	func(opts *csource.Options) bool {
   703  		if !opts.Repeat {
   704  			return false
   705  		}
   706  		opts.Repeat = false
   707  		opts.Cgroups = false
   708  		opts.NetReset = false
   709  		opts.Procs = 1
   710  		return true
   711  	},
   712  	func(opts *csource.Options) bool {
   713  		if opts.Procs == 1 {
   714  			return false
   715  		}
   716  		opts.Procs = 1
   717  		return true
   718  	},
   719  	func(opts *csource.Options) bool {
   720  		if opts.Sandbox == "none" {
   721  			return false
   722  		}
   723  		opts.Sandbox = "none"
   724  		return true
   725  	},
   726  }
   727  
   728  var cSimplifies = append(progSimplifies, []Simplify{
   729  	func(opts *csource.Options) bool {
   730  		if opts.Sandbox == "" {
   731  			return false
   732  		}
   733  		opts.Sandbox = ""
   734  		opts.NetInjection = false
   735  		opts.NetDevices = false
   736  		opts.NetReset = false
   737  		opts.Cgroups = false
   738  		opts.BinfmtMisc = false
   739  		opts.CloseFDs = false
   740  		opts.DevlinkPCI = false
   741  		opts.NicVF = false
   742  		opts.USB = false
   743  		opts.VhciInjection = false
   744  		opts.Wifi = false
   745  		opts.Swap = false
   746  		return true
   747  	},
   748  	func(opts *csource.Options) bool {
   749  		if !opts.NetInjection {
   750  			return false
   751  		}
   752  		opts.NetInjection = false
   753  		return true
   754  	},
   755  	func(opts *csource.Options) bool {
   756  		if !opts.NetDevices {
   757  			return false
   758  		}
   759  		opts.NetDevices = false
   760  		return true
   761  	},
   762  	func(opts *csource.Options) bool {
   763  		if !opts.NetReset {
   764  			return false
   765  		}
   766  		opts.NetReset = false
   767  		return true
   768  	},
   769  	func(opts *csource.Options) bool {
   770  		if !opts.Cgroups {
   771  			return false
   772  		}
   773  		opts.Cgroups = false
   774  		return true
   775  	},
   776  	func(opts *csource.Options) bool {
   777  		if !opts.BinfmtMisc {
   778  			return false
   779  		}
   780  		opts.BinfmtMisc = false
   781  		return true
   782  	},
   783  	func(opts *csource.Options) bool {
   784  		// We don't want to remove close_fds() call when repeat is enabled,
   785  		// since that can lead to deadlocks, see executor/common_linux.h.
   786  		if !opts.CloseFDs || opts.Repeat {
   787  			return false
   788  		}
   789  		opts.CloseFDs = false
   790  		return true
   791  	},
   792  	func(opts *csource.Options) bool {
   793  		if !opts.DevlinkPCI {
   794  			return false
   795  		}
   796  		opts.DevlinkPCI = false
   797  		return true
   798  	},
   799  	func(opts *csource.Options) bool {
   800  		if !opts.NicVF {
   801  			return false
   802  		}
   803  		opts.NicVF = false
   804  		return true
   805  	},
   806  	func(opts *csource.Options) bool {
   807  		if !opts.USB {
   808  			return false
   809  		}
   810  		opts.USB = false
   811  		return true
   812  	},
   813  	func(opts *csource.Options) bool {
   814  		if !opts.VhciInjection {
   815  			return false
   816  		}
   817  		opts.VhciInjection = false
   818  		return true
   819  	},
   820  	func(opts *csource.Options) bool {
   821  		if !opts.Wifi {
   822  			return false
   823  		}
   824  		opts.Wifi = false
   825  		return true
   826  	},
   827  	func(opts *csource.Options) bool {
   828  		if !opts.IEEE802154 {
   829  			return false
   830  		}
   831  		opts.IEEE802154 = false
   832  		return true
   833  	},
   834  	func(opts *csource.Options) bool {
   835  		if !opts.UseTmpDir || opts.Sandbox == "namespace" || opts.Cgroups {
   836  			return false
   837  		}
   838  		opts.UseTmpDir = false
   839  		return true
   840  	},
   841  	func(opts *csource.Options) bool {
   842  		if !opts.HandleSegv {
   843  			return false
   844  		}
   845  		opts.HandleSegv = false
   846  		return true
   847  	},
   848  	func(opts *csource.Options) bool {
   849  		if !opts.Sysctl {
   850  			return false
   851  		}
   852  		opts.Sysctl = false
   853  		return true
   854  	},
   855  	func(opts *csource.Options) bool {
   856  		if !opts.Swap {
   857  			return false
   858  		}
   859  		opts.Swap = false
   860  		return true
   861  	},
   862  }...)