github.com/google/syzkaller@v0.0.0-20240517125934-c0f1611a36d6/dashboard/app/entities.go (about)

     1  // Copyright 2017 syzkaller project authors. All rights reserved.
     2  // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
     3  
     4  package main
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"regexp"
    10  	"strconv"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/google/syzkaller/dashboard/dashapi"
    15  	"github.com/google/syzkaller/pkg/hash"
    16  	"github.com/google/syzkaller/pkg/subsystem"
    17  	db "google.golang.org/appengine/v2/datastore"
    18  )
    19  
    20  // This file contains definitions of entities stored in datastore.
    21  
    22  const (
    23  	maxTextLen   = 200
    24  	MaxStringLen = 1024
    25  
    26  	maxBugHistoryDays = 365 * 5
    27  )
    28  
    29  type Manager struct {
    30  	Namespace         string
    31  	Name              string
    32  	Link              string
    33  	CurrentBuild      string
    34  	FailedBuildBug    string
    35  	FailedSyzBuildBug string
    36  	LastAlive         time.Time
    37  	CurrentUpTime     time.Duration
    38  	LastGeneratedJob  time.Time
    39  }
    40  
    41  // ManagerStats holds per-day manager runtime stats.
    42  // Has Manager as parent entity. Keyed by Date.
    43  type ManagerStats struct {
    44  	Date              int // YYYYMMDD
    45  	MaxCorpus         int64
    46  	MaxPCs            int64 // coverage
    47  	MaxCover          int64 // what we call feedback signal everywhere else
    48  	TotalFuzzingTime  time.Duration
    49  	TotalCrashes      int64
    50  	CrashTypes        int64 // unique crash types
    51  	SuppressedCrashes int64
    52  	TotalExecs        int64
    53  	// These are only recorded once right after corpus is triaged.
    54  	TriagedCoverage int64
    55  	TriagedPCs      int64
    56  }
    57  
    58  type Asset struct {
    59  	Type        dashapi.AssetType
    60  	DownloadURL string
    61  	CreateDate  time.Time
    62  }
    63  
    64  type Build struct {
    65  	Namespace           string
    66  	Manager             string
    67  	ID                  string // unique ID generated by syz-ci
    68  	Type                BuildType
    69  	Time                time.Time
    70  	OS                  string
    71  	Arch                string
    72  	VMArch              string
    73  	SyzkallerCommit     string
    74  	SyzkallerCommitDate time.Time
    75  	CompilerID          string
    76  	KernelRepo          string
    77  	KernelBranch        string
    78  	KernelCommit        string
    79  	KernelCommitTitle   string    `datastore:",noindex"`
    80  	KernelCommitDate    time.Time `datastore:",noindex"`
    81  	KernelConfig        int64     // reference to KernelConfig text entity
    82  	Assets              []Asset   // build-related assets
    83  	AssetsLastCheck     time.Time // the last time we checked the assets for deprecation
    84  }
    85  
    86  type Bug struct {
    87  	Namespace    string
    88  	Seq          int64 // sequences of the bug with the same title
    89  	Title        string
    90  	MergedTitles []string // crash titles that we already merged into this bug
    91  	AltTitles    []string // alternative crash titles that we may merge into this bug
    92  	Status       int
    93  	StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why
    94  	DupOf        string
    95  	NumCrashes   int64
    96  	NumRepro     int64
    97  	// ReproLevel is the best ever found repro level for this bug.
    98  	// HeadReproLevel is best known repro level that still works on the HEAD commit.
    99  	ReproLevel      dashapi.ReproLevel
   100  	HeadReproLevel  dashapi.ReproLevel `datastore:"HeadReproLevel"`
   101  	BisectCause     BisectStatus
   102  	BisectFix       BisectStatus
   103  	HasReport       bool
   104  	NeedCommitInfo  bool
   105  	FirstTime       time.Time
   106  	LastTime        time.Time
   107  	LastSavedCrash  time.Time
   108  	LastReproTime   time.Time
   109  	LastCauseBisect time.Time
   110  	FixTime         time.Time // when we become aware of the fixing commit
   111  	LastActivity    time.Time // last time we observed any activity related to the bug
   112  	Closed          time.Time
   113  	SubsystemsTime  time.Time // when we have updated subsystems last time
   114  	SubsystemsRev   int
   115  	Reporting       []BugReporting
   116  	Commits         []string // titles of fixing commmits
   117  	CommitInfo      []Commit // additional info for commits (for historical reasons parallel array to Commits)
   118  	HappenedOn      []string // list of managers
   119  	PatchedOn       []string `datastore:",noindex"` // list of managers
   120  	UNCC            []string // don't CC these emails on this bug
   121  	// Kcidb publishing status bitmask:
   122  	// bit 0 - the bug is published
   123  	// bit 1 - don't want to publish it (syzkaller build/test errors)
   124  	KcidbStatus    int64
   125  	DailyStats     []BugDailyStats
   126  	Labels         []BugLabel
   127  	DiscussionInfo []BugDiscussionInfo
   128  	TreeTests      BugTreeTestInfo
   129  	// FixCandidateJob holds the key of the latest successful cross-tree fix bisection job.
   130  	FixCandidateJob string
   131  	ReproAttempts   []BugReproAttempt
   132  }
   133  
   134  type BugTreeTestInfo struct {
   135  	// NeedPoll is set to true if this bug needs to be considered ASAP.
   136  	NeedPoll bool
   137  	// NextPoll can be used to delay the next inspection of the bug.
   138  	NextPoll time.Time
   139  	// List contains latest data about cross-tree patch tests.
   140  	List []BugTreeTest
   141  }
   142  
   143  type BugTreeTest struct {
   144  	CrashID int64
   145  	Repo    string
   146  	Branch  string // May be also equal to a commit.
   147  	// If the values below are set, the testing was done on a merge base.
   148  	MergeBaseRepo   string
   149  	MergeBaseBranch string
   150  	// Below are job keys.
   151  	First      string // The first job that finished successfully.
   152  	FirstOK    string
   153  	FirstCrash string
   154  	Last       string
   155  	Error      string // If some job succeeds afterwards, it should be cleared.
   156  	Pending    string
   157  }
   158  
   159  type BugLabelType string
   160  
   161  type BugLabel struct {
   162  	Label BugLabelType
   163  	// Either empty (for flags) or contains the value.
   164  	Value string
   165  	// The email of the user who manually set this subsystem tag.
   166  	// If empty, the label was set automatically.
   167  	SetBy string
   168  	// Link to the message.
   169  	Link string
   170  }
   171  
   172  func (label BugLabel) String() string {
   173  	if label.Value == "" {
   174  		return string(label.Label)
   175  	}
   176  	return string(label.Label) + ":" + label.Value
   177  }
   178  
   179  // BugReproAttempt describes a single attempt to generate a repro for a bug.
   180  type BugReproAttempt struct {
   181  	Time    time.Time
   182  	Manager string
   183  	Log     int64
   184  }
   185  
   186  func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem, now time.Time, rev int) {
   187  	bug.SubsystemsRev = rev
   188  	bug.SubsystemsTime = now
   189  	var objects []BugLabel
   190  	for _, item := range list {
   191  		objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name})
   192  	}
   193  	bug.SetLabels(makeLabelSet(c, bug.Namespace), objects)
   194  }
   195  
   196  func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error {
   197  	tx := func(c context.Context) error {
   198  		bug := new(Bug)
   199  		if err := db.Get(c, bugKey, bug); err != nil {
   200  			return fmt.Errorf("failed to get bug: %w", err)
   201  		}
   202  		err := transform(bug)
   203  		if err != nil {
   204  			return err
   205  		}
   206  		if _, err := db.Put(c, bugKey, bug); err != nil {
   207  			return fmt.Errorf("failed to put bug: %w", err)
   208  		}
   209  		return nil
   210  	}
   211  	return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10})
   212  }
   213  
   214  func (bug *Bug) hasUserSubsystems() bool {
   215  	return bug.HasUserLabel(SubsystemLabel)
   216  }
   217  
   218  // Initially, subsystem labels were stored as Tags.Subsystems, but over time
   219  // it turned out that we'd better store all labels together.
   220  // Let's keep this conversion code until "Tags" are removed from all bugs.
   221  // Then it can be removed.
   222  
   223  type Bug202304 struct {
   224  	Tags BugTags202304
   225  }
   226  
   227  type BugTags202304 struct {
   228  	Subsystems []BugTag202304
   229  }
   230  
   231  type BugTag202304 struct {
   232  	Name  string
   233  	SetBy string
   234  }
   235  
   236  func (bug *Bug) Load(origProps []db.Property) error {
   237  	// First filer out Tag properties.
   238  	var tags, ps []db.Property
   239  	for _, p := range origProps {
   240  		if strings.HasPrefix(p.Name, "Tags.") {
   241  			tags = append(tags, p)
   242  		} else {
   243  			ps = append(ps, p)
   244  		}
   245  	}
   246  	if err := db.LoadStruct(bug, ps); err != nil {
   247  		return err
   248  	}
   249  	if len(tags) > 0 {
   250  		old := Bug202304{}
   251  		if err := db.LoadStruct(&old, tags); err != nil {
   252  			return err
   253  		}
   254  		for _, entry := range old.Tags.Subsystems {
   255  			bug.Labels = append(bug.Labels, BugLabel{
   256  				Label: SubsystemLabel,
   257  				SetBy: entry.SetBy,
   258  				Value: entry.Name,
   259  			})
   260  		}
   261  	}
   262  	headReproFound := false
   263  	for _, p := range ps {
   264  		if p.Name == "HeadReproLevel" {
   265  			headReproFound = true
   266  			break
   267  		}
   268  	}
   269  	if !headReproFound {
   270  		// The field is new, so it won't be set in all entities.
   271  		// Assume it to be equal to the best found repro for the bug.
   272  		bug.HeadReproLevel = bug.ReproLevel
   273  	}
   274  	return nil
   275  }
   276  
   277  func (bug *Bug) Save() ([]db.Property, error) {
   278  	return db.SaveStruct(bug)
   279  }
   280  
   281  type BugDailyStats struct {
   282  	Date       int // YYYYMMDD
   283  	CrashCount int
   284  }
   285  
   286  type Commit struct {
   287  	Hash       string
   288  	Title      string
   289  	Author     string
   290  	AuthorName string
   291  	CC         string `datastore:",noindex"` // (|-delimited list)
   292  	Date       time.Time
   293  }
   294  
   295  func (com Commit) toDashapi() *dashapi.Commit {
   296  	return &dashapi.Commit{
   297  		Hash:       com.Hash,
   298  		Title:      com.Title,
   299  		Author:     com.Author,
   300  		AuthorName: com.AuthorName,
   301  		Date:       com.Date,
   302  	}
   303  }
   304  
   305  type BugDiscussionInfo struct {
   306  	Source  string
   307  	Summary DiscussionSummary
   308  }
   309  
   310  type DiscussionSummary struct {
   311  	AllMessages      int
   312  	ExternalMessages int
   313  	LastMessage      time.Time
   314  	LastPatchMessage time.Time
   315  }
   316  
   317  type BugReporting struct {
   318  	Name    string // refers to Reporting.Name
   319  	ID      string // unique ID per BUG/BugReporting used in commucation with external systems
   320  	ExtID   string // arbitrary reporting ID that is passed back in dashapi.BugReport
   321  	Link    string
   322  	CC      string // additional emails added to CC list (|-delimited list)
   323  	CrashID int64  // crash that we've last reported in this reporting
   324  	Auto    bool   // was it auto-upstreamed/obsoleted?
   325  	// If Dummy is true, the corresponding Reporting stage was introduced later and the object was just
   326  	// inserted to preserve consistency across the system. Even though it's indicated as Closed and Reported,
   327  	// it never actually was.
   328  	Dummy      bool
   329  	ReproLevel dashapi.ReproLevel // may be less then bug.ReproLevel if repro arrived but we didn't report it yet
   330  	Labels     string             // a comma-separated string of already reported labels
   331  	OnHold     time.Time          // if set, the bug must not be upstreamed
   332  	Reported   time.Time
   333  	Closed     time.Time
   334  }
   335  
   336  func (r *BugReporting) GetLabels() []string {
   337  	return strings.Split(r.Labels, ",")
   338  }
   339  
   340  func (r *BugReporting) AddLabel(label string) {
   341  	newList := unique(append(r.GetLabels(), label))
   342  	r.Labels = strings.Join(newList, ",")
   343  }
   344  
   345  type Crash struct {
   346  	// May be different from bug.Title due to AltTitles.
   347  	// May be empty for old bugs, in such case bug.Title is the right title.
   348  	Title           string
   349  	Manager         string
   350  	BuildID         string
   351  	Time            time.Time
   352  	Reported        time.Time // set if this crash was ever reported
   353  	References      []CrashReference
   354  	Maintainers     []string            `datastore:",noindex"`
   355  	Log             int64               // reference to CrashLog text entity
   356  	Flags           int64               // properties of the Crash
   357  	Report          int64               // reference to CrashReport text entity
   358  	ReportElements  CrashReportElements // parsed parts of the crash report
   359  	ReproOpts       []byte              `datastore:",noindex"`
   360  	ReproSyz        int64               // reference to ReproSyz text entity
   361  	ReproC          int64               // reference to ReproC text entity
   362  	ReproIsRevoked  bool                // the repro no longer triggers the bug on HEAD
   363  	LastReproRetest time.Time           // the last time when the repro was re-checked
   364  	MachineInfo     int64               // Reference to MachineInfo text entity.
   365  	// Custom crash priority for reporting (greater values are higher priority).
   366  	// For example, a crash in mainline kernel has higher priority than a crash in a side branch.
   367  	// For historical reasons this is called ReportLen.
   368  	ReportLen       int64
   369  	Assets          []Asset   // crash-related assets
   370  	AssetsLastCheck time.Time // the last time we checked the assets for deprecation
   371  }
   372  
   373  type CrashReportElements struct {
   374  	GuiltyFiles []string // guilty files as determined during the crash report parsing
   375  }
   376  
   377  type CrashReferenceType string
   378  
   379  const (
   380  	CrashReferenceReporting = "reporting"
   381  	CrashReferenceJob       = "job"
   382  	// This one is needed for backward compatibility.
   383  	crashReferenceUnknown = "unknown"
   384  )
   385  
   386  type CrashReference struct {
   387  	Type CrashReferenceType
   388  	// For CrashReferenceReporting, it refers to Reporting.Name
   389  	// For CrashReferenceJob, it refers to extJobID(jobKey)
   390  	Key  string
   391  	Time time.Time
   392  }
   393  
   394  func (crash *Crash) AddReference(newRef CrashReference) {
   395  	crash.Reported = newRef.Time
   396  	for i, ref := range crash.References {
   397  		if ref.Type != newRef.Type || ref.Key != newRef.Key {
   398  			continue
   399  		}
   400  		crash.References[i].Time = newRef.Time
   401  		return
   402  	}
   403  	crash.References = append(crash.References, newRef)
   404  }
   405  
   406  func (crash *Crash) ClearReference(t CrashReferenceType, key string) {
   407  	newRefs := []CrashReference{}
   408  	crash.Reported = time.Time{}
   409  	for _, ref := range crash.References {
   410  		if ref.Type == t && ref.Key == key {
   411  			continue
   412  		}
   413  		if ref.Time.After(crash.Reported) {
   414  			crash.Reported = ref.Time
   415  		}
   416  		newRefs = append(newRefs, ref)
   417  	}
   418  	crash.References = newRefs
   419  }
   420  
   421  func (crash *Crash) Load(ps []db.Property) error {
   422  	if err := db.LoadStruct(crash, ps); err != nil {
   423  		return err
   424  	}
   425  	// Earlier we only relied on Reported, which does not let us reliably unreport a crash.
   426  	// We need some means of ref counting, so let's create a dummy reference to keep the
   427  	// crash from being purged.
   428  	if !crash.Reported.IsZero() && len(crash.References) == 0 {
   429  		crash.References = append(crash.References, CrashReference{
   430  			Type: crashReferenceUnknown,
   431  			Time: crash.Reported,
   432  		})
   433  	}
   434  	return nil
   435  }
   436  
   437  func (crash *Crash) Save() ([]db.Property, error) {
   438  	return db.SaveStruct(crash)
   439  }
   440  
   441  type Discussion struct {
   442  	ID      string // the base message ID
   443  	Source  string
   444  	Type    string
   445  	Subject string
   446  	BugKeys []string
   447  	// Message contains last N messages.
   448  	// N is supposed to be big enough, so that in almost all cases
   449  	// AllMessages == len(Messages) holds true.
   450  	Messages []DiscussionMessage
   451  	// Since Messages could be trimmed, we have to keep aggregate stats.
   452  	Summary DiscussionSummary
   453  }
   454  
   455  func discussionKey(c context.Context, source, id string) *db.Key {
   456  	return db.NewKey(c, "Discussion", fmt.Sprintf("%v-%v", source, id), 0, nil)
   457  }
   458  
   459  func (d *Discussion) key(c context.Context) *db.Key {
   460  	return discussionKey(c, d.Source, d.ID)
   461  }
   462  
   463  type DiscussionMessage struct {
   464  	ID string
   465  	// External is true if the message is not from the bot itself.
   466  	// Let's use a shorter name to save space.
   467  	External bool      `datastore:"e"`
   468  	Time     time.Time `datastore:",noindex"`
   469  }
   470  
   471  // ReportingState holds dynamic info associated with reporting.
   472  type ReportingState struct {
   473  	Entries []ReportingStateEntry
   474  }
   475  
   476  type ReportingStateEntry struct {
   477  	Namespace string
   478  	Name      string
   479  	// Current reporting quota consumption.
   480  	Sent int
   481  	Date int // YYYYMMDD
   482  }
   483  
   484  // Subsystem holds the history of grouped per-subsystem open bug reminders.
   485  type Subsystem struct {
   486  	Namespace string
   487  	Name      string
   488  	// ListsQueried is the last time bug lists were queried for the subsystem.
   489  	ListsQueried time.Time
   490  	// LastBugList is the last time we have actually managed to generate a bug list.
   491  	LastBugList time.Time
   492  }
   493  
   494  // SubsystemReport holds a single report about open bugs in a subsystem.
   495  // There'll be one record for moderation (if it's needed) and one for actual reporting.
   496  type SubsystemReport struct {
   497  	Created     time.Time
   498  	BugKeys     []string `datastore:",noindex"`
   499  	TotalStats  SubsystemReportStats
   500  	PeriodStats SubsystemReportStats
   501  	Stages      []SubsystemReportStage
   502  }
   503  
   504  func (r *SubsystemReport) getBugKeys() ([]*db.Key, error) {
   505  	ret := []*db.Key{}
   506  	for _, encoded := range r.BugKeys {
   507  		key, err := db.DecodeKey(encoded)
   508  		if err != nil {
   509  			return nil, fmt.Errorf("failed to parse %#v: %w", encoded, err)
   510  		}
   511  		ret = append(ret, key)
   512  	}
   513  	return ret, nil
   514  }
   515  
   516  func (r *SubsystemReport) findStage(id string) *SubsystemReportStage {
   517  	for j := range r.Stages {
   518  		stage := &r.Stages[j]
   519  		if stage.ID == id {
   520  			return stage
   521  		}
   522  	}
   523  	return nil
   524  }
   525  
   526  type SubsystemReportStats struct {
   527  	Reported int
   528  	LowPrio  int
   529  	Fixed    int
   530  }
   531  
   532  func (s *SubsystemReportStats) toDashapi() dashapi.BugListReportStats {
   533  	return dashapi.BugListReportStats{
   534  		Reported: s.Reported,
   535  		LowPrio:  s.LowPrio,
   536  		Fixed:    s.Fixed,
   537  	}
   538  }
   539  
   540  // There can be at most two stages.
   541  // One has Moderation=true, the other one has Moderation=false.
   542  type SubsystemReportStage struct {
   543  	ID         string
   544  	ExtID      string
   545  	Link       string
   546  	Reported   time.Time
   547  	Closed     time.Time
   548  	Moderation bool
   549  }
   550  
   551  // Job represent a single patch testing or bisection job for syz-ci.
   552  // Later we may want to extend this to other types of jobs:
   553  //   - test of a committed fix
   554  //   - reproduce crash
   555  //   - test that crash still happens on HEAD
   556  //
   557  // Job has Bug as parent entity.
   558  type Job struct {
   559  	Type      JobType
   560  	Created   time.Time
   561  	User      string
   562  	CC        []string
   563  	Reporting string
   564  	ExtID     string // email Message-ID
   565  	Link      string // web link for the job (e.g. email in the group)
   566  	Namespace string
   567  	Manager   string
   568  	BugTitle  string
   569  	CrashID   int64
   570  
   571  	// Provided by user:
   572  	KernelRepo   string
   573  	KernelBranch string
   574  	Patch        int64 // reference to Patch text entity
   575  	KernelConfig int64 // reference to the kernel config entity
   576  
   577  	Attempts    int       // number of times we tried to execute this job
   578  	IsRunning   bool      // the job might have been started, but never finished
   579  	LastStarted time.Time `datastore:"Started"`
   580  	Finished    time.Time // if set, job is finished
   581  	TreeOrigin  bool      // whether the job is related to tree origin detection
   582  
   583  	// If patch test should be done on the merge base between two branches.
   584  	MergeBaseRepo   string
   585  	MergeBaseBranch string
   586  
   587  	// By default, bisection starts from the revision of the associated crash.
   588  	// The BisectFrom field can override this.
   589  	BisectFrom string
   590  
   591  	// Result of execution:
   592  	CrashTitle  string // if empty, we did not hit crash during testing
   593  	CrashLog    int64  // reference to CrashLog text entity
   594  	CrashReport int64  // reference to CrashReport text entity
   595  	Commits     []Commit
   596  	BuildID     string
   597  	Log         int64 // reference to Log text entity
   598  	Error       int64 // reference to Error text entity, if set job failed
   599  	Flags       dashapi.JobDoneFlags
   600  
   601  	Reported         bool   // have we reported result back to user?
   602  	InvalidatedBy    string // user who marked this bug as invalid, empty by default
   603  	BackportedCommit Commit
   604  }
   605  
   606  func (job *Job) IsBisection() bool {
   607  	return job.Type == JobBisectCause || job.Type == JobBisectFix
   608  }
   609  
   610  func (job *Job) IsFinished() bool {
   611  	return !job.Finished.IsZero()
   612  }
   613  
   614  type JobType int
   615  
   616  const (
   617  	JobTestPatch JobType = iota
   618  	JobBisectCause
   619  	JobBisectFix
   620  )
   621  
   622  func (typ JobType) toDashapiReportType() dashapi.ReportType {
   623  	switch typ {
   624  	case JobTestPatch:
   625  		return dashapi.ReportTestPatch
   626  	case JobBisectCause:
   627  		return dashapi.ReportBisectCause
   628  	case JobBisectFix:
   629  		return dashapi.ReportBisectFix
   630  	default:
   631  		panic(fmt.Sprintf("unknown job type %v", typ))
   632  	}
   633  }
   634  
   635  func (job *Job) isUnreliableBisect() bool {
   636  	if job.Type != JobBisectCause && job.Type != JobBisectFix {
   637  		panic(fmt.Sprintf("bad job type %v", job.Type))
   638  	}
   639  	// If a bisection points to a merge or a commit that does not affect the kernel binary,
   640  	// it is considered an unreliable/wrong result and should not be reported in emails.
   641  	return job.Flags&dashapi.BisectResultMerge != 0 ||
   642  		job.Flags&dashapi.BisectResultNoop != 0 ||
   643  		job.Flags&dashapi.BisectResultRelease != 0 ||
   644  		job.Flags&dashapi.BisectResultIgnore != 0
   645  }
   646  
   647  func (job *Job) IsCrossTree() bool {
   648  	return job.MergeBaseRepo != "" && job.IsBisection()
   649  }
   650  
   651  // Text holds text blobs (crash logs, reports, reproducers, etc).
   652  type Text struct {
   653  	Namespace string
   654  	Text      []byte `datastore:",noindex"` // gzip-compressed text
   655  }
   656  
   657  const (
   658  	textCrashLog     = "CrashLog"
   659  	textCrashReport  = "CrashReport"
   660  	textReproSyz     = "ReproSyz"
   661  	textReproC       = "ReproC"
   662  	textMachineInfo  = "MachineInfo"
   663  	textKernelConfig = "KernelConfig"
   664  	textPatch        = "Patch"
   665  	textLog          = "Log"
   666  	textError        = "Error"
   667  	textReproLog     = "ReproLog"
   668  )
   669  
   670  const (
   671  	BugStatusOpen = iota
   672  )
   673  
   674  const (
   675  	BugStatusFixed = 1000 + iota
   676  	BugStatusInvalid
   677  	BugStatusDup
   678  )
   679  
   680  const (
   681  	ReproLevelNone = dashapi.ReproLevelNone
   682  	ReproLevelSyz  = dashapi.ReproLevelSyz
   683  	ReproLevelC    = dashapi.ReproLevelC
   684  )
   685  
   686  type BuildType int
   687  
   688  const (
   689  	BuildNormal BuildType = iota
   690  	BuildFailed
   691  	BuildJob
   692  )
   693  
   694  type BisectStatus int
   695  
   696  const (
   697  	BisectNot BisectStatus = iota
   698  	BisectPending
   699  	BisectError
   700  	BisectYes          // have 1 commit
   701  	BisectUnreliable   // have 1 commit, but suspect it's wrong
   702  	BisectInconclusive // multiple commits due to skips
   703  	BisectHorizont     // happens on the oldest commit we can test (or HEAD for fix bisection)
   704  	bisectStatusLast   // this value can be changed (not stored in datastore)
   705  )
   706  
   707  func (status BisectStatus) String() string {
   708  	switch status {
   709  	case BisectError:
   710  		return "error"
   711  	case BisectYes:
   712  		return "done"
   713  	case BisectUnreliable:
   714  		return "unreliable"
   715  	case BisectInconclusive:
   716  		return "inconclusive"
   717  	case BisectHorizont:
   718  		return "inconclusive"
   719  	default:
   720  		return ""
   721  	}
   722  }
   723  
   724  func mgrKey(c context.Context, ns, name string) *db.Key {
   725  	return db.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil)
   726  }
   727  
   728  func (mgr *Manager) key(c context.Context) *db.Key {
   729  	return mgrKey(c, mgr.Namespace, mgr.Name)
   730  }
   731  
   732  func loadManager(c context.Context, ns, name string) (*Manager, error) {
   733  	mgr := new(Manager)
   734  	if err := db.Get(c, mgrKey(c, ns, name), mgr); err != nil {
   735  		if err != db.ErrNoSuchEntity {
   736  			return nil, fmt.Errorf("failed to get manager %v/%v: %w", ns, name, err)
   737  		}
   738  		mgr = &Manager{
   739  			Namespace: ns,
   740  			Name:      name,
   741  		}
   742  	}
   743  	return mgr, nil
   744  }
   745  
   746  // updateManager does transactional compare-and-swap on the manager and its current stats.
   747  func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats) error) error {
   748  	date := timeDate(timeNow(c))
   749  	tx := func(c context.Context) error {
   750  		mgr, err := loadManager(c, ns, name)
   751  		if err != nil {
   752  			return err
   753  		}
   754  		mgrKey := mgr.key(c)
   755  		stats := new(ManagerStats)
   756  		statsKey := db.NewKey(c, "ManagerStats", "", int64(date), mgrKey)
   757  		if err := db.Get(c, statsKey, stats); err != nil {
   758  			if err != db.ErrNoSuchEntity {
   759  				return fmt.Errorf("failed to get stats %v/%v/%v: %w", ns, name, date, err)
   760  			}
   761  			stats = &ManagerStats{
   762  				Date: date,
   763  			}
   764  		}
   765  
   766  		if err := fn(mgr, stats); err != nil {
   767  			return err
   768  		}
   769  
   770  		if _, err := db.Put(c, mgrKey, mgr); err != nil {
   771  			return fmt.Errorf("failed to put manager: %w", err)
   772  		}
   773  		if _, err := db.Put(c, statsKey, stats); err != nil {
   774  			return fmt.Errorf("failed to put manager stats: %w", err)
   775  		}
   776  		return nil
   777  	}
   778  	return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10})
   779  }
   780  
   781  func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) {
   782  	var managers []*Manager
   783  	query := db.NewQuery("Manager")
   784  	if ns != "" {
   785  		query = query.Filter("Namespace=", ns)
   786  	}
   787  	keys, err := query.GetAll(c, &managers)
   788  	if err != nil {
   789  		return nil, nil, fmt.Errorf("failed to query managers: %w", err)
   790  	}
   791  	var result []*Manager
   792  	var resultKeys []*db.Key
   793  	for i, mgr := range managers {
   794  		if getNsConfig(c, mgr.Namespace).Managers[mgr.Name].Decommissioned {
   795  			continue
   796  		}
   797  		result = append(result, mgr)
   798  		resultKeys = append(resultKeys, keys[i])
   799  	}
   800  	return result, resultKeys, nil
   801  }
   802  
   803  func buildKey(c context.Context, ns, id string) *db.Key {
   804  	if ns == "" {
   805  		panic("requesting build key outside of namespace")
   806  	}
   807  	h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id)))
   808  	return db.NewKey(c, "Build", h, 0, nil)
   809  }
   810  
   811  func loadBuild(c context.Context, ns, id string) (*Build, error) {
   812  	build := new(Build)
   813  	if err := db.Get(c, buildKey(c, ns, id), build); err != nil {
   814  		if err == db.ErrNoSuchEntity {
   815  			return nil, fmt.Errorf("unknown build %v/%v", ns, id)
   816  		}
   817  		return nil, fmt.Errorf("failed to get build %v/%v: %w", ns, id, err)
   818  	}
   819  	return build, nil
   820  }
   821  
   822  func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) {
   823  	mgr, err := loadManager(c, ns, manager)
   824  	if err != nil {
   825  		return nil, err
   826  	}
   827  	if mgr.CurrentBuild == "" {
   828  		return nil, fmt.Errorf("failed to fetch manager build: no builds")
   829  	}
   830  	return loadBuild(c, ns, mgr.CurrentBuild)
   831  }
   832  
   833  func loadBuilds(c context.Context, ns, manager string, typ BuildType) ([]*Build, error) {
   834  	const limit = 500
   835  	var builds []*Build
   836  	_, err := db.NewQuery("Build").
   837  		Filter("Namespace=", ns).
   838  		Filter("Manager=", manager).
   839  		Filter("Type=", typ).
   840  		Order("-Time").
   841  		Limit(limit).
   842  		GetAll(c, &builds)
   843  	if err != nil {
   844  		return nil, err
   845  	}
   846  	return builds, nil
   847  }
   848  
   849  func (bug *Bug) displayTitle() string {
   850  	if bug.Seq == 0 {
   851  		return bug.Title
   852  	}
   853  	return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
   854  }
   855  
   856  var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`)
   857  
   858  func splitDisplayTitle(display string) (string, int64, error) {
   859  	match := displayTitleRe.FindStringSubmatchIndex(display)
   860  	if match == nil {
   861  		return display, 0, nil
   862  	}
   863  	title := display[match[2]:match[3]]
   864  	seqStr := display[match[4]:match[5]]
   865  	seq, err := strconv.ParseInt(seqStr, 10, 64)
   866  	if err != nil {
   867  		return "", 0, fmt.Errorf("failed to parse bug title: %w", err)
   868  	}
   869  	if seq <= 0 || seq > 1e6 {
   870  		return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
   871  	}
   872  	return title, seq - 1, nil
   873  }
   874  
   875  func canonicalBug(c context.Context, bug *Bug) (*Bug, error) {
   876  	for {
   877  		if bug.Status != BugStatusDup {
   878  			return bug, nil
   879  		}
   880  		canon := new(Bug)
   881  		bugKey := db.NewKey(c, "Bug", bug.DupOf, 0, nil)
   882  		if err := db.Get(c, bugKey, canon); err != nil {
   883  			return nil, fmt.Errorf("failed to get dup bug %q for %q: %w",
   884  				bug.DupOf, bug.keyHash(c), err)
   885  		}
   886  		bug = canon
   887  	}
   888  }
   889  
   890  func (bug *Bug) key(c context.Context) *db.Key {
   891  	return db.NewKey(c, "Bug", bug.keyHash(c), 0, nil)
   892  }
   893  
   894  func (bug *Bug) keyHash(c context.Context) string {
   895  	return bugKeyHash(c, bug.Namespace, bug.Title, bug.Seq)
   896  }
   897  
   898  func bugKeyHash(c context.Context, ns, title string, seq int64) string {
   899  	return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq)))
   900  }
   901  
   902  func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) {
   903  	domain := getNsConfig(c, bug.Namespace).SimilarityDomain
   904  	dedup := make(map[string]bool)
   905  	dedup[bug.keyHash(c)] = true
   906  
   907  	ret := []*Bug{}
   908  	for _, title := range bug.AltTitles {
   909  		var similar []*Bug
   910  		_, err := db.NewQuery("Bug").
   911  			Filter("AltTitles=", title).
   912  			GetAll(c, &similar)
   913  		if err != nil {
   914  			return nil, err
   915  		}
   916  		for _, bug := range similar {
   917  			if getNsConfig(c, bug.Namespace).SimilarityDomain != domain ||
   918  				dedup[bug.keyHash(c)] {
   919  				continue
   920  			}
   921  			dedup[bug.keyHash(c)] = true
   922  			ret = append(ret, bug)
   923  		}
   924  	}
   925  	return ret, nil
   926  }
   927  
   928  // Since these IDs appear in Reported-by tags in commit, we slightly limit their size.
   929  const reportingHashLen = 20
   930  
   931  func bugReportingHash(bugHash, reporting string) string {
   932  	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:reportingHashLen]
   933  }
   934  
   935  func looksLikeReportingHash(id string) bool {
   936  	// This is only used as best-effort check.
   937  	// Now we produce 20-chars ids, but we used to use full sha1 hash.
   938  	return len(id) == reportingHashLen || len(id) == 2*len(hash.Sig{})
   939  }
   940  
   941  func (bug *Bug) updateCommits(commits []string, now time.Time) {
   942  	bug.Commits = commits
   943  	bug.CommitInfo = nil
   944  	bug.NeedCommitInfo = true
   945  	bug.FixTime = now
   946  	bug.PatchedOn = nil
   947  }
   948  
   949  func (bug *Bug) getCommitInfo(i int) Commit {
   950  	if i < len(bug.CommitInfo) {
   951  		return bug.CommitInfo[i]
   952  	}
   953  	return Commit{}
   954  }
   955  
   956  func (bug *Bug) increaseCrashStats(now time.Time) {
   957  	bug.NumCrashes++
   958  	date := timeDate(now)
   959  	if len(bug.DailyStats) == 0 || bug.DailyStats[len(bug.DailyStats)-1].Date < date {
   960  		bug.DailyStats = append(bug.DailyStats, BugDailyStats{date, 1})
   961  	} else {
   962  		// It is theoretically possible that this method might get into a situation, when
   963  		// the latest saved date is later than now. But we assume that this can only happen
   964  		// in a small window around the start of the day and it is better to attribute a
   965  		// crash to the next day than to get a mismatch between NumCrashes and the sum of
   966  		// CrashCount.
   967  		bug.DailyStats[len(bug.DailyStats)-1].CrashCount++
   968  	}
   969  
   970  	if len(bug.DailyStats) > maxBugHistoryDays {
   971  		bug.DailyStats = bug.DailyStats[len(bug.DailyStats)-maxBugHistoryDays:]
   972  	}
   973  }
   974  
   975  func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats {
   976  	startDate := timeDate(from)
   977  	startPos := len(bug.DailyStats)
   978  	for ; startPos > 0; startPos-- {
   979  		if bug.DailyStats[startPos-1].Date < startDate {
   980  			break
   981  		}
   982  	}
   983  	return bug.DailyStats[startPos:]
   984  }
   985  
   986  func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) {
   987  	var status dashapi.BugStatus
   988  	switch bug.Status {
   989  	case BugStatusOpen:
   990  		status = dashapi.BugStatusOpen
   991  	case BugStatusFixed:
   992  		status = dashapi.BugStatusFixed
   993  	case BugStatusInvalid:
   994  		status = dashapi.BugStatusInvalid
   995  	case BugStatusDup:
   996  		status = dashapi.BugStatusDup
   997  	default:
   998  		return status, fmt.Errorf("unknown bugs status %v", bug.Status)
   999  	}
  1000  	return status, nil
  1001  }
  1002  
  1003  // If an entity of type EmergencyStop exists, syzbot's operation is paused until
  1004  // a support engineer deletes it from the DB.
  1005  type EmergencyStop struct {
  1006  	Time time.Time
  1007  	User string
  1008  }
  1009  
  1010  func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error {
  1011  	crash := new(Crash)
  1012  	crashKey := db.NewKey(c, "Crash", "", crashID, bugKey)
  1013  	if err := db.Get(c, crashKey, crash); err != nil {
  1014  		return fmt.Errorf("failed to get reported crash %v: %w", crashID, err)
  1015  	}
  1016  	crash.AddReference(ref)
  1017  	if _, err := db.Put(c, crashKey, crash); err != nil {
  1018  		return fmt.Errorf("failed to put reported crash %v: %w", crashID, err)
  1019  	}
  1020  	return nil
  1021  }
  1022  
  1023  func removeCrashReference(c context.Context, crashID int64, bugKey *db.Key,
  1024  	t CrashReferenceType, key string) error {
  1025  	crash := new(Crash)
  1026  	crashKey := db.NewKey(c, "Crash", "", crashID, bugKey)
  1027  	if err := db.Get(c, crashKey, crash); err != nil {
  1028  		return fmt.Errorf("failed to get reported crash %v: %w", crashID, err)
  1029  	}
  1030  	crash.ClearReference(t, key)
  1031  	if _, err := db.Put(c, crashKey, crash); err != nil {
  1032  		return fmt.Errorf("failed to put reported crash %v: %w", crashID, err)
  1033  	}
  1034  	return nil
  1035  }
  1036  
  1037  func kernelRepoInfo(c context.Context, build *Build) KernelRepo {
  1038  	return kernelRepoInfoRaw(c, build.Namespace, build.KernelRepo, build.KernelBranch)
  1039  }
  1040  
  1041  func kernelRepoInfoRaw(c context.Context, ns, url, branch string) KernelRepo {
  1042  	var info KernelRepo
  1043  	for _, repo := range getNsConfig(c, ns).Repos {
  1044  		if repo.URL == url && repo.Branch == branch {
  1045  			info = repo
  1046  			break
  1047  		}
  1048  	}
  1049  	if info.Alias == "" {
  1050  		info.Alias = url
  1051  		if branch != "" {
  1052  			info.Alias += " " + branch
  1053  		}
  1054  	}
  1055  	return info
  1056  }
  1057  
  1058  func textLink(tag string, id int64) string {
  1059  	if id == 0 {
  1060  		return ""
  1061  	}
  1062  	return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16))
  1063  }
  1064  
  1065  // timeDate returns t's date as a single int YYYYMMDD.
  1066  func timeDate(t time.Time) int {
  1067  	year, month, day := t.Date()
  1068  	return year*10000 + int(month)*100 + day
  1069  }
  1070  
  1071  func stringInList(list []string, str string) bool {
  1072  	for _, s := range list {
  1073  		if s == str {
  1074  			return true
  1075  		}
  1076  	}
  1077  	return false
  1078  }
  1079  
  1080  func stringListsIntersect(a, b []string) bool {
  1081  	m := map[string]bool{}
  1082  	for _, strA := range a {
  1083  		m[strA] = true
  1084  	}
  1085  	for _, strB := range b {
  1086  		if m[strB] {
  1087  			return true
  1088  		}
  1089  	}
  1090  	return false
  1091  }
  1092  
  1093  func mergeString(list []string, str string) []string {
  1094  	if !stringInList(list, str) {
  1095  		list = append(list, str)
  1096  	}
  1097  	return list
  1098  }
  1099  
  1100  func mergeStringList(list, add []string) []string {
  1101  	for _, str := range add {
  1102  		list = mergeString(list, str)
  1103  	}
  1104  	return list
  1105  }
  1106  
  1107  // dateTime converts date in YYYYMMDD format back to Time.
  1108  func dateTime(date int) time.Time {
  1109  	return time.Date(date/10000, time.Month(date/100%100), date%100, 0, 0, 0, 0, time.UTC)
  1110  }