github.com/google/syzkaller@v0.0.0-20240517125934-c0f1611a36d6/dashboard/app/entities.go (about) 1 // Copyright 2017 syzkaller project authors. All rights reserved. 2 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4 package main 5 6 import ( 7 "context" 8 "fmt" 9 "regexp" 10 "strconv" 11 "strings" 12 "time" 13 14 "github.com/google/syzkaller/dashboard/dashapi" 15 "github.com/google/syzkaller/pkg/hash" 16 "github.com/google/syzkaller/pkg/subsystem" 17 db "google.golang.org/appengine/v2/datastore" 18 ) 19 20 // This file contains definitions of entities stored in datastore. 21 22 const ( 23 maxTextLen = 200 24 MaxStringLen = 1024 25 26 maxBugHistoryDays = 365 * 5 27 ) 28 29 type Manager struct { 30 Namespace string 31 Name string 32 Link string 33 CurrentBuild string 34 FailedBuildBug string 35 FailedSyzBuildBug string 36 LastAlive time.Time 37 CurrentUpTime time.Duration 38 LastGeneratedJob time.Time 39 } 40 41 // ManagerStats holds per-day manager runtime stats. 42 // Has Manager as parent entity. Keyed by Date. 43 type ManagerStats struct { 44 Date int // YYYYMMDD 45 MaxCorpus int64 46 MaxPCs int64 // coverage 47 MaxCover int64 // what we call feedback signal everywhere else 48 TotalFuzzingTime time.Duration 49 TotalCrashes int64 50 CrashTypes int64 // unique crash types 51 SuppressedCrashes int64 52 TotalExecs int64 53 // These are only recorded once right after corpus is triaged. 54 TriagedCoverage int64 55 TriagedPCs int64 56 } 57 58 type Asset struct { 59 Type dashapi.AssetType 60 DownloadURL string 61 CreateDate time.Time 62 } 63 64 type Build struct { 65 Namespace string 66 Manager string 67 ID string // unique ID generated by syz-ci 68 Type BuildType 69 Time time.Time 70 OS string 71 Arch string 72 VMArch string 73 SyzkallerCommit string 74 SyzkallerCommitDate time.Time 75 CompilerID string 76 KernelRepo string 77 KernelBranch string 78 KernelCommit string 79 KernelCommitTitle string `datastore:",noindex"` 80 KernelCommitDate time.Time `datastore:",noindex"` 81 KernelConfig int64 // reference to KernelConfig text entity 82 Assets []Asset // build-related assets 83 AssetsLastCheck time.Time // the last time we checked the assets for deprecation 84 } 85 86 type Bug struct { 87 Namespace string 88 Seq int64 // sequences of the bug with the same title 89 Title string 90 MergedTitles []string // crash titles that we already merged into this bug 91 AltTitles []string // alternative crash titles that we may merge into this bug 92 Status int 93 StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why 94 DupOf string 95 NumCrashes int64 96 NumRepro int64 97 // ReproLevel is the best ever found repro level for this bug. 98 // HeadReproLevel is best known repro level that still works on the HEAD commit. 99 ReproLevel dashapi.ReproLevel 100 HeadReproLevel dashapi.ReproLevel `datastore:"HeadReproLevel"` 101 BisectCause BisectStatus 102 BisectFix BisectStatus 103 HasReport bool 104 NeedCommitInfo bool 105 FirstTime time.Time 106 LastTime time.Time 107 LastSavedCrash time.Time 108 LastReproTime time.Time 109 LastCauseBisect time.Time 110 FixTime time.Time // when we become aware of the fixing commit 111 LastActivity time.Time // last time we observed any activity related to the bug 112 Closed time.Time 113 SubsystemsTime time.Time // when we have updated subsystems last time 114 SubsystemsRev int 115 Reporting []BugReporting 116 Commits []string // titles of fixing commmits 117 CommitInfo []Commit // additional info for commits (for historical reasons parallel array to Commits) 118 HappenedOn []string // list of managers 119 PatchedOn []string `datastore:",noindex"` // list of managers 120 UNCC []string // don't CC these emails on this bug 121 // Kcidb publishing status bitmask: 122 // bit 0 - the bug is published 123 // bit 1 - don't want to publish it (syzkaller build/test errors) 124 KcidbStatus int64 125 DailyStats []BugDailyStats 126 Labels []BugLabel 127 DiscussionInfo []BugDiscussionInfo 128 TreeTests BugTreeTestInfo 129 // FixCandidateJob holds the key of the latest successful cross-tree fix bisection job. 130 FixCandidateJob string 131 ReproAttempts []BugReproAttempt 132 } 133 134 type BugTreeTestInfo struct { 135 // NeedPoll is set to true if this bug needs to be considered ASAP. 136 NeedPoll bool 137 // NextPoll can be used to delay the next inspection of the bug. 138 NextPoll time.Time 139 // List contains latest data about cross-tree patch tests. 140 List []BugTreeTest 141 } 142 143 type BugTreeTest struct { 144 CrashID int64 145 Repo string 146 Branch string // May be also equal to a commit. 147 // If the values below are set, the testing was done on a merge base. 148 MergeBaseRepo string 149 MergeBaseBranch string 150 // Below are job keys. 151 First string // The first job that finished successfully. 152 FirstOK string 153 FirstCrash string 154 Last string 155 Error string // If some job succeeds afterwards, it should be cleared. 156 Pending string 157 } 158 159 type BugLabelType string 160 161 type BugLabel struct { 162 Label BugLabelType 163 // Either empty (for flags) or contains the value. 164 Value string 165 // The email of the user who manually set this subsystem tag. 166 // If empty, the label was set automatically. 167 SetBy string 168 // Link to the message. 169 Link string 170 } 171 172 func (label BugLabel) String() string { 173 if label.Value == "" { 174 return string(label.Label) 175 } 176 return string(label.Label) + ":" + label.Value 177 } 178 179 // BugReproAttempt describes a single attempt to generate a repro for a bug. 180 type BugReproAttempt struct { 181 Time time.Time 182 Manager string 183 Log int64 184 } 185 186 func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem, now time.Time, rev int) { 187 bug.SubsystemsRev = rev 188 bug.SubsystemsTime = now 189 var objects []BugLabel 190 for _, item := range list { 191 objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name}) 192 } 193 bug.SetLabels(makeLabelSet(c, bug.Namespace), objects) 194 } 195 196 func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error { 197 tx := func(c context.Context) error { 198 bug := new(Bug) 199 if err := db.Get(c, bugKey, bug); err != nil { 200 return fmt.Errorf("failed to get bug: %w", err) 201 } 202 err := transform(bug) 203 if err != nil { 204 return err 205 } 206 if _, err := db.Put(c, bugKey, bug); err != nil { 207 return fmt.Errorf("failed to put bug: %w", err) 208 } 209 return nil 210 } 211 return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) 212 } 213 214 func (bug *Bug) hasUserSubsystems() bool { 215 return bug.HasUserLabel(SubsystemLabel) 216 } 217 218 // Initially, subsystem labels were stored as Tags.Subsystems, but over time 219 // it turned out that we'd better store all labels together. 220 // Let's keep this conversion code until "Tags" are removed from all bugs. 221 // Then it can be removed. 222 223 type Bug202304 struct { 224 Tags BugTags202304 225 } 226 227 type BugTags202304 struct { 228 Subsystems []BugTag202304 229 } 230 231 type BugTag202304 struct { 232 Name string 233 SetBy string 234 } 235 236 func (bug *Bug) Load(origProps []db.Property) error { 237 // First filer out Tag properties. 238 var tags, ps []db.Property 239 for _, p := range origProps { 240 if strings.HasPrefix(p.Name, "Tags.") { 241 tags = append(tags, p) 242 } else { 243 ps = append(ps, p) 244 } 245 } 246 if err := db.LoadStruct(bug, ps); err != nil { 247 return err 248 } 249 if len(tags) > 0 { 250 old := Bug202304{} 251 if err := db.LoadStruct(&old, tags); err != nil { 252 return err 253 } 254 for _, entry := range old.Tags.Subsystems { 255 bug.Labels = append(bug.Labels, BugLabel{ 256 Label: SubsystemLabel, 257 SetBy: entry.SetBy, 258 Value: entry.Name, 259 }) 260 } 261 } 262 headReproFound := false 263 for _, p := range ps { 264 if p.Name == "HeadReproLevel" { 265 headReproFound = true 266 break 267 } 268 } 269 if !headReproFound { 270 // The field is new, so it won't be set in all entities. 271 // Assume it to be equal to the best found repro for the bug. 272 bug.HeadReproLevel = bug.ReproLevel 273 } 274 return nil 275 } 276 277 func (bug *Bug) Save() ([]db.Property, error) { 278 return db.SaveStruct(bug) 279 } 280 281 type BugDailyStats struct { 282 Date int // YYYYMMDD 283 CrashCount int 284 } 285 286 type Commit struct { 287 Hash string 288 Title string 289 Author string 290 AuthorName string 291 CC string `datastore:",noindex"` // (|-delimited list) 292 Date time.Time 293 } 294 295 func (com Commit) toDashapi() *dashapi.Commit { 296 return &dashapi.Commit{ 297 Hash: com.Hash, 298 Title: com.Title, 299 Author: com.Author, 300 AuthorName: com.AuthorName, 301 Date: com.Date, 302 } 303 } 304 305 type BugDiscussionInfo struct { 306 Source string 307 Summary DiscussionSummary 308 } 309 310 type DiscussionSummary struct { 311 AllMessages int 312 ExternalMessages int 313 LastMessage time.Time 314 LastPatchMessage time.Time 315 } 316 317 type BugReporting struct { 318 Name string // refers to Reporting.Name 319 ID string // unique ID per BUG/BugReporting used in commucation with external systems 320 ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport 321 Link string 322 CC string // additional emails added to CC list (|-delimited list) 323 CrashID int64 // crash that we've last reported in this reporting 324 Auto bool // was it auto-upstreamed/obsoleted? 325 // If Dummy is true, the corresponding Reporting stage was introduced later and the object was just 326 // inserted to preserve consistency across the system. Even though it's indicated as Closed and Reported, 327 // it never actually was. 328 Dummy bool 329 ReproLevel dashapi.ReproLevel // may be less then bug.ReproLevel if repro arrived but we didn't report it yet 330 Labels string // a comma-separated string of already reported labels 331 OnHold time.Time // if set, the bug must not be upstreamed 332 Reported time.Time 333 Closed time.Time 334 } 335 336 func (r *BugReporting) GetLabels() []string { 337 return strings.Split(r.Labels, ",") 338 } 339 340 func (r *BugReporting) AddLabel(label string) { 341 newList := unique(append(r.GetLabels(), label)) 342 r.Labels = strings.Join(newList, ",") 343 } 344 345 type Crash struct { 346 // May be different from bug.Title due to AltTitles. 347 // May be empty for old bugs, in such case bug.Title is the right title. 348 Title string 349 Manager string 350 BuildID string 351 Time time.Time 352 Reported time.Time // set if this crash was ever reported 353 References []CrashReference 354 Maintainers []string `datastore:",noindex"` 355 Log int64 // reference to CrashLog text entity 356 Flags int64 // properties of the Crash 357 Report int64 // reference to CrashReport text entity 358 ReportElements CrashReportElements // parsed parts of the crash report 359 ReproOpts []byte `datastore:",noindex"` 360 ReproSyz int64 // reference to ReproSyz text entity 361 ReproC int64 // reference to ReproC text entity 362 ReproIsRevoked bool // the repro no longer triggers the bug on HEAD 363 LastReproRetest time.Time // the last time when the repro was re-checked 364 MachineInfo int64 // Reference to MachineInfo text entity. 365 // Custom crash priority for reporting (greater values are higher priority). 366 // For example, a crash in mainline kernel has higher priority than a crash in a side branch. 367 // For historical reasons this is called ReportLen. 368 ReportLen int64 369 Assets []Asset // crash-related assets 370 AssetsLastCheck time.Time // the last time we checked the assets for deprecation 371 } 372 373 type CrashReportElements struct { 374 GuiltyFiles []string // guilty files as determined during the crash report parsing 375 } 376 377 type CrashReferenceType string 378 379 const ( 380 CrashReferenceReporting = "reporting" 381 CrashReferenceJob = "job" 382 // This one is needed for backward compatibility. 383 crashReferenceUnknown = "unknown" 384 ) 385 386 type CrashReference struct { 387 Type CrashReferenceType 388 // For CrashReferenceReporting, it refers to Reporting.Name 389 // For CrashReferenceJob, it refers to extJobID(jobKey) 390 Key string 391 Time time.Time 392 } 393 394 func (crash *Crash) AddReference(newRef CrashReference) { 395 crash.Reported = newRef.Time 396 for i, ref := range crash.References { 397 if ref.Type != newRef.Type || ref.Key != newRef.Key { 398 continue 399 } 400 crash.References[i].Time = newRef.Time 401 return 402 } 403 crash.References = append(crash.References, newRef) 404 } 405 406 func (crash *Crash) ClearReference(t CrashReferenceType, key string) { 407 newRefs := []CrashReference{} 408 crash.Reported = time.Time{} 409 for _, ref := range crash.References { 410 if ref.Type == t && ref.Key == key { 411 continue 412 } 413 if ref.Time.After(crash.Reported) { 414 crash.Reported = ref.Time 415 } 416 newRefs = append(newRefs, ref) 417 } 418 crash.References = newRefs 419 } 420 421 func (crash *Crash) Load(ps []db.Property) error { 422 if err := db.LoadStruct(crash, ps); err != nil { 423 return err 424 } 425 // Earlier we only relied on Reported, which does not let us reliably unreport a crash. 426 // We need some means of ref counting, so let's create a dummy reference to keep the 427 // crash from being purged. 428 if !crash.Reported.IsZero() && len(crash.References) == 0 { 429 crash.References = append(crash.References, CrashReference{ 430 Type: crashReferenceUnknown, 431 Time: crash.Reported, 432 }) 433 } 434 return nil 435 } 436 437 func (crash *Crash) Save() ([]db.Property, error) { 438 return db.SaveStruct(crash) 439 } 440 441 type Discussion struct { 442 ID string // the base message ID 443 Source string 444 Type string 445 Subject string 446 BugKeys []string 447 // Message contains last N messages. 448 // N is supposed to be big enough, so that in almost all cases 449 // AllMessages == len(Messages) holds true. 450 Messages []DiscussionMessage 451 // Since Messages could be trimmed, we have to keep aggregate stats. 452 Summary DiscussionSummary 453 } 454 455 func discussionKey(c context.Context, source, id string) *db.Key { 456 return db.NewKey(c, "Discussion", fmt.Sprintf("%v-%v", source, id), 0, nil) 457 } 458 459 func (d *Discussion) key(c context.Context) *db.Key { 460 return discussionKey(c, d.Source, d.ID) 461 } 462 463 type DiscussionMessage struct { 464 ID string 465 // External is true if the message is not from the bot itself. 466 // Let's use a shorter name to save space. 467 External bool `datastore:"e"` 468 Time time.Time `datastore:",noindex"` 469 } 470 471 // ReportingState holds dynamic info associated with reporting. 472 type ReportingState struct { 473 Entries []ReportingStateEntry 474 } 475 476 type ReportingStateEntry struct { 477 Namespace string 478 Name string 479 // Current reporting quota consumption. 480 Sent int 481 Date int // YYYYMMDD 482 } 483 484 // Subsystem holds the history of grouped per-subsystem open bug reminders. 485 type Subsystem struct { 486 Namespace string 487 Name string 488 // ListsQueried is the last time bug lists were queried for the subsystem. 489 ListsQueried time.Time 490 // LastBugList is the last time we have actually managed to generate a bug list. 491 LastBugList time.Time 492 } 493 494 // SubsystemReport holds a single report about open bugs in a subsystem. 495 // There'll be one record for moderation (if it's needed) and one for actual reporting. 496 type SubsystemReport struct { 497 Created time.Time 498 BugKeys []string `datastore:",noindex"` 499 TotalStats SubsystemReportStats 500 PeriodStats SubsystemReportStats 501 Stages []SubsystemReportStage 502 } 503 504 func (r *SubsystemReport) getBugKeys() ([]*db.Key, error) { 505 ret := []*db.Key{} 506 for _, encoded := range r.BugKeys { 507 key, err := db.DecodeKey(encoded) 508 if err != nil { 509 return nil, fmt.Errorf("failed to parse %#v: %w", encoded, err) 510 } 511 ret = append(ret, key) 512 } 513 return ret, nil 514 } 515 516 func (r *SubsystemReport) findStage(id string) *SubsystemReportStage { 517 for j := range r.Stages { 518 stage := &r.Stages[j] 519 if stage.ID == id { 520 return stage 521 } 522 } 523 return nil 524 } 525 526 type SubsystemReportStats struct { 527 Reported int 528 LowPrio int 529 Fixed int 530 } 531 532 func (s *SubsystemReportStats) toDashapi() dashapi.BugListReportStats { 533 return dashapi.BugListReportStats{ 534 Reported: s.Reported, 535 LowPrio: s.LowPrio, 536 Fixed: s.Fixed, 537 } 538 } 539 540 // There can be at most two stages. 541 // One has Moderation=true, the other one has Moderation=false. 542 type SubsystemReportStage struct { 543 ID string 544 ExtID string 545 Link string 546 Reported time.Time 547 Closed time.Time 548 Moderation bool 549 } 550 551 // Job represent a single patch testing or bisection job for syz-ci. 552 // Later we may want to extend this to other types of jobs: 553 // - test of a committed fix 554 // - reproduce crash 555 // - test that crash still happens on HEAD 556 // 557 // Job has Bug as parent entity. 558 type Job struct { 559 Type JobType 560 Created time.Time 561 User string 562 CC []string 563 Reporting string 564 ExtID string // email Message-ID 565 Link string // web link for the job (e.g. email in the group) 566 Namespace string 567 Manager string 568 BugTitle string 569 CrashID int64 570 571 // Provided by user: 572 KernelRepo string 573 KernelBranch string 574 Patch int64 // reference to Patch text entity 575 KernelConfig int64 // reference to the kernel config entity 576 577 Attempts int // number of times we tried to execute this job 578 IsRunning bool // the job might have been started, but never finished 579 LastStarted time.Time `datastore:"Started"` 580 Finished time.Time // if set, job is finished 581 TreeOrigin bool // whether the job is related to tree origin detection 582 583 // If patch test should be done on the merge base between two branches. 584 MergeBaseRepo string 585 MergeBaseBranch string 586 587 // By default, bisection starts from the revision of the associated crash. 588 // The BisectFrom field can override this. 589 BisectFrom string 590 591 // Result of execution: 592 CrashTitle string // if empty, we did not hit crash during testing 593 CrashLog int64 // reference to CrashLog text entity 594 CrashReport int64 // reference to CrashReport text entity 595 Commits []Commit 596 BuildID string 597 Log int64 // reference to Log text entity 598 Error int64 // reference to Error text entity, if set job failed 599 Flags dashapi.JobDoneFlags 600 601 Reported bool // have we reported result back to user? 602 InvalidatedBy string // user who marked this bug as invalid, empty by default 603 BackportedCommit Commit 604 } 605 606 func (job *Job) IsBisection() bool { 607 return job.Type == JobBisectCause || job.Type == JobBisectFix 608 } 609 610 func (job *Job) IsFinished() bool { 611 return !job.Finished.IsZero() 612 } 613 614 type JobType int 615 616 const ( 617 JobTestPatch JobType = iota 618 JobBisectCause 619 JobBisectFix 620 ) 621 622 func (typ JobType) toDashapiReportType() dashapi.ReportType { 623 switch typ { 624 case JobTestPatch: 625 return dashapi.ReportTestPatch 626 case JobBisectCause: 627 return dashapi.ReportBisectCause 628 case JobBisectFix: 629 return dashapi.ReportBisectFix 630 default: 631 panic(fmt.Sprintf("unknown job type %v", typ)) 632 } 633 } 634 635 func (job *Job) isUnreliableBisect() bool { 636 if job.Type != JobBisectCause && job.Type != JobBisectFix { 637 panic(fmt.Sprintf("bad job type %v", job.Type)) 638 } 639 // If a bisection points to a merge or a commit that does not affect the kernel binary, 640 // it is considered an unreliable/wrong result and should not be reported in emails. 641 return job.Flags&dashapi.BisectResultMerge != 0 || 642 job.Flags&dashapi.BisectResultNoop != 0 || 643 job.Flags&dashapi.BisectResultRelease != 0 || 644 job.Flags&dashapi.BisectResultIgnore != 0 645 } 646 647 func (job *Job) IsCrossTree() bool { 648 return job.MergeBaseRepo != "" && job.IsBisection() 649 } 650 651 // Text holds text blobs (crash logs, reports, reproducers, etc). 652 type Text struct { 653 Namespace string 654 Text []byte `datastore:",noindex"` // gzip-compressed text 655 } 656 657 const ( 658 textCrashLog = "CrashLog" 659 textCrashReport = "CrashReport" 660 textReproSyz = "ReproSyz" 661 textReproC = "ReproC" 662 textMachineInfo = "MachineInfo" 663 textKernelConfig = "KernelConfig" 664 textPatch = "Patch" 665 textLog = "Log" 666 textError = "Error" 667 textReproLog = "ReproLog" 668 ) 669 670 const ( 671 BugStatusOpen = iota 672 ) 673 674 const ( 675 BugStatusFixed = 1000 + iota 676 BugStatusInvalid 677 BugStatusDup 678 ) 679 680 const ( 681 ReproLevelNone = dashapi.ReproLevelNone 682 ReproLevelSyz = dashapi.ReproLevelSyz 683 ReproLevelC = dashapi.ReproLevelC 684 ) 685 686 type BuildType int 687 688 const ( 689 BuildNormal BuildType = iota 690 BuildFailed 691 BuildJob 692 ) 693 694 type BisectStatus int 695 696 const ( 697 BisectNot BisectStatus = iota 698 BisectPending 699 BisectError 700 BisectYes // have 1 commit 701 BisectUnreliable // have 1 commit, but suspect it's wrong 702 BisectInconclusive // multiple commits due to skips 703 BisectHorizont // happens on the oldest commit we can test (or HEAD for fix bisection) 704 bisectStatusLast // this value can be changed (not stored in datastore) 705 ) 706 707 func (status BisectStatus) String() string { 708 switch status { 709 case BisectError: 710 return "error" 711 case BisectYes: 712 return "done" 713 case BisectUnreliable: 714 return "unreliable" 715 case BisectInconclusive: 716 return "inconclusive" 717 case BisectHorizont: 718 return "inconclusive" 719 default: 720 return "" 721 } 722 } 723 724 func mgrKey(c context.Context, ns, name string) *db.Key { 725 return db.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil) 726 } 727 728 func (mgr *Manager) key(c context.Context) *db.Key { 729 return mgrKey(c, mgr.Namespace, mgr.Name) 730 } 731 732 func loadManager(c context.Context, ns, name string) (*Manager, error) { 733 mgr := new(Manager) 734 if err := db.Get(c, mgrKey(c, ns, name), mgr); err != nil { 735 if err != db.ErrNoSuchEntity { 736 return nil, fmt.Errorf("failed to get manager %v/%v: %w", ns, name, err) 737 } 738 mgr = &Manager{ 739 Namespace: ns, 740 Name: name, 741 } 742 } 743 return mgr, nil 744 } 745 746 // updateManager does transactional compare-and-swap on the manager and its current stats. 747 func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats) error) error { 748 date := timeDate(timeNow(c)) 749 tx := func(c context.Context) error { 750 mgr, err := loadManager(c, ns, name) 751 if err != nil { 752 return err 753 } 754 mgrKey := mgr.key(c) 755 stats := new(ManagerStats) 756 statsKey := db.NewKey(c, "ManagerStats", "", int64(date), mgrKey) 757 if err := db.Get(c, statsKey, stats); err != nil { 758 if err != db.ErrNoSuchEntity { 759 return fmt.Errorf("failed to get stats %v/%v/%v: %w", ns, name, date, err) 760 } 761 stats = &ManagerStats{ 762 Date: date, 763 } 764 } 765 766 if err := fn(mgr, stats); err != nil { 767 return err 768 } 769 770 if _, err := db.Put(c, mgrKey, mgr); err != nil { 771 return fmt.Errorf("failed to put manager: %w", err) 772 } 773 if _, err := db.Put(c, statsKey, stats); err != nil { 774 return fmt.Errorf("failed to put manager stats: %w", err) 775 } 776 return nil 777 } 778 return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) 779 } 780 781 func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) { 782 var managers []*Manager 783 query := db.NewQuery("Manager") 784 if ns != "" { 785 query = query.Filter("Namespace=", ns) 786 } 787 keys, err := query.GetAll(c, &managers) 788 if err != nil { 789 return nil, nil, fmt.Errorf("failed to query managers: %w", err) 790 } 791 var result []*Manager 792 var resultKeys []*db.Key 793 for i, mgr := range managers { 794 if getNsConfig(c, mgr.Namespace).Managers[mgr.Name].Decommissioned { 795 continue 796 } 797 result = append(result, mgr) 798 resultKeys = append(resultKeys, keys[i]) 799 } 800 return result, resultKeys, nil 801 } 802 803 func buildKey(c context.Context, ns, id string) *db.Key { 804 if ns == "" { 805 panic("requesting build key outside of namespace") 806 } 807 h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id))) 808 return db.NewKey(c, "Build", h, 0, nil) 809 } 810 811 func loadBuild(c context.Context, ns, id string) (*Build, error) { 812 build := new(Build) 813 if err := db.Get(c, buildKey(c, ns, id), build); err != nil { 814 if err == db.ErrNoSuchEntity { 815 return nil, fmt.Errorf("unknown build %v/%v", ns, id) 816 } 817 return nil, fmt.Errorf("failed to get build %v/%v: %w", ns, id, err) 818 } 819 return build, nil 820 } 821 822 func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) { 823 mgr, err := loadManager(c, ns, manager) 824 if err != nil { 825 return nil, err 826 } 827 if mgr.CurrentBuild == "" { 828 return nil, fmt.Errorf("failed to fetch manager build: no builds") 829 } 830 return loadBuild(c, ns, mgr.CurrentBuild) 831 } 832 833 func loadBuilds(c context.Context, ns, manager string, typ BuildType) ([]*Build, error) { 834 const limit = 500 835 var builds []*Build 836 _, err := db.NewQuery("Build"). 837 Filter("Namespace=", ns). 838 Filter("Manager=", manager). 839 Filter("Type=", typ). 840 Order("-Time"). 841 Limit(limit). 842 GetAll(c, &builds) 843 if err != nil { 844 return nil, err 845 } 846 return builds, nil 847 } 848 849 func (bug *Bug) displayTitle() string { 850 if bug.Seq == 0 { 851 return bug.Title 852 } 853 return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1) 854 } 855 856 var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`) 857 858 func splitDisplayTitle(display string) (string, int64, error) { 859 match := displayTitleRe.FindStringSubmatchIndex(display) 860 if match == nil { 861 return display, 0, nil 862 } 863 title := display[match[2]:match[3]] 864 seqStr := display[match[4]:match[5]] 865 seq, err := strconv.ParseInt(seqStr, 10, 64) 866 if err != nil { 867 return "", 0, fmt.Errorf("failed to parse bug title: %w", err) 868 } 869 if seq <= 0 || seq > 1e6 { 870 return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq) 871 } 872 return title, seq - 1, nil 873 } 874 875 func canonicalBug(c context.Context, bug *Bug) (*Bug, error) { 876 for { 877 if bug.Status != BugStatusDup { 878 return bug, nil 879 } 880 canon := new(Bug) 881 bugKey := db.NewKey(c, "Bug", bug.DupOf, 0, nil) 882 if err := db.Get(c, bugKey, canon); err != nil { 883 return nil, fmt.Errorf("failed to get dup bug %q for %q: %w", 884 bug.DupOf, bug.keyHash(c), err) 885 } 886 bug = canon 887 } 888 } 889 890 func (bug *Bug) key(c context.Context) *db.Key { 891 return db.NewKey(c, "Bug", bug.keyHash(c), 0, nil) 892 } 893 894 func (bug *Bug) keyHash(c context.Context) string { 895 return bugKeyHash(c, bug.Namespace, bug.Title, bug.Seq) 896 } 897 898 func bugKeyHash(c context.Context, ns, title string, seq int64) string { 899 return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq))) 900 } 901 902 func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) { 903 domain := getNsConfig(c, bug.Namespace).SimilarityDomain 904 dedup := make(map[string]bool) 905 dedup[bug.keyHash(c)] = true 906 907 ret := []*Bug{} 908 for _, title := range bug.AltTitles { 909 var similar []*Bug 910 _, err := db.NewQuery("Bug"). 911 Filter("AltTitles=", title). 912 GetAll(c, &similar) 913 if err != nil { 914 return nil, err 915 } 916 for _, bug := range similar { 917 if getNsConfig(c, bug.Namespace).SimilarityDomain != domain || 918 dedup[bug.keyHash(c)] { 919 continue 920 } 921 dedup[bug.keyHash(c)] = true 922 ret = append(ret, bug) 923 } 924 } 925 return ret, nil 926 } 927 928 // Since these IDs appear in Reported-by tags in commit, we slightly limit their size. 929 const reportingHashLen = 20 930 931 func bugReportingHash(bugHash, reporting string) string { 932 return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:reportingHashLen] 933 } 934 935 func looksLikeReportingHash(id string) bool { 936 // This is only used as best-effort check. 937 // Now we produce 20-chars ids, but we used to use full sha1 hash. 938 return len(id) == reportingHashLen || len(id) == 2*len(hash.Sig{}) 939 } 940 941 func (bug *Bug) updateCommits(commits []string, now time.Time) { 942 bug.Commits = commits 943 bug.CommitInfo = nil 944 bug.NeedCommitInfo = true 945 bug.FixTime = now 946 bug.PatchedOn = nil 947 } 948 949 func (bug *Bug) getCommitInfo(i int) Commit { 950 if i < len(bug.CommitInfo) { 951 return bug.CommitInfo[i] 952 } 953 return Commit{} 954 } 955 956 func (bug *Bug) increaseCrashStats(now time.Time) { 957 bug.NumCrashes++ 958 date := timeDate(now) 959 if len(bug.DailyStats) == 0 || bug.DailyStats[len(bug.DailyStats)-1].Date < date { 960 bug.DailyStats = append(bug.DailyStats, BugDailyStats{date, 1}) 961 } else { 962 // It is theoretically possible that this method might get into a situation, when 963 // the latest saved date is later than now. But we assume that this can only happen 964 // in a small window around the start of the day and it is better to attribute a 965 // crash to the next day than to get a mismatch between NumCrashes and the sum of 966 // CrashCount. 967 bug.DailyStats[len(bug.DailyStats)-1].CrashCount++ 968 } 969 970 if len(bug.DailyStats) > maxBugHistoryDays { 971 bug.DailyStats = bug.DailyStats[len(bug.DailyStats)-maxBugHistoryDays:] 972 } 973 } 974 975 func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats { 976 startDate := timeDate(from) 977 startPos := len(bug.DailyStats) 978 for ; startPos > 0; startPos-- { 979 if bug.DailyStats[startPos-1].Date < startDate { 980 break 981 } 982 } 983 return bug.DailyStats[startPos:] 984 } 985 986 func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) { 987 var status dashapi.BugStatus 988 switch bug.Status { 989 case BugStatusOpen: 990 status = dashapi.BugStatusOpen 991 case BugStatusFixed: 992 status = dashapi.BugStatusFixed 993 case BugStatusInvalid: 994 status = dashapi.BugStatusInvalid 995 case BugStatusDup: 996 status = dashapi.BugStatusDup 997 default: 998 return status, fmt.Errorf("unknown bugs status %v", bug.Status) 999 } 1000 return status, nil 1001 } 1002 1003 // If an entity of type EmergencyStop exists, syzbot's operation is paused until 1004 // a support engineer deletes it from the DB. 1005 type EmergencyStop struct { 1006 Time time.Time 1007 User string 1008 } 1009 1010 func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error { 1011 crash := new(Crash) 1012 crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) 1013 if err := db.Get(c, crashKey, crash); err != nil { 1014 return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) 1015 } 1016 crash.AddReference(ref) 1017 if _, err := db.Put(c, crashKey, crash); err != nil { 1018 return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) 1019 } 1020 return nil 1021 } 1022 1023 func removeCrashReference(c context.Context, crashID int64, bugKey *db.Key, 1024 t CrashReferenceType, key string) error { 1025 crash := new(Crash) 1026 crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) 1027 if err := db.Get(c, crashKey, crash); err != nil { 1028 return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) 1029 } 1030 crash.ClearReference(t, key) 1031 if _, err := db.Put(c, crashKey, crash); err != nil { 1032 return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) 1033 } 1034 return nil 1035 } 1036 1037 func kernelRepoInfo(c context.Context, build *Build) KernelRepo { 1038 return kernelRepoInfoRaw(c, build.Namespace, build.KernelRepo, build.KernelBranch) 1039 } 1040 1041 func kernelRepoInfoRaw(c context.Context, ns, url, branch string) KernelRepo { 1042 var info KernelRepo 1043 for _, repo := range getNsConfig(c, ns).Repos { 1044 if repo.URL == url && repo.Branch == branch { 1045 info = repo 1046 break 1047 } 1048 } 1049 if info.Alias == "" { 1050 info.Alias = url 1051 if branch != "" { 1052 info.Alias += " " + branch 1053 } 1054 } 1055 return info 1056 } 1057 1058 func textLink(tag string, id int64) string { 1059 if id == 0 { 1060 return "" 1061 } 1062 return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16)) 1063 } 1064 1065 // timeDate returns t's date as a single int YYYYMMDD. 1066 func timeDate(t time.Time) int { 1067 year, month, day := t.Date() 1068 return year*10000 + int(month)*100 + day 1069 } 1070 1071 func stringInList(list []string, str string) bool { 1072 for _, s := range list { 1073 if s == str { 1074 return true 1075 } 1076 } 1077 return false 1078 } 1079 1080 func stringListsIntersect(a, b []string) bool { 1081 m := map[string]bool{} 1082 for _, strA := range a { 1083 m[strA] = true 1084 } 1085 for _, strB := range b { 1086 if m[strB] { 1087 return true 1088 } 1089 } 1090 return false 1091 } 1092 1093 func mergeString(list []string, str string) []string { 1094 if !stringInList(list, str) { 1095 list = append(list, str) 1096 } 1097 return list 1098 } 1099 1100 func mergeStringList(list, add []string) []string { 1101 for _, str := range add { 1102 list = mergeString(list, str) 1103 } 1104 return list 1105 } 1106 1107 // dateTime converts date in YYYYMMDD format back to Time. 1108 func dateTime(date int) time.Time { 1109 return time.Date(date/10000, time.Month(date/100%100), date%100, 0, 0, 0, 0, time.UTC) 1110 }