github.com/google/syzkaller@v0.0.0-20251211124644-a066d2bc4b02/tools/syz-lore/query_lkml.go

// Copyright 2023 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"flag"
	"log"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/email/lore"
	"github.com/google/syzkaller/pkg/hash"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/tool"
	"github.com/google/syzkaller/pkg/vcs"
	"golang.org/x/sync/errgroup"
)

// The syz-lore tool can parse Lore archives and extract syzbot-related conversations from there.

var (
	flagEmails    = flag.String("emails", "", "comma-separated list of own emails")
	flagDomains   = flag.String("domains", "", "comma-separated list of own domains")
	flagOutDir    = flag.String("out_dir", "", "a directory to save discussions as JSON files")
	flagDashboard = flag.String("dashboard", "", "dashboard address")
	flagAPIClient = flag.String("client", "", "the name of the API client")
	flagAPIKey    = flag.String("key", "", "api key")
	flagVerbose   = flag.Bool("v", false, "print more debug info")
)

func main() {
	defer tool.Init()()
	if len(flag.Args()) == 0 {
		tool.Failf("format: syz-lore [flags] dir1 [dir2 ...]")
	}
	emails := strings.Split(*flagEmails, ",")
	domains := strings.Split(*flagDomains, ",")
	threads := processArchives(flag.Args(), emails, domains)
	for i, thread := range threads {
		messages := []dashapi.DiscussionMessage{}
		for _, m := range thread.Messages {
			messages = append(messages, dashapi.DiscussionMessage{
				ID:       m.MessageID,
				External: !m.OwnEmail,
				Time:     m.Date,
				Email:    m.Author,
			})
		}
		discussion := &dashapi.Discussion{
			ID:       thread.MessageID,
			Source:   dashapi.DiscussionLore,
			Type:     thread.Type,
			Subject:  thread.Subject,
			BugIDs:   thread.BugIDs,
			Messages: messages,
		}
		log.Printf("saving %d/%d", i+1, len(threads))
		err := saveDiscussion(discussion)
		if err != nil {
			tool.Fail(err)
		}
	}
}

var dash *dashapi.Dashboard

func saveDiscussion(d *dashapi.Discussion) error {
	var err error
	if *flagDashboard != "" && dash == nil {
		dash, err = dashapi.New(*flagAPIClient, *flagDashboard, *flagAPIKey)
		if err != nil {
			tool.Failf("dashapi failed: %v", err)
		}
	}
	if *flagOutDir != "" {
		bytes, err := json.Marshal(d)
		if err != nil {
			return err
		}
		path := filepath.Join(*flagOutDir, hash.String([]byte(d.ID))+".json")
		err = osutil.WriteFile(path, bytes)
		if err != nil {
			return err
		}
	}
	if dash != nil {
		return dash.SaveDiscussion(&dashapi.SaveDiscussionReq{
			Discussion: d,
		})
	}
	return nil
}

func processArchives(paths, emails, domains []string) []*lore.Thread {
	threads := runtime.NumCPU()
	messages := make(chan lore.EmailReader, threads*2)
	wg := sync.WaitGroup{}
	g, _ := errgroup.WithContext(context.Background())

	// Generate per-email jobs.
	for _, path := range paths {
		log.Printf("reading %s", path)
		wg.Add(1)
		g.Go(func() error {
			defer wg.Done()
			repo := vcs.NewLKMLRepo(path)
			list, err := lore.ReadArchive(repo, "", time.Time{})
			if err != nil {
				return err
			}
			for _, reader := range list {
				messages <- reader
			}
			return nil
		})
	}

	// Set up some worker threads.
	var repoEmails []*lore.Email
	var mu sync.Mutex
	var skipped atomic.Int64
	for i := 0; i < threads; i++ {
		g.Go(func() error {
			for rawMsg := range messages {
				msg, err := rawMsg.Parse(emails, domains)
				if err != nil {
					// There are many broken messages in LKML,
					// no sense to print them all each time.
					skipped.Add(1)
					continue
				}
				mu.Lock()
				repoEmails = append(repoEmails, msg)
				mu.Unlock()
			}
			return nil
		})
	}

	// Once all jobs are generated, close the processing channel.
	wg.Wait()
	close(messages)
	if err := g.Wait(); err != nil {
		tool.Failf("%s", err)
	}
	if cnt := skipped.Load(); cnt > 0 {
		log.Printf("skipped %d messages because of parsing errors", cnt)
	}

	list := lore.Threads(repoEmails)
	log.Printf("collected %d email threads", len(list))

	ret := []*lore.Thread{}
	for _, d := range list {
		if d.BugIDs == nil {
			continue
		}
		ret = append(ret, d)
		if *flagVerbose {
			log.Printf("discussion ID=%s BugID=%s Type=%s Subject=%s Messages=%d",
				d.MessageID, d.BugIDs, d.Subject, d.Type, len(d.Messages))
		}
	}
	log.Printf("%d threads are related to syzbot", len(ret))
	return ret
}