github.com/google/syzkaller@v0.0.0-20251211124644-a066d2bc4b02/tools/syz-covermerger/syz_covermerger.go (about)

     1  // Copyright 2024 syzkaller project authors. All rights reserved.
     2  // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
     3  
     4  package main
     5  
     6  import (
     7  	"context"
     8  	"flag"
     9  	"fmt"
    10  	"io"
    11  	"runtime"
    12  	"strings"
    13  
    14  	"cloud.google.com/go/civil"
    15  	"github.com/google/syzkaller/dashboard/dashapi"
    16  	"github.com/google/syzkaller/pkg/coveragedb"
    17  	"github.com/google/syzkaller/pkg/covermerger"
    18  	"github.com/google/syzkaller/pkg/gcs"
    19  	"github.com/google/syzkaller/pkg/log"
    20  	_ "github.com/google/syzkaller/pkg/subsystem/lists"
    21  	"github.com/google/syzkaller/pkg/tool"
    22  )
    23  
    24  var (
    25  	flagWorkdir = flag.String("workdir", "workdir-cover-aggregation",
    26  		"[optional] used to clone repos")
    27  	flagRepo                = flag.String("repo", "", "[required] repo to be used as an aggregation point")
    28  	flagCommit              = flag.String("commit", "", "[required] commit hash to be used as an aggregation point")
    29  	flagNamespace           = flag.String("namespace", "upstream", "[optional] target namespace")
    30  	flagDuration            = flag.Int64("duration", 0, "[optional] used to mark DB records")
    31  	flagDateTo              = flag.String("date-to", "", "[optional] used to mark DB records")
    32  	flagTotalRows           = flag.Int64("total-rows", 0, "[optional] source size, is used for version contol")
    33  	flagToDashAPI           = flag.String("to-dashapi", "", "[optional] dashapi address")
    34  	flagDashboardClientName = flag.String("dashboard-client-name", "coverage-merger", "[optional]")
    35  	flagSrcProvider         = flag.String("provider", "git-clone", "[optional] git-clone or web-git")
    36  	flagFilePathPrefix      = flag.String("file-path-prefix", "", "[optional] kernel file path prefix")
    37  	flagToGCS               = flag.String("to-gcs", "", "[optional] gcs destination to save jsonl to")
    38  )
    39  
    40  func makeProvider() covermerger.FileVersProvider {
    41  	switch *flagSrcProvider {
    42  	case "git-clone":
    43  		return covermerger.MakeMonoRepo(*flagWorkdir)
    44  	case "web-git":
    45  		return covermerger.MakeWebGit(nil)
    46  	default:
    47  		panic(fmt.Sprintf("unknown provider %v", *flagSrcProvider))
    48  	}
    49  }
    50  
    51  func main() {
    52  	if err := do(); err != nil {
    53  		log.Fatalf("failed to saveCoverage: %v", err.Error())
    54  	}
    55  }
    56  
    57  func do() error {
    58  	defer tool.Init()()
    59  	config := &covermerger.Config{
    60  		Jobs:    runtime.NumCPU(),
    61  		Workdir: *flagWorkdir,
    62  		Base: covermerger.RepoCommit{
    63  			Repo:   *flagRepo,
    64  			Commit: *flagCommit,
    65  		},
    66  		FileVersProvider: makeProvider(),
    67  	}
    68  	var dateFrom, dateTo civil.Date
    69  	var err error
    70  	if dateTo, err = civil.ParseDate(*flagDateTo); err != nil {
    71  		panic(fmt.Sprintf("failed to parse time_to: %s", err.Error()))
    72  	}
    73  	dateFrom = dateTo.AddDays(-int(*flagDuration))
    74  	csvReader, err := covermerger.InitNsRecords(context.Background(),
    75  		*flagNamespace,
    76  		*flagFilePathPrefix,
    77  		"",
    78  		dateFrom,
    79  		dateTo,
    80  	)
    81  	if err != nil {
    82  		panic(fmt.Sprintf("failed to dbReader.InitNsRecords: %v", err.Error()))
    83  	}
    84  	defer csvReader.Close()
    85  	var wc io.WriteCloser
    86  	url := *flagToGCS
    87  	if *flagToDashAPI != "" {
    88  		dash, err := dashapi.New(*flagDashboardClientName, *flagToDashAPI, "")
    89  		if err != nil {
    90  			return fmt.Errorf("dashapi.New: %w", err)
    91  		}
    92  		url, err = dash.CreateUploadURL()
    93  		if err != nil {
    94  			return fmt.Errorf("dash.CreateUploadURL: %w", err)
    95  		}
    96  	}
    97  	if url != "" {
    98  		gcsClient, err := gcs.NewClient(context.Background())
    99  		if err != nil {
   100  			return fmt.Errorf("gcs.NewClient: %w", err)
   101  		}
   102  		defer gcsClient.Close()
   103  		wc, err = gcsClient.FileWriter(strings.TrimPrefix(url, "gs://"), "", "")
   104  		if err != nil {
   105  			return fmt.Errorf("gcsClient.FileWriter: %w", err)
   106  		}
   107  	}
   108  	totalInstrumentedLines, totalCoveredLines, err := covermerger.MergeCSVWriteJSONL(
   109  		config,
   110  		&coveragedb.HistoryRecord{
   111  			Namespace: *flagNamespace,
   112  			Repo:      *flagRepo,
   113  			Commit:    *flagCommit,
   114  			Duration:  *flagDuration,
   115  			DateTo:    dateTo,
   116  			TotalRows: *flagTotalRows,
   117  		},
   118  		csvReader,
   119  		wc)
   120  	if err != nil {
   121  		return fmt.Errorf("covermerger.MergeCSVWriteJSONL: %w", err)
   122  	}
   123  	if wc != nil {
   124  		if err := wc.Close(); err != nil {
   125  			return fmt.Errorf("wc.Close: %w", err)
   126  		}
   127  	}
   128  	printCoverage(totalInstrumentedLines, totalCoveredLines)
   129  	if *flagToDashAPI != "" {
   130  		// Merging may take hours. It is better to create new connection instead of reuse.
   131  		dash, err := dashapi.New(*flagDashboardClientName, *flagToDashAPI, "")
   132  		if err != nil {
   133  			return fmt.Errorf("dashapi.New: %w", err)
   134  		}
   135  		if rowsCreated, err := dash.SaveCoverage(url); err != nil {
   136  			return fmt.Errorf("dash.SaveCoverage: %w", err)
   137  		} else {
   138  			fmt.Printf("created %d DB rows\n", rowsCreated)
   139  		}
   140  	}
   141  	return nil
   142  }
   143  
   144  func printCoverage(instrumented, covered int) {
   145  	coverage := 0.0
   146  	if instrumented != 0 {
   147  		coverage = float64(covered) / float64(instrumented)
   148  	}
   149  	fmt.Printf("total instrumented(%d), covered(%d), %.2f%%\n",
   150  		instrumented, covered, coverage*100)
   151  }