github.com/google/syzkaller@v0.0.0-20251211124644-a066d2bc4b02/syz-cluster/series-tracker/manifest.go (about)

     1  // Copyright 2024 syzkaller project authors. All rights reserved.
     2  // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
     3  
     4  package main
     5  
     6  import (
     7  	"bytes"
     8  	"compress/gzip"
     9  	"context"
    10  	"encoding/json"
    11  	"fmt"
    12  	"io"
    13  	"log"
    14  	"net/http"
    15  	"regexp"
    16  	"strconv"
    17  	"sync"
    18  	"time"
    19  )
    20  
    21  type InboxInfo struct {
    22  	Prefix string
    23  	Epochs int
    24  }
    25  
    26  func (ii InboxInfo) EpochURL(id int) string {
    27  	return fmt.Sprintf("%s/git/%d.git", ii.Prefix, id)
    28  }
    29  
    30  func (ii InboxInfo) LastEpochURL() string {
    31  	return ii.EpochURL(ii.Epochs - 1)
    32  }
    33  
    34  var archiveRe = regexp.MustCompile(`/([\w-]+)/git/(\d+)\.git`)
    35  
    36  func ParseManifest(baseURL string, jsonData []byte) (map[string]*InboxInfo, error) {
    37  	var rawMap map[string]json.RawMessage
    38  	err := json.Unmarshal(jsonData, &rawMap)
    39  	if err != nil {
    40  		return nil, err
    41  	}
    42  	ret := map[string]*InboxInfo{}
    43  	for url := range rawMap {
    44  		groups := archiveRe.FindStringSubmatch(url)
    45  		if len(groups) == 0 {
    46  			// TODO: monitor these.
    47  			log.Printf("unexpected manifest.js key: %q", url)
    48  			continue
    49  		}
    50  		epoch, err := strconv.Atoi(groups[2])
    51  		if err != nil {
    52  			log.Printf("invalid manifest.js key: %q", url)
    53  			continue
    54  		}
    55  		inbox := ret[groups[1]]
    56  		if inbox == nil {
    57  			inbox = &InboxInfo{Prefix: fmt.Sprintf("%s/%s", baseURL, groups[1])}
    58  			ret[groups[1]] = inbox
    59  		}
    60  		inbox.Epochs = max(inbox.Epochs, epoch+1)
    61  	}
    62  	return ret, nil
    63  }
    64  
    65  func QueryManifest(baseURL string) (map[string]*InboxInfo, error) {
    66  	resp, err := http.Get(baseURL + "/manifest.js.gz")
    67  	if err != nil {
    68  		return nil, err
    69  	}
    70  	defer resp.Body.Close()
    71  
    72  	gzReader, err := gzip.NewReader(resp.Body)
    73  	if err != nil {
    74  		return nil, err
    75  	}
    76  	defer gzReader.Close()
    77  
    78  	var buf bytes.Buffer
    79  	_, err = io.Copy(&buf, gzReader)
    80  	if err != nil {
    81  		return nil, err
    82  	}
    83  	return ParseManifest(baseURL, buf.Bytes())
    84  }
    85  
    86  // ManifestSource keeps an up to date version of the manifest.
    87  type ManifestSource struct {
    88  	mu          sync.Mutex
    89  	url         string
    90  	latestOk    map[string]*InboxInfo
    91  	firstLoaded chan struct{} // The channel will be closed on the first successful load.
    92  }
    93  
    94  func NewManifestSource(baseURL string) *ManifestSource {
    95  	return &ManifestSource{
    96  		url:         baseURL,
    97  		firstLoaded: make(chan struct{}),
    98  	}
    99  }
   100  
   101  func (ms *ManifestSource) Loop(ctx context.Context) {
   102  	// When we try to load for the first time, retry more frequently.
   103  	const backOffPeriod = time.Minute * 15
   104  	// Then, update rarely. New epochs are very infrequent.
   105  	const refreshPeriod = time.Hour * 12
   106  
   107  	alreadyLoaded := false
   108  	nextAttemptIn := backOffPeriod
   109  	for {
   110  		info, err := QueryManifest(ms.url)
   111  		log.Printf("loaded manifest: %v", err)
   112  		if err == nil {
   113  			ms.mu.Lock()
   114  			ms.latestOk = info
   115  			ms.mu.Unlock()
   116  			if !alreadyLoaded {
   117  				alreadyLoaded = true
   118  				nextAttemptIn = refreshPeriod
   119  				close(ms.firstLoaded)
   120  			}
   121  		}
   122  		select {
   123  		case <-ctx.Done():
   124  			return
   125  		case <-time.After(nextAttemptIn):
   126  		}
   127  	}
   128  }
   129  
   130  func (ms *ManifestSource) Get(ctx context.Context) map[string]*InboxInfo {
   131  	select {
   132  	case <-ms.firstLoaded:
   133  		ms.mu.Lock()
   134  		defer ms.mu.Unlock()
   135  		return ms.latestOk
   136  	case <-ctx.Done():
   137  		return nil
   138  	}
   139  }