github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/shared/metadata_util.go (about)

     1  // Copyright 2020 The WPT Dashboard Project. All rights reserved.
     2  // Use of this source code is governed by a BSD-style license that can be
     3  // found in the LICENSE file.
     4  
     5  //go:generate mockgen -destination sharedtest/metadata_util_mock.go -package sharedtest github.com/web-platform-tests/wpt.fyi/shared MetadataFetcher
     6  
     7  package shared
     8  
     9  import (
    10  	"archive/tar"
    11  	"compress/gzip"
    12  	"context"
    13  	"fmt"
    14  	"io"
    15  	"io/ioutil"
    16  	"net/http"
    17  	"strings"
    18  
    19  	"github.com/google/go-github/v47/github"
    20  )
    21  
    22  // PendingMetadataCacheKey is the key for the set that stores a list of
    23  // pending metadata PRs in Redis.
    24  const PendingMetadataCacheKey = "WPT-PENDING-METADATA"
    25  
    26  // PendingMetadataCachePrefix is the key prefix for pending metadata
    27  // stored in Redis.
    28  const PendingMetadataCachePrefix = "PENDING-PR-"
    29  
    30  // SourceOwner is the owner name of the wpt-metadata repo.
    31  const SourceOwner string = "web-platform-tests"
    32  
    33  // SourceRepo is the wpt-metadata repo.
    34  const SourceRepo string = "wpt-metadata"
    35  const baseBranch string = "master"
    36  
    37  // MetadataFetcher is an abstract interface that encapsulates the Fetch() method. Fetch() fetches metadata
    38  // for webapp and searchcache.
    39  type MetadataFetcher interface {
    40  	Fetch() (sha *string, res map[string][]byte, err error)
    41  }
    42  
    43  // GetWPTMetadataMasterSHA returns the SHA of the master branch of the wpt-metadata repo.
    44  func GetWPTMetadataMasterSHA(ctx context.Context, gitHubClient *github.Client) (*string, error) {
    45  	baseRef, _, err := gitHubClient.Git.GetRef(ctx, SourceOwner, SourceRepo, "refs/heads/"+baseBranch)
    46  	if err != nil {
    47  		return nil, err
    48  	}
    49  
    50  	return baseRef.Object.SHA, nil
    51  }
    52  
    53  // GetWPTMetadataArchive iterates through wpt-metadata repository and returns a
    54  // map that maps a test path to its META.yml file content, using a given ref.
    55  func GetWPTMetadataArchive(client *http.Client, ref *string) (res map[string][]byte, err error) {
    56  	// See https://developer.github.com/v3/repos/contents/#get-archive-link for the archive link format.
    57  	return getWPTMetadataArchiveWithURL(client, "https://api.github.com/repos/web-platform-tests/wpt-metadata/tarball", ref)
    58  }
    59  
    60  func getWPTMetadataArchiveWithURL(client *http.Client, url string, ref *string) (res map[string][]byte, err error) {
    61  	if ref != nil && *ref != "" {
    62  		url = url + "/" + *ref
    63  	}
    64  
    65  	resp, err := client.Get(url)
    66  	if err != nil {
    67  		return nil, err
    68  	}
    69  	defer resp.Body.Close()
    70  
    71  	statusCode := resp.StatusCode
    72  	if !(statusCode >= 200 && statusCode <= 299) {
    73  		err := fmt.Errorf("Bad status code:%d, Unable to download wpt-metadata", statusCode)
    74  		return nil, err
    75  	}
    76  
    77  	gzip, err := gzip.NewReader(resp.Body)
    78  	if err != nil {
    79  		return nil, err
    80  	}
    81  	return parseMetadataFromGZip(gzip)
    82  }
    83  
    84  func parseMetadataFromGZip(gzip *gzip.Reader) (res map[string][]byte, err error) {
    85  	defer gzip.Close()
    86  
    87  	tarReader := tar.NewReader(gzip)
    88  	var metadataMap = make(map[string][]byte)
    89  	for {
    90  		header, err := tarReader.Next()
    91  
    92  		if err == io.EOF {
    93  			break
    94  		}
    95  
    96  		if err != nil {
    97  			return nil, err
    98  		}
    99  
   100  		// Not a regular file.
   101  		if header.Typeflag != tar.TypeReg {
   102  			continue
   103  		}
   104  
   105  		if !strings.HasSuffix(header.Name, "META.yml") {
   106  			continue
   107  		}
   108  
   109  		data, err := ioutil.ReadAll(tarReader)
   110  		if err != nil && err != io.EOF {
   111  			return nil, err
   112  		}
   113  
   114  		// Removes `owner-repo` prefix in the file name.
   115  		relativeFileName := header.Name[strings.Index(header.Name, "/")+1:]
   116  		relativeFileName = strings.TrimSuffix(relativeFileName, "/META.yml")
   117  		metadataMap[relativeFileName] = data
   118  	}
   119  
   120  	return metadataMap, nil
   121  }