github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/shared/metadata_util.go (about) 1 // Copyright 2020 The WPT Dashboard Project. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 //go:generate mockgen -destination sharedtest/metadata_util_mock.go -package sharedtest github.com/web-platform-tests/wpt.fyi/shared MetadataFetcher 6 7 package shared 8 9 import ( 10 "archive/tar" 11 "compress/gzip" 12 "context" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "net/http" 17 "strings" 18 19 "github.com/google/go-github/v47/github" 20 ) 21 22 // PendingMetadataCacheKey is the key for the set that stores a list of 23 // pending metadata PRs in Redis. 24 const PendingMetadataCacheKey = "WPT-PENDING-METADATA" 25 26 // PendingMetadataCachePrefix is the key prefix for pending metadata 27 // stored in Redis. 28 const PendingMetadataCachePrefix = "PENDING-PR-" 29 30 // SourceOwner is the owner name of the wpt-metadata repo. 31 const SourceOwner string = "web-platform-tests" 32 33 // SourceRepo is the wpt-metadata repo. 34 const SourceRepo string = "wpt-metadata" 35 const baseBranch string = "master" 36 37 // MetadataFetcher is an abstract interface that encapsulates the Fetch() method. Fetch() fetches metadata 38 // for webapp and searchcache. 39 type MetadataFetcher interface { 40 Fetch() (sha *string, res map[string][]byte, err error) 41 } 42 43 // GetWPTMetadataMasterSHA returns the SHA of the master branch of the wpt-metadata repo. 44 func GetWPTMetadataMasterSHA(ctx context.Context, gitHubClient *github.Client) (*string, error) { 45 baseRef, _, err := gitHubClient.Git.GetRef(ctx, SourceOwner, SourceRepo, "refs/heads/"+baseBranch) 46 if err != nil { 47 return nil, err 48 } 49 50 return baseRef.Object.SHA, nil 51 } 52 53 // GetWPTMetadataArchive iterates through wpt-metadata repository and returns a 54 // map that maps a test path to its META.yml file content, using a given ref. 55 func GetWPTMetadataArchive(client *http.Client, ref *string) (res map[string][]byte, err error) { 56 // See https://developer.github.com/v3/repos/contents/#get-archive-link for the archive link format. 57 return getWPTMetadataArchiveWithURL(client, "https://api.github.com/repos/web-platform-tests/wpt-metadata/tarball", ref) 58 } 59 60 func getWPTMetadataArchiveWithURL(client *http.Client, url string, ref *string) (res map[string][]byte, err error) { 61 if ref != nil && *ref != "" { 62 url = url + "/" + *ref 63 } 64 65 resp, err := client.Get(url) 66 if err != nil { 67 return nil, err 68 } 69 defer resp.Body.Close() 70 71 statusCode := resp.StatusCode 72 if !(statusCode >= 200 && statusCode <= 299) { 73 err := fmt.Errorf("Bad status code:%d, Unable to download wpt-metadata", statusCode) 74 return nil, err 75 } 76 77 gzip, err := gzip.NewReader(resp.Body) 78 if err != nil { 79 return nil, err 80 } 81 return parseMetadataFromGZip(gzip) 82 } 83 84 func parseMetadataFromGZip(gzip *gzip.Reader) (res map[string][]byte, err error) { 85 defer gzip.Close() 86 87 tarReader := tar.NewReader(gzip) 88 var metadataMap = make(map[string][]byte) 89 for { 90 header, err := tarReader.Next() 91 92 if err == io.EOF { 93 break 94 } 95 96 if err != nil { 97 return nil, err 98 } 99 100 // Not a regular file. 101 if header.Typeflag != tar.TypeReg { 102 continue 103 } 104 105 if !strings.HasSuffix(header.Name, "META.yml") { 106 continue 107 } 108 109 data, err := ioutil.ReadAll(tarReader) 110 if err != nil && err != io.EOF { 111 return nil, err 112 } 113 114 // Removes `owner-repo` prefix in the file name. 115 relativeFileName := header.Name[strings.Index(header.Name, "/")+1:] 116 relativeFileName = strings.TrimSuffix(relativeFileName, "/META.yml") 117 metadataMap[relativeFileName] = data 118 } 119 120 return metadataMap, nil 121 }