github.com/distbuild/reclient@v0.0.0-20240401075343-3de72e395564/internal/pkg/downloadmismatch/download.go (about) 1 // Copyright 2023 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package downloadmismatch downloads compare build mismatch outputs. 16 package downloadmismatch 17 18 import ( 19 "context" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 "strings" 25 26 "github.com/bazelbuild/remote-apis-sdks/go/pkg/client" 27 "github.com/bazelbuild/remote-apis-sdks/go/pkg/digest" 28 "github.com/bazelbuild/remote-apis-sdks/go/pkg/tool" 29 "google.golang.org/protobuf/proto" 30 31 spb "github.com/bazelbuild/reclient/api/stats" 32 ) 33 34 const ( 35 metricsFile = "rbe_metrics.pb" 36 // DownloadDir is the directory name of downloaded mismatched. 37 DownloadDir = "reclient_mismatches" 38 // LocalOutputDir stores all the local outputs of one mismatch. 39 LocalOutputDir = "local" 40 // RemoteOutputDir stores all the remote outputs of one mismatch. 41 RemoteOutputDir = "remote" 42 ) 43 44 func dedup(list []string) []string { 45 // key is cleaned path, value is original path. 46 // returns original paths. 47 fileSet := make(map[string]string) 48 for _, f := range list { 49 if _, found := fileSet[filepath.Clean(f)]; found { 50 continue 51 } 52 fileSet[filepath.Clean(f)] = f 53 } 54 var dlist []string 55 for _, f := range fileSet { 56 dlist = append(dlist, f) 57 } 58 return dlist 59 } 60 61 func readMismatchesFromFile(fp string) (*spb.Stats, error) { 62 blobs, err := ioutil.ReadFile(fp) 63 if err != nil { 64 return nil, fmt.Errorf("Failed to read mismatches from file: %v: %v", fp, err) 65 } 66 sPb := &spb.Stats{} 67 if e := proto.Unmarshal(blobs, sPb); e != nil { 68 return nil, fmt.Errorf("Failed to transform file content into stats proto: %v: %v", fp, e) 69 } 70 return sPb, nil 71 } 72 73 // DownloadMismatches finds mismatches from rbe_metrics.pb in the outputDir using the instance and service 74 // provided. The downloaded build outputs will be stored in 75 // #outputDir/reclient_mismatches/#actionDigest/#local_or_remote/#outputDigest. 76 func DownloadMismatches(logDir string, outputDir string, grpcClient *client.Client) error { 77 mismatchOutputDir := filepath.Join(outputDir, DownloadDir) 78 os.RemoveAll(mismatchOutputDir) 79 ctx := context.Background() 80 stats, err := readMismatchesFromFile(filepath.Join(logDir, metricsFile)) 81 if err != nil { 82 return err 83 } 84 if stats.Verification == nil { 85 return fmt.Errorf("No compare build stats in rbe_metrics.pb, was it a compare build?") 86 } 87 88 defer grpcClient.Close() 89 toolClient := &tool.Client{GrpcClient: grpcClient} 90 91 for _, mismatch := range stats.Verification.Mismatches { 92 actionDg, err := digest.NewFromString(mismatch.ActionDigest) 93 if err != nil { 94 return err 95 } 96 actionPath := filepath.Join(mismatchOutputDir, actionDg.Hash) 97 outputPath := filepath.Join(actionPath, strings.Replace(mismatch.Path, "/", "_", -1)) 98 if err := os.MkdirAll(outputPath, os.ModePerm); err != nil { 99 return err 100 } 101 // In old reproxy versions, RemoteDigest/LocalDigest is still used. 102 if mismatch.RemoteDigest != "" { 103 mismatch.RemoteDigests = dedup(append(mismatch.RemoteDigests, mismatch.RemoteDigest)) 104 } 105 if mismatch.LocalDigest != "" { 106 mismatch.LocalDigests = dedup(append(mismatch.LocalDigests, mismatch.LocalDigest)) 107 } 108 for _, dgStr := range mismatch.RemoteDigests { 109 if err := downloadToFile(ctx, toolClient, filepath.Join(outputPath, RemoteOutputDir), dgStr); err != nil { 110 return err 111 } 112 } 113 for _, dgStr := range mismatch.LocalDigests { 114 if err := downloadToFile(ctx, toolClient, filepath.Join(outputPath, LocalOutputDir), dgStr); err != nil { 115 return err 116 } 117 } 118 } 119 return nil 120 } 121 122 func downloadToFile(ctx context.Context, toolClient *tool.Client, downloadPath, digestStr string) error { 123 os.MkdirAll(downloadPath, os.ModePerm) 124 digestPb, _ := digest.NewFromString(digestStr) 125 blobs, err := toolClient.DownloadBlob(ctx, digestStr, "") 126 if err != nil { 127 return fmt.Errorf("Download error: %v", err) 128 } 129 fp := filepath.Join(downloadPath, digestPb.Hash) 130 return ioutil.WriteFile(fp, []byte(blobs), os.ModePerm) 131 }