github.com/distbuild/reclient@v0.0.0-20240401075343-3de72e395564/internal/pkg/downloadmismatch/download.go (about)

     1  // Copyright 2023 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package downloadmismatch downloads compare build mismatch outputs.
    16  package downloadmismatch
    17  
    18  import (
    19  	"context"
    20  	"fmt"
    21  	"io/ioutil"
    22  	"os"
    23  	"path/filepath"
    24  	"strings"
    25  
    26  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
    27  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
    28  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/tool"
    29  	"google.golang.org/protobuf/proto"
    30  
    31  	spb "github.com/bazelbuild/reclient/api/stats"
    32  )
    33  
    34  const (
    35  	metricsFile = "rbe_metrics.pb"
    36  	// DownloadDir is the directory name of downloaded mismatched.
    37  	DownloadDir = "reclient_mismatches"
    38  	// LocalOutputDir stores all the local outputs of one mismatch.
    39  	LocalOutputDir = "local"
    40  	// RemoteOutputDir stores all the remote outputs of one mismatch.
    41  	RemoteOutputDir = "remote"
    42  )
    43  
    44  func dedup(list []string) []string {
    45  	// key is cleaned path, value is original path.
    46  	// returns original paths.
    47  	fileSet := make(map[string]string)
    48  	for _, f := range list {
    49  		if _, found := fileSet[filepath.Clean(f)]; found {
    50  			continue
    51  		}
    52  		fileSet[filepath.Clean(f)] = f
    53  	}
    54  	var dlist []string
    55  	for _, f := range fileSet {
    56  		dlist = append(dlist, f)
    57  	}
    58  	return dlist
    59  }
    60  
    61  func readMismatchesFromFile(fp string) (*spb.Stats, error) {
    62  	blobs, err := ioutil.ReadFile(fp)
    63  	if err != nil {
    64  		return nil, fmt.Errorf("Failed to read mismatches from file: %v: %v", fp, err)
    65  	}
    66  	sPb := &spb.Stats{}
    67  	if e := proto.Unmarshal(blobs, sPb); e != nil {
    68  		return nil, fmt.Errorf("Failed to transform file content into stats proto: %v: %v", fp, e)
    69  	}
    70  	return sPb, nil
    71  }
    72  
    73  // DownloadMismatches finds mismatches from rbe_metrics.pb in the outputDir using the instance and service
    74  // provided. The downloaded build outputs will be stored in
    75  // #outputDir/reclient_mismatches/#actionDigest/#local_or_remote/#outputDigest.
    76  func DownloadMismatches(logDir string, outputDir string, grpcClient *client.Client) error {
    77  	mismatchOutputDir := filepath.Join(outputDir, DownloadDir)
    78  	os.RemoveAll(mismatchOutputDir)
    79  	ctx := context.Background()
    80  	stats, err := readMismatchesFromFile(filepath.Join(logDir, metricsFile))
    81  	if err != nil {
    82  		return err
    83  	}
    84  	if stats.Verification == nil {
    85  		return fmt.Errorf("No compare build stats in rbe_metrics.pb, was it a compare build?")
    86  	}
    87  
    88  	defer grpcClient.Close()
    89  	toolClient := &tool.Client{GrpcClient: grpcClient}
    90  
    91  	for _, mismatch := range stats.Verification.Mismatches {
    92  		actionDg, err := digest.NewFromString(mismatch.ActionDigest)
    93  		if err != nil {
    94  			return err
    95  		}
    96  		actionPath := filepath.Join(mismatchOutputDir, actionDg.Hash)
    97  		outputPath := filepath.Join(actionPath, strings.Replace(mismatch.Path, "/", "_", -1))
    98  		if err := os.MkdirAll(outputPath, os.ModePerm); err != nil {
    99  			return err
   100  		}
   101  		// In old reproxy versions, RemoteDigest/LocalDigest is still used.
   102  		if mismatch.RemoteDigest != "" {
   103  			mismatch.RemoteDigests = dedup(append(mismatch.RemoteDigests, mismatch.RemoteDigest))
   104  		}
   105  		if mismatch.LocalDigest != "" {
   106  			mismatch.LocalDigests = dedup(append(mismatch.LocalDigests, mismatch.LocalDigest))
   107  		}
   108  		for _, dgStr := range mismatch.RemoteDigests {
   109  			if err := downloadToFile(ctx, toolClient, filepath.Join(outputPath, RemoteOutputDir), dgStr); err != nil {
   110  				return err
   111  			}
   112  		}
   113  		for _, dgStr := range mismatch.LocalDigests {
   114  			if err := downloadToFile(ctx, toolClient, filepath.Join(outputPath, LocalOutputDir), dgStr); err != nil {
   115  				return err
   116  			}
   117  		}
   118  	}
   119  	return nil
   120  }
   121  
   122  func downloadToFile(ctx context.Context, toolClient *tool.Client, downloadPath, digestStr string) error {
   123  	os.MkdirAll(downloadPath, os.ModePerm)
   124  	digestPb, _ := digest.NewFromString(digestStr)
   125  	blobs, err := toolClient.DownloadBlob(ctx, digestStr, "")
   126  	if err != nil {
   127  		return fmt.Errorf("Download error: %v", err)
   128  	}
   129  	fp := filepath.Join(downloadPath, digestPb.Hash)
   130  	return ioutil.WriteFile(fp, []byte(blobs), os.ModePerm)
   131  }