k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/robots/coverage/downloader/downloader.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package downloader finds and downloads the coverage profile file from the latest healthy build
    18  // stored in given gcs directory
    19  package downloader
    20  
    21  import (
    22  	"context"
    23  	"encoding/json"
    24  	"fmt"
    25  	"io"
    26  	"path"
    27  	"sort"
    28  	"strconv"
    29  
    30  	"cloud.google.com/go/storage"
    31  	"github.com/sirupsen/logrus"
    32  	"google.golang.org/api/iterator"
    33  
    34  	prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
    35  )
    36  
    37  // listGcsObjects get the slice of gcs objects under a given path
    38  func listGcsObjects(ctx context.Context, client *storage.Client, bucketName, prefix, delim string) (
    39  	[]string, error) {
    40  
    41  	var objects []string
    42  	it := client.Bucket(bucketName).Objects(ctx, &storage.Query{
    43  		Prefix:    prefix,
    44  		Delimiter: delim,
    45  	})
    46  
    47  	for {
    48  		attrs, err := it.Next()
    49  		if err == iterator.Done {
    50  			break
    51  		}
    52  		if err != nil {
    53  			return objects, fmt.Errorf("error iterating: %w", err)
    54  		}
    55  
    56  		if attrs.Prefix != "" {
    57  			objects = append(objects, path.Base(attrs.Prefix))
    58  		}
    59  	}
    60  	logrus.Info("end of listGcsObjects(...)")
    61  	return objects, nil
    62  }
    63  
    64  func readGcsObject(ctx context.Context, client *storage.Client, bucket, object string) ([]byte, error) {
    65  	logrus.Infof("Trying to read gcs object '%s' in bucket '%s'\n", object, bucket)
    66  	o := client.Bucket(bucket).Object(object)
    67  	reader, err := o.NewReader(ctx)
    68  	if err != nil {
    69  		return nil, fmt.Errorf("cannot read object '%s': %w", object, err)
    70  	}
    71  	return io.ReadAll(reader)
    72  }
    73  
    74  // FindBaseProfile finds the coverage profile file from the latest healthy build
    75  // stored in given gcs directory
    76  func FindBaseProfile(ctx context.Context, client *storage.Client, bucket, prowJobName, artifactsDirName,
    77  	covProfileName string) ([]byte, error) {
    78  
    79  	dirOfJob := path.Join("logs", prowJobName)
    80  
    81  	strBuilds, err := listGcsObjects(ctx, client, bucket, dirOfJob+"/", "/")
    82  	if err != nil {
    83  		return nil, fmt.Errorf("error listing gcs objects: %w", err)
    84  	}
    85  
    86  	builds := sortBuilds(strBuilds)
    87  	profilePath := ""
    88  	for _, build := range builds {
    89  		buildDirPath := path.Join(dirOfJob, strconv.Itoa(build))
    90  		dirOfStatusJSON := path.Join(buildDirPath, prowv1.FinishedStatusFile)
    91  
    92  		statusText, err := readGcsObject(ctx, client, bucket, dirOfStatusJSON)
    93  		if err != nil {
    94  			logrus.Infof("Cannot read finished.json (%s) in bucket '%s'", dirOfStatusJSON, bucket)
    95  		} else if isBuildSucceeded(statusText) {
    96  			artifactsDirPath := path.Join(buildDirPath, artifactsDirName)
    97  			profilePath = path.Join(artifactsDirPath, covProfileName)
    98  			break
    99  		}
   100  	}
   101  	if profilePath == "" {
   102  		return nil, fmt.Errorf("no healthy build found for job '%s' in bucket '%s'; total # builds = %v", dirOfJob, bucket, len(builds))
   103  	}
   104  	return readGcsObject(ctx, client, bucket, profilePath)
   105  }
   106  
   107  // sortBuilds converts all build from str to int and sorts all builds in descending order and
   108  // returns the sorted slice
   109  func sortBuilds(strBuilds []string) []int {
   110  	var res []int
   111  	for _, buildStr := range strBuilds {
   112  		num, err := strconv.Atoi(buildStr)
   113  		if err != nil {
   114  			logrus.Infof("Non-int build number found: '%s'", buildStr)
   115  		} else {
   116  			res = append(res, num)
   117  		}
   118  	}
   119  	sort.Sort(sort.Reverse(sort.IntSlice(res)))
   120  	return res
   121  }
   122  
   123  type finishedStatus struct {
   124  	Timestamp int
   125  	Passed    bool
   126  }
   127  
   128  func isBuildSucceeded(jsonText []byte) bool {
   129  	var status finishedStatus
   130  	err := json.Unmarshal(jsonText, &status)
   131  	return err == nil && status.Passed
   132  }