github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/robots/coverage/downloader/downloader.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package downloader finds and downloads the coverage profile file from the latest healthy build
    18  // stored in given gcs directory
    19  package downloader
    20  
    21  import (
    22  	"context"
    23  	"encoding/json"
    24  	"fmt"
    25  	"io/ioutil"
    26  	"path"
    27  	"sort"
    28  	"strconv"
    29  
    30  	"cloud.google.com/go/storage"
    31  	"github.com/sirupsen/logrus"
    32  	"google.golang.org/api/iterator"
    33  )
    34  
    35  const (
    36  	//statusJSON is the JSON file that stores build success info
    37  	statusJSON = "finished.json"
    38  )
    39  
    40  //listGcsObjects get the slice of gcs objects under a given path
    41  func listGcsObjects(ctx context.Context, client *storage.Client, bucketName, prefix, delim string) (
    42  	[]string, error) {
    43  
    44  	var objects []string
    45  	it := client.Bucket(bucketName).Objects(ctx, &storage.Query{
    46  		Prefix:    prefix,
    47  		Delimiter: delim,
    48  	})
    49  
    50  	for {
    51  		attrs, err := it.Next()
    52  		if err == iterator.Done {
    53  			break
    54  		}
    55  		if err != nil {
    56  			return objects, fmt.Errorf("error iterating: %v", err)
    57  		}
    58  
    59  		if attrs.Prefix != "" {
    60  			objects = append(objects, path.Base(attrs.Prefix))
    61  		}
    62  	}
    63  	logrus.Info("end of listGcsObjects(...)")
    64  	return objects, nil
    65  }
    66  
    67  func readGcsObject(ctx context.Context, client *storage.Client, bucket, object string) ([]byte, error) {
    68  	logrus.Infof("Trying to read gcs object '%s' in bucket '%s'\n", object, bucket)
    69  	o := client.Bucket(bucket).Object(object)
    70  	reader, err := o.NewReader(ctx)
    71  	if err != nil {
    72  		return nil, fmt.Errorf("cannot read object '%s': %v", object, err)
    73  	}
    74  	return ioutil.ReadAll(reader)
    75  }
    76  
    77  // FindBaseProfile finds the coverage profile file from the latest healthy build
    78  // stored in given gcs directory
    79  func FindBaseProfile(ctx context.Context, client *storage.Client, bucket, prowJobName, artifactsDirName,
    80  	covProfileName string) ([]byte, error) {
    81  
    82  	dirOfJob := path.Join("logs", prowJobName)
    83  
    84  	strBuilds, err := listGcsObjects(ctx, client, bucket, dirOfJob+"/", "/")
    85  	if err != nil {
    86  		return nil, fmt.Errorf("error listing gcs objects: %v", err)
    87  	}
    88  
    89  	builds := sortBuilds(strBuilds)
    90  	profilePath := ""
    91  	for _, build := range builds {
    92  		buildDirPath := path.Join(dirOfJob, strconv.Itoa(build))
    93  		dirOfStatusJSON := path.Join(buildDirPath, statusJSON)
    94  
    95  		statusText, err := readGcsObject(ctx, client, bucket, dirOfStatusJSON)
    96  		if err != nil {
    97  			logrus.Infof("Cannot read finished.json (%s) in bucket '%s'", dirOfStatusJSON, bucket)
    98  		} else if isBuildSucceeded(statusText) {
    99  			artifactsDirPath := path.Join(buildDirPath, artifactsDirName)
   100  			profilePath = path.Join(artifactsDirPath, covProfileName)
   101  			break
   102  		}
   103  	}
   104  	if profilePath == "" {
   105  		return nil, fmt.Errorf("no healthy build found for job '%s' in bucket '%s'; total # builds = %v", dirOfJob, bucket, len(builds))
   106  	}
   107  	return readGcsObject(ctx, client, bucket, profilePath)
   108  }
   109  
   110  // sortBuilds converts all build from str to int and sorts all builds in descending order and
   111  // returns the sorted slice
   112  func sortBuilds(strBuilds []string) []int {
   113  	var res []int
   114  	for _, buildStr := range strBuilds {
   115  		num, err := strconv.Atoi(buildStr)
   116  		if err != nil {
   117  			logrus.Infof("Non-int build number found: '%s'", buildStr)
   118  		} else {
   119  			res = append(res, num)
   120  		}
   121  	}
   122  	sort.Sort(sort.Reverse(sort.IntSlice(res)))
   123  	return res
   124  }
   125  
   126  type finishedStatus struct {
   127  	Timestamp int
   128  	Passed    bool
   129  }
   130  
   131  func isBuildSucceeded(jsonText []byte) bool {
   132  	var status finishedStatus
   133  	err := json.Unmarshal(jsonText, &status)
   134  	return err == nil && status.Passed
   135  }