github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/spyglass/artifacts.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package spyglass
    18  
    19  import (
    20  	"fmt"
    21  	"strings"
    22  	"time"
    23  
    24  	"github.com/sirupsen/logrus"
    25  	"k8s.io/test-infra/prow/spyglass/lenses"
    26  )
    27  
    28  // ListArtifacts gets the names of all artifacts available from the given source
    29  func (s *Spyglass) ListArtifacts(src string) ([]string, error) {
    30  	keyType, key, err := splitSrc(src)
    31  	if err != nil {
    32  		return []string{}, fmt.Errorf("error parsing src: %v", err)
    33  	}
    34  	gcsKey := ""
    35  	switch keyType {
    36  	case gcsKeyType:
    37  		gcsKey = key
    38  	case prowKeyType:
    39  		if gcsKey, err = s.prowToGCS(key); err != nil {
    40  			logrus.Warningf("Failed to get gcs source for prow job: %v", err)
    41  		}
    42  	default:
    43  		return nil, fmt.Errorf("Unrecognized key type for src: %v", src)
    44  	}
    45  
    46  	artifactNames, err := s.GCSArtifactFetcher.artifacts(gcsKey)
    47  	logFound := false
    48  	for _, name := range artifactNames {
    49  		if name == "build-log.txt" {
    50  			logFound = true
    51  			break
    52  		}
    53  	}
    54  	if err != nil || !logFound {
    55  		artifactNames = append(artifactNames, "build-log.txt")
    56  	}
    57  	return artifactNames, nil
    58  }
    59  
    60  // prowToGCS returns the GCS key corresponding to the given prow key
    61  func (s *Spyglass) prowToGCS(prowKey string) (string, error) {
    62  	parsed := strings.Split(prowKey, "/")
    63  	if len(parsed) != 2 {
    64  		return "", fmt.Errorf("Could not get GCS src: prow src %q incorrectly formatted", prowKey)
    65  	}
    66  	jobName := parsed[0]
    67  	buildID := parsed[1]
    68  
    69  	job, err := s.jobAgent.GetProwJob(jobName, buildID)
    70  	if err != nil {
    71  		return "", fmt.Errorf("Failed to get prow job from src %q: %v", prowKey, err)
    72  	}
    73  
    74  	url := job.Status.URL
    75  	prefix := s.ConfigAgent.Config().Plank.JobURLPrefix
    76  	if !strings.HasPrefix(url, prefix) {
    77  		return "", fmt.Errorf("unexpected job URL %q when finding GCS path: expected something starting with %q", url, prefix)
    78  	}
    79  	return url[len(prefix):], nil
    80  }
    81  
    82  // FetchArtifacts constructs and returns Artifact objects for each artifact name in the list.
    83  // This includes getting any handles needed for read write operations, direct artifact links, etc.
    84  func (s *Spyglass) FetchArtifacts(src string, podName string, sizeLimit int64, artifactNames []string) ([]lenses.Artifact, error) {
    85  	artStart := time.Now()
    86  	arts := []lenses.Artifact{}
    87  	keyType, key, err := splitSrc(src)
    88  	if err != nil {
    89  		return arts, fmt.Errorf("error parsing src: %v", err)
    90  	}
    91  	gcsKey := ""
    92  	jobName := ""
    93  	buildID := ""
    94  	switch keyType {
    95  	case gcsKeyType:
    96  		gcsKey = strings.TrimSuffix(key, "/")
    97  		parts := strings.Split(gcsKey, "/")
    98  		if len(parts) < 2 {
    99  			logrus.WithField("gcs key", gcsKey).Warningf("invalid gcs key")
   100  		} else {
   101  			jobName = parts[len(parts)-2]
   102  			buildID = parts[len(parts)-1]
   103  		}
   104  	case prowKeyType:
   105  		parts := strings.Split(key, "/")
   106  		if len(parts) != 2 {
   107  			return arts, fmt.Errorf("key %q incorrectly formatted", key)
   108  		}
   109  		jobName = parts[0]
   110  		buildID = parts[1]
   111  		if gcsKey, err = s.prowToGCS(key); err != nil {
   112  			logrus.Warningln(err)
   113  		}
   114  	default:
   115  		return nil, fmt.Errorf("Invalid src: %v", src)
   116  	}
   117  
   118  	podLogNeeded := false
   119  	for _, name := range artifactNames {
   120  		art, err := s.GCSArtifactFetcher.artifact(gcsKey, name, sizeLimit)
   121  		if err == nil {
   122  			// Actually try making a request, because calling GCSArtifactFetcher.artifact does no I/O.
   123  			// (these files are being explicitly requested and so will presumably soon be accessed, so
   124  			// the extra network I/O should not be too problematic).
   125  			_, err = art.Size()
   126  		}
   127  		if err != nil {
   128  			if name == "build-log.txt" {
   129  				podLogNeeded = true
   130  			} else {
   131  				logrus.Errorf("Failed to fetch artifact %s: %v", name, err)
   132  			}
   133  			continue
   134  		}
   135  		arts = append(arts, art)
   136  	}
   137  
   138  	if podLogNeeded {
   139  		art, err := s.PodLogArtifactFetcher.artifact(jobName, buildID, sizeLimit)
   140  		if err != nil {
   141  			logrus.Errorf("Failed to fetch pod log: %v", err)
   142  		} else {
   143  			arts = append(arts, art)
   144  		}
   145  	}
   146  
   147  	logrus.WithField("duration", time.Since(artStart)).Infof("Retrieved artifacts for %v", src)
   148  	return arts, nil
   149  }
   150  
   151  func splitSrc(src string) (keyType, key string, err error) {
   152  	split := strings.SplitN(src, "/", 2)
   153  	if len(split) < 2 {
   154  		err = fmt.Errorf("invalid src %s: expected <key-type>/<key>", src)
   155  		return
   156  	}
   157  	keyType = split[0]
   158  	key = split[1]
   159  	return
   160  }