sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/spyglass/artifacts.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package spyglass
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  
    24  	"github.com/sirupsen/logrus"
    25  
    26  	"k8s.io/apimachinery/pkg/util/sets"
    27  	"sigs.k8s.io/prow/pkg/config"
    28  	"sigs.k8s.io/prow/pkg/io/providers"
    29  	"sigs.k8s.io/prow/pkg/spyglass/api"
    30  	"sigs.k8s.io/prow/pkg/spyglass/lenses/common"
    31  )
    32  
    33  // ListArtifacts gets the names of all artifacts available from the given source
    34  func (s *Spyglass) ListArtifacts(ctx context.Context, src string) ([]string, error) {
    35  	keyType, key, err := splitSrc(src)
    36  	if err != nil {
    37  		return []string{}, fmt.Errorf("error parsing src: %w", err)
    38  	}
    39  	gcsKey := ""
    40  	switch keyType {
    41  	case prowKeyType:
    42  		storageProvider, key, err := s.prowToGCS(key)
    43  		if err != nil {
    44  			logrus.Debugf("Failed to get gcs source for prow job: %v", err)
    45  		}
    46  		gcsKey = fmt.Sprintf("%s://%s", storageProvider, key)
    47  	default:
    48  		if keyType == gcsKeyType {
    49  			keyType = providers.GS
    50  		}
    51  		gcsKey = fmt.Sprintf("%s://%s", keyType, key)
    52  	}
    53  
    54  	artifactNames, err := s.StorageArtifactFetcher.artifacts(ctx, gcsKey)
    55  	// Don't care errors that are not supposed logged as http errors, for example
    56  	// context cancelled error due to user cancelled request.
    57  	if err != nil && err != context.Canceled {
    58  		if config.IsNotAllowedBucketError(err) {
    59  			logrus.WithError(err).WithField("gcs-key", gcsKey).Debug("error retrieving artifact names from gcs storage")
    60  		} else {
    61  			logrus.WithError(err).WithField("gcs-key", gcsKey).Warn("error retrieving artifact names from gcs storage")
    62  		}
    63  	}
    64  
    65  	artifactNamesSet := sets.New[string](artifactNames...)
    66  
    67  	jobName, buildID, err := common.KeyToJob(src)
    68  	if err != nil {
    69  		return sets.List(artifactNamesSet), fmt.Errorf("error parsing src: %w", err)
    70  	}
    71  
    72  	job, err := s.jobAgent.GetProwJob(jobName, buildID)
    73  	if err != nil {
    74  		// we don't return the error because we assume that if we cannot get the prowjob from the jobAgent,
    75  		// then we must already have all the build-logs in gcs
    76  		logrus.Infof("unable to get prowjob from Pod: %v", err)
    77  		return sets.List(artifactNamesSet), nil
    78  	}
    79  
    80  	if job.Spec.PodSpec != nil {
    81  		jobContainers := job.Spec.PodSpec.Containers
    82  
    83  		for _, container := range jobContainers {
    84  			logName := singleLogName
    85  			if len(jobContainers) > 1 {
    86  				logName = fmt.Sprintf("%s-%s", container.Name, singleLogName)
    87  			}
    88  			artifactNamesSet.Insert(logName)
    89  		}
    90  	}
    91  
    92  	return sets.List(artifactNamesSet), nil
    93  }
    94  
    95  // prowToGCS returns the GCS key corresponding to the given prow key
    96  func (s *Spyglass) prowToGCS(prowKey string) (string, string, error) {
    97  	return common.ProwToGCS(s.JobAgent, s.config, prowKey)
    98  }
    99  
   100  // FetchArtifacts constructs and returns Artifact objects for each artifact name in the list.
   101  // This includes getting any handles needed for read write operations, direct artifact links, etc.
   102  func (s *Spyglass) FetchArtifacts(ctx context.Context, src string, podName string, sizeLimit int64, artifactNames []string) ([]api.Artifact, error) {
   103  	return common.FetchArtifacts(ctx, s.JobAgent, s.config, s.StorageArtifactFetcher, s.PodLogArtifactFetcher, src, podName, sizeLimit, artifactNames)
   104  }
   105  
   106  func splitSrc(src string) (keyType, key string, err error) {
   107  	split := strings.SplitN(src, "/", 2)
   108  	if len(split) < 2 {
   109  		err = fmt.Errorf("invalid src %s: expected <key-type>/<key>", src)
   110  		return
   111  	}
   112  	keyType = split[0]
   113  	key = split[1]
   114  	return
   115  }