sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/initupload/run.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package initupload
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"errors"
    24  	"fmt"
    25  	"io"
    26  	"os"
    27  	"time"
    28  
    29  	"github.com/GoogleCloudPlatform/testgrid/metadata"
    30  	prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
    31  	"sigs.k8s.io/prow/pkg/pod-utils/clone"
    32  	"sigs.k8s.io/prow/pkg/pod-utils/downwardapi"
    33  	"sigs.k8s.io/prow/pkg/pod-utils/gcs"
    34  )
    35  
    36  // Run will start the initupload job to upload the artifacts, logs and clone status.
    37  func (o Options) Run() error {
    38  	spec, err := downwardapi.ResolveSpecFromEnv()
    39  	if err != nil {
    40  		return fmt.Errorf("could not resolve job spec: %w", err)
    41  	}
    42  
    43  	uploadTargets := map[string]gcs.UploadFunc{}
    44  
    45  	var failed bool
    46  	var cloneRecords []clone.Record
    47  	if o.Log != "" {
    48  		if failed, cloneRecords, err = processCloneLog(o.Log, uploadTargets); err != nil {
    49  			return err
    50  		}
    51  	}
    52  
    53  	started := downwardapi.SpecToStarted(spec, cloneRecords)
    54  
    55  	startedData, err := json.Marshal(&started)
    56  	if err != nil {
    57  		return fmt.Errorf("could not marshal starting data: %w", err)
    58  	}
    59  
    60  	uploadTargets[prowv1.StartedStatusFile] = gcs.DataUpload(newBytesReadCloser(startedData))
    61  
    62  	ctx := context.Background()
    63  	if err := o.Options.Run(ctx, spec, uploadTargets); err != nil {
    64  		return fmt.Errorf("failed to upload to blob storage: %w", err)
    65  	}
    66  
    67  	if failed {
    68  		return errors.New("cloning the appropriate refs failed")
    69  	}
    70  
    71  	return nil
    72  }
    73  
    74  // processCloneLog checks if clone operation succeeded or failed for a ref
    75  // and upload clone logs as build log upon failures.
    76  // returns: bool - clone status
    77  //
    78  //	[]Record - containing final SHA on successful clones
    79  //	error - when unexpected file operation happens
    80  func processCloneLog(logfile string, uploadTargets map[string]gcs.UploadFunc) (bool, []clone.Record, error) {
    81  	var cloneRecords []clone.Record
    82  	data, err := os.ReadFile(logfile)
    83  	if err != nil {
    84  		return true, cloneRecords, fmt.Errorf("could not read clone log: %w", err)
    85  	}
    86  	if err = json.Unmarshal(data, &cloneRecords); err != nil {
    87  		return true, cloneRecords, fmt.Errorf("could not unmarshal clone records: %w", err)
    88  	}
    89  	// Do not read from cloneLog directly. Instead create multiple readers from cloneLog so it can
    90  	// be uploaded to both clone-log.txt and build-log.txt on failure.
    91  	cloneLog := bytes.Buffer{}
    92  	var failed bool
    93  	for _, record := range cloneRecords {
    94  		cloneLog.WriteString(clone.FormatRecord(record))
    95  		failed = failed || record.Failed
    96  
    97  	}
    98  	uploadTargets["clone-log.txt"] = gcs.DataUpload(newBytesReadCloser(cloneLog.Bytes()))
    99  	uploadTargets[prowv1.CloneRecordFile] = gcs.FileUpload(logfile)
   100  
   101  	if failed {
   102  		uploadTargets["build-log.txt"] = gcs.DataUpload(newBytesReadCloser(cloneLog.Bytes()))
   103  
   104  		passed := !failed
   105  		now := time.Now().Unix()
   106  		finished := metadata.Finished{
   107  			Timestamp: &now,
   108  			Passed:    &passed,
   109  			Result:    "FAILURE",
   110  		}
   111  		finishedData, err := json.Marshal(&finished)
   112  		if err != nil {
   113  			return true, cloneRecords, fmt.Errorf("could not marshal finishing data: %w", err)
   114  		}
   115  		uploadTargets[prowv1.FinishedStatusFile] = gcs.DataUpload(newBytesReadCloser(finishedData))
   116  	}
   117  	return failed, cloneRecords, nil
   118  }
   119  
   120  func newBytesReadCloser(data []byte) gcs.ReaderFunc {
   121  	return func() (io.ReadCloser, error) {
   122  		return io.NopCloser(bytes.NewReader(data)), nil
   123  	}
   124  }