github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/prow/pod-utils/gcs/upload.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package gcs
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"io"
    23  	"os"
    24  	"sync"
    25  
    26  	"cloud.google.com/go/storage"
    27  	"github.com/sirupsen/logrus"
    28  
    29  	"k8s.io/test-infra/prow/errorutil"
    30  )
    31  
    32  // UploadFunc knows how to upload into an object
    33  type UploadFunc func(obj *storage.ObjectHandle) error
    34  
    35  // Upload uploads all of the data in the
    36  // uploadTargets map to GCS in parallel. The map is
    37  // keyed on GCS path under the bucket
    38  func Upload(bucket *storage.BucketHandle, uploadTargets map[string]UploadFunc) error {
    39  	errCh := make(chan error, len(uploadTargets))
    40  	group := &sync.WaitGroup{}
    41  	group.Add(len(uploadTargets))
    42  	for dest, upload := range uploadTargets {
    43  		obj := bucket.Object(dest)
    44  		logrus.WithField("dest", dest).Info("Queued for upload")
    45  		go func(f UploadFunc, obj *storage.ObjectHandle, name string) {
    46  			defer group.Done()
    47  			if err := f(obj); err != nil {
    48  				errCh <- err
    49  			}
    50  			logrus.WithField("dest", name).Info("Finished upload")
    51  		}(upload, obj, dest)
    52  	}
    53  	group.Wait()
    54  	close(errCh)
    55  	if len(errCh) != 0 {
    56  		var uploadErrors []error
    57  		for err := range errCh {
    58  			uploadErrors = append(uploadErrors, err)
    59  		}
    60  		return fmt.Errorf("encountered errors during upload: %v", uploadErrors)
    61  	}
    62  
    63  	return nil
    64  }
    65  
    66  // FileUpload returns an UploadFunc which copies all
    67  // data from the file on disk to the GCS object
    68  func FileUpload(file string) UploadFunc {
    69  	return func(obj *storage.ObjectHandle) error {
    70  		reader, err := os.Open(file)
    71  		if err != nil {
    72  			return err
    73  		}
    74  
    75  		uploadErr := DataUpload(reader)(obj)
    76  		closeErr := reader.Close()
    77  
    78  		return errorutil.NewAggregate(uploadErr, closeErr)
    79  	}
    80  }
    81  
    82  // DataUpload returns an UploadFunc which copies all
    83  // data from src reader into GCS
    84  func DataUpload(src io.Reader) UploadFunc {
    85  	return func(obj *storage.ObjectHandle) error {
    86  		writer := obj.NewWriter(context.Background())
    87  		_, copyErr := io.Copy(writer, src)
    88  		closeErr := writer.Close()
    89  
    90  		return errorutil.NewAggregate(copyErr, closeErr)
    91  	}
    92  }