github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/gcsupload/run.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package gcsupload 18 19 import ( 20 "context" 21 "fmt" 22 "os" 23 "path" 24 "path/filepath" 25 "strings" 26 27 "cloud.google.com/go/storage" 28 "github.com/sirupsen/logrus" 29 "google.golang.org/api/option" 30 31 "k8s.io/test-infra/prow/kube" 32 "k8s.io/test-infra/prow/pod-utils/downwardapi" 33 "k8s.io/test-infra/prow/pod-utils/gcs" 34 ) 35 36 // Run will upload files to GCS as prescribed by 37 // the options. Any extra files can be passed as 38 // a parameter and will have the prefix prepended 39 // to their destination in GCS, so the caller can 40 // operate relative to the base of the GCS dir. 41 func (o Options) Run(spec *downwardapi.JobSpec, extra map[string]gcs.UploadFunc) error { 42 uploadTargets := o.assembleTargets(spec, extra) 43 44 if !o.DryRun { 45 ctx := context.Background() 46 gcsClient, err := storage.NewClient(ctx, option.WithCredentialsFile(o.GcsCredentialsFile)) 47 if err != nil { 48 return fmt.Errorf("could not connect to GCS: %v", err) 49 } 50 51 if err := gcs.Upload(gcsClient.Bucket(o.Bucket), uploadTargets); err != nil { 52 return fmt.Errorf("failed to upload to GCS: %v", err) 53 } 54 } else { 55 for destination := range uploadTargets { 56 logrus.WithField("dest", destination).Info("Would upload") 57 } 58 } 59 60 logrus.Info("Finished upload to GCS") 61 return nil 62 } 63 64 func (o Options) assembleTargets(spec *downwardapi.JobSpec, extra map[string]gcs.UploadFunc) map[string]gcs.UploadFunc { 65 jobBasePath, gcsPath, builder := PathsForJob(o.GCSConfiguration, spec, o.SubDir) 66 67 uploadTargets := map[string]gcs.UploadFunc{} 68 69 // ensure that an alias exists for any 70 // job we're uploading artifacts for 71 if alias := gcs.AliasForSpec(spec); alias != "" { 72 fullBasePath := "gs://" + path.Join(o.Bucket, jobBasePath) 73 uploadTargets[alias] = gcs.DataUploadWithMetadata(strings.NewReader(fullBasePath), map[string]string{ 74 "x-goog-meta-link": fullBasePath, 75 }) 76 } 77 78 if latestBuilds := gcs.LatestBuildForSpec(spec, builder); len(latestBuilds) > 0 { 79 for _, latestBuild := range latestBuilds { 80 uploadTargets[latestBuild] = gcs.DataUpload(strings.NewReader(spec.BuildID)) 81 } 82 } 83 84 for _, item := range o.Items { 85 info, err := os.Stat(item) 86 if err != nil { 87 logrus.Warnf("Encountered error in resolving items to upload for %s: %v", item, err) 88 continue 89 } 90 if info.IsDir() { 91 gatherArtifacts(item, gcsPath, info.Name(), uploadTargets) 92 } else { 93 destination := path.Join(gcsPath, info.Name()) 94 if _, exists := uploadTargets[destination]; exists { 95 logrus.Warnf("Encountered duplicate upload of %s, skipping...", destination) 96 continue 97 } 98 uploadTargets[destination] = gcs.FileUpload(item) 99 } 100 } 101 102 for destination, upload := range extra { 103 uploadTargets[path.Join(gcsPath, destination)] = upload 104 } 105 106 return uploadTargets 107 } 108 109 // PathsForJob determines the following for a job: 110 // - path in GCS under the bucket where job artifacts will be uploaded for: 111 // - the job 112 // - this specific run of the job (if any subdir is present) 113 // The builder for the job is also returned for use in other path resolution. 114 func PathsForJob(options *kube.GCSConfiguration, spec *downwardapi.JobSpec, subdir string) (string, string, gcs.RepoPathBuilder) { 115 builder := builderForStrategy(options.PathStrategy, options.DefaultOrg, options.DefaultRepo) 116 jobBasePath := gcs.PathForSpec(spec, builder) 117 if options.PathPrefix != "" { 118 jobBasePath = path.Join(options.PathPrefix, jobBasePath) 119 } 120 var gcsPath string 121 if subdir == "" { 122 gcsPath = jobBasePath 123 } else { 124 gcsPath = path.Join(jobBasePath, subdir) 125 } 126 127 return jobBasePath, gcsPath, builder 128 } 129 130 func builderForStrategy(strategy, defaultOrg, defaultRepo string) gcs.RepoPathBuilder { 131 var builder gcs.RepoPathBuilder 132 switch strategy { 133 case kube.PathStrategyExplicit: 134 builder = gcs.NewExplicitRepoPathBuilder() 135 case kube.PathStrategyLegacy: 136 builder = gcs.NewLegacyRepoPathBuilder(defaultOrg, defaultRepo) 137 case kube.PathStrategySingle: 138 builder = gcs.NewSingleDefaultRepoPathBuilder(defaultOrg, defaultRepo) 139 } 140 141 return builder 142 } 143 144 func gatherArtifacts(artifactDir, gcsPath, subDir string, uploadTargets map[string]gcs.UploadFunc) { 145 logrus.Printf("Gathering artifacts from artifact directory: %s", artifactDir) 146 filepath.Walk(artifactDir, func(fspath string, info os.FileInfo, err error) error { 147 if info == nil || info.IsDir() { 148 return nil 149 } 150 151 // we know path will be below artifactDir, but we can't 152 // communicate that to the filepath module. We can ignore 153 // this error as we can be certain it won't occur and best- 154 // effort upload is OK in any case 155 if relPath, err := filepath.Rel(artifactDir, fspath); err == nil { 156 destination := path.Join(gcsPath, subDir, relPath) 157 if _, exists := uploadTargets[destination]; exists { 158 logrus.Warnf("Encountered duplicate upload of %s, skipping...", destination) 159 return nil 160 } 161 logrus.Printf("Found %s in artifact directory. Uploading as %s\n", fspath, destination) 162 uploadTargets[destination] = gcs.FileUpload(fspath) 163 } else { 164 logrus.Warnf("Encountered error in relative path calculation for %s under %s: %v", fspath, artifactDir, err) 165 } 166 return nil 167 }) 168 }