github.com/jdhenke/godel@v0.0.0-20161213181855-abeb3861bf0d/apps/distgo/cmd/publish/publish.go (about) 1 // Copyright 2016 Palantir Technologies, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package publish 16 17 import ( 18 "bytes" 19 "crypto/md5" 20 "crypto/sha1" 21 "crypto/sha256" 22 "encoding/hex" 23 "fmt" 24 "io" 25 "io/ioutil" 26 "net/http" 27 "net/url" 28 "os" 29 "path" 30 "strings" 31 "text/template" 32 33 "github.com/pkg/errors" 34 "gopkg.in/cheggaaa/pb.v1" 35 36 "github.com/palantir/godel/apps/distgo/cmd/dist" 37 "github.com/palantir/godel/apps/distgo/params" 38 "github.com/palantir/godel/apps/distgo/pkg/slsspec" 39 "github.com/palantir/godel/apps/distgo/templating" 40 ) 41 42 const ( 43 pomTemplate = `<?xml version="1.0" encoding="UTF-8"?> 44 <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" 45 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> 46 <modelVersion>4.0.0</modelVersion> 47 <groupId>{{.Publish.GroupID}}</groupId> 48 <artifactId>{{.ProductName}}</artifactId> 49 <version>{{.ProductVersion}}</version> 50 <packaging>{{packagingType}}</packaging> 51 </project> 52 ` 53 ) 54 55 type Publisher interface { 56 Publish(buildSpec params.ProductBuildSpec, paths ProductPaths, stdout io.Writer) (string, error) 57 } 58 59 type BasicConnectionInfo struct { 60 URL string 61 Username string 62 Password string 63 } 64 65 func Run(buildSpecWithDeps params.ProductBuildSpecWithDeps, publisher Publisher, almanacInfo *AlmanacInfo, stdout io.Writer) error { 66 buildSpec := buildSpecWithDeps.Spec 67 for _, currDistCfg := range buildSpec.Dist { 68 // verify that distribution to publish exists 69 artifactPath := dist.ArtifactPath(buildSpec, currDistCfg) 70 if _, err := os.Stat(artifactPath); os.IsNotExist(err) { 71 return errors.Errorf("distribution for %v does not exist at %v", buildSpec.ProductName, artifactPath) 72 } 73 74 paths, err := productPath(buildSpecWithDeps, currDistCfg) 75 if err != nil { 76 return errors.Wrapf(err, "failed to determine product paths") 77 } 78 79 artifactURL, err := publisher.Publish(buildSpec, paths, stdout) 80 if err != nil { 81 return fmt.Errorf("Publish failed for %v: %v", buildSpec.ProductName, err) 82 } 83 84 if almanacInfo != nil && artifactURL != "" { 85 if err := almanacPublish(artifactURL, *almanacInfo, buildSpec, currDistCfg, stdout); err != nil { 86 return fmt.Errorf("Almanac publish failed for %v: %v", buildSpec.ProductName, err) 87 } 88 } 89 } 90 return nil 91 } 92 93 func DistsNotBuilt(buildSpecWithDeps []params.ProductBuildSpecWithDeps) []params.ProductBuildSpecWithDeps { 94 var distsNotBuilt []params.ProductBuildSpecWithDeps 95 for _, currBuildSpecWithDeps := range buildSpecWithDeps { 96 currBuildSpec := currBuildSpecWithDeps.Spec 97 for _, currDistCfg := range currBuildSpec.Dist { 98 artifactPath := dist.ArtifactPath(currBuildSpec, currDistCfg) 99 if _, err := os.Stat(artifactPath); os.IsNotExist(err) { 100 distsNotBuilt = append(distsNotBuilt, currBuildSpecWithDeps) 101 } 102 } 103 } 104 return distsNotBuilt 105 } 106 107 type ProductPaths struct { 108 // path of the form "{{GroupID}}/{{ProductName}}/{{ProductVersion}}". For example, "com/group/foo-service/1.0.1". 109 productPath string 110 pomFilePath string 111 artifactPath string 112 } 113 114 func productPath(buildSpecWithDeps params.ProductBuildSpecWithDeps, distCfg params.Dist) (ProductPaths, error) { 115 buildSpec := buildSpecWithDeps.Spec 116 117 distType, err := packagingType(distCfg.Info.Type()) 118 if err != nil { 119 return ProductPaths{}, err 120 } 121 122 funcs := template.FuncMap{ 123 "packagingType": func() string { return distType }, 124 } 125 t := template.Must(template.New("pom").Funcs(funcs).Parse(pomTemplate)) 126 127 pomFileBuf := bytes.Buffer{} 128 if err := t.Execute(&pomFileBuf, templating.ConvertSpec(buildSpec, distCfg)); err != nil { 129 return ProductPaths{}, errors.Wrapf(err, "failed to execute template") 130 } 131 132 pomFilePath := pomFilePath(buildSpec, distCfg) 133 if err := ioutil.WriteFile(pomFilePath, pomFileBuf.Bytes(), 0644); err != nil { 134 return ProductPaths{}, errors.Wrapf(err, "failed to write POM file to %v", pomFilePath) 135 } 136 137 return ProductPaths{ 138 productPath: path.Join(path.Join(strings.Split(distCfg.Publish.GroupID, ".")...), buildSpec.ProductName, buildSpec.ProductVersion), 139 pomFilePath: pomFilePath, 140 artifactPath: dist.ArtifactPath(buildSpec, distCfg), 141 }, nil 142 } 143 144 func packagingType(distType params.DistInfoType) (string, error) { 145 switch distType { 146 case params.SLSDistType: 147 return "sls.tgz", nil 148 case params.BinDistType: 149 return "tgz", nil 150 case params.RPMDistType: 151 return "rpm", nil 152 default: 153 return "", fmt.Errorf("unknown dist type: %v", distType) 154 } 155 } 156 157 func (b BasicConnectionInfo) uploadArtifacts(baseURL string, paths ProductPaths, artifactExists artifactExistsFunc, stdout io.Writer) (string, error) { 158 artifactURL, err := b.uploadFile(paths.artifactPath, baseURL, paths.artifactPath, artifactExists, stdout) 159 if err != nil { 160 return artifactURL, err 161 } 162 if _, err := b.uploadFile(paths.pomFilePath, baseURL, paths.pomFilePath, artifactExists, stdout); err != nil { 163 return artifactURL, err 164 } 165 return artifactURL, nil 166 } 167 168 type fileInfo struct { 169 path string 170 bytes []byte 171 checksums checksums 172 } 173 174 type checksums struct { 175 SHA1 string 176 SHA256 string 177 MD5 string 178 } 179 180 func (c checksums) match(other checksums) bool { 181 nonEmptyEqual := nonEmptyEqual(c.MD5, other.MD5) || nonEmptyEqual(c.SHA1, other.SHA1) || nonEmptyEqual(c.SHA256, other.SHA256) 182 // if no non-empty checksums are equal, checksums are not equal 183 if !nonEmptyEqual { 184 return false 185 } 186 187 // if non-empty checksums are different, treat as suspect and return false 188 if nonEmptyDiffer(c.MD5, other.MD5) || nonEmptyDiffer(c.SHA1, other.SHA1) || nonEmptyDiffer(c.SHA256, other.SHA256) { 189 return false 190 } 191 192 // at least one non-empty checksum was equal, and no non-empty checksums differed 193 return true 194 } 195 196 func nonEmptyEqual(s1, s2 string) bool { 197 return s1 != "" && s2 != "" && s1 == s2 198 } 199 200 func nonEmptyDiffer(s1, s2 string) bool { 201 return s1 != "" && s2 != "" && s1 != s2 202 } 203 204 // function type returns true if the file represented by the given fileInfo object 205 type artifactExistsFunc func(fi fileInfo, dstFileName, username, password string) bool 206 207 func newFileInfo(pathToFile string) (fileInfo, error) { 208 bytes, err := ioutil.ReadFile(pathToFile) 209 if err != nil { 210 return fileInfo{}, errors.Wrapf(err, "Failed to read file %v", pathToFile) 211 } 212 213 sha1Bytes := sha1.Sum(bytes) 214 sha256Bytes := sha256.Sum256(bytes) 215 md5Bytes := md5.Sum(bytes) 216 217 return fileInfo{ 218 path: pathToFile, 219 bytes: bytes, 220 checksums: checksums{ 221 SHA1: hex.EncodeToString(sha1Bytes[:]), 222 SHA256: hex.EncodeToString(sha256Bytes[:]), 223 MD5: hex.EncodeToString(md5Bytes[:]), 224 }, 225 }, nil 226 } 227 228 func (b BasicConnectionInfo) uploadFile(filePath, baseURL, artifactPath string, artifactExists artifactExistsFunc, stdout io.Writer) (rURL string, rErr error) { 229 rawUploadURL := strings.Join([]string{baseURL, path.Base(artifactPath)}, "/") 230 231 fileInfo, err := newFileInfo(filePath) 232 if err != nil { 233 return rawUploadURL, err 234 } 235 236 if artifactExists != nil && artifactExists(fileInfo, path.Base(artifactPath), b.Username, b.Password) { 237 fmt.Fprintf(stdout, "File %s already exists at %s, skipping upload.\n", filePath, rawUploadURL) 238 return rawUploadURL, nil 239 } 240 241 uploadURL, err := url.Parse(rawUploadURL) 242 if err != nil { 243 return rawUploadURL, errors.Wrapf(err, "Failed to parse %v as URL", rawUploadURL) 244 } 245 246 fmt.Fprintf(stdout, "Uploading %v to %v...\n", fileInfo.path, rawUploadURL) 247 248 header := http.Header{} 249 addChecksumToHeader(header, "Md5", fileInfo.checksums.MD5) 250 addChecksumToHeader(header, "Sha1", fileInfo.checksums.SHA1) 251 addChecksumToHeader(header, "Sha256", fileInfo.checksums.SHA256) 252 253 bar := pb.New(len(fileInfo.bytes)).SetUnits(pb.U_BYTES) 254 bar.Output = stdout 255 bar.SetMaxWidth(120) 256 bar.Start() 257 defer bar.Finish() 258 reader := bar.NewProxyReader(bytes.NewReader(fileInfo.bytes)) 259 260 req := http.Request{ 261 Method: http.MethodPut, 262 URL: uploadURL, 263 Header: header, 264 Body: ioutil.NopCloser(reader), 265 ContentLength: int64(len(fileInfo.bytes)), 266 } 267 req.SetBasicAuth(b.Username, b.Password) 268 269 resp, err := http.DefaultClient.Do(&req) 270 if err != nil { 271 return rawUploadURL, errors.Wrapf(err, "failed to upload %v to %v", fileInfo.path, rawUploadURL) 272 } 273 defer func() { 274 if err := resp.Body.Close(); err != nil && rErr == nil { 275 rErr = errors.Wrapf(err, "failed to close response body for URL %s", rawUploadURL) 276 } 277 }() 278 279 if resp.StatusCode >= http.StatusBadRequest { 280 msg := fmt.Sprintf("uploading %v to %v resulted in response %q", fileInfo.path, rawUploadURL, resp.Status) 281 if body, err := ioutil.ReadAll(resp.Body); err == nil { 282 bodyStr := string(body) 283 if bodyStr != "" { 284 msg += ":\n" + bodyStr 285 } 286 } 287 return rawUploadURL, fmt.Errorf(msg) 288 } 289 290 return rawUploadURL, nil 291 } 292 293 func addChecksumToHeader(header http.Header, checksumName string, checksum string) { 294 header.Add(fmt.Sprintf("X-Checksum-%v", checksumName), checksum) 295 } 296 297 func pomFilePath(buildSpec params.ProductBuildSpec, distCfg params.Dist) string { 298 outputDir := path.Join(buildSpec.ProjectDir, distCfg.OutputDir) 299 values := slsspec.TemplateValues(buildSpec.ProductName, buildSpec.ProductVersion) 300 outputSlsDir := slsspec.New().RootDirName(values) 301 return path.Join(outputDir, path.Base(outputSlsDir)+".pom") 302 }