github.com/jonsyu1/godel@v0.0.0-20171017211503-64567a0cf169/apps/distgo/cmd/publish/publish.go (about) 1 // Copyright 2016 Palantir Technologies, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package publish 16 17 import ( 18 "bytes" 19 "crypto/md5" 20 "crypto/sha1" 21 "crypto/sha256" 22 "encoding/hex" 23 "fmt" 24 "io" 25 "io/ioutil" 26 "net/http" 27 "net/url" 28 "os" 29 "path" 30 "strings" 31 32 "github.com/pkg/errors" 33 "gopkg.in/cheggaaa/pb.v1" 34 35 "github.com/palantir/godel/apps/distgo/cmd/dist" 36 "github.com/palantir/godel/apps/distgo/params" 37 "github.com/palantir/godel/apps/distgo/pkg/slsspec" 38 "github.com/palantir/godel/apps/distgo/templating" 39 ) 40 41 type Publisher interface { 42 Publish(buildSpec params.ProductBuildSpec, paths ProductPaths, stdout io.Writer) ([]string, error) 43 } 44 45 type BasicConnectionInfo struct { 46 URL string 47 Username string 48 Password string 49 } 50 51 func Run(buildSpecWithDeps params.ProductBuildSpecWithDeps, publisher Publisher, almanacInfo *AlmanacInfo, stdout io.Writer) error { 52 buildSpec := buildSpecWithDeps.Spec 53 for _, currDistCfg := range buildSpec.Dist { 54 // verify that distribution to publish exists 55 for _, artifactPath := range dist.FullArtifactsPaths(dist.ToDister(currDistCfg.Info), buildSpec, currDistCfg) { 56 if _, err := os.Stat(artifactPath); os.IsNotExist(err) { 57 return errors.Errorf("distribution for %s does not exist at %s", buildSpec.ProductName, artifactPath) 58 } 59 } 60 61 paths, err := productPath(buildSpecWithDeps, currDistCfg) 62 if err != nil { 63 return errors.Wrapf(err, "failed to determine product paths") 64 } 65 66 artifactURLs, err := publisher.Publish(buildSpec, paths, stdout) 67 if err != nil { 68 return fmt.Errorf("Publish failed for %s: %v", buildSpec.ProductName, err) 69 } 70 71 if almanacInfo != nil { 72 for _, currArtifactURL := range artifactURLs { 73 if err := almanacPublish(currArtifactURL, *almanacInfo, buildSpec, currDistCfg, stdout); err != nil { 74 return fmt.Errorf("Almanac publish failed for %s: %v", buildSpec.ProductName, err) 75 } 76 } 77 } 78 } 79 return nil 80 } 81 82 func DistsNotBuilt(buildSpecWithDeps []params.ProductBuildSpecWithDeps) []params.ProductBuildSpecWithDeps { 83 var distsNotBuilt []params.ProductBuildSpecWithDeps 84 for _, currBuildSpecWithDeps := range buildSpecWithDeps { 85 currBuildSpec := currBuildSpecWithDeps.Spec 86 for _, currDistCfg := range currBuildSpec.Dist { 87 for _, artifactPath := range dist.FullArtifactsPaths(dist.ToDister(currDistCfg.Info), currBuildSpec, currDistCfg) { 88 if _, err := os.Stat(artifactPath); os.IsNotExist(err) { 89 distsNotBuilt = append(distsNotBuilt, currBuildSpecWithDeps) 90 } 91 } 92 } 93 } 94 return distsNotBuilt 95 } 96 97 type ProductPaths struct { 98 // path of the form "{{GroupID}}/{{ProductName}}/{{ProductVersion}}". For example, "com/group/foo-service/1.0.1". 99 productPath string 100 pomFilePath string 101 artifactPaths []string 102 } 103 104 func productPath(buildSpecWithDeps params.ProductBuildSpecWithDeps, distCfg params.Dist) (ProductPaths, error) { 105 buildSpec := buildSpecWithDeps.Spec 106 107 pomBytes, err := generatePOM(templating.ConvertSpec(buildSpec, distCfg), dist.ToDister(distCfg.Info).DistPackageType()) 108 if err != nil { 109 return ProductPaths{}, errors.Wrapf(err, "failed to generate POM") 110 } 111 112 pomFilePath := pomFilePath(buildSpec, distCfg) 113 if err := ioutil.WriteFile(pomFilePath, pomBytes, 0644); err != nil { 114 return ProductPaths{}, errors.Wrapf(err, "failed to write POM file to %s", pomFilePath) 115 } 116 117 return ProductPaths{ 118 productPath: path.Join(path.Join(strings.Split(distCfg.Publish.GroupID, ".")...), buildSpec.ProductName, buildSpec.ProductVersion), 119 pomFilePath: pomFilePath, 120 artifactPaths: dist.FullArtifactsPaths(dist.ToDister(distCfg.Info), buildSpec, distCfg), 121 }, nil 122 } 123 124 func (b BasicConnectionInfo) uploadArtifacts(baseURL string, paths ProductPaths, artifactExists artifactExistsFunc, stdout io.Writer) ([]string, error) { 125 var artifactURLs []string 126 for _, currArtifactPath := range paths.artifactPaths { 127 artifactURL, err := b.uploadFile(currArtifactPath, baseURL, currArtifactPath, artifactExists, stdout) 128 if err != nil { 129 return artifactURLs, err 130 } 131 artifactURLs = append(artifactURLs, artifactURL) 132 } 133 if _, err := b.uploadFile(paths.pomFilePath, baseURL, paths.pomFilePath, artifactExists, stdout); err != nil { 134 return artifactURLs, err 135 } 136 return artifactURLs, nil 137 } 138 139 type fileInfo struct { 140 path string 141 bytes []byte 142 checksums checksums 143 } 144 145 type checksums struct { 146 SHA1 string 147 SHA256 string 148 MD5 string 149 } 150 151 func (c checksums) match(other checksums) bool { 152 nonEmptyEqual := nonEmptyEqual(c.MD5, other.MD5) || nonEmptyEqual(c.SHA1, other.SHA1) || nonEmptyEqual(c.SHA256, other.SHA256) 153 // if no non-empty checksums are equal, checksums are not equal 154 if !nonEmptyEqual { 155 return false 156 } 157 158 // if non-empty checksums are different, treat as suspect and return false 159 if nonEmptyDiffer(c.MD5, other.MD5) || nonEmptyDiffer(c.SHA1, other.SHA1) || nonEmptyDiffer(c.SHA256, other.SHA256) { 160 return false 161 } 162 163 // at least one non-empty checksum was equal, and no non-empty checksums differed 164 return true 165 } 166 167 func nonEmptyEqual(s1, s2 string) bool { 168 return s1 != "" && s2 != "" && s1 == s2 169 } 170 171 func nonEmptyDiffer(s1, s2 string) bool { 172 return s1 != "" && s2 != "" && s1 != s2 173 } 174 175 // function type returns true if the file represented by the given fileInfo object 176 type artifactExistsFunc func(fi fileInfo, dstFileName, username, password string) bool 177 178 func newFileInfo(pathToFile string) (fileInfo, error) { 179 bytes, err := ioutil.ReadFile(pathToFile) 180 if err != nil { 181 return fileInfo{}, errors.Wrapf(err, "Failed to read file %s", pathToFile) 182 } 183 184 sha1Bytes := sha1.Sum(bytes) 185 sha256Bytes := sha256.Sum256(bytes) 186 md5Bytes := md5.Sum(bytes) 187 188 return fileInfo{ 189 path: pathToFile, 190 bytes: bytes, 191 checksums: checksums{ 192 SHA1: hex.EncodeToString(sha1Bytes[:]), 193 SHA256: hex.EncodeToString(sha256Bytes[:]), 194 MD5: hex.EncodeToString(md5Bytes[:]), 195 }, 196 }, nil 197 } 198 199 func (b BasicConnectionInfo) uploadFile(filePath, baseURL, artifactPath string, artifactExists artifactExistsFunc, stdout io.Writer) (rURL string, rErr error) { 200 rawUploadURL := strings.Join([]string{baseURL, path.Base(artifactPath)}, "/") 201 202 fileInfo, err := newFileInfo(filePath) 203 if err != nil { 204 return rawUploadURL, err 205 } 206 207 if artifactExists != nil && artifactExists(fileInfo, path.Base(artifactPath), b.Username, b.Password) { 208 fmt.Fprintf(stdout, "File %s already exists at %s, skipping upload.\n", filePath, rawUploadURL) 209 return rawUploadURL, nil 210 } 211 212 uploadURL, err := url.Parse(rawUploadURL) 213 if err != nil { 214 return rawUploadURL, errors.Wrapf(err, "Failed to parse %s as URL", rawUploadURL) 215 } 216 217 fmt.Fprintf(stdout, "Uploading %s to %s\n", fileInfo.path, rawUploadURL) 218 219 header := http.Header{} 220 addChecksumToHeader(header, "Md5", fileInfo.checksums.MD5) 221 addChecksumToHeader(header, "Sha1", fileInfo.checksums.SHA1) 222 addChecksumToHeader(header, "Sha256", fileInfo.checksums.SHA256) 223 224 bar := pb.New(len(fileInfo.bytes)).SetUnits(pb.U_BYTES) 225 bar.Output = stdout 226 bar.SetMaxWidth(120) 227 bar.Start() 228 defer bar.Finish() 229 reader := bar.NewProxyReader(bytes.NewReader(fileInfo.bytes)) 230 231 req := http.Request{ 232 Method: http.MethodPut, 233 URL: uploadURL, 234 Header: header, 235 Body: ioutil.NopCloser(reader), 236 ContentLength: int64(len(fileInfo.bytes)), 237 } 238 req.SetBasicAuth(b.Username, b.Password) 239 240 resp, err := http.DefaultClient.Do(&req) 241 if err != nil { 242 return rawUploadURL, errors.Wrapf(err, "failed to upload %s to %s", fileInfo.path, rawUploadURL) 243 } 244 defer func() { 245 if err := resp.Body.Close(); err != nil && rErr == nil { 246 rErr = errors.Wrapf(err, "failed to close response body for URL %s", rawUploadURL) 247 } 248 }() 249 250 if resp.StatusCode >= http.StatusBadRequest { 251 msg := fmt.Sprintf("uploading %s to %s resulted in response %q", fileInfo.path, rawUploadURL, resp.Status) 252 if body, err := ioutil.ReadAll(resp.Body); err == nil { 253 bodyStr := string(body) 254 if bodyStr != "" { 255 msg += ":\n" + bodyStr 256 } 257 } 258 return rawUploadURL, fmt.Errorf(msg) 259 } 260 261 return rawUploadURL, nil 262 } 263 264 func addChecksumToHeader(header http.Header, checksumName, checksum string) { 265 header.Add(fmt.Sprintf("X-Checksum-%s", checksumName), checksum) 266 } 267 268 func pomFilePath(buildSpec params.ProductBuildSpec, distCfg params.Dist) string { 269 outputDir := path.Join(buildSpec.ProjectDir, distCfg.OutputDir) 270 values := slsspec.TemplateValues(buildSpec.ProductName, buildSpec.ProductVersion) 271 outputSlsDir := slsspec.New().RootDirName(values) 272 return path.Join(outputDir, path.Base(outputSlsDir)+".pom") 273 }