github.com/jdhenke/godel@v0.0.0-20161213181855-abeb3861bf0d/apps/distgo/cmd/publish/cmd_test.go (about) 1 // Copyright 2016 Palantir Technologies, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package publish 16 17 import ( 18 "bytes" 19 "encoding/json" 20 "fmt" 21 "io/ioutil" 22 "net/http" 23 "net/http/httptest" 24 "os" 25 "path" 26 "path/filepath" 27 "regexp" 28 "strings" 29 "testing" 30 31 "github.com/nmiyake/pkg/dirs" 32 "github.com/palantir/pkg/cli/cfgcli" 33 "github.com/stretchr/testify/assert" 34 "github.com/stretchr/testify/require" 35 36 "github.com/palantir/godel/apps/distgo/cmd/artifacts" 37 "github.com/palantir/godel/apps/distgo/cmd/build" 38 "github.com/palantir/godel/apps/distgo/config" 39 "github.com/palantir/godel/apps/distgo/pkg/git/gittest" 40 ) 41 42 const testMain = "package main; func main(){}" 43 44 func TestPublishBatchErrors(t *testing.T) { 45 var handlerFunc func(w http.ResponseWriter, r *http.Request) 46 handlerFuncPtr := &handlerFunc 47 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 48 localHandlerFunc := *handlerFuncPtr 49 localHandlerFunc(w, r) 50 })) 51 defer ts.Close() 52 53 tmp, cleanup, err := dirs.TempDir(".", "") 54 defer cleanup() 55 require.NoError(t, err) 56 57 wd, err := os.Getwd() 58 defer func() { 59 if err := os.Chdir(wd); err != nil { 60 fmt.Printf("Failed to restore working directory to %v: %v\n", wd, err) 61 } 62 }() 63 require.NoError(t, err) 64 65 for i, currCase := range []struct { 66 cfg string 67 mainFiles []string 68 publishProducts []string 69 handler func(w http.ResponseWriter, r *http.Request) 70 failFast bool 71 wantOutputRegexp string 72 notWantOutputRegexp string 73 wantErrorRegexps []string 74 }{ 75 // if failFast is false, all products should attempt publish 76 { 77 cfg: ` 78 products: 79 test-bar: 80 build: 81 main-pkg: ./bar 82 test-baz: 83 build: 84 main-pkg: ./baz 85 test-foo: 86 build: 87 main-pkg: ./foo 88 group-id: com.palantir.distgo-cmd-test`, 89 mainFiles: []string{"foo/main.go", "bar/main.go", "baz/main.go"}, 90 publishProducts: []string{"test-foo", "test-bar", "test-baz"}, 91 handler: func(w http.ResponseWriter, r *http.Request) { 92 status := http.StatusOK 93 if strings.Contains(r.URL.String(), "test-bar") || strings.Contains(r.URL.String(), "test-baz") { 94 // fail all test-bar and test-baz publishes 95 status = http.StatusNotFound 96 } 97 w.WriteHeader(status) 98 }, 99 wantOutputRegexp: `(?s).+Uploading dist/test-foo-unspecified.pom to .+`, 100 notWantOutputRegexp: `(?s).+Uploading dist/test-bar-unspecified.pom to .+`, 101 wantErrorRegexps: []string{`Publish failed for test-bar: uploading .+ to .+ resulted in response "404 Not Found"`, `Publish failed for test-baz: uploading .+ to .+ resulted in response "404 Not Found"`}, 102 }, 103 // if failFast is true, first fail should terminate publish 104 { 105 cfg: ` 106 products: 107 test-bar: 108 build: 109 main-pkg: ./bar 110 test-baz: 111 build: 112 main-pkg: ./baz 113 test-foo: 114 build: 115 main-pkg: ./foo 116 group-id: com.palantir.distgo-cmd-test`, 117 mainFiles: []string{"foo/main.go", "bar/main.go", "baz/main.go"}, 118 publishProducts: []string{"test-foo", "test-bar", "test-baz"}, 119 handler: func(w http.ResponseWriter, r *http.Request) { 120 status := http.StatusOK 121 if strings.Contains(r.URL.String(), "test-bar") || strings.Contains(r.URL.String(), "test-baz") { 122 // fail all test-bar and test-baz publishes 123 status = http.StatusNotFound 124 } 125 w.WriteHeader(status) 126 }, 127 failFast: true, 128 notWantOutputRegexp: `(?s).+Uploading dist/test-bar-unspecified.tgz to .+`, 129 wantErrorRegexps: []string{`^Publish failed for test-bar: uploading .+ to .+ resulted in response "404 Not Found"$`}, 130 }, 131 } { 132 err = os.Chdir(wd) 133 require.NoError(t, err) 134 135 handlerFunc = currCase.handler 136 137 currTmp, err := ioutil.TempDir(tmp, "") 138 require.NoError(t, err) 139 140 gittest.InitGitDir(t, currTmp) 141 142 for _, currMain := range currCase.mainFiles { 143 err = os.MkdirAll(path.Dir(path.Join(currTmp, currMain)), 0755) 144 require.NoError(t, err) 145 err = ioutil.WriteFile(path.Join(currTmp, currMain), []byte(testMain), 0644) 146 require.NoError(t, err) 147 } 148 149 err = ioutil.WriteFile(path.Join(currTmp, "dist.yml"), []byte(currCase.cfg), 0644) 150 require.NoError(t, err) 151 152 cfgcli.ConfigPath = "dist.yml" 153 154 err = os.Chdir(currTmp) 155 require.NoError(t, err) 156 157 p := ArtifactoryConnectionInfo{ 158 BasicConnectionInfo: BasicConnectionInfo{ 159 URL: ts.URL, 160 Username: "username", 161 Password: "password", 162 }, 163 Repository: "repo", 164 } 165 166 buf := &bytes.Buffer{} 167 err = publishAction(p, currCase.publishProducts, nil, currCase.failFast, buf, ".") 168 assert.Regexp(t, regexp.MustCompile(currCase.wantOutputRegexp), buf.String(), "Case %d", i) 169 assert.NotRegexp(t, regexp.MustCompile(currCase.notWantOutputRegexp), buf.String(), "Case %d", i) 170 for _, currWantRegexp := range currCase.wantErrorRegexps { 171 assert.Regexp(t, regexp.MustCompile(currWantRegexp), err.Error(), "Case %d", i) 172 } 173 } 174 } 175 176 func TestArtifactoryPublishChecksums(t *testing.T) { 177 var handlerFunc func(w http.ResponseWriter, r *http.Request) 178 handlerFuncPtr := &handlerFunc 179 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 180 localHandlerFunc := *handlerFuncPtr 181 localHandlerFunc(w, r) 182 })) 183 defer ts.Close() 184 185 tmpDir, cleanup, err := dirs.TempDir(".", "") 186 defer cleanup() 187 require.NoError(t, err) 188 189 wd, err := os.Getwd() 190 defer func() { 191 if err := os.Chdir(wd); err != nil { 192 fmt.Printf("Failed to restore working directory to %v: %v\n", wd, err) 193 } 194 }() 195 require.NoError(t, err) 196 197 for i, currCase := range []struct { 198 cfg string 199 mainFiles []string 200 publishProducts []string 201 handler func(productToArtifact map[string]string) func(w http.ResponseWriter, r *http.Request) 202 wantRegexp func(productToArtifact map[string]string) string 203 notWantRegexp func(productToArtifact map[string]string) string 204 }{ 205 // upload for products with matching checksums are skipped 206 { 207 cfg: ` 208 products: 209 foo: 210 build: 211 main-pkg: ./foo 212 group-id: com.palantir.distgo-cmd-test`, 213 mainFiles: []string{"foo/main.go"}, 214 publishProducts: []string{"foo"}, 215 handler: func(productToArtifact map[string]string) func(w http.ResponseWriter, r *http.Request) { 216 fooArtifactPath := productToArtifact["foo"] 217 fooArtifactName := path.Base(fooArtifactPath) 218 return func(w http.ResponseWriter, r *http.Request) { 219 if strings.Contains(r.URL.String(), fooArtifactName) { 220 fileInfo, err := newFileInfo(fooArtifactPath) 221 require.NoError(t, err) 222 bytes, err := json.Marshal(map[string]checksums{ 223 "checksums": fileInfo.checksums, 224 }) 225 require.NoError(t, err) 226 _, err = w.Write(bytes) 227 require.NoError(t, err) 228 } else { 229 w.WriteHeader(http.StatusOK) 230 } 231 } 232 }, 233 wantRegexp: func(productToArtifact map[string]string) string { 234 fooArtifactName := path.Base(productToArtifact["foo"]) 235 return fmt.Sprintf("File dist/%s already exists at .+, skipping upload", fooArtifactName) 236 }, 237 }, 238 // upload for products is skipped even if only single checksum matches 239 { 240 cfg: ` 241 products: 242 foo: 243 build: 244 main-pkg: ./foo 245 group-id: com.palantir.distgo-cmd-test`, 246 mainFiles: []string{"foo/main.go"}, 247 publishProducts: []string{"foo"}, 248 handler: func(productToArtifact map[string]string) func(w http.ResponseWriter, r *http.Request) { 249 fooArtifactPath := productToArtifact["foo"] 250 fooArtifactName := path.Base(fooArtifactPath) 251 return func(w http.ResponseWriter, r *http.Request) { 252 if strings.Contains(r.URL.String(), fooArtifactName) { 253 fileInfo, err := newFileInfo(fooArtifactPath) 254 require.NoError(t, err) 255 hashes := fileInfo.checksums 256 // set 2 hashes to blank, but keep one matching hash 257 hashes.SHA256 = "" 258 hashes.MD5 = "" 259 bytes, err := json.Marshal(map[string]checksums{ 260 "checksums": hashes, 261 }) 262 require.NoError(t, err) 263 _, err = w.Write(bytes) 264 require.NoError(t, err) 265 } else { 266 w.WriteHeader(http.StatusOK) 267 } 268 } 269 }, 270 wantRegexp: func(productToArtifact map[string]string) string { 271 fooArtifactName := path.Base(productToArtifact["foo"]) 272 return fmt.Sprintf("File dist/%s already exists at .+, skipping upload", fooArtifactName) 273 }, 274 }, 275 // product is uploaded if checksum does not match 276 { 277 cfg: ` 278 products: 279 foo: 280 build: 281 main-pkg: ./foo 282 group-id: com.palantir.distgo-cmd-test`, 283 mainFiles: []string{"foo/main.go"}, 284 publishProducts: []string{"foo"}, 285 handler: func(productToArtifact map[string]string) func(w http.ResponseWriter, r *http.Request) { 286 fooArtifactPath := productToArtifact["foo"] 287 fooArtifactName := path.Base(fooArtifactPath) 288 return func(w http.ResponseWriter, r *http.Request) { 289 if strings.Contains(r.URL.String(), fooArtifactName) { 290 fileInfo, err := newFileInfo(fooArtifactPath) 291 require.NoError(t, err) 292 hashes := fileInfo.checksums 293 // 2 hashes match, but one does not 294 hashes.SHA1 = "invalid" 295 bytes, err := json.Marshal(map[string]checksums{ 296 "checksums": hashes, 297 }) 298 require.NoError(t, err) 299 _, err = w.Write(bytes) 300 require.NoError(t, err) 301 } else { 302 w.WriteHeader(http.StatusOK) 303 } 304 } 305 }, 306 notWantRegexp: func(productToArtifact map[string]string) string { 307 return "File .+ already exists at .+, skipping upload" 308 }, 309 }, 310 // product is uploaded if response does not contain checksums 311 { 312 cfg: ` 313 products: 314 foo: 315 build: 316 main-pkg: ./foo 317 group-id: com.palantir.distgo-cmd-test`, 318 mainFiles: []string{"foo/main.go"}, 319 publishProducts: []string{"foo"}, 320 handler: func(productToArtifact map[string]string) func(w http.ResponseWriter, r *http.Request) { 321 fooArtifactPath := productToArtifact["foo"] 322 fooArtifactName := path.Base(fooArtifactPath) 323 return func(w http.ResponseWriter, r *http.Request) { 324 if strings.Contains(r.URL.String(), fooArtifactName) { 325 bytes, err := json.Marshal(map[string]string{ 326 "no-checksum": "placeholder", 327 }) 328 require.NoError(t, err) 329 _, err = w.Write(bytes) 330 require.NoError(t, err) 331 } else { 332 w.WriteHeader(http.StatusOK) 333 } 334 } 335 }, 336 notWantRegexp: func(productToArtifact map[string]string) string { 337 return "File .+ already exists at .+, skipping upload" 338 }, 339 }, 340 } { 341 err = os.Chdir(wd) 342 require.NoError(t, err) 343 344 currTmp, err := ioutil.TempDir(tmpDir, "") 345 require.NoError(t, err) 346 347 currTmp, err = filepath.Abs(currTmp) 348 require.NoError(t, err) 349 350 gittest.InitGitDir(t, currTmp) 351 352 for _, currMain := range currCase.mainFiles { 353 err = os.MkdirAll(path.Dir(path.Join(currTmp, currMain)), 0755) 354 require.NoError(t, err) 355 err = ioutil.WriteFile(path.Join(currTmp, currMain), []byte(testMain), 0644) 356 require.NoError(t, err) 357 } 358 359 err = ioutil.WriteFile(path.Join(currTmp, "dist.yml"), []byte(currCase.cfg), 0644) 360 require.NoError(t, err) 361 362 cfgcli.ConfigPath = "dist.yml" 363 364 err = os.Chdir(currTmp) 365 require.NoError(t, err) 366 367 cfg, err := config.Load(cfgcli.ConfigPath, cfgcli.ConfigJSON) 368 require.NoError(t, err) 369 370 buildSpecsWithDeps, err := build.SpecsWithDepsForArgs(cfg, currCase.publishProducts, currTmp) 371 require.NoError(t, err) 372 373 artifacts, err := artifacts.DistArtifacts(buildSpecsWithDeps, true) 374 require.NoError(t, err) 375 376 productToArtifact := make(map[string]string) 377 for k, v := range artifacts { 378 require.Equal(t, 1, len(v.Keys())) 379 productToArtifact[k] = v.Get(v.Keys()[0]) 380 } 381 382 handlerFunc = currCase.handler(productToArtifact) 383 384 p := ArtifactoryConnectionInfo{ 385 BasicConnectionInfo: BasicConnectionInfo{ 386 URL: ts.URL, 387 Username: "username", 388 Password: "password", 389 }, 390 Repository: "repo", 391 } 392 393 buf := &bytes.Buffer{} 394 395 err = publishAction(p, currCase.publishProducts, nil, true, buf, ".") 396 require.NoError(t, err, "Case %d", i) 397 398 if currCase.wantRegexp != nil { 399 assert.Regexp(t, regexp.MustCompile(currCase.wantRegexp(productToArtifact)), buf.String(), "Case %d", i) 400 } 401 402 if currCase.notWantRegexp != nil { 403 assert.NotRegexp(t, regexp.MustCompile(currCase.notWantRegexp(productToArtifact)), buf.String(), "Case %d", i) 404 } 405 } 406 } 407 408 func TestAlmanacPublishCheckURL(t *testing.T) { 409 var handlerFunc func(w http.ResponseWriter, r *http.Request) 410 handlerFuncPtr := &handlerFunc 411 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 412 localHandlerFunc := *handlerFuncPtr 413 localHandlerFunc(w, r) 414 })) 415 defer ts.Close() 416 417 tmpDir, cleanup, err := dirs.TempDir(".", "") 418 defer cleanup() 419 require.NoError(t, err) 420 421 wd, err := os.Getwd() 422 defer func() { 423 if err := os.Chdir(wd); err != nil { 424 fmt.Printf("Failed to restore working directory to %v: %v\n", wd, err) 425 } 426 }() 427 require.NoError(t, err) 428 429 for i, currCase := range []struct { 430 cfg string 431 mainFiles []string 432 publishProducts []string 433 handler func(w http.ResponseWriter, r *http.Request) 434 wantRegexp string 435 notWantRegexp string 436 wantErrorRegexp string 437 }{ 438 // Almanac publish for products with matching URLs are skipped 439 { 440 cfg: ` 441 products: 442 foo: 443 build: 444 main-pkg: ./foo 445 group-id: com.palantir.distgo-cmd-test`, 446 mainFiles: []string{"foo/main.go"}, 447 publishProducts: []string{"foo"}, 448 handler: func(w http.ResponseWriter, r *http.Request) { 449 if r.URL.String() == "/v1/units/foo/unspecified/1" { 450 urlMap := map[string]string{ 451 "url": ts.URL + "/artifactory/repo/com/palantir/distgo-cmd-test/foo/unspecified/foo-unspecified.sls.tgz", 452 } 453 bytes, err := json.Marshal(urlMap) 454 require.NoError(t, err) 455 _, err = w.Write(bytes) 456 require.NoError(t, err) 457 } else { 458 w.WriteHeader(http.StatusOK) 459 } 460 }, 461 wantRegexp: fmt.Sprintf(`(?s).+Unit for product foo branch unspecified revision 1 with URL %s already exists; skipping publish.+`, ts.URL+"/artifactory/repo/com/palantir/distgo-cmd-test/foo/unspecified/foo-unspecified.sls.tgz"), 462 }, 463 // Almanac publish for product that do not exist in Almanac succeeds 464 { 465 cfg: ` 466 products: 467 foo: 468 build: 469 main-pkg: ./foo 470 group-id: com.palantir.distgo-cmd-test`, 471 mainFiles: []string{"foo/main.go"}, 472 publishProducts: []string{"foo"}, 473 handler: func(w http.ResponseWriter, r *http.Request) { 474 status := http.StatusOK 475 if r.URL.String() == "/v1/units/foo/unspecified/1" { 476 // return error for unit 477 status = http.StatusBadRequest 478 } 479 w.WriteHeader(status) 480 }, 481 notWantRegexp: `(?s).+Unit for product .+ branch .+ revision .+ with the URL .+ already exists; skipping publish.+`, 482 }, 483 // Almanac publish for product that exists in Almanac (but has no URL) fails 484 { 485 cfg: ` 486 products: 487 foo: 488 build: 489 main-pkg: ./foo 490 group-id: com.palantir.distgo-cmd-test`, 491 mainFiles: []string{"foo/main.go"}, 492 publishProducts: []string{"foo"}, 493 handler: func(w http.ResponseWriter, r *http.Request) { 494 w.WriteHeader(http.StatusOK) 495 }, 496 notWantRegexp: `(?s).+Unit for product .+ branch .+ revision .+ with the URL .+ already exists; skipping publish.+`, 497 wantErrorRegexp: `^Almanac publish failed for foo: unit for product foo branch unspecified revision 1 already exists; not overwriting it$`, 498 }, 499 // Almanac publish for product that exists in Almanac (but has different URL) fails 500 { 501 cfg: ` 502 products: 503 foo: 504 build: 505 main-pkg: ./foo 506 group-id: com.palantir.distgo-cmd-test`, 507 mainFiles: []string{"foo/main.go"}, 508 publishProducts: []string{"foo"}, 509 handler: func(w http.ResponseWriter, r *http.Request) { 510 if r.URL.String() == "/v1/units/foo/unspecified/1" { 511 urlMap := map[string]string{ 512 "url": "nonMatchingURL", 513 } 514 bytes, err := json.Marshal(urlMap) 515 require.NoError(t, err) 516 _, err = w.Write(bytes) 517 require.NoError(t, err) 518 } else { 519 w.WriteHeader(http.StatusOK) 520 } 521 }, 522 notWantRegexp: `(?s).+Unit for product .+ branch .+ revision .+ with the URL .+ already exists; skipping publish.+`, 523 wantErrorRegexp: `^Almanac publish failed for foo: unit for product foo branch unspecified revision 1 already exists; not overwriting it$`, 524 }, 525 } { 526 err = os.Chdir(wd) 527 require.NoError(t, err) 528 529 currTmp, err := ioutil.TempDir(tmpDir, "") 530 require.NoError(t, err) 531 532 gittest.InitGitDir(t, currTmp) 533 534 for _, currMain := range currCase.mainFiles { 535 err = os.MkdirAll(path.Dir(path.Join(currTmp, currMain)), 0755) 536 require.NoError(t, err) 537 err = ioutil.WriteFile(path.Join(currTmp, currMain), []byte(testMain), 0644) 538 require.NoError(t, err) 539 } 540 541 err = ioutil.WriteFile(path.Join(currTmp, "dist.yml"), []byte(currCase.cfg), 0644) 542 require.NoError(t, err) 543 544 cfgcli.ConfigPath = "dist.yml" 545 546 err = os.Chdir(currTmp) 547 require.NoError(t, err) 548 549 handlerFunc = currCase.handler 550 551 p := ArtifactoryConnectionInfo{ 552 BasicConnectionInfo: BasicConnectionInfo{ 553 URL: ts.URL, 554 Username: "username", 555 Password: "password", 556 }, 557 Repository: "repo", 558 } 559 560 a := &AlmanacInfo{ 561 URL: ts.URL, 562 AccessID: "username", 563 Secret: "password", 564 } 565 566 buf := &bytes.Buffer{} 567 568 err = publishAction(p, currCase.publishProducts, a, true, buf, ".") 569 570 if currCase.wantErrorRegexp != "" { 571 assert.Regexp(t, regexp.MustCompile(currCase.wantErrorRegexp), err.Error(), "Case %d", i) 572 } else { 573 require.NoError(t, err, "Case %d", i) 574 } 575 576 if currCase.wantRegexp != "" { 577 assert.Regexp(t, regexp.MustCompile(currCase.wantRegexp), buf.String(), "Case %d", i) 578 } 579 580 if currCase.notWantRegexp != "" { 581 assert.NotRegexp(t, regexp.MustCompile(currCase.notWantRegexp), buf.String(), "Case %d", i) 582 } 583 } 584 }