sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/spyglass/spyglass_test.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package spyglass 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "os" 24 "reflect" 25 "sort" 26 "strings" 27 "testing" 28 29 "k8s.io/apimachinery/pkg/util/sets" 30 31 coreapi "k8s.io/api/core/v1" 32 "sigs.k8s.io/prow/pkg/gcsupload" 33 "sigs.k8s.io/prow/pkg/pod-utils/downwardapi" 34 35 "github.com/fsouza/fake-gcs-server/fakestorage" 36 "github.com/sirupsen/logrus" 37 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 38 39 tgconf "github.com/GoogleCloudPlatform/testgrid/pb/config" 40 41 ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" 42 prowapi "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" 43 "sigs.k8s.io/prow/pkg/config" 44 "sigs.k8s.io/prow/pkg/deck/jobs" 45 "sigs.k8s.io/prow/pkg/io" 46 "sigs.k8s.io/prow/pkg/kube" 47 "sigs.k8s.io/prow/pkg/spyglass/api" 48 "sigs.k8s.io/prow/pkg/spyglass/lenses" 49 "sigs.k8s.io/prow/pkg/spyglass/lenses/common" 50 ) 51 52 var ( 53 fakeJa *jobs.JobAgent 54 fakeGCSServer *fakestorage.Server 55 ) 56 57 type fkc []prowapi.ProwJob 58 59 func (f fkc) List(ctx context.Context, pjs *prowapi.ProwJobList, _ ...ctrlruntimeclient.ListOption) error { 60 pjs.Items = f 61 return nil 62 } 63 64 type fpkc string 65 66 func (f fpkc) GetLogs(name, container string) ([]byte, error) { 67 if name == "wowowow" || name == "powowow" { 68 return []byte(fmt.Sprintf("%s.%s", f, container)), nil 69 } 70 return nil, fmt.Errorf("pod not found: %s", name) 71 } 72 73 type fca struct { 74 c config.Config 75 } 76 77 func (ca fca) Config() *config.Config { 78 return &ca.c 79 } 80 81 func TestMain(m *testing.M) { 82 var longLog string 83 for i := 0; i < 300; i++ { 84 longLog += "here a log\nthere a log\neverywhere a log log\n" 85 } 86 fakeGCSServer = fakestorage.NewServer([]fakestorage.Object{ 87 { 88 BucketName: "test-bucket", 89 Name: "logs/example-ci-run/403/build-log.txt", 90 Content: []byte("Oh wow\nlogs\nthis is\ncrazy"), 91 Metadata: map[string]string{ 92 "foo": "bar", 93 }, 94 }, 95 { 96 BucketName: "test-bucket", 97 Name: "logs/example-ci-run/403/long-log.txt", 98 Content: []byte(longLog), 99 }, 100 { 101 BucketName: "test-bucket", 102 Name: "logs/example-ci-run/403/junit_01.xml", 103 Content: []byte(`<testsuite tests="1017" failures="1017" time="0.016981535"> 104 <testcase name="BeforeSuite" classname="Kubernetes e2e suite" time="0.006343795"> 105 <failure type="Failure"> 106 test/e2e/e2e.go:137 BeforeSuite on Node 1 failed test/e2e/e2e.go:137 107 </failure> 108 </testcase> 109 </testsuite>`), 110 }, 111 { 112 BucketName: "test-bucket", 113 Name: "logs/example-ci-run/403/started.json", 114 Content: []byte(`{ 115 "node": "gke-prow-default-pool-3c8994a8-qfhg", 116 "repo-version": "v1.12.0-alpha.0.985+e6f64d0a79243c", 117 "timestamp": 1528742858, 118 "repos": { 119 "k8s.io/kubernetes": "master", 120 "k8s.io/release": "master" 121 }, 122 "version": "v1.12.0-alpha.0.985+e6f64d0a79243c", 123 "metadata": { 124 "pod": "cbc53d8e-6da7-11e8-a4ff-0a580a6c0269" 125 } 126 }`), 127 }, 128 { 129 BucketName: "test-bucket", 130 Name: "logs/example-ci-run/403/finished.json", 131 Content: []byte(`{ 132 "timestamp": 1528742943, 133 "version": "v1.12.0-alpha.0.985+e6f64d0a79243c", 134 "result": "SUCCESS", 135 "passed": true, 136 "job-version": "v1.12.0-alpha.0.985+e6f64d0a79243c", 137 "metadata": { 138 "repo": "k8s.io/kubernetes", 139 "repos": { 140 "k8s.io/kubernetes": "master", 141 "k8s.io/release": "master" 142 }, 143 "infra-commit": "260081852", 144 "pod": "cbc53d8e-6da7-11e8-a4ff-0a580a6c0269", 145 "repo-commit": "e6f64d0a79243c834babda494151fc5d66582240" 146 }, 147 },`), 148 }, 149 { 150 BucketName: "test-bucket", 151 Name: "logs/symlink-party/123.txt", 152 Content: []byte(`gs://test-bucket/logs/the-actual-place/123`), 153 }, 154 { 155 BucketName: "multi-container-one-log", 156 Name: "logs/job/123/test-1-build-log.txt", 157 Content: []byte("this log exists in gcs!"), 158 }, 159 }) 160 defer fakeGCSServer.Stop() 161 kc := fkc{ 162 prowapi.ProwJob{ 163 Spec: prowapi.ProwJobSpec{ 164 Agent: prowapi.KubernetesAgent, 165 Job: "job", 166 }, 167 Status: prowapi.ProwJobStatus{ 168 PodName: "wowowow", 169 BuildID: "123", 170 }, 171 }, 172 prowapi.ProwJob{ 173 Spec: prowapi.ProwJobSpec{ 174 Agent: prowapi.KubernetesAgent, 175 Job: "jib", 176 Cluster: "trusted", 177 }, 178 Status: prowapi.ProwJobStatus{ 179 PodName: "powowow", 180 BuildID: "123", 181 }, 182 }, 183 prowapi.ProwJob{ 184 Spec: prowapi.ProwJobSpec{ 185 Agent: prowapi.KubernetesAgent, 186 Job: "example-ci-run", 187 PodSpec: &coreapi.PodSpec{ 188 Containers: []coreapi.Container{ 189 { 190 Image: "tester", 191 }, 192 }, 193 }, 194 }, 195 Status: prowapi.ProwJobStatus{ 196 PodName: "wowowow", 197 BuildID: "404", 198 }, 199 }, 200 prowapi.ProwJob{ 201 Spec: prowapi.ProwJobSpec{ 202 Agent: prowapi.KubernetesAgent, 203 Job: "multiple-container-job", 204 PodSpec: &coreapi.PodSpec{ 205 Containers: []coreapi.Container{ 206 { 207 Name: "test-1", 208 }, 209 { 210 Name: "test-2", 211 }, 212 }, 213 }, 214 }, 215 Status: prowapi.ProwJobStatus{ 216 PodName: "wowowow", 217 BuildID: "123", 218 }, 219 }, 220 } 221 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 222 fakeJa.Start() 223 os.Exit(m.Run()) 224 } 225 226 type dumpLens struct{} 227 228 func (dumpLens) Config() lenses.LensConfig { 229 return lenses.LensConfig{ 230 Name: "dump", 231 Title: "Dump View", 232 } 233 } 234 235 func (dumpLens) Header(artifacts []api.Artifact, resourceDir string, config json.RawMessage, spyglassConfig config.Spyglass) string { 236 return "" 237 } 238 239 func (dumpLens) Body(artifacts []api.Artifact, resourceDir string, data string, config json.RawMessage, spyglassConfig config.Spyglass) string { 240 var view []byte 241 for _, a := range artifacts { 242 data, err := a.ReadAll() 243 if err != nil { 244 logrus.WithError(err).Error("Error reading artifact") 245 continue 246 } 247 view = append(view, data...) 248 } 249 return string(view) 250 } 251 252 func (dumpLens) Callback(artifacts []api.Artifact, resourceDir string, data string, config json.RawMessage, spyglassConfig config.Spyglass) string { 253 return "" 254 } 255 256 func TestViews(t *testing.T) { 257 fakeGCSClient := fakeGCSServer.Client() 258 testCases := []struct { 259 name string 260 registeredViewers []lenses.Lens 261 lenses []int 262 expectedLensTitles []string 263 }{ 264 { 265 name: "Spyglass basic test", 266 registeredViewers: []lenses.Lens{dumpLens{}}, 267 lenses: []int{0}, 268 expectedLensTitles: []string{"Dump View"}, 269 }, 270 } 271 272 for _, tc := range testCases { 273 t.Run(tc.name, func(t *testing.T) { 274 for _, l := range tc.registeredViewers { 275 lenses.RegisterLens(l) 276 } 277 c := fca{ 278 c: config.Config{ 279 ProwConfig: config.ProwConfig{ 280 Deck: config.Deck{ 281 Spyglass: config.Spyglass{ 282 Lenses: []config.LensFileConfig{ 283 { 284 Lens: config.LensConfig{ 285 Name: "dump", 286 }, 287 }, 288 }, 289 }, 290 }, 291 }, 292 }, 293 } 294 sg := New(context.Background(), fakeJa, c.Config, io.NewGCSOpener(fakeGCSClient), false) 295 _, ls := sg.Lenses(tc.lenses) 296 for _, l := range ls { 297 var found bool 298 for _, title := range tc.expectedLensTitles { 299 if title == l.Config().Title { 300 found = true 301 } 302 } 303 if !found { 304 t.Errorf("lens title %s not found in expected titles.", l.Config().Title) 305 } 306 } 307 for _, title := range tc.expectedLensTitles { 308 var found bool 309 for _, l := range ls { 310 if title == l.Config().Title { 311 found = true 312 } 313 } 314 if !found { 315 t.Errorf("expected title %s not found in produced lenses.", title) 316 } 317 } 318 }) 319 } 320 } 321 322 func TestSplitSrc(t *testing.T) { 323 testCases := []struct { 324 name string 325 src string 326 expKeyType string 327 expKey string 328 expError bool 329 }{ 330 { 331 name: "empty string", 332 src: "", 333 expError: true, 334 }, 335 { 336 name: "missing key", 337 src: "gcs", 338 expError: true, 339 }, 340 { 341 name: "prow key", 342 src: "prowjob/example-job-name/123456", 343 expKeyType: "prowjob", 344 expKey: "example-job-name/123456", 345 }, 346 { 347 name: "gcs key", 348 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/", 349 expKeyType: "gcs", 350 expKey: "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/", 351 }, 352 } 353 for _, tc := range testCases { 354 keyType, key, err := splitSrc(tc.src) 355 if tc.expError && err == nil { 356 t.Errorf("test %q expected error", tc.name) 357 } 358 if !tc.expError && err != nil { 359 t.Errorf("test %q encountered unexpected error: %v", tc.name, err) 360 } 361 if keyType != tc.expKeyType || key != tc.expKey { 362 t.Errorf("test %q: splitting src %q: Expected <%q, %q>, got <%q, %q>", 363 tc.name, tc.src, tc.expKeyType, tc.expKey, keyType, key) 364 } 365 } 366 } 367 368 func TestJobPath(t *testing.T) { 369 kc := fkc{ 370 prowapi.ProwJob{ 371 Spec: prowapi.ProwJobSpec{ 372 Type: prowapi.PeriodicJob, 373 Job: "example-periodic-job", 374 DecorationConfig: &prowapi.DecorationConfig{ 375 GCSConfiguration: &prowapi.GCSConfiguration{ 376 Bucket: "chum-bucket", 377 }, 378 }, 379 }, 380 Status: prowapi.ProwJobStatus{ 381 PodName: "flying-whales", 382 BuildID: "1111", 383 }, 384 }, 385 prowapi.ProwJob{ 386 Spec: prowapi.ProwJobSpec{ 387 Type: prowapi.PresubmitJob, 388 Job: "example-presubmit-job", 389 DecorationConfig: &prowapi.DecorationConfig{ 390 GCSConfiguration: &prowapi.GCSConfiguration{ 391 Bucket: "chum-bucket", 392 }, 393 }, 394 }, 395 Status: prowapi.ProwJobStatus{ 396 PodName: "flying-whales", 397 BuildID: "2222", 398 }, 399 }, 400 prowapi.ProwJob{ 401 Spec: prowapi.ProwJobSpec{ 402 Type: prowapi.PresubmitJob, 403 Job: "undecorated-job", 404 }, 405 Status: prowapi.ProwJobStatus{ 406 PodName: "flying-whales", 407 BuildID: "1", 408 }, 409 }, 410 prowapi.ProwJob{ 411 Spec: prowapi.ProwJobSpec{ 412 Type: prowapi.PresubmitJob, 413 Job: "missing-gcs-job", 414 DecorationConfig: &prowapi.DecorationConfig{}, 415 }, 416 Status: prowapi.ProwJobStatus{ 417 PodName: "flying-whales", 418 BuildID: "1", 419 }, 420 }, 421 } 422 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 423 fakeJa.Start() 424 testCases := []struct { 425 name string 426 src string 427 expJobPath string 428 expError bool 429 }{ 430 { 431 name: "non-presubmit job in GCS with trailing /", 432 src: "gcs/kubernetes-jenkins/logs/example-job-name/123/", 433 expJobPath: "gs/kubernetes-jenkins/logs/example-job-name", 434 }, 435 { 436 name: "non-presubmit job in GCS without trailing /", 437 src: "gcs/kubernetes-jenkins/logs/example-job-name/123", 438 expJobPath: "gs/kubernetes-jenkins/logs/example-job-name", 439 }, 440 { 441 name: "presubmit job in GCS with trailing /", 442 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/", 443 expJobPath: "gs/kubernetes-jenkins/pr-logs/directory/example-job-name", 444 }, 445 { 446 name: "presubmit job in GCS without trailing /", 447 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159", 448 expJobPath: "gs/kubernetes-jenkins/pr-logs/directory/example-job-name", 449 }, 450 { 451 name: "non-presubmit Prow job", 452 src: "prowjob/example-periodic-job/1111", 453 expJobPath: "gs/chum-bucket/logs/example-periodic-job", 454 }, 455 { 456 name: "Prow presubmit job", 457 src: "prowjob/example-presubmit-job/2222", 458 expJobPath: "gs/chum-bucket/pr-logs/directory/example-presubmit-job", 459 }, 460 { 461 name: "nonexistent job", 462 src: "prowjob/example-periodic-job/0000", 463 expError: true, 464 }, 465 { 466 name: "invalid key type", 467 src: "oh/my/glob/drama/bomb", 468 expError: true, 469 }, 470 { 471 name: "invalid GCS path", 472 src: "gcs/kubernetes-jenkins/bad-path", 473 expError: true, 474 }, 475 { 476 name: "job missing decoration", 477 src: "prowjob/undecorated-job/1", 478 expError: true, 479 }, 480 { 481 name: "job missing GCS config", 482 src: "prowjob/missing-gcs-job/1", 483 expError: true, 484 }, 485 } 486 for _, tc := range testCases { 487 fakeGCSClient := fakeGCSServer.Client() 488 fakeOpener := io.NewGCSOpener(fakeGCSClient) 489 fca := config.Agent{} 490 sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false) 491 jobPath, err := sg.JobPath(tc.src) 492 if tc.expError && err == nil { 493 t.Errorf("test %q: JobPath(%q) expected error", tc.name, tc.src) 494 continue 495 } 496 if !tc.expError && err != nil { 497 t.Errorf("test %q: JobPath(%q) returned unexpected error %v", tc.name, tc.src, err) 498 continue 499 } 500 if jobPath != tc.expJobPath { 501 t.Errorf("test %q: JobPath(%q) expected %q, got %q", tc.name, tc.src, tc.expJobPath, jobPath) 502 } 503 } 504 } 505 506 func TestProwJob(t *testing.T) { 507 kc := fkc{ 508 prowapi.ProwJob{ 509 ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-1"}, 510 Spec: prowapi.ProwJobSpec{ 511 Type: prowapi.PeriodicJob, 512 Job: "example-periodic-job", 513 DecorationConfig: &prowapi.DecorationConfig{ 514 GCSConfiguration: &prowapi.GCSConfiguration{ 515 Bucket: "chum-bucket", 516 }, 517 }, 518 }, 519 Status: prowapi.ProwJobStatus{ 520 State: prowapi.TriggeredState, 521 PodName: "flying-whales", 522 BuildID: "1111", 523 }, 524 }, 525 prowapi.ProwJob{ 526 ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-2"}, 527 Spec: prowapi.ProwJobSpec{ 528 Type: prowapi.PresubmitJob, 529 Job: "example-presubmit-job", 530 DecorationConfig: &prowapi.DecorationConfig{ 531 GCSConfiguration: &prowapi.GCSConfiguration{ 532 Bucket: "chum-bucket", 533 }, 534 }, 535 }, 536 Status: prowapi.ProwJobStatus{ 537 State: prowapi.PendingState, 538 PodName: "flying-whales", 539 BuildID: "2222", 540 }, 541 }, 542 prowapi.ProwJob{ 543 ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-3"}, 544 Spec: prowapi.ProwJobSpec{ 545 Type: prowapi.PresubmitJob, 546 Job: "undecorated-job", 547 }, 548 Status: prowapi.ProwJobStatus{ 549 State: prowapi.SuccessState, 550 PodName: "flying-whales", 551 BuildID: "1", 552 }, 553 }, 554 prowapi.ProwJob{ 555 Spec: prowapi.ProwJobSpec{ 556 Type: prowapi.PresubmitJob, 557 Job: "missing-name-job", 558 DecorationConfig: &prowapi.DecorationConfig{}, 559 }, 560 Status: prowapi.ProwJobStatus{ 561 PodName: "flying-whales", 562 BuildID: "1", 563 }, 564 }, 565 } 566 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 567 fakeJa.Start() 568 testCases := []struct { 569 name string 570 src string 571 expJob string 572 expJobPath string 573 expJobState prowapi.ProwJobState 574 expError bool 575 }{ 576 { 577 name: "non-presubmit job in GCS without trailing /", 578 src: "gcs/kubernetes-jenkins/logs/example-periodic-job/1111/", 579 expJob: "example-periodic-job", 580 expJobPath: "flying-whales-1", 581 expJobState: prowapi.TriggeredState, 582 }, 583 { 584 name: "presubmit job in GCS with trailing /", 585 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-presubmit-job/2222/", 586 expJob: "example-presubmit-job", 587 expJobPath: "flying-whales-2", 588 expJobState: prowapi.PendingState, 589 }, 590 { 591 name: "non-presubmit Prow job", 592 src: "prowjob/example-periodic-job/1111", 593 expJob: "example-periodic-job", 594 expJobPath: "flying-whales-1", 595 expJobState: prowapi.TriggeredState, 596 }, 597 { 598 name: "Prow presubmit job", 599 src: "prowjob/example-presubmit-job/2222", 600 expJob: "example-presubmit-job", 601 expJobPath: "flying-whales-2", 602 expJobState: prowapi.PendingState, 603 }, 604 { 605 name: "nonexistent job", 606 src: "prowjob/example-periodic-job/0000", 607 expJob: "", 608 expJobPath: "", 609 expJobState: "", 610 }, 611 { 612 name: "job missing name", 613 src: "prowjob/missing-name-job/1", 614 expJob: "missing-name-job", 615 expJobPath: "", 616 expJobState: "", 617 }, 618 { 619 name: "previously invalid key type is now valid but nonexistent", 620 src: "oh/my/glob/drama/bomb", 621 expJob: "", 622 expJobPath: "", 623 expJobState: "", 624 }, 625 { 626 name: "invalid GCS path", 627 src: "gcs/kubernetes-jenkins/bad-path", 628 expError: true, 629 }, 630 } 631 for _, tc := range testCases { 632 fakeGCSClient := fakeGCSServer.Client() 633 fakeOpener := io.NewGCSOpener(fakeGCSClient) 634 fca := config.Agent{} 635 sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false) 636 job, jobPath, jobState, err := sg.ProwJob(tc.src) 637 if tc.expError && err == nil { 638 t.Errorf("test %q: JobPath(%q) expected error", tc.name, tc.src) 639 continue 640 } 641 if !tc.expError && err != nil { 642 t.Errorf("test %q: JobPath(%q) returned unexpected error %v", tc.name, tc.src, err) 643 continue 644 } 645 if job != tc.expJob { 646 t.Errorf("test %q: Job(%q) expected %q, got %q", tc.name, tc.src, tc.expJob, job) 647 } 648 if jobPath != tc.expJobPath { 649 t.Errorf("test %q: JobPath(%q) expected %q, got %q", tc.name, tc.src, tc.expJobPath, jobPath) 650 } 651 if jobState != tc.expJobState { 652 t.Errorf("test %q: JobState(%q) expected %q, got %q", tc.name, tc.src, tc.expJobState, jobState) 653 } 654 } 655 } 656 657 func TestRunPath(t *testing.T) { 658 kc := fkc{ 659 prowapi.ProwJob{ 660 Spec: prowapi.ProwJobSpec{ 661 Type: prowapi.PeriodicJob, 662 Job: "example-periodic-job", 663 DecorationConfig: &prowapi.DecorationConfig{ 664 GCSConfiguration: &prowapi.GCSConfiguration{ 665 Bucket: "chum-bucket", 666 }, 667 }, 668 }, 669 Status: prowapi.ProwJobStatus{ 670 PodName: "flying-whales", 671 BuildID: "1111", 672 URL: "http://magic/view/gcs/chum-bucket/logs/example-periodic-job/1111", 673 }, 674 }, 675 prowapi.ProwJob{ 676 Spec: prowapi.ProwJobSpec{ 677 Type: prowapi.PresubmitJob, 678 Job: "example-presubmit-job", 679 DecorationConfig: &prowapi.DecorationConfig{ 680 GCSConfiguration: &prowapi.GCSConfiguration{ 681 Bucket: "chum-bucket", 682 }, 683 }, 684 Refs: &prowapi.Refs{ 685 Org: "some-org", 686 Repo: "some-repo", 687 Pulls: []prowapi.Pull{ 688 { 689 Number: 42, 690 }, 691 }, 692 }, 693 }, 694 Status: prowapi.ProwJobStatus{ 695 PodName: "flying-whales", 696 BuildID: "2222", 697 URL: "http://magic/view/gcs/chum-bucket/pr-logs/pull/some-org_some-repo/42/example-presubmit-job/2222", 698 }, 699 }, 700 } 701 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 702 fakeJa.Start() 703 testCases := []struct { 704 name string 705 src string 706 expRunPath string 707 expError bool 708 }{ 709 { 710 name: "non-presubmit job in GCS with trailing /", 711 src: "gcs/kubernetes-jenkins/logs/example-job-name/123/", 712 expRunPath: "kubernetes-jenkins/logs/example-job-name/123", 713 }, 714 { 715 name: "non-presubmit job in GCS without trailing /", 716 src: "gcs/kubernetes-jenkins/logs/example-job-name/123", 717 expRunPath: "kubernetes-jenkins/logs/example-job-name/123", 718 }, 719 { 720 name: "presubmit job in GCS with trailing /", 721 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/", 722 expRunPath: "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159", 723 }, 724 { 725 name: "presubmit job in GCS without trailing /", 726 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159", 727 expRunPath: "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159", 728 }, 729 { 730 name: "non-presubmit Prow job", 731 src: "prowjob/example-periodic-job/1111", 732 expRunPath: "chum-bucket/logs/example-periodic-job/1111", 733 }, 734 { 735 name: "Prow presubmit job with full path", 736 src: "prowjob/example-presubmit-job/2222", 737 expRunPath: "chum-bucket/pr-logs/pull/some-org_some-repo/42/example-presubmit-job/2222", 738 }, 739 { 740 name: "nonexistent job", 741 src: "prowjob/example-periodic-job/0000", 742 expError: true, 743 }, 744 { 745 name: "previously invalid key type is now valid", 746 src: "oh/my/glob/drama/bomb", 747 expRunPath: "my/glob/drama/bomb", 748 }, 749 { 750 name: "nonsense string errors", 751 src: "this is not useful", 752 expError: true, 753 }, 754 } 755 for _, tc := range testCases { 756 fakeGCSClient := fakeGCSServer.Client() 757 fakeOpener := io.NewGCSOpener(fakeGCSClient) 758 fca := config.Agent{} 759 fca.Set(&config.Config{ 760 ProwConfig: config.ProwConfig{ 761 Plank: config.Plank{ 762 JobURLPrefixConfig: map[string]string{"*": "http://magic/view/gcs/"}, 763 }, 764 }, 765 }) 766 sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false) 767 jobPath, err := sg.RunPath(tc.src) 768 if tc.expError && err == nil { 769 t.Errorf("test %q: RunPath(%q) expected error, got %q", tc.name, tc.src, jobPath) 770 continue 771 } 772 if !tc.expError && err != nil { 773 t.Errorf("test %q: RunPath(%q) returned unexpected error %v", tc.name, tc.src, err) 774 continue 775 } 776 if jobPath != tc.expRunPath { 777 t.Errorf("test %q: RunPath(%q) expected %q, got %q", tc.name, tc.src, tc.expRunPath, jobPath) 778 } 779 } 780 } 781 782 func TestRunToPR(t *testing.T) { 783 kc := fkc{ 784 prowapi.ProwJob{ 785 Spec: prowapi.ProwJobSpec{ 786 Type: prowapi.PeriodicJob, 787 Job: "example-periodic-job", 788 DecorationConfig: &prowapi.DecorationConfig{ 789 GCSConfiguration: &prowapi.GCSConfiguration{ 790 Bucket: "chum-bucket", 791 }, 792 }, 793 }, 794 Status: prowapi.ProwJobStatus{ 795 PodName: "flying-whales", 796 BuildID: "1111", 797 URL: "http://magic/view/gcs/chum-bucket/logs/example-periodic-job/1111", 798 }, 799 }, 800 prowapi.ProwJob{ 801 Spec: prowapi.ProwJobSpec{ 802 Type: prowapi.PresubmitJob, 803 Job: "example-presubmit-job", 804 DecorationConfig: &prowapi.DecorationConfig{ 805 GCSConfiguration: &prowapi.GCSConfiguration{ 806 Bucket: "chum-bucket", 807 }, 808 }, 809 Refs: &prowapi.Refs{ 810 Org: "some-org", 811 Repo: "some-repo", 812 Pulls: []prowapi.Pull{ 813 { 814 Number: 42, 815 }, 816 }, 817 }, 818 }, 819 Status: prowapi.ProwJobStatus{ 820 PodName: "flying-whales", 821 BuildID: "2222", 822 }, 823 }, 824 } 825 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 826 fakeJa.Start() 827 testCases := []struct { 828 name string 829 src string 830 expOrg string 831 expRepo string 832 expNumber int 833 expError bool 834 }{ 835 { 836 name: "presubmit job in GCS with trailing /", 837 src: "gcs/kubernetes-jenkins/pr-logs/pull/Katharine_test-infra/1234/example-job-name/314159/", 838 expOrg: "Katharine", 839 expRepo: "test-infra", 840 expNumber: 1234, 841 }, 842 { 843 name: "presubmit job in GCS without trailing /", 844 src: "gcs/kubernetes-jenkins/pr-logs/pull/Katharine_test-infra/1234/example-job-name/314159", 845 expOrg: "Katharine", 846 expRepo: "test-infra", 847 expNumber: 1234, 848 }, 849 { 850 name: "presubmit job in GCS without org name", 851 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/2345/example-job-name/314159", 852 expOrg: "kubernetes", 853 expRepo: "test-infra", 854 expNumber: 2345, 855 }, 856 { 857 name: "presubmit job in GCS without org or repo name", 858 src: "gcs/kubernetes-jenkins/pr-logs/pull/3456/example-job-name/314159", 859 expOrg: "kubernetes", 860 expRepo: "kubernetes", 861 expNumber: 3456, 862 }, 863 { 864 name: "Prow presubmit job", 865 src: "prowjob/example-presubmit-job/2222", 866 expOrg: "some-org", 867 expRepo: "some-repo", 868 expNumber: 42, 869 }, 870 { 871 name: "Prow periodic job errors", 872 src: "prowjob/example-periodic-job/1111", 873 expError: true, 874 }, 875 { 876 name: "GCS periodic job errors", 877 src: "gcs/kuberneretes-jenkins/logs/example-periodic-job/1111", 878 expError: true, 879 }, 880 { 881 name: "GCS job with non-numeric PR number errors", 882 src: "gcs/kubernetes-jenkins/pr-logs/pull/asdf/example-job-name/314159", 883 expError: true, 884 }, 885 { 886 name: "GCS PR job in directory errors", 887 src: "gcs/kubernetes-jenkins/pr-logs/directory/example-job-name/314159", 888 expError: true, 889 }, 890 { 891 name: "Bad GCS key errors", 892 src: "gcs/this is just nonsense", 893 expError: true, 894 }, 895 { 896 name: "Longer bad GCS key errors", 897 src: "gcs/kubernetes-jenkins/pr-logs", 898 expError: true, 899 }, 900 { 901 name: "Nonsense string errors", 902 src: "friendship is magic", 903 expError: true, 904 }, 905 } 906 for _, tc := range testCases { 907 fakeGCSClient := fakeGCSServer.Client() 908 fca := config.Agent{} 909 fca.Set(&config.Config{ 910 ProwConfig: config.ProwConfig{ 911 Plank: config.Plank{ 912 DefaultDecorationConfigs: config.DefaultDecorationMapToSliceTesting( 913 map[string]*prowapi.DecorationConfig{ 914 "*": { 915 GCSConfiguration: &prowapi.GCSConfiguration{ 916 Bucket: "kubernetes-jenkins", 917 DefaultOrg: "kubernetes", 918 DefaultRepo: "kubernetes", 919 PathStrategy: "legacy", 920 }, 921 }, 922 }), 923 }, 924 }, 925 }) 926 sg := New(context.Background(), fakeJa, fca.Config, io.NewGCSOpener(fakeGCSClient), false) 927 org, repo, num, err := sg.RunToPR(tc.src) 928 if tc.expError && err == nil { 929 t.Errorf("test %q: RunToPR(%q) expected error", tc.name, tc.src) 930 continue 931 } 932 if !tc.expError && err != nil { 933 t.Errorf("test %q: RunToPR(%q) returned unexpected error %v", tc.name, tc.src, err) 934 continue 935 } 936 if org != tc.expOrg || repo != tc.expRepo || num != tc.expNumber { 937 t.Errorf("test %q: RunToPR(%q) expected %s/%s#%d, got %s/%s#%d", tc.name, tc.src, tc.expOrg, tc.expRepo, tc.expNumber, org, repo, num) 938 } 939 } 940 } 941 942 func TestProwToGCS(t *testing.T) { 943 testCases := []struct { 944 name string 945 key string 946 configPrefix string 947 expectedPath string 948 expectError bool 949 }{ 950 { 951 name: "extraction from gubernator-like URL", 952 key: "gubernator-job/1111", 953 configPrefix: "https://gubernator.example.com/build/", 954 expectedPath: "some-bucket/gubernator-job/1111/", 955 expectError: false, 956 }, 957 { 958 name: "extraction from spyglass-like URL", 959 key: "spyglass-job/2222", 960 configPrefix: "https://prow.example.com/view/gcs/", 961 expectedPath: "some-bucket/spyglass-job/2222/", 962 expectError: false, 963 }, 964 { 965 name: "failed extraction from wrong URL", 966 key: "spyglass-job/1111", 967 configPrefix: "https://gubernator.example.com/build/", 968 expectedPath: "", 969 expectError: true, 970 }, 971 { 972 name: "prefix longer than URL", 973 key: "spyglass-job/2222", 974 configPrefix: strings.Repeat("!", 100), 975 expectError: true, 976 }, 977 } 978 979 for _, tc := range testCases { 980 kc := fkc{ 981 prowapi.ProwJob{ 982 Spec: prowapi.ProwJobSpec{ 983 Job: "gubernator-job", 984 }, 985 Status: prowapi.ProwJobStatus{ 986 URL: "https://gubernator.example.com/build/some-bucket/gubernator-job/1111/", 987 BuildID: "1111", 988 }, 989 }, 990 prowapi.ProwJob{ 991 Spec: prowapi.ProwJobSpec{ 992 Job: "spyglass-job", 993 }, 994 Status: prowapi.ProwJobStatus{ 995 URL: "https://prow.example.com/view/gcs/some-bucket/spyglass-job/2222/", 996 BuildID: "2222", 997 }, 998 }, 999 } 1000 1001 fakeGCSClient := fakeGCSServer.Client() 1002 fakeConfigAgent := fca{ 1003 c: config.Config{ 1004 ProwConfig: config.ProwConfig{ 1005 Plank: config.Plank{ 1006 JobURLPrefixConfig: map[string]string{"*": tc.configPrefix}, 1007 }, 1008 }, 1009 }, 1010 } 1011 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config) 1012 fakeJa.Start() 1013 sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false) 1014 1015 _, p, err := sg.prowToGCS(tc.key) 1016 if err != nil && !tc.expectError { 1017 t.Errorf("test %q: unexpected error: %v", tc.key, err) 1018 continue 1019 } 1020 if err == nil && tc.expectError { 1021 t.Errorf("test %q: expected an error but instead got success and path '%s'", tc.key, p) 1022 continue 1023 } 1024 if p != tc.expectedPath { 1025 t.Errorf("test %q: expected '%s' but got '%s'", tc.key, tc.expectedPath, p) 1026 } 1027 } 1028 } 1029 1030 func TestGCSPathRoundTrip(t *testing.T) { 1031 testCases := []struct { 1032 name string 1033 pathStrategy string 1034 defaultOrg string 1035 defaultRepo string 1036 org string 1037 repo string 1038 }{ 1039 { 1040 name: "simple explicit path", 1041 pathStrategy: "explicit", 1042 org: "test-org", 1043 repo: "test-repo", 1044 }, 1045 { 1046 name: "explicit path with underscores", 1047 pathStrategy: "explicit", 1048 org: "test-org", 1049 repo: "underscore_repo", 1050 }, 1051 { 1052 name: "'single' path with default repo", 1053 pathStrategy: "single", 1054 defaultOrg: "default-org", 1055 defaultRepo: "default-repo", 1056 org: "default-org", 1057 repo: "default-repo", 1058 }, 1059 { 1060 name: "'single' path with non-default repo", 1061 pathStrategy: "single", 1062 defaultOrg: "default-org", 1063 defaultRepo: "default-repo", 1064 org: "default-org", 1065 repo: "random-repo", 1066 }, 1067 { 1068 name: "'single' path with non-default org but default repo", 1069 pathStrategy: "single", 1070 defaultOrg: "default-org", 1071 defaultRepo: "default-repo", 1072 org: "random-org", 1073 repo: "default-repo", 1074 }, 1075 { 1076 name: "'single' path with non-default org and repo", 1077 pathStrategy: "single", 1078 defaultOrg: "default-org", 1079 defaultRepo: "default-repo", 1080 org: "random-org", 1081 repo: "random-repo", 1082 }, 1083 { 1084 name: "legacy path with default repo", 1085 pathStrategy: "legacy", 1086 defaultOrg: "default-org", 1087 defaultRepo: "default-repo", 1088 org: "default-org", 1089 repo: "default-repo", 1090 }, 1091 { 1092 name: "legacy path with non-default repo", 1093 pathStrategy: "legacy", 1094 defaultOrg: "default-org", 1095 defaultRepo: "default-repo", 1096 org: "default-org", 1097 repo: "random-repo", 1098 }, 1099 { 1100 name: "legacy path with non-default org but default repo", 1101 pathStrategy: "legacy", 1102 defaultOrg: "default-org", 1103 defaultRepo: "default-repo", 1104 org: "random-org", 1105 repo: "default-repo", 1106 }, 1107 { 1108 name: "legacy path with non-default org and repo", 1109 pathStrategy: "legacy", 1110 defaultOrg: "default-org", 1111 defaultRepo: "default-repo", 1112 org: "random-org", 1113 repo: "random-repo", 1114 }, 1115 { 1116 name: "legacy path with non-default org and repo with underscores", 1117 pathStrategy: "legacy", 1118 defaultOrg: "default-org", 1119 defaultRepo: "default-repo", 1120 org: "random-org", 1121 repo: "underscore_repo", 1122 }, 1123 } 1124 1125 for _, tc := range testCases { 1126 kc := fkc{} 1127 fakeConfigAgent := fca{ 1128 c: config.Config{ 1129 ProwConfig: config.ProwConfig{ 1130 Plank: config.Plank{ 1131 DefaultDecorationConfigs: config.DefaultDecorationMapToSliceTesting( 1132 map[string]*prowapi.DecorationConfig{ 1133 "*": { 1134 GCSConfiguration: &prowapi.GCSConfiguration{ 1135 DefaultOrg: tc.defaultOrg, 1136 DefaultRepo: tc.defaultRepo, 1137 }, 1138 }, 1139 }), 1140 }, 1141 }, 1142 }, 1143 } 1144 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config) 1145 fakeJa.Start() 1146 1147 fakeGCSClient := fakeGCSServer.Client() 1148 1149 sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false) 1150 gcspath, _, _ := gcsupload.PathsForJob( 1151 &prowapi.GCSConfiguration{Bucket: "test-bucket", PathStrategy: tc.pathStrategy}, 1152 &downwardapi.JobSpec{ 1153 Job: "test-job", 1154 BuildID: "1234", 1155 Type: prowapi.PresubmitJob, 1156 Refs: &prowapi.Refs{ 1157 Org: tc.org, Repo: tc.repo, 1158 Pulls: []prowapi.Pull{{Number: 42}}, 1159 }, 1160 }, "") 1161 fmt.Println(gcspath) 1162 org, repo, prnum, err := sg.RunToPR("gcs/test-bucket/" + gcspath) 1163 if err != nil { 1164 t.Errorf("unexpected error: %v", err) 1165 continue 1166 } 1167 if org != tc.org || repo != tc.repo || prnum != 42 { 1168 t.Errorf("expected %s/%s#42, got %s/%s#%d", tc.org, tc.repo, org, repo, prnum) 1169 } 1170 } 1171 } 1172 1173 func TestTestGridLink(t *testing.T) { 1174 testCases := []struct { 1175 name string 1176 src string 1177 expQuery string 1178 expError bool 1179 }{ 1180 { 1181 name: "non-presubmit job in GCS with trailing /", 1182 src: "gcs/kubernetes-jenkins/logs/periodic-job/123/", 1183 expQuery: "some-dashboard#periodic", 1184 }, 1185 { 1186 name: "non-presubmit job in GCS without trailing /", 1187 src: "gcs/kubernetes-jenkins/logs/periodic-job/123", 1188 expQuery: "some-dashboard#periodic", 1189 }, 1190 { 1191 name: "presubmit job in GCS", 1192 src: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/presubmit-job/314159/", 1193 expQuery: "some-dashboard#presubmit", 1194 }, 1195 { 1196 name: "non-presubmit Prow job", 1197 src: "prowjob/periodic-job/1111", 1198 expQuery: "some-dashboard#periodic", 1199 }, 1200 { 1201 name: "presubmit Prow job", 1202 src: "prowjob/presubmit-job/2222", 1203 expQuery: "some-dashboard#presubmit", 1204 }, 1205 { 1206 name: "nonexistent job", 1207 src: "prowjob/nonexistent-job/0000", 1208 expError: true, 1209 }, 1210 { 1211 name: "invalid key type", 1212 src: "oh/my/glob/drama/bomb", 1213 expError: true, 1214 }, 1215 { 1216 name: "nonsense string errors", 1217 src: "this is not useful", 1218 expError: true, 1219 }, 1220 } 1221 1222 kc := fkc{} 1223 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config) 1224 fakeJa.Start() 1225 1226 tg := TestGrid{c: &tgconf.Configuration{ 1227 Dashboards: []*tgconf.Dashboard{ 1228 { 1229 Name: "some-dashboard", 1230 DashboardTab: []*tgconf.DashboardTab{ 1231 { 1232 Name: "periodic", 1233 TestGroupName: "periodic-job", 1234 }, 1235 { 1236 Name: "presubmit", 1237 TestGroupName: "presubmit-job", 1238 }, 1239 { 1240 Name: "some-other-job", 1241 TestGroupName: "some-other-job", 1242 }, 1243 }, 1244 }, 1245 }, 1246 }} 1247 1248 for _, tc := range testCases { 1249 fakeGCSClient := fakeGCSServer.Client() 1250 fca := config.Agent{} 1251 fca.Set(&config.Config{ 1252 ProwConfig: config.ProwConfig{ 1253 Deck: config.Deck{ 1254 Spyglass: config.Spyglass{ 1255 TestGridRoot: "https://testgrid.com/", 1256 }, 1257 }, 1258 }, 1259 }) 1260 sg := New(context.Background(), fakeJa, fca.Config, io.NewGCSOpener(fakeGCSClient), false) 1261 sg.testgrid = &tg 1262 link, err := sg.TestGridLink(tc.src) 1263 if tc.expError { 1264 if err == nil { 1265 t.Errorf("test %q: TestGridLink(%q) expected error, got %q", tc.name, tc.src, link) 1266 } 1267 continue 1268 } 1269 if err != nil { 1270 t.Errorf("test %q: TestGridLink(%q) returned unexpected error %v", tc.name, tc.src, err) 1271 continue 1272 } 1273 if link != "https://testgrid.com/"+tc.expQuery { 1274 t.Errorf("test %q: TestGridLink(%q) expected %q, got %q", tc.name, tc.src, "https://testgrid.com/"+tc.expQuery, link) 1275 } 1276 } 1277 } 1278 1279 func TestFetchArtifactsPodLog(t *testing.T) { 1280 kc := fkc{ 1281 prowapi.ProwJob{ 1282 Spec: prowapi.ProwJobSpec{ 1283 Agent: prowapi.KubernetesAgent, 1284 Job: "job", 1285 }, 1286 Status: prowapi.ProwJobStatus{ 1287 PodName: "wowowow", 1288 BuildID: "123", 1289 URL: "https://gubernator.example.com/build/job/123", 1290 }, 1291 }, 1292 prowapi.ProwJob{ 1293 Spec: prowapi.ProwJobSpec{ 1294 Agent: prowapi.KubernetesAgent, 1295 Job: "multi-container-one-log", 1296 }, 1297 Status: prowapi.ProwJobStatus{ 1298 PodName: "wowowow", 1299 BuildID: "123", 1300 URL: "https://gubernator.example.com/build/multi-container/123", 1301 }, 1302 }, 1303 } 1304 fakeConfigAgent := fca{ 1305 c: config.Config{ 1306 ProwConfig: config.ProwConfig{ 1307 Deck: config.Deck{ 1308 AllKnownStorageBuckets: sets.New[string]("job", "kubernetes-jenkins", "multi-container-one-log"), 1309 }, 1310 Plank: config.Plank{ 1311 JobURLPrefixConfig: map[string]string{"*": "https://gubernator.example.com/build/"}, 1312 }, 1313 }, 1314 }, 1315 } 1316 fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config) 1317 fakeJa.Start() 1318 1319 fakeGCSClient := fakeGCSServer.Client() 1320 1321 sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false) 1322 testKeys := []string{ 1323 "prowjob/job/123", 1324 "gcs/kubernetes-jenkins/logs/job/123/", 1325 "gcs/kubernetes-jenkins/logs/job/123", 1326 } 1327 1328 for _, key := range testKeys { 1329 result, err := sg.FetchArtifacts(context.Background(), key, "", 500e6, []string{"build-log.txt"}) 1330 if err != nil { 1331 t.Errorf("Unexpected error grabbing pod log for %s: %v", key, err) 1332 continue 1333 } 1334 if len(result) != 1 { 1335 t.Errorf("Expected 1 artifact for %s, got %d", key, len(result)) 1336 continue 1337 } 1338 content, err := result[0].ReadAll() 1339 if err != nil { 1340 t.Errorf("Unexpected error reading pod log for %s: %v", key, err) 1341 continue 1342 } 1343 if string(content) != fmt.Sprintf("clusterA.%s", kube.TestContainerName) { 1344 t.Errorf("Bad pod log content for %s: %q (expected 'clusterA')", key, content) 1345 } 1346 } 1347 1348 multiContainerOneLogKey := "gcs/multi-container-one-log/logs/job/123" 1349 1350 testKeys = append(testKeys, multiContainerOneLogKey) 1351 1352 for _, key := range testKeys { 1353 containers := []string{"test-1", "test-2"} 1354 result, err := sg.FetchArtifacts(context.Background(), key, "", 500e6, []string{fmt.Sprintf("%s-%s", containers[0], singleLogName), fmt.Sprintf("%s-%s", containers[1], singleLogName)}) 1355 if err != nil { 1356 t.Errorf("Unexpected error grabbing pod log for %s: %v", key, err) 1357 continue 1358 } 1359 for i, art := range result { 1360 content, err := art.ReadAll() 1361 if err != nil { 1362 t.Errorf("Unexpected error reading pod log for %s: %v", key, err) 1363 continue 1364 } 1365 expected := fmt.Sprintf("clusterA.%s", containers[i]) 1366 if key == multiContainerOneLogKey && containers[i] == "test-1" { 1367 expected = "this log exists in gcs!" 1368 } 1369 if string(content) != expected { 1370 t.Errorf("Bad pod log content for %s: %q (expected '%s')", key, content, expected) 1371 } 1372 } 1373 } 1374 } 1375 1376 func TestKeyToJob(t *testing.T) { 1377 testCases := []struct { 1378 name string 1379 path string 1380 jobName string 1381 buildID string 1382 expectErr bool 1383 }{ 1384 { 1385 name: "GCS periodic path with trailing slash", 1386 path: "gcs/kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-1-14/40/", 1387 jobName: "periodic-kubernetes-bazel-test-1-14", 1388 buildID: "40", 1389 }, 1390 { 1391 name: "GCS periodic path without trailing slash", 1392 path: "gcs/kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-1-14/40", 1393 jobName: "periodic-kubernetes-bazel-test-1-14", 1394 buildID: "40", 1395 }, 1396 { 1397 name: "GCS PR path with trailing slash", 1398 path: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/11573/pull-test-infra-bazel/25366/", 1399 jobName: "pull-test-infra-bazel", 1400 buildID: "25366", 1401 }, 1402 { 1403 name: "GCS PR path without trailing slash", 1404 path: "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/11573/pull-test-infra-bazel/25366", 1405 jobName: "pull-test-infra-bazel", 1406 buildID: "25366", 1407 }, 1408 { 1409 name: "Prowjob path with trailing slash", 1410 path: "prowjob/pull-test-infra-bazel/25366/", 1411 jobName: "pull-test-infra-bazel", 1412 buildID: "25366", 1413 }, 1414 { 1415 name: "Prowjob path without trailing slash", 1416 path: "prowjob/pull-test-infra-bazel/25366", 1417 jobName: "pull-test-infra-bazel", 1418 buildID: "25366", 1419 }, 1420 { 1421 name: "Path with only one component", 1422 path: "nope", 1423 expectErr: true, 1424 }, 1425 } 1426 1427 for _, tc := range testCases { 1428 jobName, buildID, err := common.KeyToJob(tc.path) 1429 if err != nil { 1430 if !tc.expectErr { 1431 t.Errorf("%s: unexpected error %v", tc.name, err) 1432 } 1433 continue 1434 } 1435 if tc.expectErr { 1436 t.Errorf("%s: expected an error, but got result %s #%s", tc.name, jobName, buildID) 1437 continue 1438 } 1439 if jobName != tc.jobName { 1440 t.Errorf("%s: expected job name %q, but got %q", tc.name, tc.jobName, jobName) 1441 continue 1442 } 1443 if buildID != tc.buildID { 1444 t.Errorf("%s: expected build ID %q, but got %q", tc.name, tc.buildID, buildID) 1445 } 1446 } 1447 } 1448 1449 func TestResolveSymlink(t *testing.T) { 1450 testCases := []struct { 1451 name string 1452 path string 1453 result string 1454 expectErr bool 1455 }{ 1456 { 1457 name: "symlink without trailing slash is resolved", 1458 path: "gcs/test-bucket/logs/symlink-party/123", 1459 result: "gs/test-bucket/logs/the-actual-place/123", 1460 }, 1461 { 1462 name: "symlink with trailing slash is resolved", 1463 path: "gcs/test-bucket/logs/symlink-party/123/", 1464 result: "gs/test-bucket/logs/the-actual-place/123", 1465 }, 1466 { 1467 name: "non-symlink without trailing slash is unchanged", 1468 path: "gcs/test-bucket/better-logs/42", 1469 result: "gs/test-bucket/better-logs/42", 1470 }, 1471 { 1472 name: "non-symlink with trailing slash drops the slash", 1473 path: "gcs/test-bucket/better-logs/42/", 1474 result: "gs/test-bucket/better-logs/42", 1475 }, 1476 { 1477 name: "non-symlink with aliased bucket is replaced", 1478 path: "gcs/alias/better-logs/42", 1479 result: "gs/test-bucket/better-logs/42", 1480 }, 1481 { 1482 name: "prowjob without trailing slash is unchanged", 1483 path: "prowjob/better-logs/42", 1484 result: "prowjob/better-logs/42", 1485 }, 1486 { 1487 name: "prowjob with trailing slash drops the slash", 1488 path: "prowjob/better-logs/42/", 1489 result: "prowjob/better-logs/42", 1490 }, 1491 { 1492 name: "unknown key type is an error", 1493 path: "wtf/what-is-this/send-help", 1494 expectErr: true, 1495 }, 1496 { 1497 name: "insufficient path components are an error", 1498 path: "gcs/hi", 1499 expectErr: true, 1500 }, 1501 } 1502 1503 for _, tc := range testCases { 1504 fakeConfigAgent := fca{ 1505 c: config.Config{ 1506 ProwConfig: config.ProwConfig{ 1507 Deck: config.Deck{ 1508 Spyglass: config.Spyglass{ 1509 BucketAliases: map[string]string{"alias": "test-bucket"}}, 1510 }, 1511 }, 1512 }, 1513 } 1514 //fakeConfigAgent.Config().Deck.Spyglass.BucketAliases = map[string]string{"alias": "test-bucket"} 1515 fakeJa = jobs.NewJobAgent(context.Background(), fkc{}, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config) 1516 fakeJa.Start() 1517 1518 fakeGCSClient := fakeGCSServer.Client() 1519 1520 sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false) 1521 1522 result, err := sg.ResolveSymlink(tc.path) 1523 if err != nil { 1524 if !tc.expectErr { 1525 t.Errorf("test %q: unexpected error: %v", tc.name, err) 1526 } 1527 continue 1528 } 1529 if tc.expectErr { 1530 t.Errorf("test %q: expected an error, but got result %q", tc.name, result) 1531 continue 1532 } 1533 if result != tc.result { 1534 t.Errorf("test %q: expected %q, but got %q", tc.name, tc.result, result) 1535 continue 1536 } 1537 } 1538 } 1539 1540 func TestExtraLinks(t *testing.T) { 1541 testCases := []struct { 1542 name string 1543 content string 1544 links []ExtraLink 1545 expectErr bool 1546 }{ 1547 { 1548 name: "does nothing without error given no started.json", 1549 links: nil, 1550 }, 1551 { 1552 name: "errors given a malformed started.json", 1553 content: "this isn't json", 1554 expectErr: true, 1555 }, 1556 { 1557 name: "does nothing given metadata with no links", 1558 content: `{"metadata": {"somethingThatIsntLinks": 23}}`, 1559 links: nil, 1560 }, 1561 { 1562 name: "returns well-formed links", 1563 content: `{"metadata": {"links": {"ResultStore": {"url": "http://resultstore", "description": "The thing that isn't spyglass"}}}}`, 1564 links: []ExtraLink{{Name: "ResultStore", URL: "http://resultstore", Description: "The thing that isn't spyglass"}}, 1565 }, 1566 { 1567 name: "returns links without a description", 1568 content: `{"metadata": {"links": {"ResultStore": {"url": "http://resultstore"}}}}`, 1569 links: []ExtraLink{{Name: "ResultStore", URL: "http://resultstore"}}, 1570 }, 1571 { 1572 name: "skips links without a URL", 1573 content: `{"metadata": {"links": {"No Link": {"description": "bad link"}, "ResultStore": {"url": "http://resultstore"}}}}`, 1574 links: []ExtraLink{{Name: "ResultStore", URL: "http://resultstore"}}, 1575 }, 1576 { 1577 name: "skips links without a name", 1578 content: `{"metadata": {"links": {"": {"url": "http://resultstore"}}}}`, 1579 links: []ExtraLink{}, 1580 }, 1581 { 1582 name: "returns no links when links is empty", 1583 content: `{"metadata": {"links": {}}}`, 1584 links: []ExtraLink{}, 1585 }, 1586 { 1587 name: "returns multiple links", 1588 content: `{"metadata": {"links": {"A": {"url": "http://a", "description": "A!"}, "B": {"url": "http://b"}}}}`, 1589 links: []ExtraLink{{Name: "A", URL: "http://a", Description: "A!"}, {Name: "B", URL: "http://b"}}, 1590 }, 1591 } 1592 for _, tc := range testCases { 1593 t.Run(tc.name, func(t *testing.T) { 1594 var objects []fakestorage.Object 1595 if tc.content != "" { 1596 objects = []fakestorage.Object{ 1597 { 1598 BucketName: "test-bucket", 1599 Name: "logs/some-job/42/started.json", 1600 Content: []byte(tc.content), 1601 }, 1602 } 1603 } 1604 gcsServer := fakestorage.NewServer(objects) 1605 defer gcsServer.Stop() 1606 1607 gcsClient := gcsServer.Client() 1608 fakeConfigAgent := fca{ 1609 c: config.Config{ 1610 ProwConfig: config.ProwConfig{ 1611 Deck: config.Deck{ 1612 AllKnownStorageBuckets: sets.New[string]("test-bucket"), 1613 }, 1614 }, 1615 }, 1616 } 1617 fakeJa = jobs.NewJobAgent(context.Background(), fkc{}, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config) 1618 fakeJa.Start() 1619 sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(gcsClient), false) 1620 1621 result, err := sg.ExtraLinks(context.Background(), "gcs/test-bucket/logs/some-job/42") 1622 if err != nil { 1623 if !tc.expectErr { 1624 t.Fatalf("unexpected error: %v", err) 1625 } 1626 return 1627 } 1628 sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name }) 1629 sort.Slice(tc.links, func(i, j int) bool { return tc.links[i].Name < tc.links[j].Name }) 1630 if !reflect.DeepEqual(result, tc.links) { 1631 t.Fatalf("Expected links %#v, got %#v", tc.links, result) 1632 } 1633 }) 1634 } 1635 }