sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/spyglass/spyglass_test.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package spyglass
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"os"
    24  	"reflect"
    25  	"sort"
    26  	"strings"
    27  	"testing"
    28  
    29  	"k8s.io/apimachinery/pkg/util/sets"
    30  
    31  	coreapi "k8s.io/api/core/v1"
    32  	"sigs.k8s.io/prow/pkg/gcsupload"
    33  	"sigs.k8s.io/prow/pkg/pod-utils/downwardapi"
    34  
    35  	"github.com/fsouza/fake-gcs-server/fakestorage"
    36  	"github.com/sirupsen/logrus"
    37  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    38  
    39  	tgconf "github.com/GoogleCloudPlatform/testgrid/pb/config"
    40  
    41  	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
    42  	prowapi "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
    43  	"sigs.k8s.io/prow/pkg/config"
    44  	"sigs.k8s.io/prow/pkg/deck/jobs"
    45  	"sigs.k8s.io/prow/pkg/io"
    46  	"sigs.k8s.io/prow/pkg/kube"
    47  	"sigs.k8s.io/prow/pkg/spyglass/api"
    48  	"sigs.k8s.io/prow/pkg/spyglass/lenses"
    49  	"sigs.k8s.io/prow/pkg/spyglass/lenses/common"
    50  )
    51  
    52  var (
    53  	fakeJa        *jobs.JobAgent
    54  	fakeGCSServer *fakestorage.Server
    55  )
    56  
    57  type fkc []prowapi.ProwJob
    58  
    59  func (f fkc) List(ctx context.Context, pjs *prowapi.ProwJobList, _ ...ctrlruntimeclient.ListOption) error {
    60  	pjs.Items = f
    61  	return nil
    62  }
    63  
    64  type fpkc string
    65  
    66  func (f fpkc) GetLogs(name, container string) ([]byte, error) {
    67  	if name == "wowowow" || name == "powowow" {
    68  		return []byte(fmt.Sprintf("%s.%s", f, container)), nil
    69  	}
    70  	return nil, fmt.Errorf("pod not found: %s", name)
    71  }
    72  
    73  type fca struct {
    74  	c config.Config
    75  }
    76  
    77  func (ca fca) Config() *config.Config {
    78  	return &ca.c
    79  }
    80  
    81  func TestMain(m *testing.M) {
    82  	var longLog string
    83  	for i := 0; i < 300; i++ {
    84  		longLog += "here a log\nthere a log\neverywhere a log log\n"
    85  	}
    86  	fakeGCSServer = fakestorage.NewServer([]fakestorage.Object{
    87  		{
    88  			BucketName: "test-bucket",
    89  			Name:       "logs/example-ci-run/403/build-log.txt",
    90  			Content:    []byte("Oh wow\nlogs\nthis is\ncrazy"),
    91  			Metadata: map[string]string{
    92  				"foo": "bar",
    93  			},
    94  		},
    95  		{
    96  			BucketName: "test-bucket",
    97  			Name:       "logs/example-ci-run/403/long-log.txt",
    98  			Content:    []byte(longLog),
    99  		},
   100  		{
   101  			BucketName: "test-bucket",
   102  			Name:       "logs/example-ci-run/403/junit_01.xml",
   103  			Content: []byte(`<testsuite tests="1017" failures="1017" time="0.016981535">
   104  <testcase name="BeforeSuite" classname="Kubernetes e2e suite" time="0.006343795">
   105  <failure type="Failure">
   106  test/e2e/e2e.go:137 BeforeSuite on Node 1 failed test/e2e/e2e.go:137
   107  </failure>
   108  </testcase>
   109  </testsuite>`),
   110  		},
   111  		{
   112  			BucketName: "test-bucket",
   113  			Name:       "logs/example-ci-run/403/started.json",
   114  			Content: []byte(`{
   115  						  "node": "gke-prow-default-pool-3c8994a8-qfhg",
   116  						  "repo-version": "v1.12.0-alpha.0.985+e6f64d0a79243c",
   117  						  "timestamp": 1528742858,
   118  						  "repos": {
   119  						    "k8s.io/kubernetes": "master",
   120  						    "k8s.io/release": "master"
   121  						  },
   122  						  "version": "v1.12.0-alpha.0.985+e6f64d0a79243c",
   123  						  "metadata": {
   124  						    "pod": "cbc53d8e-6da7-11e8-a4ff-0a580a6c0269"
   125  						  }
   126  						}`),
   127  		},
   128  		{
   129  			BucketName: "test-bucket",
   130  			Name:       "logs/example-ci-run/403/finished.json",
   131  			Content: []byte(`{
   132  						  "timestamp": 1528742943,
   133  						  "version": "v1.12.0-alpha.0.985+e6f64d0a79243c",
   134  						  "result": "SUCCESS",
   135  						  "passed": true,
   136  						  "job-version": "v1.12.0-alpha.0.985+e6f64d0a79243c",
   137  						  "metadata": {
   138  						    "repo": "k8s.io/kubernetes",
   139  						    "repos": {
   140  						      "k8s.io/kubernetes": "master",
   141  						      "k8s.io/release": "master"
   142  						    },
   143  						    "infra-commit": "260081852",
   144  						    "pod": "cbc53d8e-6da7-11e8-a4ff-0a580a6c0269",
   145  						    "repo-commit": "e6f64d0a79243c834babda494151fc5d66582240"
   146  						  },
   147  						},`),
   148  		},
   149  		{
   150  			BucketName: "test-bucket",
   151  			Name:       "logs/symlink-party/123.txt",
   152  			Content:    []byte(`gs://test-bucket/logs/the-actual-place/123`),
   153  		},
   154  		{
   155  			BucketName: "multi-container-one-log",
   156  			Name:       "logs/job/123/test-1-build-log.txt",
   157  			Content:    []byte("this log exists in gcs!"),
   158  		},
   159  	})
   160  	defer fakeGCSServer.Stop()
   161  	kc := fkc{
   162  		prowapi.ProwJob{
   163  			Spec: prowapi.ProwJobSpec{
   164  				Agent: prowapi.KubernetesAgent,
   165  				Job:   "job",
   166  			},
   167  			Status: prowapi.ProwJobStatus{
   168  				PodName: "wowowow",
   169  				BuildID: "123",
   170  			},
   171  		},
   172  		prowapi.ProwJob{
   173  			Spec: prowapi.ProwJobSpec{
   174  				Agent:   prowapi.KubernetesAgent,
   175  				Job:     "jib",
   176  				Cluster: "trusted",
   177  			},
   178  			Status: prowapi.ProwJobStatus{
   179  				PodName: "powowow",
   180  				BuildID: "123",
   181  			},
   182  		},
   183  		prowapi.ProwJob{
   184  			Spec: prowapi.ProwJobSpec{
   185  				Agent: prowapi.KubernetesAgent,
   186  				Job:   "example-ci-run",
   187  				PodSpec: &coreapi.PodSpec{
   188  					Containers: []coreapi.Container{
   189  						{
   190  							Image: "tester",
   191  						},
   192  					},
   193  				},
   194  			},
   195  			Status: prowapi.ProwJobStatus{
   196  				PodName: "wowowow",
   197  				BuildID: "404",
   198  			},
   199  		},
   200  		prowapi.ProwJob{
   201  			Spec: prowapi.ProwJobSpec{
   202  				Agent: prowapi.KubernetesAgent,
   203  				Job:   "multiple-container-job",
   204  				PodSpec: &coreapi.PodSpec{
   205  					Containers: []coreapi.Container{
   206  						{
   207  							Name: "test-1",
   208  						},
   209  						{
   210  							Name: "test-2",
   211  						},
   212  					},
   213  				},
   214  			},
   215  			Status: prowapi.ProwJobStatus{
   216  				PodName: "wowowow",
   217  				BuildID: "123",
   218  			},
   219  		},
   220  	}
   221  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
   222  	fakeJa.Start()
   223  	os.Exit(m.Run())
   224  }
   225  
   226  type dumpLens struct{}
   227  
   228  func (dumpLens) Config() lenses.LensConfig {
   229  	return lenses.LensConfig{
   230  		Name:  "dump",
   231  		Title: "Dump View",
   232  	}
   233  }
   234  
   235  func (dumpLens) Header(artifacts []api.Artifact, resourceDir string, config json.RawMessage, spyglassConfig config.Spyglass) string {
   236  	return ""
   237  }
   238  
   239  func (dumpLens) Body(artifacts []api.Artifact, resourceDir string, data string, config json.RawMessage, spyglassConfig config.Spyglass) string {
   240  	var view []byte
   241  	for _, a := range artifacts {
   242  		data, err := a.ReadAll()
   243  		if err != nil {
   244  			logrus.WithError(err).Error("Error reading artifact")
   245  			continue
   246  		}
   247  		view = append(view, data...)
   248  	}
   249  	return string(view)
   250  }
   251  
   252  func (dumpLens) Callback(artifacts []api.Artifact, resourceDir string, data string, config json.RawMessage, spyglassConfig config.Spyglass) string {
   253  	return ""
   254  }
   255  
   256  func TestViews(t *testing.T) {
   257  	fakeGCSClient := fakeGCSServer.Client()
   258  	testCases := []struct {
   259  		name               string
   260  		registeredViewers  []lenses.Lens
   261  		lenses             []int
   262  		expectedLensTitles []string
   263  	}{
   264  		{
   265  			name:               "Spyglass basic test",
   266  			registeredViewers:  []lenses.Lens{dumpLens{}},
   267  			lenses:             []int{0},
   268  			expectedLensTitles: []string{"Dump View"},
   269  		},
   270  	}
   271  
   272  	for _, tc := range testCases {
   273  		t.Run(tc.name, func(t *testing.T) {
   274  			for _, l := range tc.registeredViewers {
   275  				lenses.RegisterLens(l)
   276  			}
   277  			c := fca{
   278  				c: config.Config{
   279  					ProwConfig: config.ProwConfig{
   280  						Deck: config.Deck{
   281  							Spyglass: config.Spyglass{
   282  								Lenses: []config.LensFileConfig{
   283  									{
   284  										Lens: config.LensConfig{
   285  											Name: "dump",
   286  										},
   287  									},
   288  								},
   289  							},
   290  						},
   291  					},
   292  				},
   293  			}
   294  			sg := New(context.Background(), fakeJa, c.Config, io.NewGCSOpener(fakeGCSClient), false)
   295  			_, ls := sg.Lenses(tc.lenses)
   296  			for _, l := range ls {
   297  				var found bool
   298  				for _, title := range tc.expectedLensTitles {
   299  					if title == l.Config().Title {
   300  						found = true
   301  					}
   302  				}
   303  				if !found {
   304  					t.Errorf("lens title %s not found in expected titles.", l.Config().Title)
   305  				}
   306  			}
   307  			for _, title := range tc.expectedLensTitles {
   308  				var found bool
   309  				for _, l := range ls {
   310  					if title == l.Config().Title {
   311  						found = true
   312  					}
   313  				}
   314  				if !found {
   315  					t.Errorf("expected title %s not found in produced lenses.", title)
   316  				}
   317  			}
   318  		})
   319  	}
   320  }
   321  
   322  func TestSplitSrc(t *testing.T) {
   323  	testCases := []struct {
   324  		name       string
   325  		src        string
   326  		expKeyType string
   327  		expKey     string
   328  		expError   bool
   329  	}{
   330  		{
   331  			name:     "empty string",
   332  			src:      "",
   333  			expError: true,
   334  		},
   335  		{
   336  			name:     "missing key",
   337  			src:      "gcs",
   338  			expError: true,
   339  		},
   340  		{
   341  			name:       "prow key",
   342  			src:        "prowjob/example-job-name/123456",
   343  			expKeyType: "prowjob",
   344  			expKey:     "example-job-name/123456",
   345  		},
   346  		{
   347  			name:       "gcs key",
   348  			src:        "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/",
   349  			expKeyType: "gcs",
   350  			expKey:     "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/",
   351  		},
   352  	}
   353  	for _, tc := range testCases {
   354  		keyType, key, err := splitSrc(tc.src)
   355  		if tc.expError && err == nil {
   356  			t.Errorf("test %q expected error", tc.name)
   357  		}
   358  		if !tc.expError && err != nil {
   359  			t.Errorf("test %q encountered unexpected error: %v", tc.name, err)
   360  		}
   361  		if keyType != tc.expKeyType || key != tc.expKey {
   362  			t.Errorf("test %q: splitting src %q: Expected <%q, %q>, got <%q, %q>",
   363  				tc.name, tc.src, tc.expKeyType, tc.expKey, keyType, key)
   364  		}
   365  	}
   366  }
   367  
   368  func TestJobPath(t *testing.T) {
   369  	kc := fkc{
   370  		prowapi.ProwJob{
   371  			Spec: prowapi.ProwJobSpec{
   372  				Type: prowapi.PeriodicJob,
   373  				Job:  "example-periodic-job",
   374  				DecorationConfig: &prowapi.DecorationConfig{
   375  					GCSConfiguration: &prowapi.GCSConfiguration{
   376  						Bucket: "chum-bucket",
   377  					},
   378  				},
   379  			},
   380  			Status: prowapi.ProwJobStatus{
   381  				PodName: "flying-whales",
   382  				BuildID: "1111",
   383  			},
   384  		},
   385  		prowapi.ProwJob{
   386  			Spec: prowapi.ProwJobSpec{
   387  				Type: prowapi.PresubmitJob,
   388  				Job:  "example-presubmit-job",
   389  				DecorationConfig: &prowapi.DecorationConfig{
   390  					GCSConfiguration: &prowapi.GCSConfiguration{
   391  						Bucket: "chum-bucket",
   392  					},
   393  				},
   394  			},
   395  			Status: prowapi.ProwJobStatus{
   396  				PodName: "flying-whales",
   397  				BuildID: "2222",
   398  			},
   399  		},
   400  		prowapi.ProwJob{
   401  			Spec: prowapi.ProwJobSpec{
   402  				Type: prowapi.PresubmitJob,
   403  				Job:  "undecorated-job",
   404  			},
   405  			Status: prowapi.ProwJobStatus{
   406  				PodName: "flying-whales",
   407  				BuildID: "1",
   408  			},
   409  		},
   410  		prowapi.ProwJob{
   411  			Spec: prowapi.ProwJobSpec{
   412  				Type:             prowapi.PresubmitJob,
   413  				Job:              "missing-gcs-job",
   414  				DecorationConfig: &prowapi.DecorationConfig{},
   415  			},
   416  			Status: prowapi.ProwJobStatus{
   417  				PodName: "flying-whales",
   418  				BuildID: "1",
   419  			},
   420  		},
   421  	}
   422  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
   423  	fakeJa.Start()
   424  	testCases := []struct {
   425  		name       string
   426  		src        string
   427  		expJobPath string
   428  		expError   bool
   429  	}{
   430  		{
   431  			name:       "non-presubmit job in GCS with trailing /",
   432  			src:        "gcs/kubernetes-jenkins/logs/example-job-name/123/",
   433  			expJobPath: "gs/kubernetes-jenkins/logs/example-job-name",
   434  		},
   435  		{
   436  			name:       "non-presubmit job in GCS without trailing /",
   437  			src:        "gcs/kubernetes-jenkins/logs/example-job-name/123",
   438  			expJobPath: "gs/kubernetes-jenkins/logs/example-job-name",
   439  		},
   440  		{
   441  			name:       "presubmit job in GCS with trailing /",
   442  			src:        "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/",
   443  			expJobPath: "gs/kubernetes-jenkins/pr-logs/directory/example-job-name",
   444  		},
   445  		{
   446  			name:       "presubmit job in GCS without trailing /",
   447  			src:        "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159",
   448  			expJobPath: "gs/kubernetes-jenkins/pr-logs/directory/example-job-name",
   449  		},
   450  		{
   451  			name:       "non-presubmit Prow job",
   452  			src:        "prowjob/example-periodic-job/1111",
   453  			expJobPath: "gs/chum-bucket/logs/example-periodic-job",
   454  		},
   455  		{
   456  			name:       "Prow presubmit job",
   457  			src:        "prowjob/example-presubmit-job/2222",
   458  			expJobPath: "gs/chum-bucket/pr-logs/directory/example-presubmit-job",
   459  		},
   460  		{
   461  			name:     "nonexistent job",
   462  			src:      "prowjob/example-periodic-job/0000",
   463  			expError: true,
   464  		},
   465  		{
   466  			name:     "invalid key type",
   467  			src:      "oh/my/glob/drama/bomb",
   468  			expError: true,
   469  		},
   470  		{
   471  			name:     "invalid GCS path",
   472  			src:      "gcs/kubernetes-jenkins/bad-path",
   473  			expError: true,
   474  		},
   475  		{
   476  			name:     "job missing decoration",
   477  			src:      "prowjob/undecorated-job/1",
   478  			expError: true,
   479  		},
   480  		{
   481  			name:     "job missing GCS config",
   482  			src:      "prowjob/missing-gcs-job/1",
   483  			expError: true,
   484  		},
   485  	}
   486  	for _, tc := range testCases {
   487  		fakeGCSClient := fakeGCSServer.Client()
   488  		fakeOpener := io.NewGCSOpener(fakeGCSClient)
   489  		fca := config.Agent{}
   490  		sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false)
   491  		jobPath, err := sg.JobPath(tc.src)
   492  		if tc.expError && err == nil {
   493  			t.Errorf("test %q: JobPath(%q) expected error", tc.name, tc.src)
   494  			continue
   495  		}
   496  		if !tc.expError && err != nil {
   497  			t.Errorf("test %q: JobPath(%q) returned unexpected error %v", tc.name, tc.src, err)
   498  			continue
   499  		}
   500  		if jobPath != tc.expJobPath {
   501  			t.Errorf("test %q: JobPath(%q) expected %q, got %q", tc.name, tc.src, tc.expJobPath, jobPath)
   502  		}
   503  	}
   504  }
   505  
   506  func TestProwJob(t *testing.T) {
   507  	kc := fkc{
   508  		prowapi.ProwJob{
   509  			ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-1"},
   510  			Spec: prowapi.ProwJobSpec{
   511  				Type: prowapi.PeriodicJob,
   512  				Job:  "example-periodic-job",
   513  				DecorationConfig: &prowapi.DecorationConfig{
   514  					GCSConfiguration: &prowapi.GCSConfiguration{
   515  						Bucket: "chum-bucket",
   516  					},
   517  				},
   518  			},
   519  			Status: prowapi.ProwJobStatus{
   520  				State:   prowapi.TriggeredState,
   521  				PodName: "flying-whales",
   522  				BuildID: "1111",
   523  			},
   524  		},
   525  		prowapi.ProwJob{
   526  			ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-2"},
   527  			Spec: prowapi.ProwJobSpec{
   528  				Type: prowapi.PresubmitJob,
   529  				Job:  "example-presubmit-job",
   530  				DecorationConfig: &prowapi.DecorationConfig{
   531  					GCSConfiguration: &prowapi.GCSConfiguration{
   532  						Bucket: "chum-bucket",
   533  					},
   534  				},
   535  			},
   536  			Status: prowapi.ProwJobStatus{
   537  				State:   prowapi.PendingState,
   538  				PodName: "flying-whales",
   539  				BuildID: "2222",
   540  			},
   541  		},
   542  		prowapi.ProwJob{
   543  			ObjectMeta: metav1.ObjectMeta{Name: "flying-whales-3"},
   544  			Spec: prowapi.ProwJobSpec{
   545  				Type: prowapi.PresubmitJob,
   546  				Job:  "undecorated-job",
   547  			},
   548  			Status: prowapi.ProwJobStatus{
   549  				State:   prowapi.SuccessState,
   550  				PodName: "flying-whales",
   551  				BuildID: "1",
   552  			},
   553  		},
   554  		prowapi.ProwJob{
   555  			Spec: prowapi.ProwJobSpec{
   556  				Type:             prowapi.PresubmitJob,
   557  				Job:              "missing-name-job",
   558  				DecorationConfig: &prowapi.DecorationConfig{},
   559  			},
   560  			Status: prowapi.ProwJobStatus{
   561  				PodName: "flying-whales",
   562  				BuildID: "1",
   563  			},
   564  		},
   565  	}
   566  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
   567  	fakeJa.Start()
   568  	testCases := []struct {
   569  		name        string
   570  		src         string
   571  		expJob      string
   572  		expJobPath  string
   573  		expJobState prowapi.ProwJobState
   574  		expError    bool
   575  	}{
   576  		{
   577  			name:        "non-presubmit job in GCS without trailing /",
   578  			src:         "gcs/kubernetes-jenkins/logs/example-periodic-job/1111/",
   579  			expJob:      "example-periodic-job",
   580  			expJobPath:  "flying-whales-1",
   581  			expJobState: prowapi.TriggeredState,
   582  		},
   583  		{
   584  			name:        "presubmit job in GCS with trailing /",
   585  			src:         "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-presubmit-job/2222/",
   586  			expJob:      "example-presubmit-job",
   587  			expJobPath:  "flying-whales-2",
   588  			expJobState: prowapi.PendingState,
   589  		},
   590  		{
   591  			name:        "non-presubmit Prow job",
   592  			src:         "prowjob/example-periodic-job/1111",
   593  			expJob:      "example-periodic-job",
   594  			expJobPath:  "flying-whales-1",
   595  			expJobState: prowapi.TriggeredState,
   596  		},
   597  		{
   598  			name:        "Prow presubmit job",
   599  			src:         "prowjob/example-presubmit-job/2222",
   600  			expJob:      "example-presubmit-job",
   601  			expJobPath:  "flying-whales-2",
   602  			expJobState: prowapi.PendingState,
   603  		},
   604  		{
   605  			name:        "nonexistent job",
   606  			src:         "prowjob/example-periodic-job/0000",
   607  			expJob:      "",
   608  			expJobPath:  "",
   609  			expJobState: "",
   610  		},
   611  		{
   612  			name:        "job missing name",
   613  			src:         "prowjob/missing-name-job/1",
   614  			expJob:      "missing-name-job",
   615  			expJobPath:  "",
   616  			expJobState: "",
   617  		},
   618  		{
   619  			name:        "previously invalid key type is now valid but nonexistent",
   620  			src:         "oh/my/glob/drama/bomb",
   621  			expJob:      "",
   622  			expJobPath:  "",
   623  			expJobState: "",
   624  		},
   625  		{
   626  			name:     "invalid GCS path",
   627  			src:      "gcs/kubernetes-jenkins/bad-path",
   628  			expError: true,
   629  		},
   630  	}
   631  	for _, tc := range testCases {
   632  		fakeGCSClient := fakeGCSServer.Client()
   633  		fakeOpener := io.NewGCSOpener(fakeGCSClient)
   634  		fca := config.Agent{}
   635  		sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false)
   636  		job, jobPath, jobState, err := sg.ProwJob(tc.src)
   637  		if tc.expError && err == nil {
   638  			t.Errorf("test %q: JobPath(%q) expected error", tc.name, tc.src)
   639  			continue
   640  		}
   641  		if !tc.expError && err != nil {
   642  			t.Errorf("test %q: JobPath(%q) returned unexpected error %v", tc.name, tc.src, err)
   643  			continue
   644  		}
   645  		if job != tc.expJob {
   646  			t.Errorf("test %q: Job(%q) expected %q, got %q", tc.name, tc.src, tc.expJob, job)
   647  		}
   648  		if jobPath != tc.expJobPath {
   649  			t.Errorf("test %q: JobPath(%q) expected %q, got %q", tc.name, tc.src, tc.expJobPath, jobPath)
   650  		}
   651  		if jobState != tc.expJobState {
   652  			t.Errorf("test %q: JobState(%q) expected %q, got %q", tc.name, tc.src, tc.expJobState, jobState)
   653  		}
   654  	}
   655  }
   656  
   657  func TestRunPath(t *testing.T) {
   658  	kc := fkc{
   659  		prowapi.ProwJob{
   660  			Spec: prowapi.ProwJobSpec{
   661  				Type: prowapi.PeriodicJob,
   662  				Job:  "example-periodic-job",
   663  				DecorationConfig: &prowapi.DecorationConfig{
   664  					GCSConfiguration: &prowapi.GCSConfiguration{
   665  						Bucket: "chum-bucket",
   666  					},
   667  				},
   668  			},
   669  			Status: prowapi.ProwJobStatus{
   670  				PodName: "flying-whales",
   671  				BuildID: "1111",
   672  				URL:     "http://magic/view/gcs/chum-bucket/logs/example-periodic-job/1111",
   673  			},
   674  		},
   675  		prowapi.ProwJob{
   676  			Spec: prowapi.ProwJobSpec{
   677  				Type: prowapi.PresubmitJob,
   678  				Job:  "example-presubmit-job",
   679  				DecorationConfig: &prowapi.DecorationConfig{
   680  					GCSConfiguration: &prowapi.GCSConfiguration{
   681  						Bucket: "chum-bucket",
   682  					},
   683  				},
   684  				Refs: &prowapi.Refs{
   685  					Org:  "some-org",
   686  					Repo: "some-repo",
   687  					Pulls: []prowapi.Pull{
   688  						{
   689  							Number: 42,
   690  						},
   691  					},
   692  				},
   693  			},
   694  			Status: prowapi.ProwJobStatus{
   695  				PodName: "flying-whales",
   696  				BuildID: "2222",
   697  				URL:     "http://magic/view/gcs/chum-bucket/pr-logs/pull/some-org_some-repo/42/example-presubmit-job/2222",
   698  			},
   699  		},
   700  	}
   701  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
   702  	fakeJa.Start()
   703  	testCases := []struct {
   704  		name       string
   705  		src        string
   706  		expRunPath string
   707  		expError   bool
   708  	}{
   709  		{
   710  			name:       "non-presubmit job in GCS with trailing /",
   711  			src:        "gcs/kubernetes-jenkins/logs/example-job-name/123/",
   712  			expRunPath: "kubernetes-jenkins/logs/example-job-name/123",
   713  		},
   714  		{
   715  			name:       "non-presubmit job in GCS without trailing /",
   716  			src:        "gcs/kubernetes-jenkins/logs/example-job-name/123",
   717  			expRunPath: "kubernetes-jenkins/logs/example-job-name/123",
   718  		},
   719  		{
   720  			name:       "presubmit job in GCS with trailing /",
   721  			src:        "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159/",
   722  			expRunPath: "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159",
   723  		},
   724  		{
   725  			name:       "presubmit job in GCS without trailing /",
   726  			src:        "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159",
   727  			expRunPath: "kubernetes-jenkins/pr-logs/pull/test-infra/0000/example-job-name/314159",
   728  		},
   729  		{
   730  			name:       "non-presubmit Prow job",
   731  			src:        "prowjob/example-periodic-job/1111",
   732  			expRunPath: "chum-bucket/logs/example-periodic-job/1111",
   733  		},
   734  		{
   735  			name:       "Prow presubmit job with full path",
   736  			src:        "prowjob/example-presubmit-job/2222",
   737  			expRunPath: "chum-bucket/pr-logs/pull/some-org_some-repo/42/example-presubmit-job/2222",
   738  		},
   739  		{
   740  			name:     "nonexistent job",
   741  			src:      "prowjob/example-periodic-job/0000",
   742  			expError: true,
   743  		},
   744  		{
   745  			name:       "previously invalid key type is now valid",
   746  			src:        "oh/my/glob/drama/bomb",
   747  			expRunPath: "my/glob/drama/bomb",
   748  		},
   749  		{
   750  			name:     "nonsense string errors",
   751  			src:      "this is not useful",
   752  			expError: true,
   753  		},
   754  	}
   755  	for _, tc := range testCases {
   756  		fakeGCSClient := fakeGCSServer.Client()
   757  		fakeOpener := io.NewGCSOpener(fakeGCSClient)
   758  		fca := config.Agent{}
   759  		fca.Set(&config.Config{
   760  			ProwConfig: config.ProwConfig{
   761  				Plank: config.Plank{
   762  					JobURLPrefixConfig: map[string]string{"*": "http://magic/view/gcs/"},
   763  				},
   764  			},
   765  		})
   766  		sg := New(context.Background(), fakeJa, fca.Config, fakeOpener, false)
   767  		jobPath, err := sg.RunPath(tc.src)
   768  		if tc.expError && err == nil {
   769  			t.Errorf("test %q: RunPath(%q) expected error, got  %q", tc.name, tc.src, jobPath)
   770  			continue
   771  		}
   772  		if !tc.expError && err != nil {
   773  			t.Errorf("test %q: RunPath(%q) returned unexpected error %v", tc.name, tc.src, err)
   774  			continue
   775  		}
   776  		if jobPath != tc.expRunPath {
   777  			t.Errorf("test %q: RunPath(%q) expected %q, got %q", tc.name, tc.src, tc.expRunPath, jobPath)
   778  		}
   779  	}
   780  }
   781  
   782  func TestRunToPR(t *testing.T) {
   783  	kc := fkc{
   784  		prowapi.ProwJob{
   785  			Spec: prowapi.ProwJobSpec{
   786  				Type: prowapi.PeriodicJob,
   787  				Job:  "example-periodic-job",
   788  				DecorationConfig: &prowapi.DecorationConfig{
   789  					GCSConfiguration: &prowapi.GCSConfiguration{
   790  						Bucket: "chum-bucket",
   791  					},
   792  				},
   793  			},
   794  			Status: prowapi.ProwJobStatus{
   795  				PodName: "flying-whales",
   796  				BuildID: "1111",
   797  				URL:     "http://magic/view/gcs/chum-bucket/logs/example-periodic-job/1111",
   798  			},
   799  		},
   800  		prowapi.ProwJob{
   801  			Spec: prowapi.ProwJobSpec{
   802  				Type: prowapi.PresubmitJob,
   803  				Job:  "example-presubmit-job",
   804  				DecorationConfig: &prowapi.DecorationConfig{
   805  					GCSConfiguration: &prowapi.GCSConfiguration{
   806  						Bucket: "chum-bucket",
   807  					},
   808  				},
   809  				Refs: &prowapi.Refs{
   810  					Org:  "some-org",
   811  					Repo: "some-repo",
   812  					Pulls: []prowapi.Pull{
   813  						{
   814  							Number: 42,
   815  						},
   816  					},
   817  				},
   818  			},
   819  			Status: prowapi.ProwJobStatus{
   820  				PodName: "flying-whales",
   821  				BuildID: "2222",
   822  			},
   823  		},
   824  	}
   825  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
   826  	fakeJa.Start()
   827  	testCases := []struct {
   828  		name      string
   829  		src       string
   830  		expOrg    string
   831  		expRepo   string
   832  		expNumber int
   833  		expError  bool
   834  	}{
   835  		{
   836  			name:      "presubmit job in GCS with trailing /",
   837  			src:       "gcs/kubernetes-jenkins/pr-logs/pull/Katharine_test-infra/1234/example-job-name/314159/",
   838  			expOrg:    "Katharine",
   839  			expRepo:   "test-infra",
   840  			expNumber: 1234,
   841  		},
   842  		{
   843  			name:      "presubmit job in GCS without trailing /",
   844  			src:       "gcs/kubernetes-jenkins/pr-logs/pull/Katharine_test-infra/1234/example-job-name/314159",
   845  			expOrg:    "Katharine",
   846  			expRepo:   "test-infra",
   847  			expNumber: 1234,
   848  		},
   849  		{
   850  			name:      "presubmit job in GCS without org name",
   851  			src:       "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/2345/example-job-name/314159",
   852  			expOrg:    "kubernetes",
   853  			expRepo:   "test-infra",
   854  			expNumber: 2345,
   855  		},
   856  		{
   857  			name:      "presubmit job in GCS without org or repo name",
   858  			src:       "gcs/kubernetes-jenkins/pr-logs/pull/3456/example-job-name/314159",
   859  			expOrg:    "kubernetes",
   860  			expRepo:   "kubernetes",
   861  			expNumber: 3456,
   862  		},
   863  		{
   864  			name:      "Prow presubmit job",
   865  			src:       "prowjob/example-presubmit-job/2222",
   866  			expOrg:    "some-org",
   867  			expRepo:   "some-repo",
   868  			expNumber: 42,
   869  		},
   870  		{
   871  			name:     "Prow periodic job errors",
   872  			src:      "prowjob/example-periodic-job/1111",
   873  			expError: true,
   874  		},
   875  		{
   876  			name:     "GCS periodic job errors",
   877  			src:      "gcs/kuberneretes-jenkins/logs/example-periodic-job/1111",
   878  			expError: true,
   879  		},
   880  		{
   881  			name:     "GCS job with non-numeric PR number errors",
   882  			src:      "gcs/kubernetes-jenkins/pr-logs/pull/asdf/example-job-name/314159",
   883  			expError: true,
   884  		},
   885  		{
   886  			name:     "GCS PR job in directory errors",
   887  			src:      "gcs/kubernetes-jenkins/pr-logs/directory/example-job-name/314159",
   888  			expError: true,
   889  		},
   890  		{
   891  			name:     "Bad GCS key errors",
   892  			src:      "gcs/this is just nonsense",
   893  			expError: true,
   894  		},
   895  		{
   896  			name:     "Longer bad GCS key errors",
   897  			src:      "gcs/kubernetes-jenkins/pr-logs",
   898  			expError: true,
   899  		},
   900  		{
   901  			name:     "Nonsense string errors",
   902  			src:      "friendship is magic",
   903  			expError: true,
   904  		},
   905  	}
   906  	for _, tc := range testCases {
   907  		fakeGCSClient := fakeGCSServer.Client()
   908  		fca := config.Agent{}
   909  		fca.Set(&config.Config{
   910  			ProwConfig: config.ProwConfig{
   911  				Plank: config.Plank{
   912  					DefaultDecorationConfigs: config.DefaultDecorationMapToSliceTesting(
   913  						map[string]*prowapi.DecorationConfig{
   914  							"*": {
   915  								GCSConfiguration: &prowapi.GCSConfiguration{
   916  									Bucket:       "kubernetes-jenkins",
   917  									DefaultOrg:   "kubernetes",
   918  									DefaultRepo:  "kubernetes",
   919  									PathStrategy: "legacy",
   920  								},
   921  							},
   922  						}),
   923  				},
   924  			},
   925  		})
   926  		sg := New(context.Background(), fakeJa, fca.Config, io.NewGCSOpener(fakeGCSClient), false)
   927  		org, repo, num, err := sg.RunToPR(tc.src)
   928  		if tc.expError && err == nil {
   929  			t.Errorf("test %q: RunToPR(%q) expected error", tc.name, tc.src)
   930  			continue
   931  		}
   932  		if !tc.expError && err != nil {
   933  			t.Errorf("test %q: RunToPR(%q) returned unexpected error %v", tc.name, tc.src, err)
   934  			continue
   935  		}
   936  		if org != tc.expOrg || repo != tc.expRepo || num != tc.expNumber {
   937  			t.Errorf("test %q: RunToPR(%q) expected %s/%s#%d, got %s/%s#%d", tc.name, tc.src, tc.expOrg, tc.expRepo, tc.expNumber, org, repo, num)
   938  		}
   939  	}
   940  }
   941  
   942  func TestProwToGCS(t *testing.T) {
   943  	testCases := []struct {
   944  		name         string
   945  		key          string
   946  		configPrefix string
   947  		expectedPath string
   948  		expectError  bool
   949  	}{
   950  		{
   951  			name:         "extraction from gubernator-like URL",
   952  			key:          "gubernator-job/1111",
   953  			configPrefix: "https://gubernator.example.com/build/",
   954  			expectedPath: "some-bucket/gubernator-job/1111/",
   955  			expectError:  false,
   956  		},
   957  		{
   958  			name:         "extraction from spyglass-like URL",
   959  			key:          "spyglass-job/2222",
   960  			configPrefix: "https://prow.example.com/view/gcs/",
   961  			expectedPath: "some-bucket/spyglass-job/2222/",
   962  			expectError:  false,
   963  		},
   964  		{
   965  			name:         "failed extraction from wrong URL",
   966  			key:          "spyglass-job/1111",
   967  			configPrefix: "https://gubernator.example.com/build/",
   968  			expectedPath: "",
   969  			expectError:  true,
   970  		},
   971  		{
   972  			name:         "prefix longer than URL",
   973  			key:          "spyglass-job/2222",
   974  			configPrefix: strings.Repeat("!", 100),
   975  			expectError:  true,
   976  		},
   977  	}
   978  
   979  	for _, tc := range testCases {
   980  		kc := fkc{
   981  			prowapi.ProwJob{
   982  				Spec: prowapi.ProwJobSpec{
   983  					Job: "gubernator-job",
   984  				},
   985  				Status: prowapi.ProwJobStatus{
   986  					URL:     "https://gubernator.example.com/build/some-bucket/gubernator-job/1111/",
   987  					BuildID: "1111",
   988  				},
   989  			},
   990  			prowapi.ProwJob{
   991  				Spec: prowapi.ProwJobSpec{
   992  					Job: "spyglass-job",
   993  				},
   994  				Status: prowapi.ProwJobStatus{
   995  					URL:     "https://prow.example.com/view/gcs/some-bucket/spyglass-job/2222/",
   996  					BuildID: "2222",
   997  				},
   998  			},
   999  		}
  1000  
  1001  		fakeGCSClient := fakeGCSServer.Client()
  1002  		fakeConfigAgent := fca{
  1003  			c: config.Config{
  1004  				ProwConfig: config.ProwConfig{
  1005  					Plank: config.Plank{
  1006  						JobURLPrefixConfig: map[string]string{"*": tc.configPrefix},
  1007  					},
  1008  				},
  1009  			},
  1010  		}
  1011  		fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config)
  1012  		fakeJa.Start()
  1013  		sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false)
  1014  
  1015  		_, p, err := sg.prowToGCS(tc.key)
  1016  		if err != nil && !tc.expectError {
  1017  			t.Errorf("test %q: unexpected error: %v", tc.key, err)
  1018  			continue
  1019  		}
  1020  		if err == nil && tc.expectError {
  1021  			t.Errorf("test %q: expected an error but instead got success and path '%s'", tc.key, p)
  1022  			continue
  1023  		}
  1024  		if p != tc.expectedPath {
  1025  			t.Errorf("test %q: expected '%s' but got '%s'", tc.key, tc.expectedPath, p)
  1026  		}
  1027  	}
  1028  }
  1029  
  1030  func TestGCSPathRoundTrip(t *testing.T) {
  1031  	testCases := []struct {
  1032  		name         string
  1033  		pathStrategy string
  1034  		defaultOrg   string
  1035  		defaultRepo  string
  1036  		org          string
  1037  		repo         string
  1038  	}{
  1039  		{
  1040  			name:         "simple explicit path",
  1041  			pathStrategy: "explicit",
  1042  			org:          "test-org",
  1043  			repo:         "test-repo",
  1044  		},
  1045  		{
  1046  			name:         "explicit path with underscores",
  1047  			pathStrategy: "explicit",
  1048  			org:          "test-org",
  1049  			repo:         "underscore_repo",
  1050  		},
  1051  		{
  1052  			name:         "'single' path with default repo",
  1053  			pathStrategy: "single",
  1054  			defaultOrg:   "default-org",
  1055  			defaultRepo:  "default-repo",
  1056  			org:          "default-org",
  1057  			repo:         "default-repo",
  1058  		},
  1059  		{
  1060  			name:         "'single' path with non-default repo",
  1061  			pathStrategy: "single",
  1062  			defaultOrg:   "default-org",
  1063  			defaultRepo:  "default-repo",
  1064  			org:          "default-org",
  1065  			repo:         "random-repo",
  1066  		},
  1067  		{
  1068  			name:         "'single' path with non-default org but default repo",
  1069  			pathStrategy: "single",
  1070  			defaultOrg:   "default-org",
  1071  			defaultRepo:  "default-repo",
  1072  			org:          "random-org",
  1073  			repo:         "default-repo",
  1074  		},
  1075  		{
  1076  			name:         "'single' path with non-default org and repo",
  1077  			pathStrategy: "single",
  1078  			defaultOrg:   "default-org",
  1079  			defaultRepo:  "default-repo",
  1080  			org:          "random-org",
  1081  			repo:         "random-repo",
  1082  		},
  1083  		{
  1084  			name:         "legacy path with default repo",
  1085  			pathStrategy: "legacy",
  1086  			defaultOrg:   "default-org",
  1087  			defaultRepo:  "default-repo",
  1088  			org:          "default-org",
  1089  			repo:         "default-repo",
  1090  		},
  1091  		{
  1092  			name:         "legacy path with non-default repo",
  1093  			pathStrategy: "legacy",
  1094  			defaultOrg:   "default-org",
  1095  			defaultRepo:  "default-repo",
  1096  			org:          "default-org",
  1097  			repo:         "random-repo",
  1098  		},
  1099  		{
  1100  			name:         "legacy path with non-default org but default repo",
  1101  			pathStrategy: "legacy",
  1102  			defaultOrg:   "default-org",
  1103  			defaultRepo:  "default-repo",
  1104  			org:          "random-org",
  1105  			repo:         "default-repo",
  1106  		},
  1107  		{
  1108  			name:         "legacy path with non-default org and repo",
  1109  			pathStrategy: "legacy",
  1110  			defaultOrg:   "default-org",
  1111  			defaultRepo:  "default-repo",
  1112  			org:          "random-org",
  1113  			repo:         "random-repo",
  1114  		},
  1115  		{
  1116  			name:         "legacy path with non-default org and repo with underscores",
  1117  			pathStrategy: "legacy",
  1118  			defaultOrg:   "default-org",
  1119  			defaultRepo:  "default-repo",
  1120  			org:          "random-org",
  1121  			repo:         "underscore_repo",
  1122  		},
  1123  	}
  1124  
  1125  	for _, tc := range testCases {
  1126  		kc := fkc{}
  1127  		fakeConfigAgent := fca{
  1128  			c: config.Config{
  1129  				ProwConfig: config.ProwConfig{
  1130  					Plank: config.Plank{
  1131  						DefaultDecorationConfigs: config.DefaultDecorationMapToSliceTesting(
  1132  							map[string]*prowapi.DecorationConfig{
  1133  								"*": {
  1134  									GCSConfiguration: &prowapi.GCSConfiguration{
  1135  										DefaultOrg:  tc.defaultOrg,
  1136  										DefaultRepo: tc.defaultRepo,
  1137  									},
  1138  								},
  1139  							}),
  1140  					},
  1141  				},
  1142  			},
  1143  		}
  1144  		fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config)
  1145  		fakeJa.Start()
  1146  
  1147  		fakeGCSClient := fakeGCSServer.Client()
  1148  
  1149  		sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false)
  1150  		gcspath, _, _ := gcsupload.PathsForJob(
  1151  			&prowapi.GCSConfiguration{Bucket: "test-bucket", PathStrategy: tc.pathStrategy},
  1152  			&downwardapi.JobSpec{
  1153  				Job:     "test-job",
  1154  				BuildID: "1234",
  1155  				Type:    prowapi.PresubmitJob,
  1156  				Refs: &prowapi.Refs{
  1157  					Org: tc.org, Repo: tc.repo,
  1158  					Pulls: []prowapi.Pull{{Number: 42}},
  1159  				},
  1160  			}, "")
  1161  		fmt.Println(gcspath)
  1162  		org, repo, prnum, err := sg.RunToPR("gcs/test-bucket/" + gcspath)
  1163  		if err != nil {
  1164  			t.Errorf("unexpected error: %v", err)
  1165  			continue
  1166  		}
  1167  		if org != tc.org || repo != tc.repo || prnum != 42 {
  1168  			t.Errorf("expected %s/%s#42, got %s/%s#%d", tc.org, tc.repo, org, repo, prnum)
  1169  		}
  1170  	}
  1171  }
  1172  
  1173  func TestTestGridLink(t *testing.T) {
  1174  	testCases := []struct {
  1175  		name     string
  1176  		src      string
  1177  		expQuery string
  1178  		expError bool
  1179  	}{
  1180  		{
  1181  			name:     "non-presubmit job in GCS with trailing /",
  1182  			src:      "gcs/kubernetes-jenkins/logs/periodic-job/123/",
  1183  			expQuery: "some-dashboard#periodic",
  1184  		},
  1185  		{
  1186  			name:     "non-presubmit job in GCS without trailing /",
  1187  			src:      "gcs/kubernetes-jenkins/logs/periodic-job/123",
  1188  			expQuery: "some-dashboard#periodic",
  1189  		},
  1190  		{
  1191  			name:     "presubmit job in GCS",
  1192  			src:      "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/0000/presubmit-job/314159/",
  1193  			expQuery: "some-dashboard#presubmit",
  1194  		},
  1195  		{
  1196  			name:     "non-presubmit Prow job",
  1197  			src:      "prowjob/periodic-job/1111",
  1198  			expQuery: "some-dashboard#periodic",
  1199  		},
  1200  		{
  1201  			name:     "presubmit Prow job",
  1202  			src:      "prowjob/presubmit-job/2222",
  1203  			expQuery: "some-dashboard#presubmit",
  1204  		},
  1205  		{
  1206  			name:     "nonexistent job",
  1207  			src:      "prowjob/nonexistent-job/0000",
  1208  			expError: true,
  1209  		},
  1210  		{
  1211  			name:     "invalid key type",
  1212  			src:      "oh/my/glob/drama/bomb",
  1213  			expError: true,
  1214  		},
  1215  		{
  1216  			name:     "nonsense string errors",
  1217  			src:      "this is not useful",
  1218  			expError: true,
  1219  		},
  1220  	}
  1221  
  1222  	kc := fkc{}
  1223  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fca{}.Config)
  1224  	fakeJa.Start()
  1225  
  1226  	tg := TestGrid{c: &tgconf.Configuration{
  1227  		Dashboards: []*tgconf.Dashboard{
  1228  			{
  1229  				Name: "some-dashboard",
  1230  				DashboardTab: []*tgconf.DashboardTab{
  1231  					{
  1232  						Name:          "periodic",
  1233  						TestGroupName: "periodic-job",
  1234  					},
  1235  					{
  1236  						Name:          "presubmit",
  1237  						TestGroupName: "presubmit-job",
  1238  					},
  1239  					{
  1240  						Name:          "some-other-job",
  1241  						TestGroupName: "some-other-job",
  1242  					},
  1243  				},
  1244  			},
  1245  		},
  1246  	}}
  1247  
  1248  	for _, tc := range testCases {
  1249  		fakeGCSClient := fakeGCSServer.Client()
  1250  		fca := config.Agent{}
  1251  		fca.Set(&config.Config{
  1252  			ProwConfig: config.ProwConfig{
  1253  				Deck: config.Deck{
  1254  					Spyglass: config.Spyglass{
  1255  						TestGridRoot: "https://testgrid.com/",
  1256  					},
  1257  				},
  1258  			},
  1259  		})
  1260  		sg := New(context.Background(), fakeJa, fca.Config, io.NewGCSOpener(fakeGCSClient), false)
  1261  		sg.testgrid = &tg
  1262  		link, err := sg.TestGridLink(tc.src)
  1263  		if tc.expError {
  1264  			if err == nil {
  1265  				t.Errorf("test %q: TestGridLink(%q) expected error, got  %q", tc.name, tc.src, link)
  1266  			}
  1267  			continue
  1268  		}
  1269  		if err != nil {
  1270  			t.Errorf("test %q: TestGridLink(%q) returned unexpected error %v", tc.name, tc.src, err)
  1271  			continue
  1272  		}
  1273  		if link != "https://testgrid.com/"+tc.expQuery {
  1274  			t.Errorf("test %q: TestGridLink(%q) expected %q, got %q", tc.name, tc.src, "https://testgrid.com/"+tc.expQuery, link)
  1275  		}
  1276  	}
  1277  }
  1278  
  1279  func TestFetchArtifactsPodLog(t *testing.T) {
  1280  	kc := fkc{
  1281  		prowapi.ProwJob{
  1282  			Spec: prowapi.ProwJobSpec{
  1283  				Agent: prowapi.KubernetesAgent,
  1284  				Job:   "job",
  1285  			},
  1286  			Status: prowapi.ProwJobStatus{
  1287  				PodName: "wowowow",
  1288  				BuildID: "123",
  1289  				URL:     "https://gubernator.example.com/build/job/123",
  1290  			},
  1291  		},
  1292  		prowapi.ProwJob{
  1293  			Spec: prowapi.ProwJobSpec{
  1294  				Agent: prowapi.KubernetesAgent,
  1295  				Job:   "multi-container-one-log",
  1296  			},
  1297  			Status: prowapi.ProwJobStatus{
  1298  				PodName: "wowowow",
  1299  				BuildID: "123",
  1300  				URL:     "https://gubernator.example.com/build/multi-container/123",
  1301  			},
  1302  		},
  1303  	}
  1304  	fakeConfigAgent := fca{
  1305  		c: config.Config{
  1306  			ProwConfig: config.ProwConfig{
  1307  				Deck: config.Deck{
  1308  					AllKnownStorageBuckets: sets.New[string]("job", "kubernetes-jenkins", "multi-container-one-log"),
  1309  				},
  1310  				Plank: config.Plank{
  1311  					JobURLPrefixConfig: map[string]string{"*": "https://gubernator.example.com/build/"},
  1312  				},
  1313  			},
  1314  		},
  1315  	}
  1316  	fakeJa = jobs.NewJobAgent(context.Background(), kc, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config)
  1317  	fakeJa.Start()
  1318  
  1319  	fakeGCSClient := fakeGCSServer.Client()
  1320  
  1321  	sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false)
  1322  	testKeys := []string{
  1323  		"prowjob/job/123",
  1324  		"gcs/kubernetes-jenkins/logs/job/123/",
  1325  		"gcs/kubernetes-jenkins/logs/job/123",
  1326  	}
  1327  
  1328  	for _, key := range testKeys {
  1329  		result, err := sg.FetchArtifacts(context.Background(), key, "", 500e6, []string{"build-log.txt"})
  1330  		if err != nil {
  1331  			t.Errorf("Unexpected error grabbing pod log for %s: %v", key, err)
  1332  			continue
  1333  		}
  1334  		if len(result) != 1 {
  1335  			t.Errorf("Expected 1 artifact for %s, got %d", key, len(result))
  1336  			continue
  1337  		}
  1338  		content, err := result[0].ReadAll()
  1339  		if err != nil {
  1340  			t.Errorf("Unexpected error reading pod log for %s: %v", key, err)
  1341  			continue
  1342  		}
  1343  		if string(content) != fmt.Sprintf("clusterA.%s", kube.TestContainerName) {
  1344  			t.Errorf("Bad pod log content for %s: %q (expected 'clusterA')", key, content)
  1345  		}
  1346  	}
  1347  
  1348  	multiContainerOneLogKey := "gcs/multi-container-one-log/logs/job/123"
  1349  
  1350  	testKeys = append(testKeys, multiContainerOneLogKey)
  1351  
  1352  	for _, key := range testKeys {
  1353  		containers := []string{"test-1", "test-2"}
  1354  		result, err := sg.FetchArtifacts(context.Background(), key, "", 500e6, []string{fmt.Sprintf("%s-%s", containers[0], singleLogName), fmt.Sprintf("%s-%s", containers[1], singleLogName)})
  1355  		if err != nil {
  1356  			t.Errorf("Unexpected error grabbing pod log for %s: %v", key, err)
  1357  			continue
  1358  		}
  1359  		for i, art := range result {
  1360  			content, err := art.ReadAll()
  1361  			if err != nil {
  1362  				t.Errorf("Unexpected error reading pod log for %s: %v", key, err)
  1363  				continue
  1364  			}
  1365  			expected := fmt.Sprintf("clusterA.%s", containers[i])
  1366  			if key == multiContainerOneLogKey && containers[i] == "test-1" {
  1367  				expected = "this log exists in gcs!"
  1368  			}
  1369  			if string(content) != expected {
  1370  				t.Errorf("Bad pod log content for %s: %q (expected '%s')", key, content, expected)
  1371  			}
  1372  		}
  1373  	}
  1374  }
  1375  
  1376  func TestKeyToJob(t *testing.T) {
  1377  	testCases := []struct {
  1378  		name      string
  1379  		path      string
  1380  		jobName   string
  1381  		buildID   string
  1382  		expectErr bool
  1383  	}{
  1384  		{
  1385  			name:    "GCS periodic path with trailing slash",
  1386  			path:    "gcs/kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-1-14/40/",
  1387  			jobName: "periodic-kubernetes-bazel-test-1-14",
  1388  			buildID: "40",
  1389  		},
  1390  		{
  1391  			name:    "GCS periodic path without trailing slash",
  1392  			path:    "gcs/kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-1-14/40",
  1393  			jobName: "periodic-kubernetes-bazel-test-1-14",
  1394  			buildID: "40",
  1395  		},
  1396  		{
  1397  			name:    "GCS PR path with trailing slash",
  1398  			path:    "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/11573/pull-test-infra-bazel/25366/",
  1399  			jobName: "pull-test-infra-bazel",
  1400  			buildID: "25366",
  1401  		},
  1402  		{
  1403  			name:    "GCS PR path without trailing slash",
  1404  			path:    "gcs/kubernetes-jenkins/pr-logs/pull/test-infra/11573/pull-test-infra-bazel/25366",
  1405  			jobName: "pull-test-infra-bazel",
  1406  			buildID: "25366",
  1407  		},
  1408  		{
  1409  			name:    "Prowjob path with trailing slash",
  1410  			path:    "prowjob/pull-test-infra-bazel/25366/",
  1411  			jobName: "pull-test-infra-bazel",
  1412  			buildID: "25366",
  1413  		},
  1414  		{
  1415  			name:    "Prowjob path without trailing slash",
  1416  			path:    "prowjob/pull-test-infra-bazel/25366",
  1417  			jobName: "pull-test-infra-bazel",
  1418  			buildID: "25366",
  1419  		},
  1420  		{
  1421  			name:      "Path with only one component",
  1422  			path:      "nope",
  1423  			expectErr: true,
  1424  		},
  1425  	}
  1426  
  1427  	for _, tc := range testCases {
  1428  		jobName, buildID, err := common.KeyToJob(tc.path)
  1429  		if err != nil {
  1430  			if !tc.expectErr {
  1431  				t.Errorf("%s: unexpected error %v", tc.name, err)
  1432  			}
  1433  			continue
  1434  		}
  1435  		if tc.expectErr {
  1436  			t.Errorf("%s: expected an error, but got result %s #%s", tc.name, jobName, buildID)
  1437  			continue
  1438  		}
  1439  		if jobName != tc.jobName {
  1440  			t.Errorf("%s: expected job name %q, but got %q", tc.name, tc.jobName, jobName)
  1441  			continue
  1442  		}
  1443  		if buildID != tc.buildID {
  1444  			t.Errorf("%s: expected build ID %q, but got %q", tc.name, tc.buildID, buildID)
  1445  		}
  1446  	}
  1447  }
  1448  
  1449  func TestResolveSymlink(t *testing.T) {
  1450  	testCases := []struct {
  1451  		name      string
  1452  		path      string
  1453  		result    string
  1454  		expectErr bool
  1455  	}{
  1456  		{
  1457  			name:   "symlink without trailing slash is resolved",
  1458  			path:   "gcs/test-bucket/logs/symlink-party/123",
  1459  			result: "gs/test-bucket/logs/the-actual-place/123",
  1460  		},
  1461  		{
  1462  			name:   "symlink with trailing slash is resolved",
  1463  			path:   "gcs/test-bucket/logs/symlink-party/123/",
  1464  			result: "gs/test-bucket/logs/the-actual-place/123",
  1465  		},
  1466  		{
  1467  			name:   "non-symlink without trailing slash is unchanged",
  1468  			path:   "gcs/test-bucket/better-logs/42",
  1469  			result: "gs/test-bucket/better-logs/42",
  1470  		},
  1471  		{
  1472  			name:   "non-symlink with trailing slash drops the slash",
  1473  			path:   "gcs/test-bucket/better-logs/42/",
  1474  			result: "gs/test-bucket/better-logs/42",
  1475  		},
  1476  		{
  1477  			name:   "non-symlink with aliased bucket is replaced",
  1478  			path:   "gcs/alias/better-logs/42",
  1479  			result: "gs/test-bucket/better-logs/42",
  1480  		},
  1481  		{
  1482  			name:   "prowjob without trailing slash is unchanged",
  1483  			path:   "prowjob/better-logs/42",
  1484  			result: "prowjob/better-logs/42",
  1485  		},
  1486  		{
  1487  			name:   "prowjob with trailing slash drops the slash",
  1488  			path:   "prowjob/better-logs/42/",
  1489  			result: "prowjob/better-logs/42",
  1490  		},
  1491  		{
  1492  			name:      "unknown key type is an error",
  1493  			path:      "wtf/what-is-this/send-help",
  1494  			expectErr: true,
  1495  		},
  1496  		{
  1497  			name:      "insufficient path components are an error",
  1498  			path:      "gcs/hi",
  1499  			expectErr: true,
  1500  		},
  1501  	}
  1502  
  1503  	for _, tc := range testCases {
  1504  		fakeConfigAgent := fca{
  1505  			c: config.Config{
  1506  				ProwConfig: config.ProwConfig{
  1507  					Deck: config.Deck{
  1508  						Spyglass: config.Spyglass{
  1509  							BucketAliases: map[string]string{"alias": "test-bucket"}},
  1510  					},
  1511  				},
  1512  			},
  1513  		}
  1514  		//fakeConfigAgent.Config().Deck.Spyglass.BucketAliases = map[string]string{"alias": "test-bucket"}
  1515  		fakeJa = jobs.NewJobAgent(context.Background(), fkc{}, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config)
  1516  		fakeJa.Start()
  1517  
  1518  		fakeGCSClient := fakeGCSServer.Client()
  1519  
  1520  		sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(fakeGCSClient), false)
  1521  
  1522  		result, err := sg.ResolveSymlink(tc.path)
  1523  		if err != nil {
  1524  			if !tc.expectErr {
  1525  				t.Errorf("test %q: unexpected error: %v", tc.name, err)
  1526  			}
  1527  			continue
  1528  		}
  1529  		if tc.expectErr {
  1530  			t.Errorf("test %q: expected an error, but got result %q", tc.name, result)
  1531  			continue
  1532  		}
  1533  		if result != tc.result {
  1534  			t.Errorf("test %q: expected %q, but got %q", tc.name, tc.result, result)
  1535  			continue
  1536  		}
  1537  	}
  1538  }
  1539  
  1540  func TestExtraLinks(t *testing.T) {
  1541  	testCases := []struct {
  1542  		name      string
  1543  		content   string
  1544  		links     []ExtraLink
  1545  		expectErr bool
  1546  	}{
  1547  		{
  1548  			name:  "does nothing without error given no started.json",
  1549  			links: nil,
  1550  		},
  1551  		{
  1552  			name:      "errors given a malformed started.json",
  1553  			content:   "this isn't json",
  1554  			expectErr: true,
  1555  		},
  1556  		{
  1557  			name:    "does nothing given metadata with no links",
  1558  			content: `{"metadata": {"somethingThatIsntLinks": 23}}`,
  1559  			links:   nil,
  1560  		},
  1561  		{
  1562  			name:    "returns well-formed links",
  1563  			content: `{"metadata": {"links": {"ResultStore": {"url": "http://resultstore", "description": "The thing that isn't spyglass"}}}}`,
  1564  			links:   []ExtraLink{{Name: "ResultStore", URL: "http://resultstore", Description: "The thing that isn't spyglass"}},
  1565  		},
  1566  		{
  1567  			name:    "returns links without a description",
  1568  			content: `{"metadata": {"links": {"ResultStore": {"url": "http://resultstore"}}}}`,
  1569  			links:   []ExtraLink{{Name: "ResultStore", URL: "http://resultstore"}},
  1570  		},
  1571  		{
  1572  			name:    "skips links without a URL",
  1573  			content: `{"metadata": {"links": {"No Link": {"description": "bad link"}, "ResultStore": {"url": "http://resultstore"}}}}`,
  1574  			links:   []ExtraLink{{Name: "ResultStore", URL: "http://resultstore"}},
  1575  		},
  1576  		{
  1577  			name:    "skips links without a name",
  1578  			content: `{"metadata": {"links": {"": {"url": "http://resultstore"}}}}`,
  1579  			links:   []ExtraLink{},
  1580  		},
  1581  		{
  1582  			name:    "returns no links when links is empty",
  1583  			content: `{"metadata": {"links": {}}}`,
  1584  			links:   []ExtraLink{},
  1585  		},
  1586  		{
  1587  			name:    "returns multiple links",
  1588  			content: `{"metadata": {"links": {"A": {"url": "http://a", "description": "A!"}, "B": {"url": "http://b"}}}}`,
  1589  			links:   []ExtraLink{{Name: "A", URL: "http://a", Description: "A!"}, {Name: "B", URL: "http://b"}},
  1590  		},
  1591  	}
  1592  	for _, tc := range testCases {
  1593  		t.Run(tc.name, func(t *testing.T) {
  1594  			var objects []fakestorage.Object
  1595  			if tc.content != "" {
  1596  				objects = []fakestorage.Object{
  1597  					{
  1598  						BucketName: "test-bucket",
  1599  						Name:       "logs/some-job/42/started.json",
  1600  						Content:    []byte(tc.content),
  1601  					},
  1602  				}
  1603  			}
  1604  			gcsServer := fakestorage.NewServer(objects)
  1605  			defer gcsServer.Stop()
  1606  
  1607  			gcsClient := gcsServer.Client()
  1608  			fakeConfigAgent := fca{
  1609  				c: config.Config{
  1610  					ProwConfig: config.ProwConfig{
  1611  						Deck: config.Deck{
  1612  							AllKnownStorageBuckets: sets.New[string]("test-bucket"),
  1613  						},
  1614  					},
  1615  				},
  1616  			}
  1617  			fakeJa = jobs.NewJobAgent(context.Background(), fkc{}, false, true, []string{}, map[string]jobs.PodLogClient{kube.DefaultClusterAlias: fpkc("clusterA"), "trusted": fpkc("clusterB")}, fakeConfigAgent.Config)
  1618  			fakeJa.Start()
  1619  			sg := New(context.Background(), fakeJa, fakeConfigAgent.Config, io.NewGCSOpener(gcsClient), false)
  1620  
  1621  			result, err := sg.ExtraLinks(context.Background(), "gcs/test-bucket/logs/some-job/42")
  1622  			if err != nil {
  1623  				if !tc.expectErr {
  1624  					t.Fatalf("unexpected error: %v", err)
  1625  				}
  1626  				return
  1627  			}
  1628  			sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name })
  1629  			sort.Slice(tc.links, func(i, j int) bool { return tc.links[i].Name < tc.links[j].Name })
  1630  			if !reflect.DeepEqual(result, tc.links) {
  1631  				t.Fatalf("Expected links %#v, got %#v", tc.links, result)
  1632  			}
  1633  		})
  1634  	}
  1635  }