go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/buildbucket/appengine/model/build_test.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package model
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strconv"
    21  	"strings"
    22  	"testing"
    23  	"time"
    24  
    25  	"google.golang.org/protobuf/proto"
    26  	"google.golang.org/protobuf/types/known/structpb"
    27  	"google.golang.org/protobuf/types/known/timestamppb"
    28  
    29  	"go.chromium.org/luci/common/clock/testclock"
    30  	"go.chromium.org/luci/gae/impl/memory"
    31  	"go.chromium.org/luci/gae/service/datastore"
    32  
    33  	pb "go.chromium.org/luci/buildbucket/proto"
    34  
    35  	. "github.com/smartystreets/goconvey/convey"
    36  
    37  	. "go.chromium.org/luci/common/testing/assertions"
    38  )
    39  
    40  func mustStruct(data map[string]any) *structpb.Struct {
    41  	ret, err := structpb.NewStruct(data)
    42  	if err != nil {
    43  		panic(err)
    44  	}
    45  	return ret
    46  }
    47  
    48  func TestBuild(t *testing.T) {
    49  	t.Parallel()
    50  
    51  	Convey("Build", t, func() {
    52  		ctx := memory.Use(context.Background())
    53  		ctx, tclock := testclock.UseTime(ctx, testclock.TestRecentTimeUTC)
    54  
    55  		t0 := tclock.Now()
    56  		t0pb := timestamppb.New(t0)
    57  
    58  		datastore.GetTestable(ctx).AutoIndex(true)
    59  		datastore.GetTestable(ctx).Consistent(true)
    60  		m := NoopBuildMask
    61  
    62  		Convey("read/write", func() {
    63  			So(datastore.Put(ctx, &Build{
    64  				ID: 1,
    65  				Proto: &pb.Build{
    66  					Id: 1,
    67  					Builder: &pb.BuilderID{
    68  						Project: "project",
    69  						Bucket:  "bucket",
    70  						Builder: "builder",
    71  					},
    72  					Status:      pb.Status_SUCCESS,
    73  					CreateTime:  t0pb,
    74  					UpdateTime:  t0pb,
    75  					AncestorIds: []int64{2, 3, 4},
    76  				},
    77  			}), ShouldBeNil)
    78  
    79  			b := &Build{
    80  				ID: 1,
    81  			}
    82  			So(datastore.Get(ctx, b), ShouldBeNil)
    83  			p := proto.Clone(b.Proto).(*pb.Build)
    84  			b.Proto = &pb.Build{}
    85  			b.NextBackendSyncTime = ""
    86  			So(b, ShouldResemble, &Build{
    87  				ID:                1,
    88  				Proto:             &pb.Build{},
    89  				BucketID:          "project/bucket",
    90  				BuilderID:         "project/bucket/builder",
    91  				Canary:            false,
    92  				CreateTime:        datastore.RoundTime(t0),
    93  				StatusChangedTime: datastore.RoundTime(t0),
    94  				Experimental:      false,
    95  				Incomplete:        false,
    96  				Status:            pb.Status_SUCCESS,
    97  				Project:           "project",
    98  				LegacyProperties: LegacyProperties{
    99  					Result: Success,
   100  					Status: Completed,
   101  				},
   102  				AncestorIds: []int64{2, 3, 4},
   103  				ParentID:    4,
   104  			})
   105  			So(p, ShouldResembleProto, &pb.Build{
   106  				Id: 1,
   107  				Builder: &pb.BuilderID{
   108  					Project: "project",
   109  					Bucket:  "bucket",
   110  					Builder: "builder",
   111  				},
   112  				Status:      pb.Status_SUCCESS,
   113  				CreateTime:  t0pb,
   114  				UpdateTime:  t0pb,
   115  				AncestorIds: []int64{2, 3, 4},
   116  			})
   117  		})
   118  
   119  		Convey("legacy", func() {
   120  			Convey("infra failure", func() {
   121  				So(datastore.Put(ctx, &Build{
   122  					ID: 1,
   123  					Proto: &pb.Build{
   124  						Id: 1,
   125  						Builder: &pb.BuilderID{
   126  							Project: "project",
   127  							Bucket:  "bucket",
   128  							Builder: "builder",
   129  						},
   130  						Status:     pb.Status_INFRA_FAILURE,
   131  						CreateTime: t0pb,
   132  						UpdateTime: t0pb,
   133  					},
   134  				}), ShouldBeNil)
   135  
   136  				b := &Build{
   137  					ID: 1,
   138  				}
   139  				So(datastore.Get(ctx, b), ShouldBeNil)
   140  				p := proto.Clone(b.Proto).(*pb.Build)
   141  				b.Proto = &pb.Build{}
   142  				b.NextBackendSyncTime = ""
   143  				So(b, ShouldResemble, &Build{
   144  					ID:                1,
   145  					Proto:             &pb.Build{},
   146  					BucketID:          "project/bucket",
   147  					BuilderID:         "project/bucket/builder",
   148  					Canary:            false,
   149  					CreateTime:        datastore.RoundTime(t0),
   150  					StatusChangedTime: datastore.RoundTime(t0),
   151  					Experimental:      false,
   152  					Incomplete:        false,
   153  					Status:            pb.Status_INFRA_FAILURE,
   154  					Project:           "project",
   155  					LegacyProperties: LegacyProperties{
   156  						FailureReason: InfraFailure,
   157  						Result:        Failure,
   158  						Status:        Completed,
   159  					},
   160  				})
   161  				So(p, ShouldResembleProto, &pb.Build{
   162  					Id: 1,
   163  					Builder: &pb.BuilderID{
   164  						Project: "project",
   165  						Bucket:  "bucket",
   166  						Builder: "builder",
   167  					},
   168  					Status:     pb.Status_INFRA_FAILURE,
   169  					CreateTime: t0pb,
   170  					UpdateTime: t0pb,
   171  				})
   172  			})
   173  
   174  			Convey("timeout", func() {
   175  				So(datastore.Put(ctx, &Build{
   176  					ID: 1,
   177  					Proto: &pb.Build{
   178  						Id: 1,
   179  						Builder: &pb.BuilderID{
   180  							Project: "project",
   181  							Bucket:  "bucket",
   182  							Builder: "builder",
   183  						},
   184  						Status: pb.Status_INFRA_FAILURE,
   185  						StatusDetails: &pb.StatusDetails{
   186  							Timeout: &pb.StatusDetails_Timeout{},
   187  						},
   188  						CreateTime: t0pb,
   189  						UpdateTime: t0pb,
   190  					},
   191  				}), ShouldBeNil)
   192  
   193  				b := &Build{
   194  					ID: 1,
   195  				}
   196  				So(datastore.Get(ctx, b), ShouldBeNil)
   197  				p := proto.Clone(b.Proto).(*pb.Build)
   198  				b.Proto = &pb.Build{}
   199  				b.NextBackendSyncTime = ""
   200  				So(b, ShouldResemble, &Build{
   201  					ID:                1,
   202  					Proto:             &pb.Build{},
   203  					BucketID:          "project/bucket",
   204  					BuilderID:         "project/bucket/builder",
   205  					Canary:            false,
   206  					CreateTime:        datastore.RoundTime(t0),
   207  					StatusChangedTime: datastore.RoundTime(t0),
   208  					Experimental:      false,
   209  					Incomplete:        false,
   210  					Status:            pb.Status_INFRA_FAILURE,
   211  					Project:           "project",
   212  					LegacyProperties: LegacyProperties{
   213  						CancelationReason: TimeoutCanceled,
   214  						Result:            Canceled,
   215  						Status:            Completed,
   216  					},
   217  				})
   218  				So(p, ShouldResembleProto, &pb.Build{
   219  					Id: 1,
   220  					Builder: &pb.BuilderID{
   221  						Project: "project",
   222  						Bucket:  "bucket",
   223  						Builder: "builder",
   224  					},
   225  					Status: pb.Status_INFRA_FAILURE,
   226  					StatusDetails: &pb.StatusDetails{
   227  						Timeout: &pb.StatusDetails_Timeout{},
   228  					},
   229  					CreateTime: t0pb,
   230  					UpdateTime: t0pb,
   231  				})
   232  			})
   233  
   234  			Convey("canceled", func() {
   235  				So(datastore.Put(ctx, &Build{
   236  					ID: 1,
   237  					Proto: &pb.Build{
   238  						Id: 1,
   239  						Builder: &pb.BuilderID{
   240  							Project: "project",
   241  							Bucket:  "bucket",
   242  							Builder: "builder",
   243  						},
   244  						Status:     pb.Status_CANCELED,
   245  						CreateTime: t0pb,
   246  						UpdateTime: t0pb,
   247  					},
   248  				}), ShouldBeNil)
   249  
   250  				b := &Build{
   251  					ID: 1,
   252  				}
   253  				So(datastore.Get(ctx, b), ShouldBeNil)
   254  				p := proto.Clone(b.Proto).(*pb.Build)
   255  				b.Proto = &pb.Build{}
   256  				b.NextBackendSyncTime = ""
   257  				So(b, ShouldResemble, &Build{
   258  					ID:                1,
   259  					Proto:             &pb.Build{},
   260  					BucketID:          "project/bucket",
   261  					BuilderID:         "project/bucket/builder",
   262  					Canary:            false,
   263  					CreateTime:        datastore.RoundTime(t0),
   264  					StatusChangedTime: datastore.RoundTime(t0),
   265  					Experimental:      false,
   266  					Incomplete:        false,
   267  					Status:            pb.Status_CANCELED,
   268  					Project:           "project",
   269  					LegacyProperties: LegacyProperties{
   270  						CancelationReason: ExplicitlyCanceled,
   271  						Result:            Canceled,
   272  						Status:            Completed,
   273  					},
   274  				})
   275  				So(p, ShouldResembleProto, &pb.Build{
   276  					Id: 1,
   277  					Builder: &pb.BuilderID{
   278  						Project: "project",
   279  						Bucket:  "bucket",
   280  						Builder: "builder",
   281  					},
   282  					Status:     pb.Status_CANCELED,
   283  					CreateTime: t0pb,
   284  					UpdateTime: t0pb,
   285  				})
   286  			})
   287  		})
   288  
   289  		Convey("Realm", func() {
   290  			b := &Build{
   291  				ID: 1,
   292  				Proto: &pb.Build{
   293  					Id: 1,
   294  					Builder: &pb.BuilderID{
   295  						Project: "project",
   296  						Bucket:  "bucket",
   297  						Builder: "builder",
   298  					},
   299  				},
   300  			}
   301  			So(b.Realm(), ShouldEqual, "project:bucket")
   302  		})
   303  
   304  		Convey("ToProto", func() {
   305  			b := &Build{
   306  				ID: 1,
   307  				Proto: &pb.Build{
   308  					Id: 1,
   309  				},
   310  				Tags: []string{
   311  					"key1:value1",
   312  					"builder:hidden",
   313  					"key2:value2",
   314  				},
   315  			}
   316  			key := datastore.KeyForObj(ctx, b)
   317  			So(datastore.Put(ctx, &BuildInfra{
   318  				Build: key,
   319  				Proto: &pb.BuildInfra{
   320  					Buildbucket: &pb.BuildInfra_Buildbucket{
   321  						Hostname: "example.com",
   322  					},
   323  				},
   324  			}), ShouldBeNil)
   325  			So(datastore.Put(ctx, &BuildInputProperties{
   326  				Build: key,
   327  				Proto: &structpb.Struct{
   328  					Fields: map[string]*structpb.Value{
   329  						"input": {
   330  							Kind: &structpb.Value_StringValue{
   331  								StringValue: "input value",
   332  							},
   333  						},
   334  					},
   335  				},
   336  			}), ShouldBeNil)
   337  
   338  			Convey("mask", func() {
   339  				Convey("include", func() {
   340  					m := HardcodedBuildMask("id")
   341  					p, err := b.ToProto(ctx, m, nil)
   342  					So(err, ShouldBeNil)
   343  					So(p.Id, ShouldEqual, 1)
   344  				})
   345  
   346  				Convey("exclude", func() {
   347  					m := HardcodedBuildMask("builder")
   348  					p, err := b.ToProto(ctx, m, nil)
   349  					So(err, ShouldBeNil)
   350  					So(p.Id, ShouldEqual, 0)
   351  				})
   352  			})
   353  
   354  			Convey("tags", func() {
   355  				p, err := b.ToProto(ctx, m, nil)
   356  				So(err, ShouldBeNil)
   357  				So(p.Tags, ShouldResembleProto, []*pb.StringPair{
   358  					{
   359  						Key:   "key1",
   360  						Value: "value1",
   361  					},
   362  					{
   363  						Key:   "key2",
   364  						Value: "value2",
   365  					},
   366  				})
   367  				So(b.Proto.Tags, ShouldBeEmpty)
   368  			})
   369  
   370  			Convey("infra", func() {
   371  				p, err := b.ToProto(ctx, m, nil)
   372  				So(err, ShouldBeNil)
   373  				So(p.Infra, ShouldResembleProto, &pb.BuildInfra{
   374  					Buildbucket: &pb.BuildInfra_Buildbucket{
   375  						Hostname: "example.com",
   376  					},
   377  				})
   378  				So(b.Proto.Infra, ShouldBeNil)
   379  			})
   380  
   381  			Convey("input properties", func() {
   382  				p, err := b.ToProto(ctx, m, nil)
   383  				So(err, ShouldBeNil)
   384  				So(p.Input.Properties, ShouldResembleProto, mustStruct(map[string]any{
   385  					"input": "input value",
   386  				}))
   387  				So(b.Proto.Input, ShouldBeNil)
   388  			})
   389  
   390  			Convey("output properties", func() {
   391  				So(datastore.Put(ctx, &BuildOutputProperties{
   392  					Build: key,
   393  					Proto: &structpb.Struct{
   394  						Fields: map[string]*structpb.Value{
   395  							"output": {
   396  								Kind: &structpb.Value_StringValue{
   397  									StringValue: "output value",
   398  								},
   399  							},
   400  						},
   401  					},
   402  				}), ShouldBeNil)
   403  				p, err := b.ToProto(ctx, m, nil)
   404  				So(err, ShouldBeNil)
   405  				So(p.Output.Properties, ShouldResembleProto, mustStruct(map[string]any{
   406  					"output": "output value",
   407  				}))
   408  				So(b.Proto.Output, ShouldBeNil)
   409  
   410  				Convey("one missing, one found", func() {
   411  					b1 := &pb.Build{
   412  						Id: 1,
   413  					}
   414  					b2 := &pb.Build{
   415  						Id: 2,
   416  					}
   417  					m := HardcodedBuildMask("output.properties")
   418  					So(LoadBuildDetails(ctx, m, nil, b1, b2), ShouldBeNil)
   419  					So(b1.Output.Properties, ShouldResembleProto, mustStruct(map[string]any{
   420  						"output": "output value",
   421  					}))
   422  					So(b2.Output.GetProperties(), ShouldBeNil)
   423  				})
   424  			})
   425  
   426  			Convey("output properties(large)", func() {
   427  				largeProps, err := structpb.NewStruct(map[string]any{})
   428  				So(err, ShouldBeNil)
   429  				k := "laaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaarge_key"
   430  				v := "laaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaarge_value"
   431  				for i := 0; i < 10000; i++ {
   432  					largeProps.Fields[k+strconv.Itoa(i)] = &structpb.Value{
   433  						Kind: &structpb.Value_StringValue{
   434  							StringValue: v,
   435  						},
   436  					}
   437  				}
   438  				outProp := &BuildOutputProperties{
   439  					Build: key,
   440  					Proto: largeProps,
   441  				}
   442  				So(outProp.Put(ctx), ShouldBeNil)
   443  
   444  				p, err := b.ToProto(ctx, m, nil)
   445  				So(err, ShouldBeNil)
   446  				So(p.Output.Properties, ShouldResembleProto, largeProps)
   447  				So(b.Proto.Output, ShouldBeNil)
   448  			})
   449  
   450  			Convey("steps", func() {
   451  				s, err := proto.Marshal(&pb.Build{
   452  					Steps: []*pb.Step{
   453  						{
   454  							Name: "step",
   455  						},
   456  					},
   457  				})
   458  				So(err, ShouldBeNil)
   459  				So(datastore.Put(ctx, &BuildSteps{
   460  					Build:    key,
   461  					Bytes:    s,
   462  					IsZipped: false,
   463  				}), ShouldBeNil)
   464  				p, err := b.ToProto(ctx, m, nil)
   465  				So(err, ShouldBeNil)
   466  				So(p.Steps, ShouldResembleProto, []*pb.Step{
   467  					{
   468  						Name: "step",
   469  					},
   470  				})
   471  				So(b.Proto.Steps, ShouldBeEmpty)
   472  			})
   473  		})
   474  
   475  		Convey("ToSimpleBuildProto", func() {
   476  			b := &Build{
   477  				ID: 1,
   478  				Proto: &pb.Build{
   479  					Id: 1,
   480  					Builder: &pb.BuilderID{
   481  						Project: "project",
   482  						Bucket:  "bucket",
   483  						Builder: "builder",
   484  					},
   485  					Tags: []*pb.StringPair{
   486  						{
   487  							Key:   "k1",
   488  							Value: "v1",
   489  						},
   490  					},
   491  				},
   492  				Project:   "project",
   493  				BucketID:  "project/bucket",
   494  				BuilderID: "project/bucket/builder",
   495  				Tags: []string{
   496  					"k1:v1",
   497  				},
   498  			}
   499  
   500  			actual := b.ToSimpleBuildProto(ctx)
   501  			So(actual, ShouldResembleProto, &pb.Build{
   502  				Id: 1,
   503  				Builder: &pb.BuilderID{
   504  					Project: "project",
   505  					Bucket:  "bucket",
   506  					Builder: "builder",
   507  				},
   508  				Tags: []*pb.StringPair{
   509  					{
   510  						Key:   "k1",
   511  						Value: "v1",
   512  					},
   513  				},
   514  			})
   515  		})
   516  
   517  		Convey("ExperimentsString", func() {
   518  			b := &Build{}
   519  			check := func(exps []string, enabled string) {
   520  				b.Experiments = exps
   521  				So(b.ExperimentsString(), ShouldEqual, enabled)
   522  			}
   523  
   524  			Convey("Returns None", func() {
   525  				check([]string{}, "None")
   526  			})
   527  
   528  			Convey("Sorted", func() {
   529  				exps := []string{"+exp4", "-exp3", "+exp1", "-exp10"}
   530  				check(exps, "exp1|exp4")
   531  			})
   532  		})
   533  
   534  		Convey("NextBackendSyncTime", func() {
   535  			b := &Build{
   536  				ID:      1,
   537  				Project: "project",
   538  				Proto: &pb.Build{
   539  					Id: 1,
   540  					Builder: &pb.BuilderID{
   541  						Project: "project",
   542  						Bucket:  "bucket",
   543  						Builder: "builder",
   544  					},
   545  					Status:      pb.Status_STARTED,
   546  					CreateTime:  t0pb,
   547  					UpdateTime:  t0pb,
   548  					AncestorIds: []int64{2, 3, 4},
   549  				},
   550  				BackendTarget: "backend",
   551  			}
   552  			b.GenerateNextBackendSyncTime(ctx, 1)
   553  			So(datastore.Put(ctx, b), ShouldBeNil)
   554  
   555  			// First save.
   556  			So(datastore.Get(ctx, b), ShouldBeNil)
   557  			ut0 := b.NextBackendSyncTime
   558  			parts := strings.Split(ut0, syncTimeSep)
   559  			So(parts, ShouldHaveLength, 4)
   560  			So(parts[3], ShouldEqual, fmt.Sprint(t0.Add(b.BackendSyncInterval).Truncate(time.Minute).Unix()))
   561  			So(ut0, ShouldEqual, "backend--project--0--1454472600")
   562  
   563  			// update soon after, NextBackendSyncTime unchanged.
   564  			b.Proto.UpdateTime = timestamppb.New(t0.Add(time.Second))
   565  			So(datastore.Put(ctx, b), ShouldBeNil)
   566  			So(datastore.Get(ctx, b), ShouldBeNil)
   567  			So(b.NextBackendSyncTime, ShouldEqual, ut0)
   568  			So(b.BackendSyncInterval, ShouldEqual, defaultBuildSyncInterval)
   569  
   570  			// update after 30sec, NextBackendSyncTime unchanged.
   571  			b.Proto.UpdateTime = timestamppb.New(t0.Add(40 * time.Second))
   572  			So(datastore.Put(ctx, b), ShouldBeNil)
   573  			So(datastore.Get(ctx, b), ShouldBeNil)
   574  			So(b.NextBackendSyncTime, ShouldBeGreaterThan, ut0)
   575  
   576  			// update after 2min, NextBackendSyncTime changed.
   577  			t1 := t0.Add(2 * time.Minute)
   578  			b.Proto.UpdateTime = timestamppb.New(t1)
   579  			So(datastore.Put(ctx, b), ShouldBeNil)
   580  			So(datastore.Get(ctx, b), ShouldBeNil)
   581  			So(b.NextBackendSyncTime, ShouldBeGreaterThan, ut0)
   582  			parts = strings.Split(b.NextBackendSyncTime, syncTimeSep)
   583  			So(parts, ShouldHaveLength, 4)
   584  			So(parts[3], ShouldEqual, fmt.Sprint(t1.Add(b.BackendSyncInterval).Truncate(time.Minute).Unix()))
   585  		})
   586  	})
   587  }