go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/gce/appengine/backend/queues_test.go (about)

     1  // Copyright 2018 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package backend
    16  
    17  import (
    18  	"context"
    19  	"math/rand"
    20  	"net/http"
    21  	"testing"
    22  	"time"
    23  
    24  	"google.golang.org/api/compute/v1"
    25  	"google.golang.org/genproto/googleapis/type/dayofweek"
    26  	"google.golang.org/protobuf/testing/protocmp"
    27  	"google.golang.org/protobuf/types/known/emptypb"
    28  
    29  	"go.chromium.org/luci/appengine/tq"
    30  	"go.chromium.org/luci/appengine/tq/tqtesting"
    31  	"go.chromium.org/luci/common/clock/testclock"
    32  	"go.chromium.org/luci/common/data/rand/mathrand"
    33  	"go.chromium.org/luci/gae/impl/memory"
    34  	"go.chromium.org/luci/gae/service/datastore"
    35  
    36  	"go.chromium.org/luci/gce/api/config/v1"
    37  	"go.chromium.org/luci/gce/api/projects/v1"
    38  	"go.chromium.org/luci/gce/api/tasks/v1"
    39  	"go.chromium.org/luci/gce/appengine/model"
    40  	"go.chromium.org/luci/gce/appengine/testing/roundtripper"
    41  
    42  	"github.com/google/go-cmp/cmp"
    43  	"github.com/google/go-cmp/cmp/cmpopts"
    44  	. "github.com/smartystreets/goconvey/convey"
    45  	. "go.chromium.org/luci/common/testing/assertions"
    46  )
    47  
    48  func TestQueues(t *testing.T) {
    49  	t.Parallel()
    50  
    51  	Convey("queues", t, func() {
    52  		dsp := &tq.Dispatcher{}
    53  		registerTasks(dsp)
    54  		rt := &roundtripper.JSONRoundTripper{}
    55  		gce, err := compute.New(&http.Client{Transport: rt})
    56  		So(err, ShouldBeNil)
    57  		c := withCompute(withDispatcher(memory.Use(context.Background()), dsp), ComputeService{Stable: gce})
    58  		datastore.GetTestable(c).AutoIndex(true)
    59  		datastore.GetTestable(c).Consistent(true)
    60  		tqt := tqtesting.GetTestable(c, dsp)
    61  		tqt.CreateQueues()
    62  
    63  		Convey("countVMs", func() {
    64  			Convey("invalid", func() {
    65  				Convey("nil", func() {
    66  					err := countVMs(c, nil)
    67  					So(err, ShouldErrLike, "unexpected payload")
    68  				})
    69  
    70  				Convey("empty", func() {
    71  					err := countVMs(c, &tasks.CountVMs{})
    72  					So(err, ShouldErrLike, "ID is required")
    73  				})
    74  			})
    75  
    76  			Convey("valid", func() {
    77  				err := countVMs(c, &tasks.CountVMs{
    78  					Id: "id",
    79  				})
    80  				So(err, ShouldBeNil)
    81  			})
    82  		})
    83  
    84  		Convey("createVM", func() {
    85  			c, _ = testclock.UseTime(c, testclock.TestTimeUTC)
    86  
    87  			Convey("invalid", func() {
    88  				Convey("nil", func() {
    89  					err := createVM(c, nil)
    90  					So(err, ShouldErrLike, "unexpected payload")
    91  				})
    92  
    93  				Convey("empty", func() {
    94  					err := createVM(c, &tasks.CreateVM{})
    95  					So(err, ShouldErrLike, "is required")
    96  				})
    97  
    98  				Convey("ID", func() {
    99  					err := createVM(c, &tasks.CreateVM{
   100  						Config: "config",
   101  					})
   102  					So(err, ShouldErrLike, "ID is required")
   103  				})
   104  
   105  				Convey("config", func() {
   106  					err := createVM(c, &tasks.CreateVM{
   107  						Id: "id",
   108  					})
   109  					So(err, ShouldErrLike, "config is required")
   110  				})
   111  			})
   112  
   113  			Convey("valid", func() {
   114  				Convey("nil", func() {
   115  					err := createVM(c, &tasks.CreateVM{
   116  						Id:     "id",
   117  						Index:  2,
   118  						Config: "config",
   119  					})
   120  					So(err, ShouldBeNil)
   121  					v := &model.VM{
   122  						ID: "id",
   123  					}
   124  					So(datastore.Get(c, v), ShouldBeNil)
   125  					So(v.Index, ShouldEqual, 2)
   126  					So(v.Config, ShouldEqual, "config")
   127  				})
   128  
   129  				Convey("empty", func() {
   130  					err := createVM(c, &tasks.CreateVM{
   131  						Id:         "id",
   132  						Attributes: &config.VM{},
   133  						Index:      2,
   134  						Config:     "config",
   135  					})
   136  					So(err, ShouldBeNil)
   137  					v := &model.VM{
   138  						ID: "id",
   139  					}
   140  					So(datastore.Get(c, v), ShouldBeNil)
   141  					So(v.Index, ShouldEqual, 2)
   142  				})
   143  
   144  				Convey("non-empty", func() {
   145  					c = mathrand.Set(c, rand.New(rand.NewSource(1)))
   146  					err := createVM(c, &tasks.CreateVM{
   147  						Id: "id",
   148  						Attributes: &config.VM{
   149  							Disk: []*config.Disk{
   150  								{
   151  									Image: "image",
   152  								},
   153  							},
   154  						},
   155  						Index:  2,
   156  						Config: "config",
   157  						Prefix: "prefix",
   158  					})
   159  					So(err, ShouldBeNil)
   160  					v := &model.VM{
   161  						ID: "id",
   162  					}
   163  					So(datastore.Get(c, v), ShouldBeNil)
   164  					So(cmp.Diff(v, &model.VM{
   165  						ID: "id",
   166  						Attributes: config.VM{
   167  							Disk: []*config.Disk{
   168  								{
   169  									Image: "image",
   170  								},
   171  							},
   172  						},
   173  						AttributesIndexed: []string{
   174  							"disk.image:image",
   175  						},
   176  						Config:     "config",
   177  						Configured: testclock.TestTimeUTC.Unix(),
   178  						Hostname:   "prefix-2-fpll",
   179  						Index:      2,
   180  						Prefix:     "prefix",
   181  					}, cmpopts.IgnoreUnexported(*v), protocmp.Transform()), ShouldBeEmpty)
   182  				})
   183  
   184  				Convey("not updated", func() {
   185  					datastore.Put(c, &model.VM{
   186  						ID: "id",
   187  						Attributes: config.VM{
   188  							Zone: "zone",
   189  						},
   190  						Drained: true,
   191  					})
   192  					err := createVM(c, &tasks.CreateVM{
   193  						Id: "id",
   194  						Attributes: &config.VM{
   195  							Project: "project",
   196  						},
   197  						Config: "config",
   198  						Index:  2,
   199  					})
   200  					So(err, ShouldBeNil)
   201  					v := &model.VM{
   202  						ID: "id",
   203  					}
   204  					So(datastore.Get(c, v), ShouldBeNil)
   205  					So(cmp.Diff(v, &model.VM{
   206  						ID: "id",
   207  						Attributes: config.VM{
   208  							Zone: "zone",
   209  						},
   210  						Drained: true,
   211  					}, cmpopts.IgnoreUnexported(*v), protocmp.Transform()), ShouldBeEmpty)
   212  				})
   213  
   214  				Convey("sets zone", func() {
   215  					err := createVM(c, &tasks.CreateVM{
   216  						Id: "id",
   217  						Attributes: &config.VM{
   218  							Disk: []*config.Disk{
   219  								{
   220  									Type: "{{.Zone}}/type",
   221  								},
   222  							},
   223  							MachineType: "{{.Zone}}/type",
   224  							Zone:        "zone",
   225  						},
   226  						Config: "config",
   227  						Index:  2,
   228  					})
   229  					So(err, ShouldBeNil)
   230  					v := &model.VM{
   231  						ID: "id",
   232  					}
   233  					So(datastore.Get(c, v), ShouldBeNil)
   234  					So(&v.Attributes, ShouldResembleProto, &config.VM{
   235  						Disk: []*config.Disk{
   236  							{
   237  								Type: "zone/type",
   238  							},
   239  						},
   240  						MachineType: "zone/type",
   241  						Zone:        "zone",
   242  					})
   243  				})
   244  			})
   245  		})
   246  
   247  		Convey("drainVM", func() {
   248  			Convey("invalid", func() {
   249  				Convey("config", func() {
   250  					err := drainVM(c, &model.VM{
   251  						ID: "id",
   252  					})
   253  					So(err, ShouldErrLike, "failed to fetch config")
   254  				})
   255  			})
   256  
   257  			Convey("valid", func() {
   258  				Convey("config", func() {
   259  					Convey("drained", func() {
   260  						datastore.Put(c, &model.Config{
   261  							ID: "config",
   262  							Config: &config.Config{
   263  								CurrentAmount: 2,
   264  							},
   265  						})
   266  						v := &model.VM{
   267  							ID:      "id",
   268  							Config:  "config",
   269  							Drained: true,
   270  						}
   271  						So(datastore.Put(c, v), ShouldBeNil)
   272  						So(drainVM(c, v), ShouldBeNil)
   273  						So(v.Drained, ShouldBeTrue)
   274  						So(datastore.Get(c, v), ShouldBeNil)
   275  						So(v.Drained, ShouldBeTrue)
   276  					})
   277  
   278  					Convey("deleted", func() {
   279  						v := &model.VM{
   280  							ID:     "id",
   281  							Config: "config",
   282  						}
   283  						So(datastore.Put(c, v), ShouldBeNil)
   284  						So(drainVM(c, v), ShouldBeNil)
   285  						So(v.Drained, ShouldBeTrue)
   286  						So(datastore.Get(c, v), ShouldBeNil)
   287  						So(v.Drained, ShouldBeTrue)
   288  					})
   289  
   290  					Convey("amount", func() {
   291  						Convey("unspecified", func() {
   292  							datastore.Put(c, &model.Config{
   293  								ID: "config",
   294  							})
   295  							v := &model.VM{
   296  								ID:     "id",
   297  								Config: "config",
   298  							}
   299  							So(datastore.Put(c, v), ShouldBeNil)
   300  							So(drainVM(c, v), ShouldBeNil)
   301  							So(v.Drained, ShouldBeTrue)
   302  							So(err, ShouldBeNil)
   303  							So(datastore.Get(c, v), ShouldBeNil)
   304  							So(v.Drained, ShouldBeTrue)
   305  						})
   306  
   307  						Convey("lesser", func() {
   308  							datastore.Put(c, &model.Config{
   309  								ID: "config",
   310  								Config: &config.Config{
   311  									CurrentAmount: 1,
   312  								},
   313  							})
   314  							v := &model.VM{
   315  								ID:     "id",
   316  								Config: "config",
   317  								Index:  2,
   318  							}
   319  							So(datastore.Put(c, v), ShouldBeNil)
   320  							So(drainVM(c, v), ShouldBeNil)
   321  							So(v.Drained, ShouldBeTrue)
   322  							So(datastore.Get(c, v), ShouldBeNil)
   323  							So(v.Drained, ShouldBeTrue)
   324  						})
   325  
   326  						Convey("equal", func() {
   327  							datastore.Put(c, &model.Config{
   328  								ID: "config",
   329  								Config: &config.Config{
   330  									CurrentAmount: 2,
   331  								},
   332  							})
   333  							v := &model.VM{
   334  								ID:     "id",
   335  								Config: "config",
   336  								Index:  2,
   337  							}
   338  							So(datastore.Put(c, v), ShouldBeNil)
   339  							So(drainVM(c, v), ShouldBeNil)
   340  							So(v.Drained, ShouldBeTrue)
   341  							So(datastore.Get(c, v), ShouldBeNil)
   342  							So(v.Drained, ShouldBeTrue)
   343  						})
   344  
   345  						Convey("greater", func() {
   346  							datastore.Put(c, &model.Config{
   347  								ID: "config",
   348  								Config: &config.Config{
   349  									CurrentAmount: 3,
   350  								},
   351  							})
   352  							v := &model.VM{
   353  								ID:     "id",
   354  								Config: "config",
   355  								Index:  2,
   356  							}
   357  							So(datastore.Put(c, v), ShouldBeNil)
   358  							So(drainVM(c, v), ShouldBeNil)
   359  							So(v.Drained, ShouldBeFalse)
   360  							So(datastore.Get(c, v), ShouldBeNil)
   361  							So(v.Drained, ShouldBeFalse)
   362  						})
   363  					})
   364  
   365  					Convey("DUTs", func() {
   366  						Convey("DUT is in config", func() {
   367  							datastore.Put(c, &model.Config{
   368  								ID: "config",
   369  								Config: &config.Config{
   370  									Duts: map[string]*emptypb.Empty{
   371  										"dut1": {},
   372  										"dut2": {},
   373  										"dut3": {},
   374  									},
   375  								},
   376  							})
   377  							v := &model.VM{
   378  								ID:     "id",
   379  								Config: "config",
   380  								DUT:    "dut1",
   381  							}
   382  							So(datastore.Put(c, v), ShouldBeNil)
   383  							So(drainVM(c, v), ShouldBeNil)
   384  							So(v.Drained, ShouldBeFalse)
   385  							So(datastore.Get(c, v), ShouldBeNil)
   386  							So(v.Drained, ShouldBeFalse)
   387  						})
   388  
   389  						Convey("DUT is not in config", func() {
   390  							datastore.Put(c, &model.Config{
   391  								ID: "config",
   392  								Config: &config.Config{
   393  									Duts: map[string]*emptypb.Empty{
   394  										"dut1": {},
   395  										"dut2": {},
   396  										"dut3": {},
   397  									},
   398  								},
   399  							})
   400  							v := &model.VM{
   401  								ID:     "id",
   402  								Config: "config",
   403  								DUT:    "dut4",
   404  							}
   405  							So(datastore.Put(c, v), ShouldBeNil)
   406  							So(drainVM(c, v), ShouldBeNil)
   407  							So(v.Drained, ShouldBeTrue)
   408  							So(datastore.Get(c, v), ShouldBeNil)
   409  							So(v.Drained, ShouldBeTrue)
   410  						})
   411  					})
   412  				})
   413  
   414  				Convey("deleted", func() {
   415  					v := &model.VM{
   416  						ID:     "id",
   417  						Config: "config",
   418  					}
   419  					So(drainVM(c, v), ShouldBeNil)
   420  					So(v.Drained, ShouldBeTrue)
   421  					So(datastore.Get(c, v), ShouldEqual, datastore.ErrNoSuchEntity)
   422  				})
   423  			})
   424  		})
   425  
   426  		Convey("expandConfig", func() {
   427  			Convey("invalid", func() {
   428  				Convey("nil", func() {
   429  					err := expandConfig(c, nil)
   430  					So(err, ShouldErrLike, "unexpected payload")
   431  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   432  				})
   433  
   434  				Convey("empty", func() {
   435  					err := expandConfig(c, &tasks.ExpandConfig{})
   436  					So(err, ShouldErrLike, "ID is required")
   437  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   438  				})
   439  
   440  				Convey("missing", func() {
   441  					err := expandConfig(c, &tasks.ExpandConfig{
   442  						Id: "id",
   443  					})
   444  					So(err, ShouldErrLike, "failed to fetch config")
   445  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   446  					cfg := &model.Config{
   447  						ID: "id",
   448  					}
   449  					So(datastore.Get(c, cfg), ShouldEqual, datastore.ErrNoSuchEntity)
   450  				})
   451  			})
   452  
   453  			Convey("valid", func() {
   454  				Convey("none", func() {
   455  					So(datastore.Put(c, &model.Config{
   456  						ID: "id",
   457  						Config: &config.Config{
   458  							Attributes: &config.VM{
   459  								Project: "project",
   460  							},
   461  							Prefix: "prefix",
   462  						},
   463  					}), ShouldBeNil)
   464  					err := expandConfig(c, &tasks.ExpandConfig{
   465  						Id: "id",
   466  					})
   467  					So(err, ShouldBeNil)
   468  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   469  					cfg := &model.Config{
   470  						ID: "id",
   471  					}
   472  					So(datastore.Get(c, cfg), ShouldBeNil)
   473  					So(cfg.Config.CurrentAmount, ShouldEqual, 0)
   474  				})
   475  
   476  				Convey("DUTs have priority", func() {
   477  					So(datastore.Put(c, &model.Config{
   478  						ID: "id",
   479  						Config: &config.Config{
   480  							Attributes: &config.VM{
   481  								Project: "project",
   482  							},
   483  							Amount: &config.Amount{
   484  								Min: 5,
   485  								Max: 6,
   486  							},
   487  							Prefix: "prefix",
   488  							Duts: map[string]*emptypb.Empty{
   489  								"dut1": {},
   490  								"dut2": {},
   491  								"dut3": {},
   492  							},
   493  						},
   494  					}), ShouldBeNil)
   495  					err := expandConfig(c, &tasks.ExpandConfig{
   496  						Id: "id",
   497  					})
   498  					So(err, ShouldBeNil)
   499  					So(tqt.GetScheduledTasks(), ShouldHaveLength, 3)
   500  					cfg := &model.Config{
   501  						ID: "id",
   502  					}
   503  					So(datastore.Get(c, cfg), ShouldBeNil)
   504  					So(cfg.Config.Duts, ShouldResembleProto, map[string]*emptypb.Empty{
   505  						"dut1": {},
   506  						"dut2": {},
   507  						"dut3": {},
   508  					})
   509  				})
   510  
   511  				Convey("schedule", func() {
   512  					So(datastore.Put(c, &model.Config{
   513  						ID: "id",
   514  						Config: &config.Config{
   515  							Attributes: &config.VM{
   516  								Project: "project",
   517  							},
   518  							Amount: &config.Amount{
   519  								Min: 2,
   520  								Max: 2,
   521  								Change: []*config.Schedule{
   522  									{
   523  										Min: 5,
   524  										Max: 5,
   525  										Length: &config.TimePeriod{
   526  											Time: &config.TimePeriod_Duration{
   527  												Duration: "1h",
   528  											},
   529  										},
   530  										Start: &config.TimeOfDay{
   531  											Day:  dayofweek.DayOfWeek_MONDAY,
   532  											Time: "1:00",
   533  										},
   534  									},
   535  								},
   536  							},
   537  							Prefix: "prefix",
   538  						},
   539  					}), ShouldBeNil)
   540  
   541  					Convey("default", func() {
   542  						now := time.Time{}
   543  						So(now.Weekday(), ShouldEqual, time.Monday)
   544  						c, _ = testclock.UseTime(c, now)
   545  						err := expandConfig(c, &tasks.ExpandConfig{
   546  							Id: "id",
   547  						})
   548  						So(err, ShouldBeNil)
   549  						So(tqt.GetScheduledTasks(), ShouldHaveLength, 2)
   550  						cfg := &model.Config{
   551  							ID: "id",
   552  						}
   553  						So(datastore.Get(c, cfg), ShouldBeNil)
   554  						So(cfg.Config.CurrentAmount, ShouldEqual, 2)
   555  					})
   556  
   557  					Convey("scheduled", func() {
   558  						now := time.Time{}.Add(time.Hour)
   559  						So(now.Weekday(), ShouldEqual, time.Monday)
   560  						So(now.Hour(), ShouldEqual, 1)
   561  						c, _ = testclock.UseTime(c, now)
   562  						err := expandConfig(c, &tasks.ExpandConfig{
   563  							Id: "id",
   564  						})
   565  						So(err, ShouldBeNil)
   566  						So(tqt.GetScheduledTasks(), ShouldHaveLength, 5)
   567  						cfg := &model.Config{
   568  							ID: "id",
   569  						}
   570  						So(datastore.Get(c, cfg), ShouldBeNil)
   571  						So(cfg.Config.CurrentAmount, ShouldEqual, 5)
   572  					})
   573  				})
   574  			})
   575  		})
   576  
   577  		Convey("createTasksPerAmount", func() {
   578  			Convey("invalid", func() {
   579  				Convey("config.Duts is not empty", func() {
   580  					vms := []*model.VM{}
   581  					m := &model.Config{
   582  						ID: "id",
   583  						Config: &config.Config{
   584  							Attributes: &config.VM{
   585  								Project: "project",
   586  							},
   587  							CurrentAmount: 3,
   588  							Duts: map[string]*emptypb.Empty{
   589  								"dut1": {},
   590  								"dut2": {},
   591  							},
   592  							Prefix: "prefix",
   593  						},
   594  					}
   595  					n := time.Now()
   596  					t, err := createTasksPerAmount(c, vms, m, n)
   597  					So(err, ShouldErrLike, "config.Duts should be empty")
   598  					So(t, ShouldBeEmpty)
   599  				})
   600  			})
   601  
   602  			Convey("valid", func() {
   603  				Convey("default", func() {
   604  					vms := []*model.VM{}
   605  					m := &model.Config{
   606  						ID: "id",
   607  						Config: &config.Config{
   608  							Attributes: &config.VM{
   609  								Project: "project",
   610  							},
   611  							CurrentAmount: 3,
   612  							Prefix:        "prefix",
   613  						},
   614  					}
   615  					n := time.Now()
   616  					t, err := createTasksPerAmount(c, vms, m, n)
   617  					So(err, ShouldBeNil)
   618  					So(len(t), ShouldEqual, 3)
   619  				})
   620  
   621  				Convey("default - skip existing vms", func() {
   622  					vms := []*model.VM{
   623  						{
   624  							ID:     "prefix-1",
   625  							Config: "id",
   626  							Prefix: "prefix",
   627  						},
   628  					}
   629  					m := &model.Config{
   630  						ID: "id",
   631  						Config: &config.Config{
   632  							Attributes: &config.VM{
   633  								Project: "project",
   634  							},
   635  							CurrentAmount: 3,
   636  							Prefix:        "prefix",
   637  						},
   638  					}
   639  					n := time.Now()
   640  					t, err := createTasksPerAmount(c, vms, m, n)
   641  					So(err, ShouldBeNil)
   642  					So(t, ShouldHaveLength, 2)
   643  				})
   644  
   645  				Convey("default - skip all vms", func() {
   646  					vms := []*model.VM{
   647  						{
   648  							ID:     "prefix-0",
   649  							Config: "id",
   650  							Prefix: "prefix",
   651  						},
   652  						{
   653  							ID:     "prefix-1",
   654  							Config: "id",
   655  							Prefix: "prefix",
   656  						},
   657  						{
   658  							ID:     "prefix-2",
   659  							Config: "id",
   660  							Prefix: "prefix",
   661  						},
   662  					}
   663  					m := &model.Config{
   664  						ID: "id",
   665  						Config: &config.Config{
   666  							Attributes: &config.VM{
   667  								Project: "project",
   668  							},
   669  							CurrentAmount: 3,
   670  							Prefix:        "prefix",
   671  						},
   672  					}
   673  					n := time.Now()
   674  					t, err := createTasksPerAmount(c, vms, m, n)
   675  					So(err, ShouldBeNil)
   676  					So(t, ShouldHaveLength, 0)
   677  				})
   678  			})
   679  		})
   680  
   681  		Convey("createTasksPerDUT", func() {
   682  			Convey("invalid", func() {
   683  				Convey("config.Duts is nil", func() {
   684  					vms := []*model.VM{}
   685  					m := &model.Config{
   686  						ID: "id",
   687  						Config: &config.Config{
   688  							Attributes: &config.VM{
   689  								Project: "project",
   690  							},
   691  							Prefix: "prefix",
   692  						},
   693  					}
   694  					n := time.Now()
   695  					t, err := createTasksPerDUT(c, vms, m, n)
   696  					So(err, ShouldErrLike, "config.DUTs cannot be empty")
   697  					So(t, ShouldBeEmpty)
   698  				})
   699  				Convey("config.Duts is empty", func() {
   700  					vms := []*model.VM{}
   701  					m := &model.Config{
   702  						ID: "id",
   703  						Config: &config.Config{
   704  							Attributes: &config.VM{
   705  								Project: "project",
   706  							},
   707  							Duts:   map[string]*emptypb.Empty{},
   708  							Prefix: "prefix",
   709  						},
   710  					}
   711  					n := time.Now()
   712  					t, err := createTasksPerDUT(c, vms, m, n)
   713  					So(err, ShouldErrLike, "config.DUTs cannot be empty")
   714  					So(t, ShouldBeEmpty)
   715  				})
   716  			})
   717  		})
   718  
   719  		Convey("valid", func() {
   720  			Convey("default", func() {
   721  				vms := []*model.VM{}
   722  				m := &model.Config{
   723  					ID: "id",
   724  					Config: &config.Config{
   725  						Attributes: &config.VM{
   726  							Project: "project",
   727  						},
   728  						Duts: map[string]*emptypb.Empty{
   729  							"dut1": {},
   730  							"dut2": {},
   731  							"dut3": {},
   732  						},
   733  						Prefix: "prefix",
   734  					},
   735  				}
   736  				n := time.Now()
   737  				t, err := createTasksPerDUT(c, vms, m, n)
   738  				So(err, ShouldBeNil)
   739  				So(len(t), ShouldEqual, 3)
   740  			})
   741  
   742  			Convey("dispatched task contain DUT info", func() {
   743  				vms := []*model.VM{}
   744  				m := &model.Config{
   745  					ID: "id",
   746  					Config: &config.Config{
   747  						Attributes: &config.VM{
   748  							Project: "project",
   749  						},
   750  						Duts: map[string]*emptypb.Empty{
   751  							"dut1": {},
   752  						},
   753  						Prefix: "prefix",
   754  					},
   755  				}
   756  				n := time.Now()
   757  				t, err := createTasksPerDUT(c, vms, m, n)
   758  				So(err, ShouldBeNil)
   759  				vm := t[0].Payload.(*tasks.CreateVM)
   760  				So(vm.DUT, ShouldEqual, "dut1")
   761  			})
   762  		})
   763  
   764  		Convey("reportQuota", func() {
   765  			Convey("invalid", func() {
   766  				Convey("nil", func() {
   767  					err := reportQuota(c, nil)
   768  					So(err, ShouldErrLike, "unexpected payload")
   769  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   770  				})
   771  
   772  				Convey("empty", func() {
   773  					err := reportQuota(c, &tasks.ReportQuota{})
   774  					So(err, ShouldErrLike, "ID is required")
   775  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   776  				})
   777  
   778  				Convey("missing", func() {
   779  					err := reportQuota(c, &tasks.ReportQuota{
   780  						Id: "id",
   781  					})
   782  					So(err, ShouldErrLike, "failed to fetch project")
   783  					So(tqt.GetScheduledTasks(), ShouldBeEmpty)
   784  				})
   785  			})
   786  
   787  			Convey("valid", func() {
   788  				rt.Handler = func(req any) (int, any) {
   789  					return http.StatusOK, &compute.RegionList{
   790  						Items: []*compute.Region{
   791  							{
   792  								Name: "ignore",
   793  							},
   794  							{
   795  								Name: "region",
   796  								Quotas: []*compute.Quota{
   797  									{
   798  										Limit:  100.0,
   799  										Metric: "ignore",
   800  										Usage:  0.0,
   801  									},
   802  									{
   803  										Limit:  100.0,
   804  										Metric: "metric",
   805  										Usage:  25.0,
   806  									},
   807  								},
   808  							},
   809  						},
   810  					}
   811  				}
   812  				datastore.Put(c, &model.Project{
   813  					ID: "id",
   814  					Config: &projects.Config{
   815  						Metric:  []string{"metric"},
   816  						Project: "project",
   817  						Region:  []string{"region"},
   818  					},
   819  				})
   820  				err := reportQuota(c, &tasks.ReportQuota{
   821  					Id: "id",
   822  				})
   823  				So(err, ShouldBeNil)
   824  			})
   825  		})
   826  
   827  		Convey("getUniqueID", func() {
   828  			c, _ = testclock.UseTime(c, testclock.TestRecentTimeUTC)
   829  			c = mathrand.Set(c, rand.New(rand.NewSource(1)))
   830  			id := getUniqueID(c, "prefix")
   831  			So(id, ShouldEqual, "prefix-1454472306000-fpll")
   832  		})
   833  	})
   834  }