github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/engine/buildcontroller_test.go (about)

     1  package engine
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"runtime"
     7  	"strings"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/stretchr/testify/assert"
    12  	"github.com/stretchr/testify/require"
    13  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    14  	"k8s.io/apimachinery/pkg/types"
    15  
    16  	"github.com/tilt-dev/tilt/internal/container"
    17  	"github.com/tilt-dev/tilt/internal/controllers/apis/uibutton"
    18  	"github.com/tilt-dev/tilt/internal/k8s"
    19  	"github.com/tilt-dev/tilt/internal/k8s/testyaml"
    20  	"github.com/tilt-dev/tilt/internal/store"
    21  	"github.com/tilt-dev/tilt/internal/testutils/configmap"
    22  	"github.com/tilt-dev/tilt/internal/testutils/manifestbuilder"
    23  	"github.com/tilt-dev/tilt/internal/testutils/podbuilder"
    24  	"github.com/tilt-dev/tilt/internal/watch"
    25  	"github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1"
    26  	"github.com/tilt-dev/tilt/pkg/model"
    27  )
    28  
    29  func TestBuildControllerLocalResource(t *testing.T) {
    30  	f := newTestFixture(t)
    31  
    32  	dep := f.JoinPath("stuff.json")
    33  	manifest := manifestbuilder.New(f, "local").
    34  		WithLocalResource("echo beep boop", []string{dep}).
    35  		Build()
    36  	f.Start([]model.Manifest{manifest})
    37  
    38  	call := f.nextCallComplete()
    39  	lt := manifest.LocalTarget()
    40  	assert.Equal(t, lt, call.local())
    41  
    42  	f.fsWatcher.Events <- watch.NewFileEvent(dep)
    43  
    44  	call = f.nextCallComplete()
    45  	assert.Equal(t, lt, call.local())
    46  
    47  	f.WaitUntilManifestState("local target manifest state not updated", "local", func(ms store.ManifestState) bool {
    48  		lrs := ms.RuntimeState.(store.LocalRuntimeState)
    49  		return !lrs.LastReadyOrSucceededTime.IsZero() && lrs.RuntimeStatus() == v1alpha1.RuntimeStatusNotApplicable
    50  	})
    51  
    52  	err := f.Stop()
    53  	assert.NoError(t, err)
    54  	f.assertAllBuildsConsumed()
    55  }
    56  
    57  func TestBuildControllerManualTriggerBuildReasonInit(t *testing.T) {
    58  	for _, tc := range []struct {
    59  		name        string
    60  		triggerMode model.TriggerMode
    61  	}{
    62  		{"fully manual", model.TriggerModeManual},
    63  		{"manual with auto init", model.TriggerModeManualWithAutoInit},
    64  	} {
    65  		t.Run(tc.name, func(t *testing.T) {
    66  			f := newTestFixture(t)
    67  			mName := model.ManifestName("foobar")
    68  			manifest := f.newManifest(mName.String()).WithTriggerMode(tc.triggerMode)
    69  			manifests := []model.Manifest{manifest}
    70  			f.Start(manifests)
    71  
    72  			// make sure there's a first build
    73  			if !manifest.TriggerMode.AutoInitial() {
    74  				f.store.Dispatch(store.AppendToTriggerQueueAction{Name: mName})
    75  			}
    76  
    77  			f.nextCallComplete()
    78  
    79  			f.withManifestState(mName, func(ms store.ManifestState) {
    80  				require.Equal(t, tc.triggerMode.AutoInitial(), ms.LastBuild().Reason.Has(model.BuildReasonFlagInit))
    81  			})
    82  		})
    83  	}
    84  }
    85  
    86  func TestTriggerModes(t *testing.T) {
    87  	for _, tc := range []struct {
    88  		name                       string
    89  		triggerMode                model.TriggerMode
    90  		expectInitialBuild         bool
    91  		expectBuildWhenFilesChange bool
    92  	}{
    93  		{name: "fully auto", triggerMode: model.TriggerModeAuto, expectInitialBuild: true, expectBuildWhenFilesChange: true},
    94  		{name: "auto with manual init", triggerMode: model.TriggerModeAutoWithManualInit, expectInitialBuild: false, expectBuildWhenFilesChange: true},
    95  		{name: "manual with auto init", triggerMode: model.TriggerModeManualWithAutoInit, expectInitialBuild: true, expectBuildWhenFilesChange: false},
    96  		{name: "fully manual", triggerMode: model.TriggerModeManual, expectInitialBuild: false, expectBuildWhenFilesChange: false},
    97  	} {
    98  		t.Run(tc.name, func(t *testing.T) {
    99  			f := newTestFixture(t)
   100  
   101  			manifest := f.simpleManifestWithTriggerMode("foobar", tc.triggerMode)
   102  			manifests := []model.Manifest{manifest}
   103  			f.Start(manifests)
   104  
   105  			// basic check of trigger mode properties
   106  			assert.Equal(t, tc.expectInitialBuild, tc.triggerMode.AutoInitial())
   107  			assert.Equal(t, tc.expectBuildWhenFilesChange, tc.triggerMode.AutoOnChange())
   108  
   109  			// if we expect an initial build from the manifest, wait for it to complete
   110  			if tc.expectInitialBuild {
   111  				f.nextCallComplete("initial build")
   112  			}
   113  
   114  			f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("main.go"))
   115  			f.WaitUntil("pending change appears", func(st store.EngineState) bool {
   116  				return len(st.BuildStatus(manifest.ImageTargetAt(0).ID()).PendingFileChanges) >= 1
   117  			})
   118  
   119  			if !tc.expectBuildWhenFilesChange {
   120  				f.assertNoCall("even tho there are pending changes, manual manifest shouldn't build w/o explicit trigger")
   121  				return
   122  			}
   123  
   124  			call := f.nextCallComplete("build after file change")
   125  			state := call.oneImageState()
   126  			assert.Equal(t, []string{f.JoinPath("main.go")}, state.FilesChanged())
   127  		})
   128  	}
   129  }
   130  
   131  func TestBuildControllerImageBuildTrigger(t *testing.T) {
   132  	for _, tc := range []struct {
   133  		name               string
   134  		triggerMode        model.TriggerMode
   135  		filesChanged       bool
   136  		expectedImageBuild bool
   137  	}{
   138  		{name: "fully manual with change", triggerMode: model.TriggerModeManual, filesChanged: true, expectedImageBuild: false},
   139  		{name: "manual with auto init with change", triggerMode: model.TriggerModeManualWithAutoInit, filesChanged: true, expectedImageBuild: false},
   140  		{name: "fully manual without change", triggerMode: model.TriggerModeManual, filesChanged: false, expectedImageBuild: true},
   141  		{name: "manual with auto init without change", triggerMode: model.TriggerModeManualWithAutoInit, filesChanged: false, expectedImageBuild: true},
   142  		{name: "fully auto without change", triggerMode: model.TriggerModeAuto, filesChanged: false, expectedImageBuild: true},
   143  		{name: "auto with manual init without change", triggerMode: model.TriggerModeAutoWithManualInit, filesChanged: false, expectedImageBuild: true},
   144  	} {
   145  		t.Run(tc.name, func(t *testing.T) {
   146  			f := newTestFixture(t)
   147  			mName := model.ManifestName("foobar")
   148  
   149  			manifest := f.simpleManifestWithTriggerMode(mName, tc.triggerMode)
   150  			manifests := []model.Manifest{manifest}
   151  			f.Start(manifests)
   152  
   153  			// if we expect an initial build from the manifest, wait for it to complete
   154  			if manifest.TriggerMode.AutoInitial() {
   155  				f.nextCallComplete()
   156  			}
   157  
   158  			expectedFiles := []string{}
   159  			if tc.filesChanged {
   160  				expectedFiles = append(expectedFiles, f.JoinPath("main.go"))
   161  				f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("main.go"))
   162  			}
   163  			f.WaitUntil("pending change appears", func(st store.EngineState) bool {
   164  				return len(st.BuildStatus(manifest.ImageTargetAt(0).ID()).PendingFileChanges) >= len(expectedFiles)
   165  			})
   166  
   167  			if manifest.TriggerMode.AutoOnChange() {
   168  				f.assertNoCall("even tho there are pending changes, manual manifest shouldn't build w/o explicit trigger")
   169  			}
   170  
   171  			f.store.Dispatch(store.AppendToTriggerQueueAction{Name: mName})
   172  			call := f.nextCallComplete()
   173  			state := call.oneImageState()
   174  			assert.Equal(t, expectedFiles, state.FilesChanged())
   175  			assert.Equal(t, tc.expectedImageBuild, state.FullBuildTriggered)
   176  
   177  			f.WaitUntil("manifest removed from queue", func(st store.EngineState) bool {
   178  				for _, mn := range st.TriggerQueue {
   179  					if mn == mName {
   180  						return false
   181  					}
   182  				}
   183  				return true
   184  			})
   185  		})
   186  	}
   187  }
   188  
   189  func TestBuildQueueOrdering(t *testing.T) {
   190  	f := newTestFixture(t)
   191  
   192  	m1 := f.newManifestWithRef("manifest1", container.MustParseNamed("manifest1")).
   193  		WithTriggerMode(model.TriggerModeManualWithAutoInit)
   194  	m2 := f.newManifestWithRef("manifest2", container.MustParseNamed("manifest2")).
   195  		WithTriggerMode(model.TriggerModeManualWithAutoInit)
   196  	m3 := f.newManifestWithRef("manifest3", container.MustParseNamed("manifest3")).
   197  		WithTriggerMode(model.TriggerModeManual)
   198  	m4 := f.newManifestWithRef("manifest4", container.MustParseNamed("manifest4")).
   199  		WithTriggerMode(model.TriggerModeManual)
   200  
   201  	// attach to state in different order than we plan to trigger them
   202  	manifests := []model.Manifest{m4, m2, m3, m1}
   203  	f.Start(manifests)
   204  
   205  	expectedInitialBuildCount := 0
   206  	for _, m := range manifests {
   207  		if m.TriggerMode.AutoInitial() {
   208  			expectedInitialBuildCount++
   209  			f.nextCall()
   210  		}
   211  	}
   212  
   213  	f.waitForCompletedBuildCount(expectedInitialBuildCount)
   214  
   215  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("main.go"))
   216  	f.WaitUntil("pending change appears", func(st store.EngineState) bool {
   217  		return len(st.BuildStatus(m1.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   218  			len(st.BuildStatus(m2.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   219  			len(st.BuildStatus(m3.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   220  			len(st.BuildStatus(m4.ImageTargetAt(0).ID()).PendingFileChanges) > 0
   221  	})
   222  	f.assertNoCall("even tho there are pending changes, manual manifest shouldn't build w/o explicit trigger")
   223  
   224  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest1"})
   225  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest2"})
   226  	time.Sleep(10 * time.Millisecond)
   227  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest3"})
   228  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest4"})
   229  
   230  	for i := range manifests {
   231  		expName := fmt.Sprintf("manifest%d", i+1)
   232  		call := f.nextCall()
   233  		imgID := call.firstImgTarg().ID().String()
   234  		if assert.True(t, strings.HasSuffix(imgID, expName),
   235  			"expected to get manifest '%s' but instead got: '%s' (checking suffix for manifest name)", expName, imgID) {
   236  			assert.Equal(t, []string{f.JoinPath("main.go")}, call.oneImageState().FilesChanged(),
   237  				"for manifest '%s", expName)
   238  		}
   239  	}
   240  	f.waitForCompletedBuildCount(expectedInitialBuildCount + len(manifests))
   241  }
   242  
   243  func TestBuildQueueAndAutobuildOrdering(t *testing.T) {
   244  	f := newTestFixture(t)
   245  
   246  	// changes to this dir. will register with our manual manifests
   247  	dirManual := f.JoinPath("dirManual/")
   248  	// changes to this dir. will register with our automatic manifests
   249  	dirAuto := f.JoinPath("dirAuto/")
   250  
   251  	m1 := f.newDockerBuildManifestWithBuildPath("manifest1", dirManual).WithTriggerMode(model.TriggerModeManualWithAutoInit)
   252  	m2 := f.newDockerBuildManifestWithBuildPath("manifest2", dirManual).WithTriggerMode(model.TriggerModeManualWithAutoInit)
   253  	m3 := f.newDockerBuildManifestWithBuildPath("manifest3", dirManual).WithTriggerMode(model.TriggerModeManual)
   254  	m4 := f.newDockerBuildManifestWithBuildPath("manifest4", dirManual).WithTriggerMode(model.TriggerModeManual)
   255  	m5 := f.newDockerBuildManifestWithBuildPath("manifest5", dirAuto).WithTriggerMode(model.TriggerModeAuto)
   256  
   257  	// attach to state in different order than we plan to trigger them
   258  	manifests := []model.Manifest{m5, m4, m2, m3, m1}
   259  	f.Start(manifests)
   260  
   261  	expectedInitialBuildCount := 0
   262  	for _, m := range manifests {
   263  		if m.TriggerMode.AutoInitial() {
   264  			expectedInitialBuildCount++
   265  			f.nextCall()
   266  		}
   267  	}
   268  
   269  	f.waitForCompletedBuildCount(expectedInitialBuildCount)
   270  
   271  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("dirManual/main.go"))
   272  	f.WaitUntil("pending change appears", func(st store.EngineState) bool {
   273  		return len(st.BuildStatus(m1.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   274  			len(st.BuildStatus(m2.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   275  			len(st.BuildStatus(m3.ImageTargetAt(0).ID()).PendingFileChanges) > 0 &&
   276  			len(st.BuildStatus(m4.ImageTargetAt(0).ID()).PendingFileChanges) > 0
   277  	})
   278  	f.assertNoCall("even tho there are pending changes, manual manifest shouldn't build w/o explicit trigger")
   279  
   280  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest1"})
   281  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest2"})
   282  	// make our one auto-trigger manifest build - should be evaluated LAST, after
   283  	// all the manual manifests waiting in the queue
   284  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("dirAuto/main.go"))
   285  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest3"})
   286  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: "manifest4"})
   287  
   288  	for i := range manifests {
   289  		call := f.nextCall()
   290  		imgTargID := call.firstImgTarg().ID().String()
   291  		expectSuffix := fmt.Sprintf("manifest%d", i+1)
   292  		assert.True(t, strings.HasSuffix(imgTargID, expectSuffix), "expect this call to have image target ...%s (got: %s)", expectSuffix, imgTargID)
   293  
   294  		if i < 4 {
   295  			assert.Equal(t, []string{f.JoinPath("dirManual/main.go")}, call.oneImageState().FilesChanged(), "for manifest %d", i+1)
   296  		} else {
   297  			// the automatic manifest
   298  			assert.Equal(t, []string{f.JoinPath("dirAuto/main.go")}, call.oneImageState().FilesChanged(), "for manifest %d", i+1)
   299  		}
   300  	}
   301  	f.waitForCompletedBuildCount(len(manifests) + expectedInitialBuildCount)
   302  }
   303  
   304  // any manifests without image targets should be deployed before any manifests WITH image targets
   305  func TestBuildControllerNoBuildManifestsFirst(t *testing.T) {
   306  	f := newTestFixture(t)
   307  
   308  	manifests := make([]model.Manifest, 10)
   309  	for i := 0; i < 10; i++ {
   310  		manifests[i] = f.newManifest(fmt.Sprintf("built%d", i+1))
   311  	}
   312  
   313  	for _, i := range []int{3, 7, 8} {
   314  		manifests[i] = manifestbuilder.New(f, model.ManifestName(fmt.Sprintf("unbuilt%d", i+1))).
   315  			WithK8sYAML(SanchoYAML).
   316  			Build()
   317  	}
   318  	f.Start(manifests)
   319  
   320  	var observedBuildOrder []string
   321  	for i := 0; i < len(manifests); i++ {
   322  		call := f.nextCall()
   323  		observedBuildOrder = append(observedBuildOrder, call.k8s().Name.String())
   324  	}
   325  
   326  	// throwing a bunch of elements at it to increase confidence we maintain order between built and unbuilt
   327  	// this might miss bugs since we might just get these elements back in the right order via luck
   328  	expectedBuildOrder := []string{
   329  		"unbuilt4",
   330  		"unbuilt8",
   331  		"unbuilt9",
   332  		"built1",
   333  		"built2",
   334  		"built3",
   335  		"built5",
   336  		"built6",
   337  		"built7",
   338  		"built10",
   339  	}
   340  	assert.Equal(t, expectedBuildOrder, observedBuildOrder)
   341  }
   342  
   343  func TestBuildControllerUnresourcedYAMLFirst(t *testing.T) {
   344  	f := newTestFixture(t)
   345  
   346  	manifests := []model.Manifest{
   347  		f.newManifest("built1"),
   348  		f.newManifest("built2"),
   349  		f.newManifest("built3"),
   350  		f.newManifest("built4"),
   351  	}
   352  
   353  	manifests = append(manifests, manifestbuilder.New(f, model.UnresourcedYAMLManifestName).
   354  		WithK8sYAML(testyaml.SecretYaml).Build())
   355  	f.Start(manifests)
   356  
   357  	var observedBuildOrder []string
   358  	for i := 0; i < len(manifests); i++ {
   359  		call := f.nextCall()
   360  		observedBuildOrder = append(observedBuildOrder, call.k8s().Name.String())
   361  	}
   362  
   363  	expectedBuildOrder := []string{
   364  		model.UnresourcedYAMLManifestName.String(),
   365  		"built1",
   366  		"built2",
   367  		"built3",
   368  		"built4",
   369  	}
   370  	assert.Equal(t, expectedBuildOrder, observedBuildOrder)
   371  }
   372  
   373  func TestBuildControllerRespectDockerComposeOrder(t *testing.T) {
   374  	f := newTestFixture(t)
   375  
   376  	sancho := NewSanchoLiveUpdateDCManifest(f)
   377  	redis := manifestbuilder.New(f, "redis").WithDockerCompose().Build()
   378  	donQuixote := manifestbuilder.New(f, "don-quixote").WithDockerCompose().Build()
   379  	manifests := []model.Manifest{redis, sancho, donQuixote}
   380  	f.Start(manifests)
   381  
   382  	var observedBuildOrder []string
   383  	for i := 0; i < len(manifests); i++ {
   384  		call := f.nextCall()
   385  		observedBuildOrder = append(observedBuildOrder, call.dc().Name.String())
   386  	}
   387  
   388  	// If these were Kubernetes resources, we would try to deploy don-quixote
   389  	// before sancho, because it doesn't have an image build.
   390  	//
   391  	// But this would be wrong, because Docker Compose has stricter ordering requirements, see:
   392  	// https://docs.docker.com/compose/startup-order/
   393  	expectedBuildOrder := []string{
   394  		"redis",
   395  		"sancho",
   396  		"don-quixote",
   397  	}
   398  	assert.Equal(t, expectedBuildOrder, observedBuildOrder)
   399  }
   400  
   401  func TestBuildControllerLocalResourcesBeforeClusterResources(t *testing.T) {
   402  	f := newTestFixture(t)
   403  
   404  	manifests := []model.Manifest{
   405  		f.newManifest("clusterBuilt1"),
   406  		f.newManifest("clusterBuilt2"),
   407  		manifestbuilder.New(f, "clusterUnbuilt").
   408  			WithK8sYAML(SanchoYAML).Build(),
   409  		manifestbuilder.New(f, "local1").
   410  			WithLocalResource("echo local1", nil).Build(),
   411  		f.newManifest("clusterBuilt3"),
   412  		manifestbuilder.New(f, "local2").
   413  			WithLocalResource("echo local2", nil).Build(),
   414  	}
   415  
   416  	manifests = append(manifests, manifestbuilder.New(f, model.UnresourcedYAMLManifestName).
   417  		WithK8sYAML(testyaml.SecretYaml).Build())
   418  	f.Start(manifests)
   419  
   420  	var observedBuildOrder []string
   421  	for i := 0; i < len(manifests); i++ {
   422  		call := f.nextCall()
   423  		if !call.k8s().Empty() {
   424  			observedBuildOrder = append(observedBuildOrder, call.k8s().Name.String())
   425  			continue
   426  		}
   427  		observedBuildOrder = append(observedBuildOrder, call.local().Name.String())
   428  	}
   429  
   430  	expectedBuildOrder := []string{
   431  		"local1",
   432  		"local2",
   433  		model.UnresourcedYAMLManifestName.String(),
   434  		"clusterUnbuilt",
   435  		"clusterBuilt1",
   436  		"clusterBuilt2",
   437  		"clusterBuilt3",
   438  	}
   439  	assert.Equal(t, expectedBuildOrder, observedBuildOrder)
   440  }
   441  
   442  func TestBuildControllerResourceDeps(t *testing.T) {
   443  	f := newTestFixture(t)
   444  
   445  	depGraph := map[string][]string{
   446  		"a": {"e"},
   447  		"b": {"e"},
   448  		"c": {"d", "g"},
   449  		"d": {},
   450  		"e": {"d", "f"},
   451  		"f": {"c"},
   452  		"g": {},
   453  	}
   454  
   455  	var manifests []model.Manifest
   456  	podBuilders := make(map[string]podbuilder.PodBuilder)
   457  	for name, deps := range depGraph {
   458  		m := f.newManifest(name)
   459  		for _, dep := range deps {
   460  			m.ResourceDependencies = append(m.ResourceDependencies, model.ManifestName(dep))
   461  		}
   462  		manifests = append(manifests, m)
   463  		podBuilders[name] = f.registerForDeployer(m)
   464  	}
   465  
   466  	f.Start(manifests)
   467  
   468  	var observedOrder []string
   469  	for i := range manifests {
   470  		call := f.nextCall("%dth build. have built: %v", i, observedOrder)
   471  		name := call.k8s().Name.String()
   472  		observedOrder = append(observedOrder, name)
   473  		f.podEvent(podBuilders[name].WithContainerReady(true).Build())
   474  	}
   475  
   476  	var expectedManifests []string
   477  	for name := range depGraph {
   478  		expectedManifests = append(expectedManifests, name)
   479  	}
   480  
   481  	// make sure everything built
   482  	require.ElementsMatch(t, expectedManifests, observedOrder)
   483  
   484  	buildIndexes := make(map[string]int)
   485  	for i, n := range observedOrder {
   486  		buildIndexes[n] = i
   487  	}
   488  
   489  	// make sure it happened in an acceptable order
   490  	for name, deps := range depGraph {
   491  		for _, dep := range deps {
   492  			require.Truef(t, buildIndexes[name] > buildIndexes[dep], "%s built before %s, contrary to resource deps", name, dep)
   493  		}
   494  	}
   495  }
   496  
   497  // normally, local builds go before k8s builds
   498  // if the local build depends on the k8s build, the k8s build should go first
   499  func TestBuildControllerResourceDepTrumpsLocalResourcePriority(t *testing.T) {
   500  	f := newTestFixture(t)
   501  
   502  	k8sManifest := f.newManifest("foo")
   503  	pb := f.registerForDeployer(k8sManifest)
   504  	localManifest := manifestbuilder.New(f, "bar").
   505  		WithLocalResource("echo bar", nil).
   506  		WithResourceDeps("foo").Build()
   507  	manifests := []model.Manifest{localManifest, k8sManifest}
   508  	f.Start(manifests)
   509  
   510  	var observedBuildOrder []string
   511  	for i := 0; i < len(manifests); i++ {
   512  		call := f.nextCall()
   513  		if !call.k8s().Empty() {
   514  			observedBuildOrder = append(observedBuildOrder, call.k8s().Name.String())
   515  			pb = pb.WithContainerReady(true)
   516  			f.podEvent(pb.Build())
   517  			continue
   518  		}
   519  		observedBuildOrder = append(observedBuildOrder, call.local().Name.String())
   520  	}
   521  
   522  	expectedBuildOrder := []string{"foo", "bar"}
   523  	assert.Equal(t, expectedBuildOrder, observedBuildOrder)
   524  }
   525  
   526  // bar depends on foo, we build foo three times before marking it ready, and make sure bar waits
   527  func TestBuildControllerResourceDepTrumpsInitialBuild(t *testing.T) {
   528  	f := newTestFixture(t)
   529  
   530  	foo := manifestbuilder.New(f, "foo").
   531  		WithLocalResource("foo cmd", []string{f.JoinPath("foo")}).
   532  		Build()
   533  	bar := manifestbuilder.New(f, "bar").
   534  		WithLocalResource("bar cmd", []string{f.JoinPath("bar")}).
   535  		WithResourceDeps("foo").
   536  		Build()
   537  	manifests := []model.Manifest{foo, bar}
   538  	f.SetNextBuildError(errors.New("failure"))
   539  	f.Start(manifests)
   540  
   541  	call := f.nextCall()
   542  	require.Equal(t, "foo", call.local().Name.String())
   543  
   544  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("foo", "main.go"))
   545  	f.SetNextBuildError(errors.New("failure"))
   546  	call = f.nextCall()
   547  	require.Equal(t, "foo", call.local().Name.String())
   548  
   549  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("foo", "main.go"))
   550  	call = f.nextCall()
   551  	require.Equal(t, "foo", call.local().Name.String())
   552  
   553  	// now that the foo build has succeeded, bar should get queued
   554  	call = f.nextCall()
   555  	require.Equal(t, "bar", call.local().Name.String())
   556  }
   557  
   558  // bar depends on foo. make sure bar waits on foo even as foo fails
   559  func TestBuildControllerResourceDepTrumpsPendingBuild(t *testing.T) {
   560  	f := newTestFixture(t)
   561  
   562  	foo := manifestbuilder.New(f, "foo").
   563  		WithLocalResource("foo cmd", []string{f.JoinPath("foo")}).
   564  		Build()
   565  	bar := manifestbuilder.New(f, "bar").
   566  		WithLocalResource("bar cmd", []string{f.JoinPath("bar")}).
   567  		WithResourceDeps("foo").
   568  		Build()
   569  
   570  	manifests := []model.Manifest{bar, foo}
   571  	f.SetNextBuildError(errors.New("failure"))
   572  	f.Start(manifests)
   573  
   574  	// trigger a change for bar so that it would try to build if not for its resource dep
   575  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("bar", "main.go"))
   576  
   577  	call := f.nextCall()
   578  	require.Equal(t, "foo", call.local().Name.String())
   579  
   580  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("foo", "main.go"))
   581  	call = f.nextCall()
   582  	require.Equal(t, "foo", call.local().Name.String())
   583  
   584  	// since the foo build succeeded, bar should now queue
   585  	call = f.nextCall()
   586  	require.Equal(t, "bar", call.local().Name.String())
   587  }
   588  
   589  func TestBuildControllerWontBuildManifestIfNoSlotsAvailable(t *testing.T) {
   590  	f := newTestFixture(t)
   591  	f.b.completeBuildsManually = true
   592  	f.setMaxParallelUpdates(2)
   593  
   594  	manA := f.newDockerBuildManifestWithBuildPath("manA", f.JoinPath("a"))
   595  	manB := f.newDockerBuildManifestWithBuildPath("manB", f.JoinPath("b"))
   596  	manC := f.newDockerBuildManifestWithBuildPath("manC", f.JoinPath("c"))
   597  	f.Start([]model.Manifest{manA, manB, manC})
   598  	f.completeAndCheckBuildsForManifests(manA, manB, manC)
   599  
   600  	// start builds for all manifests (we only have 2 build slots)
   601  	f.editFileAndWaitForManifestBuilding("manA", "a/main.go")
   602  	f.editFileAndWaitForManifestBuilding("manB", "b/main.go")
   603  	f.editFileAndAssertManifestNotBuilding("manC", "c/main.go")
   604  
   605  	// Complete one build...
   606  	f.completeBuildForManifest(manA)
   607  	call := f.nextCall("expect manA build complete")
   608  	f.assertCallIsForManifestAndFiles(call, manA, "a/main.go")
   609  
   610  	// ...and now there's a free build slot for 'manC'
   611  	f.waitUntilManifestBuilding("manC")
   612  
   613  	// complete the rest (can't guarantee order)
   614  	f.completeAndCheckBuildsForManifests(manB, manC)
   615  
   616  	err := f.Stop()
   617  	assert.NoError(t, err)
   618  	f.assertAllBuildsConsumed()
   619  }
   620  
   621  // It should be legal for a user to change maxParallelUpdates while builds
   622  // are in progress (e.g. if there are 5 builds in progress and user sets
   623  // maxParallelUpdates=3, nothing should explode.)
   624  func TestCurrentlyBuildingMayExceedMaxParallelUpdates(t *testing.T) {
   625  	f := newTestFixture(t)
   626  	f.b.completeBuildsManually = true
   627  	f.setMaxParallelUpdates(3)
   628  
   629  	manA := f.newDockerBuildManifestWithBuildPath("manA", f.JoinPath("a"))
   630  	manB := f.newDockerBuildManifestWithBuildPath("manB", f.JoinPath("b"))
   631  	manC := f.newDockerBuildManifestWithBuildPath("manC", f.JoinPath("c"))
   632  	f.Start([]model.Manifest{manA, manB, manC})
   633  	f.completeAndCheckBuildsForManifests(manA, manB, manC)
   634  
   635  	// start builds for all manifests
   636  	f.editFileAndWaitForManifestBuilding("manA", "a/main.go")
   637  	f.editFileAndWaitForManifestBuilding("manB", "b/main.go")
   638  	f.editFileAndWaitForManifestBuilding("manC", "c/main.go")
   639  	f.waitUntilNumBuildSlots(0)
   640  
   641  	// decrease maxParallelUpdates (now less than the number of current builds, but this is okay)
   642  	f.setMaxParallelUpdates(2)
   643  	f.waitUntilNumBuildSlots(0)
   644  
   645  	// another file change for manB -- will try to start another build as soon as possible
   646  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("b/other.go"))
   647  
   648  	f.completeBuildForManifest(manB)
   649  	call := f.nextCall("expect manB build complete")
   650  	f.assertCallIsForManifestAndFiles(call, manB, "b/main.go")
   651  
   652  	// we should NOT see another build for manB, even though it has a pending file change,
   653  	// b/c we don't have enough slots (since we decreased maxParallelUpdates)
   654  	f.waitUntilNumBuildSlots(0)
   655  	f.waitUntilManifestNotBuilding("manB")
   656  
   657  	// complete another build...
   658  	f.completeBuildForManifest(manA)
   659  	call = f.nextCall("expect manA build complete")
   660  	f.assertCallIsForManifestAndFiles(call, manA, "a/main.go")
   661  
   662  	// ...now that we have an available slots again, manB will rebuild
   663  	f.waitUntilManifestBuilding("manB")
   664  
   665  	f.completeBuildForManifest(manB)
   666  	call = f.nextCall("expect manB build complete (second build)")
   667  	f.assertCallIsForManifestAndFiles(call, manB, "b/other.go")
   668  
   669  	f.completeBuildForManifest(manC)
   670  	call = f.nextCall("expect manC build complete")
   671  	f.assertCallIsForManifestAndFiles(call, manC, "c/main.go")
   672  
   673  	err := f.Stop()
   674  	assert.NoError(t, err)
   675  	f.assertAllBuildsConsumed()
   676  }
   677  
   678  func TestDontStartBuildIfControllerAndEngineUnsynced(t *testing.T) {
   679  	f := newTestFixture(t)
   680  
   681  	f.b.completeBuildsManually = true
   682  	f.setMaxParallelUpdates(3)
   683  
   684  	manA := f.newDockerBuildManifestWithBuildPath("manA", f.JoinPath("a"))
   685  	manB := f.newDockerBuildManifestWithBuildPath("manB", f.JoinPath("b"))
   686  	f.Start([]model.Manifest{manA, manB})
   687  	f.completeAndCheckBuildsForManifests(manA, manB)
   688  
   689  	f.editFileAndWaitForManifestBuilding("manA", "a/main.go")
   690  
   691  	// deliberately de-sync engine state and build controller
   692  	st := f.store.LockMutableStateForTesting()
   693  	st.BuildControllerStartCount--
   694  	f.store.UnlockMutableState()
   695  
   696  	// this build won't start while state and build controller are out of sync
   697  	f.editFileAndAssertManifestNotBuilding("manB", "b/main.go")
   698  
   699  	// resync the two counts...
   700  	st = f.store.LockMutableStateForTesting()
   701  	st.BuildControllerStartCount++
   702  	f.store.UnlockMutableState()
   703  
   704  	// ...and manB build will start as expected
   705  	f.waitUntilManifestBuilding("manB")
   706  
   707  	// complete all builds (can't guarantee order)
   708  	f.completeAndCheckBuildsForManifests(manA, manB)
   709  
   710  	err := f.Stop()
   711  	assert.NoError(t, err)
   712  	f.assertAllBuildsConsumed()
   713  }
   714  
   715  func TestErrorHandlingWithMultipleBuilds(t *testing.T) {
   716  	if runtime.GOOS == "windows" {
   717  		t.Skip("TODO(nick): fix this")
   718  	}
   719  	f := newTestFixture(t)
   720  	f.b.completeBuildsManually = true
   721  	f.setMaxParallelUpdates(2)
   722  
   723  	errA := fmt.Errorf("errA")
   724  	errB := fmt.Errorf("errB")
   725  
   726  	manA := f.newDockerBuildManifestWithBuildPath("manA", f.JoinPath("a"))
   727  	manB := f.newDockerBuildManifestWithBuildPath("manB", f.JoinPath("b"))
   728  	manC := f.newDockerBuildManifestWithBuildPath("manC", f.JoinPath("c"))
   729  	f.Start([]model.Manifest{manA, manB, manC})
   730  	f.completeAndCheckBuildsForManifests(manA, manB, manC)
   731  
   732  	// start builds for all manifests (we only have 2 build slots)
   733  	f.SetNextBuildError(errA)
   734  	f.editFileAndWaitForManifestBuilding("manA", "a/main.go")
   735  	f.SetNextBuildError(errB)
   736  	f.editFileAndWaitForManifestBuilding("manB", "b/main.go")
   737  	f.editFileAndAssertManifestNotBuilding("manC", "c/main.go")
   738  
   739  	// Complete one build...
   740  	f.completeBuildForManifest(manA)
   741  	call := f.nextCall("expect manA build complete")
   742  	f.assertCallIsForManifestAndFiles(call, manA, "a/main.go")
   743  	f.WaitUntilManifestState("last manA build reflects expected error", "manA", func(ms store.ManifestState) bool {
   744  		return ms.LastBuild().Error == errA
   745  	})
   746  
   747  	// ...'manC' should start building, even though the manA build ended with an error
   748  	f.waitUntilManifestBuilding("manC")
   749  
   750  	// complete the rest
   751  	f.completeAndCheckBuildsForManifests(manB, manC)
   752  	f.WaitUntilManifestState("last manB build reflects expected error", "manB", func(ms store.ManifestState) bool {
   753  		return ms.LastBuild().Error == errB
   754  	})
   755  	f.WaitUntilManifestState("last manC build recorded and has no error", "manC", func(ms store.ManifestState) bool {
   756  		return len(ms.BuildHistory) == 2 && ms.LastBuild().Error == nil
   757  	})
   758  
   759  	err := f.Stop()
   760  	assert.NoError(t, err)
   761  	f.assertAllBuildsConsumed()
   762  }
   763  
   764  func TestManifestsWithSameTwoImages(t *testing.T) {
   765  	f := newTestFixture(t)
   766  	m1, m2 := NewManifestsWithSameTwoImages(f)
   767  	f.Start([]model.Manifest{m1, m2})
   768  
   769  	f.waitForCompletedBuildCount(2)
   770  
   771  	call := f.nextCall("m1 build1")
   772  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   773  
   774  	call = f.nextCall("m2 build1")
   775  	assert.Equal(t, m2.K8sTarget(), call.k8s())
   776  
   777  	aPath := f.JoinPath("common", "a.txt")
   778  	f.fsWatcher.Events <- watch.NewFileEvent(aPath)
   779  
   780  	f.waitForCompletedBuildCount(4)
   781  
   782  	// Make sure that both builds are triggered, and that they
   783  	// are triggered in a particular order.
   784  	call = f.nextCall("m1 build2")
   785  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   786  
   787  	state := call.state[m1.ImageTargets[0].ID()]
   788  	assert.Equal(t, map[string]bool{aPath: true}, state.FilesChangedSet)
   789  
   790  	// Make sure that when the second build is triggered, we did the bookkeeping
   791  	// correctly around marking the first and second image built and only deploying
   792  	// the k8s resources.
   793  	call = f.nextCall("m2 build2")
   794  	assert.Equal(t, m2.K8sTarget(), call.k8s())
   795  
   796  	id := m2.ImageTargets[0].ID()
   797  	result := f.b.resultsByID[id]
   798  	assert.Equal(t, result, call.state[id].LastResult)
   799  	assert.Equal(t, 0, len(call.state[id].FilesChangedSet))
   800  
   801  	id = m2.ImageTargets[1].ID()
   802  	result = f.b.resultsByID[id]
   803  	assert.Equal(t, result, call.state[id].LastResult)
   804  	assert.Equal(t, 0, len(call.state[id].FilesChangedSet))
   805  
   806  	err := f.Stop()
   807  	assert.NoError(t, err)
   808  	f.assertAllBuildsConsumed()
   809  }
   810  
   811  func TestManifestsWithTwoCommonAncestors(t *testing.T) {
   812  	f := newTestFixture(t)
   813  	m1, m2 := NewManifestsWithTwoCommonAncestors(f)
   814  	f.Start([]model.Manifest{m1, m2})
   815  
   816  	f.waitForCompletedBuildCount(2)
   817  
   818  	call := f.nextCall("m1 build1")
   819  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   820  
   821  	call = f.nextCall("m2 build1")
   822  	assert.Equal(t, m2.K8sTarget(), call.k8s())
   823  
   824  	aPath := f.JoinPath("base", "a.txt")
   825  	f.fsWatcher.Events <- watch.NewFileEvent(aPath)
   826  
   827  	f.waitForCompletedBuildCount(4)
   828  
   829  	// Make sure that both builds are triggered, and that they
   830  	// are triggered in a particular order.
   831  	call = f.nextCall("m1 build2")
   832  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   833  
   834  	state := call.state[m1.ImageTargets[0].ID()]
   835  	assert.Equal(t, map[string]bool{aPath: true}, state.FilesChangedSet)
   836  
   837  	// Make sure that when the second build is triggered, we did the bookkeeping
   838  	// correctly around marking the first and second image built, and only
   839  	// rebuilding the third image and k8s deploy.
   840  	call = f.nextCall("m2 build2")
   841  	assert.Equal(t, m2.K8sTarget(), call.k8s())
   842  
   843  	id := m2.ImageTargets[0].ID()
   844  	result := f.b.resultsByID[id]
   845  	assert.Equal(t, result, call.state[id].LastResult)
   846  	assert.Equal(t, 0, len(call.state[id].FilesChangedSet))
   847  
   848  	id = m2.ImageTargets[1].ID()
   849  	result = f.b.resultsByID[id]
   850  	assert.Equal(t, result, call.state[id].LastResult)
   851  	assert.Equal(t, 0, len(call.state[id].FilesChangedSet))
   852  
   853  	id = m2.ImageTargets[2].ID()
   854  	result = f.b.resultsByID[id]
   855  
   856  	// Assert the 3rd image was not reused from the previous build.
   857  	assert.NotEqual(t, result, call.state[id].LastResult)
   858  	assert.Equal(t,
   859  		map[model.TargetID]bool{m2.ImageTargets[1].ID(): true},
   860  		call.state[id].DepsChangedSet)
   861  
   862  	err := f.Stop()
   863  	assert.NoError(t, err)
   864  	f.assertAllBuildsConsumed()
   865  }
   866  
   867  func TestLocalDependsOnNonWorkloadK8s(t *testing.T) {
   868  	f := newTestFixture(t)
   869  
   870  	local1 := manifestbuilder.New(f, "local").
   871  		WithLocalResource("exec-local", nil).
   872  		WithResourceDeps("k8s1").
   873  		Build()
   874  	k8s1 := manifestbuilder.New(f, "k8s1").
   875  		WithK8sYAML(testyaml.SanchoYAML).
   876  		WithK8sPodReadiness(model.PodReadinessIgnore).
   877  		Build()
   878  	f.Start([]model.Manifest{local1, k8s1})
   879  
   880  	f.waitForCompletedBuildCount(2)
   881  
   882  	call := f.nextCall("k8s1 build")
   883  	assert.Equal(t, k8s1.K8sTarget(), call.k8s())
   884  
   885  	call = f.nextCall("local build")
   886  	assert.Equal(t, local1.LocalTarget(), call.local())
   887  
   888  	err := f.Stop()
   889  	assert.NoError(t, err)
   890  	f.assertAllBuildsConsumed()
   891  }
   892  
   893  func TestManifestsWithCommonAncestorAndTrigger(t *testing.T) {
   894  	f := newTestFixture(t)
   895  	m1, m2 := NewManifestsWithCommonAncestor(f)
   896  	f.Start([]model.Manifest{m1, m2})
   897  
   898  	f.waitForCompletedBuildCount(2)
   899  
   900  	call := f.nextCall("m1 build1")
   901  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   902  
   903  	call = f.nextCall("m2 build1")
   904  	assert.Equal(t, m2.K8sTarget(), call.k8s())
   905  
   906  	f.store.Dispatch(store.AppendToTriggerQueueAction{Name: m1.Name})
   907  	f.waitForCompletedBuildCount(3)
   908  
   909  	// Make sure that only one build was triggered.
   910  	call = f.nextCall("m1 build2")
   911  	assert.Equal(t, m1.K8sTarget(), call.k8s())
   912  
   913  	f.assertNoCall("m2 should not be rebuilt")
   914  
   915  	err := f.Stop()
   916  	assert.NoError(t, err)
   917  	f.assertAllBuildsConsumed()
   918  }
   919  
   920  func TestDisablingCancelsBuild(t *testing.T) {
   921  	f := newTestFixture(t)
   922  	manifest := manifestbuilder.New(f, "local").
   923  		WithLocalResource("sleep 10000", nil).
   924  		Build()
   925  	f.b.completeBuildsManually = true
   926  
   927  	f.Start([]model.Manifest{manifest})
   928  	f.waitUntilManifestBuilding("local")
   929  
   930  	ds := manifest.DeployTarget.(model.LocalTarget).ServeCmdDisableSource
   931  	err := configmap.UpsertDisableConfigMap(f.ctx, f.ctrlClient, ds.ConfigMap.Name, ds.ConfigMap.Key, true)
   932  	require.NoError(t, err)
   933  
   934  	f.waitForCompletedBuildCount(1)
   935  
   936  	f.withManifestState("local", func(ms store.ManifestState) {
   937  		require.EqualError(t, ms.LastBuild().Error, "build canceled")
   938  	})
   939  
   940  	err = f.Stop()
   941  	require.NoError(t, err)
   942  }
   943  
   944  func TestCancelButton(t *testing.T) {
   945  	f := newTestFixture(t)
   946  	f.b.completeBuildsManually = true
   947  	f.useRealTiltfileLoader()
   948  	f.WriteFile("Tiltfile", `
   949  local_resource('local', 'sleep 10000')
   950  `)
   951  	f.loadAndStart()
   952  	f.waitUntilManifestBuilding("local")
   953  
   954  	var cancelButton v1alpha1.UIButton
   955  	err := f.ctrlClient.Get(f.ctx, types.NamespacedName{Name: uibutton.StopBuildButtonName("local")}, &cancelButton)
   956  	require.NoError(t, err)
   957  	cancelButton.Status.LastClickedAt = metav1.NowMicro()
   958  	err = f.ctrlClient.Status().Update(f.ctx, &cancelButton)
   959  	require.NoError(t, err)
   960  
   961  	f.waitForCompletedBuildCount(1)
   962  
   963  	f.withManifestState("local", func(ms store.ManifestState) {
   964  		require.EqualError(t, ms.LastBuild().Error, "build canceled")
   965  	})
   966  
   967  	err = f.Stop()
   968  	require.NoError(t, err)
   969  }
   970  
   971  func TestCancelButtonClickedBeforeBuild(t *testing.T) {
   972  	f := newTestFixture(t)
   973  	f.b.completeBuildsManually = true
   974  	f.useRealTiltfileLoader()
   975  	f.WriteFile("Tiltfile", `
   976  local_resource('local', 'sleep 10000')
   977  `)
   978  	// grab a timestamp now to represent clicking the button before the build started
   979  	ts := metav1.NowMicro()
   980  
   981  	f.loadAndStart()
   982  	f.waitUntilManifestBuilding("local")
   983  
   984  	var cancelButton v1alpha1.UIButton
   985  	err := f.ctrlClient.Get(f.ctx, types.NamespacedName{Name: uibutton.StopBuildButtonName("local")}, &cancelButton)
   986  	require.NoError(t, err)
   987  	cancelButton.Status.LastClickedAt = ts
   988  	err = f.ctrlClient.Status().Update(f.ctx, &cancelButton)
   989  	require.NoError(t, err)
   990  
   991  	// give the build controller a little time to process the button click
   992  	require.Never(t, func() bool {
   993  		state := f.store.RLockState()
   994  		defer f.store.RUnlockState()
   995  		return state.CompletedBuildCount > 0
   996  	}, 20*time.Millisecond, 2*time.Millisecond, "build finished on its own even though manual build completion is enabled")
   997  
   998  	f.b.completeBuild("local:local")
   999  
  1000  	f.waitForCompletedBuildCount(1)
  1001  
  1002  	f.withManifestState("local", func(ms store.ManifestState) {
  1003  		require.NoError(t, ms.LastBuild().Error)
  1004  	})
  1005  
  1006  	err = f.Stop()
  1007  	require.NoError(t, err)
  1008  }
  1009  
  1010  func TestBuildControllerK8sFileDependencies(t *testing.T) {
  1011  	f := newTestFixture(t)
  1012  
  1013  	kt := k8s.MustTarget("fe", testyaml.SanchoYAML).
  1014  		WithPathDependencies([]string{f.JoinPath("k8s-dep")}).
  1015  		WithIgnores([]v1alpha1.IgnoreDef{
  1016  			{BasePath: f.JoinPath("k8s-dep", ".git")},
  1017  			{
  1018  				BasePath: f.JoinPath("k8s-dep"),
  1019  				Patterns: []string{"ignore-me"},
  1020  			},
  1021  		})
  1022  	m := model.Manifest{Name: "fe"}.WithDeployTarget(kt)
  1023  
  1024  	f.Start([]model.Manifest{m})
  1025  
  1026  	call := f.nextCall()
  1027  	assert.Empty(t, call.k8sState().FilesChanged())
  1028  
  1029  	// path dependency is on ./k8s-dep/** with a local repo of ./k8s-dep/.git/** (ignored)
  1030  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("k8s-dep", "ignore-me"))
  1031  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("k8s-dep", ".git", "file"))
  1032  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath("k8s-dep", "file"))
  1033  
  1034  	call = f.nextCall()
  1035  	assert.Equal(t, []string{f.JoinPath("k8s-dep", "file")}, call.k8sState().FilesChanged())
  1036  
  1037  	err := f.Stop()
  1038  	assert.NoError(t, err)
  1039  	f.assertAllBuildsConsumed()
  1040  }
  1041  
  1042  func (f *testFixture) waitUntilManifestBuilding(name model.ManifestName) {
  1043  	f.t.Helper()
  1044  	msg := fmt.Sprintf("manifest %q is building", name)
  1045  	f.WaitUntilManifestState(msg, name, func(ms store.ManifestState) bool {
  1046  		return ms.IsBuilding()
  1047  	})
  1048  
  1049  	f.withState(func(st store.EngineState) {
  1050  		ok := st.CurrentBuildSet[name]
  1051  		require.True(f.t, ok, "expected EngineState to reflect that %q is currently building", name)
  1052  	})
  1053  }
  1054  
  1055  func (f *testFixture) waitUntilManifestNotBuilding(name model.ManifestName) {
  1056  	msg := fmt.Sprintf("manifest %q is NOT building", name)
  1057  	f.WaitUntilManifestState(msg, name, func(ms store.ManifestState) bool {
  1058  		return !ms.IsBuilding()
  1059  	})
  1060  
  1061  	f.withState(func(st store.EngineState) {
  1062  		ok := st.CurrentBuildSet[name]
  1063  		require.False(f.t, ok, "expected EngineState to reflect that %q is NOT currently building", name)
  1064  	})
  1065  }
  1066  
  1067  func (f *testFixture) waitUntilNumBuildSlots(expected int) {
  1068  	msg := fmt.Sprintf("%d build slots available", expected)
  1069  	f.WaitUntil(msg, func(st store.EngineState) bool {
  1070  		return expected == st.AvailableBuildSlots()
  1071  	})
  1072  }
  1073  
  1074  func (f *testFixture) editFileAndWaitForManifestBuilding(name model.ManifestName, path string) {
  1075  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath(path))
  1076  	f.waitUntilManifestBuilding(name)
  1077  }
  1078  
  1079  func (f *testFixture) editFileAndAssertManifestNotBuilding(name model.ManifestName, path string) {
  1080  	f.fsWatcher.Events <- watch.NewFileEvent(f.JoinPath(path))
  1081  	f.waitUntilManifestNotBuilding(name)
  1082  }
  1083  
  1084  func (f *testFixture) assertCallIsForManifestAndFiles(call buildAndDeployCall, m model.Manifest, files ...string) {
  1085  	assert.Equal(f.t, m.ImageTargetAt(0).ID(), call.firstImgTarg().ID())
  1086  	assert.Equal(f.t, f.JoinPaths(files), call.oneImageState().FilesChanged())
  1087  }
  1088  
  1089  func (f *testFixture) completeAndCheckBuildsForManifests(manifests ...model.Manifest) {
  1090  	for _, m := range manifests {
  1091  		f.completeBuildForManifest(m)
  1092  	}
  1093  
  1094  	expectedImageTargets := make([][]model.ImageTarget, len(manifests))
  1095  	var actualImageTargets [][]model.ImageTarget
  1096  	for i, m := range manifests {
  1097  		expectedImageTargets[i] = m.ImageTargets
  1098  
  1099  		call := f.nextCall("timed out waiting for call %d/%d", i+1, len(manifests))
  1100  		actualImageTargets = append(actualImageTargets, call.imageTargets())
  1101  	}
  1102  	require.ElementsMatch(f.t, expectedImageTargets, actualImageTargets)
  1103  
  1104  	for _, m := range manifests {
  1105  		f.waitUntilManifestNotBuilding(m.Name)
  1106  	}
  1107  }
  1108  
  1109  func (f *testFixture) simpleManifestWithTriggerMode(name model.ManifestName, tm model.TriggerMode) model.Manifest {
  1110  	return manifestbuilder.New(f, name).WithTriggerMode(tm).
  1111  		WithImageTarget(NewSanchoDockerBuildImageTarget(f)).
  1112  		WithK8sYAML(SanchoYAML).Build()
  1113  }