github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/worker/runtime/backend_test.go (about)

     1  package runtime_test
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"sync"
     7  	"testing"
     8  	"time"
     9  
    10  	"code.cloudfoundry.org/garden"
    11  	"code.cloudfoundry.org/garden/gardenfakes"
    12  	"github.com/pf-qiu/concourse/v6/worker/runtime"
    13  	"github.com/pf-qiu/concourse/v6/worker/runtime/libcontainerd/libcontainerdfakes"
    14  	"github.com/pf-qiu/concourse/v6/worker/runtime/runtimefakes"
    15  	"github.com/containerd/containerd"
    16  	"github.com/containerd/containerd/errdefs"
    17  	"github.com/opencontainers/runtime-spec/specs-go"
    18  	"github.com/stretchr/testify/require"
    19  	"github.com/stretchr/testify/suite"
    20  )
    21  
    22  type BackendSuite struct {
    23  	suite.Suite
    24  	*require.Assertions
    25  
    26  	backend runtime.GardenBackend
    27  	client  *libcontainerdfakes.FakeClient
    28  	network *runtimefakes.FakeNetwork
    29  	userns  *runtimefakes.FakeUserNamespace
    30  	killer  *runtimefakes.FakeKiller
    31  }
    32  
    33  func (s *BackendSuite) SetupTest() {
    34  	s.client = new(libcontainerdfakes.FakeClient)
    35  	s.killer = new(runtimefakes.FakeKiller)
    36  	s.network = new(runtimefakes.FakeNetwork)
    37  	s.userns = new(runtimefakes.FakeUserNamespace)
    38  
    39  	var err error
    40  	s.backend, err = runtime.NewGardenBackend(s.client,
    41  		runtime.WithKiller(s.killer),
    42  		runtime.WithNetwork(s.network),
    43  		runtime.WithUserNamespace(s.userns),
    44  	)
    45  	s.NoError(err)
    46  }
    47  
    48  func (s *BackendSuite) TestNew() {
    49  	_, err := runtime.NewGardenBackend(nil)
    50  	s.EqualError(err, "nil client")
    51  }
    52  
    53  func (s *BackendSuite) TestPing() {
    54  	for _, tc := range []struct {
    55  		desc          string
    56  		versionReturn error
    57  		succeeds      bool
    58  	}{
    59  		{
    60  			desc:          "fail from containerd version service",
    61  			succeeds:      true,
    62  			versionReturn: nil,
    63  		},
    64  		{
    65  			desc:          "ok from containerd's version service",
    66  			succeeds:      false,
    67  			versionReturn: errors.New("error returning version"),
    68  		},
    69  	} {
    70  		s.T().Run(tc.desc, func(t *testing.T) {
    71  			s.client.VersionReturns(tc.versionReturn)
    72  
    73  			err := s.backend.Ping()
    74  			if tc.succeeds {
    75  				s.NoError(err)
    76  				return
    77  			}
    78  
    79  			s.EqualError(errors.Unwrap(err), "error returning version")
    80  		})
    81  	}
    82  }
    83  
    84  var (
    85  	invalidGdnSpec      = garden.ContainerSpec{}
    86  	minimumValidGdnSpec = garden.ContainerSpec{
    87  		Handle: "handle", RootFSPath: "raw:///rootfs",
    88  	}
    89  )
    90  
    91  func (s *BackendSuite) TestCreateWithInvalidSpec() {
    92  	_, err := s.backend.Create(invalidGdnSpec)
    93  
    94  	s.Error(err)
    95  	s.Equal(0, s.client.NewContainerCallCount())
    96  }
    97  
    98  func (s *BackendSuite) TestCreateWithNewContainerFailure() {
    99  	s.client.NewContainerReturns(nil, errors.New("err"))
   100  
   101  	_, err := s.backend.Create(minimumValidGdnSpec)
   102  	s.Error(err)
   103  
   104  	s.Equal(1, s.client.NewContainerCallCount())
   105  }
   106  
   107  func (s *BackendSuite) TestCreateContainerNewTaskFailure() {
   108  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   109  
   110  	expectedErr := errors.New("task-err")
   111  	fakeContainer.NewTaskReturns(nil, expectedErr)
   112  
   113  	s.client.NewContainerReturns(fakeContainer, nil)
   114  
   115  	_, err := s.backend.Create(minimumValidGdnSpec)
   116  	s.EqualError(errors.Unwrap(errors.Unwrap(err)), expectedErr.Error())
   117  
   118  	s.Equal(1, fakeContainer.NewTaskCallCount())
   119  }
   120  
   121  func (s *BackendSuite) TestCreateContainerTaskStartFailure() {
   122  	fakeTask := new(libcontainerdfakes.FakeTask)
   123  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   124  
   125  	s.client.NewContainerReturns(fakeContainer, nil)
   126  	fakeContainer.NewTaskReturns(fakeTask, nil)
   127  	fakeTask.StartReturns(errors.New("start-err"))
   128  
   129  	_, err := s.backend.Create(minimumValidGdnSpec)
   130  	s.Error(err)
   131  
   132  	s.EqualError(errors.Unwrap(err), "start-err")
   133  }
   134  
   135  func (s *BackendSuite) TestCreateContainerSetsHandle() {
   136  	fakeTask := new(libcontainerdfakes.FakeTask)
   137  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   138  
   139  	fakeContainer.IDReturns("handle")
   140  	fakeContainer.NewTaskReturns(fakeTask, nil)
   141  
   142  	s.client.NewContainerReturns(fakeContainer, nil)
   143  	cont, err := s.backend.Create(minimumValidGdnSpec)
   144  	s.NoError(err)
   145  
   146  	s.Equal("handle", cont.Handle())
   147  }
   148  
   149  func (s *BackendSuite) TestCreateMaxContainersReached() {
   150  	backend, err := runtime.NewGardenBackend(s.client,
   151  		runtime.WithKiller(s.killer),
   152  		runtime.WithNetwork(s.network),
   153  		runtime.WithUserNamespace(s.userns),
   154  		runtime.WithMaxContainers(1),
   155  		runtime.WithRequestTimeout(1*time.Second),
   156  	)
   157  	s.NoError(err)
   158  
   159  	fakeTask := new(libcontainerdfakes.FakeTask)
   160  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   161  
   162  	fakeContainer.NewTaskReturns(fakeTask, nil)
   163  	s.client.NewContainerReturns(fakeContainer, nil)
   164  
   165  	s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil)
   166  	_, err = backend.Create(minimumValidGdnSpec)
   167  	s.Error(err)
   168  	s.Contains(err.Error(), "max containers reached")
   169  }
   170  
   171  func (s *BackendSuite) TestCreateMaxContainersReachedConcurrent() {
   172  	fakeTask := new(libcontainerdfakes.FakeTask)
   173  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   174  
   175  	fakeContainer.NewTaskReturns(fakeTask, nil)
   176  
   177  	s.client.NewContainerStub = func(context context.Context, str string, strings map[string]string, spec *specs.Spec) (container containerd.Container, e error) {
   178  		s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil)
   179  		return fakeContainer, nil
   180  	}
   181  
   182  	backend, err := runtime.NewGardenBackend(s.client,
   183  		runtime.WithKiller(s.killer),
   184  		runtime.WithNetwork(s.network),
   185  		runtime.WithUserNamespace(s.userns),
   186  		runtime.WithMaxContainers(1),
   187  		runtime.WithRequestTimeout(1*time.Second),
   188  	)
   189  	s.NoError(err)
   190  
   191  	numberOfRequests := 10
   192  	requestErrors := make(chan error, numberOfRequests)
   193  	wg := sync.WaitGroup{}
   194  	wg.Add(numberOfRequests)
   195  
   196  	for i := 0; i < numberOfRequests; i++ {
   197  		go func() {
   198  			_, err := backend.Create(minimumValidGdnSpec)
   199  			if err != nil {
   200  				requestErrors <- err
   201  			}
   202  			wg.Done()
   203  		}()
   204  	}
   205  	wg.Wait()
   206  	close(requestErrors)
   207  
   208  	s.Len(requestErrors, numberOfRequests-1)
   209  	s.Equal(s.client.NewContainerCallCount(), 1)
   210  	for err := range requestErrors {
   211  		s.Contains(err.Error(), "max containers reached")
   212  	}
   213  }
   214  
   215  func (s *BackendSuite) TestCreateContainerLockTimeout() {
   216  	fakeTask := new(libcontainerdfakes.FakeTask)
   217  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   218  
   219  	fakeContainer.IDReturns("handle")
   220  	fakeContainer.NewTaskReturns(fakeTask, nil)
   221  
   222  	s.client.NewContainerStub = func(context context.Context, str string, strings map[string]string, spec *specs.Spec) (container containerd.Container, e error) {
   223  		s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil)
   224  		time.Sleep(500 * time.Millisecond)
   225  		return fakeContainer, nil
   226  	}
   227  
   228  	numberOfRequests := 10
   229  
   230  	backend, err := runtime.NewGardenBackend(s.client,
   231  		runtime.WithKiller(s.killer),
   232  		runtime.WithNetwork(s.network),
   233  		runtime.WithUserNamespace(s.userns),
   234  		runtime.WithRequestTimeout(10*time.Millisecond),
   235  		runtime.WithMaxContainers(numberOfRequests),
   236  	)
   237  	s.NoError(err)
   238  
   239  	requestErrors := make(chan error, numberOfRequests)
   240  	wg := sync.WaitGroup{}
   241  	wg.Add(numberOfRequests)
   242  
   243  	for i := 0; i < numberOfRequests; i++ {
   244  		go func() {
   245  			_, err := backend.Create(minimumValidGdnSpec)
   246  			if err != nil {
   247  				requestErrors <- err
   248  			}
   249  			wg.Done()
   250  		}()
   251  	}
   252  	wg.Wait()
   253  	close(requestErrors)
   254  
   255  	s.Len(requestErrors, numberOfRequests-1)
   256  	for err := range requestErrors {
   257  		s.Contains(err.Error(), "acquiring create container lock")
   258  	}
   259  }
   260  func (s *BackendSuite) TestContainersWithContainerdFailure() {
   261  	s.client.ContainersReturns(nil, errors.New("err"))
   262  
   263  	_, err := s.backend.Containers(nil)
   264  	s.Error(err)
   265  	s.Equal(1, s.client.ContainersCallCount())
   266  }
   267  
   268  func (s *BackendSuite) TestContainersWithInvalidPropertyFilters() {
   269  	for _, tc := range []struct {
   270  		desc   string
   271  		filter map[string]string
   272  	}{
   273  		{
   274  			desc: "empty key",
   275  			filter: map[string]string{
   276  				"": "bar",
   277  			},
   278  		},
   279  		{
   280  			desc: "empty value",
   281  			filter: map[string]string{
   282  				"foo": "",
   283  			},
   284  		},
   285  	} {
   286  		s.T().Run(tc.desc, func(t *testing.T) {
   287  			_, err := s.backend.Containers(tc.filter)
   288  
   289  			s.Error(err)
   290  			s.Equal(0, s.client.ContainersCallCount())
   291  		})
   292  	}
   293  }
   294  
   295  func (s *BackendSuite) TestContainersWithProperProperties() {
   296  	_, _ = s.backend.Containers(map[string]string{"foo": "bar", "caz": "zaz"})
   297  	s.Equal(1, s.client.ContainersCallCount())
   298  
   299  	_, labelSet := s.client.ContainersArgsForCall(0)
   300  	s.ElementsMatch([]string{"labels.foo==bar", "labels.caz==zaz"}, labelSet)
   301  }
   302  
   303  func (s *BackendSuite) TestContainersConversion() {
   304  	fakeContainer1 := new(libcontainerdfakes.FakeContainer)
   305  	fakeContainer2 := new(libcontainerdfakes.FakeContainer)
   306  
   307  	s.client.ContainersReturns([]containerd.Container{
   308  		fakeContainer1, fakeContainer2,
   309  	}, nil)
   310  
   311  	containers, err := s.backend.Containers(nil)
   312  	s.NoError(err)
   313  	s.Equal(1, s.client.ContainersCallCount())
   314  	s.Len(containers, 2)
   315  }
   316  
   317  func (s *BackendSuite) TestLookupEmptyHandleError() {
   318  	_, err := s.backend.Lookup("")
   319  	s.Equal("empty handle", err.Error())
   320  }
   321  
   322  func (s *BackendSuite) TestLookupCallGetContainerWithHandle() {
   323  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   324  	fakeContainer.IDReturns("handle")
   325  	s.client.GetContainerReturns(fakeContainer, nil)
   326  
   327  	_, _ = s.backend.Lookup("handle")
   328  	s.Equal(1, s.client.GetContainerCallCount())
   329  
   330  	_, handle := s.client.GetContainerArgsForCall(0)
   331  	s.Equal("handle", handle)
   332  }
   333  
   334  func (s *BackendSuite) TestLookupGetContainerError() {
   335  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   336  	fakeContainer.IDReturns("handle")
   337  	s.client.GetContainerReturns(fakeContainer, nil)
   338  
   339  	s.client.GetContainerReturns(nil, errors.New("containerd-err"))
   340  
   341  	_, err := s.backend.Lookup("handle")
   342  	s.Error(err)
   343  	s.EqualError(errors.Unwrap(err), "containerd-err")
   344  }
   345  
   346  func (s *BackendSuite) TestLookupGetContainerFails() {
   347  	s.client.GetContainerReturns(nil, errors.New("err"))
   348  	_, err := s.backend.Lookup("non-existent-handle")
   349  	s.Error(err)
   350  	s.EqualError(errors.Unwrap(err), "err")
   351  }
   352  
   353  func (s *BackendSuite) TestLookupGetNoContainerReturned() {
   354  	s.client.GetContainerReturns(nil, errors.New("not found"))
   355  	container, err := s.backend.Lookup("non-existent-handle")
   356  	s.Error(err)
   357  	s.Nil(container)
   358  }
   359  
   360  func (s *BackendSuite) TestLookupGetContainer() {
   361  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   362  	fakeContainer.IDReturns("handle")
   363  	s.client.GetContainerReturns(fakeContainer, nil)
   364  	container, err := s.backend.Lookup("handle")
   365  	s.NoError(err)
   366  	s.NotNil(container)
   367  	s.Equal("handle", container.Handle())
   368  }
   369  
   370  func (s *BackendSuite) TestDestroyEmptyHandleError() {
   371  	err := s.backend.Destroy("")
   372  	s.EqualError(err, "empty handle")
   373  }
   374  
   375  func (s *BackendSuite) TestDestroyGetContainerError() {
   376  	s.client.GetContainerReturns(nil, errors.New("get-container-failed"))
   377  
   378  	err := s.backend.Destroy("some-handle")
   379  	s.EqualError(errors.Unwrap(err), "get-container-failed")
   380  }
   381  
   382  func (s *BackendSuite) TestDestroyGetTaskError() {
   383  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   384  
   385  	s.client.GetContainerReturns(fakeContainer, nil)
   386  
   387  	expectedError := errors.New("get-task-failed")
   388  	fakeContainer.TaskReturns(nil, expectedError)
   389  
   390  	err := s.backend.Destroy("some handle")
   391  	s.True(errors.Is(err, expectedError))
   392  }
   393  
   394  func (s *BackendSuite) TestDestroyGetTaskErrorNotFoundAndDeleteFails() {
   395  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   396  
   397  	s.client.GetContainerReturns(fakeContainer, nil)
   398  	fakeContainer.TaskReturns(nil, errdefs.ErrNotFound)
   399  
   400  	expectedError := errors.New("delete-container-failed")
   401  	fakeContainer.DeleteReturns(expectedError)
   402  
   403  	err := s.backend.Destroy("some handle")
   404  	s.True(errors.Is(err, expectedError))
   405  }
   406  
   407  func (s *BackendSuite) TestDestroyGetTaskErrorNotFoundAndDeleteSucceeds() {
   408  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   409  
   410  	s.client.GetContainerReturns(fakeContainer, nil)
   411  	fakeContainer.TaskReturns(nil, errdefs.ErrNotFound)
   412  
   413  	err := s.backend.Destroy("some handle")
   414  
   415  	s.Equal(1, fakeContainer.DeleteCallCount())
   416  	s.NoError(err)
   417  }
   418  
   419  func (s *BackendSuite) TestDestroyKillTaskFails() {
   420  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   421  	fakeTask := new(libcontainerdfakes.FakeTask)
   422  
   423  	s.client.GetContainerReturns(fakeContainer, nil)
   424  	fakeContainer.TaskReturns(fakeTask, nil)
   425  
   426  	expectedError := errors.New("kill-task-failed")
   427  	s.killer.KillReturns(expectedError)
   428  
   429  	err := s.backend.Destroy("some handle")
   430  	s.True(errors.Is(err, expectedError))
   431  	_, _, behaviour := s.killer.KillArgsForCall(0)
   432  	s.Equal(runtime.KillGracefully, behaviour)
   433  }
   434  
   435  func (s *BackendSuite) TestDestroyRemoveNetworkFails() {
   436  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   437  	fakeTask := new(libcontainerdfakes.FakeTask)
   438  
   439  	s.client.GetContainerReturns(fakeContainer, nil)
   440  	fakeContainer.TaskReturns(fakeTask, nil)
   441  
   442  	expectedError := errors.New("remove-network-failed")
   443  	s.network.RemoveReturns(expectedError)
   444  
   445  	err := s.backend.Destroy("some handle")
   446  	s.True(errors.Is(err, expectedError))
   447  }
   448  
   449  func (s *BackendSuite) TestDestroyDeleteTaskFails() {
   450  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   451  	fakeTask := new(libcontainerdfakes.FakeTask)
   452  
   453  	s.client.GetContainerReturns(fakeContainer, nil)
   454  	fakeContainer.TaskReturns(fakeTask, nil)
   455  
   456  	expectedError := errors.New("delete-task-failed")
   457  	fakeTask.DeleteReturns(nil, expectedError)
   458  
   459  	err := s.backend.Destroy("some handle")
   460  	s.True(errors.Is(err, expectedError))
   461  }
   462  
   463  func (s *BackendSuite) TestDestroyContainerDeleteFailsAndDeleteTaskSucceeds() {
   464  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   465  	fakeTask := new(libcontainerdfakes.FakeTask)
   466  
   467  	s.client.GetContainerReturns(fakeContainer, nil)
   468  	fakeContainer.TaskReturns(fakeTask, nil)
   469  
   470  	expectedError := errors.New("delete-container-failed")
   471  	fakeContainer.DeleteReturns(expectedError)
   472  
   473  	err := s.backend.Destroy("some handle")
   474  	s.True(errors.Is(err, expectedError))
   475  }
   476  
   477  func (s *BackendSuite) TestDestroySucceeds() {
   478  	fakeContainer := new(libcontainerdfakes.FakeContainer)
   479  	fakeTask := new(libcontainerdfakes.FakeTask)
   480  	s.client.GetContainerReturns(fakeContainer, nil)
   481  	fakeContainer.TaskReturns(fakeTask, nil)
   482  
   483  	err := s.backend.Destroy("some handle")
   484  	s.NoError(err)
   485  }
   486  
   487  func (s *BackendSuite) TestStartInitsClientAndSetsUpRestrictedNetworks() {
   488  	err := s.backend.Start()
   489  	s.NoError(err)
   490  	s.Equal(1, s.client.InitCallCount())
   491  	s.Equal(1, s.network.SetupRestrictedNetworksCallCount())
   492  }
   493  
   494  func (s *BackendSuite) TestStartInitError() {
   495  	s.client.InitReturns(errors.New("init failed"))
   496  	err := s.backend.Start()
   497  	s.EqualError(errors.Unwrap(err), "init failed")
   498  }
   499  
   500  func (s *BackendSuite) TestStop() {
   501  	s.backend.Stop()
   502  	s.Equal(1, s.client.StopCallCount())
   503  }
   504  
   505  func (s *BackendSuite) TestGraceTimeGetPropertyFails() {
   506  	fakeContainer := new(gardenfakes.FakeContainer)
   507  	fakeContainer.PropertyReturns("", errors.New("error"))
   508  	result := s.backend.GraceTime(fakeContainer)
   509  	s.Equal(time.Duration(0), result)
   510  }
   511  
   512  func (s *BackendSuite) TestGraceTimeInvalidInteger() {
   513  	fakeContainer := new(gardenfakes.FakeContainer)
   514  	fakeContainer.PropertyReturns("not a number", nil)
   515  	result := s.backend.GraceTime(fakeContainer)
   516  	s.Equal(time.Duration(0), result)
   517  }
   518  
   519  func (s *BackendSuite) TestGraceTimeReturnsDuration() {
   520  	fakeContainer := new(gardenfakes.FakeContainer)
   521  	fakeContainer.PropertyReturns("123", nil)
   522  	result := s.backend.GraceTime(fakeContainer)
   523  	s.Equal(time.Duration(123), result)
   524  }