github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/worker/runtime/backend_test.go (about) 1 package runtime_test 2 3 import ( 4 "context" 5 "errors" 6 "sync" 7 "testing" 8 "time" 9 10 "code.cloudfoundry.org/garden" 11 "code.cloudfoundry.org/garden/gardenfakes" 12 "github.com/pf-qiu/concourse/v6/worker/runtime" 13 "github.com/pf-qiu/concourse/v6/worker/runtime/libcontainerd/libcontainerdfakes" 14 "github.com/pf-qiu/concourse/v6/worker/runtime/runtimefakes" 15 "github.com/containerd/containerd" 16 "github.com/containerd/containerd/errdefs" 17 "github.com/opencontainers/runtime-spec/specs-go" 18 "github.com/stretchr/testify/require" 19 "github.com/stretchr/testify/suite" 20 ) 21 22 type BackendSuite struct { 23 suite.Suite 24 *require.Assertions 25 26 backend runtime.GardenBackend 27 client *libcontainerdfakes.FakeClient 28 network *runtimefakes.FakeNetwork 29 userns *runtimefakes.FakeUserNamespace 30 killer *runtimefakes.FakeKiller 31 } 32 33 func (s *BackendSuite) SetupTest() { 34 s.client = new(libcontainerdfakes.FakeClient) 35 s.killer = new(runtimefakes.FakeKiller) 36 s.network = new(runtimefakes.FakeNetwork) 37 s.userns = new(runtimefakes.FakeUserNamespace) 38 39 var err error 40 s.backend, err = runtime.NewGardenBackend(s.client, 41 runtime.WithKiller(s.killer), 42 runtime.WithNetwork(s.network), 43 runtime.WithUserNamespace(s.userns), 44 ) 45 s.NoError(err) 46 } 47 48 func (s *BackendSuite) TestNew() { 49 _, err := runtime.NewGardenBackend(nil) 50 s.EqualError(err, "nil client") 51 } 52 53 func (s *BackendSuite) TestPing() { 54 for _, tc := range []struct { 55 desc string 56 versionReturn error 57 succeeds bool 58 }{ 59 { 60 desc: "fail from containerd version service", 61 succeeds: true, 62 versionReturn: nil, 63 }, 64 { 65 desc: "ok from containerd's version service", 66 succeeds: false, 67 versionReturn: errors.New("error returning version"), 68 }, 69 } { 70 s.T().Run(tc.desc, func(t *testing.T) { 71 s.client.VersionReturns(tc.versionReturn) 72 73 err := s.backend.Ping() 74 if tc.succeeds { 75 s.NoError(err) 76 return 77 } 78 79 s.EqualError(errors.Unwrap(err), "error returning version") 80 }) 81 } 82 } 83 84 var ( 85 invalidGdnSpec = garden.ContainerSpec{} 86 minimumValidGdnSpec = garden.ContainerSpec{ 87 Handle: "handle", RootFSPath: "raw:///rootfs", 88 } 89 ) 90 91 func (s *BackendSuite) TestCreateWithInvalidSpec() { 92 _, err := s.backend.Create(invalidGdnSpec) 93 94 s.Error(err) 95 s.Equal(0, s.client.NewContainerCallCount()) 96 } 97 98 func (s *BackendSuite) TestCreateWithNewContainerFailure() { 99 s.client.NewContainerReturns(nil, errors.New("err")) 100 101 _, err := s.backend.Create(minimumValidGdnSpec) 102 s.Error(err) 103 104 s.Equal(1, s.client.NewContainerCallCount()) 105 } 106 107 func (s *BackendSuite) TestCreateContainerNewTaskFailure() { 108 fakeContainer := new(libcontainerdfakes.FakeContainer) 109 110 expectedErr := errors.New("task-err") 111 fakeContainer.NewTaskReturns(nil, expectedErr) 112 113 s.client.NewContainerReturns(fakeContainer, nil) 114 115 _, err := s.backend.Create(minimumValidGdnSpec) 116 s.EqualError(errors.Unwrap(errors.Unwrap(err)), expectedErr.Error()) 117 118 s.Equal(1, fakeContainer.NewTaskCallCount()) 119 } 120 121 func (s *BackendSuite) TestCreateContainerTaskStartFailure() { 122 fakeTask := new(libcontainerdfakes.FakeTask) 123 fakeContainer := new(libcontainerdfakes.FakeContainer) 124 125 s.client.NewContainerReturns(fakeContainer, nil) 126 fakeContainer.NewTaskReturns(fakeTask, nil) 127 fakeTask.StartReturns(errors.New("start-err")) 128 129 _, err := s.backend.Create(minimumValidGdnSpec) 130 s.Error(err) 131 132 s.EqualError(errors.Unwrap(err), "start-err") 133 } 134 135 func (s *BackendSuite) TestCreateContainerSetsHandle() { 136 fakeTask := new(libcontainerdfakes.FakeTask) 137 fakeContainer := new(libcontainerdfakes.FakeContainer) 138 139 fakeContainer.IDReturns("handle") 140 fakeContainer.NewTaskReturns(fakeTask, nil) 141 142 s.client.NewContainerReturns(fakeContainer, nil) 143 cont, err := s.backend.Create(minimumValidGdnSpec) 144 s.NoError(err) 145 146 s.Equal("handle", cont.Handle()) 147 } 148 149 func (s *BackendSuite) TestCreateMaxContainersReached() { 150 backend, err := runtime.NewGardenBackend(s.client, 151 runtime.WithKiller(s.killer), 152 runtime.WithNetwork(s.network), 153 runtime.WithUserNamespace(s.userns), 154 runtime.WithMaxContainers(1), 155 runtime.WithRequestTimeout(1*time.Second), 156 ) 157 s.NoError(err) 158 159 fakeTask := new(libcontainerdfakes.FakeTask) 160 fakeContainer := new(libcontainerdfakes.FakeContainer) 161 162 fakeContainer.NewTaskReturns(fakeTask, nil) 163 s.client.NewContainerReturns(fakeContainer, nil) 164 165 s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil) 166 _, err = backend.Create(minimumValidGdnSpec) 167 s.Error(err) 168 s.Contains(err.Error(), "max containers reached") 169 } 170 171 func (s *BackendSuite) TestCreateMaxContainersReachedConcurrent() { 172 fakeTask := new(libcontainerdfakes.FakeTask) 173 fakeContainer := new(libcontainerdfakes.FakeContainer) 174 175 fakeContainer.NewTaskReturns(fakeTask, nil) 176 177 s.client.NewContainerStub = func(context context.Context, str string, strings map[string]string, spec *specs.Spec) (container containerd.Container, e error) { 178 s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil) 179 return fakeContainer, nil 180 } 181 182 backend, err := runtime.NewGardenBackend(s.client, 183 runtime.WithKiller(s.killer), 184 runtime.WithNetwork(s.network), 185 runtime.WithUserNamespace(s.userns), 186 runtime.WithMaxContainers(1), 187 runtime.WithRequestTimeout(1*time.Second), 188 ) 189 s.NoError(err) 190 191 numberOfRequests := 10 192 requestErrors := make(chan error, numberOfRequests) 193 wg := sync.WaitGroup{} 194 wg.Add(numberOfRequests) 195 196 for i := 0; i < numberOfRequests; i++ { 197 go func() { 198 _, err := backend.Create(minimumValidGdnSpec) 199 if err != nil { 200 requestErrors <- err 201 } 202 wg.Done() 203 }() 204 } 205 wg.Wait() 206 close(requestErrors) 207 208 s.Len(requestErrors, numberOfRequests-1) 209 s.Equal(s.client.NewContainerCallCount(), 1) 210 for err := range requestErrors { 211 s.Contains(err.Error(), "max containers reached") 212 } 213 } 214 215 func (s *BackendSuite) TestCreateContainerLockTimeout() { 216 fakeTask := new(libcontainerdfakes.FakeTask) 217 fakeContainer := new(libcontainerdfakes.FakeContainer) 218 219 fakeContainer.IDReturns("handle") 220 fakeContainer.NewTaskReturns(fakeTask, nil) 221 222 s.client.NewContainerStub = func(context context.Context, str string, strings map[string]string, spec *specs.Spec) (container containerd.Container, e error) { 223 s.client.ContainersReturns([]containerd.Container{fakeContainer}, nil) 224 time.Sleep(500 * time.Millisecond) 225 return fakeContainer, nil 226 } 227 228 numberOfRequests := 10 229 230 backend, err := runtime.NewGardenBackend(s.client, 231 runtime.WithKiller(s.killer), 232 runtime.WithNetwork(s.network), 233 runtime.WithUserNamespace(s.userns), 234 runtime.WithRequestTimeout(10*time.Millisecond), 235 runtime.WithMaxContainers(numberOfRequests), 236 ) 237 s.NoError(err) 238 239 requestErrors := make(chan error, numberOfRequests) 240 wg := sync.WaitGroup{} 241 wg.Add(numberOfRequests) 242 243 for i := 0; i < numberOfRequests; i++ { 244 go func() { 245 _, err := backend.Create(minimumValidGdnSpec) 246 if err != nil { 247 requestErrors <- err 248 } 249 wg.Done() 250 }() 251 } 252 wg.Wait() 253 close(requestErrors) 254 255 s.Len(requestErrors, numberOfRequests-1) 256 for err := range requestErrors { 257 s.Contains(err.Error(), "acquiring create container lock") 258 } 259 } 260 func (s *BackendSuite) TestContainersWithContainerdFailure() { 261 s.client.ContainersReturns(nil, errors.New("err")) 262 263 _, err := s.backend.Containers(nil) 264 s.Error(err) 265 s.Equal(1, s.client.ContainersCallCount()) 266 } 267 268 func (s *BackendSuite) TestContainersWithInvalidPropertyFilters() { 269 for _, tc := range []struct { 270 desc string 271 filter map[string]string 272 }{ 273 { 274 desc: "empty key", 275 filter: map[string]string{ 276 "": "bar", 277 }, 278 }, 279 { 280 desc: "empty value", 281 filter: map[string]string{ 282 "foo": "", 283 }, 284 }, 285 } { 286 s.T().Run(tc.desc, func(t *testing.T) { 287 _, err := s.backend.Containers(tc.filter) 288 289 s.Error(err) 290 s.Equal(0, s.client.ContainersCallCount()) 291 }) 292 } 293 } 294 295 func (s *BackendSuite) TestContainersWithProperProperties() { 296 _, _ = s.backend.Containers(map[string]string{"foo": "bar", "caz": "zaz"}) 297 s.Equal(1, s.client.ContainersCallCount()) 298 299 _, labelSet := s.client.ContainersArgsForCall(0) 300 s.ElementsMatch([]string{"labels.foo==bar", "labels.caz==zaz"}, labelSet) 301 } 302 303 func (s *BackendSuite) TestContainersConversion() { 304 fakeContainer1 := new(libcontainerdfakes.FakeContainer) 305 fakeContainer2 := new(libcontainerdfakes.FakeContainer) 306 307 s.client.ContainersReturns([]containerd.Container{ 308 fakeContainer1, fakeContainer2, 309 }, nil) 310 311 containers, err := s.backend.Containers(nil) 312 s.NoError(err) 313 s.Equal(1, s.client.ContainersCallCount()) 314 s.Len(containers, 2) 315 } 316 317 func (s *BackendSuite) TestLookupEmptyHandleError() { 318 _, err := s.backend.Lookup("") 319 s.Equal("empty handle", err.Error()) 320 } 321 322 func (s *BackendSuite) TestLookupCallGetContainerWithHandle() { 323 fakeContainer := new(libcontainerdfakes.FakeContainer) 324 fakeContainer.IDReturns("handle") 325 s.client.GetContainerReturns(fakeContainer, nil) 326 327 _, _ = s.backend.Lookup("handle") 328 s.Equal(1, s.client.GetContainerCallCount()) 329 330 _, handle := s.client.GetContainerArgsForCall(0) 331 s.Equal("handle", handle) 332 } 333 334 func (s *BackendSuite) TestLookupGetContainerError() { 335 fakeContainer := new(libcontainerdfakes.FakeContainer) 336 fakeContainer.IDReturns("handle") 337 s.client.GetContainerReturns(fakeContainer, nil) 338 339 s.client.GetContainerReturns(nil, errors.New("containerd-err")) 340 341 _, err := s.backend.Lookup("handle") 342 s.Error(err) 343 s.EqualError(errors.Unwrap(err), "containerd-err") 344 } 345 346 func (s *BackendSuite) TestLookupGetContainerFails() { 347 s.client.GetContainerReturns(nil, errors.New("err")) 348 _, err := s.backend.Lookup("non-existent-handle") 349 s.Error(err) 350 s.EqualError(errors.Unwrap(err), "err") 351 } 352 353 func (s *BackendSuite) TestLookupGetNoContainerReturned() { 354 s.client.GetContainerReturns(nil, errors.New("not found")) 355 container, err := s.backend.Lookup("non-existent-handle") 356 s.Error(err) 357 s.Nil(container) 358 } 359 360 func (s *BackendSuite) TestLookupGetContainer() { 361 fakeContainer := new(libcontainerdfakes.FakeContainer) 362 fakeContainer.IDReturns("handle") 363 s.client.GetContainerReturns(fakeContainer, nil) 364 container, err := s.backend.Lookup("handle") 365 s.NoError(err) 366 s.NotNil(container) 367 s.Equal("handle", container.Handle()) 368 } 369 370 func (s *BackendSuite) TestDestroyEmptyHandleError() { 371 err := s.backend.Destroy("") 372 s.EqualError(err, "empty handle") 373 } 374 375 func (s *BackendSuite) TestDestroyGetContainerError() { 376 s.client.GetContainerReturns(nil, errors.New("get-container-failed")) 377 378 err := s.backend.Destroy("some-handle") 379 s.EqualError(errors.Unwrap(err), "get-container-failed") 380 } 381 382 func (s *BackendSuite) TestDestroyGetTaskError() { 383 fakeContainer := new(libcontainerdfakes.FakeContainer) 384 385 s.client.GetContainerReturns(fakeContainer, nil) 386 387 expectedError := errors.New("get-task-failed") 388 fakeContainer.TaskReturns(nil, expectedError) 389 390 err := s.backend.Destroy("some handle") 391 s.True(errors.Is(err, expectedError)) 392 } 393 394 func (s *BackendSuite) TestDestroyGetTaskErrorNotFoundAndDeleteFails() { 395 fakeContainer := new(libcontainerdfakes.FakeContainer) 396 397 s.client.GetContainerReturns(fakeContainer, nil) 398 fakeContainer.TaskReturns(nil, errdefs.ErrNotFound) 399 400 expectedError := errors.New("delete-container-failed") 401 fakeContainer.DeleteReturns(expectedError) 402 403 err := s.backend.Destroy("some handle") 404 s.True(errors.Is(err, expectedError)) 405 } 406 407 func (s *BackendSuite) TestDestroyGetTaskErrorNotFoundAndDeleteSucceeds() { 408 fakeContainer := new(libcontainerdfakes.FakeContainer) 409 410 s.client.GetContainerReturns(fakeContainer, nil) 411 fakeContainer.TaskReturns(nil, errdefs.ErrNotFound) 412 413 err := s.backend.Destroy("some handle") 414 415 s.Equal(1, fakeContainer.DeleteCallCount()) 416 s.NoError(err) 417 } 418 419 func (s *BackendSuite) TestDestroyKillTaskFails() { 420 fakeContainer := new(libcontainerdfakes.FakeContainer) 421 fakeTask := new(libcontainerdfakes.FakeTask) 422 423 s.client.GetContainerReturns(fakeContainer, nil) 424 fakeContainer.TaskReturns(fakeTask, nil) 425 426 expectedError := errors.New("kill-task-failed") 427 s.killer.KillReturns(expectedError) 428 429 err := s.backend.Destroy("some handle") 430 s.True(errors.Is(err, expectedError)) 431 _, _, behaviour := s.killer.KillArgsForCall(0) 432 s.Equal(runtime.KillGracefully, behaviour) 433 } 434 435 func (s *BackendSuite) TestDestroyRemoveNetworkFails() { 436 fakeContainer := new(libcontainerdfakes.FakeContainer) 437 fakeTask := new(libcontainerdfakes.FakeTask) 438 439 s.client.GetContainerReturns(fakeContainer, nil) 440 fakeContainer.TaskReturns(fakeTask, nil) 441 442 expectedError := errors.New("remove-network-failed") 443 s.network.RemoveReturns(expectedError) 444 445 err := s.backend.Destroy("some handle") 446 s.True(errors.Is(err, expectedError)) 447 } 448 449 func (s *BackendSuite) TestDestroyDeleteTaskFails() { 450 fakeContainer := new(libcontainerdfakes.FakeContainer) 451 fakeTask := new(libcontainerdfakes.FakeTask) 452 453 s.client.GetContainerReturns(fakeContainer, nil) 454 fakeContainer.TaskReturns(fakeTask, nil) 455 456 expectedError := errors.New("delete-task-failed") 457 fakeTask.DeleteReturns(nil, expectedError) 458 459 err := s.backend.Destroy("some handle") 460 s.True(errors.Is(err, expectedError)) 461 } 462 463 func (s *BackendSuite) TestDestroyContainerDeleteFailsAndDeleteTaskSucceeds() { 464 fakeContainer := new(libcontainerdfakes.FakeContainer) 465 fakeTask := new(libcontainerdfakes.FakeTask) 466 467 s.client.GetContainerReturns(fakeContainer, nil) 468 fakeContainer.TaskReturns(fakeTask, nil) 469 470 expectedError := errors.New("delete-container-failed") 471 fakeContainer.DeleteReturns(expectedError) 472 473 err := s.backend.Destroy("some handle") 474 s.True(errors.Is(err, expectedError)) 475 } 476 477 func (s *BackendSuite) TestDestroySucceeds() { 478 fakeContainer := new(libcontainerdfakes.FakeContainer) 479 fakeTask := new(libcontainerdfakes.FakeTask) 480 s.client.GetContainerReturns(fakeContainer, nil) 481 fakeContainer.TaskReturns(fakeTask, nil) 482 483 err := s.backend.Destroy("some handle") 484 s.NoError(err) 485 } 486 487 func (s *BackendSuite) TestStartInitsClientAndSetsUpRestrictedNetworks() { 488 err := s.backend.Start() 489 s.NoError(err) 490 s.Equal(1, s.client.InitCallCount()) 491 s.Equal(1, s.network.SetupRestrictedNetworksCallCount()) 492 } 493 494 func (s *BackendSuite) TestStartInitError() { 495 s.client.InitReturns(errors.New("init failed")) 496 err := s.backend.Start() 497 s.EqualError(errors.Unwrap(err), "init failed") 498 } 499 500 func (s *BackendSuite) TestStop() { 501 s.backend.Stop() 502 s.Equal(1, s.client.StopCallCount()) 503 } 504 505 func (s *BackendSuite) TestGraceTimeGetPropertyFails() { 506 fakeContainer := new(gardenfakes.FakeContainer) 507 fakeContainer.PropertyReturns("", errors.New("error")) 508 result := s.backend.GraceTime(fakeContainer) 509 s.Equal(time.Duration(0), result) 510 } 511 512 func (s *BackendSuite) TestGraceTimeInvalidInteger() { 513 fakeContainer := new(gardenfakes.FakeContainer) 514 fakeContainer.PropertyReturns("not a number", nil) 515 result := s.backend.GraceTime(fakeContainer) 516 s.Equal(time.Duration(0), result) 517 } 518 519 func (s *BackendSuite) TestGraceTimeReturnsDuration() { 520 fakeContainer := new(gardenfakes.FakeContainer) 521 fakeContainer.PropertyReturns("123", nil) 522 result := s.backend.GraceTime(fakeContainer) 523 s.Equal(time.Duration(123), result) 524 }