github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/topgun/k8s/worker_lifecycle_test.go (about)

     1  package k8s_test
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/onsi/gomega/gbytes"
     7  
     8  	. "github.com/pf-qiu/concourse/v6/topgun"
     9  	. "github.com/onsi/ginkgo"
    10  	. "github.com/onsi/gomega"
    11  )
    12  
    13  var _ = Describe("Worker lifecycle", func() {
    14  
    15  	var (
    16  		atc         Endpoint
    17  		gracePeriod string
    18  	)
    19  
    20  	JustBeforeEach(func() {
    21  		setReleaseNameAndNamespace("wl")
    22  
    23  		deployConcourseChart(releaseName,
    24  			`--set=worker.replicas=1`,
    25  			`--set=persistence.enabled=false`,
    26  			`--set=worker.terminationGracePeriodSeconds=`+gracePeriod,
    27  		)
    28  
    29  		atc = waitAndLogin(namespace, releaseName+"-web")
    30  
    31  		fly.Run("set-pipeline", "-n",
    32  			"-c", "pipelines/task-waiting.yml",
    33  			"-p", "some-pipeline",
    34  		)
    35  
    36  		fly.Run("unpause-pipeline", "-p", "some-pipeline")
    37  		fly.Run("trigger-job", "-j", "some-pipeline/simple-job")
    38  
    39  		By("waiting container to be created")
    40  		Eventually(func() bool {
    41  			containers := fly.GetContainers()
    42  
    43  			for _, container := range containers {
    44  				if container.Type == "task" && container.State == "created" {
    45  					return true
    46  				}
    47  			}
    48  
    49  			return false
    50  		}, 2*time.Minute, 10*time.Second).
    51  			Should(BeTrue())
    52  
    53  		Run(nil, "kubectl", "scale", "--namespace", namespace,
    54  			"statefulset", releaseName+"-worker", "--replicas=0",
    55  		)
    56  	})
    57  
    58  	AfterEach(func() {
    59  		atc.Close()
    60  		cleanupReleases()
    61  	})
    62  
    63  	Context("terminating the worker", func() {
    64  
    65  		Context("gracefully", func() {
    66  			BeforeEach(func() {
    67  				gracePeriod = "600"
    68  			})
    69  
    70  			It("finishes tasks gracefully with termination", func() {
    71  				By("seeing that the worker state is retiring")
    72  				Eventually(func() string {
    73  					workers := fly.GetWorkers()
    74  					Expect(workers).To(HaveLen(1))
    75  					return workers[0].State
    76  				}, 10*time.Second, 2*time.Second).
    77  					Should(Equal("retiring"))
    78  
    79  				By("letting the task finish")
    80  				fly.Run("hijack", "--verbose", "-j", "some-pipeline/simple-job", "-s", "simple-task",
    81  					"--", "/bin/sh", "-ce",
    82  					`touch /tmp/stop-waiting`,
    83  				)
    84  
    85  				By("seeing that there are no workers")
    86  				Eventually(func() []Worker {
    87  					return fly.GetWorkers()
    88  				}, 1*time.Minute, 1*time.Second).
    89  					Should(HaveLen(0))
    90  
    91  				By("seeing that the build succeeded")
    92  				fly.Run("watch", "-j", "some-pipeline/simple-job")
    93  			})
    94  		})
    95  
    96  		Context("ungracefully", func() {
    97  			BeforeEach(func() {
    98  				gracePeriod = "1"
    99  			})
   100  
   101  			It("interrupts the task execution", func() {
   102  				Skip("skipping because it always fails due to https://github.com/pf-qiu/concourse/v6/issues/3011")
   103  				By("seeing that there are no workers")
   104  
   105  				Eventually(func() []Worker {
   106  					return getRunningWorkers(fly.GetWorkers())
   107  				}, 1*time.Minute, 1*time.Second).
   108  					Should(HaveLen(0))
   109  
   110  				By("seeing that the worker disappeared")
   111  				startSession := fly.Start("watch", "-j", "some-pipeline/simple-job")
   112  				<-startSession.Exited
   113  				Expect(startSession.Out).To(gbytes.Say("errored"))
   114  			})
   115  		})
   116  	})
   117  })