github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/topgun/both/atc_shutdown_test.go (about)

     1  package topgun_test
     2  
     3  import (
     4  	"regexp"
     5  	"time"
     6  
     7  	. "github.com/pf-qiu/concourse/v6/topgun/common"
     8  	_ "github.com/lib/pq"
     9  	. "github.com/onsi/ginkgo"
    10  	. "github.com/onsi/gomega"
    11  	"github.com/onsi/gomega/gbytes"
    12  	"github.com/onsi/gomega/gexec"
    13  )
    14  
    15  var _ = Describe("ATC Shutting down", func() {
    16  	Context("with two atcs available", func() {
    17  		var atcs []BoshInstance
    18  		var atc0URL string
    19  		var atc1URL string
    20  
    21  		BeforeEach(func() {
    22  			Deploy(
    23  				"deployments/concourse.yml",
    24  				"-o", "operations/web-instances.yml",
    25  				"-v", "web_instances=2",
    26  			)
    27  
    28  			WaitForRunningWorker()
    29  
    30  			atcs = JobInstances("web")
    31  			atc0URL = "http://" + atcs[0].IP + ":8080"
    32  			atc1URL = "http://" + atcs[1].IP + ":8080"
    33  
    34  			Fly.Login(AtcUsername, AtcPassword, atc0URL)
    35  		})
    36  
    37  		Context("when one of the ATCS is stopped", func() {
    38  			var stopSession *gexec.Session
    39  
    40  			BeforeEach(func() {
    41  				By("stopping one of the web instances")
    42  				stopSession = SpawnBosh("stop", atcs[1].Name)
    43  				Eventually(stopSession).Should(gexec.Exit(0))
    44  			})
    45  
    46  			AfterEach(func() {
    47  				restartSession := SpawnBosh("start", atcs[0].Name)
    48  				<-restartSession.Exited
    49  				Eventually(restartSession).Should(gexec.Exit(0))
    50  			})
    51  
    52  			Describe("workers registering with random TSA address", func() {
    53  				It("recovers from the TSA going down by registering with a random TSA", func() {
    54  					WaitForRunningWorker()
    55  
    56  					By("stopping the other web instance")
    57  					stopSession = SpawnBosh("stop", atcs[0].Name)
    58  					Eventually(stopSession).Should(gexec.Exit(0))
    59  
    60  					By("starting the stopped web instance")
    61  					startSession := SpawnBosh("start", atcs[1].Name)
    62  					Eventually(startSession).Should(gexec.Exit(0))
    63  
    64  					atcs = JobInstances("web")
    65  					atc0URL = "http://" + atcs[0].IP + ":8080"
    66  					atc1URL = "http://" + atcs[1].IP + ":8080"
    67  
    68  					Fly.Login(AtcUsername, AtcPassword, atc1URL)
    69  
    70  					WaitForRunningWorker()
    71  				})
    72  			})
    73  		})
    74  
    75  		Describe("tracking builds previously tracked by shutdown ATC", func() {
    76  			var buildID string
    77  
    78  			BeforeEach(func() {
    79  				By("executing a task")
    80  				buildSession := Fly.Start("execute", "-c", "tasks/wait.yml")
    81  				Eventually(buildSession).Should(gbytes.Say("executing build"))
    82  
    83  				buildRegex := regexp.MustCompile(`executing build (\d+)`)
    84  				matches := buildRegex.FindSubmatch(buildSession.Out.Contents())
    85  				buildID = string(matches[1])
    86  
    87  				//For the initializing block
    88  				Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
    89  				//For the output from the running step
    90  				Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
    91  			})
    92  
    93  			Context("when the web node tracking the build shuts down", func() {
    94  				JustBeforeEach(func() {
    95  					By("restarting both web nodes")
    96  					Bosh("restart", atcs[0].Group)
    97  				})
    98  
    99  				It("continues tracking the build progress", func() {
   100  					By("waiting for another web node to attach to process")
   101  					watchSession := Fly.Start("watch", "-b", buildID)
   102  					Eventually(watchSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
   103  					time.Sleep(10 * time.Second)
   104  
   105  					By("hijacking the build to tell it to finish")
   106  					Eventually(func() int {
   107  						hijackSession := Fly.Start(
   108  							"hijack",
   109  							"-b", buildID,
   110  							"-s", "one-off", "--",
   111  							"touch", "/tmp/stop-waiting",
   112  						)
   113  						<-hijackSession.Exited
   114  						return hijackSession.ExitCode()
   115  					}).Should(Equal(0))
   116  
   117  					By("waiting for the build to exit")
   118  					Eventually(watchSession, 1*time.Minute).Should(gbytes.Say("done"))
   119  					<-watchSession.Exited
   120  					Expect(watchSession.ExitCode()).To(Equal(0))
   121  				})
   122  			})
   123  		})
   124  	})
   125  })