github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/topgun/k8s/baggageclaim_drivers_test.go (about)

     1  package k8s_test
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	. "github.com/onsi/ginkgo"
     9  	. "github.com/onsi/gomega"
    10  	corev1 "k8s.io/api/core/v1"
    11  	v1 "k8s.io/api/storage/v1"
    12  	k8sErrs "k8s.io/apimachinery/pkg/api/errors"
    13  	"k8s.io/apimachinery/pkg/api/resource"
    14  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    15  )
    16  
    17  var _ = Describe("baggageclaim drivers", func() {
    18  
    19  	AfterEach(func() {
    20  		cleanupReleases()
    21  	})
    22  
    23  	onPks(func() {
    24  		baggageclaimWorks("btrfs")
    25  		baggageclaimWorks("overlay")
    26  		baggageclaimWorks("naive")
    27  	})
    28  
    29  	onGke(func() {
    30  
    31  		const (
    32  			COS    = "--set=worker.nodeSelector.nodeImage=cos"
    33  			UBUNTU = "--set=worker.nodeSelector.nodeImage=ubuntu"
    34  		)
    35  
    36  		Context("cos image", func() {
    37  			baggageclaimFails("btrfs", COS)
    38  			baggageclaimWorks("overlay", COS)
    39  			baggageclaimWorks("naive", COS)
    40  		})
    41  
    42  		Context("ubuntu image", func() {
    43  			baggageclaimWorks("btrfs", UBUNTU)
    44  			baggageclaimWorks("overlay", UBUNTU)
    45  			baggageclaimWorks("naive", UBUNTU)
    46  		})
    47  
    48  		Context("with a real btrfs partition", func() {
    49  			It("successfully recreates the worker", func() {
    50  				By("deploying concourse with ONLY one worker and having the worker pod use the gcloud disk and format it with btrfs")
    51  
    52  				scName := "btrfs"
    53  				createBtrfsStorageClass(scName)
    54  
    55  				setReleaseNameAndNamespace("real-btrfs-disk")
    56  				pvcName := "disk-" + namespace
    57  
    58  				deployWithDriverAndSelectors("btrfs", UBUNTU,
    59  					"--set=persistence.enabled=false",
    60  					"--set=worker.additionalVolumes[0].name=concourse-work-dir",
    61  					"--set=worker.additionalVolumes[0].persistentVolumeClaim.claimName="+pvcName,
    62  				)
    63  
    64  				// We create the PVC after deploying because we need the namespace to be created first by helm
    65  				createPVC(pvcName, scName)
    66  
    67  				atc := waitAndLogin(namespace, releaseName+"-web")
    68  				defer atc.Close()
    69  
    70  				By("Setting and triggering a pipeline that always fails which creates volumes on the persistent disk")
    71  				fly.Run("set-pipeline", "-n", "-c", "pipelines/pipeline-that-fails.yml", "-p", "failing-pipeline")
    72  				fly.Run("unpause-pipeline", "-p", "failing-pipeline")
    73  				sessionTriggerJob := fly.Start("trigger-job", "-w", "-j", "failing-pipeline/simple-job")
    74  				<-sessionTriggerJob.Exited
    75  
    76  				By("deleting the worker pod which triggers the initContainer script")
    77  				deletePods(releaseName, fmt.Sprintf("--selector=app=%s-worker", releaseName))
    78  
    79  				By("all pods should be running")
    80  				waitAllPodsInNamespaceToBeReady(namespace)
    81  			})
    82  		})
    83  
    84  	})
    85  })
    86  
    87  func baggageclaimWorks(driver string, selectorFlags ...string) {
    88  	Context(driver, func() {
    89  		It("works", func() {
    90  			setReleaseNameAndNamespace("bd-" + driver)
    91  			deployWithDriverAndSelectors(driver, selectorFlags...)
    92  
    93  			atc := waitAndLogin(namespace, releaseName+"-web")
    94  			defer atc.Close()
    95  
    96  			By("Setting and triggering a dumb pipeline")
    97  			fly.Run("set-pipeline", "-n", "-c", "pipelines/get-task.yml", "-p", "some-pipeline")
    98  			fly.Run("unpause-pipeline", "-p", "some-pipeline")
    99  			fly.Run("trigger-job", "-w", "-j", "some-pipeline/simple-job")
   100  		})
   101  	})
   102  }
   103  
   104  func baggageclaimFails(driver string, selectorFlags ...string) {
   105  	Context(driver, func() {
   106  		It("fails", func() {
   107  			setReleaseNameAndNamespace("bd-" + driver)
   108  			deployWithDriverAndSelectors(driver, selectorFlags...)
   109  
   110  			Eventually(func() []byte {
   111  				var logs []byte
   112  				pods := getPods(namespace, metav1.ListOptions{LabelSelector: "app=" + namespace + "-worker"})
   113  				for _, p := range pods {
   114  					contents, _ := kubeClient.CoreV1().Pods(namespace).GetLogs(p.Name, &corev1.PodLogOptions{}).Do(context.TODO()).Raw()
   115  					logs = append(logs, contents...)
   116  				}
   117  
   118  				return logs
   119  
   120  			}, 2*time.Minute, 1*time.Second).Should(ContainSubstring("failed-to-set-up-driver"))
   121  		})
   122  	})
   123  }
   124  
   125  func deployWithDriverAndSelectors(driver string, selectorFlags ...string) {
   126  	helmDeployTestFlags := []string{
   127  		"--set=concourse.web.kubernetes.enabled=false",
   128  		"--set=concourse.worker.baggageclaim.driver=" + driver,
   129  		"--set=worker.replicas=1",
   130  	}
   131  
   132  	deployConcourseChart(releaseName, append(helmDeployTestFlags, selectorFlags...)...)
   133  }
   134  
   135  func createBtrfsStorageClass(name string) {
   136  	_, err := kubeClient.StorageV1().StorageClasses().Create(context.TODO(), &v1.StorageClass{
   137  		ObjectMeta:  metav1.ObjectMeta{Name: "btrfs"},
   138  		Provisioner: "kubernetes.io/gce-pd",
   139  		Parameters: map[string]string{
   140  			"type":   "pd-standard",
   141  			"fstype": name,
   142  		},
   143  	}, metav1.CreateOptions{})
   144  	if err != nil && !k8sErrs.IsAlreadyExists(err) {
   145  		Fail("failed to create btrfs storage class: " + err.Error())
   146  	}
   147  }
   148  func createPVC(name string, scName string) {
   149  	_, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).
   150  		Create(context.TODO(), &corev1.PersistentVolumeClaim{
   151  			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
   152  			Spec: corev1.PersistentVolumeClaimSpec{
   153  				StorageClassName: &scName,
   154  				AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
   155  				Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
   156  					corev1.ResourceStorage: resource.MustParse("1Gi")}},
   157  			}}, metav1.CreateOptions{})
   158  	Expect(err).To(BeNil(), "failed to create persistent volume claim "+name)
   159  }