vitess.io/vitess@v0.16.2/go/vt/topo/k8stopo/server_flaky_test.go (about)

     1  /*
     2  Copyright 2020 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package k8stopo
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"os/exec"
    24  	"path"
    25  	"runtime"
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/stretchr/testify/require"
    30  	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    31  	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	kubeyaml "k8s.io/apimachinery/pkg/util/yaml"
    34  	"k8s.io/client-go/tools/clientcmd"
    35  
    36  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    37  	"vitess.io/vitess/go/vt/topo"
    38  	"vitess.io/vitess/go/vt/topo/test"
    39  )
    40  
    41  func TestKubernetesTopo(t *testing.T) {
    42  	if runtime.GOOS != "linux" {
    43  		t.Skip("k3s not supported on non-linux platforms. Skipping k8stopo integration tests")
    44  	}
    45  
    46  	// Create a data dir for test data
    47  	testDataDir := t.TempDir()
    48  
    49  	// Gen a temp file name for the config
    50  	testConfig, err := os.CreateTemp("", "vt-test-k3s-config")
    51  	if err != nil {
    52  		t.Fatal(err)
    53  	}
    54  	testConfigPath := testConfig.Name()
    55  	defer os.Remove(testConfigPath) // clean up
    56  
    57  	k3sArgs := []string{
    58  		"server", "start",
    59  		"--write-kubeconfig=" + testConfigPath,
    60  		"--data-dir=" + testDataDir,
    61  		"--https-listen-port=6663",
    62  		"--disable-agent", "--flannel-backend=none",
    63  		"--disable-network-policy",
    64  		"--disable-cloud-controller",
    65  		"--disable-scheduler",
    66  		"--no-deploy=coredns,servicelb,traefik,local-storage,metrics-server",
    67  		"--kube-controller-manager-arg=port=10253",
    68  
    69  		"--log=/tmp/k3svtlog",
    70  	}
    71  
    72  	// Start a minimal k3s daemon, and close it after all tests are done.
    73  	ctx, killK3s := context.WithCancel(context.Background())
    74  	c := exec.CommandContext(ctx, "k3s", k3sArgs...)
    75  
    76  	// Start in the background and kill when tests end
    77  	t.Log("Starting k3s")
    78  	err = c.Start()
    79  	if err != nil {
    80  		t.Fatal("Unable to start k3s", err)
    81  	}
    82  	defer killK3s()
    83  
    84  	// Wait for server to be ready
    85  	for {
    86  		t.Log("Waiting for server to be ready")
    87  		time.Sleep(time.Second)
    88  		config, err := clientcmd.BuildConfigFromFlags("", testConfigPath)
    89  		if err != nil {
    90  			continue
    91  		}
    92  
    93  		// Create the vitesstoponode crd
    94  		apiextensionsClientSet, err := apiextensionsclient.NewForConfig(config)
    95  		if err != nil {
    96  			t.Fatal(err)
    97  		}
    98  
    99  		crdFile, err := os.Open("./VitessTopoNodes-crd.yaml")
   100  		require.NoError(t, err)
   101  		defer crdFile.Close()
   102  
   103  		crd := &extensionsv1.CustomResourceDefinition{}
   104  
   105  		kubeyaml.NewYAMLOrJSONDecoder(crdFile, 2048).Decode(crd)
   106  
   107  		_, err = apiextensionsClientSet.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
   108  		if err != nil {
   109  			t.Fatal(err)
   110  		}
   111  
   112  		break
   113  	}
   114  
   115  	serverAddr := "default"
   116  
   117  	oldKubeConfigPath := kubeconfigPath
   118  	kubeconfigPath = testConfigPath
   119  	defer func() {
   120  		kubeconfigPath = oldKubeConfigPath
   121  	}()
   122  
   123  	// Run the test suite.
   124  	testIndex := 0
   125  	test.TopoServerTestSuite(t, func() *topo.Server {
   126  		// Each test will use its own sub-directories.
   127  		// The directories will be created when used the first time.
   128  		testRoot := fmt.Sprintf("/test-%v", testIndex)
   129  		testIndex++
   130  
   131  		globalRoot := path.Join(testRoot, topo.GlobalCell)
   132  		cellRoot := path.Join(testRoot, test.LocalCellName)
   133  
   134  		ts, err := topo.OpenServer("k8s", serverAddr, globalRoot)
   135  		if err != nil {
   136  			t.Fatalf("OpenServer() failed: %v", err)
   137  		}
   138  		if err := ts.CreateCellInfo(context.Background(), test.LocalCellName, &topodatapb.CellInfo{
   139  			ServerAddress: serverAddr,
   140  			Root:          cellRoot,
   141  		}); err != nil {
   142  			t.Fatalf("CreateCellInfo() failed: %v", err)
   143  		}
   144  
   145  		return ts
   146  	}, []string{"checkTryLock", "checkShardWithLock"})
   147  }