github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/cmd/roachtest/cli.go (about)

     1  // Copyright 2018 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package main
    12  
    13  import (
    14  	"context"
    15  	"reflect"
    16  	"strings"
    17  	"time"
    18  )
    19  
    20  func runCLINodeStatus(ctx context.Context, t *test, c *cluster) {
    21  	c.Put(ctx, cockroach, "./cockroach")
    22  	c.Start(ctx, t, c.Range(1, 3))
    23  
    24  	db := c.Conn(ctx, 1)
    25  	defer db.Close()
    26  
    27  	waitForFullReplication(t, db)
    28  
    29  	lastWords := func(s string) []string {
    30  		var result []string
    31  		for _, line := range strings.Split(s, "\n") {
    32  			words := strings.Fields(line)
    33  			if n := len(words); n > 0 {
    34  				result = append(result, words[n-2]+" "+words[n-1])
    35  			}
    36  		}
    37  		return result
    38  	}
    39  
    40  	nodeStatus := func() (raw string, _ []string) {
    41  		out, err := c.RunWithBuffer(ctx, t.l, c.Node(1),
    42  			"./cockroach node status --insecure -p {pgport:1}")
    43  		if err != nil {
    44  			t.Fatalf("%v\n%s", err, out)
    45  		}
    46  		raw = string(out)
    47  		return raw, lastWords(string(out))
    48  	}
    49  
    50  	{
    51  		expected := []string{
    52  			"is_available is_live",
    53  			"true true",
    54  			"true true",
    55  			"true true",
    56  		}
    57  		raw, actual := nodeStatus()
    58  		if !reflect.DeepEqual(expected, actual) {
    59  			t.Fatalf("expected %s, but found %s:\nfrom:\n%s", expected, actual, raw)
    60  		}
    61  	}
    62  
    63  	waitUntil := func(expected []string) {
    64  		var raw string
    65  		var actual []string
    66  		// Node liveness takes ~9s to time out. Give the test double that time.
    67  		for i := 0; i < 20; i++ {
    68  			raw, actual = nodeStatus()
    69  			if reflect.DeepEqual(expected, actual) {
    70  				break
    71  			}
    72  			t.l.Printf("not done: %s vs %s\n", expected, actual)
    73  			time.Sleep(time.Second)
    74  		}
    75  		if !reflect.DeepEqual(expected, actual) {
    76  			t.Fatalf("expected %s, but found %s from:\n%s", expected, actual, raw)
    77  		}
    78  	}
    79  
    80  	// Kill node 2 and wait for it to be marked as !is_available and !is_live.
    81  	c.Stop(ctx, c.Node(2))
    82  	waitUntil([]string{
    83  		"is_available is_live",
    84  		"true true",
    85  		"false false",
    86  		"true true",
    87  	})
    88  
    89  	// Kill node 3 and wait for all of the nodes to be marked as
    90  	// !is_available. Node 1 is not available because the liveness check can no
    91  	// longer write to the liveness range due to lack of quorum. This test is
    92  	// verifying that "node status" still returns info in this situation since
    93  	// it only accesses gossip info.
    94  	c.Stop(ctx, c.Node(3))
    95  	waitUntil([]string{
    96  		"is_available is_live",
    97  		"false true",
    98  		"false false",
    99  		"false false",
   100  	})
   101  
   102  	// Stop the cluster and restart only 2 of the nodes. Verify that three nodes
   103  	// show up in the node status output.
   104  	c.Stop(ctx, c.Range(1, 3))
   105  	c.Start(ctx, t, c.Range(1, 2))
   106  
   107  	// Wait for the cluster to come back up.
   108  	waitForFullReplication(t, db)
   109  
   110  	waitUntil([]string{
   111  		"is_available is_live",
   112  		"true true",
   113  		"true true",
   114  		"false false",
   115  	})
   116  
   117  	// Start node again to satisfy roachtest.
   118  	c.Start(ctx, t, c.Node(3))
   119  }