github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/cmd/roachtest/scaledata.go (about)

     1  // Copyright 2018 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package main
    12  
    13  import (
    14  	"context"
    15  	"fmt"
    16  	"runtime"
    17  	"strings"
    18  	"time"
    19  
    20  	"github.com/cockroachdb/cockroach/pkg/util/binfetcher"
    21  )
    22  
    23  func registerScaleData(r *testRegistry) {
    24  	// apps is a suite of Sqlapp applications designed to be used to check the
    25  	// consistency of a database under load. Each Sqlapp application launches a
    26  	// set of workers who perform database operations while another worker
    27  	// periodically checks invariants to capture any inconsistencies. The
    28  	// application suite has been pulled from:
    29  	// github.com/scaledata/rksql/tree/master/src/go/src/rubrik/sqlapp
    30  	//
    31  	// The map provides a mapping between application name and command-line
    32  	// flags unique to that application.
    33  	apps := map[string]string{
    34  		"distributed_semaphore": "",
    35  		"filesystem_simulator":  "",
    36  		"jobcoordinator":        "--num_jobs_per_worker=8 --job_period_scale_millis=100",
    37  	}
    38  
    39  	for app, flags := range apps {
    40  		app, flags := app, flags // copy loop iterator vars
    41  		const duration = 10 * time.Minute
    42  		for _, n := range []int{3, 6} {
    43  			r.Add(testSpec{
    44  				Name:    fmt.Sprintf("scaledata/%s/nodes=%d", app, n),
    45  				Owner:   OwnerKV,
    46  				Timeout: 2 * duration,
    47  				Cluster: makeClusterSpec(n + 1),
    48  				Run: func(ctx context.Context, t *test, c *cluster) {
    49  					runSqlapp(ctx, t, c, app, flags, duration)
    50  				},
    51  			})
    52  		}
    53  	}
    54  }
    55  
    56  func runSqlapp(ctx context.Context, t *test, c *cluster, app, flags string, dur time.Duration) {
    57  	roachNodeCount := c.spec.NodeCount - 1
    58  	roachNodes := c.Range(1, roachNodeCount)
    59  	appNode := c.Node(c.spec.NodeCount)
    60  
    61  	if local && runtime.GOOS != "linux" {
    62  		t.Fatalf("must run on linux os, found %s", runtime.GOOS)
    63  	}
    64  	b, err := binfetcher.Download(ctx, binfetcher.Options{
    65  		Component: "rubrik",
    66  		Binary:    app,
    67  		Version:   "LATEST",
    68  		GOOS:      "linux",
    69  		GOARCH:    "amd64",
    70  	})
    71  	if err != nil {
    72  		t.Fatal(err)
    73  	}
    74  
    75  	c.Put(ctx, b, app, appNode)
    76  	c.Put(ctx, cockroach, "./cockroach", roachNodes)
    77  	c.Start(ctx, t, roachNodes)
    78  
    79  	// TODO(nvanbenschoten): We are currently running these consistency checks with
    80  	// basic chaos. We should also run them in more chaotic environments which
    81  	// could introduce network partitions, ENOSPACE, clock issues, etc.
    82  
    83  	// Sqlapps each take a `--cockroach_ip_addresses_csv` flag, which is a
    84  	// comma-separated list of node IP addresses with optional port specifiers.
    85  	addrStr := strings.Join(c.InternalAddr(ctx, c.Range(1, roachNodeCount)), ",")
    86  
    87  	m := newMonitor(ctx, c, roachNodes)
    88  	{
    89  		// Kill one node at a time, with a minute of healthy cluster and thirty
    90  		// seconds of down node.
    91  		ch := Chaos{
    92  			Timer:   Periodic{Period: 90 * time.Second, DownTime: 30 * time.Second},
    93  			Target:  roachNodes.randNode,
    94  			Stopper: time.After(dur),
    95  		}
    96  		m.Go(ch.Runner(c, m))
    97  	}
    98  	m.Go(func(ctx context.Context) error {
    99  		t.Status("installing schema")
   100  		err := c.RunE(ctx, appNode, fmt.Sprintf("./%s --install_schema "+
   101  			"--cockroach_ip_addresses_csv='%s' %s", app, addrStr, flags))
   102  		if err != nil {
   103  			return err
   104  		}
   105  
   106  		t.Status("running consistency checker")
   107  		const workers = 16
   108  		return c.RunE(ctx, appNode, fmt.Sprintf("./%s  --duration_secs=%d "+
   109  			"--num_workers=%d --cockroach_ip_addresses_csv='%s' %s",
   110  			app, int(dur.Seconds()), workers, addrStr, flags))
   111  	})
   112  	m.Wait()
   113  }