github.com/thanos-io/thanos@v0.32.5/examples/interactive/interactive_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package e2ebench_test
     5  
     6  import (
     7  	"fmt"
     8  	"os"
     9  	execlib "os/exec"
    10  	"path/filepath"
    11  	"testing"
    12  
    13  	"github.com/efficientgo/e2e"
    14  	e2edb "github.com/efficientgo/e2e/db"
    15  	e2einteractive "github.com/efficientgo/e2e/interactive"
    16  	e2emon "github.com/efficientgo/e2e/monitoring"
    17  	"github.com/efficientgo/e2e/monitoring/promconfig"
    18  	"github.com/pkg/errors"
    19  	"github.com/prometheus/common/model"
    20  	"gopkg.in/yaml.v2"
    21  
    22  	"github.com/efficientgo/core/testutil"
    23  	"github.com/thanos-io/objstore/client"
    24  	"github.com/thanos-io/objstore/providers/s3"
    25  	tracingclient "github.com/thanos-io/thanos/pkg/tracing/client"
    26  	"github.com/thanos-io/thanos/pkg/tracing/jaeger"
    27  	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
    28  )
    29  
    30  const (
    31  	data           = "data"
    32  	defaultProfile = "continuous-30d-tiny"
    33  )
    34  
    35  var (
    36  	maxTimeFresh = `2021-07-27T00:00:00Z`
    37  	maxTimeOld   = `2021-07-20T00:00:00Z`
    38  
    39  	store1Data = func() string { a, _ := filepath.Abs(filepath.Join(data, "store1")); return a }()
    40  	store2Data = func() string { a, _ := filepath.Abs(filepath.Join(data, "store2")); return a }()
    41  	prom1Data  = func() string { a, _ := filepath.Abs(filepath.Join(data, "prom1")); return a }()
    42  	prom2Data  = func() string { a, _ := filepath.Abs(filepath.Join(data, "prom2")); return a }()
    43  )
    44  
    45  func exec(cmd string, args ...string) error {
    46  	if o, err := execlib.Command(cmd, args...).CombinedOutput(); err != nil {
    47  		return errors.Wrap(err, string(o))
    48  	}
    49  	return nil
    50  }
    51  
    52  // createData generates some blocks for us to play with and makes them
    53  // available to store and Prometheus instances.
    54  //
    55  // You can choose different profiles by setting the BLOCK_PROFILE environment variable.
    56  // Available profiles can be found at https://github.com/thanos-io/thanosbench/blob/master/pkg/blockgen/profiles.go#L28
    57  func createData() (perr error) {
    58  	profile := os.Getenv("BLOCK_PROFILE")
    59  	if profile == "" {
    60  		profile = defaultProfile //	Use "continuous-1w-1series-10000apps" if you have ~10GB of memory for creation phase.
    61  	}
    62  
    63  	fmt.Println("Re-creating data (can take minutes)...")
    64  	defer func() {
    65  		if perr != nil {
    66  			_ = os.RemoveAll(data)
    67  		}
    68  	}()
    69  
    70  	if err := exec(
    71  		"sh", "-c",
    72  		fmt.Sprintf("mkdir -p %s && "+
    73  			"docker run -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block plan -p %s --labels 'cluster=\"eu-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
    74  			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block gen --output.dir /shared", store1Data, profile, maxTimeOld, store1Data),
    75  	); err != nil {
    76  		return err
    77  	}
    78  	if err := exec(
    79  		"sh", "-c",
    80  		fmt.Sprintf("mkdir -p %s && "+
    81  			"docker run -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block plan -p %s --labels 'cluster=\"us-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
    82  			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block gen --output.dir /shared", store2Data, profile, maxTimeOld, store2Data),
    83  	); err != nil {
    84  		return err
    85  	}
    86  
    87  	if err := exec(
    88  		"sh", "-c",
    89  		fmt.Sprintf("mkdir -p %s && "+
    90  			"docker run -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block plan -p %s --max-time=%s | "+
    91  			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block gen --output.dir /shared", prom1Data, profile, maxTimeFresh, prom1Data),
    92  	); err != nil {
    93  		return err
    94  	}
    95  	if err := exec(
    96  		"sh", "-c",
    97  		fmt.Sprintf("mkdir -p %s && "+
    98  			"docker run -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block plan -p %s --max-time=%s | "+
    99  			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.3.0-rc.0 block gen --output.dir /shared", prom2Data, profile, maxTimeFresh, prom2Data),
   100  	); err != nil {
   101  		return err
   102  	}
   103  	return nil
   104  }
   105  
   106  // TestReadOnlyThanosSetup runs read only Thanos setup that has data from `maxTimeFresh - 2w` to `maxTimeOld`, with extra monitoring and tracing for full playground experience.
   107  // Run with test args `-timeout 9999m`.
   108  func TestReadOnlyThanosSetup(t *testing.T) {
   109  	t.Skip("This is interactive test - it will run until you will kill it or curl 'finish' endpoint. Comment and run as normal test to use it!")
   110  
   111  	// Create series of TSDB blocks. Cache them to 'data' dir so we don't need to re-create on every run.
   112  	_, err := os.Stat(data)
   113  	if os.IsNotExist(err) {
   114  		testutil.Ok(t, createData())
   115  	} else {
   116  		testutil.Ok(t, err)
   117  		fmt.Println("Skipping blocks generation, found data directory.")
   118  	}
   119  
   120  	e, err := e2e.NewDockerEnvironment("interactive")
   121  	testutil.Ok(t, err)
   122  	t.Cleanup(e.Close)
   123  
   124  	m, err := e2emon.Start(e)
   125  	testutil.Ok(t, err)
   126  
   127  	// Initialize object storage with two buckets (our long term storage).
   128  	//
   129  	//	┌──────────────┐
   130  	//	│              │
   131  	//	│    Minio     │
   132  	//	│              │
   133  	//	├──────────────┼──────────────────────────────────────────────────┐
   134  	//	│ Bucket: bkt1 │ {cluster=eu1, replica=0} 10k series [t-2w, t-1w] │
   135  	//	├──────────────┼──────────────────────────────────────────────────┘
   136  	//	│              │
   137  	//	├──────────────┼──────────────────────────────────────────────────┐
   138  	//	│ Bucket: bkt2 │ {cluster=us1, replica=0} 10k series [t-2w, t-1w] │
   139  	//	└──────────────┴──────────────────────────────────────────────────┘
   140  	//
   141  	m1 := e2edb.NewMinio(e, "minio-1", "default")
   142  	testutil.Ok(t, exec("cp", "-r", store1Data+"/.", filepath.Join(m1.Dir(), "bkt1")))
   143  	testutil.Ok(t, exec("cp", "-r", store2Data+"/.", filepath.Join(m1.Dir(), "bkt2")))
   144  
   145  	// Setup Jaeger.
   146  	j := e.Runnable("tracing").WithPorts(map[string]int{"http-front": 16686, "jaeger.thrift": 14268}).Init(e2e.StartOptions{Image: "jaegertracing/all-in-one:1.25"})
   147  	testutil.Ok(t, e2e.StartAndWaitReady(j))
   148  
   149  	jaegerConfig, err := yaml.Marshal(tracingclient.TracingConfig{
   150  		Type: tracingclient.Jaeger,
   151  		Config: jaeger.Config{
   152  			ServiceName:  "thanos",
   153  			SamplerType:  "const",
   154  			SamplerParam: 1,
   155  			Endpoint:     "http://" + j.InternalEndpoint("jaeger.thrift") + "/api/traces",
   156  		},
   157  	})
   158  	testutil.Ok(t, err)
   159  
   160  	// Create two store gateways, one for each bucket (access point to long term storage).
   161  	//	                    ┌───────────┐
   162  	//	                    │           │
   163  	//	┌──────────────┐    │  Store 1  │
   164  	//	│ Bucket: bkt1 │◄───┤           │
   165  	//	├──────────────┼    └───────────┘
   166  	//	│              │
   167  	//	├──────────────┼    ┌───────────┐
   168  	//	│ Bucket: bkt2 │◄───┤           │
   169  	//	└──────────────┴    │  Store 2  │
   170  	//	                    │           │
   171  	//	                    └───────────┘
   172  	bkt1Config, err := yaml.Marshal(client.BucketConfig{
   173  		Type: client.S3,
   174  		Config: s3.Config{
   175  			Bucket:    "bkt1",
   176  			AccessKey: e2edb.MinioAccessKey,
   177  			SecretKey: e2edb.MinioSecretKey,
   178  			Endpoint:  m1.InternalEndpoint("http"),
   179  			Insecure:  true,
   180  		},
   181  	})
   182  	testutil.Ok(t, err)
   183  	store1 := e2edb.NewThanosStore(
   184  		e,
   185  		"store1",
   186  		bkt1Config,
   187  		e2edb.WithImage("thanos:latest"),
   188  		e2edb.WithFlagOverride(map[string]string{
   189  			"--tracing.config":    string(jaegerConfig),
   190  			"--consistency-delay": "0s",
   191  		}),
   192  	)
   193  
   194  	bkt2Config, err := yaml.Marshal(client.BucketConfig{
   195  		Type: client.S3,
   196  		Config: s3.Config{
   197  			Bucket:    "bkt2",
   198  			AccessKey: e2edb.MinioAccessKey,
   199  			SecretKey: e2edb.MinioSecretKey,
   200  			Endpoint:  m1.InternalEndpoint("http"),
   201  			Insecure:  true,
   202  		},
   203  	})
   204  	testutil.Ok(t, err)
   205  
   206  	store2 := e2edb.NewThanosStore(
   207  		e,
   208  		"store2",
   209  		bkt2Config,
   210  		e2edb.WithImage("thanos:latest"),
   211  		e2edb.WithFlagOverride(map[string]string{
   212  			"--tracing.config":    string(jaegerConfig),
   213  			"--consistency-delay": "0s",
   214  		}),
   215  	)
   216  
   217  	// Create two Prometheus replicas in HA, and one separate one (short term storage + scraping).
   218  	// Add a Thanos sidecar.
   219  	//
   220  	//                                                         ┌────────────┐
   221  	//    ┌───────────────────────────────────────────────┐    │            │
   222  	//    │ {cluster=eu1, replica=0} 10k series [t-1w, t] │◄───┤  Prom-ha0  │
   223  	//    └───────────────────────────────────────────────┘    │            │
   224  	//                                                         ├────────────┤
   225  	//                                                         │   Sidecar  │
   226  	//                                                         └────────────┘
   227  	//
   228  	//                                                         ┌────────────┐
   229  	//    ┌───────────────────────────────────────────────┐    │            │
   230  	//    │ {cluster=eu1, replica=1} 10k series [t-1w, t] │◄───┤  Prom-ha1  │
   231  	//    └───────────────────────────────────────────────┘    │            │
   232  	//                                                         ├────────────┤
   233  	//                                                         │   Sidecar  │
   234  	//                                                         └────────────┘
   235  	//
   236  	//                                                         ┌────────────┐
   237  	//    ┌───────────────────────────────────────────────┐    │            │
   238  	//    │ {cluster=us1, replica=0} 10k series [t-1w, t] │◄───┤  Prom 2    │
   239  	//    └───────────────────────────────────────────────┘    │            │
   240  	//                                                         ├────────────┤
   241  	//                                                         │   Sidecar  │
   242  	//                                                         └────────────┘
   243  	promHA0 := e2edb.NewPrometheus(e, "prom-ha0")
   244  	promHA1 := e2edb.NewPrometheus(e, "prom-ha1")
   245  	prom2 := e2edb.NewPrometheus(e, "prom2")
   246  
   247  	sidecarHA0 := e2edb.NewThanosSidecar(e, "sidecar-prom-ha0", promHA0, e2edb.WithImage("thanos:latest"), e2edb.WithFlagOverride(map[string]string{"--tracing.config": string(jaegerConfig)}))
   248  	sidecarHA1 := e2edb.NewThanosSidecar(e, "sidecar-prom-ha1", promHA1, e2edb.WithImage("thanos:latest"), e2edb.WithFlagOverride(map[string]string{"--tracing.config": string(jaegerConfig)}))
   249  	sidecar2 := e2edb.NewThanosSidecar(e, "sidecar2", prom2, e2edb.WithImage("thanos:latest"))
   250  
   251  	receive1 := e2ethanos.NewReceiveBuilder(e, "receiver-1").WithIngestionEnabled().Init()
   252  
   253  	testutil.Ok(t, exec("cp", "-r", prom1Data+"/.", promHA0.Dir()))
   254  	testutil.Ok(t, exec("sh", "-c", "find "+prom1Data+"/ -maxdepth 1 -type d | tail -5 | xargs -I {} cp -r {} "+promHA1.Dir())) // Copy only 5 blocks from 9 to mimic replica 1 with partial data set.
   255  	testutil.Ok(t, exec("cp", "-r", prom2Data+"/.", prom2.Dir()))
   256  
   257  	testutil.Ok(t, promHA0.SetConfig(promconfig.Config{
   258  		GlobalConfig: promconfig.GlobalConfig{
   259  			ExternalLabels: map[model.LabelName]model.LabelValue{
   260  				"cluster": "eu-1",
   261  				"replica": "0",
   262  			},
   263  		},
   264  	}))
   265  	testutil.Ok(t, promHA1.SetConfig(promconfig.Config{
   266  		GlobalConfig: promconfig.GlobalConfig{
   267  			ExternalLabels: map[model.LabelName]model.LabelValue{
   268  				"cluster": "eu-1",
   269  				"replica": "1",
   270  			},
   271  		},
   272  	}))
   273  	testutil.Ok(t, prom2.SetConfig(promconfig.Config{
   274  		GlobalConfig: promconfig.GlobalConfig{
   275  			ExternalLabels: map[model.LabelName]model.LabelValue{
   276  				"cluster": "us-1",
   277  				"replica": "0",
   278  			},
   279  		},
   280  	}))
   281  
   282  	testutil.Ok(t, e2e.StartAndWaitReady(m1))
   283  	testutil.Ok(t, e2e.StartAndWaitReady(promHA0, promHA1, prom2, sidecarHA0, sidecarHA1, sidecar2, store1, store2, receive1))
   284  
   285  	// Let's start query on top of all those 6 store APIs (global query engine).
   286  	//
   287  	//  ┌──────────────┐
   288  	//  │              │
   289  	//  │    Minio     │                                                       ┌───────────┐
   290  	//  │              │                                                       │           │
   291  	//  ├──────────────┼──────────────────────────────────────────────────┐    │  Store 1  │◄──────┐
   292  	//  │ Bucket: bkt1 │ {cluster=eu1, replica=0} 10k series [t-2w, t-1w] │◄───┤           │       │
   293  	//  ├──────────────┼──────────────────────────────────────────────────┘    └───────────┘       │
   294  	//  │              │                                                                           │
   295  	//  ├──────────────┼──────────────────────────────────────────────────┐    ┌───────────┐       │
   296  	//  │ Bucket: bkt2 │ {cluster=us1, replica=0} 10k series [t-2w, t-1w] │◄───┤           │       │
   297  	//  └──────────────┴──────────────────────────────────────────────────┘    │  Store 2  │◄──────┤
   298  	//                                                                         │           │       │
   299  	//                                                                         └───────────┘       │
   300  	//                                                                                             │
   301  	//                                                                                             │
   302  	//                                                                                             │      ┌───────────────┐
   303  	//                                                                         ┌────────────┐      │      │               │
   304  	//                    ┌───────────────────────────────────────────────┐    │            │      ├──────┤    Querier    │◄────── PromQL
   305  	//                    │ {cluster=eu1, replica=0} 10k series [t-1w, t] │◄───┤  Prom-ha0  │      │      │               │
   306  	//                    └───────────────────────────────────────────────┘    │            │      │      └───────────────┘
   307  	//                                                                         ├────────────┤      │
   308  	//                                                                         │   Sidecar  │◄─────┤
   309  	//                                                                         └────────────┘      │
   310  	//                                                                                             │
   311  	//                                                                         ┌────────────┐      │
   312  	//                    ┌───────────────────────────────────────────────┐    │            │      │
   313  	//                    │ {cluster=eu1, replica=1} 10k series [t-1w, t] │◄───┤  Prom-ha1  │      │
   314  	//                    └───────────────────────────────────────────────┘    │            │      │
   315  	//                                                                         ├────────────┤      │
   316  	//                                                                         │   Sidecar  │◄─────┤
   317  	//                                                                         └────────────┘      │
   318  	//                                                                                             │
   319  	//                                                                         ┌────────────┐      │
   320  	//                    ┌───────────────────────────────────────────────┐    │            │      │
   321  	//                    │ {cluster=us1, replica=0} 10k series [t-1w, t] │◄───┤  Prom 2    │      │
   322  	//                    └───────────────────────────────────────────────┘    │            │      │
   323  	//                                                                         ├────────────┤      │
   324  	//                                                                         │   Sidecar  │◄─────┘
   325  	//                                                                         └────────────┘
   326  	//
   327  	query1 := e2edb.NewThanosQuerier(
   328  		e,
   329  		"query1",
   330  		[]string{
   331  			store1.InternalEndpoint("grpc"),
   332  			store2.InternalEndpoint("grpc"),
   333  			sidecarHA0.InternalEndpoint("grpc"),
   334  			sidecarHA1.InternalEndpoint("grpc"),
   335  			sidecar2.InternalEndpoint("grpc"),
   336  			receive1.InternalEndpoint("grpc"),
   337  		},
   338  		e2edb.WithImage("thanos:latest"),
   339  		e2edb.WithFlagOverride(map[string]string{"--tracing.config": string(jaegerConfig)}),
   340  	)
   341  	testutil.Ok(t, e2e.StartAndWaitReady(query1))
   342  
   343  	// Wait until we have 6 gRPC connections.
   344  	testutil.Ok(t, query1.WaitSumMetricsWithOptions(e2emon.Equals(6), []string{"thanos_store_nodes_grpc_connections"}, e2emon.WaitMissingMetrics()))
   345  
   346  	const path = "graph?g0.expr=sum(continuous_app_metric0)%20by%20(cluster%2C%20replica)&g0.tab=0&g0.stacked=0&g0.range_input=2w&g0.max_source_resolution=0s&g0.deduplicate=0&g0.partial_response=0&g0.store_matches=%5B%5D&g0.end_input=2021-07-27%2000%3A00%3A00"
   347  	testutil.Ok(t, e2einteractive.OpenInBrowser(fmt.Sprintf("http://%s/%s", query1.Endpoint("http"), path)))
   348  	testutil.Ok(t, e2einteractive.OpenInBrowser(fmt.Sprintf("http://%s/%s", prom2.Endpoint("http"), path)))
   349  
   350  	// Tracing endpoint.
   351  	testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+j.Endpoint("http-front")))
   352  	// Monitoring Endpoint.
   353  	testutil.Ok(t, m.OpenUserInterfaceInBrowser())
   354  	testutil.Ok(t, e2einteractive.RunUntilEndpointHit())
   355  }