vitess.io/vitess@v0.16.2/go/vt/vtctld/api_test.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vtctld
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"io"
    24  	"net/http"
    25  	"net/http/httptest"
    26  	"strings"
    27  	"testing"
    28  
    29  	"github.com/stretchr/testify/require"
    30  
    31  	"vitess.io/vitess/go/vt/discovery"
    32  	"vitess.io/vitess/go/vt/topo/memorytopo"
    33  	"vitess.io/vitess/go/vt/wrangler"
    34  
    35  	querypb "vitess.io/vitess/go/vt/proto/query"
    36  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    37  	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
    38  )
    39  
    40  // tabletStats will create a discovery.TabletHealth object based on the given tablet configuration.
    41  func tabletStats(keyspace, cell, shard string, tabletType topodatapb.TabletType, uid uint32) (*topodatapb.Tablet, *discovery.TabletHealth) {
    42  	target := &querypb.Target{
    43  		Keyspace:   keyspace,
    44  		Shard:      shard,
    45  		TabletType: tabletType,
    46  	}
    47  	tablet := &topodatapb.Tablet{
    48  		Alias:    &topodatapb.TabletAlias{Cell: cell, Uid: uid},
    49  		Keyspace: keyspace,
    50  		Shard:    shard,
    51  		Type:     tabletType,
    52  		PortMap:  map[string]int32{"vt": int32(uid), "grpc": int32(uid + 1)},
    53  	}
    54  	realtimeStats := &querypb.RealtimeStats{
    55  		HealthError: "",
    56  		// uid is used for ReplicationLagSeconds to give it a unique value.
    57  		ReplicationLagSeconds: uid,
    58  	}
    59  	stats := &discovery.TabletHealth{
    60  		Tablet: tablet,
    61  		Target: target,
    62  		// Up:        true,
    63  		Serving:   true,
    64  		Stats:     realtimeStats,
    65  		LastError: nil,
    66  	}
    67  	return tablet, stats
    68  }
    69  
    70  func compactJSON(in []byte) string {
    71  	buf := &bytes.Buffer{}
    72  	json.Compact(buf, in)
    73  	return strings.ReplaceAll(buf.String(), "\n", "")
    74  }
    75  
    76  func TestAPI(t *testing.T) {
    77  	ctx := context.Background()
    78  	cells := []string{"cell1", "cell2"}
    79  	ts := memorytopo.NewServer(cells...)
    80  	actionRepo := NewActionRepository(ts)
    81  	server := httptest.NewServer(nil)
    82  	defer server.Close()
    83  
    84  	// Populate topo. Remove ServedTypes from shards to avoid ordering issues.
    85  	ts.CreateKeyspace(ctx, "ks1", &topodatapb.Keyspace{DurabilityPolicy: "semi_sync"})
    86  	ts.CreateShard(ctx, "ks1", "-80")
    87  	ts.CreateShard(ctx, "ks1", "80-")
    88  
    89  	// SaveVSchema to test that creating a snapshot keyspace copies VSchema
    90  	vs := &vschemapb.Keyspace{
    91  		Sharded: true,
    92  		Vindexes: map[string]*vschemapb.Vindex{
    93  			"name1": {
    94  				Type: "hash",
    95  			},
    96  		},
    97  		Tables: map[string]*vschemapb.Table{
    98  			"table1": {
    99  				ColumnVindexes: []*vschemapb.ColumnVindex{
   100  					{
   101  						Column: "column1",
   102  						Name:   "name1",
   103  					},
   104  				},
   105  			},
   106  		},
   107  	}
   108  	ts.SaveVSchema(ctx, "ks1", vs)
   109  
   110  	tablet1 := topodatapb.Tablet{
   111  		Alias:         &topodatapb.TabletAlias{Cell: "cell1", Uid: 100},
   112  		Keyspace:      "ks1",
   113  		Shard:         "-80",
   114  		Type:          topodatapb.TabletType_REPLICA,
   115  		KeyRange:      &topodatapb.KeyRange{Start: nil, End: []byte{0x80}},
   116  		PortMap:       map[string]int32{"vt": 100},
   117  		Hostname:      "mysql1-cell1.test.net",
   118  		MysqlHostname: "mysql1-cell1.test.net",
   119  		MysqlPort:     int32(3306),
   120  	}
   121  	ts.CreateTablet(ctx, &tablet1)
   122  
   123  	tablet2 := topodatapb.Tablet{
   124  		Alias:         &topodatapb.TabletAlias{Cell: "cell2", Uid: 200},
   125  		Keyspace:      "ks1",
   126  		Shard:         "-80",
   127  		Type:          topodatapb.TabletType_REPLICA,
   128  		KeyRange:      &topodatapb.KeyRange{Start: nil, End: []byte{0x80}},
   129  		PortMap:       map[string]int32{"vt": 200},
   130  		Hostname:      "mysql2-cell2.test.net",
   131  		MysqlHostname: "mysql2-cell2.test.net",
   132  		MysqlPort:     int32(3306),
   133  	}
   134  	ts.CreateTablet(ctx, &tablet2)
   135  
   136  	// Populate fake actions.
   137  	actionRepo.RegisterKeyspaceAction("TestKeyspaceAction",
   138  		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
   139  			return "TestKeyspaceAction Result", nil
   140  		})
   141  	actionRepo.RegisterShardAction("TestShardAction",
   142  		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
   143  			return "TestShardAction Result", nil
   144  		})
   145  	actionRepo.RegisterTabletAction("TestTabletAction", "",
   146  		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
   147  			return "TestTabletAction Result", nil
   148  		})
   149  
   150  	healthcheck := discovery.NewFakeHealthCheck(nil)
   151  	initAPI(ctx, ts, actionRepo, healthcheck)
   152  
   153  	t1, ts1 := tabletStats("ks1", "cell1", "-80", topodatapb.TabletType_REPLICA, 100)
   154  	t2, ts2 := tabletStats("ks1", "cell1", "-80", topodatapb.TabletType_RDONLY, 200)
   155  	t3, ts3 := tabletStats("ks1", "cell2", "80-", topodatapb.TabletType_REPLICA, 300)
   156  	t4, ts4 := tabletStats("ks1", "cell2", "80-", topodatapb.TabletType_RDONLY, 400)
   157  
   158  	t5, ts5 := tabletStats("ks2", "cell1", "0", topodatapb.TabletType_REPLICA, 500)
   159  	t6, ts6 := tabletStats("ks2", "cell2", "0", topodatapb.TabletType_REPLICA, 600)
   160  
   161  	healthcheck.AddTablet(t1)
   162  	healthcheck.AddTablet(t2)
   163  	healthcheck.AddTablet(t3)
   164  	healthcheck.AddTablet(t4)
   165  
   166  	healthcheck.AddTablet(t5)
   167  	healthcheck.AddTablet(t6)
   168  
   169  	healthcheck.UpdateHealth(ts1)
   170  	healthcheck.UpdateHealth(ts2)
   171  	healthcheck.UpdateHealth(ts3)
   172  	healthcheck.UpdateHealth(ts4)
   173  
   174  	healthcheck.UpdateHealth(ts5)
   175  	healthcheck.UpdateHealth(ts6)
   176  
   177  	// all-tablets response for keyspace/ks1/tablets/ endpoints
   178  	keyspaceKs1AllTablets := `[
   179  		{
   180  			"alias": {
   181  				"cell": "cell1",
   182  				"uid": 100
   183  			},
   184  			"hostname": "mysql1-cell1.test.net",
   185  			"port_map": {
   186  				"vt": 100
   187  			},
   188  			"keyspace": "ks1",
   189  			"shard": "-80",
   190  			"key_range": {
   191  				"end": "gA=="
   192  			},
   193  			"type": 2,
   194  			"mysql_hostname": "mysql1-cell1.test.net",
   195  			"mysql_port": 3306,
   196  			"stats": {
   197  				"realtime": {
   198  					"replication_lag_seconds": 100
   199  				},
   200  				"serving": true,
   201  				"up": true
   202  			},
   203  			"url": "http://mysql1-cell1.test.net:100"
   204  		},
   205  		{
   206  			"alias": {
   207  				"cell": "cell2",
   208  				"uid": 200
   209  			},
   210  			"hostname": "mysql2-cell2.test.net",
   211  			"port_map": {
   212  				"vt": 200
   213  			},
   214  			"keyspace": "ks1",
   215  			"shard": "-80",
   216  			"key_range": {
   217  				"end": "gA=="
   218  			},
   219  			"type": 2,
   220  			"mysql_hostname": "mysql2-cell2.test.net",
   221  			"mysql_port": 3306,
   222  			"url": "http://mysql2-cell2.test.net:200"
   223  		}
   224  	]`
   225  
   226  	// Test cases.
   227  	table := []struct {
   228  		method, path, body, want string
   229  		statusCode               int
   230  	}{
   231  		// Create snapshot keyspace with durability policy specified
   232  		{"POST", "vtctl/", `["CreateKeyspace", "--keyspace_type=SNAPSHOT", "--base_keyspace=ks1", "--snapshot_time=2006-01-02T15:04:05+00:00", "--durability-policy=semi_sync", "ks3"]`, `{
   233    "Error": "durability-policy cannot be specified while creating a snapshot keyspace"`, http.StatusOK},
   234  		// Create snapshot keyspace using API
   235  		{"POST", "vtctl/", `["CreateKeyspace", "--keyspace_type=SNAPSHOT", "--base_keyspace=ks1", "--snapshot_time=2006-01-02T15:04:05+00:00", "ks3"]`, `{
   236  		   "Error": "",
   237  		   "Output": ""
   238  		}`, http.StatusOK},
   239  
   240  		// Cells
   241  		{"GET", "cells", "", `["cell1","cell2"]`, http.StatusOK},
   242  
   243  		// Keyspace
   244  		{"GET", "keyspace/doesnt-exist/tablets/", "", ``, http.StatusNotFound},
   245  		{"GET", "keyspace/ks1/tablets/", "", keyspaceKs1AllTablets, http.StatusOK},
   246  		{"GET", "keyspace/ks1/tablets/-80", "", keyspaceKs1AllTablets, http.StatusOK},
   247  		{"GET", "keyspace/ks1/tablets/80-", "", `[]`, http.StatusOK},
   248  		{"GET", "keyspace/ks1/tablets/?cells=cell1,cell2", "", keyspaceKs1AllTablets, http.StatusOK},
   249  		{"GET", "keyspace/ks1/tablets/?cells=cell1", "", `[
   250  			{
   251  				"alias": {
   252  					"cell": "cell1",
   253  					"uid": 100
   254  				},
   255  				"hostname": "mysql1-cell1.test.net",
   256  				"port_map": {
   257  					"vt": 100
   258  				},
   259  				"keyspace": "ks1",
   260  				"shard": "-80",
   261  				"key_range": {
   262  					"end": "gA=="
   263  				},
   264  				"type": 2,
   265  				"mysql_hostname": "mysql1-cell1.test.net",
   266  				"mysql_port": 3306,
   267  				"stats": {
   268  					"realtime": {
   269  						"replication_lag_seconds": 100
   270  					},
   271  					"serving": true,
   272  					"up": true
   273  				},
   274  				"url": "http://mysql1-cell1.test.net:100"
   275  			}
   276  		]`, http.StatusOK},
   277  		{"GET", "keyspace/ks1/tablets/?cells=cell3", "", `[]`, http.StatusOK},
   278  		{"GET", "keyspace/ks1/tablets/?cell=cell2", "", `[
   279  			{
   280  				"alias": {
   281  					"cell": "cell2",
   282  					"uid": 200
   283  				},
   284  				"hostname": "mysql2-cell2.test.net",
   285  				"port_map": {
   286  					"vt": 200
   287  				},
   288  				"keyspace": "ks1",
   289  				"shard": "-80",
   290  				"key_range": {
   291  					"end": "gA=="
   292  				},
   293  				"type": 2,
   294  				"mysql_hostname": "mysql2-cell2.test.net",
   295  				"mysql_port": 3306,
   296  				"url": "http://mysql2-cell2.test.net:200"
   297  			}
   298  		]`, http.StatusOK},
   299  		{"GET", "keyspace/ks1/tablets/?cell=cell3", "", `[]`, http.StatusOK},
   300  
   301  		// Keyspaces
   302  		{"GET", "keyspaces", "", `["ks1", "ks3"]`, http.StatusOK},
   303  		{"GET", "keyspaces/ks1", "", `{
   304  				"served_froms": [],
   305  				"keyspace_type":0,
   306  				"base_keyspace":"",
   307  				"snapshot_time":null,
   308  				"durability_policy":"semi_sync",
   309  				"throttler_config": null
   310  			}`, http.StatusOK},
   311  		{"GET", "keyspaces/nonexistent", "", "404 page not found", http.StatusNotFound},
   312  		{"POST", "keyspaces/ks1?action=TestKeyspaceAction", "", `{
   313  				"Name": "TestKeyspaceAction",
   314  				"Parameters": "ks1",
   315  				"Output": "TestKeyspaceAction Result",
   316  				"Error": false
   317  			}`, http.StatusOK},
   318  
   319  		// Shards
   320  		{"GET", "shards/ks1/", "", `["-80","80-"]`, http.StatusOK},
   321  		{"GET", "shards/ks1/-80", "", `{
   322  				"primary_alias": null,
   323  				"primary_term_start_time":null,
   324  				"key_range": {
   325  					"start": "",
   326  					"end":"gA=="
   327  				},
   328  				"source_shards": [],
   329  				"tablet_controls": [],
   330  				"is_primary_serving": true
   331  			}`, http.StatusOK},
   332  		{"GET", "shards/ks1/-DEAD", "", "404 page not found", http.StatusNotFound},
   333  		{"POST", "shards/ks1/-80?action=TestShardAction", "", `{
   334  				"Name": "TestShardAction",
   335  				"Parameters": "ks1/-80",
   336  				"Output": "TestShardAction Result",
   337  				"Error": false
   338  			}`, http.StatusOK},
   339  
   340  		// Tablets
   341  		{"GET", "tablets/?shard=ks1%2F-80", "", `[
   342  				{"cell":"cell1","uid":100},
   343  				{"cell":"cell2","uid":200}
   344  			]`, http.StatusOK},
   345  		{"GET", "tablets/?cell=cell1", "", `[
   346  				{"cell":"cell1","uid":100}
   347  			]`, http.StatusOK},
   348  		{"GET", "tablets/?shard=ks1%2F-80&cell=cell2", "", `[
   349  				{"cell":"cell2","uid":200}
   350  			]`, http.StatusOK},
   351  		{"GET", "tablets/?shard=ks1%2F80-&cell=cell1", "", `[]`, http.StatusOK},
   352  		{"GET", "tablets/cell1-100", "", `{
   353  				"alias": {"cell": "cell1", "uid": 100},
   354  				"hostname": "mysql1-cell1.test.net",
   355  				"port_map": {"vt": 100},
   356  				"keyspace": "ks1",
   357  				"shard": "-80",
   358  				"key_range": {
   359  					"end": "gA=="
   360  				},
   361  				"type": 2,
   362  				"mysql_hostname": "mysql1-cell1.test.net",
   363  				"mysql_port": 3306,
   364  				"url":"http://mysql1-cell1.test.net:100"
   365  			}`, http.StatusOK},
   366  		{"GET", "tablets/nonexistent-999", "", "404 page not found", http.StatusNotFound},
   367  		{"POST", "tablets/cell1-100?action=TestTabletAction", "", `{
   368  				"Name": "TestTabletAction",
   369  				"Parameters": "cell1-0000000100",
   370  				"Output": "TestTabletAction Result",
   371  				"Error": false
   372  			}`, http.StatusOK},
   373  
   374  		// Tablet Updates
   375  		{"GET", "tablet_statuses/?keyspace=ks1&cell=cell1&type=REPLICA&metric=lag", "", `[
   376  		{
   377  		    "Data": [ [100, -1] ],
   378  		    "Aliases": [[ { "cell": "cell1", "uid": 100 }, null ]],
   379  		    "KeyspaceLabel": { "Name": "ks1", "Rowspan": 1 },
   380  		    "CellAndTypeLabels": [{ "CellLabel": { "Name": "cell1",  "Rowspan": 1 }, "TypeLabels": [{"Name": "REPLICA", "Rowspan": 1}] }] ,
   381  		    "ShardLabels": ["-80", "80-"],
   382  		    "YGridLines": [0.5]
   383  		  }
   384  		]`, http.StatusOK},
   385  		{"GET", "tablet_statuses/?keyspace=ks1&cell=all&type=all&metric=lag", "", `[
   386  		{
   387  		  "Data":[[-1,400],[-1,300],[200,-1],[100,-1]],
   388  		  "Aliases":[[null,{"cell":"cell2","uid":400}],[null,{"cell":"cell2","uid":300}],[{"cell":"cell1","uid":200},null],[{"cell":"cell1","uid":100},null]],
   389  		  "KeyspaceLabel":{"Name":"ks1","Rowspan":4},
   390  		  "CellAndTypeLabels":[
   391  		     {"CellLabel":{"Name":"cell1","Rowspan":2},"TypeLabels":[{"Name":"REPLICA","Rowspan":1},{"Name":"RDONLY","Rowspan":1}]},
   392  		     {"CellLabel":{"Name":"cell2","Rowspan":2},"TypeLabels":[{"Name":"REPLICA","Rowspan":1},{"Name":"RDONLY","Rowspan":1}]}],
   393  		  "ShardLabels":["-80","80-"],
   394  		  "YGridLines":[0.5,1.5,2.5,3.5]
   395  		}
   396  		]`, http.StatusOK},
   397  		{"GET", "tablet_statuses/?keyspace=all&cell=all&type=all&metric=lag", "", `[
   398  		  {
   399  		   "Data":[[-1,300],[200,-1]],
   400  		   "Aliases":null,
   401  		   "KeyspaceLabel":{"Name":"ks1","Rowspan":2},
   402  		  "CellAndTypeLabels":[
   403  		    {"CellLabel":{"Name":"cell1","Rowspan":1},"TypeLabels":null},
   404  		    {"CellLabel":{"Name":"cell2","Rowspan":1},"TypeLabels":null}],
   405  		  "ShardLabels":["-80","80-"],
   406  		  "YGridLines":[0.5,1.5]
   407  		  },
   408  		  {
   409  		    "Data":[[600],[500]],
   410  		   "Aliases":null,
   411  		   "KeyspaceLabel":{"Name":"ks2","Rowspan":2},
   412  		  "CellAndTypeLabels":[
   413  		    {"CellLabel":{"Name":"cell1","Rowspan":1},"TypeLabels":null},
   414  		    {"CellLabel":{"Name":"cell2","Rowspan":1},"TypeLabels":null}],
   415  		  "ShardLabels":["0"],
   416  		  "YGridLines":[0.5, 1.5]
   417  		  }
   418  		]`, http.StatusOK},
   419  		{"GET", "tablet_statuses/cell1/REPLICA/lag", "", "can't get tablet_statuses: invalid target path: \"cell1/REPLICA/lag\"  expected path: ?keyspace=<keyspace>&cell=<cell>&type=<type>&metric=<metric>", http.StatusInternalServerError},
   420  		{"GET", "tablet_statuses/?keyspace=ks1&cell=cell1&type=hello&metric=lag", "", "can't get tablet_statuses: invalid tablet type: unknown TabletType hello", http.StatusInternalServerError},
   421  
   422  		// Tablet Health
   423  		{"GET", "tablet_health/cell1/100", "", `{ "Key": ",grpc:101,vt:100", "Tablet": { "alias": { "cell": "cell1", "uid": 100 },"port_map": { "grpc": 101, "vt": 100 }, "keyspace": "ks1", "shard": "-80", "type": 2},
   424  		  "Name": "cell1-0000000100", "Target": { "keyspace": "ks1", "shard": "-80", "tablet_type": 2 }, "Up": true, "Serving": true, "PrimaryTermStartTime": 0, "TabletExternallyReparentedTimestamp": 0,
   425  		  "Stats": { "replication_lag_seconds": 100 }, "LastError": null }`, http.StatusOK},
   426  		{"GET", "tablet_health/cell1", "", "can't get tablet_health: invalid tablet_health path: \"cell1\"  expected path: /tablet_health/<cell>/<uid>", http.StatusInternalServerError},
   427  		{"GET", "tablet_health/cell1/gh", "", "can't get tablet_health: incorrect uid", http.StatusInternalServerError},
   428  
   429  		// Topology Info
   430  		{"GET", "topology_info/?keyspace=all&cell=all", "", `{
   431  		   "Keyspaces": ["ks1", "ks2"],
   432  		   "Cells": ["cell1","cell2"],
   433  		   "TabletTypes": ["REPLICA","RDONLY"]
   434  		}`, http.StatusOK},
   435  		{"GET", "topology_info/?keyspace=ks1&cell=cell1", "", `{
   436  		   "Keyspaces": ["ks1", "ks2"],
   437  		   "Cells": ["cell1","cell2"],
   438  		   "TabletTypes": ["REPLICA", "RDONLY"]
   439  		}`, http.StatusOK},
   440  
   441  		// vtctl RunCommand
   442  		{"POST", "vtctl/", `["GetKeyspace","ks1"]`, `{
   443  		   "Error": "",
   444  		   "Output": "{\n  \"served_froms\": [],\n  \"keyspace_type\": 0,\n  \"base_keyspace\": \"\",\n  \"snapshot_time\": null,\n  \"durability_policy\": \"semi_sync\",\n  \"throttler_config\": null\n}\n\n"
   445  		}`, http.StatusOK},
   446  		{"POST", "vtctl/", `["GetKeyspace","ks3"]`, `{
   447  		   "Error": "",
   448  		   "Output": "{\n  \"served_froms\": [],\n  \"keyspace_type\": 1,\n  \"base_keyspace\": \"ks1\",\n  \"snapshot_time\": {\n    \"seconds\": \"1136214245\",\n    \"nanoseconds\": 0\n  },\n  \"durability_policy\": \"none\",\n  \"throttler_config\": null\n}\n\n"
   449  		}`, http.StatusOK},
   450  		{"POST", "vtctl/", `["GetVSchema","ks3"]`, `{
   451  		   "Error": "",
   452  		   "Output": "{\n  \"sharded\": true,\n  \"vindexes\": {\n    \"name1\": {\n      \"type\": \"hash\"\n    }\n  },\n  \"tables\": {\n    \"table1\": {\n      \"columnVindexes\": [\n        {\n          \"column\": \"column1\",\n          \"name\": \"name1\"\n        }\n      ]\n    }\n  },\n  \"requireExplicitRouting\": true\n}\n\n"
   453  		}`, http.StatusOK},
   454  		{"POST", "vtctl/", `["GetKeyspace","does_not_exist"]`, `{
   455  			"Error": "node doesn't exist: keyspaces/does_not_exist/Keyspace",
   456  		   "Output": ""
   457  		}`, http.StatusOK},
   458  		{"POST", "vtctl/", `["Panic"]`, `uncaught panic: this command panics on purpose`, http.StatusInternalServerError},
   459  	}
   460  	for _, in := range table {
   461  		t.Run(in.method+in.path, func(t *testing.T) {
   462  			var resp *http.Response
   463  			var err error
   464  
   465  			switch in.method {
   466  			case "GET":
   467  				resp, err = http.Get(server.URL + apiPrefix + in.path)
   468  				require.NoError(t, err)
   469  				defer resp.Body.Close()
   470  			case "POST":
   471  				resp, err = http.Post(server.URL+apiPrefix+in.path, "application/json", strings.NewReader(in.body))
   472  				require.NoError(t, err)
   473  				defer resp.Body.Close()
   474  			default:
   475  				t.Fatalf("[%v] unknown method: %v", in.path, in.method)
   476  			}
   477  
   478  			body, err := io.ReadAll(resp.Body)
   479  			require.NoError(t, err)
   480  			require.Equal(t, in.statusCode, resp.StatusCode)
   481  
   482  			got := compactJSON(body)
   483  			want := compactJSON([]byte(in.want))
   484  			if want == "" {
   485  				// want is not valid JSON. Fallback to a string comparison.
   486  				want = in.want
   487  				// For unknown reasons errors have a trailing "\n\t\t". Remove it.
   488  				got = strings.TrimSpace(string(body))
   489  			}
   490  			if !strings.HasPrefix(got, want) {
   491  				t.Fatalf("For path [%v] got\n'%v', want\n'%v'", in.path, got, want)
   492  				return
   493  			}
   494  		})
   495  
   496  	}
   497  }