github.com/outbrain/consul@v1.4.5/agent/session_endpoint_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"net/http"
     7  	"net/http/httptest"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/hashicorp/consul/agent/structs"
    12  	"github.com/hashicorp/consul/api"
    13  	"github.com/hashicorp/consul/testrpc"
    14  	"github.com/hashicorp/consul/testutil/retry"
    15  	"github.com/hashicorp/consul/types"
    16  	"github.com/pascaldekloe/goe/verify"
    17  )
    18  
    19  func verifySession(r *retry.R, a *TestAgent, want structs.Session) {
    20  	args := &structs.SessionSpecificRequest{
    21  		Datacenter: "dc1",
    22  		Session:    want.ID,
    23  	}
    24  	var out structs.IndexedSessions
    25  	if err := a.RPC("Session.Get", args, &out); err != nil {
    26  		r.Fatalf("err: %v", err)
    27  	}
    28  	if len(out.Sessions) != 1 {
    29  		r.Fatalf("bad: %#v", out.Sessions)
    30  	}
    31  
    32  	// Make a copy so we don't modify the state store copy for an in-mem
    33  	// RPC and zero out the Raft info for the compare.
    34  	got := *(out.Sessions[0])
    35  	got.CreateIndex = 0
    36  	got.ModifyIndex = 0
    37  	verify.Values(r, "", got, want)
    38  }
    39  
    40  func TestSessionCreate(t *testing.T) {
    41  	t.Parallel()
    42  	a := NewTestAgent(t, t.Name(), "")
    43  	defer a.Shutdown()
    44  
    45  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
    46  
    47  	// Create a health check
    48  	args := &structs.RegisterRequest{
    49  		Datacenter: "dc1",
    50  		Node:       a.Config.NodeName,
    51  		Address:    "127.0.0.1",
    52  		Check: &structs.HealthCheck{
    53  			CheckID:   "consul",
    54  			Node:      a.Config.NodeName,
    55  			Name:      "consul",
    56  			ServiceID: "consul",
    57  			Status:    api.HealthPassing,
    58  		},
    59  	}
    60  
    61  	retry.Run(t, func(r *retry.R) {
    62  		var out struct{}
    63  		if err := a.RPC("Catalog.Register", args, &out); err != nil {
    64  			r.Fatalf("err: %v", err)
    65  		}
    66  
    67  		// Associate session with node and 2 health checks
    68  		body := bytes.NewBuffer(nil)
    69  		enc := json.NewEncoder(body)
    70  		raw := map[string]interface{}{
    71  			"Name":      "my-cool-session",
    72  			"Node":      a.Config.NodeName,
    73  			"Checks":    []types.CheckID{structs.SerfCheckID, "consul"},
    74  			"LockDelay": "20s",
    75  		}
    76  		enc.Encode(raw)
    77  
    78  		req, _ := http.NewRequest("PUT", "/v1/session/create", body)
    79  		resp := httptest.NewRecorder()
    80  		obj, err := a.srv.SessionCreate(resp, req)
    81  		if err != nil {
    82  			r.Fatalf("err: %v", err)
    83  		}
    84  
    85  		want := structs.Session{
    86  			ID:        obj.(sessionCreateResponse).ID,
    87  			Name:      "my-cool-session",
    88  			Node:      a.Config.NodeName,
    89  			Checks:    []types.CheckID{structs.SerfCheckID, "consul"},
    90  			LockDelay: 20 * time.Second,
    91  			Behavior:  structs.SessionKeysRelease,
    92  		}
    93  		verifySession(r, a, want)
    94  	})
    95  }
    96  
    97  func TestSessionCreate_Delete(t *testing.T) {
    98  	t.Parallel()
    99  	a := NewTestAgent(t, t.Name(), "")
   100  	defer a.Shutdown()
   101  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   102  
   103  	// Create a health check
   104  	args := &structs.RegisterRequest{
   105  		Datacenter: "dc1",
   106  		Node:       a.Config.NodeName,
   107  		Address:    "127.0.0.1",
   108  		Check: &structs.HealthCheck{
   109  			CheckID:   "consul",
   110  			Node:      a.Config.NodeName,
   111  			Name:      "consul",
   112  			ServiceID: "consul",
   113  			Status:    api.HealthPassing,
   114  		},
   115  	}
   116  	retry.Run(t, func(r *retry.R) {
   117  		var out struct{}
   118  		if err := a.RPC("Catalog.Register", args, &out); err != nil {
   119  			r.Fatalf("err: %v", err)
   120  		}
   121  
   122  		// Associate session with node and 2 health checks, and make it delete on session destroy
   123  		body := bytes.NewBuffer(nil)
   124  		enc := json.NewEncoder(body)
   125  		raw := map[string]interface{}{
   126  			"Name":      "my-cool-session",
   127  			"Node":      a.Config.NodeName,
   128  			"Checks":    []types.CheckID{structs.SerfCheckID, "consul"},
   129  			"LockDelay": "20s",
   130  			"Behavior":  structs.SessionKeysDelete,
   131  		}
   132  		enc.Encode(raw)
   133  
   134  		req, _ := http.NewRequest("PUT", "/v1/session/create", body)
   135  		resp := httptest.NewRecorder()
   136  		obj, err := a.srv.SessionCreate(resp, req)
   137  		if err != nil {
   138  			r.Fatalf("err: %v", err)
   139  		}
   140  
   141  		want := structs.Session{
   142  			ID:        obj.(sessionCreateResponse).ID,
   143  			Name:      "my-cool-session",
   144  			Node:      a.Config.NodeName,
   145  			Checks:    []types.CheckID{structs.SerfCheckID, "consul"},
   146  			LockDelay: 20 * time.Second,
   147  			Behavior:  structs.SessionKeysDelete,
   148  		}
   149  		verifySession(r, a, want)
   150  	})
   151  }
   152  
   153  func TestSessionCreate_DefaultCheck(t *testing.T) {
   154  	t.Parallel()
   155  	a := NewTestAgent(t, t.Name(), "")
   156  	defer a.Shutdown()
   157  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   158  
   159  	// Associate session with node and 2 health checks
   160  	body := bytes.NewBuffer(nil)
   161  	enc := json.NewEncoder(body)
   162  	raw := map[string]interface{}{
   163  		"Name":      "my-cool-session",
   164  		"Node":      a.Config.NodeName,
   165  		"LockDelay": "20s",
   166  	}
   167  	enc.Encode(raw)
   168  
   169  	req, _ := http.NewRequest("PUT", "/v1/session/create", body)
   170  	resp := httptest.NewRecorder()
   171  	retry.Run(t, func(r *retry.R) {
   172  		obj, err := a.srv.SessionCreate(resp, req)
   173  		if err != nil {
   174  			r.Fatalf("err: %v", err)
   175  		}
   176  
   177  		want := structs.Session{
   178  			ID:        obj.(sessionCreateResponse).ID,
   179  			Name:      "my-cool-session",
   180  			Node:      a.Config.NodeName,
   181  			Checks:    []types.CheckID{structs.SerfCheckID},
   182  			LockDelay: 20 * time.Second,
   183  			Behavior:  structs.SessionKeysRelease,
   184  		}
   185  		verifySession(r, a, want)
   186  	})
   187  }
   188  
   189  func TestSessionCreate_NoCheck(t *testing.T) {
   190  	t.Parallel()
   191  	a := NewTestAgent(t, t.Name(), "")
   192  	defer a.Shutdown()
   193  	testrpc.WaitForLeader(t, a.RPC, "dc1")
   194  
   195  	// Associate session with node and 2 health checks
   196  	body := bytes.NewBuffer(nil)
   197  	enc := json.NewEncoder(body)
   198  	raw := map[string]interface{}{
   199  		"Name":      "my-cool-session",
   200  		"Node":      a.Config.NodeName,
   201  		"Checks":    []types.CheckID{},
   202  		"LockDelay": "20s",
   203  	}
   204  	enc.Encode(raw)
   205  
   206  	req, _ := http.NewRequest("PUT", "/v1/session/create", body)
   207  	resp := httptest.NewRecorder()
   208  	retry.Run(t, func(r *retry.R) {
   209  		obj, err := a.srv.SessionCreate(resp, req)
   210  		if err != nil {
   211  			r.Fatalf("err: %v", err)
   212  		}
   213  
   214  		want := structs.Session{
   215  			ID:        obj.(sessionCreateResponse).ID,
   216  			Name:      "my-cool-session",
   217  			Node:      a.Config.NodeName,
   218  			Checks:    []types.CheckID{},
   219  			LockDelay: 20 * time.Second,
   220  			Behavior:  structs.SessionKeysRelease,
   221  		}
   222  		verifySession(r, a, want)
   223  	})
   224  }
   225  
   226  func TestFixupLockDelay(t *testing.T) {
   227  	t.Parallel()
   228  	inp := map[string]interface{}{
   229  		"lockdelay": float64(15),
   230  	}
   231  	if err := FixupLockDelay(inp); err != nil {
   232  		t.Fatalf("err: %v", err)
   233  	}
   234  	if inp["lockdelay"] != 15*time.Second {
   235  		t.Fatalf("bad: %v", inp)
   236  	}
   237  
   238  	inp = map[string]interface{}{
   239  		"lockDelay": float64(15 * time.Second),
   240  	}
   241  	if err := FixupLockDelay(inp); err != nil {
   242  		t.Fatalf("err: %v", err)
   243  	}
   244  	if inp["lockDelay"] != 15*time.Second {
   245  		t.Fatalf("bad: %v", inp)
   246  	}
   247  
   248  	inp = map[string]interface{}{
   249  		"LockDelay": "15s",
   250  	}
   251  	if err := FixupLockDelay(inp); err != nil {
   252  		t.Fatalf("err: %v", err)
   253  	}
   254  	if inp["LockDelay"] != 15*time.Second {
   255  		t.Fatalf("bad: %v", inp)
   256  	}
   257  }
   258  
   259  func makeTestSession(t *testing.T, srv *HTTPServer) string {
   260  	req, _ := http.NewRequest("PUT", "/v1/session/create", nil)
   261  	resp := httptest.NewRecorder()
   262  	obj, err := srv.SessionCreate(resp, req)
   263  	if err != nil {
   264  		t.Fatalf("err: %v", err)
   265  	}
   266  	sessResp := obj.(sessionCreateResponse)
   267  	return sessResp.ID
   268  }
   269  
   270  func makeTestSessionDelete(t *testing.T, srv *HTTPServer) string {
   271  	// Create Session with delete behavior
   272  	body := bytes.NewBuffer(nil)
   273  	enc := json.NewEncoder(body)
   274  	raw := map[string]interface{}{
   275  		"Behavior": "delete",
   276  	}
   277  	enc.Encode(raw)
   278  
   279  	req, _ := http.NewRequest("PUT", "/v1/session/create", body)
   280  	resp := httptest.NewRecorder()
   281  	obj, err := srv.SessionCreate(resp, req)
   282  	if err != nil {
   283  		t.Fatalf("err: %v", err)
   284  	}
   285  	sessResp := obj.(sessionCreateResponse)
   286  	return sessResp.ID
   287  }
   288  
   289  func makeTestSessionTTL(t *testing.T, srv *HTTPServer, ttl string) string {
   290  	// Create Session with TTL
   291  	body := bytes.NewBuffer(nil)
   292  	enc := json.NewEncoder(body)
   293  	raw := map[string]interface{}{
   294  		"TTL": ttl,
   295  	}
   296  	enc.Encode(raw)
   297  
   298  	req, _ := http.NewRequest("PUT", "/v1/session/create", body)
   299  	resp := httptest.NewRecorder()
   300  	obj, err := srv.SessionCreate(resp, req)
   301  	if err != nil {
   302  		t.Fatalf("err: %v", err)
   303  	}
   304  	sessResp := obj.(sessionCreateResponse)
   305  	return sessResp.ID
   306  }
   307  
   308  func TestSessionDestroy(t *testing.T) {
   309  	t.Parallel()
   310  	a := NewTestAgent(t, t.Name(), "")
   311  	defer a.Shutdown()
   312  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   313  
   314  	id := makeTestSession(t, a.srv)
   315  
   316  	req, _ := http.NewRequest("PUT", "/v1/session/destroy/"+id, nil)
   317  	resp := httptest.NewRecorder()
   318  	obj, err := a.srv.SessionDestroy(resp, req)
   319  	if err != nil {
   320  		t.Fatalf("err: %v", err)
   321  	}
   322  	if resp := obj.(bool); !resp {
   323  		t.Fatalf("should work")
   324  	}
   325  }
   326  
   327  func TestSessionCustomTTL(t *testing.T) {
   328  	t.Parallel()
   329  	ttl := 250 * time.Millisecond
   330  	a := NewTestAgent(t, t.Name(), `
   331  		session_ttl_min = "250ms"
   332  	`)
   333  	defer a.Shutdown()
   334  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   335  
   336  	retry.Run(t, func(r *retry.R) {
   337  		id := makeTestSessionTTL(t, a.srv, ttl.String())
   338  
   339  		req, _ := http.NewRequest("GET", "/v1/session/info/"+id, nil)
   340  		resp := httptest.NewRecorder()
   341  		obj, err := a.srv.SessionGet(resp, req)
   342  		if err != nil {
   343  			r.Fatalf("err: %v", err)
   344  		}
   345  		respObj, ok := obj.(structs.Sessions)
   346  		if !ok {
   347  			r.Fatalf("should work")
   348  		}
   349  		if len(respObj) != 1 {
   350  			r.Fatalf("bad: %v", respObj)
   351  		}
   352  		if respObj[0].TTL != ttl.String() {
   353  			r.Fatalf("Incorrect TTL: %s", respObj[0].TTL)
   354  		}
   355  
   356  		time.Sleep(ttl*structs.SessionTTLMultiplier + ttl)
   357  
   358  		req, _ = http.NewRequest("GET", "/v1/session/info/"+id, nil)
   359  		resp = httptest.NewRecorder()
   360  		obj, err = a.srv.SessionGet(resp, req)
   361  		if err != nil {
   362  			r.Fatalf("err: %v", err)
   363  		}
   364  		respObj, ok = obj.(structs.Sessions)
   365  		if len(respObj) != 0 {
   366  			r.Fatalf("session '%s' should have been destroyed", id)
   367  		}
   368  	})
   369  }
   370  
   371  func TestSessionTTLRenew(t *testing.T) {
   372  	// t.Parallel() // timing test. no parallel
   373  	ttl := 250 * time.Millisecond
   374  	a := NewTestAgent(t, t.Name(), `
   375  		session_ttl_min = "250ms"
   376  	`)
   377  	defer a.Shutdown()
   378  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   379  
   380  	id := makeTestSessionTTL(t, a.srv, ttl.String())
   381  
   382  	req, _ := http.NewRequest("GET", "/v1/session/info/"+id, nil)
   383  	resp := httptest.NewRecorder()
   384  	obj, err := a.srv.SessionGet(resp, req)
   385  	if err != nil {
   386  		t.Fatalf("err: %v", err)
   387  	}
   388  	respObj, ok := obj.(structs.Sessions)
   389  	if !ok {
   390  		t.Fatalf("should work")
   391  	}
   392  	if len(respObj) != 1 {
   393  		t.Fatalf("bad: %v", respObj)
   394  	}
   395  	if respObj[0].TTL != ttl.String() {
   396  		t.Fatalf("Incorrect TTL: %s", respObj[0].TTL)
   397  	}
   398  
   399  	// Sleep to consume some time before renew
   400  	time.Sleep(ttl * (structs.SessionTTLMultiplier / 3))
   401  
   402  	req, _ = http.NewRequest("PUT", "/v1/session/renew/"+id, nil)
   403  	resp = httptest.NewRecorder()
   404  	obj, err = a.srv.SessionRenew(resp, req)
   405  	if err != nil {
   406  		t.Fatalf("err: %v", err)
   407  	}
   408  	respObj, ok = obj.(structs.Sessions)
   409  	if !ok {
   410  		t.Fatalf("should work")
   411  	}
   412  	if len(respObj) != 1 {
   413  		t.Fatalf("bad: %v", respObj)
   414  	}
   415  
   416  	// Sleep for ttl * TTL Multiplier
   417  	time.Sleep(ttl * structs.SessionTTLMultiplier)
   418  
   419  	req, _ = http.NewRequest("GET", "/v1/session/info/"+id, nil)
   420  	resp = httptest.NewRecorder()
   421  	obj, err = a.srv.SessionGet(resp, req)
   422  	if err != nil {
   423  		t.Fatalf("err: %v", err)
   424  	}
   425  	respObj, ok = obj.(structs.Sessions)
   426  	if !ok {
   427  		t.Fatalf("session '%s' should have renewed", id)
   428  	}
   429  	if len(respObj) != 1 {
   430  		t.Fatalf("session '%s' should have renewed", id)
   431  	}
   432  
   433  	// now wait for timeout and expect session to get destroyed
   434  	time.Sleep(ttl * structs.SessionTTLMultiplier)
   435  
   436  	req, _ = http.NewRequest("GET", "/v1/session/info/"+id, nil)
   437  	resp = httptest.NewRecorder()
   438  	obj, err = a.srv.SessionGet(resp, req)
   439  	if err != nil {
   440  		t.Fatalf("err: %v", err)
   441  	}
   442  	respObj, ok = obj.(structs.Sessions)
   443  	if !ok {
   444  		t.Fatalf("session '%s' should have destroyed", id)
   445  	}
   446  	if len(respObj) != 0 {
   447  		t.Fatalf("session '%s' should have destroyed", id)
   448  	}
   449  }
   450  
   451  func TestSessionGet(t *testing.T) {
   452  	t.Parallel()
   453  	t.Run("", func(t *testing.T) {
   454  		a := NewTestAgent(t, t.Name(), "")
   455  		defer a.Shutdown()
   456  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   457  
   458  		req, _ := http.NewRequest("GET", "/v1/session/info/adf4238a-882b-9ddc-4a9d-5b6758e4159e", nil)
   459  		resp := httptest.NewRecorder()
   460  		retry.Run(t, func(r *retry.R) {
   461  			obj, err := a.srv.SessionGet(resp, req)
   462  			if err != nil {
   463  				r.Fatalf("err: %v", err)
   464  			}
   465  			respObj, ok := obj.(structs.Sessions)
   466  			if !ok {
   467  				r.Fatalf("should work")
   468  			}
   469  			if respObj == nil || len(respObj) != 0 {
   470  				r.Fatalf("bad: %v", respObj)
   471  			}
   472  		})
   473  	})
   474  
   475  	t.Run("", func(t *testing.T) {
   476  		a := NewTestAgent(t, t.Name(), "")
   477  		defer a.Shutdown()
   478  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   479  
   480  		id := makeTestSession(t, a.srv)
   481  
   482  		req, _ := http.NewRequest("GET", "/v1/session/info/"+id, nil)
   483  		resp := httptest.NewRecorder()
   484  		obj, err := a.srv.SessionGet(resp, req)
   485  		if err != nil {
   486  			t.Fatalf("err: %v", err)
   487  		}
   488  		respObj, ok := obj.(structs.Sessions)
   489  		if !ok {
   490  			t.Fatalf("should work")
   491  		}
   492  		if len(respObj) != 1 {
   493  			t.Fatalf("bad: %v", respObj)
   494  		}
   495  	})
   496  }
   497  
   498  func TestSessionList(t *testing.T) {
   499  	t.Run("", func(t *testing.T) {
   500  		a := NewTestAgent(t, t.Name(), "")
   501  		defer a.Shutdown()
   502  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   503  
   504  		req, _ := http.NewRequest("GET", "/v1/session/list", nil)
   505  		resp := httptest.NewRecorder()
   506  		obj, err := a.srv.SessionList(resp, req)
   507  		if err != nil {
   508  			t.Fatalf("err: %v", err)
   509  		}
   510  		respObj, ok := obj.(structs.Sessions)
   511  		if !ok {
   512  			t.Fatalf("should work")
   513  		}
   514  		if respObj == nil || len(respObj) != 0 {
   515  			t.Fatalf("bad: %v", respObj)
   516  		}
   517  	})
   518  
   519  	t.Run("", func(t *testing.T) {
   520  		a := NewTestAgent(t, t.Name(), "")
   521  		defer a.Shutdown()
   522  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   523  
   524  		var ids []string
   525  		for i := 0; i < 10; i++ {
   526  			ids = append(ids, makeTestSession(t, a.srv))
   527  		}
   528  
   529  		req, _ := http.NewRequest("GET", "/v1/session/list", nil)
   530  		resp := httptest.NewRecorder()
   531  		obj, err := a.srv.SessionList(resp, req)
   532  		if err != nil {
   533  			t.Fatalf("err: %v", err)
   534  		}
   535  		respObj, ok := obj.(structs.Sessions)
   536  		if !ok {
   537  			t.Fatalf("should work")
   538  		}
   539  		if len(respObj) != 10 {
   540  			t.Fatalf("bad: %v", respObj)
   541  		}
   542  	})
   543  }
   544  
   545  func TestSessionsForNode(t *testing.T) {
   546  	t.Parallel()
   547  	t.Run("", func(t *testing.T) {
   548  		a := NewTestAgent(t, t.Name(), "")
   549  		defer a.Shutdown()
   550  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   551  
   552  		req, _ := http.NewRequest("GET", "/v1/session/node/"+a.Config.NodeName, nil)
   553  		resp := httptest.NewRecorder()
   554  		obj, err := a.srv.SessionsForNode(resp, req)
   555  		if err != nil {
   556  			t.Fatalf("err: %v", err)
   557  		}
   558  		respObj, ok := obj.(structs.Sessions)
   559  		if !ok {
   560  			t.Fatalf("should work")
   561  		}
   562  		if respObj == nil || len(respObj) != 0 {
   563  			t.Fatalf("bad: %v", respObj)
   564  		}
   565  	})
   566  
   567  	t.Run("", func(t *testing.T) {
   568  		a := NewTestAgent(t, t.Name(), "")
   569  		defer a.Shutdown()
   570  		testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   571  
   572  		var ids []string
   573  		for i := 0; i < 10; i++ {
   574  			ids = append(ids, makeTestSession(t, a.srv))
   575  		}
   576  
   577  		req, _ := http.NewRequest("GET", "/v1/session/node/"+a.Config.NodeName, nil)
   578  		resp := httptest.NewRecorder()
   579  		obj, err := a.srv.SessionsForNode(resp, req)
   580  		if err != nil {
   581  			t.Fatalf("err: %v", err)
   582  		}
   583  		respObj, ok := obj.(structs.Sessions)
   584  		if !ok {
   585  			t.Fatalf("should work")
   586  		}
   587  		if len(respObj) != 10 {
   588  			t.Fatalf("bad: %v", respObj)
   589  		}
   590  	})
   591  }
   592  
   593  func TestSessionDeleteDestroy(t *testing.T) {
   594  	t.Parallel()
   595  	a := NewTestAgent(t, t.Name(), "")
   596  	defer a.Shutdown()
   597  	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
   598  
   599  	id := makeTestSessionDelete(t, a.srv)
   600  
   601  	// now create a new key for the session and acquire it
   602  	buf := bytes.NewBuffer([]byte("test"))
   603  	req, _ := http.NewRequest("PUT", "/v1/kv/ephemeral?acquire="+id, buf)
   604  	resp := httptest.NewRecorder()
   605  	obj, err := a.srv.KVSEndpoint(resp, req)
   606  	if err != nil {
   607  		t.Fatalf("err: %v", err)
   608  	}
   609  
   610  	if res := obj.(bool); !res {
   611  		t.Fatalf("should work")
   612  	}
   613  
   614  	// now destroy the session, this should delete the key created above
   615  	req, _ = http.NewRequest("PUT", "/v1/session/destroy/"+id, nil)
   616  	resp = httptest.NewRecorder()
   617  	obj, err = a.srv.SessionDestroy(resp, req)
   618  	if err != nil {
   619  		t.Fatalf("err: %v", err)
   620  	}
   621  	if resp := obj.(bool); !resp {
   622  		t.Fatalf("should work")
   623  	}
   624  
   625  	// Verify that the key is gone
   626  	req, _ = http.NewRequest("GET", "/v1/kv/ephemeral", nil)
   627  	resp = httptest.NewRecorder()
   628  	obj, _ = a.srv.KVSEndpoint(resp, req)
   629  	res, found := obj.(structs.DirEntries)
   630  	if found || len(res) != 0 {
   631  		t.Fatalf("bad: %v found, should be nothing", res)
   632  	}
   633  }