github.com/janelia-flyem/dvid@v1.0.0/datatype/keyvalue/keyvalue_test.go (about)

     1  package keyvalue
     2  
     3  import (
     4  	"archive/tar"
     5  	"bytes"
     6  	"crypto/rand"
     7  	"encoding/json"
     8  	"fmt"
     9  	"io"
    10  	"log"
    11  	"net/http"
    12  	"strings"
    13  	"sync"
    14  	"testing"
    15  
    16  	pb "google.golang.org/protobuf/proto"
    17  
    18  	"github.com/janelia-flyem/dvid/datastore"
    19  	"github.com/janelia-flyem/dvid/datatype/common/proto"
    20  	"github.com/janelia-flyem/dvid/dvid"
    21  	"github.com/janelia-flyem/dvid/server"
    22  )
    23  
    24  var (
    25  	kvtype datastore.TypeService
    26  	testMu sync.Mutex
    27  )
    28  
    29  // Sets package-level testRepo and TestVersionID
    30  func initTestRepo() (dvid.UUID, dvid.VersionID) {
    31  	testMu.Lock()
    32  	defer testMu.Unlock()
    33  	if kvtype == nil {
    34  		var err error
    35  		kvtype, err = datastore.TypeServiceByName(TypeName)
    36  		if err != nil {
    37  			log.Fatalf("Can't get keyvalue type: %s\n", err)
    38  		}
    39  	}
    40  	return datastore.NewTestRepo()
    41  }
    42  
    43  // Make sure new keyvalue data have different IDs.
    44  func TestNewKeyvalueDifferent(t *testing.T) {
    45  	if err := server.OpenTest(); err != nil {
    46  		t.Fatalf("can't open test server: %v\n", err)
    47  	}
    48  	defer server.CloseTest()
    49  
    50  	uuid, _ := initTestRepo()
    51  
    52  	// Add data
    53  	config := dvid.NewConfig()
    54  	dataservice1, err := datastore.NewData(uuid, kvtype, "instance1", config)
    55  	if err != nil {
    56  		t.Errorf("Error creating new keyvalue instance: %v\n", err)
    57  	}
    58  	kv1, ok := dataservice1.(*Data)
    59  	if !ok {
    60  		t.Errorf("Returned new data instance 1 is not keyvalue.Data\n")
    61  	}
    62  	if kv1.DataName() != "instance1" {
    63  		t.Errorf("New keyvalue data instance name set incorrectly: %q != %q\n",
    64  			kv1.DataName(), "instance1")
    65  	}
    66  
    67  	dataservice2, err := datastore.NewData(uuid, kvtype, "instance2", config)
    68  	if err != nil {
    69  		t.Errorf("Error creating new keyvalue instance: %v\n", err)
    70  	}
    71  	kv2, ok := dataservice2.(*Data)
    72  	if !ok {
    73  		t.Errorf("Returned new data instance 2 is not keyvalue.Data\n")
    74  	}
    75  
    76  	if kv1.InstanceID() == kv2.InstanceID() {
    77  		t.Errorf("Instance IDs should be different: %d == %d\n",
    78  			kv1.InstanceID(), kv2.InstanceID())
    79  	}
    80  }
    81  
    82  func TestKeyvalueRoundTrip(t *testing.T) {
    83  	if err := server.OpenTest(); err != nil {
    84  		t.Fatalf("can't open test server: %v\n", err)
    85  	}
    86  	defer server.CloseTest()
    87  
    88  	uuid, versionID := initTestRepo()
    89  
    90  	// Add data
    91  	config := dvid.NewConfig()
    92  	dataservice, err := datastore.NewData(uuid, kvtype, "roundtripper", config)
    93  	if err != nil {
    94  		t.Errorf("Error creating new keyvalue instance: %v\n", err)
    95  	}
    96  	kvdata, ok := dataservice.(*Data)
    97  	if !ok {
    98  		t.Errorf("Returned new data instance is not keyvalue.Data\n")
    99  	}
   100  
   101  	ctx := datastore.NewVersionedCtx(dataservice, versionID)
   102  
   103  	keyStr := "testkey.-{}03`~| %@\x01"
   104  	value := []byte("I like Japan and this is some unicode: \u65e5\u672c\u8a9e")
   105  
   106  	if err = kvdata.PutData(ctx, keyStr, value); err != nil {
   107  		t.Errorf("Could not put keyvalue data: %v\n", err)
   108  	}
   109  
   110  	retrieved, found, err := kvdata.GetData(ctx, keyStr)
   111  	if err != nil {
   112  		t.Fatalf("Could not get keyvalue data: %v\n", err)
   113  	}
   114  	if !found {
   115  		t.Fatalf("Could not find put keyvalue\n")
   116  	}
   117  	if bytes.Compare(value, retrieved) != 0 {
   118  		t.Errorf("keyvalue retrieved %q != put %q\n", string(retrieved), string(value))
   119  	}
   120  }
   121  
   122  func TestKeyvalueRepoPersistence(t *testing.T) {
   123  	if err := server.OpenTest(); err != nil {
   124  		t.Fatalf("can't open test server: %v\n", err)
   125  	}
   126  	defer server.CloseTest()
   127  
   128  	uuid, _ := initTestRepo()
   129  
   130  	// Make labels and set various properties
   131  	config := dvid.NewConfig()
   132  	dataservice, err := datastore.NewData(uuid, kvtype, "mykv", config)
   133  	if err != nil {
   134  		t.Errorf("Unable to create keyvalue instance: %v\n", err)
   135  	}
   136  	kvdata, ok := dataservice.(*Data)
   137  	if !ok {
   138  		t.Errorf("Can't cast keyvalue data service into keyvalue.Data\n")
   139  	}
   140  	oldData := *kvdata
   141  
   142  	// Restart test datastore and see if datasets are still there.
   143  	if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil {
   144  		t.Fatalf("Unable to save repo during keyvalue persistence test: %v\n", err)
   145  	}
   146  	datastore.CloseReopenTest()
   147  
   148  	dataservice2, err := datastore.GetDataByUUIDName(uuid, "mykv")
   149  	if err != nil {
   150  		t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err)
   151  	}
   152  	kvdata2, ok := dataservice2.(*Data)
   153  	if !ok {
   154  		t.Errorf("Returned new data instance 2 is not keyvalue.Data\n")
   155  	}
   156  	if !oldData.Equals(kvdata2) {
   157  		t.Errorf("Expected %v, got %v\n", oldData, *kvdata2)
   158  	}
   159  }
   160  
   161  func testRequest(t *testing.T, uuid dvid.UUID, versionID dvid.VersionID, name dvid.InstanceName) {
   162  	config := dvid.NewConfig()
   163  	dataservice, err := datastore.NewData(uuid, kvtype, name, config)
   164  	if err != nil {
   165  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   166  	}
   167  	data, ok := dataservice.(*Data)
   168  	if !ok {
   169  		t.Fatalf("Returned new data instance is not keyvalue.Data\n")
   170  	}
   171  
   172  	key1 := "mykey"
   173  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
   174  	resp := server.TestHTTPResponse(t, "HEAD", key1req, nil)
   175  	if resp.Code != http.StatusNotFound {
   176  		t.Errorf("HEAD on %s did not return 404 (File not found).  Status = %d\n", key1req, resp.Code)
   177  	}
   178  
   179  	// PUT a value
   180  	value1 := "some stuff"
   181  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   182  
   183  	resp = server.TestHTTPResponse(t, "HEAD", key1req, nil)
   184  	if resp.Code != http.StatusOK {
   185  		t.Errorf("HEAD on %s did not return 200 (OK).  Status = %d\n", key1req, resp.Code)
   186  	}
   187  
   188  	// Get back k/v
   189  	returnValue := server.TestHTTP(t, "GET", key1req, nil)
   190  	if string(returnValue) != value1 {
   191  		t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue))
   192  	}
   193  
   194  	// Expect error if no key used.
   195  	badrequest := fmt.Sprintf("%snode/%s/%s/key/", server.WebAPIPath, uuid, data.DataName())
   196  	server.TestBadHTTP(t, "GET", badrequest, nil)
   197  
   198  	// Add 2nd k/v
   199  	key2 := "my2ndkey"
   200  	key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
   201  
   202  	resp = server.TestHTTPResponse(t, "HEAD", key2req, nil)
   203  	if resp.Code != http.StatusNotFound {
   204  		t.Errorf("HEAD on %s did not return 404 (File Not Found).  Status = %d\n", key2req, resp.Code)
   205  	}
   206  
   207  	value2 := "more good stuff"
   208  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   209  
   210  	resp = server.TestHTTPResponse(t, "HEAD", key2req, nil)
   211  	if resp.Code != http.StatusOK {
   212  		t.Errorf("HEAD on %s did not return 200 (OK).  Status = %d\n", key2req, resp.Code)
   213  	}
   214  
   215  	// Add 3rd k/v
   216  	key3 := "heresanotherkey"
   217  	value3 := "my 3rd value"
   218  	key3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key3)
   219  	server.TestHTTP(t, "POST", key3req, strings.NewReader(value3))
   220  
   221  	// Check return of first two keys in range.
   222  	rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
   223  		"my", "zebra")
   224  	returnValue = server.TestHTTP(t, "GET", rangereq, nil)
   225  
   226  	var retrievedKeys []string
   227  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   228  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   229  	}
   230  	if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
   231  		t.Errorf("Bad key range request return.  Expected: [%q,%q].  Got: %s\n",
   232  			key2, key1, string(returnValue))
   233  	}
   234  
   235  	// Check return of all keys
   236  	allkeyreq := fmt.Sprintf("%snode/%s/%s/keys", server.WebAPIPath, uuid, data.DataName())
   237  	returnValue = server.TestHTTP(t, "GET", allkeyreq, nil)
   238  
   239  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   240  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   241  	}
   242  	if len(retrievedKeys) != 3 || retrievedKeys[0] != "heresanotherkey" && retrievedKeys[1] != "my2ndKey" && retrievedKeys[2] != "mykey" {
   243  		t.Errorf("Bad all key request return.  Expected: [%q,%q,%q].  Got: %s\n",
   244  			key3, key2, key1, string(returnValue))
   245  	}
   246  }
   247  
   248  func TestKeyvalueRequests(t *testing.T) {
   249  	if err := server.OpenTest(); err != nil {
   250  		t.Fatalf("can't open test server: %v\n", err)
   251  	}
   252  	defer server.CloseTest()
   253  
   254  	uuid, versionID := initTestRepo()
   255  
   256  	testRequest(t, uuid, versionID, "mykeyvalue")
   257  }
   258  
   259  type resolveResp struct {
   260  	Child dvid.UUID `json:"child"`
   261  }
   262  
   263  func TestKeyvalueRange(t *testing.T) {
   264  	if err := server.OpenTest(); err != nil {
   265  		t.Fatalf("can't open test server: %v\n", err)
   266  	}
   267  	defer server.CloseTest()
   268  
   269  	uuid, _ := initTestRepo()
   270  
   271  	config := dvid.NewConfig()
   272  	config.Set("versioned", "false")
   273  	_, err := datastore.NewData(uuid, kvtype, "unversiontest", config)
   274  	if err != nil {
   275  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   276  	}
   277  
   278  	// PUT a value
   279  	key1 := "key1"
   280  	value1 := `[1, 2, 3]`
   281  	key1req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, key1)
   282  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   283  
   284  	returnValue := server.TestHTTP(t, "GET", key1req, nil)
   285  	if string(returnValue) != value1 {
   286  		t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue))
   287  	}
   288  
   289  	// Add 2nd k/v
   290  	key2 := "key2"
   291  	value2 := `{"foo":"a string", "bar":"another string", "baz":[1, 2, 3]}`
   292  	key2req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, key2)
   293  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   294  
   295  	// Test
   296  	rangeReq := fmt.Sprintf("%snode/%s/unversiontest/keyrangevalues/a/zz", server.WebAPIPath, uuid)
   297  	expectedJSON := `{"key1":[1, 2, 3],"key2":` + value2 + "}"
   298  
   299  	returnValue = server.TestHTTP(t, "GET", rangeReq+"?json=true", nil)
   300  	if string(returnValue) != expectedJSON {
   301  		t.Errorf("Error on keyrangevalues: got %s, expected %s\n", string(returnValue), expectedJSON)
   302  	}
   303  }
   304  
   305  func TestKeyvalueUnversioned(t *testing.T) {
   306  	if err := server.OpenTest(); err != nil {
   307  		t.Fatalf("can't open test server: %v\n", err)
   308  	}
   309  	defer server.CloseTest()
   310  
   311  	uuid, _ := initTestRepo()
   312  
   313  	config := dvid.NewConfig()
   314  	config.Set("versioned", "false")
   315  	dataservice, err := datastore.NewData(uuid, kvtype, "unversiontest", config)
   316  	if err != nil {
   317  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   318  	}
   319  	data, ok := dataservice.(*Data)
   320  	if !ok {
   321  		t.Fatalf("Returned new data instance is not keyvalue.Data\n")
   322  	}
   323  
   324  	// PUT a value
   325  	key1 := "mykey"
   326  	value1 := "some stuff"
   327  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
   328  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   329  
   330  	// Add 2nd k/v
   331  	key2 := "my2ndkey"
   332  	value2 := "more good stuff"
   333  	key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
   334  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   335  
   336  	// Create a new version in repo
   337  	if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
   338  		t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
   339  	}
   340  	uuid2, err := datastore.NewVersion(uuid, "some child", "", nil)
   341  	if err != nil {
   342  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
   343  	}
   344  	_, err = datastore.VersionFromUUID(uuid2)
   345  	if err != nil {
   346  		t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err)
   347  	}
   348  
   349  	// Change the 2nd k/v
   350  	uuid2val := "this is completely different"
   351  	uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2)
   352  	server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val))
   353  
   354  	// Now the first version value should equal the new value
   355  	returnValue := server.TestHTTP(t, "GET", key2req, nil)
   356  	if string(returnValue) != uuid2val {
   357  		t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   358  	}
   359  
   360  	// Get the second version value
   361  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
   362  	if string(returnValue) != uuid2val {
   363  		t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   364  	}
   365  
   366  	// Check return of first two keys in range.
   367  	rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
   368  		"my", "zebra")
   369  	returnValue = server.TestHTTP(t, "GET", rangereq, nil)
   370  
   371  	var retrievedKeys []string
   372  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   373  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   374  	}
   375  	if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
   376  		t.Errorf("Bad key range request return.  Expected: [%q,%q].  Got: %s\n",
   377  			key1, key2, string(returnValue))
   378  	}
   379  
   380  	// Commit the repo
   381  	if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil {
   382  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
   383  	}
   384  
   385  	// Make grandchild of root
   386  	uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil)
   387  	if err != nil {
   388  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
   389  	}
   390  
   391  	// Delete the 2nd k/v
   392  	uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2)
   393  	server.TestHTTP(t, "DELETE", uuid3req, nil)
   394  
   395  	server.TestBadHTTP(t, "GET", uuid3req, nil)
   396  
   397  	// Make sure the 2nd k/v is now missing for previous versions.
   398  	server.TestBadHTTP(t, "GET", key2req, nil)
   399  	server.TestBadHTTP(t, "GET", uuid2req, nil)
   400  
   401  	// Make a child
   402  	if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil {
   403  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
   404  	}
   405  	uuid4, err := datastore.NewVersion(uuid3, "some child", "", nil)
   406  	if err != nil {
   407  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err)
   408  	}
   409  
   410  	// Change the 2nd k/v
   411  	uuid4val := "we are reintroducing this k/v"
   412  	uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2)
   413  	server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val))
   414  
   415  	if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil {
   416  		t.Errorf("Unable to commit node %s: %v\n", uuid4, err)
   417  	}
   418  
   419  	// Make sure the 2nd k/v is correct for each of previous versions.
   420  	returnValue = server.TestHTTP(t, "GET", key2req, nil)
   421  	if string(returnValue) != uuid4val {
   422  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
   423  	}
   424  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
   425  	if string(returnValue) != uuid4val {
   426  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
   427  	}
   428  	returnValue = server.TestHTTP(t, "GET", uuid3req, nil)
   429  	if string(returnValue) != uuid4val {
   430  		t.Errorf("Error on third version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
   431  	}
   432  	returnValue = server.TestHTTP(t, "GET", uuid4req, nil)
   433  	if string(returnValue) != uuid4val {
   434  		t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
   435  	}
   436  }
   437  
   438  func TestKeyvalueVersioning(t *testing.T) {
   439  	if err := server.OpenTest(); err != nil {
   440  		t.Fatalf("can't open test server: %v\n", err)
   441  	}
   442  	defer server.CloseTest()
   443  
   444  	uuid, _ := initTestRepo()
   445  
   446  	config := dvid.NewConfig()
   447  	dataservice, err := datastore.NewData(uuid, kvtype, "versiontest", config)
   448  	if err != nil {
   449  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   450  	}
   451  	data, ok := dataservice.(*Data)
   452  	if !ok {
   453  		t.Fatalf("Returned new data instance is not keyvalue.Data\n")
   454  	}
   455  
   456  	// PUT a value
   457  	key1 := "mykey"
   458  	value1 := `"some stuff"`
   459  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
   460  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   461  
   462  	// Add 2nd k/v
   463  	key2 := "my2ndkey"
   464  	value2 := `"more good stuff"`
   465  	key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2)
   466  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   467  
   468  	// Use the batch POST
   469  	payloadSize := 1000
   470  	var kvs proto.KeyValues
   471  	kvs.Kvs = make([]*proto.KeyValue, 5)
   472  	var payload [5][]byte
   473  	for i := 0; i < 5; i++ {
   474  		payload[i] = make([]byte, i*payloadSize+10)
   475  		n, err := rand.Read(payload[i])
   476  		if n != i*payloadSize+10 {
   477  			t.Fatalf("couldn't create payload %d\n", i)
   478  		}
   479  		if err != nil {
   480  			t.Fatalf("couldn't create payload %d: %v\n", i, err)
   481  		}
   482  		kvs.Kvs[i] = &proto.KeyValue{
   483  			Key:   fmt.Sprintf("batchkey-%d", i),
   484  			Value: payload[i],
   485  		}
   486  	}
   487  	serialization, err := pb.Marshal(&kvs)
   488  	if err != nil {
   489  		t.Fatalf("couldn't serialize keyvalues: %v\n", err)
   490  	}
   491  	kvsPostReq := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName())
   492  	server.TestHTTP(t, "POST", kvsPostReq, bytes.NewReader(serialization))
   493  
   494  	// Create a new version in repo
   495  	if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
   496  		t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
   497  	}
   498  	uuid2, err := datastore.NewVersion(uuid, "some child", "", nil)
   499  	if err != nil {
   500  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
   501  	}
   502  	_, err = datastore.VersionFromUUID(uuid2)
   503  	if err != nil {
   504  		t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err)
   505  	}
   506  
   507  	// Change the 2nd k/v
   508  	uuid2val := "this is completely different"
   509  	uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2)
   510  	server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val))
   511  
   512  	// Get the first version value
   513  	returnValue := server.TestHTTP(t, "GET", key2req, nil)
   514  	if string(returnValue) != value2 {
   515  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
   516  	}
   517  
   518  	// Get the second version value
   519  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
   520  	if string(returnValue) != uuid2val {
   521  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   522  	}
   523  
   524  	// Check return of first two keys in range.
   525  	rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(),
   526  		"my", "zebra")
   527  	returnValue = server.TestHTTP(t, "GET", rangereq, nil)
   528  
   529  	var retrievedKeys []string
   530  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   531  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   532  	}
   533  	if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" {
   534  		t.Errorf("Bad key range request return.  Expected: [%q,%q].  Got: %s\n",
   535  			key1, key2, string(returnValue))
   536  	}
   537  
   538  	// Check some values using GET /keyvalues (json)
   539  	getreq0 := fmt.Sprintf("%snode/%s/%s/keyvalues?json=true", server.WebAPIPath, uuid, data.DataName())
   540  	expectedKeys := []string{key1, key2, "missing-key"}
   541  
   542  	jsonKeys, err := json.Marshal(expectedKeys)
   543  	if err != nil {
   544  		t.Fatalf("couldn't parse expectedKeys\n")
   545  	}
   546  
   547  	returnValue = server.TestHTTP(t, "GET", getreq0, bytes.NewBuffer(jsonKeys))
   548  	expectedJSON := fmt.Sprintf(`{%q:%s,%q:%s,"missing-key":{}}`, key1, value1, key2, value2)
   549  	if string(returnValue) != expectedJSON {
   550  		t.Errorf("Error on keyvalues JSON return: got %s, expected %s\n", string(returnValue), expectedJSON)
   551  	}
   552  
   553  	// Check values from batch POST using individual key gets
   554  	for i := 0; i < 5; i++ {
   555  		k := fmt.Sprintf("batchkey-%d", i)
   556  		keyreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), k)
   557  		returnValue := server.TestHTTP(t, "GET", keyreq, nil)
   558  		if len(returnValue) != i*payloadSize+10 {
   559  			t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", k, i+10, len(returnValue))
   560  		}
   561  		if !bytes.Equal(payload[i], returnValue) {
   562  			t.Fatalf("bad response for key %q\n", k)
   563  		}
   564  	}
   565  
   566  	// Check some values from batch POST using GET /keyvalues?jsontar=true
   567  	getreq1 := fmt.Sprintf("%snode/%s/%s/keyvalues?jsontar=true", server.WebAPIPath, uuid, data.DataName())
   568  	tardata := server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["batchkey-0","batchkey-1","batchkey-4","batchkey-1000"]`))
   569  	tarbuf := bytes.NewBuffer(tardata)
   570  	tr := tar.NewReader(tarbuf)
   571  	expectedKeys = []string{"batchkey-0", "batchkey-1", "batchkey-4", "batchkey-1000"}
   572  	keyNum := 0
   573  	for {
   574  		hdr, err := tr.Next()
   575  		if err == io.EOF {
   576  			break
   577  		}
   578  		if err != nil {
   579  			t.Fatalf("error parsing tar: %v\n", err)
   580  		}
   581  		if hdr.Name != expectedKeys[keyNum] {
   582  			t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name)
   583  		}
   584  		var i int
   585  		if _, err = fmt.Sscanf(hdr.Name, "batchkey-%d", &i); err != nil {
   586  			t.Fatalf("error parsing tar file hdr %q: %v\n", hdr.Name, err)
   587  		}
   588  		var val bytes.Buffer
   589  		if _, err := io.Copy(&val, tr); err != nil {
   590  			t.Fatalf("error reading tar data: %v\n", err)
   591  		}
   592  		returnValue := val.Bytes()
   593  		if i != 1000 {
   594  			if len(returnValue) != i*payloadSize+10 {
   595  				t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", hdr.Name, i*payloadSize+10, len(returnValue))
   596  			}
   597  			if !bytes.Equal(payload[i], returnValue) {
   598  				t.Fatalf("bad response for key %q\n", hdr.Name)
   599  			}
   600  		} else {
   601  			if len(returnValue) != 0 {
   602  				t.Fatalf("expected 0 byte response for key %q, got %d bytes\n", hdr.Name, len(returnValue))
   603  			}
   604  		}
   605  		keyNum++
   606  	}
   607  	if keyNum != 4 {
   608  		t.Fatalf("Got %d keys when there should have been 4\n", keyNum)
   609  	}
   610  	tardata = server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["batchkey-3"]`))
   611  	tarbuf = bytes.NewBuffer(tardata)
   612  	tr = tar.NewReader(tarbuf)
   613  	expectedKeys = []string{"batchkey-3"}
   614  	keyNum = 0
   615  	for {
   616  		hdr, err := tr.Next()
   617  		if err == io.EOF {
   618  			break
   619  		}
   620  		if err != nil {
   621  			t.Fatalf("error parsing tar: %v\n", err)
   622  		}
   623  		if hdr.Name != expectedKeys[keyNum] {
   624  			t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name)
   625  		}
   626  		var i int
   627  		if _, err = fmt.Sscanf(hdr.Name, "batchkey-%d", &i); err != nil {
   628  			t.Fatalf("error parsing tar file hdr %q: %v\n", hdr.Name, err)
   629  		}
   630  		var val bytes.Buffer
   631  		if _, err := io.Copy(&val, tr); err != nil {
   632  			t.Fatalf("error reading tar data: %v\n", err)
   633  		}
   634  		returnValue := val.Bytes()
   635  		if len(returnValue) != i*payloadSize+10 {
   636  			t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", hdr.Name, i*payloadSize+10, len(returnValue))
   637  		}
   638  		if !bytes.Equal(payload[i], returnValue) {
   639  			t.Fatalf("bad response for key %q\n", hdr.Name)
   640  		}
   641  		keyNum++
   642  	}
   643  	if keyNum != 1 {
   644  		t.Fatalf("Got %d keys when there should have been 1\n", keyNum)
   645  	}
   646  
   647  	// Check some values from batch POST using GET /keyvalues (protobuf3)
   648  	getreq2 := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName())
   649  	expectedKeys = []string{"batchkey-1", "batchkey-2", "batchkey-3", "batchkey-1000"}
   650  	pbufKeys := proto.Keys{
   651  		Keys: expectedKeys,
   652  	}
   653  	keysSerialization, err := pb.Marshal(&pbufKeys)
   654  	if err != nil {
   655  		t.Fatalf("couldn't serialized protobuf keys: %v\n", err)
   656  	}
   657  	keyvaluesSerialization := server.TestHTTP(t, "GET", getreq2, bytes.NewBuffer(keysSerialization))
   658  	var pbKVs proto.KeyValues
   659  	if err := pb.Unmarshal(keyvaluesSerialization, &pbKVs); err != nil {
   660  		t.Fatalf("couldn't unmarshal keyvalues protobuf: %v\n", err)
   661  	}
   662  	if len(pbKVs.Kvs) != 4 {
   663  		t.Fatalf("expected 4 kv pairs returned, got %d\n", len(pbKVs.Kvs))
   664  	}
   665  	for keyNum, kv := range pbKVs.Kvs {
   666  		if kv.Key != expectedKeys[keyNum] {
   667  			t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], kv.Key)
   668  		}
   669  		var i int
   670  		if _, err = fmt.Sscanf(kv.Key, "batchkey-%d", &i); err != nil {
   671  			t.Fatalf("error parsing key %d %q: %v\n", keyNum, kv.Key, err)
   672  		}
   673  		if i != 1000 {
   674  			if len(kv.Value) != i*payloadSize+10 {
   675  				t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", kv.Key, i*payloadSize+10, len(kv.Value))
   676  			}
   677  			if !bytes.Equal(payload[i], kv.Value) {
   678  				t.Fatalf("bad response for key %q\n", kv.Key)
   679  			}
   680  		} else {
   681  			if len(kv.Value) != 0 {
   682  				t.Fatalf("expected 0 byte value for key %q but got %d bytes instead\n", kv.Key, len(kv.Value))
   683  			}
   684  		}
   685  	}
   686  
   687  	// Commit the repo
   688  	if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil {
   689  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
   690  	}
   691  
   692  	// Make grandchild of root
   693  	uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil)
   694  	if err != nil {
   695  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
   696  	}
   697  
   698  	// Delete the 2nd k/v
   699  	uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2)
   700  	server.TestHTTP(t, "DELETE", uuid3req, nil)
   701  
   702  	server.TestBadHTTP(t, "GET", uuid3req, nil)
   703  
   704  	// Make sure the 2nd k/v is correct for each of previous versions.
   705  	returnValue = server.TestHTTP(t, "GET", key2req, nil)
   706  	if string(returnValue) != value2 {
   707  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
   708  	}
   709  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
   710  	if string(returnValue) != uuid2val {
   711  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   712  	}
   713  
   714  	// Make a child
   715  	if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil {
   716  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
   717  	}
   718  	uuid4, err := datastore.NewVersion(uuid3, "some child", "", nil)
   719  	if err != nil {
   720  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err)
   721  	}
   722  
   723  	// Change the 2nd k/v
   724  	uuid4val := "we are reintroducing this k/v"
   725  	uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2)
   726  	server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val))
   727  
   728  	if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil {
   729  		t.Errorf("Unable to commit node %s: %v\n", uuid4, err)
   730  	}
   731  
   732  	// Make sure the 2nd k/v is correct for each of previous versions.
   733  	returnValue = server.TestHTTP(t, "GET", key2req, nil)
   734  	if string(returnValue) != value2 {
   735  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue))
   736  	}
   737  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
   738  	if string(returnValue) != uuid2val {
   739  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   740  	}
   741  	server.TestBadHTTP(t, "GET", uuid3req, nil)
   742  	returnValue = server.TestHTTP(t, "GET", uuid4req, nil)
   743  	if string(returnValue) != uuid4val {
   744  		t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue))
   745  	}
   746  
   747  	// Let's try a merge!
   748  
   749  	// Make a child off the 2nd version from root.
   750  	uuid5, err := datastore.NewVersion(uuid2, "some child", "some child", nil)
   751  	if err != nil {
   752  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
   753  	}
   754  
   755  	// Store new stuff in 2nd k/v
   756  	uuid5val := "this is forked value"
   757  	uuid5req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid5, data.DataName(), key2)
   758  	server.TestHTTP(t, "POST", uuid5req, strings.NewReader(uuid5val))
   759  
   760  	returnValue = server.TestHTTP(t, "GET", uuid5req, nil)
   761  	if string(returnValue) != uuid5val {
   762  		t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue))
   763  	}
   764  
   765  	// Commit node
   766  	if err = datastore.Commit(uuid5, "forked node", []string{"we modified stuff"}); err != nil {
   767  		t.Errorf("Unable to commit node %s: %v\n", uuid5, err)
   768  	}
   769  
   770  	// Should be able to merge using conflict-free (disjoint at key level) merge even though
   771  	// its conflicted.  Will get lazy error on request.
   772  	badChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid5}, "some child", datastore.MergeConflictFree)
   773  	if err != nil {
   774  		t.Errorf("Error doing merge: %v\n", err)
   775  	}
   776  	childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, badChild, data.DataName(), key2)
   777  	server.TestBadHTTP(t, "GET", childreq, nil)
   778  
   779  	// Manually fix conflict: Branch, and then delete 2nd k/v and commit.
   780  	uuid6, err := datastore.NewVersion(uuid5, "some child", "", nil)
   781  	if err != nil {
   782  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid5, err)
   783  	}
   784  
   785  	uuid6req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid6, data.DataName(), key2)
   786  	server.TestHTTP(t, "DELETE", uuid6req, nil)
   787  	server.TestBadHTTP(t, "GET", uuid6req, nil)
   788  
   789  	if err = datastore.Commit(uuid6, "deleted forked node 2nd k/v", []string{"we modified stuff"}); err != nil {
   790  		t.Errorf("Unable to commit node %s: %s\n", uuid6, err)
   791  	}
   792  
   793  	// Should now be able to correctly merge the two branches.
   794  	goodChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid6}, "merging stuff", datastore.MergeConflictFree)
   795  	if err != nil {
   796  		t.Errorf("Error doing merge: %v\n", err)
   797  	}
   798  
   799  	// We should be able to see just the original uuid4 value of the 2nd k/v
   800  	childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, goodChild, data.DataName(), key2)
   801  	returnValue = server.TestHTTP(t, "GET", childreq, nil)
   802  	if string(returnValue) != uuid4val {
   803  		t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
   804  	}
   805  
   806  	// Apply the automatic conflict resolution using ordering.
   807  	resolveNote := fmt.Sprintf(`{"data":["versiontest"],"parents":[%q,%q],"note":"automatic resolved merge"}`, uuid5, uuid4)
   808  	resolveReq := fmt.Sprintf("%srepo/%s/resolve", server.WebAPIPath, uuid4)
   809  	returnValue = server.TestHTTP(t, "POST", resolveReq, bytes.NewBufferString(resolveNote))
   810  	resolveResp := struct {
   811  		Child dvid.UUID `json:"child"`
   812  	}{}
   813  	if err := json.Unmarshal(returnValue, &resolveResp); err != nil {
   814  		t.Fatalf("Can't parse return of resolve request: %s\n", string(returnValue))
   815  	}
   816  
   817  	// We should now see the uuid5 version of the 2nd k/v in the returned merged node.
   818  	childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, resolveResp.Child, data.DataName(), key2)
   819  	returnValue = server.TestHTTP(t, "GET", childreq, nil)
   820  	if string(returnValue) != uuid5val {
   821  		t.Errorf("Error on auto merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue))
   822  	}
   823  
   824  	// Introduce a child off root but don't add 2nd k/v to it.
   825  	uuid7, err := datastore.NewVersion(uuid, "2nd child off root", "2nd child off root", nil)
   826  	if err != nil {
   827  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
   828  	}
   829  	if err = datastore.Commit(uuid7, "useless node", []string{"we modified nothing!"}); err != nil {
   830  		t.Errorf("Unable to commit node %s: %v\n", uuid7, err)
   831  	}
   832  
   833  	// Now merge the previously merged node with the newly created "blank" child off root.
   834  	if err = datastore.Commit(goodChild, "this was a good merge", []string{}); err != nil {
   835  		t.Errorf("Unable to commit node %s: %v\n", goodChild, err)
   836  	}
   837  	merge2, err := datastore.Merge([]dvid.UUID{goodChild, uuid7}, "merging a useless path", datastore.MergeConflictFree)
   838  	if err != nil {
   839  		t.Errorf("Error doing merge: %v\n", err)
   840  	}
   841  	merge3, err := datastore.Merge([]dvid.UUID{uuid7, goodChild}, "merging a useless path in reverse order", datastore.MergeConflictFree)
   842  	if err != nil {
   843  		t.Errorf("Error doing merge: %v\n", err)
   844  	}
   845  
   846  	// We should still be conflict free since 2nd key in left parent path will take precedent over shared 2nd key
   847  	// in root.  This tests our invalidation of ancestors.
   848  	toughreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge2, data.DataName(), key2)
   849  	returnValue = server.TestHTTP(t, "GET", toughreq, nil)
   850  	if string(returnValue) != uuid4val {
   851  		t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
   852  	}
   853  	toughreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge3, data.DataName(), key2)
   854  	returnValue = server.TestHTTP(t, "GET", toughreq, nil)
   855  	if string(returnValue) != uuid4val {
   856  		t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue))
   857  	}
   858  
   859  	// Create a new keyvalue data instance at an interior non-root node.
   860  	uuid8, err := datastore.NewVersion(uuid7, "open leaf child", "", nil)
   861  	if err != nil {
   862  		t.Fatalf("Unable to create new version off %s: %v\n", uuid7, err)
   863  	}
   864  	dataservice, err = datastore.NewData(uuid8, kvtype, "leafdata", config)
   865  	if err != nil {
   866  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   867  	}
   868  	leafdata, ok := dataservice.(*Data)
   869  	if !ok {
   870  		t.Fatalf("Returned new data instance is not keyvalue.Data\n")
   871  	}
   872  
   873  	// PUT a value
   874  	key1req = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid8, leafdata.DataName(), key1)
   875  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   876  
   877  	// Add 2nd k/v
   878  	key2req = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid8, leafdata.DataName(), key2)
   879  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   880  
   881  	// Change the 2nd k/v
   882  	server.TestHTTP(t, "POST", key2req, strings.NewReader(uuid2val))
   883  
   884  	// Check the values
   885  	returnValue = server.TestHTTP(t, "GET", key1req, nil)
   886  	if string(returnValue) != value1 {
   887  		t.Errorf("Key %q: expected %s, got %s\n", key1, value1, string(returnValue))
   888  	}
   889  	returnValue = server.TestHTTP(t, "GET", key2req, nil)
   890  	if string(returnValue) != uuid2val {
   891  		t.Errorf("Key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   892  	}
   893  
   894  	// Change the name of the interior data.
   895  	if err = datastore.RenameData(uuid8, "leafdata", "versiontest", "foobar"); err == nil {
   896  		t.Fatalf("Should have been prevented from renaming data 'leafdata' to existing data 'versiontest'!\n")
   897  	}
   898  	if err = datastore.RenameData(uuid8, "leafdata", "renamedData", "foobar"); err != nil {
   899  		t.Fatalf("Error renaming leafdata: %v\n", err)
   900  	}
   901  
   902  	// Check the values
   903  	key1req = fmt.Sprintf("%snode/%s/renamedData/key/%s", server.WebAPIPath, uuid8, key1)
   904  	key2req = fmt.Sprintf("%snode/%s/renamedData/key/%s", server.WebAPIPath, uuid8, key2)
   905  	returnValue = server.TestHTTP(t, "GET", key1req, nil)
   906  	if string(returnValue) != value1 {
   907  		t.Errorf("Key %q: expected %s, got %s\n", key1, value1, string(returnValue))
   908  	}
   909  	returnValue = server.TestHTTP(t, "GET", key2req, nil)
   910  	if string(returnValue) != uuid2val {
   911  		t.Errorf("Key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue))
   912  	}
   913  }
   914  
   915  // Test added after error in getting two paths to the same ancestor k/v after merge.
   916  func TestDiamondGetOnMerge(t *testing.T) {
   917  	if err := server.OpenTest(); err != nil {
   918  		t.Fatalf("can't open test server: %v\n", err)
   919  	}
   920  	defer server.CloseTest()
   921  
   922  	uuid, _ := initTestRepo()
   923  
   924  	config := dvid.NewConfig()
   925  	dataservice, err := datastore.NewData(uuid, kvtype, "mergetest", config)
   926  	if err != nil {
   927  		t.Fatalf("Error creating new keyvalue instance: %v\n", err)
   928  	}
   929  	data, ok := dataservice.(*Data)
   930  	if !ok {
   931  		t.Fatalf("Returned new data instance is not keyvalue.Data\n")
   932  	}
   933  
   934  	// PUT a value
   935  	key1 := "mykey"
   936  	value1 := "some stuff"
   937  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1)
   938  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   939  
   940  	if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
   941  		t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
   942  	}
   943  	uuid2, err := datastore.NewVersion(uuid, "first child", "", nil)
   944  	if err != nil {
   945  		t.Fatalf("Unable to create 1st child off root %s: %v\n", uuid, err)
   946  	}
   947  	if err = datastore.Commit(uuid2, "first child", nil); err != nil {
   948  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
   949  	}
   950  	uuid3, err := datastore.NewVersion(uuid, "second child", "newbranch", nil)
   951  	if err != nil {
   952  		t.Fatalf("Unable to create 2nd child off root %s: %v\n", uuid, err)
   953  	}
   954  	if err = datastore.Commit(uuid3, "second child", nil); err != nil {
   955  		t.Errorf("Unable to commit node %s: %v\n", uuid3, err)
   956  	}
   957  
   958  	child, err := datastore.Merge([]dvid.UUID{uuid2, uuid3}, "merging stuff", datastore.MergeConflictFree)
   959  	if err != nil {
   960  		t.Errorf("Error doing merge: %v\n", err)
   961  	}
   962  
   963  	// We should be able to see just the original uuid value of the k/v
   964  	childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, child, data.DataName(), key1)
   965  	returnValue := server.TestHTTP(t, "GET", childreq, nil)
   966  	if string(returnValue) != value1 {
   967  		t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key1, value1, string(returnValue))
   968  	}
   969  }
   970  
   971  /*
   972  TODO -- Complete when mutation log access added, so we can check mutation is logged and test blobstore
   973  		fetch with reference.
   974  
   975  func TestBlobstoreMutationLog(t *testing.T) {
   976  	if err := server.OpenTest(); err != nil {
   977  		t.Fatalf("can't open test server: %v\n", err)
   978  	}
   979  	defer server.CloseTest()
   980  
   981  	uuid, _ := initTestRepo()
   982  	var config dvid.Config
   983  	server.CreateTestInstance(t, uuid, "keyvalue", "mykv", config)
   984  
   985  	key := "mykey"
   986  	value := "some stuff"
   987  	url := fmt.Sprintf("%snode/%s/mykv/key/%s", server.WebAPIPath, uuid, key)
   988  	server.TestHTTP(t, "POST", url, strings.NewReader(value))
   989  }
   990  
   991  */