github.com/janelia-flyem/dvid@v1.0.0/datatype/neuronjson/neuronjson_test.go (about)

     1  package neuronjson
     2  
     3  import (
     4  	"archive/tar"
     5  	bytes "bytes"
     6  	"encoding/json"
     7  	"fmt"
     8  	io "io"
     9  	"log"
    10  	"net/http"
    11  	"reflect"
    12  	"sort"
    13  	"strings"
    14  	"sync"
    15  	"testing"
    16  
    17  	pb "google.golang.org/protobuf/proto"
    18  
    19  	"github.com/janelia-flyem/dvid/datastore"
    20  	"github.com/janelia-flyem/dvid/datatype/common/proto"
    21  	"github.com/janelia-flyem/dvid/dvid"
    22  	"github.com/janelia-flyem/dvid/server"
    23  )
    24  
    25  var (
    26  	jsontype datastore.TypeService
    27  	testMu   sync.Mutex
    28  )
    29  
    30  var sampleData = map[uint64]string{
    31  	1000: `"bodyid": 1000, "position": [100, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`,
    32  	1001: `"bodyid": 1001, "position": [110, 111, 112], "avg_location": "113, 114, 115", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`,
    33  	1002: `"bodyid": 1002, "position": [100, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`,
    34  	1003: `"bodyid": 1003, "position": [102, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`,
    35  	2000: `"bodyid": 2000, "position": [200, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`,
    36  	2001: `"bodyid": 2001, "position": [200, 111, 112], "avg_location": "213, 214, 215", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`,
    37  	2002: `"bodyid": 2002, "position": [200, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`,
    38  	2003: `"bodyid": 2003, "position": [202, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`,
    39  	3000: `"bodyid": 3000, "position": [300, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`,
    40  	3001: `"bodyid": 3001, "position": [300, 111, 113], "avg_location": "313, 314, 315", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`,
    41  	3002: `"bodyid": 3002, "position": [302, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`,
    42  	3003: `"bodyid": 3003, "position": [303, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`,
    43  }
    44  
    45  var testJsonSchema = `
    46  {
    47      "$schema": "https://json-schema.org/draft/2020-12/schema",
    48      "type": "object",
    49      "additionalProperties": true,
    50      "default": {},
    51      "required": ["bodyid"],
    52      "properties": {
    53          "bodyid": {
    54              "description": "the body id",
    55              "type": "integer"
    56          },
    57          "group": {
    58              "description": "a group id",
    59              "type": "integer"
    60          },
    61          "status": {
    62              "description": "neuron status",
    63              "type": "string"
    64          },
    65          "position": {
    66              "description": "a coordinate somewhere on the body",
    67              "type": "array",
    68              "items": {"type": "integer"},
    69              "minItems": 3,
    70              "maxItems": 3
    71          },
    72          "soma_position": {
    73              "description": "a coordinate in the neuron soma",
    74              "type": "array",
    75              "items": {"type": "integer"},
    76              "minItems": 3,
    77              "maxItems": 3
    78          },
    79          "tosoma_position": {
    80              "description": "a coordinate on the neuron's cell body fiber, as near to the soma as possible",
    81              "type": "array",
    82              "items": {"type": "integer"},
    83              "minItems": 3,
    84              "maxItems": 3
    85          },
    86          "root_position": {
    87              "description": "some 'root' position for the neuron when the soma and 'tosoma' aren't segmented.",
    88              "type": "array",
    89              "items": {"type": "integer"},
    90              "minItems": 3,
    91              "maxItems": 3
    92          }
    93      }
    94  }`
    95  
    96  // Sets package-level testRepo and TestVersionID
    97  func initTestRepo() (dvid.UUID, dvid.VersionID) {
    98  	testMu.Lock()
    99  	defer testMu.Unlock()
   100  	if jsontype == nil {
   101  		var err error
   102  		jsontype, err = datastore.TypeServiceByName(TypeName)
   103  		if err != nil {
   104  			log.Fatalf("Can't get neuronjson type: %s\n", err)
   105  		}
   106  	}
   107  	return datastore.NewTestRepo()
   108  }
   109  
   110  func checkBasicAndAll(t *testing.T, basicJSON string, allJSON []byte, user string) {
   111  	var basic NeuronJSON
   112  	if err := json.Unmarshal([]byte(basicJSON), &basic); err != nil {
   113  		t.Fatalf("Couldn't unmarshal basic JSON: %s\n", basicJSON)
   114  	}
   115  	var allList ListNeuronJSON
   116  	if err := json.Unmarshal(allJSON, &allList); err != nil {
   117  		t.Fatalf("Couldn't unmarshal all JSON: %s\n", string(allJSON))
   118  	}
   119  	if len(allList) != 1 {
   120  		t.Fatalf("Can't check allJSON without 1 element, received:\n%s\n", string(allJSON))
   121  	}
   122  	all := allList[0]
   123  	for field, value := range basic {
   124  		if _, found := all[field+"_user"]; !found {
   125  			t.Fatalf("Couldn't find %q field\n", field+"_user")
   126  		}
   127  		if _, found := all[field+"_time"]; !found {
   128  			t.Fatalf("Couldn't find %q field\n", field+"_time")
   129  		}
   130  		if all[field+"_user"] != user {
   131  			t.Fatalf("%q field got %q, not expected %q\n", field+"_user", all[field+"_user"], user)
   132  		}
   133  		if _, found := all[field]; !found {
   134  			t.Fatalf("Couldn't find %q field\n", field)
   135  		}
   136  		typeAll := reflect.TypeOf(all[field])
   137  		typeBasic := reflect.TypeOf(value)
   138  		if typeAll != typeBasic {
   139  			t.Fatalf("%q field has different types %q vs %q: %v != %v\n", field, typeBasic, typeAll, value, all[field])
   140  		}
   141  		if !reflect.DeepEqual(all[field], value) {
   142  			t.Fatalf("%q field got %q (type %s), not expected %q (type %s)\n", field, all[field], typeAll, value, typeBasic)
   143  		}
   144  	}
   145  }
   146  
   147  // returns []byte of updated JSON
   148  func updatedJSONBytes(t *testing.T, origJSON, newJSON string) []byte {
   149  	var vx, vy NeuronJSON
   150  	if err := json.Unmarshal([]byte(origJSON), &vx); err != nil {
   151  		t.Fatalf("can't unmarshal origJSON: %v\n", err)
   152  	}
   153  	if err := json.Unmarshal([]byte(newJSON), &vy); err != nil {
   154  		t.Fatalf("can't unmarshal newJSON: %v\n", err)
   155  	}
   156  	for k, v := range vy {
   157  		if _, found := vx[k]; !found {
   158  			vx[k] = v
   159  		}
   160  	}
   161  	updatedJSON, err := json.Marshal(vx)
   162  	if err != nil {
   163  		t.Fatalf("Couldn't serialize updated JSON (%v): %v\n", vx, err)
   164  	}
   165  	return updatedJSON
   166  }
   167  
   168  // equalJSONString compares two JSON strings, ignoring ordering but removing time fields.
   169  func equalJSONString(x, y string) bool {
   170  	var vx, vy map[string]ListNeuronJSON
   171  	if err := json.Unmarshal([]byte(x), &vx); err != nil {
   172  		return false
   173  	}
   174  	o1 := make(map[string]ListNeuronJSON, len(vx))
   175  	for uuid, jsonList := range vx {
   176  		o1[uuid] = jsonList.makeTimeless()
   177  	}
   178  	if err := json.Unmarshal([]byte(y), &vy); err != nil {
   179  		return false
   180  	}
   181  	o2 := make(map[string]ListNeuronJSON, len(vy))
   182  	for uuid, jsonList := range vy {
   183  		o2[uuid] = jsonList.makeTimeless()
   184  	}
   185  	return reflect.DeepEqual(o1, o2)
   186  }
   187  
   188  // equalObjectJSON compares two []byte of JSON objects, ignoring ordering.
   189  func equalObjectJSON(x, y []byte, showFields Fields) bool {
   190  	var vx, vy NeuronJSON
   191  	if err := json.Unmarshal(x, &vx); err != nil {
   192  		return false
   193  	}
   194  	if err := json.Unmarshal(y, &vy); err != nil {
   195  		return false
   196  	}
   197  	return reflect.DeepEqual(removeReservedFields(vx, showFields), removeReservedFields(vy, showFields))
   198  }
   199  
   200  // equalListJSON compares two []byte of JSON lists.
   201  func equalListJSON(x, y []byte, showFields Fields) bool {
   202  	var vx, vy []NeuronJSON
   203  	if err := json.Unmarshal(x, &vx); err != nil {
   204  		return false
   205  	}
   206  	if err := json.Unmarshal(y, &vy); err != nil {
   207  		return false
   208  	}
   209  	if len(vx) != len(vy) {
   210  		return false
   211  	}
   212  	if len(vx) == 0 {
   213  		return true // both have 0 objects.
   214  	}
   215  	for i := range vx {
   216  		vx[i] = removeReservedFields(vx[i], showFields)
   217  	}
   218  	for i := range vy {
   219  		vy[i] = removeReservedFields(vy[i], showFields)
   220  	}
   221  	dvid.Infof("equalListJSON: vx = %v\n", vx)
   222  	dvid.Infof("equalListJSON: vy = %v\n", vy)
   223  	return reflect.DeepEqual(vx, vy)
   224  }
   225  
   226  func TestFields(t *testing.T) {
   227  	foo := NeuronJSON{
   228  		"foo":      "foo value",
   229  		"foo_user": "foo_user value",
   230  		"foo_time": "foo_time value",
   231  		"moo":      "moo value",
   232  		"moo_user": "moo_user value",
   233  		"moo_time": "moo_time value",
   234  	}
   235  	testData := make(NeuronJSON, len(foo))
   236  	for k, v := range foo {
   237  		testData[k] = v
   238  	}
   239  	out := removeReservedFields(testData, ShowAll)
   240  	if !reflect.DeepEqual(out, foo) {
   241  		t.Fatalf("Expected %v, got %v", foo, testData)
   242  	}
   243  
   244  	expected := NeuronJSON{
   245  		"foo":      "foo value",
   246  		"foo_user": "foo_user value",
   247  		"moo":      "moo value",
   248  		"moo_user": "moo_user value",
   249  	}
   250  	out = removeReservedFields(testData, ShowUsers)
   251  	if !reflect.DeepEqual(out, expected) {
   252  		t.Fatalf("Expected %v\ngot %v\n", expected, testData)
   253  	}
   254  
   255  	for k, v := range foo {
   256  		testData[k] = v
   257  	}
   258  	expected = NeuronJSON{
   259  		"foo":      "foo value",
   260  		"foo_time": "foo_time value",
   261  		"moo":      "moo value",
   262  		"moo_time": "moo_time value",
   263  	}
   264  	out = removeReservedFields(testData, ShowTime)
   265  	if !reflect.DeepEqual(out, expected) {
   266  		t.Fatalf("Expected %v\ngot %v\n", expected, testData)
   267  	}
   268  
   269  	for k, v := range foo {
   270  		testData[k] = v
   271  	}
   272  	expected = NeuronJSON{
   273  		"foo": "foo value",
   274  		"moo": "moo value",
   275  	}
   276  	out = removeReservedFields(testData, ShowBasic)
   277  	if !reflect.DeepEqual(out, expected) {
   278  		t.Fatalf("Expected %v\ngot %v\n", expected, testData)
   279  	}
   280  }
   281  
   282  // Make sure new neuronjson data have different IDs.
   283  func TestNewNeuronjsonDifferent(t *testing.T) {
   284  	if err := server.OpenTest(); err != nil {
   285  		t.Fatalf("can't open test server: %v\n", err)
   286  	}
   287  	defer server.CloseTest()
   288  
   289  	uuid, _ := initTestRepo()
   290  
   291  	// Add data
   292  	config := dvid.NewConfig()
   293  	dataservice1, err := datastore.NewData(uuid, jsontype, "instance1", config)
   294  	if err != nil {
   295  		t.Errorf("Error creating new neuronjson instance: %v\n", err)
   296  	}
   297  	kv1, ok := dataservice1.(*Data)
   298  	if !ok {
   299  		t.Errorf("Returned new data instance 1 is not neuronjson.Data\n")
   300  	}
   301  	if kv1.DataName() != "instance1" {
   302  		t.Errorf("New neuronjson data instance name set incorrectly: %q != %q\n",
   303  			kv1.DataName(), "instance1")
   304  	}
   305  
   306  	dataservice2, err := datastore.NewData(uuid, jsontype, "instance2", config)
   307  	if err != nil {
   308  		t.Errorf("Error creating new neuronjson instance: %v\n", err)
   309  	}
   310  	kv2, ok := dataservice2.(*Data)
   311  	if !ok {
   312  		t.Errorf("Returned new data instance 2 is not neuronjson.Data\n")
   313  	}
   314  
   315  	if kv1.InstanceID() == kv2.InstanceID() {
   316  		t.Errorf("Instance IDs should be different: %d == %d\n",
   317  			kv1.InstanceID(), kv2.InstanceID())
   318  	}
   319  }
   320  
   321  func TestNeuronjsonRoundTrip(t *testing.T) {
   322  	if err := server.OpenTest(); err != nil {
   323  		t.Fatalf("can't open test server: %v\n", err)
   324  	}
   325  	defer server.CloseTest()
   326  
   327  	uuid, versionID := initTestRepo()
   328  
   329  	// Add data
   330  	config := dvid.NewConfig()
   331  	dataservice, err := datastore.NewData(uuid, jsontype, "roundtripper", config)
   332  	if err != nil {
   333  		t.Errorf("Error creating new neuronjson instance: %v\n", err)
   334  	}
   335  	kvdata, ok := dataservice.(*Data)
   336  	if !ok {
   337  		t.Errorf("Returned new data instance is not neuronjson.Data\n")
   338  	}
   339  
   340  	ctx := datastore.NewVersionedCtx(dataservice, versionID)
   341  
   342  	keyStr := "1234"
   343  	value := []byte(`{"a string": "foo", "a number": 1234, "a list": [1, 2, 3]}`)
   344  
   345  	if err = kvdata.PutData(ctx, keyStr, value, nil, true); err != nil {
   346  		t.Errorf("Could not put neuronjson data: %v\n", err)
   347  	}
   348  
   349  	retrieved, found, err := kvdata.GetData(ctx, keyStr, nil, ShowBasic)
   350  	if err != nil {
   351  		t.Fatalf("Could not get neuronjson data: %v\n", err)
   352  	}
   353  	if !found {
   354  		t.Fatalf("Could not find put neuronjson\n")
   355  	}
   356  	if !equalObjectJSON(value, retrieved, ShowUsers) {
   357  		t.Errorf("neuronjson retrieved %q != put %q\n", string(retrieved), string(value))
   358  	}
   359  }
   360  
   361  func TestNeuronjsonRepoPersistence(t *testing.T) {
   362  	if err := server.OpenTest(); err != nil {
   363  		t.Fatalf("can't open test server: %v\n", err)
   364  	}
   365  	defer server.CloseTest()
   366  
   367  	uuid, _ := initTestRepo()
   368  
   369  	// Make labels and set various properties
   370  	config := dvid.NewConfig()
   371  	dataservice, err := datastore.NewData(uuid, jsontype, "annotations", config)
   372  	if err != nil {
   373  		t.Errorf("Unable to create neuronjson instance: %v\n", err)
   374  	}
   375  	kvdata, ok := dataservice.(*Data)
   376  	if !ok {
   377  		t.Errorf("Can't cast neuronjson data service into neuronjson.Data\n")
   378  	}
   379  	oldData := *kvdata
   380  
   381  	// Restart test datastore and see if datasets are still there.
   382  	if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil {
   383  		t.Fatalf("Unable to save repo during neuronjson persistence test: %v\n", err)
   384  	}
   385  	datastore.CloseReopenTest()
   386  
   387  	dataservice2, err := datastore.GetDataByUUIDName(uuid, "annotations")
   388  	if err != nil {
   389  		t.Fatalf("Can't get neuronjson instance from reloaded test db: %v\n", err)
   390  	}
   391  	kvdata2, ok := dataservice2.(*Data)
   392  	if !ok {
   393  		t.Errorf("Returned new data instance 2 is not neuronjson.Data\n")
   394  	}
   395  	if !oldData.Equals(kvdata2) {
   396  		t.Errorf("Expected %v, got %v\n", oldData, *kvdata2)
   397  	}
   398  }
   399  
   400  func TestMetadataSupport(t *testing.T) {
   401  	if err := server.OpenTest(); err != nil {
   402  		t.Fatalf("can't open test server: %v\n", err)
   403  	}
   404  	defer server.CloseTest()
   405  
   406  	uuid, _ := initTestRepo()
   407  	server.CreateTestInstance(t, uuid, "neuronjson", "neurons", dvid.Config{})
   408  
   409  	// Test handling of different metadata types.
   410  	reqJSONSchema := fmt.Sprintf("%snode/%s/neurons/json_schema?u=frank", server.WebAPIPath, uuid)
   411  	resp := server.TestHTTPResponse(t, "POST", reqJSONSchema, strings.NewReader(testJsonSchema))
   412  	if resp.Code != http.StatusOK {
   413  		t.Errorf("POST on %s returned %d, not 200: %s\n", reqJSONSchema, resp.Code, resp.Body.String())
   414  	}
   415  	returnValue := server.TestHTTP(t, "GET", reqJSONSchema, nil)
   416  	if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) {
   417  		t.Errorf("Error in getting json schema: got %s\n", string(returnValue))
   418  	}
   419  
   420  	reqSchema1 := fmt.Sprintf("%snode/%s/neurons/schema?u=frank", server.WebAPIPath, uuid)
   421  	resp = server.TestHTTPResponse(t, "POST", reqSchema1, strings.NewReader(testJsonSchema))
   422  	if resp.Code != http.StatusOK {
   423  		t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema1, resp.Code, resp.Body.String())
   424  	}
   425  	returnValue = server.TestHTTP(t, "GET", reqSchema1, nil)
   426  	if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) {
   427  		t.Errorf("Error in getting json schema: got %s\n", string(returnValue))
   428  	}
   429  	// -- check legacy /key/schema works for backwards-compatibility
   430  	reqSchema2 := fmt.Sprintf("%snode/%s/neurons/key/schema?u=frank", server.WebAPIPath, uuid)
   431  	resp = server.TestHTTPResponse(t, "POST", reqSchema2, strings.NewReader(testJsonSchema))
   432  	if resp.Code != http.StatusOK {
   433  		t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String())
   434  	}
   435  	returnValue = server.TestHTTP(t, "GET", reqSchema2, nil)
   436  	if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) {
   437  		t.Errorf("Error in getting json schema: got %s\n", string(returnValue))
   438  	}
   439  	resp = server.TestHTTPResponse(t, "DELETE", reqSchema2, nil)
   440  	if resp.Code != http.StatusOK {
   441  		t.Errorf("DELETE on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String())
   442  	}
   443  	resp = server.TestHTTPResponse(t, "GET", reqSchema2, nil)
   444  	if resp.Code != http.StatusNotFound {
   445  		t.Errorf("GET on %s returned %d, not 404: %s\n", reqSchema2, resp.Code, resp.Body.String())
   446  	}
   447  
   448  	reqSchema1 = fmt.Sprintf("%snode/%s/neurons/schema_batch?u=frank", server.WebAPIPath, uuid)
   449  	resp = server.TestHTTPResponse(t, "POST", reqSchema1, strings.NewReader(testJsonSchema))
   450  	if resp.Code != http.StatusOK {
   451  		t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema1, resp.Code, resp.Body.String())
   452  	}
   453  	returnValue = server.TestHTTP(t, "GET", reqSchema1, nil)
   454  	if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) {
   455  		t.Errorf("Error in getting json schema: got %s\n", string(returnValue))
   456  	}
   457  	// -- check legacy /key/schema works for backwards-compatibility
   458  	reqSchema2 = fmt.Sprintf("%snode/%s/neurons/key/schema_batch?u=frank", server.WebAPIPath, uuid)
   459  	resp = server.TestHTTPResponse(t, "POST", reqSchema2, strings.NewReader(testJsonSchema))
   460  	if resp.Code != http.StatusOK {
   461  		t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String())
   462  	}
   463  	returnValue = server.TestHTTP(t, "GET", reqSchema2, nil)
   464  	if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) {
   465  		t.Errorf("Error in getting json schema: got %s\n", string(returnValue))
   466  	}
   467  	resp = server.TestHTTPResponse(t, "DELETE", reqSchema2, nil)
   468  	if resp.Code != http.StatusOK {
   469  		t.Errorf("DELETE on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String())
   470  	}
   471  	resp = server.TestHTTPResponse(t, "GET", reqSchema2, nil)
   472  	if resp.Code != http.StatusNotFound {
   473  		t.Errorf("GET on %s returned %d, not 404: %s\n", reqSchema2, resp.Code, resp.Body.String())
   474  	}
   475  }
   476  
   477  func TestValidation(t *testing.T) {
   478  	if err := server.OpenTest(); err != nil {
   479  		t.Fatalf("can't open test server: %v\n", err)
   480  	}
   481  	defer server.CloseTest()
   482  
   483  	uuid, _ := initTestRepo()
   484  
   485  	name := dvid.InstanceName("neurons")
   486  	config := dvid.NewConfig()
   487  	dataservice, err := datastore.NewData(uuid, jsontype, name, config)
   488  	if err != nil {
   489  		t.Fatalf("Error creating new neuronjson instance: %v\n", err)
   490  	}
   491  	data, ok := dataservice.(*Data)
   492  	if !ok {
   493  		t.Fatalf("Returned new data instance is not neuronjson.Data\n")
   494  	}
   495  
   496  	// Before json schema is installed, bad values should be permitted
   497  	key1 := "1000"
   498  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid, data.DataName(), key1)
   499  	badValue := `
   500  	{
   501  		"group": 130911,
   502  		"status": "Anchor",
   503  		"root_position": [15, 15, 15],
   504  		"something_else": "foo"
   505  	}`
   506  	resp := server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   507  	if resp.Code != http.StatusOK {
   508  		t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String())
   509  	}
   510  	badValue = `
   511  	{
   512  		"bodyid": 13087493,
   513  		"group": 130911,
   514  		"status": "Anchor",
   515  		"position": [23,23]
   516  	}`
   517  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   518  	if resp.Code != http.StatusOK {
   519  		t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String())
   520  	}
   521  
   522  	// Store JSON Schema
   523  	schemaReq := fmt.Sprintf("%snode/%s/%s/json_schema?u=frank", server.WebAPIPath, uuid, data.DataName())
   524  	resp = server.TestHTTPResponse(t, "POST", schemaReq, strings.NewReader(testJsonSchema))
   525  	if resp.Code != http.StatusOK {
   526  		t.Errorf("POST on %s returned %d, not 200: %s\n", schemaReq, resp.Code, resp.Body.String())
   527  	}
   528  
   529  	// PUT a value that should conform to schema set.
   530  	okValue := `
   531  	{
   532  		"bodyid": 13087493,
   533  		"group": 130911,
   534  		"status": "Anchor",
   535  		"position": [23,23,32],
   536  		"soma_position": [30, 30, 30],
   537  		"tosoma_position": [14, 1000, 1000],
   538  		"root_position": [15, 15, 15],
   539  		"something_else": "foo"
   540  	}`
   541  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(okValue))
   542  	if resp.Code != http.StatusOK {
   543  		t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String())
   544  	}
   545  	okValue = `
   546  	{
   547  		"bodyid": 13087493,
   548  		"root_position": [15, 15, 15],
   549  		"something_else": "foo"
   550  	}`
   551  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(okValue))
   552  	if resp.Code != http.StatusOK {
   553  		t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String())
   554  	}
   555  
   556  	// Test values that should not pass validation
   557  	badValue = `
   558  	{
   559  		"group": 130911,
   560  		"status": "Anchor",
   561  		"root_position": [15, 15, 15],
   562  		"something_else": "foo"
   563  	}`
   564  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   565  	if resp.Code == http.StatusOK {
   566  		t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req)
   567  	}
   568  	badValue = `
   569  	{
   570  		"bodyid": 13087493,
   571  		"group": 130911,
   572  		"status": "Anchor",
   573  		"position": [23,23]
   574  	}`
   575  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   576  	if resp.Code == http.StatusOK {
   577  		t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req)
   578  	}
   579  	badValue = `
   580  	{
   581  		"group": 130911,
   582  		"status": "Anchor",
   583  		"root_position": [15, 15, 15],
   584  		"something_else": "foo"
   585  	}` // missing bodyid
   586  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   587  	if resp.Code == http.StatusOK {
   588  		t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req)
   589  	}
   590  	// test auto-convert from string to int
   591  	badValue = `
   592  	{
   593  		"bodyid": 13087493,
   594  		"group": "130911",
   595  		"status": "Anchor",
   596  		"root_position": [15, 15, 15],
   597  		"something_else": "foo"
   598  	}` // group is string, not int
   599  	resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue))
   600  	if resp.Code != http.StatusOK {
   601  		t.Errorf("POST on %s should have been ok after auto-conversion\n", key1req)
   602  	}
   603  }
   604  
   605  func TestKeyvalueRequests(t *testing.T) {
   606  	if err := server.OpenTest(); err != nil {
   607  		t.Fatalf("can't open test server: %v\n", err)
   608  	}
   609  	defer server.CloseTest()
   610  
   611  	uuid, _ := initTestRepo()
   612  	name := dvid.InstanceName("annotations")
   613  
   614  	config := dvid.NewConfig()
   615  	_, err := datastore.NewData(uuid, jsontype, name, config)
   616  	if err != nil {
   617  		t.Fatalf("Error creating new neuronjson instance: %v\n", err)
   618  	}
   619  
   620  	key1 := "1000"
   621  	key1req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid, name, key1)
   622  	resp := server.TestHTTPResponse(t, "HEAD", key1req, nil)
   623  	if resp.Code != http.StatusNotFound {
   624  		t.Errorf("HEAD on %s did not return 404 (File not found).  Status = %d\n", key1req, resp.Code)
   625  	}
   626  
   627  	// PUT a value
   628  	value1 := `{"a string": "foo", "a number": 1234, "a list": [1, 2, 3]}`
   629  	server.TestHTTP(t, "POST", key1req, strings.NewReader(value1))
   630  
   631  	// Expect error if key 0 is used
   632  	badrequest := fmt.Sprintf("%snode/%s/%s/key/0", server.WebAPIPath, uuid, name)
   633  	server.TestBadHTTP(t, "POST", badrequest, strings.NewReader(`{"bodyid": 0, "data": "foo"}`))
   634  
   635  	// Check HEAD response
   636  	resp = server.TestHTTPResponse(t, "HEAD", key1req, nil)
   637  	if resp.Code != http.StatusOK {
   638  		t.Errorf("HEAD on %s did not return 200 (OK).  Status = %d\n", key1req, resp.Code)
   639  	}
   640  
   641  	// Get back k/v
   642  	returnValue := server.TestHTTP(t, "GET", key1req, nil)
   643  	if !equalObjectJSON(returnValue, []byte(value1), ShowBasic) {
   644  		t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue))
   645  	}
   646  
   647  	// Expect error if no key used.
   648  	badrequest = fmt.Sprintf("%snode/%s/%s/key/", server.WebAPIPath, uuid, name)
   649  	server.TestBadHTTP(t, "GET", badrequest, nil)
   650  
   651  	// Add 2nd k/v
   652  	key2 := "2000"
   653  	key2req := fmt.Sprintf("%snode/%s/%s/key/%s?u=martha", server.WebAPIPath, uuid, name, key2)
   654  
   655  	resp = server.TestHTTPResponse(t, "HEAD", key2req, nil)
   656  	if resp.Code != http.StatusNotFound {
   657  		t.Errorf("HEAD on %s did not return 404 (File Not Found).  Status = %d\n", key2req, resp.Code)
   658  	}
   659  
   660  	value2 := `{"a string": "moo", "a number": 2345, "a list": ["mom", "pop"]}`
   661  	server.TestHTTP(t, "POST", key2req, strings.NewReader(value2))
   662  
   663  	resp = server.TestHTTPResponse(t, "HEAD", key2req, nil)
   664  	if resp.Code != http.StatusOK {
   665  		t.Errorf("HEAD on %s did not return 200 (OK).  Status = %d\n", key2req, resp.Code)
   666  	}
   667  
   668  	// Add 3rd k/v
   669  	key3 := "3000"
   670  	value3 := `{"a string": "goo", "a number": 3456, "a list": [23]}`
   671  	key3req := fmt.Sprintf("%snode/%s/%s/key/%s?u=shawna", server.WebAPIPath, uuid, name, key3)
   672  	server.TestHTTP(t, "POST", key3req, strings.NewReader(value3))
   673  
   674  	// Check return of first two keys in range.
   675  	rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, name, "0", "2100")
   676  	returnValue = server.TestHTTP(t, "GET", rangereq, nil)
   677  
   678  	var retrievedKeys []string
   679  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   680  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   681  	}
   682  	if len(retrievedKeys) != 2 || retrievedKeys[1] != key2 && retrievedKeys[0] != key1 {
   683  		t.Errorf("Bad key range request return.  Expected: [%q,%q].  Got: %s\n",
   684  			key2, key1, string(returnValue))
   685  	}
   686  
   687  	// Check return of all keys
   688  	allkeyreq := fmt.Sprintf("%snode/%s/%s/keys", server.WebAPIPath, uuid, name)
   689  	returnValue = server.TestHTTP(t, "GET", allkeyreq, nil)
   690  
   691  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
   692  		t.Errorf("Bad key range request unmarshal: %v\n", err)
   693  	}
   694  	if len(retrievedKeys) != 3 || retrievedKeys[0] != key1 && retrievedKeys[1] != key2 && retrievedKeys[2] != key3 {
   695  		t.Errorf("Bad all key request return.  Expected: [%q,%q,%q].  Got: %s\n",
   696  			key3, key2, key1, string(returnValue))
   697  	}
   698  
   699  	// Check JSON keyvalues request using both list of ids and strings, latter is for keyvalue datatype compatibility.
   700  	jsonIds := `[2000, 3000]`
   701  	kvreq := fmt.Sprintf("%snode/%s/%s/keyvalues?json=true", server.WebAPIPath, uuid, name)
   702  	returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds))
   703  
   704  	expectedValue := []byte(`{"2000":` + value2 + `,"3000":` + value3 + "}")
   705  	if !equalObjectJSON(returnValue, expectedValue, ShowBasic) {
   706  		t.Errorf("Bad keyvalues request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   707  	}
   708  
   709  	jsonIds = `["2000", "3000"]`
   710  	returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds))
   711  	if !equalObjectJSON(returnValue, expectedValue, ShowBasic) {
   712  		t.Errorf("Bad keyvalues request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   713  	}
   714  
   715  	// Make sure non-existant keys return appropriately
   716  	jsonIds = `["23910", "23", "2000", "14", "3000", "10000"]`
   717  	returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds))
   718  	if !equalObjectJSON(returnValue, expectedValue, ShowBasic) {
   719  		t.Errorf("Bad keyvalues request return using unknown key.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   720  	}
   721  
   722  	// Check query
   723  	query := `{"a string": ["moo", "goo"]}`
   724  	queryreq := fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   725  	returnValue = server.TestHTTP(t, "GET", queryreq, strings.NewReader(query))
   726  
   727  	expectedValue = []byte("[" + value2 + "," + value3 + "]")
   728  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   729  		t.Fatalf("Bad GET query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   730  	}
   731  
   732  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   733  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   734  		t.Fatalf("Bad POST query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   735  	}
   736  
   737  	query = `{"a string": ["moo", "goo"], "a number": 2345}`
   738  	queryreq = fmt.Sprintf("%snode/%s/%s/query?show=all", server.WebAPIPath, uuid, name)
   739  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   740  	checkBasicAndAll(t, value2, returnValue, "martha")
   741  
   742  	query = `{"a string": ["moo", "goo"], "a number": [2345, 3456]}`
   743  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   744  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   745  
   746  	expectedValue = []byte("[" + value2 + "," + value3 + "]")
   747  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   748  		t.Fatalf("Bad query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   749  	}
   750  
   751  	query = `{"unused field": "foo"}`
   752  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   753  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   754  
   755  	expectedValue = []byte("[]")
   756  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   757  		t.Fatalf("Bad query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   758  	}
   759  
   760  	query = `{"a string": "moo", "unused field": "foo"}`
   761  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   762  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   763  
   764  	expectedValue = []byte("[]")
   765  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   766  		t.Fatalf("Bad query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   767  	}
   768  
   769  	// Check regex query
   770  	query = `{"a string": "re/^(f|m)oo"}`
   771  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   772  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   773  
   774  	expectedValue = []byte("[" + value1 + "," + value2 + "]")
   775  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   776  		t.Fatalf("Bad regex query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   777  	}
   778  
   779  	query = `{"a string": "re/^(f|m)oo", "a list": "mom"}`
   780  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   781  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   782  
   783  	expectedValue = []byte("[" + value2 + "]")
   784  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   785  		t.Fatalf("Bad regex query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   786  	}
   787  
   788  	query = `{"a string": "re/^(f|m)oo", "a list": ["re/.*x", "re/om"]}`
   789  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name)
   790  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
   791  
   792  	expectedValue = []byte("[" + value2 + "]")
   793  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
   794  		t.Fatalf("Bad regex query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
   795  	}
   796  
   797  	// Check if keys are re-POSTed using default update or replace=true.
   798  	value3mod := `{"a string": "goo modified", "a 2nd list": [26]}`
   799  	key3modreq := fmt.Sprintf("%snode/%s/%s/key/%s?u=bill&show=all", server.WebAPIPath, uuid, name, key3)
   800  	server.TestHTTP(t, "POST", key3modreq, strings.NewReader(value3mod))
   801  
   802  	returnValue = server.TestHTTP(t, "GET", key3modreq, nil)
   803  
   804  	expectedValue = []byte(`{"a string": "goo modified", "a number": 3456, "a list": [23], "a 2nd list": [26]}`)
   805  	var expectedJSON NeuronJSON
   806  	if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil {
   807  		t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON)
   808  	}
   809  	var responseJSON NeuronJSON
   810  	if err := json.Unmarshal(returnValue, &responseJSON); err != nil {
   811  		t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue))
   812  	}
   813  	if value, found := responseJSON["a string"]; !found || value != "goo modified" {
   814  		t.Fatalf("Bad response: %v\n", responseJSON)
   815  	}
   816  	if value, found := responseJSON["a string_user"]; !found || value != "bill" {
   817  		t.Fatalf("Bad response: %v\n", responseJSON)
   818  	}
   819  	if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) {
   820  		t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   821  	}
   822  	if value, found := responseJSON["a number_user"]; !found || value != "shawna" {
   823  		t.Fatalf("Bad response: %v\n", responseJSON)
   824  	}
   825  	if value, found := responseJSON["a list"]; !found || !reflect.DeepEqual(value, []int64{23}) {
   826  		t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   827  	}
   828  	if value, found := responseJSON["a list_user"]; !found || value != "shawna" {
   829  		t.Fatalf("Bad response: %v\n", responseJSON)
   830  	}
   831  	if value, found := responseJSON["a 2nd list"]; !found || !reflect.DeepEqual(value, []int64{26}) {
   832  		t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   833  	}
   834  	if value, found := responseJSON["a 2nd list_user"]; !found || value != "bill" {
   835  		t.Fatalf("Bad response: %v\n", responseJSON)
   836  	}
   837  
   838  	// Check if we replace or keep old _user and _time while changing and reusing value
   839  	value3modA := fmt.Sprintf(`{"a string": "goo modified", "a string_user": "bill", "a string_time": "%s", "a number_user": "donald", "a 2nd list": [26]}`, responseJSON["a string_time"])
   840  	key3modAreq := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank&show=all", server.WebAPIPath, uuid, name, key3)
   841  	server.TestHTTP(t, "POST", key3modAreq, strings.NewReader(value3modA))
   842  
   843  	returnValue = server.TestHTTP(t, "GET", key3modAreq, nil)
   844  	dvid.Infof("After 1st mod got back: %s\n", string(returnValue))
   845  
   846  	expectedValue = []byte(fmt.Sprintf(`{"a string": "goo modified", "a string_user": "bill", "a string_time": "%s", "a number": 3456, "a list": [23], "a 2nd list": [26]}`, responseJSON["a string_time"]))
   847  	if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil {
   848  		t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON)
   849  	}
   850  	if err := json.Unmarshal(returnValue, &responseJSON); err != nil {
   851  		t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue))
   852  	}
   853  	if value, found := responseJSON["a string"]; !found || value != "goo modified" {
   854  		t.Fatalf("Bad response: %v\n", responseJSON)
   855  	}
   856  	if value, found := responseJSON["a string_user"]; !found || value != "bill" {
   857  		t.Fatalf("Bad response: %v\n", responseJSON)
   858  	}
   859  	if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) {
   860  		t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   861  	}
   862  	if value, found := responseJSON["a number_user"]; !found || value != "donald" {
   863  		t.Fatalf("Bad response: %v\n", responseJSON)
   864  	}
   865  	old_time := responseJSON["a number_time"]
   866  	if value, found := responseJSON["a list"]; !found || !reflect.DeepEqual(value, []int64{23}) {
   867  		t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   868  	}
   869  	if value, found := responseJSON["a list_user"]; !found || value != "shawna" {
   870  		t.Fatalf("Bad response: %v\n", responseJSON)
   871  	}
   872  	if value, found := responseJSON["a 2nd list"]; !found || !reflect.DeepEqual(value, []int64{26}) {
   873  		t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   874  	}
   875  	if value, found := responseJSON["a 2nd list_user"]; !found || value != "bill" {
   876  		t.Fatalf("Bad response: %v\n", responseJSON)
   877  	}
   878  
   879  	// Check if keys are re-POSTed using replace=true. Don't modify "a number" field.
   880  	value3mod = `{"a string": "goo replaced", "only list": [1, 2], "a number": 3456}`
   881  	key3modreq = fmt.Sprintf("%snode/%s/%s/key/%s?u=sandra&show=all&replace=true", server.WebAPIPath, uuid, name, key3)
   882  	server.TestHTTP(t, "POST", key3modreq, strings.NewReader(value3mod))
   883  
   884  	returnValue = server.TestHTTP(t, "GET", key3modreq, nil)
   885  	dvid.Infof("After replace=true, got back: %s\n", string(returnValue))
   886  
   887  	expectedValue = []byte(value3mod)
   888  	if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil {
   889  		t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON)
   890  	}
   891  	if err := json.Unmarshal(returnValue, &responseJSON); err != nil {
   892  		t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue))
   893  	}
   894  	if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) {
   895  		t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   896  	}
   897  	if value, found := responseJSON["a number_user"]; !found || value != "donald" {
   898  		t.Fatalf("Bad response: %v\n", responseJSON)
   899  	}
   900  	if value, found := responseJSON["a number_time"]; !found || value != old_time {
   901  		t.Fatalf("Bad response: %v\n", responseJSON)
   902  	}
   903  	if value, found := responseJSON["a string"]; !found || value != "goo replaced" {
   904  		t.Fatalf("Bad response: %v\n", responseJSON)
   905  	}
   906  	if value, found := responseJSON["only list"]; !found || !reflect.DeepEqual(value, []int64{1, 2}) {
   907  		t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON)
   908  	}
   909  	if value, found := responseJSON["a string_user"]; !found || value != "sandra" {
   910  		t.Fatalf("Bad response: %v\n", responseJSON)
   911  	}
   912  	if value, found := responseJSON["only list_user"]; !found || value != "sandra" {
   913  		t.Fatalf("Bad response: %v\n", responseJSON)
   914  	}
   915  	if len(responseJSON) != 9 {
   916  		t.Fatalf("Expected 9 fields in response after replace, got: %v\n", responseJSON)
   917  	}
   918  }
   919  
   920  var testData = []struct {
   921  	key string
   922  	val string
   923  }{
   924  	{"1000", `{"bodyid": 1000, "a number": 3456, "position": [150,250,380], "baz": ""}`},
   925  	{"2000", `{"bodyid": 2000, "bar":"another string", "baz":[1, 2, 3], "nullfield": "im here"}`},
   926  	{"3000", `{"bodyid": 3000, "a number": 3456, "a list": [23], "nullfield": null}`},
   927  	{"4000", `{"position": [151, 251, 301], "bodyid": 4000, "soma_side": "LHS", "baz": "some string"}`},
   928  }
   929  
   930  var testData2 = []struct {
   931  	key string
   932  	val string
   933  }{
   934  	{"10000", `{"bodyid": 10000, "a number": 3456, "position": [150,250,380], "baz": ""}`},
   935  	{"20000", `{"bodyid": 20000, "bar":"another string", "baz":[1, 2, 3], "nullfield": "im here"}`},
   936  	{"30000", `{"bodyid": 30000, "a number": 3456, "a list": [23], "nullfield": null}`},
   937  	{"40000", `{"position": [151, 251, 301], "bodyid": 40000, "soma_side": "LHS", "baz": "some string"}`},
   938  }
   939  
   940  func TestStressConcurrentRW(t *testing.T) {
   941  	if err := server.OpenTest(); err != nil {
   942  		t.Fatalf("can't open test server: %v\n", err)
   943  	}
   944  	defer server.CloseTest()
   945  
   946  	uuid, _ := initTestRepo()
   947  	payload := bytes.NewBufferString(`{"typename": "neuronjson", "dataname": "neurons"}`)
   948  	apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid)
   949  	server.TestHTTP(t, "POST", apiStr, payload)
   950  
   951  	for n := 0; n < 100; n++ {
   952  		done := make(chan struct{})
   953  		go func(done <-chan struct{}) {
   954  			i := 0
   955  			for {
   956  				i++
   957  				keyreq := fmt.Sprintf("%snode/%s/neurons/key/%d", server.WebAPIPath, uuid, i)
   958  				keyval := fmt.Sprintf(`{"bodyid": %d, "somedata": "value-%d"}`, i, i)
   959  				server.TestHTTP(t, "POST", keyreq, strings.NewReader(keyval))
   960  				select {
   961  				case <-done:
   962  					return
   963  				default:
   964  				}
   965  			}
   966  		}(done)
   967  
   968  		go func(done <-chan struct{}) {
   969  			i := 0
   970  			for {
   971  				i++
   972  				keyreq := fmt.Sprintf("%snode/%s/neurons/key/%d", server.WebAPIPath, uuid, i)
   973  				keyval := fmt.Sprintf(`{"bodyid": %d, "somedata": "value-%d"}`, i, i)
   974  				server.TestHTTPResponse(t, "GET", keyreq, strings.NewReader(keyval))
   975  				select {
   976  				case <-done:
   977  					return
   978  				default:
   979  				}
   980  			}
   981  		}(done)
   982  
   983  		for m := 0; m < 10; m++ {
   984  			rangeReq := fmt.Sprintf("%snode/%s/neurons/keyrangevalues/0/1000", server.WebAPIPath, uuid)
   985  			server.TestHTTP(t, "GET", rangeReq+"?json=true", nil)
   986  			allreq := fmt.Sprintf("%snode/%s/neurons/all", server.WebAPIPath, uuid)
   987  			server.TestHTTP(t, "GET", allreq, nil)
   988  		}
   989  		close(done)
   990  	}
   991  }
   992  
   993  func TestAll(t *testing.T) {
   994  	if err := server.OpenTest(); err != nil {
   995  		t.Fatalf("can't open test server: %v\n", err)
   996  	}
   997  	defer server.CloseTest()
   998  
   999  	uuid, _ := initTestRepo()
  1000  
  1001  	payload := bytes.NewBufferString(`{"typename": "neuronjson", "dataname": "neurons"}`)
  1002  	apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid)
  1003  	server.TestHTTP(t, "POST", apiStr, payload)
  1004  
  1005  	allNeurons := make(ListNeuronJSON, len(testData))
  1006  	var keyreq = make([]string, len(testData))
  1007  	for i := 0; i < len(testData); i++ {
  1008  		keyreq[i] = fmt.Sprintf("%snode/%s/neurons/key/%s", server.WebAPIPath, uuid, testData[i].key)
  1009  		server.TestHTTP(t, "POST", keyreq[i], strings.NewReader(testData[i].val))
  1010  		if err := json.Unmarshal([]byte(testData[i].val), &(allNeurons[i])); err != nil {
  1011  			t.Fatalf("Unable to parse test annotation %d: %v\n", i, err)
  1012  		}
  1013  	}
  1014  
  1015  	// Get all neuronjson
  1016  	allreq := fmt.Sprintf("%snode/%s/neurons/all", server.WebAPIPath, uuid)
  1017  	returnValue := server.TestHTTP(t, "GET", allreq, nil)
  1018  	var neurons ListNeuronJSON
  1019  	if err := json.Unmarshal(returnValue, &neurons); err != nil {
  1020  		t.Fatalf("Unable to parse return from /all request: %v\n", err)
  1021  	}
  1022  	sort.Sort(&neurons)
  1023  	if !reflect.DeepEqual(neurons, allNeurons) {
  1024  		t.Fatalf("Response to /all is incorrect. Expected: %v, Got: %v from JSON: %s\n", allNeurons, neurons, string(returnValue))
  1025  	}
  1026  
  1027  	// Get only ["baz", "position"] fields from /all neuronjson
  1028  	expectedVal := ListNeuronJSON{
  1029  		NeuronJSON{
  1030  			"bodyid":   uint64(1000),
  1031  			"baz":      "",
  1032  			"position": []int64{150, 250, 380},
  1033  		},
  1034  		NeuronJSON{
  1035  			"bodyid": uint64(2000),
  1036  			"baz":    []int64{1, 2, 3},
  1037  		},
  1038  		NeuronJSON{
  1039  			"bodyid":   uint64(4000),
  1040  			"baz":      "some string",
  1041  			"position": []int64{151, 251, 301},
  1042  		},
  1043  	}
  1044  	allreq = fmt.Sprintf("%snode/%s/neurons/all?fields=baz,position", server.WebAPIPath, uuid)
  1045  	returnValue = server.TestHTTP(t, "GET", allreq, nil)
  1046  	if err := json.Unmarshal(returnValue, &neurons); err != nil {
  1047  		t.Fatalf("Unable to parse return from /all request: %v\n", err)
  1048  	}
  1049  	sort.Sort(&neurons)
  1050  	for i := 0; i < 3; i++ {
  1051  		dvid.Infof("Expected %d: bodyid %s, baz %s, position %s", i, reflect.TypeOf(expectedVal[i]["bodyid"]), reflect.TypeOf(expectedVal[i]["position"]), reflect.TypeOf(expectedVal[i]["baz"]))
  1052  		dvid.Infof("Received %d: bodyid %s, baz %s, position %s", i, reflect.TypeOf(neurons[i]["bodyid"]), reflect.TypeOf(neurons[i]["position"]), reflect.TypeOf(neurons[i]["baz"]))
  1053  	}
  1054  	if !reflect.DeepEqual(neurons, expectedVal) {
  1055  		t.Fatalf("Response to /all is incorrect. Expected: %v, Got: %v\n", expectedVal, neurons)
  1056  	}
  1057  }
  1058  
  1059  func TestKeyvalueRange(t *testing.T) {
  1060  	if err := server.OpenTest(); err != nil {
  1061  		t.Fatalf("can't open test server: %v\n", err)
  1062  	}
  1063  	defer server.CloseTest()
  1064  
  1065  	uuid, _ := initTestRepo()
  1066  
  1067  	config := dvid.NewConfig()
  1068  	config.Set("versioned", "false")
  1069  	_, err := datastore.NewData(uuid, jsontype, "unversiontest", config)
  1070  	if err != nil {
  1071  		t.Fatalf("Error creating new neuronjson instance: %v\n", err)
  1072  	}
  1073  
  1074  	// PUT a value
  1075  	key1req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, testData[0].key)
  1076  	server.TestHTTP(t, "POST", key1req, strings.NewReader(testData[0].val))
  1077  
  1078  	returnValue := server.TestHTTP(t, "GET", key1req, nil)
  1079  	if !equalObjectJSON(returnValue, []byte(testData[0].val), ShowBasic) {
  1080  		t.Errorf("Error on key %q: expected %s, got %s\n", testData[0].key, testData[0].val, string(returnValue))
  1081  	}
  1082  
  1083  	// Add 2nd k/v
  1084  	key2req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, testData[1].key)
  1085  	server.TestHTTP(t, "POST", key2req, strings.NewReader(testData[1].val))
  1086  
  1087  	// Test
  1088  	rangeReq := fmt.Sprintf("%snode/%s/unversiontest/keyrangevalues/0/2001", server.WebAPIPath, uuid)
  1089  	expectedJSON := fmt.Sprintf(`{"%s":%s,"%s":%s}`, testData[0].key, testData[0].val, testData[1].key, testData[1].val)
  1090  
  1091  	returnValue = server.TestHTTP(t, "GET", rangeReq+"?json=true", nil)
  1092  	if !equalObjectJSON(returnValue, []byte(expectedJSON), ShowBasic) {
  1093  		t.Errorf("Error on keyrangevalues: got %s, expected %s\n", string(returnValue), expectedJSON)
  1094  	}
  1095  }
  1096  
  1097  func TestFieldExistenceAndVersioning(t *testing.T) {
  1098  	if err := server.OpenTest(); err != nil {
  1099  		t.Fatalf("can't open test server: %v\n", err)
  1100  	}
  1101  	defer server.CloseTest()
  1102  
  1103  	uuid, _ := initTestRepo()
  1104  
  1105  	config := dvid.NewConfig()
  1106  	dataservice, err := datastore.NewData(uuid, jsontype, "versiontest", config)
  1107  	if err != nil {
  1108  		t.Fatalf("Error creating new neuronjson instance: %v\n", err)
  1109  	}
  1110  	data, ok := dataservice.(*Data)
  1111  	if !ok {
  1112  		t.Fatalf("Returned new data instance is not neuronjson.Data\n")
  1113  	}
  1114  
  1115  	// Add the first 4 annotations
  1116  	var keyreq = make([]string, len(testData))
  1117  	for i := 0; i < len(testData); i++ {
  1118  		keyreq[i] = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), testData[i].key)
  1119  		server.TestHTTP(t, "POST", keyreq[i], strings.NewReader(testData[i].val))
  1120  	}
  1121  
  1122  	// Check field existance query
  1123  	query := `{"a number": "exists/0"}`
  1124  	queryreq := fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, data.DataName())
  1125  	returnValue := server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
  1126  
  1127  	expectedValue := []byte("[" + testData[1].val + "," + testData[3].val + "]")
  1128  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
  1129  		t.Errorf("Bad existence query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
  1130  	}
  1131  
  1132  	query = `{"a number": "exists/1", "position": "exists/0"}`
  1133  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
  1134  
  1135  	expectedValue = []byte("[" + testData[2].val + "]")
  1136  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
  1137  		t.Errorf("Bad existence query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
  1138  	}
  1139  
  1140  	query = `{"nullfield": "exists/0"}`
  1141  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
  1142  
  1143  	expectedValue = []byte("[" + testData[0].val + "," + testData[2].val + "," + testData[3].val + "]")
  1144  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
  1145  		t.Fatalf("Bad existence query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
  1146  	}
  1147  
  1148  	// Check if field is missing or empty string.
  1149  	query = `[{"baz": "exists/0"}, {"baz": ""}]`
  1150  	queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, data.DataName())
  1151  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
  1152  
  1153  	expectedValue = []byte("[" + testData[0].val + "," + testData[2].val + "]")
  1154  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
  1155  		t.Errorf("Bad ORed baz existence query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
  1156  	}
  1157  
  1158  	// Use the batch POST for all 4 annotations with key += 10000
  1159  	var kvs proto.KeyValues
  1160  	kvs.Kvs = make([]*proto.KeyValue, 4)
  1161  	for i := 0; i < 4; i++ {
  1162  		kvs.Kvs[i] = &proto.KeyValue{
  1163  			Key:   testData2[i].key,
  1164  			Value: []byte(testData2[i].val),
  1165  		}
  1166  	}
  1167  	serialization, err := pb.Marshal(&kvs)
  1168  	if err != nil {
  1169  		t.Fatalf("couldn't serialize keyvalues: %v\n", err)
  1170  	}
  1171  	kvsPostReq := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName())
  1172  	server.TestHTTP(t, "POST", kvsPostReq, bytes.NewReader(serialization))
  1173  
  1174  	// Commit current version
  1175  	if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
  1176  		t.Errorf("Unable to lock root node %s: %v\n", uuid, err)
  1177  	}
  1178  
  1179  	// Verify we can still do queries on committed version.
  1180  	query = `{"a number": "exists/1", "position": "exists/0"}`
  1181  	returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))
  1182  
  1183  	expectedValue = []byte("[" + testData[2].val + "," + testData2[2].val + "]")
  1184  	if !equalListJSON(returnValue, expectedValue, ShowBasic) {
  1185  		t.Errorf("Bad existence query request return.  Expected:%v.  Got: %v\n", string(expectedValue), string(returnValue))
  1186  	}
  1187  
  1188  	// Make new version for additional testing.
  1189  	uuid2, err := datastore.NewVersion(uuid, "some child", "", nil)
  1190  	if err != nil {
  1191  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
  1192  	}
  1193  	_, err = datastore.VersionFromUUID(uuid2)
  1194  	if err != nil {
  1195  		t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err)
  1196  	}
  1197  
  1198  	// Change key 2000
  1199  	uuid2val := `{"bodyid": 2000, "data": "new stuff"}`
  1200  	uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid2, data.DataName(), testData[1].key)
  1201  	server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val))
  1202  
  1203  	// Get the first version value
  1204  	returnValue = server.TestHTTP(t, "GET", keyreq[0], nil)
  1205  	if !equalObjectJSON(returnValue, []byte(testData[0].val), ShowBasic) {
  1206  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", testData[0].key, testData[0].val, string(returnValue))
  1207  	}
  1208  
  1209  	// Get the second version value
  1210  	expected2val := updatedJSONBytes(t, testData[1].val, uuid2val)
  1211  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
  1212  	if !equalObjectJSON(returnValue, expected2val, ShowBasic) {
  1213  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", testData[1].key, uuid2val, string(returnValue))
  1214  	}
  1215  
  1216  	// Check return of first two keys in range.
  1217  	rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(), "10", "2010")
  1218  	returnValue = server.TestHTTP(t, "GET", rangereq, nil)
  1219  
  1220  	var retrievedKeys []string
  1221  	if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil {
  1222  		t.Errorf("Bad key range request unmarshal: %v\n", err)
  1223  	}
  1224  	if len(retrievedKeys) != 2 || retrievedKeys[1] != testData[1].key && retrievedKeys[0] != testData[0].key {
  1225  		t.Errorf("Bad key range request return.  Expected: [%q,%q].  Got: %s\n",
  1226  			testData[0].key, testData[1].key, string(returnValue))
  1227  	}
  1228  
  1229  	// Check values from batch POST using individual key gets
  1230  	for i := 0; i < 4; i++ {
  1231  		k := testData2[i].key
  1232  		keyreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), k)
  1233  		returnValue := server.TestHTTP(t, "GET", keyreq, nil)
  1234  		if !equalObjectJSON(returnValue, []byte(testData2[i].val), ShowBasic) {
  1235  			t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", k, testData[i].val, returnValue)
  1236  		}
  1237  	}
  1238  
  1239  	// Check some values from batch POST using GET /keyvalues?jsontar=true
  1240  	getreq1 := fmt.Sprintf("%snode/%s/%s/keyvalues?jsontar=true", server.WebAPIPath, uuid, data.DataName())
  1241  	tardata := server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["1000","2000","4000"]`))
  1242  	tarbuf := bytes.NewBuffer(tardata)
  1243  	tr := tar.NewReader(tarbuf)
  1244  	expectedKeys := []string{"1000", "2000", "4000"}
  1245  	expectedVals := []string{testData[0].val, testData[1].val, testData[3].val}
  1246  	keyNum := 0
  1247  	for {
  1248  		hdr, err := tr.Next()
  1249  		if err == io.EOF {
  1250  			break
  1251  		}
  1252  		if err != nil {
  1253  			t.Fatalf("error parsing tar: %v\n", err)
  1254  		}
  1255  		if hdr.Name != expectedKeys[keyNum] {
  1256  			t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name)
  1257  		}
  1258  		var val bytes.Buffer
  1259  		if _, err := io.Copy(&val, tr); err != nil {
  1260  			t.Fatalf("error reading tar data: %v\n", err)
  1261  		}
  1262  		returnValue := val.Bytes()
  1263  		if !equalObjectJSON(returnValue, []byte(expectedVals[keyNum]), ShowBasic) {
  1264  			t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", expectedKeys[keyNum], expectedVals[keyNum], returnValue)
  1265  		}
  1266  		dvid.Infof("Key: %s, Value: %s\n", hdr.Name, returnValue)
  1267  		keyNum++
  1268  	}
  1269  	if keyNum != 3 {
  1270  		t.Fatalf("Got %d keys when there should have been 3\n", keyNum)
  1271  	}
  1272  
  1273  	// Check some values from batch POST using GET /keyvalues (protobuf3)
  1274  	getreq2 := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName())
  1275  	pbufKeys := proto.Keys{
  1276  		Keys: expectedKeys,
  1277  	}
  1278  	keysSerialization, err := pb.Marshal(&pbufKeys)
  1279  	if err != nil {
  1280  		t.Fatalf("couldn't serialized protobuf keys: %v\n", err)
  1281  	}
  1282  	keyvaluesSerialization := server.TestHTTP(t, "GET", getreq2, bytes.NewBuffer(keysSerialization))
  1283  	var pbKVs proto.KeyValues
  1284  	if err := pb.Unmarshal(keyvaluesSerialization, &pbKVs); err != nil {
  1285  		t.Fatalf("couldn't unmarshal keyvalues protobuf: %v\n", err)
  1286  	}
  1287  	if len(pbKVs.Kvs) != 3 {
  1288  		t.Fatalf("expected 3 kv pairs returned, got %d\n", len(pbKVs.Kvs))
  1289  	}
  1290  	for keyNum, kv := range pbKVs.Kvs {
  1291  		if kv.Key != expectedKeys[keyNum] {
  1292  			t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], kv.Key)
  1293  		}
  1294  		if !equalObjectJSON(kv.Value, []byte(expectedVals[keyNum]), ShowBasic) {
  1295  			t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", expectedKeys[keyNum], testData[keyNum].val, returnValue)
  1296  		}
  1297  	}
  1298  
  1299  	// Commit the repo
  1300  	if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil {
  1301  		t.Errorf("Unable to commit node %s: %v\n", uuid2, err)
  1302  	}
  1303  
  1304  	// Make grandchild of root
  1305  	uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil)
  1306  	if err != nil {
  1307  		t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err)
  1308  	}
  1309  
  1310  	// Delete the 2nd k/v
  1311  	uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), testData[1].key)
  1312  	server.TestHTTP(t, "DELETE", uuid3req, nil)
  1313  
  1314  	server.TestBadHTTP(t, "GET", uuid3req, nil)
  1315  
  1316  	// Make sure the 2nd k/v is correct for each of previous versions.
  1317  	returnValue = server.TestHTTP(t, "GET", keyreq[1], nil)
  1318  	if !equalObjectJSON(returnValue, []byte(testData[1].val), ShowBasic) {
  1319  		t.Errorf("Error on first version, key %q: expected %s, got %s\n", testData[1].key, testData[1].val, string(returnValue))
  1320  	}
  1321  	returnValue = server.TestHTTP(t, "GET", uuid2req, nil)
  1322  	if !equalObjectJSON(returnValue, expected2val, ShowBasic) {
  1323  		t.Errorf("Error on second version, key %q: expected %s, got %s\n", testData[1].key, expected2val, string(returnValue))
  1324  	}
  1325  }