github.com/janelia-flyem/dvid@v1.0.0/datatype/neuronjson/neuronjson_test.go (about) 1 package neuronjson 2 3 import ( 4 "archive/tar" 5 bytes "bytes" 6 "encoding/json" 7 "fmt" 8 io "io" 9 "log" 10 "net/http" 11 "reflect" 12 "sort" 13 "strings" 14 "sync" 15 "testing" 16 17 pb "google.golang.org/protobuf/proto" 18 19 "github.com/janelia-flyem/dvid/datastore" 20 "github.com/janelia-flyem/dvid/datatype/common/proto" 21 "github.com/janelia-flyem/dvid/dvid" 22 "github.com/janelia-flyem/dvid/server" 23 ) 24 25 var ( 26 jsontype datastore.TypeService 27 testMu sync.Mutex 28 ) 29 30 var sampleData = map[uint64]string{ 31 1000: `"bodyid": 1000, "position": [100, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`, 32 1001: `"bodyid": 1001, "position": [110, 111, 112], "avg_location": "113, 114, 115", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`, 33 1002: `"bodyid": 1002, "position": [100, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`, 34 1003: `"bodyid": 1003, "position": [102, 101, 102], "avg_location": "103, 104, 105", "_user": "nobody@gmail.com", "_timestamp": 1619751219.934025, "class": "Interneuron (TBD)", "tags": ["group1"]`, 35 2000: `"bodyid": 2000, "position": [200, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`, 36 2001: `"bodyid": 2001, "position": [200, 111, 112], "avg_location": "213, 214, 215", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`, 37 2002: `"bodyid": 2002, "position": [200, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`, 38 2003: `"bodyid": 2003, "position": [202, 201, 202], "avg_location": "203, 204, 205", "_user": "another@gmail.com", "_timestamp": 1619751219.934025, "class": "9A", "tags": ["group2"]`, 39 3000: `"bodyid": 3000, "position": [300, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`, 40 3001: `"bodyid": 3001, "position": [300, 111, 113], "avg_location": "313, 314, 315", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`, 41 3002: `"bodyid": 3002, "position": [302, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`, 42 3003: `"bodyid": 3003, "position": [303, 301, 303], "avg_location": "303, 304, 305", "_user": "third@gmail.com", "_timestamp": 1619751219.934025, "class": "9B", "tags": ["group3"]`, 43 } 44 45 var testJsonSchema = ` 46 { 47 "$schema": "https://json-schema.org/draft/2020-12/schema", 48 "type": "object", 49 "additionalProperties": true, 50 "default": {}, 51 "required": ["bodyid"], 52 "properties": { 53 "bodyid": { 54 "description": "the body id", 55 "type": "integer" 56 }, 57 "group": { 58 "description": "a group id", 59 "type": "integer" 60 }, 61 "status": { 62 "description": "neuron status", 63 "type": "string" 64 }, 65 "position": { 66 "description": "a coordinate somewhere on the body", 67 "type": "array", 68 "items": {"type": "integer"}, 69 "minItems": 3, 70 "maxItems": 3 71 }, 72 "soma_position": { 73 "description": "a coordinate in the neuron soma", 74 "type": "array", 75 "items": {"type": "integer"}, 76 "minItems": 3, 77 "maxItems": 3 78 }, 79 "tosoma_position": { 80 "description": "a coordinate on the neuron's cell body fiber, as near to the soma as possible", 81 "type": "array", 82 "items": {"type": "integer"}, 83 "minItems": 3, 84 "maxItems": 3 85 }, 86 "root_position": { 87 "description": "some 'root' position for the neuron when the soma and 'tosoma' aren't segmented.", 88 "type": "array", 89 "items": {"type": "integer"}, 90 "minItems": 3, 91 "maxItems": 3 92 } 93 } 94 }` 95 96 // Sets package-level testRepo and TestVersionID 97 func initTestRepo() (dvid.UUID, dvid.VersionID) { 98 testMu.Lock() 99 defer testMu.Unlock() 100 if jsontype == nil { 101 var err error 102 jsontype, err = datastore.TypeServiceByName(TypeName) 103 if err != nil { 104 log.Fatalf("Can't get neuronjson type: %s\n", err) 105 } 106 } 107 return datastore.NewTestRepo() 108 } 109 110 func checkBasicAndAll(t *testing.T, basicJSON string, allJSON []byte, user string) { 111 var basic NeuronJSON 112 if err := json.Unmarshal([]byte(basicJSON), &basic); err != nil { 113 t.Fatalf("Couldn't unmarshal basic JSON: %s\n", basicJSON) 114 } 115 var allList ListNeuronJSON 116 if err := json.Unmarshal(allJSON, &allList); err != nil { 117 t.Fatalf("Couldn't unmarshal all JSON: %s\n", string(allJSON)) 118 } 119 if len(allList) != 1 { 120 t.Fatalf("Can't check allJSON without 1 element, received:\n%s\n", string(allJSON)) 121 } 122 all := allList[0] 123 for field, value := range basic { 124 if _, found := all[field+"_user"]; !found { 125 t.Fatalf("Couldn't find %q field\n", field+"_user") 126 } 127 if _, found := all[field+"_time"]; !found { 128 t.Fatalf("Couldn't find %q field\n", field+"_time") 129 } 130 if all[field+"_user"] != user { 131 t.Fatalf("%q field got %q, not expected %q\n", field+"_user", all[field+"_user"], user) 132 } 133 if _, found := all[field]; !found { 134 t.Fatalf("Couldn't find %q field\n", field) 135 } 136 typeAll := reflect.TypeOf(all[field]) 137 typeBasic := reflect.TypeOf(value) 138 if typeAll != typeBasic { 139 t.Fatalf("%q field has different types %q vs %q: %v != %v\n", field, typeBasic, typeAll, value, all[field]) 140 } 141 if !reflect.DeepEqual(all[field], value) { 142 t.Fatalf("%q field got %q (type %s), not expected %q (type %s)\n", field, all[field], typeAll, value, typeBasic) 143 } 144 } 145 } 146 147 // returns []byte of updated JSON 148 func updatedJSONBytes(t *testing.T, origJSON, newJSON string) []byte { 149 var vx, vy NeuronJSON 150 if err := json.Unmarshal([]byte(origJSON), &vx); err != nil { 151 t.Fatalf("can't unmarshal origJSON: %v\n", err) 152 } 153 if err := json.Unmarshal([]byte(newJSON), &vy); err != nil { 154 t.Fatalf("can't unmarshal newJSON: %v\n", err) 155 } 156 for k, v := range vy { 157 if _, found := vx[k]; !found { 158 vx[k] = v 159 } 160 } 161 updatedJSON, err := json.Marshal(vx) 162 if err != nil { 163 t.Fatalf("Couldn't serialize updated JSON (%v): %v\n", vx, err) 164 } 165 return updatedJSON 166 } 167 168 // equalJSONString compares two JSON strings, ignoring ordering but removing time fields. 169 func equalJSONString(x, y string) bool { 170 var vx, vy map[string]ListNeuronJSON 171 if err := json.Unmarshal([]byte(x), &vx); err != nil { 172 return false 173 } 174 o1 := make(map[string]ListNeuronJSON, len(vx)) 175 for uuid, jsonList := range vx { 176 o1[uuid] = jsonList.makeTimeless() 177 } 178 if err := json.Unmarshal([]byte(y), &vy); err != nil { 179 return false 180 } 181 o2 := make(map[string]ListNeuronJSON, len(vy)) 182 for uuid, jsonList := range vy { 183 o2[uuid] = jsonList.makeTimeless() 184 } 185 return reflect.DeepEqual(o1, o2) 186 } 187 188 // equalObjectJSON compares two []byte of JSON objects, ignoring ordering. 189 func equalObjectJSON(x, y []byte, showFields Fields) bool { 190 var vx, vy NeuronJSON 191 if err := json.Unmarshal(x, &vx); err != nil { 192 return false 193 } 194 if err := json.Unmarshal(y, &vy); err != nil { 195 return false 196 } 197 return reflect.DeepEqual(removeReservedFields(vx, showFields), removeReservedFields(vy, showFields)) 198 } 199 200 // equalListJSON compares two []byte of JSON lists. 201 func equalListJSON(x, y []byte, showFields Fields) bool { 202 var vx, vy []NeuronJSON 203 if err := json.Unmarshal(x, &vx); err != nil { 204 return false 205 } 206 if err := json.Unmarshal(y, &vy); err != nil { 207 return false 208 } 209 if len(vx) != len(vy) { 210 return false 211 } 212 if len(vx) == 0 { 213 return true // both have 0 objects. 214 } 215 for i := range vx { 216 vx[i] = removeReservedFields(vx[i], showFields) 217 } 218 for i := range vy { 219 vy[i] = removeReservedFields(vy[i], showFields) 220 } 221 dvid.Infof("equalListJSON: vx = %v\n", vx) 222 dvid.Infof("equalListJSON: vy = %v\n", vy) 223 return reflect.DeepEqual(vx, vy) 224 } 225 226 func TestFields(t *testing.T) { 227 foo := NeuronJSON{ 228 "foo": "foo value", 229 "foo_user": "foo_user value", 230 "foo_time": "foo_time value", 231 "moo": "moo value", 232 "moo_user": "moo_user value", 233 "moo_time": "moo_time value", 234 } 235 testData := make(NeuronJSON, len(foo)) 236 for k, v := range foo { 237 testData[k] = v 238 } 239 out := removeReservedFields(testData, ShowAll) 240 if !reflect.DeepEqual(out, foo) { 241 t.Fatalf("Expected %v, got %v", foo, testData) 242 } 243 244 expected := NeuronJSON{ 245 "foo": "foo value", 246 "foo_user": "foo_user value", 247 "moo": "moo value", 248 "moo_user": "moo_user value", 249 } 250 out = removeReservedFields(testData, ShowUsers) 251 if !reflect.DeepEqual(out, expected) { 252 t.Fatalf("Expected %v\ngot %v\n", expected, testData) 253 } 254 255 for k, v := range foo { 256 testData[k] = v 257 } 258 expected = NeuronJSON{ 259 "foo": "foo value", 260 "foo_time": "foo_time value", 261 "moo": "moo value", 262 "moo_time": "moo_time value", 263 } 264 out = removeReservedFields(testData, ShowTime) 265 if !reflect.DeepEqual(out, expected) { 266 t.Fatalf("Expected %v\ngot %v\n", expected, testData) 267 } 268 269 for k, v := range foo { 270 testData[k] = v 271 } 272 expected = NeuronJSON{ 273 "foo": "foo value", 274 "moo": "moo value", 275 } 276 out = removeReservedFields(testData, ShowBasic) 277 if !reflect.DeepEqual(out, expected) { 278 t.Fatalf("Expected %v\ngot %v\n", expected, testData) 279 } 280 } 281 282 // Make sure new neuronjson data have different IDs. 283 func TestNewNeuronjsonDifferent(t *testing.T) { 284 if err := server.OpenTest(); err != nil { 285 t.Fatalf("can't open test server: %v\n", err) 286 } 287 defer server.CloseTest() 288 289 uuid, _ := initTestRepo() 290 291 // Add data 292 config := dvid.NewConfig() 293 dataservice1, err := datastore.NewData(uuid, jsontype, "instance1", config) 294 if err != nil { 295 t.Errorf("Error creating new neuronjson instance: %v\n", err) 296 } 297 kv1, ok := dataservice1.(*Data) 298 if !ok { 299 t.Errorf("Returned new data instance 1 is not neuronjson.Data\n") 300 } 301 if kv1.DataName() != "instance1" { 302 t.Errorf("New neuronjson data instance name set incorrectly: %q != %q\n", 303 kv1.DataName(), "instance1") 304 } 305 306 dataservice2, err := datastore.NewData(uuid, jsontype, "instance2", config) 307 if err != nil { 308 t.Errorf("Error creating new neuronjson instance: %v\n", err) 309 } 310 kv2, ok := dataservice2.(*Data) 311 if !ok { 312 t.Errorf("Returned new data instance 2 is not neuronjson.Data\n") 313 } 314 315 if kv1.InstanceID() == kv2.InstanceID() { 316 t.Errorf("Instance IDs should be different: %d == %d\n", 317 kv1.InstanceID(), kv2.InstanceID()) 318 } 319 } 320 321 func TestNeuronjsonRoundTrip(t *testing.T) { 322 if err := server.OpenTest(); err != nil { 323 t.Fatalf("can't open test server: %v\n", err) 324 } 325 defer server.CloseTest() 326 327 uuid, versionID := initTestRepo() 328 329 // Add data 330 config := dvid.NewConfig() 331 dataservice, err := datastore.NewData(uuid, jsontype, "roundtripper", config) 332 if err != nil { 333 t.Errorf("Error creating new neuronjson instance: %v\n", err) 334 } 335 kvdata, ok := dataservice.(*Data) 336 if !ok { 337 t.Errorf("Returned new data instance is not neuronjson.Data\n") 338 } 339 340 ctx := datastore.NewVersionedCtx(dataservice, versionID) 341 342 keyStr := "1234" 343 value := []byte(`{"a string": "foo", "a number": 1234, "a list": [1, 2, 3]}`) 344 345 if err = kvdata.PutData(ctx, keyStr, value, nil, true); err != nil { 346 t.Errorf("Could not put neuronjson data: %v\n", err) 347 } 348 349 retrieved, found, err := kvdata.GetData(ctx, keyStr, nil, ShowBasic) 350 if err != nil { 351 t.Fatalf("Could not get neuronjson data: %v\n", err) 352 } 353 if !found { 354 t.Fatalf("Could not find put neuronjson\n") 355 } 356 if !equalObjectJSON(value, retrieved, ShowUsers) { 357 t.Errorf("neuronjson retrieved %q != put %q\n", string(retrieved), string(value)) 358 } 359 } 360 361 func TestNeuronjsonRepoPersistence(t *testing.T) { 362 if err := server.OpenTest(); err != nil { 363 t.Fatalf("can't open test server: %v\n", err) 364 } 365 defer server.CloseTest() 366 367 uuid, _ := initTestRepo() 368 369 // Make labels and set various properties 370 config := dvid.NewConfig() 371 dataservice, err := datastore.NewData(uuid, jsontype, "annotations", config) 372 if err != nil { 373 t.Errorf("Unable to create neuronjson instance: %v\n", err) 374 } 375 kvdata, ok := dataservice.(*Data) 376 if !ok { 377 t.Errorf("Can't cast neuronjson data service into neuronjson.Data\n") 378 } 379 oldData := *kvdata 380 381 // Restart test datastore and see if datasets are still there. 382 if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil { 383 t.Fatalf("Unable to save repo during neuronjson persistence test: %v\n", err) 384 } 385 datastore.CloseReopenTest() 386 387 dataservice2, err := datastore.GetDataByUUIDName(uuid, "annotations") 388 if err != nil { 389 t.Fatalf("Can't get neuronjson instance from reloaded test db: %v\n", err) 390 } 391 kvdata2, ok := dataservice2.(*Data) 392 if !ok { 393 t.Errorf("Returned new data instance 2 is not neuronjson.Data\n") 394 } 395 if !oldData.Equals(kvdata2) { 396 t.Errorf("Expected %v, got %v\n", oldData, *kvdata2) 397 } 398 } 399 400 func TestMetadataSupport(t *testing.T) { 401 if err := server.OpenTest(); err != nil { 402 t.Fatalf("can't open test server: %v\n", err) 403 } 404 defer server.CloseTest() 405 406 uuid, _ := initTestRepo() 407 server.CreateTestInstance(t, uuid, "neuronjson", "neurons", dvid.Config{}) 408 409 // Test handling of different metadata types. 410 reqJSONSchema := fmt.Sprintf("%snode/%s/neurons/json_schema?u=frank", server.WebAPIPath, uuid) 411 resp := server.TestHTTPResponse(t, "POST", reqJSONSchema, strings.NewReader(testJsonSchema)) 412 if resp.Code != http.StatusOK { 413 t.Errorf("POST on %s returned %d, not 200: %s\n", reqJSONSchema, resp.Code, resp.Body.String()) 414 } 415 returnValue := server.TestHTTP(t, "GET", reqJSONSchema, nil) 416 if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) { 417 t.Errorf("Error in getting json schema: got %s\n", string(returnValue)) 418 } 419 420 reqSchema1 := fmt.Sprintf("%snode/%s/neurons/schema?u=frank", server.WebAPIPath, uuid) 421 resp = server.TestHTTPResponse(t, "POST", reqSchema1, strings.NewReader(testJsonSchema)) 422 if resp.Code != http.StatusOK { 423 t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema1, resp.Code, resp.Body.String()) 424 } 425 returnValue = server.TestHTTP(t, "GET", reqSchema1, nil) 426 if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) { 427 t.Errorf("Error in getting json schema: got %s\n", string(returnValue)) 428 } 429 // -- check legacy /key/schema works for backwards-compatibility 430 reqSchema2 := fmt.Sprintf("%snode/%s/neurons/key/schema?u=frank", server.WebAPIPath, uuid) 431 resp = server.TestHTTPResponse(t, "POST", reqSchema2, strings.NewReader(testJsonSchema)) 432 if resp.Code != http.StatusOK { 433 t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String()) 434 } 435 returnValue = server.TestHTTP(t, "GET", reqSchema2, nil) 436 if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) { 437 t.Errorf("Error in getting json schema: got %s\n", string(returnValue)) 438 } 439 resp = server.TestHTTPResponse(t, "DELETE", reqSchema2, nil) 440 if resp.Code != http.StatusOK { 441 t.Errorf("DELETE on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String()) 442 } 443 resp = server.TestHTTPResponse(t, "GET", reqSchema2, nil) 444 if resp.Code != http.StatusNotFound { 445 t.Errorf("GET on %s returned %d, not 404: %s\n", reqSchema2, resp.Code, resp.Body.String()) 446 } 447 448 reqSchema1 = fmt.Sprintf("%snode/%s/neurons/schema_batch?u=frank", server.WebAPIPath, uuid) 449 resp = server.TestHTTPResponse(t, "POST", reqSchema1, strings.NewReader(testJsonSchema)) 450 if resp.Code != http.StatusOK { 451 t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema1, resp.Code, resp.Body.String()) 452 } 453 returnValue = server.TestHTTP(t, "GET", reqSchema1, nil) 454 if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) { 455 t.Errorf("Error in getting json schema: got %s\n", string(returnValue)) 456 } 457 // -- check legacy /key/schema works for backwards-compatibility 458 reqSchema2 = fmt.Sprintf("%snode/%s/neurons/key/schema_batch?u=frank", server.WebAPIPath, uuid) 459 resp = server.TestHTTPResponse(t, "POST", reqSchema2, strings.NewReader(testJsonSchema)) 460 if resp.Code != http.StatusOK { 461 t.Errorf("POST on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String()) 462 } 463 returnValue = server.TestHTTP(t, "GET", reqSchema2, nil) 464 if !equalObjectJSON(returnValue, []byte(testJsonSchema), ShowAll) { 465 t.Errorf("Error in getting json schema: got %s\n", string(returnValue)) 466 } 467 resp = server.TestHTTPResponse(t, "DELETE", reqSchema2, nil) 468 if resp.Code != http.StatusOK { 469 t.Errorf("DELETE on %s returned %d, not 200: %s\n", reqSchema2, resp.Code, resp.Body.String()) 470 } 471 resp = server.TestHTTPResponse(t, "GET", reqSchema2, nil) 472 if resp.Code != http.StatusNotFound { 473 t.Errorf("GET on %s returned %d, not 404: %s\n", reqSchema2, resp.Code, resp.Body.String()) 474 } 475 } 476 477 func TestValidation(t *testing.T) { 478 if err := server.OpenTest(); err != nil { 479 t.Fatalf("can't open test server: %v\n", err) 480 } 481 defer server.CloseTest() 482 483 uuid, _ := initTestRepo() 484 485 name := dvid.InstanceName("neurons") 486 config := dvid.NewConfig() 487 dataservice, err := datastore.NewData(uuid, jsontype, name, config) 488 if err != nil { 489 t.Fatalf("Error creating new neuronjson instance: %v\n", err) 490 } 491 data, ok := dataservice.(*Data) 492 if !ok { 493 t.Fatalf("Returned new data instance is not neuronjson.Data\n") 494 } 495 496 // Before json schema is installed, bad values should be permitted 497 key1 := "1000" 498 key1req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid, data.DataName(), key1) 499 badValue := ` 500 { 501 "group": 130911, 502 "status": "Anchor", 503 "root_position": [15, 15, 15], 504 "something_else": "foo" 505 }` 506 resp := server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 507 if resp.Code != http.StatusOK { 508 t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String()) 509 } 510 badValue = ` 511 { 512 "bodyid": 13087493, 513 "group": 130911, 514 "status": "Anchor", 515 "position": [23,23] 516 }` 517 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 518 if resp.Code != http.StatusOK { 519 t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String()) 520 } 521 522 // Store JSON Schema 523 schemaReq := fmt.Sprintf("%snode/%s/%s/json_schema?u=frank", server.WebAPIPath, uuid, data.DataName()) 524 resp = server.TestHTTPResponse(t, "POST", schemaReq, strings.NewReader(testJsonSchema)) 525 if resp.Code != http.StatusOK { 526 t.Errorf("POST on %s returned %d, not 200: %s\n", schemaReq, resp.Code, resp.Body.String()) 527 } 528 529 // PUT a value that should conform to schema set. 530 okValue := ` 531 { 532 "bodyid": 13087493, 533 "group": 130911, 534 "status": "Anchor", 535 "position": [23,23,32], 536 "soma_position": [30, 30, 30], 537 "tosoma_position": [14, 1000, 1000], 538 "root_position": [15, 15, 15], 539 "something_else": "foo" 540 }` 541 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(okValue)) 542 if resp.Code != http.StatusOK { 543 t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String()) 544 } 545 okValue = ` 546 { 547 "bodyid": 13087493, 548 "root_position": [15, 15, 15], 549 "something_else": "foo" 550 }` 551 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(okValue)) 552 if resp.Code != http.StatusOK { 553 t.Errorf("POST on %s returned %d, not 200: %s\n", key1req, resp.Code, resp.Body.String()) 554 } 555 556 // Test values that should not pass validation 557 badValue = ` 558 { 559 "group": 130911, 560 "status": "Anchor", 561 "root_position": [15, 15, 15], 562 "something_else": "foo" 563 }` 564 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 565 if resp.Code == http.StatusOK { 566 t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req) 567 } 568 badValue = ` 569 { 570 "bodyid": 13087493, 571 "group": 130911, 572 "status": "Anchor", 573 "position": [23,23] 574 }` 575 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 576 if resp.Code == http.StatusOK { 577 t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req) 578 } 579 badValue = ` 580 { 581 "group": 130911, 582 "status": "Anchor", 583 "root_position": [15, 15, 15], 584 "something_else": "foo" 585 }` // missing bodyid 586 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 587 if resp.Code == http.StatusOK { 588 t.Errorf("POST on %s returned 200 when it shouldn't have been validated\n", key1req) 589 } 590 // test auto-convert from string to int 591 badValue = ` 592 { 593 "bodyid": 13087493, 594 "group": "130911", 595 "status": "Anchor", 596 "root_position": [15, 15, 15], 597 "something_else": "foo" 598 }` // group is string, not int 599 resp = server.TestHTTPResponse(t, "POST", key1req, strings.NewReader(badValue)) 600 if resp.Code != http.StatusOK { 601 t.Errorf("POST on %s should have been ok after auto-conversion\n", key1req) 602 } 603 } 604 605 func TestKeyvalueRequests(t *testing.T) { 606 if err := server.OpenTest(); err != nil { 607 t.Fatalf("can't open test server: %v\n", err) 608 } 609 defer server.CloseTest() 610 611 uuid, _ := initTestRepo() 612 name := dvid.InstanceName("annotations") 613 614 config := dvid.NewConfig() 615 _, err := datastore.NewData(uuid, jsontype, name, config) 616 if err != nil { 617 t.Fatalf("Error creating new neuronjson instance: %v\n", err) 618 } 619 620 key1 := "1000" 621 key1req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid, name, key1) 622 resp := server.TestHTTPResponse(t, "HEAD", key1req, nil) 623 if resp.Code != http.StatusNotFound { 624 t.Errorf("HEAD on %s did not return 404 (File not found). Status = %d\n", key1req, resp.Code) 625 } 626 627 // PUT a value 628 value1 := `{"a string": "foo", "a number": 1234, "a list": [1, 2, 3]}` 629 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 630 631 // Expect error if key 0 is used 632 badrequest := fmt.Sprintf("%snode/%s/%s/key/0", server.WebAPIPath, uuid, name) 633 server.TestBadHTTP(t, "POST", badrequest, strings.NewReader(`{"bodyid": 0, "data": "foo"}`)) 634 635 // Check HEAD response 636 resp = server.TestHTTPResponse(t, "HEAD", key1req, nil) 637 if resp.Code != http.StatusOK { 638 t.Errorf("HEAD on %s did not return 200 (OK). Status = %d\n", key1req, resp.Code) 639 } 640 641 // Get back k/v 642 returnValue := server.TestHTTP(t, "GET", key1req, nil) 643 if !equalObjectJSON(returnValue, []byte(value1), ShowBasic) { 644 t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue)) 645 } 646 647 // Expect error if no key used. 648 badrequest = fmt.Sprintf("%snode/%s/%s/key/", server.WebAPIPath, uuid, name) 649 server.TestBadHTTP(t, "GET", badrequest, nil) 650 651 // Add 2nd k/v 652 key2 := "2000" 653 key2req := fmt.Sprintf("%snode/%s/%s/key/%s?u=martha", server.WebAPIPath, uuid, name, key2) 654 655 resp = server.TestHTTPResponse(t, "HEAD", key2req, nil) 656 if resp.Code != http.StatusNotFound { 657 t.Errorf("HEAD on %s did not return 404 (File Not Found). Status = %d\n", key2req, resp.Code) 658 } 659 660 value2 := `{"a string": "moo", "a number": 2345, "a list": ["mom", "pop"]}` 661 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 662 663 resp = server.TestHTTPResponse(t, "HEAD", key2req, nil) 664 if resp.Code != http.StatusOK { 665 t.Errorf("HEAD on %s did not return 200 (OK). Status = %d\n", key2req, resp.Code) 666 } 667 668 // Add 3rd k/v 669 key3 := "3000" 670 value3 := `{"a string": "goo", "a number": 3456, "a list": [23]}` 671 key3req := fmt.Sprintf("%snode/%s/%s/key/%s?u=shawna", server.WebAPIPath, uuid, name, key3) 672 server.TestHTTP(t, "POST", key3req, strings.NewReader(value3)) 673 674 // Check return of first two keys in range. 675 rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, name, "0", "2100") 676 returnValue = server.TestHTTP(t, "GET", rangereq, nil) 677 678 var retrievedKeys []string 679 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 680 t.Errorf("Bad key range request unmarshal: %v\n", err) 681 } 682 if len(retrievedKeys) != 2 || retrievedKeys[1] != key2 && retrievedKeys[0] != key1 { 683 t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n", 684 key2, key1, string(returnValue)) 685 } 686 687 // Check return of all keys 688 allkeyreq := fmt.Sprintf("%snode/%s/%s/keys", server.WebAPIPath, uuid, name) 689 returnValue = server.TestHTTP(t, "GET", allkeyreq, nil) 690 691 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 692 t.Errorf("Bad key range request unmarshal: %v\n", err) 693 } 694 if len(retrievedKeys) != 3 || retrievedKeys[0] != key1 && retrievedKeys[1] != key2 && retrievedKeys[2] != key3 { 695 t.Errorf("Bad all key request return. Expected: [%q,%q,%q]. Got: %s\n", 696 key3, key2, key1, string(returnValue)) 697 } 698 699 // Check JSON keyvalues request using both list of ids and strings, latter is for keyvalue datatype compatibility. 700 jsonIds := `[2000, 3000]` 701 kvreq := fmt.Sprintf("%snode/%s/%s/keyvalues?json=true", server.WebAPIPath, uuid, name) 702 returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds)) 703 704 expectedValue := []byte(`{"2000":` + value2 + `,"3000":` + value3 + "}") 705 if !equalObjectJSON(returnValue, expectedValue, ShowBasic) { 706 t.Errorf("Bad keyvalues request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 707 } 708 709 jsonIds = `["2000", "3000"]` 710 returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds)) 711 if !equalObjectJSON(returnValue, expectedValue, ShowBasic) { 712 t.Errorf("Bad keyvalues request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 713 } 714 715 // Make sure non-existant keys return appropriately 716 jsonIds = `["23910", "23", "2000", "14", "3000", "10000"]` 717 returnValue = server.TestHTTP(t, "GET", kvreq, strings.NewReader(jsonIds)) 718 if !equalObjectJSON(returnValue, expectedValue, ShowBasic) { 719 t.Errorf("Bad keyvalues request return using unknown key. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 720 } 721 722 // Check query 723 query := `{"a string": ["moo", "goo"]}` 724 queryreq := fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 725 returnValue = server.TestHTTP(t, "GET", queryreq, strings.NewReader(query)) 726 727 expectedValue = []byte("[" + value2 + "," + value3 + "]") 728 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 729 t.Fatalf("Bad GET query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 730 } 731 732 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 733 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 734 t.Fatalf("Bad POST query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 735 } 736 737 query = `{"a string": ["moo", "goo"], "a number": 2345}` 738 queryreq = fmt.Sprintf("%snode/%s/%s/query?show=all", server.WebAPIPath, uuid, name) 739 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 740 checkBasicAndAll(t, value2, returnValue, "martha") 741 742 query = `{"a string": ["moo", "goo"], "a number": [2345, 3456]}` 743 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 744 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 745 746 expectedValue = []byte("[" + value2 + "," + value3 + "]") 747 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 748 t.Fatalf("Bad query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 749 } 750 751 query = `{"unused field": "foo"}` 752 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 753 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 754 755 expectedValue = []byte("[]") 756 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 757 t.Fatalf("Bad query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 758 } 759 760 query = `{"a string": "moo", "unused field": "foo"}` 761 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 762 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 763 764 expectedValue = []byte("[]") 765 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 766 t.Fatalf("Bad query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 767 } 768 769 // Check regex query 770 query = `{"a string": "re/^(f|m)oo"}` 771 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 772 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 773 774 expectedValue = []byte("[" + value1 + "," + value2 + "]") 775 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 776 t.Fatalf("Bad regex query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 777 } 778 779 query = `{"a string": "re/^(f|m)oo", "a list": "mom"}` 780 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 781 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 782 783 expectedValue = []byte("[" + value2 + "]") 784 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 785 t.Fatalf("Bad regex query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 786 } 787 788 query = `{"a string": "re/^(f|m)oo", "a list": ["re/.*x", "re/om"]}` 789 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, name) 790 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 791 792 expectedValue = []byte("[" + value2 + "]") 793 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 794 t.Fatalf("Bad regex query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 795 } 796 797 // Check if keys are re-POSTed using default update or replace=true. 798 value3mod := `{"a string": "goo modified", "a 2nd list": [26]}` 799 key3modreq := fmt.Sprintf("%snode/%s/%s/key/%s?u=bill&show=all", server.WebAPIPath, uuid, name, key3) 800 server.TestHTTP(t, "POST", key3modreq, strings.NewReader(value3mod)) 801 802 returnValue = server.TestHTTP(t, "GET", key3modreq, nil) 803 804 expectedValue = []byte(`{"a string": "goo modified", "a number": 3456, "a list": [23], "a 2nd list": [26]}`) 805 var expectedJSON NeuronJSON 806 if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil { 807 t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON) 808 } 809 var responseJSON NeuronJSON 810 if err := json.Unmarshal(returnValue, &responseJSON); err != nil { 811 t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue)) 812 } 813 if value, found := responseJSON["a string"]; !found || value != "goo modified" { 814 t.Fatalf("Bad response: %v\n", responseJSON) 815 } 816 if value, found := responseJSON["a string_user"]; !found || value != "bill" { 817 t.Fatalf("Bad response: %v\n", responseJSON) 818 } 819 if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) { 820 t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON) 821 } 822 if value, found := responseJSON["a number_user"]; !found || value != "shawna" { 823 t.Fatalf("Bad response: %v\n", responseJSON) 824 } 825 if value, found := responseJSON["a list"]; !found || !reflect.DeepEqual(value, []int64{23}) { 826 t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON) 827 } 828 if value, found := responseJSON["a list_user"]; !found || value != "shawna" { 829 t.Fatalf("Bad response: %v\n", responseJSON) 830 } 831 if value, found := responseJSON["a 2nd list"]; !found || !reflect.DeepEqual(value, []int64{26}) { 832 t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON) 833 } 834 if value, found := responseJSON["a 2nd list_user"]; !found || value != "bill" { 835 t.Fatalf("Bad response: %v\n", responseJSON) 836 } 837 838 // Check if we replace or keep old _user and _time while changing and reusing value 839 value3modA := fmt.Sprintf(`{"a string": "goo modified", "a string_user": "bill", "a string_time": "%s", "a number_user": "donald", "a 2nd list": [26]}`, responseJSON["a string_time"]) 840 key3modAreq := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank&show=all", server.WebAPIPath, uuid, name, key3) 841 server.TestHTTP(t, "POST", key3modAreq, strings.NewReader(value3modA)) 842 843 returnValue = server.TestHTTP(t, "GET", key3modAreq, nil) 844 dvid.Infof("After 1st mod got back: %s\n", string(returnValue)) 845 846 expectedValue = []byte(fmt.Sprintf(`{"a string": "goo modified", "a string_user": "bill", "a string_time": "%s", "a number": 3456, "a list": [23], "a 2nd list": [26]}`, responseJSON["a string_time"])) 847 if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil { 848 t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON) 849 } 850 if err := json.Unmarshal(returnValue, &responseJSON); err != nil { 851 t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue)) 852 } 853 if value, found := responseJSON["a string"]; !found || value != "goo modified" { 854 t.Fatalf("Bad response: %v\n", responseJSON) 855 } 856 if value, found := responseJSON["a string_user"]; !found || value != "bill" { 857 t.Fatalf("Bad response: %v\n", responseJSON) 858 } 859 if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) { 860 t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON) 861 } 862 if value, found := responseJSON["a number_user"]; !found || value != "donald" { 863 t.Fatalf("Bad response: %v\n", responseJSON) 864 } 865 old_time := responseJSON["a number_time"] 866 if value, found := responseJSON["a list"]; !found || !reflect.DeepEqual(value, []int64{23}) { 867 t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON) 868 } 869 if value, found := responseJSON["a list_user"]; !found || value != "shawna" { 870 t.Fatalf("Bad response: %v\n", responseJSON) 871 } 872 if value, found := responseJSON["a 2nd list"]; !found || !reflect.DeepEqual(value, []int64{26}) { 873 t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON) 874 } 875 if value, found := responseJSON["a 2nd list_user"]; !found || value != "bill" { 876 t.Fatalf("Bad response: %v\n", responseJSON) 877 } 878 879 // Check if keys are re-POSTed using replace=true. Don't modify "a number" field. 880 value3mod = `{"a string": "goo replaced", "only list": [1, 2], "a number": 3456}` 881 key3modreq = fmt.Sprintf("%snode/%s/%s/key/%s?u=sandra&show=all&replace=true", server.WebAPIPath, uuid, name, key3) 882 server.TestHTTP(t, "POST", key3modreq, strings.NewReader(value3mod)) 883 884 returnValue = server.TestHTTP(t, "GET", key3modreq, nil) 885 dvid.Infof("After replace=true, got back: %s\n", string(returnValue)) 886 887 expectedValue = []byte(value3mod) 888 if err := json.Unmarshal(expectedValue, &expectedJSON); err != nil { 889 t.Fatalf("Couldn't unmarshal expected basic JSON: %s\n", expectedJSON) 890 } 891 if err := json.Unmarshal(returnValue, &responseJSON); err != nil { 892 t.Fatalf("Couldn't unmarshal response JSON: %s\n", string(returnValue)) 893 } 894 if value, found := responseJSON["a number"]; !found || !reflect.DeepEqual(value, uint64(3456)) { 895 t.Fatalf("Bad response for number (type %s): %v\n", reflect.TypeOf(value), responseJSON) 896 } 897 if value, found := responseJSON["a number_user"]; !found || value != "donald" { 898 t.Fatalf("Bad response: %v\n", responseJSON) 899 } 900 if value, found := responseJSON["a number_time"]; !found || value != old_time { 901 t.Fatalf("Bad response: %v\n", responseJSON) 902 } 903 if value, found := responseJSON["a string"]; !found || value != "goo replaced" { 904 t.Fatalf("Bad response: %v\n", responseJSON) 905 } 906 if value, found := responseJSON["only list"]; !found || !reflect.DeepEqual(value, []int64{1, 2}) { 907 t.Fatalf("Bad response (type %s): %v\n", reflect.TypeOf(value), responseJSON) 908 } 909 if value, found := responseJSON["a string_user"]; !found || value != "sandra" { 910 t.Fatalf("Bad response: %v\n", responseJSON) 911 } 912 if value, found := responseJSON["only list_user"]; !found || value != "sandra" { 913 t.Fatalf("Bad response: %v\n", responseJSON) 914 } 915 if len(responseJSON) != 9 { 916 t.Fatalf("Expected 9 fields in response after replace, got: %v\n", responseJSON) 917 } 918 } 919 920 var testData = []struct { 921 key string 922 val string 923 }{ 924 {"1000", `{"bodyid": 1000, "a number": 3456, "position": [150,250,380], "baz": ""}`}, 925 {"2000", `{"bodyid": 2000, "bar":"another string", "baz":[1, 2, 3], "nullfield": "im here"}`}, 926 {"3000", `{"bodyid": 3000, "a number": 3456, "a list": [23], "nullfield": null}`}, 927 {"4000", `{"position": [151, 251, 301], "bodyid": 4000, "soma_side": "LHS", "baz": "some string"}`}, 928 } 929 930 var testData2 = []struct { 931 key string 932 val string 933 }{ 934 {"10000", `{"bodyid": 10000, "a number": 3456, "position": [150,250,380], "baz": ""}`}, 935 {"20000", `{"bodyid": 20000, "bar":"another string", "baz":[1, 2, 3], "nullfield": "im here"}`}, 936 {"30000", `{"bodyid": 30000, "a number": 3456, "a list": [23], "nullfield": null}`}, 937 {"40000", `{"position": [151, 251, 301], "bodyid": 40000, "soma_side": "LHS", "baz": "some string"}`}, 938 } 939 940 func TestStressConcurrentRW(t *testing.T) { 941 if err := server.OpenTest(); err != nil { 942 t.Fatalf("can't open test server: %v\n", err) 943 } 944 defer server.CloseTest() 945 946 uuid, _ := initTestRepo() 947 payload := bytes.NewBufferString(`{"typename": "neuronjson", "dataname": "neurons"}`) 948 apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid) 949 server.TestHTTP(t, "POST", apiStr, payload) 950 951 for n := 0; n < 100; n++ { 952 done := make(chan struct{}) 953 go func(done <-chan struct{}) { 954 i := 0 955 for { 956 i++ 957 keyreq := fmt.Sprintf("%snode/%s/neurons/key/%d", server.WebAPIPath, uuid, i) 958 keyval := fmt.Sprintf(`{"bodyid": %d, "somedata": "value-%d"}`, i, i) 959 server.TestHTTP(t, "POST", keyreq, strings.NewReader(keyval)) 960 select { 961 case <-done: 962 return 963 default: 964 } 965 } 966 }(done) 967 968 go func(done <-chan struct{}) { 969 i := 0 970 for { 971 i++ 972 keyreq := fmt.Sprintf("%snode/%s/neurons/key/%d", server.WebAPIPath, uuid, i) 973 keyval := fmt.Sprintf(`{"bodyid": %d, "somedata": "value-%d"}`, i, i) 974 server.TestHTTPResponse(t, "GET", keyreq, strings.NewReader(keyval)) 975 select { 976 case <-done: 977 return 978 default: 979 } 980 } 981 }(done) 982 983 for m := 0; m < 10; m++ { 984 rangeReq := fmt.Sprintf("%snode/%s/neurons/keyrangevalues/0/1000", server.WebAPIPath, uuid) 985 server.TestHTTP(t, "GET", rangeReq+"?json=true", nil) 986 allreq := fmt.Sprintf("%snode/%s/neurons/all", server.WebAPIPath, uuid) 987 server.TestHTTP(t, "GET", allreq, nil) 988 } 989 close(done) 990 } 991 } 992 993 func TestAll(t *testing.T) { 994 if err := server.OpenTest(); err != nil { 995 t.Fatalf("can't open test server: %v\n", err) 996 } 997 defer server.CloseTest() 998 999 uuid, _ := initTestRepo() 1000 1001 payload := bytes.NewBufferString(`{"typename": "neuronjson", "dataname": "neurons"}`) 1002 apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid) 1003 server.TestHTTP(t, "POST", apiStr, payload) 1004 1005 allNeurons := make(ListNeuronJSON, len(testData)) 1006 var keyreq = make([]string, len(testData)) 1007 for i := 0; i < len(testData); i++ { 1008 keyreq[i] = fmt.Sprintf("%snode/%s/neurons/key/%s", server.WebAPIPath, uuid, testData[i].key) 1009 server.TestHTTP(t, "POST", keyreq[i], strings.NewReader(testData[i].val)) 1010 if err := json.Unmarshal([]byte(testData[i].val), &(allNeurons[i])); err != nil { 1011 t.Fatalf("Unable to parse test annotation %d: %v\n", i, err) 1012 } 1013 } 1014 1015 // Get all neuronjson 1016 allreq := fmt.Sprintf("%snode/%s/neurons/all", server.WebAPIPath, uuid) 1017 returnValue := server.TestHTTP(t, "GET", allreq, nil) 1018 var neurons ListNeuronJSON 1019 if err := json.Unmarshal(returnValue, &neurons); err != nil { 1020 t.Fatalf("Unable to parse return from /all request: %v\n", err) 1021 } 1022 sort.Sort(&neurons) 1023 if !reflect.DeepEqual(neurons, allNeurons) { 1024 t.Fatalf("Response to /all is incorrect. Expected: %v, Got: %v from JSON: %s\n", allNeurons, neurons, string(returnValue)) 1025 } 1026 1027 // Get only ["baz", "position"] fields from /all neuronjson 1028 expectedVal := ListNeuronJSON{ 1029 NeuronJSON{ 1030 "bodyid": uint64(1000), 1031 "baz": "", 1032 "position": []int64{150, 250, 380}, 1033 }, 1034 NeuronJSON{ 1035 "bodyid": uint64(2000), 1036 "baz": []int64{1, 2, 3}, 1037 }, 1038 NeuronJSON{ 1039 "bodyid": uint64(4000), 1040 "baz": "some string", 1041 "position": []int64{151, 251, 301}, 1042 }, 1043 } 1044 allreq = fmt.Sprintf("%snode/%s/neurons/all?fields=baz,position", server.WebAPIPath, uuid) 1045 returnValue = server.TestHTTP(t, "GET", allreq, nil) 1046 if err := json.Unmarshal(returnValue, &neurons); err != nil { 1047 t.Fatalf("Unable to parse return from /all request: %v\n", err) 1048 } 1049 sort.Sort(&neurons) 1050 for i := 0; i < 3; i++ { 1051 dvid.Infof("Expected %d: bodyid %s, baz %s, position %s", i, reflect.TypeOf(expectedVal[i]["bodyid"]), reflect.TypeOf(expectedVal[i]["position"]), reflect.TypeOf(expectedVal[i]["baz"])) 1052 dvid.Infof("Received %d: bodyid %s, baz %s, position %s", i, reflect.TypeOf(neurons[i]["bodyid"]), reflect.TypeOf(neurons[i]["position"]), reflect.TypeOf(neurons[i]["baz"])) 1053 } 1054 if !reflect.DeepEqual(neurons, expectedVal) { 1055 t.Fatalf("Response to /all is incorrect. Expected: %v, Got: %v\n", expectedVal, neurons) 1056 } 1057 } 1058 1059 func TestKeyvalueRange(t *testing.T) { 1060 if err := server.OpenTest(); err != nil { 1061 t.Fatalf("can't open test server: %v\n", err) 1062 } 1063 defer server.CloseTest() 1064 1065 uuid, _ := initTestRepo() 1066 1067 config := dvid.NewConfig() 1068 config.Set("versioned", "false") 1069 _, err := datastore.NewData(uuid, jsontype, "unversiontest", config) 1070 if err != nil { 1071 t.Fatalf("Error creating new neuronjson instance: %v\n", err) 1072 } 1073 1074 // PUT a value 1075 key1req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, testData[0].key) 1076 server.TestHTTP(t, "POST", key1req, strings.NewReader(testData[0].val)) 1077 1078 returnValue := server.TestHTTP(t, "GET", key1req, nil) 1079 if !equalObjectJSON(returnValue, []byte(testData[0].val), ShowBasic) { 1080 t.Errorf("Error on key %q: expected %s, got %s\n", testData[0].key, testData[0].val, string(returnValue)) 1081 } 1082 1083 // Add 2nd k/v 1084 key2req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, testData[1].key) 1085 server.TestHTTP(t, "POST", key2req, strings.NewReader(testData[1].val)) 1086 1087 // Test 1088 rangeReq := fmt.Sprintf("%snode/%s/unversiontest/keyrangevalues/0/2001", server.WebAPIPath, uuid) 1089 expectedJSON := fmt.Sprintf(`{"%s":%s,"%s":%s}`, testData[0].key, testData[0].val, testData[1].key, testData[1].val) 1090 1091 returnValue = server.TestHTTP(t, "GET", rangeReq+"?json=true", nil) 1092 if !equalObjectJSON(returnValue, []byte(expectedJSON), ShowBasic) { 1093 t.Errorf("Error on keyrangevalues: got %s, expected %s\n", string(returnValue), expectedJSON) 1094 } 1095 } 1096 1097 func TestFieldExistenceAndVersioning(t *testing.T) { 1098 if err := server.OpenTest(); err != nil { 1099 t.Fatalf("can't open test server: %v\n", err) 1100 } 1101 defer server.CloseTest() 1102 1103 uuid, _ := initTestRepo() 1104 1105 config := dvid.NewConfig() 1106 dataservice, err := datastore.NewData(uuid, jsontype, "versiontest", config) 1107 if err != nil { 1108 t.Fatalf("Error creating new neuronjson instance: %v\n", err) 1109 } 1110 data, ok := dataservice.(*Data) 1111 if !ok { 1112 t.Fatalf("Returned new data instance is not neuronjson.Data\n") 1113 } 1114 1115 // Add the first 4 annotations 1116 var keyreq = make([]string, len(testData)) 1117 for i := 0; i < len(testData); i++ { 1118 keyreq[i] = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), testData[i].key) 1119 server.TestHTTP(t, "POST", keyreq[i], strings.NewReader(testData[i].val)) 1120 } 1121 1122 // Check field existance query 1123 query := `{"a number": "exists/0"}` 1124 queryreq := fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, data.DataName()) 1125 returnValue := server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 1126 1127 expectedValue := []byte("[" + testData[1].val + "," + testData[3].val + "]") 1128 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 1129 t.Errorf("Bad existence query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 1130 } 1131 1132 query = `{"a number": "exists/1", "position": "exists/0"}` 1133 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 1134 1135 expectedValue = []byte("[" + testData[2].val + "]") 1136 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 1137 t.Errorf("Bad existence query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 1138 } 1139 1140 query = `{"nullfield": "exists/0"}` 1141 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 1142 1143 expectedValue = []byte("[" + testData[0].val + "," + testData[2].val + "," + testData[3].val + "]") 1144 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 1145 t.Fatalf("Bad existence query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 1146 } 1147 1148 // Check if field is missing or empty string. 1149 query = `[{"baz": "exists/0"}, {"baz": ""}]` 1150 queryreq = fmt.Sprintf("%snode/%s/%s/query", server.WebAPIPath, uuid, data.DataName()) 1151 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 1152 1153 expectedValue = []byte("[" + testData[0].val + "," + testData[2].val + "]") 1154 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 1155 t.Errorf("Bad ORed baz existence query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 1156 } 1157 1158 // Use the batch POST for all 4 annotations with key += 10000 1159 var kvs proto.KeyValues 1160 kvs.Kvs = make([]*proto.KeyValue, 4) 1161 for i := 0; i < 4; i++ { 1162 kvs.Kvs[i] = &proto.KeyValue{ 1163 Key: testData2[i].key, 1164 Value: []byte(testData2[i].val), 1165 } 1166 } 1167 serialization, err := pb.Marshal(&kvs) 1168 if err != nil { 1169 t.Fatalf("couldn't serialize keyvalues: %v\n", err) 1170 } 1171 kvsPostReq := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName()) 1172 server.TestHTTP(t, "POST", kvsPostReq, bytes.NewReader(serialization)) 1173 1174 // Commit current version 1175 if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil { 1176 t.Errorf("Unable to lock root node %s: %v\n", uuid, err) 1177 } 1178 1179 // Verify we can still do queries on committed version. 1180 query = `{"a number": "exists/1", "position": "exists/0"}` 1181 returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query)) 1182 1183 expectedValue = []byte("[" + testData[2].val + "," + testData2[2].val + "]") 1184 if !equalListJSON(returnValue, expectedValue, ShowBasic) { 1185 t.Errorf("Bad existence query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue)) 1186 } 1187 1188 // Make new version for additional testing. 1189 uuid2, err := datastore.NewVersion(uuid, "some child", "", nil) 1190 if err != nil { 1191 t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err) 1192 } 1193 _, err = datastore.VersionFromUUID(uuid2) 1194 if err != nil { 1195 t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err) 1196 } 1197 1198 // Change key 2000 1199 uuid2val := `{"bodyid": 2000, "data": "new stuff"}` 1200 uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s?u=frank", server.WebAPIPath, uuid2, data.DataName(), testData[1].key) 1201 server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val)) 1202 1203 // Get the first version value 1204 returnValue = server.TestHTTP(t, "GET", keyreq[0], nil) 1205 if !equalObjectJSON(returnValue, []byte(testData[0].val), ShowBasic) { 1206 t.Errorf("Error on first version, key %q: expected %s, got %s\n", testData[0].key, testData[0].val, string(returnValue)) 1207 } 1208 1209 // Get the second version value 1210 expected2val := updatedJSONBytes(t, testData[1].val, uuid2val) 1211 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 1212 if !equalObjectJSON(returnValue, expected2val, ShowBasic) { 1213 t.Errorf("Error on second version, key %q: expected %s, got %s\n", testData[1].key, uuid2val, string(returnValue)) 1214 } 1215 1216 // Check return of first two keys in range. 1217 rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(), "10", "2010") 1218 returnValue = server.TestHTTP(t, "GET", rangereq, nil) 1219 1220 var retrievedKeys []string 1221 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 1222 t.Errorf("Bad key range request unmarshal: %v\n", err) 1223 } 1224 if len(retrievedKeys) != 2 || retrievedKeys[1] != testData[1].key && retrievedKeys[0] != testData[0].key { 1225 t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n", 1226 testData[0].key, testData[1].key, string(returnValue)) 1227 } 1228 1229 // Check values from batch POST using individual key gets 1230 for i := 0; i < 4; i++ { 1231 k := testData2[i].key 1232 keyreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), k) 1233 returnValue := server.TestHTTP(t, "GET", keyreq, nil) 1234 if !equalObjectJSON(returnValue, []byte(testData2[i].val), ShowBasic) { 1235 t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", k, testData[i].val, returnValue) 1236 } 1237 } 1238 1239 // Check some values from batch POST using GET /keyvalues?jsontar=true 1240 getreq1 := fmt.Sprintf("%snode/%s/%s/keyvalues?jsontar=true", server.WebAPIPath, uuid, data.DataName()) 1241 tardata := server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["1000","2000","4000"]`)) 1242 tarbuf := bytes.NewBuffer(tardata) 1243 tr := tar.NewReader(tarbuf) 1244 expectedKeys := []string{"1000", "2000", "4000"} 1245 expectedVals := []string{testData[0].val, testData[1].val, testData[3].val} 1246 keyNum := 0 1247 for { 1248 hdr, err := tr.Next() 1249 if err == io.EOF { 1250 break 1251 } 1252 if err != nil { 1253 t.Fatalf("error parsing tar: %v\n", err) 1254 } 1255 if hdr.Name != expectedKeys[keyNum] { 1256 t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name) 1257 } 1258 var val bytes.Buffer 1259 if _, err := io.Copy(&val, tr); err != nil { 1260 t.Fatalf("error reading tar data: %v\n", err) 1261 } 1262 returnValue := val.Bytes() 1263 if !equalObjectJSON(returnValue, []byte(expectedVals[keyNum]), ShowBasic) { 1264 t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", expectedKeys[keyNum], expectedVals[keyNum], returnValue) 1265 } 1266 dvid.Infof("Key: %s, Value: %s\n", hdr.Name, returnValue) 1267 keyNum++ 1268 } 1269 if keyNum != 3 { 1270 t.Fatalf("Got %d keys when there should have been 3\n", keyNum) 1271 } 1272 1273 // Check some values from batch POST using GET /keyvalues (protobuf3) 1274 getreq2 := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName()) 1275 pbufKeys := proto.Keys{ 1276 Keys: expectedKeys, 1277 } 1278 keysSerialization, err := pb.Marshal(&pbufKeys) 1279 if err != nil { 1280 t.Fatalf("couldn't serialized protobuf keys: %v\n", err) 1281 } 1282 keyvaluesSerialization := server.TestHTTP(t, "GET", getreq2, bytes.NewBuffer(keysSerialization)) 1283 var pbKVs proto.KeyValues 1284 if err := pb.Unmarshal(keyvaluesSerialization, &pbKVs); err != nil { 1285 t.Fatalf("couldn't unmarshal keyvalues protobuf: %v\n", err) 1286 } 1287 if len(pbKVs.Kvs) != 3 { 1288 t.Fatalf("expected 3 kv pairs returned, got %d\n", len(pbKVs.Kvs)) 1289 } 1290 for keyNum, kv := range pbKVs.Kvs { 1291 if kv.Key != expectedKeys[keyNum] { 1292 t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], kv.Key) 1293 } 1294 if !equalObjectJSON(kv.Value, []byte(expectedVals[keyNum]), ShowBasic) { 1295 t.Errorf("Expected batch POST key %q to have value %s, got %d instead\n", expectedKeys[keyNum], testData[keyNum].val, returnValue) 1296 } 1297 } 1298 1299 // Commit the repo 1300 if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil { 1301 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 1302 } 1303 1304 // Make grandchild of root 1305 uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil) 1306 if err != nil { 1307 t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err) 1308 } 1309 1310 // Delete the 2nd k/v 1311 uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), testData[1].key) 1312 server.TestHTTP(t, "DELETE", uuid3req, nil) 1313 1314 server.TestBadHTTP(t, "GET", uuid3req, nil) 1315 1316 // Make sure the 2nd k/v is correct for each of previous versions. 1317 returnValue = server.TestHTTP(t, "GET", keyreq[1], nil) 1318 if !equalObjectJSON(returnValue, []byte(testData[1].val), ShowBasic) { 1319 t.Errorf("Error on first version, key %q: expected %s, got %s\n", testData[1].key, testData[1].val, string(returnValue)) 1320 } 1321 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 1322 if !equalObjectJSON(returnValue, expected2val, ShowBasic) { 1323 t.Errorf("Error on second version, key %q: expected %s, got %s\n", testData[1].key, expected2val, string(returnValue)) 1324 } 1325 }