github.com/janelia-flyem/dvid@v1.0.0/datatype/neuronjson/neuronjson.go (about) 1 /* 2 Package neuronjson implements DVID support for neuron JSON annotations 3 */ 4 package neuronjson 5 6 import ( 7 "archive/tar" 8 "bytes" 9 "encoding/gob" 10 "encoding/json" 11 "fmt" 12 "io/ioutil" 13 "math" 14 "net/http" 15 reflect "reflect" 16 "sort" 17 "strconv" 18 "strings" 19 "sync" 20 "time" 21 22 pb "google.golang.org/protobuf/proto" 23 24 "github.com/santhosh-tekuri/jsonschema/v5" 25 26 "github.com/janelia-flyem/dvid/datastore" 27 "github.com/janelia-flyem/dvid/datatype/common/proto" 28 "github.com/janelia-flyem/dvid/dvid" 29 "github.com/janelia-flyem/dvid/server" 30 "github.com/janelia-flyem/dvid/storage" 31 ) 32 33 const ( 34 Version = "0.1" 35 RepoURL = "github.com/janelia-flyem/dvid/datatype/neuronjson" 36 TypeName = "neuronjson" 37 ) 38 39 const helpMessage = ` 40 API for 'neuronjson' datatype (github.com/janelia-flyem/dvid/datatype/neuronjson) 41 ============================================================================= 42 43 The neuronjson datatype is similar supports most of the keyvalue datatype methods 44 but extends them to include queries. 45 46 The keys are body identifier uint64 that are represented as strings for 47 backward-compatibility with clients that used to use the keyvalue datatype 48 for these neuron JSON annotations. The values are assumed to be JSON data, 49 and the queries are similar to how Firestore handles queries. 50 51 Note: UUIDs referenced below are strings that may either be a unique prefix of a 52 hexadecimal UUID string (e.g., 3FA22) or a branch leaf specification that adds 53 a colon (":") followed by the case-dependent branch name. In the case of a 54 branch leaf specification, the unique UUID prefix just identifies the repo of 55 the branch, and the UUID referenced is really the leaf of the branch name. 56 For example, if we have a DAG with root A -> B -> C where C is the current 57 HEAD or leaf of the "master" (default) branch, then asking for "B:master" is 58 the same as asking for "C". If we add another version so A -> B -> C -> D, then 59 references to "B:master" now return the data from "D". 60 61 Command-line: 62 63 $ dvid repo <UUID> new neuronjson <data name> <settings...> 64 65 Adds newly named neuronjson data to repo with specified UUID. 66 67 Example: 68 69 $ dvid repo 3f8c new neuronjson stuff 70 71 Arguments: 72 73 UUID Hexadecimal string with enough characters to uniquely identify a version node. 74 data name Name of data to create, e.g., "myblobs" 75 settings Configuration settings in "key=value" format separated by spaces. 76 77 Configuration Settings (case-insensitive keys): 78 79 Versioned Set to "false" or "0" if the neuronjson instance is unversioned (repo-wide). 80 An unversioned neuronjson will only use the UUIDs to look up the repo and 81 not differentiate between versions in the same repo. Note that unlike 82 versioned data, distribution (push/pull) of unversioned data is not defined 83 at this time. 84 85 $ dvid -stdin node <UUID> <data name> put <key> < data 86 87 Puts stdin data into the neuronjson data instance under the given key. 88 89 $ dvid node <UUID> <dataname> import-kv <keyvalue instance name> 90 91 Imports the data from a keyvalue instance within the same repo into a 92 new neuronjson instance. 93 94 Example: 95 96 $ dvid repo 3f8c myNeuronJSON import-kv myOldKV 97 98 The above imports data from the keyvalue instance "myOldKV" into the neuronjson 99 instance "myNeuronJSON". 100 101 $ dvid node <UUID> <data name> version-changes <output-dir-path> 102 103 Creates a directory at the given output-dir-path if one doesn't already exist, 104 then writes a file per version that has annotation changes. 105 106 The annotation changes are a JSON object containing a list of all JSON annotations 107 added/modified in that version as well as a special tombstone annotation for deleted 108 annotations. 109 110 Example JSON for each "<uuid>.json" file within output directory: 111 112 [ {<annotation1>}, {<annotation2>}, {"bodyid":2000, "tombstone":true}, ...] 113 114 Note the tombstone example at end where bodyid 2000 annotation was deleted. 115 116 ------------------ 117 118 HTTP API (Level 2 REST): 119 120 Note that browsers support HTTP PUT and DELETE via javascript but only GET/POST are 121 included in HTML specs. For ease of use in constructing clients, HTTP POST is used 122 to create or modify resources in an idempotent fashion. 123 124 GET <api URL>/node/<UUID>/<data name>/help 125 126 Returns data-specific help message. 127 128 129 GET <api URL>/node/<UUID>/<data name>/info 130 POST <api URL>/node/<UUID>/<data name>/info 131 132 Retrieves or puts data properties. 133 134 Example: 135 136 GET <api URL>/node/3f8c/stuff/info 137 138 Returns JSON with configuration settings. 139 140 Arguments: 141 142 UUID Hexadecimal string with enough characters to uniquely identify a version node. 143 data name Name of neuronjson data instance. 144 145 GET <api URL>/node/<UUID>/<data name>/tags 146 POST <api URL>/node/<UUID>/<data name>/tags?<options> 147 148 GET retrieves JSON of tags for this instance. 149 POST appends or replaces tags provided in POST body. Expects JSON to be POSTed 150 with the following format: 151 152 { "tag1": "anything you want", "tag2": "something else" } 153 154 To delete tags, pass an empty object with query string "replace=true". 155 156 POST Query-string Options: 157 158 replace Set to "true" if you want passed tags to replace and not be appended to current tags. 159 Default operation is false (append). 160 161 GET <api URL>/node/<UUID>/<data name>/<schema type> 162 POST <api URL>/node/<UUID>/<data name>/<schema type> 163 DEL <api URL>/node/<UUID>/<data name>/<schema type> 164 HEAD <api URL>/node/<UUID>/<data name>/<schema type> 165 166 Performs operations on metadata schema depending on the HTTP verb. 167 If the "json_schema" type is POSTed, it will be used to validate 168 future writes of neuron annotations via POST /key, /keyvalues, etc. 169 170 Example: 171 172 GET <api URL>/node/3f8c/neuron_annotations/json_schema 173 174 Returns any JSON schema for validation stored for version node 3f8c. 175 176 The "Content-type" of the HTTP response (and usually the request) are "application/json". 177 178 Arguments: 179 180 UUID Hexadecimal string with enough characters to uniquely identify a version node. 181 data name Name of keyvalue data instance. 182 schema type One of "json_schema" (validation), "schema" (neutu/neu3), "schema_batch" (neutu/neu3) 183 184 GET <api URL>/node/<UUID>/<data name>/all[?query-options] 185 186 Returns a list of all JSON annotations 187 188 GET Query-string Options: 189 190 show If "user", shows *_user fields. 191 If "time", shows *_time fields. 192 If "all", shows both *_user and *_time fields. 193 If unset (default), shows neither *_user or *_time fields. 194 195 fields Limit return to this list of field names separated by commas. 196 Example: ?fields=type,instance 197 Note that the above "show" query string still applies to the fields. 198 199 GET <api URL>/node/<UUID>/<data name>/keys 200 201 Returns all keys for this data instance in JSON format: 202 203 [key1, key2, ...] 204 205 GET <api URL>/node/<UUID>/<data name>/fields 206 207 Returns all field names in annotations for the most recent version: 208 209 ["field1", "field2", ...] 210 211 GET <api URL>/node/<UUID>/<data name>/keyrange/<key1>/<key2> 212 213 Returns all keys between 'key1' and 'key2' for this data instance in JSON format: 214 215 [key1, key2, ...] 216 217 Arguments: 218 219 UUID Hexadecimal string with enough characters to uniquely identify a version node. 220 data name Name of neuronjson data instance. 221 key1 Lexicographically lowest alphanumeric key in range. 222 key2 Lexicographically highest alphanumeric key in range. 223 224 GET <api URL>/node/<UUID>/<data name>/keyrangevalues/<key1>/<key2>?<options> 225 226 This has the same response as the GET /keyvalues endpoint but a different way of 227 specifying the keys. In this endpoint, you specify a range of keys. In the other 228 endpoint, you must explicitly send the keys in a GET payload, which may not be 229 fully supported. 230 231 Note that this endpoint streams data to the requester, which prevents setting HTTP 232 status to error if the streaming has already started. Instead, malformed output 233 will inform the requester of an error. 234 235 Response types: 236 237 1) json (values are expected to be valid JSON or an error is returned) 238 239 { 240 "key1": value1, 241 "key2": value2, 242 ... 243 } 244 245 2) tar 246 247 A tarfile is returned with each keys specifying the file names and 248 values as the file bytes. 249 250 3) protobuf3 251 252 neuronjson data needs to be serialized in a format defined by the following 253 protobuf3 definitions: 254 255 message KeyValue { 256 string key = 1; 257 bytes value = 2; 258 } 259 260 message KeyValues { 261 repeated KeyValue kvs = 1; 262 } 263 264 Arguments: 265 266 UUID Hexadecimal string with enough characters to uniquely identify a version node. 267 data name Name of neuronjson data instance. 268 key1 Lexicographically lowest alphanumeric key in range. 269 key2 Lexicographically highest alphanumeric key in range. 270 271 Query-string Options (only one of these allowed): 272 273 json If set to "true", the response will be JSON as above and the values must 274 be valid JSON or an error will be returned. 275 tar If set to "true", the response will be a tarfile with keys as file names. 276 protobuf Default, or can be set to "true". Response will be protobuf KeyValues response 277 278 Additional query option: 279 280 check If json=true, setting check=false will tell server to trust that the 281 values will be valid JSON instead of parsing it as a check. 282 283 show If "user", shows *_user fields. 284 If "time", shows *_time fields. 285 If "all", shows both *_user and *_time fields. 286 If unset (default), shows neither *_user or *_time fields. 287 288 fields Limit return to this list of field names separated by commas. 289 Example: ?fields=type,instance 290 Note that the above "show" query string still applies to the fields. 291 292 293 GET <api URL>/node/<UUID>/<data name>/key/<key>[?query-options] 294 295 For a given neuron id key, returns a value depending on the options. 296 297 Example: 298 299 GET <api URL>/node/3f8c/stuff/key/myfile.dat 300 301 Returns the data associated with the key "myfile.dat" of instance "stuff" in version 302 node 3f8c. 303 304 The "Content-type" of the HTTP response (and usually the request) are 305 "application/octet-stream" for arbitrary binary data. 306 307 Arguments: 308 309 UUID Hexadecimal string with enough characters to uniquely identify a version node. 310 data name Name of neuronjson data instance. 311 key The uint64 of a neuron identifier 312 313 GET Query-string Options: 314 315 show If "user", shows *_user fields. 316 If "time", shows *_time fields. 317 If "all", shows both *_user and *_time fields. 318 If unset (default), shows neither *_user or *_time fields. 319 320 fields Limit return to this list of field names separated by commas. 321 Example: ?fields=type,instance 322 Note that the above "show" query string still applies to the fields. 323 324 325 POST <api URL>/node/<UUID>/<data name>/key/<key> 326 327 Updates a key-value pair, modifying the fields with the POSTed JSON fields. 328 Note that unlike POST /key in keyvalue datatype instances, this operation updates 329 fields by defaults (using old fields not overwritten) rather than replacing 330 the entire annotation. The replace behavior can be explicitly set if desired 331 to match old keyvalue semantics. 332 333 For each field, a *_user and *_time field will be added to the annotation unless 334 one is already present. The *_user field will be set to the user making the 335 request and the *_time field will be set to the current time. If the current 336 field value is the same as the new value, the *_user and *_time fields will 337 not be updated. 338 339 Example: 340 341 POST <api URL>/node/3f8c/stuff/key/15319 342 343 Arguments: 344 345 UUID Hexadecimal string with enough characters to uniquely identify a version node. 346 data name Name of neuronjson data instance. 347 key The uint64 of a neuron identifier 348 349 POSTs will be logged as a Kafka JSON message with the following format: 350 { 351 "Action": "postkv", 352 "Key": <key>, 353 "Bytes": <number of bytes in data>, 354 "UUID": <UUID on which POST was done> 355 } 356 357 POST Query-string Options: 358 359 conditional List of fields separated by commas that should not be overwritten if set. 360 361 replace If "true" will remove any fields not present 362 363 364 DELETE <api URL>/node/<UUID>/<data name>/key/<key> 365 HEAD <api URL>/node/<UUID>/<data name>/key/<key> 366 367 Performs operations on a key-value pair depending on the HTTP verb. 368 369 For HEAD returns: 370 200 (OK) if a sparse volume of the given label exists within any optional bounds. 371 404 (File not Found) if there is no sparse volume for the given label within any optional bounds. 372 373 Arguments: 374 375 UUID Hexadecimal string with enough characters to uniquely identify a version node. 376 data name Name of neuronjson data instance. 377 key The uint64 of a neuron identifier 378 379 380 GET <api URL>/node/<UUID>/<data name>/keyvalues[?query-options] 381 382 Allows batch query of data. 383 384 Unless using one of the JSON query options listed below, requested keys and 385 returned neuronjson data is serialized in a format defined by the following 386 protobuf3 definitions: 387 388 message KeyValue { 389 string key = 1; 390 bytes value = 2; 391 } 392 393 message Keys { 394 repeated string keys = 1; 395 } 396 397 message KeyValues { 398 repeated KeyValue kvs = 1; 399 } 400 401 The query body must include a Keys serialization and a KeyValues serialization is 402 returned. 403 404 Arguments: 405 406 UUID Hexadecimal string with enough characters to uniquely identify a version node. 407 data name Name of neuronjson data instance. 408 409 Query-string Options: 410 411 show If "user", shows *_user fields. 412 If "time", shows *_time fields. 413 If "all", shows both *_user and *_time fields. 414 If unset (default), shows neither *_user or *_time fields. 415 416 fields Limit return to this list of field names separated by commas. 417 Example: ?fields=type,instance 418 Note that the above "show" query string still applies to the fields. 419 420 Only one of the following are allowed in a single query: 421 422 json If true (default false), query body must be JSON array of keys and returns JSON. 423 jsontar If set to any value for GET, query body must be JSON array of string keys 424 and the returned data will be a tarfile with keys as file names. 425 protobuf If set to "true", the response will be protobuf KeyValues response 426 427 Response types: 428 429 1) json (values are expected to be valid JSON or an error is returned) 430 431 { 432 "key1": value1, 433 "key2": value2, 434 ... 435 } 436 437 2) tar 438 439 A tarfile is returned with each keys specifying the file names and 440 values as the file bytes. 441 442 3) protobuf3 443 444 KeyValue data needs to be serialized in a format defined by the following 445 protobuf3 definitions: 446 447 message KeyValue { 448 string key = 1; 449 bytes value = 2; 450 } 451 452 message KeyValues { 453 repeated KeyValue kvs = 1; 454 } 455 456 457 POST <api URL>/node/<UUID>/<data name>/keyvalues[?query-options] 458 459 Allows batch ingest of data. Each POSTed neuron annotation is handled in same 460 was as decribed in POST /key. 461 462 The POST body must include a KeyValues serialization as defined by the following 463 protobuf3 definitions: 464 465 message KeyValue { 466 string key = 1; 467 bytes value = 2; 468 } 469 470 message Keys { 471 repeated string keys = 1; 472 } 473 474 message KeyValues { 475 repeated KeyValue kvs = 1; 476 } 477 478 479 POSTs will be logged as a series of Kafka JSON messages, each with the format equivalent 480 to the single POST /key: 481 { 482 "Action": "postkv", 483 "Key": <key>, 484 "Bytes": <number of bytes in data>, 485 "UUID": <UUID on which POST was done> 486 } 487 488 Arguments: 489 490 UUID Hexadecimal string with enough characters to uniquely identify a version node. 491 data name Name of neuronjson data instance. 492 493 Query-string Options: 494 495 replace If "true" will remove any fields not present 496 497 498 GET <api URL>/node/<UUID>/<data name>/query[?show=...] 499 POST <api URL>/node/<UUID>/<data name>/query[?show=...] 500 501 Both GET and POST methods are permitted to launch queries, however the 502 POST method is deprecated because it will be blocked for committed versions. 503 The JSON query format uses field names as the keys, and desired values. 504 Example: 505 { "bodyid": 23, "hemilineage": "0B", ... } 506 Each field value must be true, i.e., the conditions are ANDed together. 507 508 If a list of queries (JSON object per query) is POSTed, the results for each query are ORed 509 together with duplicate annotations removed. 510 511 A JSON list of objects that matches the query is returned in ascending order of body ID. 512 513 Query fields can include two special types of values: 514 1. Regular expressions: a string value that starts with "re/" is treated as a regex with 515 the remainder of the string being the regex. The regex is anchored to the beginning. 516 2. Field existence: a string value that starts with "exists/" checks if a field exists. 517 If "exists/0" is specified, the field must not exist or be set to null. If "exists/1" 518 is specified, the field must exist. 519 520 Arguments: 521 522 UUID Hexadecimal string with enough characters to uniquely identify a version node. 523 data name Name of neuronjson data instance. 524 525 GET Query-string Options: 526 527 onlyid If true (false by default), will only return a list of body ids that match. 528 529 show If "user", shows *_user fields. 530 If "time", shows *_time fields. 531 If "all", shows both *_user and *_time fields. 532 If unset (default), shows neither *_user or *_time fields. 533 534 fields Limit return to this list of field names separated by commas. 535 Example: ?fields=type,instance 536 Note that the above "show" query string still applies to the fields. 537 ` 538 539 func init() { 540 datastore.Register(NewType()) 541 542 // Need to register types that will be used to fulfill interfaces. 543 gob.Register(&Type{}) 544 gob.Register(&Data{}) 545 gob.Register(map[string]interface{}{}) 546 } 547 548 // Schema describe various formats for neuron annotations 549 type Schema uint8 550 551 const ( 552 // JSONSchema is validation JSON schema for annotations 553 JSONSchema Schema = iota 554 555 // NeuSchema is JSON for neutu/neu3 clients 556 NeuSchema 557 558 // NeuSchemaBatch is JSON for neutu/neu3 clients 559 NeuSchemaBatch 560 ) 561 562 func (m Schema) String() string { 563 switch m { 564 case JSONSchema: 565 return "json_schema" 566 case NeuSchema: 567 return "schema" 568 case NeuSchemaBatch: 569 return "schema_batch" 570 default: 571 return "unknown metadata" 572 } 573 } 574 575 // Type embeds the datastore's Type to create a unique type for neuronjson functions. 576 type Type struct { 577 datastore.Type 578 } 579 580 // NewType returns a pointer to a new neuronjson Type with default values set. 581 func NewType() *Type { 582 dtype := new(Type) 583 dtype.Type = datastore.Type{ 584 Name: TypeName, 585 URL: RepoURL, 586 Version: Version, 587 Requirements: &storage.Requirements{}, 588 } 589 return dtype 590 } 591 592 // --- TypeService interface --- 593 594 // NewDataService returns a pointer to new neuronjson data with default values. 595 func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) { 596 basedata, err := datastore.NewDataService(dtype, uuid, id, name, c) 597 if err != nil { 598 return nil, err 599 } 600 data := &Data{Data: basedata} 601 dvid.Infof("Creating new neuronjson %q with UUID %s\n", name, uuid) 602 data.Initialize() 603 return data, nil 604 } 605 606 func (dtype *Type) Help() string { 607 return fmt.Sprint(helpMessage) 608 } 609 610 // GetByUUIDName returns a pointer to labelblk data given a UUID and data name. 611 func GetByUUIDName(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) { 612 source, err := datastore.GetDataByUUIDName(uuid, name) 613 if err != nil { 614 return nil, err 615 } 616 data, ok := source.(*Data) 617 if !ok { 618 return nil, fmt.Errorf("instance '%s' is not a neuronjson datatype", name) 619 } 620 return data, nil 621 } 622 623 // Parses keys as body ids, including things like 'a' that might be used in keyrange/0/a. 624 func parseKeyStr(key string) (uint64, error) { 625 if len(key) == 0 { 626 return 0, fmt.Errorf("key string is empty") 627 } 628 if key[0] > '9' { 629 return math.MaxUint64, nil 630 } else if key[0] < '0' { 631 return 0, nil 632 } 633 return strconv.ParseUint(key, 10, 64) 634 } 635 636 // Get bodyid from a JSON-like map 637 func getBodyID(data map[string]interface{}) (uint64, error) { 638 bodyidVal, ok := data["bodyid"] 639 if !ok { 640 return 0, fmt.Errorf("neuronjson record has no 'bodyid' field") 641 } 642 bodyid, ok := bodyidVal.(int64) 643 if !ok { 644 return 0, fmt.Errorf("neuronjson record 'bodyid' is not a uint64 value: %v", bodyidVal) 645 } 646 return uint64(bodyid), nil 647 } 648 649 type NeuronJSON map[string]interface{} 650 651 func (nj NeuronJSON) copy() NeuronJSON { 652 dup := make(NeuronJSON, len(nj)) 653 for k, v := range nj { 654 dup[k] = v 655 } 656 return dup 657 } 658 659 // UnmarshalJSON parses JSON with numbers preferentially converted to uint64 660 // or int64 if negative. 661 func (nj *NeuronJSON) UnmarshalJSON(jsonText []byte) error { 662 var raw map[string]json.RawMessage 663 if err := json.Unmarshal([]byte(jsonText), &raw); err != nil { 664 return err 665 } 666 *nj = make(NeuronJSON, len(raw)) 667 668 // NOTE: An incoming JSON integer could be uint64, int64, or float64 and we 669 // test in that order. We could force all integers to be int64, but then 670 // any field for body IDs would be int64 instead of uint64. 671 // Since we persist as JSON and only the in-memory HEAD makes these distinctions, 672 // we just need to make sure queries on in-memory NeuronJSONs are consistent. 673 for key, val := range raw { 674 s := string(val) 675 if s == "null" { 676 (*nj)[key] = nil 677 continue 678 } 679 u, err := strconv.ParseUint(s, 10, 64) 680 if err == nil { 681 (*nj)[key] = u 682 continue 683 } 684 i, err := strconv.ParseInt(s, 10, 64) 685 if err == nil { 686 (*nj)[key] = i 687 continue 688 } 689 f, err := strconv.ParseFloat(s, 64) 690 if err == nil { 691 (*nj)[key] = f 692 continue 693 } 694 var int64list []int64 695 if err = json.Unmarshal(val, &int64list); err == nil { 696 (*nj)[key] = int64list 697 continue 698 } 699 var strlist []string 700 if err = json.Unmarshal(val, &strlist); err == nil { 701 (*nj)[key] = strlist 702 continue 703 } 704 var listVal interface{} 705 if err = json.Unmarshal(val, &listVal); err == nil { 706 (*nj)[key] = listVal 707 continue 708 } 709 return fmt.Errorf("unable to parse JSON value %q: %v", s, err) 710 } 711 return nil 712 } 713 714 type ListNeuronJSON []NeuronJSON 715 716 func (lnj ListNeuronJSON) makeTimeless() ListNeuronJSON { 717 timelessJSON := make(ListNeuronJSON, len(lnj)) 718 for i, data := range lnj { 719 out := data.copy() 720 for field := range data { 721 if strings.HasSuffix(field, "_time") { 722 delete(out, field) 723 } 724 } 725 timelessJSON[i] = out 726 } 727 return timelessJSON 728 } 729 730 // --- implement sort interface 731 732 func (lnj *ListNeuronJSON) Len() int { 733 return len(*lnj) 734 } 735 736 func (lnj *ListNeuronJSON) Swap(i, j int) { 737 (*lnj)[i], (*lnj)[j] = (*lnj)[j], (*lnj)[i] 738 fmt.Printf("swapping %d and %d", i, j) 739 } 740 741 func (lnj *ListNeuronJSON) Less(i, j int) bool { 742 bodyid_i, ok := (*lnj)[i]["bodyid"].(uint64) 743 if !ok { 744 dvid.Criticalf("ListNeuronJSON bodyid not of uint64 type: %v", (*lnj)[i]["bodyid"]) 745 } 746 bodyid_j, ok := (*lnj)[j]["bodyid"].(uint64) 747 if !ok { 748 dvid.Criticalf("ListNeuronJSON bodyid not of uint64 type: %v", (*lnj)[j]["bodyid"]) 749 } 750 fmt.Printf("Comparing bodyid %d (%d) < %d (%d) \n", i, bodyid_i, j, bodyid_j) 751 return bodyid_i < bodyid_j 752 } 753 754 // Data embeds the datastore's Data and extends it with neuronjson properties. 755 type Data struct { 756 *datastore.Data 757 758 // The in-memory dbs for main HEAD and any other important versions. 759 dbs *memdbs 760 dbsMu sync.RWMutex 761 762 // The in-memory metadata for HEAD version 763 compiledSchema *jsonschema.Schema // cached on setting of JSONSchema value for rapid validate 764 765 metadata map[Schema][]byte 766 metadataMu sync.RWMutex 767 } 768 769 // IsMutationRequest overrides the default behavior to specify POST /query as an immutable 770 // request. 771 func (d *Data) IsMutationRequest(action, endpoint string) bool { 772 lc := strings.ToLower(action) 773 if endpoint == "query" && lc == "post" { 774 return false 775 } 776 return d.Data.IsMutationRequest(action, endpoint) // default for rest. 777 } 778 779 func (d *Data) Equals(d2 *Data) bool { 780 return d.Data.Equals(d2.Data) 781 } 782 783 func (d *Data) MarshalJSON() ([]byte, error) { 784 return json.Marshal(struct { 785 Base *datastore.Data 786 Extended struct{} 787 }{ 788 d.Data, 789 struct{}{}, 790 }) 791 } 792 793 func (d *Data) GobDecode(b []byte) error { 794 buf := bytes.NewBuffer(b) 795 dec := gob.NewDecoder(buf) 796 if err := dec.Decode(&(d.Data)); err != nil { 797 return err 798 } 799 return nil 800 } 801 802 func (d *Data) GobEncode() ([]byte, error) { 803 var buf bytes.Buffer 804 enc := gob.NewEncoder(&buf) 805 if err := enc.Encode(d.Data); err != nil { 806 return nil, err 807 } 808 return buf.Bytes(), nil 809 } 810 811 // JSONString returns the JSON for this Data's configuration 812 func (d *Data) JSONString() (jsonStr string, err error) { 813 m, err := json.Marshal(d) 814 if err != nil { 815 return "", err 816 } 817 return string(m), nil 818 } 819 820 // ----- 821 822 // putCmd handles a PUT command-line request. 823 func (d *Data) putCmd(cmd datastore.Request, reply *datastore.Response) error { 824 if len(cmd.Command) < 5 { 825 return fmt.Errorf("key name must be specified after 'put'") 826 } 827 if len(cmd.Input) == 0 { 828 return fmt.Errorf("no data was passed into standard input") 829 } 830 var uuidStr, dataName, cmdStr, keyStr string 831 cmd.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &keyStr) 832 833 _, versionID, err := datastore.MatchingUUID(uuidStr) 834 if err != nil { 835 return err 836 } 837 838 // Store data 839 if !d.Versioned() { 840 // Map everything to root version. 841 versionID, err = datastore.GetRepoRootVersion(versionID) 842 if err != nil { 843 return err 844 } 845 } 846 ctx := datastore.NewVersionedCtx(d, versionID) 847 if err = d.PutData(ctx, keyStr, cmd.Input, nil, false); err != nil { 848 return fmt.Errorf("error on put to key %q for neuronjson %q: %v", keyStr, d.DataName(), err) 849 } 850 851 reply.Output = []byte(fmt.Sprintf("Put %d bytes into key %q for neuronjson %q, uuid %s\n", 852 len(cmd.Input), keyStr, d.DataName(), uuidStr)) 853 return nil 854 } 855 856 ///// Persistence of neuronjson data to storage 857 858 // getStoreData gets a map value using a key 859 func (d *Data) getStoreData(ctx storage.Context, keyStr string) (value NeuronJSON, found bool, err error) { 860 var db storage.OrderedKeyValueDB 861 db, err = datastore.GetOrderedKeyValueDB(d) 862 if err != nil { 863 return 864 } 865 tk, err := NewTKey(keyStr) 866 if err != nil { 867 return 868 } 869 data, err := db.Get(ctx, tk) 870 if err != nil { 871 return nil, false, fmt.Errorf("error in retrieving key '%s': %v", keyStr, err) 872 } 873 if data == nil { 874 return 875 } 876 if err = json.Unmarshal(data, &value); err != nil { 877 return 878 } 879 return value, true, nil 880 } 881 882 // putStoreData puts a key / map value at a given uuid 883 func (d *Data) putStoreData(ctx storage.Context, keyStr string, value NeuronJSON) error { 884 db, err := datastore.GetOrderedKeyValueDB(d) 885 if err != nil { 886 return err 887 } 888 data, err := json.Marshal(value) 889 if err != nil { 890 return err 891 } 892 tk, err := NewTKey(keyStr) 893 if err != nil { 894 return err 895 } 896 return db.Put(ctx, tk, data) 897 } 898 899 // deleteStoreData deletes a key-value pair 900 func (d *Data) deleteStoreData(ctx storage.Context, keyStr string) error { 901 db, err := datastore.GetOrderedKeyValueDB(d) 902 if err != nil { 903 return err 904 } 905 tk, err := NewTKey(keyStr) 906 if err != nil { 907 return err 908 } 909 return db.Delete(ctx, tk) 910 } 911 912 // process a range of keys from store using supplied function. 913 func (d *Data) processStoreAllKeys(ctx storage.Context, f func(key string)) error { 914 minTKey := storage.MinTKey(keyAnnotation) 915 maxTKey := storage.MaxTKey(keyAnnotation) 916 return d.processStoreKeysInRange(ctx, minTKey, maxTKey, f) 917 } 918 919 // process a range of keys using supplied function. 920 func (d *Data) processStoreKeysInRange(ctx storage.Context, minTKey, maxTKey storage.TKey, f func(key string)) error { 921 db, err := datastore.GetOrderedKeyValueDB(d) 922 if err != nil { 923 return err 924 } 925 tkeys, err := db.KeysInRange(ctx, minTKey, maxTKey) 926 if err != nil { 927 return err 928 } 929 for _, tkey := range tkeys { 930 key, err := DecodeTKey(tkey) 931 if err != nil { 932 return err 933 } 934 f(key) 935 } 936 return nil 937 } 938 939 // process a range of key-value pairs using supplied function. 940 func (d *Data) processStoreRange(ctx storage.Context, f func(key string, value NeuronJSON)) error { 941 db, err := datastore.GetOrderedKeyValueDB(d) 942 if err != nil { 943 return err 944 } 945 first := storage.MinTKey(keyAnnotation) 946 last := storage.MaxTKey(keyAnnotation) 947 err = db.ProcessRange(ctx, first, last, &storage.ChunkOp{}, func(c *storage.Chunk) error { 948 if c == nil || c.TKeyValue == nil { 949 return nil 950 } 951 kv := c.TKeyValue 952 if kv.V == nil { 953 return nil 954 } 955 key, err := DecodeTKey(kv.K) 956 if err != nil { 957 return err 958 } 959 var value NeuronJSON 960 if err := value.UnmarshalJSON(kv.V); err != nil { 961 return err 962 } 963 f(key, value) 964 return nil 965 }) 966 return err 967 } 968 969 // Initialize loads mutable properties of the neuronjson data instance, 970 // which in this case is the in-memory neuron json map for the specified versions. 971 func (d *Data) Initialize() { 972 leafUUID, leafV, err := datastore.GetBranchHead(d.RootUUID(), "master") 973 if err != nil { 974 dvid.Infof("Can't find the leaf node of the main/master branch... skipping neuronjson initialization\n") 975 leafUUID = dvid.NilUUID 976 } 977 978 d.metadata = make(map[Schema][]byte, 3) 979 980 if leafUUID != dvid.NilUUID { 981 // Load the metadata 982 ctx := datastore.NewVersionedCtx(d, leafV) 983 if sch, err := d.getJSONSchema(ctx); err == nil { 984 if sch != nil { 985 d.compiledSchema = sch 986 } 987 } else { 988 dvid.Criticalf("Can't load JSON schema for neuronjson %q: %v\n", d.DataName(), err) 989 } 990 if value, err := d.loadMetadata(ctx, NeuSchema); err == nil { 991 dvid.Infof("Metadata load of neutu/neu3 JSON schema for %s: %d bytes\n", leafUUID[:6], len(value)) 992 if value != nil { 993 d.metadata[NeuSchema] = value 994 } 995 } else { 996 dvid.Criticalf("Can't load neutu/neu3 schema for neuronjson %q: %v\n", d.DataName(), err) 997 } 998 if value, err := d.loadMetadata(ctx, NeuSchemaBatch); err == nil { 999 dvid.Infof("Metadata load of neutu/neu3 JSON batch schema for %s: %d bytes\n", leafUUID[:6], len(value)) 1000 if value != nil { 1001 d.metadata[NeuSchemaBatch] = value 1002 } 1003 } else { 1004 dvid.Criticalf("Can't load neutu/neu3 batch schema for neuronjson %q: %v\n", d.DataName(), err) 1005 } 1006 } 1007 1008 // Load the in-memory databases for specified versions or branch HEADs 1009 store, err := storage.GetAssignedStore(d) 1010 if err != nil { 1011 dvid.Criticalf("Can't get assigned store for neuronjson %q: %v\n", d.DataName(), err) 1012 return 1013 } 1014 1015 storeConfig := store.GetStoreConfig() 1016 uuidList := []string{} 1017 uuidListI, found := storeConfig.Get("inmemory") 1018 if found { 1019 dvid.Infof("Found configuration for additional in-memory UUIDs for neuronjson %q: %v\n", 1020 d.DataName(), uuidListI) 1021 switch v := uuidListI.(type) { 1022 case []string: 1023 uuidList = v 1024 case []interface{}: 1025 for i, version := range v { 1026 versionStr, ok := version.(string) 1027 if ok { 1028 uuidList = append(uuidList, versionStr) 1029 } else { 1030 dvid.Criticalf("can't parse neuronjson inmemory config for %dth version: %v\n", i, version) 1031 } 1032 } 1033 } 1034 } 1035 if err := d.initMemoryDB(uuidList); err != nil { 1036 dvid.Criticalf("Can't initialize in-memory databases for neuronjson %q: %v\n", d.DataName(), err) 1037 } 1038 } 1039 1040 // KeyExists returns true if a key is found. 1041 func (d *Data) KeyExists(ctx storage.VersionedCtx, keyStr string) (found bool, err error) { 1042 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1043 if found { 1044 var bodyid uint64 1045 bodyid, err = strconv.ParseUint(keyStr, 10, 64) 1046 if err != nil { 1047 return false, err 1048 } 1049 mdb.mu.RLock() 1050 _, found = mdb.data[bodyid] 1051 mdb.mu.RUnlock() 1052 return found, nil 1053 } 1054 db, err := datastore.GetKeyValueDB(d) 1055 if err != nil { 1056 return false, err 1057 } 1058 tk, err := NewTKey(keyStr) 1059 if err != nil { 1060 return false, err 1061 } 1062 return db.Exists(ctx, tk) 1063 } 1064 1065 // GetKeysInRange returns all keys in the range [keyBeg, keyEnd]. Results on HEAD are ordered 1066 // by integer key, while results on other branches are ordered lexicographically. 1067 func (d *Data) GetKeysInRange(ctx storage.VersionedCtx, keyBeg, keyEnd string) (keys []string, err error) { 1068 var bodyidBeg, bodyidEnd uint64 1069 if bodyidBeg, err = parseKeyStr(keyBeg); err != nil { 1070 return 1071 } 1072 if bodyidEnd, err = parseKeyStr(keyEnd); err != nil { 1073 return 1074 } 1075 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1076 if found { 1077 mdb.mu.RLock() 1078 defer mdb.mu.RUnlock() 1079 begI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] >= bodyidBeg }) 1080 endI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] > bodyidEnd }) 1081 size := endI - begI 1082 if size <= 0 { 1083 keys = []string{} 1084 return 1085 } 1086 keys = make([]string, size) 1087 pos := 0 1088 for i := begI; i < endI; i++ { 1089 bodyid := mdb.ids[i] 1090 keys[pos] = strconv.FormatUint(bodyid, 10) 1091 pos++ 1092 } 1093 } else { 1094 var begTKey, endTKey storage.TKey 1095 begTKey, err = NewTKey(keyBeg) 1096 if err != nil { 1097 return nil, err 1098 } 1099 endTKey, err = NewTKey(keyEnd) 1100 if err != nil { 1101 return nil, err 1102 } 1103 process_func := func(key string) { 1104 bodyid, err := parseKeyStr(key) 1105 if err == nil && bodyid >= bodyidBeg && bodyid <= bodyidEnd { 1106 keys = append(keys, key) 1107 } 1108 } 1109 err = d.processStoreKeysInRange(ctx, begTKey, endTKey, process_func) 1110 } 1111 return 1112 } 1113 1114 func (d *Data) GetAll(ctx storage.VersionedCtx, fieldMap map[string]struct{}, showFields Fields) (ListNeuronJSON, error) { 1115 showUser, showTime := showFields.Bools() 1116 1117 var all ListNeuronJSON 1118 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1119 if found { 1120 mdb.mu.RLock() 1121 for _, value := range mdb.data { 1122 out := selectFields(value, fieldMap, showUser, showTime) 1123 if len(out) > 1 { 1124 all = append(all, out) 1125 } 1126 } 1127 mdb.mu.RUnlock() 1128 } else { 1129 process_func := func(key string, value NeuronJSON) { 1130 out := selectFields(value, fieldMap, showUser, showTime) 1131 if len(out) > 1 { 1132 all = append(all, out) 1133 } 1134 } 1135 if err := d.processStoreRange(ctx, process_func); err != nil { 1136 return nil, err 1137 } 1138 } 1139 return all, nil 1140 } 1141 1142 func (d *Data) GetKeys(ctx storage.VersionedCtx) (out []string, err error) { 1143 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1144 if found { 1145 mdb.mu.RLock() 1146 out = make([]string, len(mdb.ids)) 1147 for i, bodyid := range mdb.ids { 1148 out[i] = strconv.FormatUint(bodyid, 10) 1149 } 1150 mdb.mu.RUnlock() 1151 } else { 1152 process_func := func(key string) { 1153 out = append(out, key) 1154 } 1155 if err := d.processStoreAllKeys(ctx, process_func); err != nil { 1156 return nil, err 1157 } 1158 } 1159 return out, nil 1160 } 1161 1162 func (d *Data) GetFields(ctx storage.VersionedCtx) ([]string, error) { 1163 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1164 if !found { 1165 return nil, fmt.Errorf("unable to get fields because no in-memory db for neuronjson %q, version %d", d.DataName(), ctx.VersionID()) 1166 } 1167 mdb.mu.RLock() 1168 fields := make([]string, len(mdb.fields)) 1169 i := 0 1170 for field := range mdb.fields { 1171 fields[i] = field 1172 i++ 1173 } 1174 mdb.mu.RUnlock() 1175 return fields, nil 1176 } 1177 1178 // GetData gets a byte value using a key 1179 func (d *Data) GetData(ctx storage.VersionedCtx, keyStr string, fieldMap map[string]struct{}, showFields Fields) ([]byte, bool, error) { 1180 // Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances. 1181 switch keyStr { 1182 case NeuSchema.String(): 1183 data, err := d.getMetadata(ctx, NeuSchema) 1184 if err != nil { 1185 return nil, false, fmt.Errorf("unable to retrieve neutu/neu3 JSON schema: %v", err) 1186 } 1187 if data != nil { 1188 return data, true, nil 1189 } 1190 return nil, false, nil 1191 case NeuSchemaBatch.String(): 1192 data, err := d.getMetadata(ctx, NeuSchemaBatch) 1193 if err != nil { 1194 return nil, false, fmt.Errorf("unable to retrieve neutu/neu3 JSON batch schema: %v", err) 1195 } 1196 if data != nil { 1197 return data, true, nil 1198 } 1199 return nil, false, nil 1200 } 1201 bodyid, err := strconv.ParseUint(keyStr, 10, 64) 1202 if err != nil { 1203 return nil, false, err 1204 } 1205 var value map[string]interface{} 1206 var found bool 1207 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1208 if found { 1209 mdb.mu.RLock() 1210 value, found = mdb.data[bodyid] 1211 mdb.mu.RUnlock() 1212 if !found { 1213 return nil, false, nil 1214 } 1215 } else { 1216 value, found, err = d.getStoreData(ctx, keyStr) 1217 if !found || err != nil { 1218 return nil, false, err 1219 } 1220 } 1221 showUser, showTime := showFields.Bools() 1222 out := selectFields(value, fieldMap, showUser, showTime) 1223 data, err := json.Marshal(out) 1224 return data, true, err 1225 } 1226 1227 // update _user and _time fields for any fields newly set or modified. 1228 func updateJSON(origData, newData NeuronJSON, user string, conditionals []string, replace bool) { 1229 // determine if any fields are being set for the first time or modified 1230 newlySet := make(map[string]struct{}, len(newData)) // fields that aren't same as old data 1231 newFields := make(map[string]struct{}, len(newData)) // fields that are not _user or _time 1232 if origData == nil { 1233 for field := range newData { 1234 newlySet[field] = struct{}{} 1235 } 1236 } else { 1237 for field, value := range newData { 1238 if origValue, found := origData[field]; !found || !reflect.DeepEqual(value, origValue) { 1239 newlySet[field] = struct{}{} 1240 } 1241 if !strings.HasSuffix(field, "_user") && !strings.HasSuffix(field, "_time") { 1242 newFields[field] = struct{}{} 1243 } 1244 } 1245 1246 // carry forward any fields not being modified if replace option is not set 1247 if !replace { 1248 protectedFields := make(map[string]struct{}, len(conditionals)) 1249 for _, field := range conditionals { 1250 protectedFields[field] = struct{}{} 1251 } 1252 for field, origValue := range origData { 1253 if _, found := newData[field]; !found { 1254 newData[field] = origValue 1255 continue 1256 } 1257 if _, found := protectedFields[field]; found { 1258 newData[field] = origValue 1259 delete(newlySet, field) 1260 } 1261 } 1262 } 1263 } 1264 1265 // add _user and _time fields for newly set and not prevented via conditionals 1266 t := time.Now() 1267 timeStr := t.Format(time.RFC3339) 1268 for field := range newlySet { 1269 if field == "bodyid" || field == "user" { 1270 continue // these fields shouldn't have _user or _time fields added 1271 } 1272 if strings.HasSuffix(field, "_time") || strings.HasSuffix(field, "_user") { 1273 continue // we will handle this with main field 1274 } 1275 if _, foundUser := newlySet[field+"_user"]; !foundUser && user != "" { 1276 newData[field+"_user"] = user 1277 } 1278 if _, foundTime := newlySet[field+"_time"]; !foundTime { 1279 newData[field+"_time"] = timeStr 1280 } 1281 } 1282 if replace { // keep _user and _time fields for unchanged fields 1283 for field := range newFields { 1284 if _, foundUser := newData[field+"_user"]; !foundUser { 1285 oldUser, found := origData[field+"_user"] 1286 if !found { 1287 oldUser = user 1288 } 1289 newData[field+"_user"] = oldUser 1290 } 1291 if _, foundTime := newData[field+"_time"]; !foundTime { 1292 oldTime, found := origData[field+"_time"] 1293 if !found { 1294 oldTime = timeStr 1295 } 1296 newData[field+"_time"] = oldTime 1297 } 1298 } 1299 } 1300 } 1301 1302 func (d *Data) storeAndUpdate(ctx *datastore.VersionedCtx, keyStr string, newData NeuronJSON, conditionals []string, replace bool) error { 1303 bodyid, err := strconv.ParseUint(keyStr, 10, 64) 1304 if err != nil { 1305 return err 1306 } 1307 1308 // get original data so we can handle default update and tell which values change for _user/_time fields. 1309 origData, found, err := d.getStoreData(ctx, keyStr) 1310 if err != nil { 1311 return err 1312 } 1313 if !found { 1314 origData = nil 1315 } 1316 updateJSON(origData, newData, ctx.User, conditionals, replace) 1317 dvid.Infof("neuronjson %s put by user %q, conditionals %v, replace %t:\nOrig: %v\n New: %v\n", 1318 d.DataName(), ctx.User, conditionals, replace, origData, newData) 1319 1320 // write result 1321 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1322 if found { 1323 mdb.mu.Lock() 1324 mdb.data[bodyid] = newData 1325 for field := range newData { 1326 mdb.fields[field] = struct{}{} 1327 } 1328 mdb.addBodyID(bodyid) 1329 mdb.mu.Unlock() 1330 } 1331 return d.putStoreData(ctx, keyStr, newData) 1332 } 1333 1334 // PutData puts a valid JSON []byte into a neuron key at a given uuid. 1335 // If replace is true, will use given value instead of updating fields that were given. 1336 // If field values are given but do not change, the _user and _time fields will not be updated. 1337 func (d *Data) PutData(ctx *datastore.VersionedCtx, keyStr string, value []byte, conditionals []string, replace bool) error { 1338 // Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances. 1339 switch keyStr { 1340 case "0": 1341 return fmt.Errorf("body id 0 is reserved and so cannot be stored") 1342 case NeuSchema.String(): 1343 if err := d.putMetadata(ctx, value, NeuSchema); err != nil { 1344 return fmt.Errorf("unable to handle POST neutu/neu3 schema metadata: %v", err) 1345 } 1346 return nil 1347 case NeuSchemaBatch.String(): 1348 if err := d.putMetadata(ctx, value, NeuSchemaBatch); err != nil { 1349 return fmt.Errorf("unable to handle POST neutu/neu3 batch schema metadata: %v", err) 1350 } 1351 return nil 1352 } 1353 1354 // validate if we have a JSON schema 1355 if sch, err := d.getJSONSchema(ctx); err == nil { 1356 var v interface{} 1357 if err = json.Unmarshal(value, &v); err != nil { 1358 return err 1359 } 1360 for err = sch.Validate(v); err != nil; { 1361 if verr, ok := err.(*jsonschema.ValidationError); ok { 1362 if !strings.HasSuffix(verr.Error(), "expected integer, but got string") { 1363 return err 1364 } 1365 // Try to convert string to integer for fields that need conversion. 1366 var field string 1367 if _, scanerr := fmt.Sscanf(err.Error(), `jsonschema: %s does`, &field); scanerr != nil { 1368 return err 1369 } 1370 field = strings.Trim(field, `'/`) 1371 dvid.Infof("Converting string to integer for field %q\n", field) 1372 var newData NeuronJSON 1373 if err := json.Unmarshal(value, &newData); err != nil { 1374 return err 1375 } 1376 if newData[field], err = strconv.ParseInt(newData[field].(string), 10, 64); err != nil { 1377 return err 1378 } 1379 if value, err = json.Marshal(newData); err != nil { 1380 return err 1381 } 1382 } else { 1383 return err 1384 } 1385 } 1386 } else { 1387 dvid.Infof("Skipping validation of POST %q neuron annotation: %v\n", d.DataName(), err) 1388 } 1389 1390 var newData NeuronJSON 1391 if err := json.Unmarshal(value, &newData); err != nil { 1392 return err 1393 } 1394 return d.storeAndUpdate(ctx, keyStr, newData, conditionals, replace) 1395 } 1396 1397 // DeleteData deletes a key-value pair 1398 func (d *Data) DeleteData(ctx storage.VersionedCtx, keyStr string) error { 1399 // Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances. 1400 switch keyStr { 1401 case NeuSchema.String(): 1402 if err := d.deleteMetadata(ctx, NeuSchema); err != nil { 1403 return fmt.Errorf("unable to handle DELETE neutu/neu3 schema metadata: %v", err) 1404 } 1405 return nil 1406 case NeuSchemaBatch.String(): 1407 if err := d.deleteMetadata(ctx, NeuSchemaBatch); err != nil { 1408 return fmt.Errorf("unable to handle DELETE neutu/neu3 batch schema metadata: %v", err) 1409 } 1410 return nil 1411 } 1412 1413 bodyid, err := strconv.ParseUint(keyStr, 10, 64) 1414 if err != nil { 1415 return err 1416 } 1417 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1418 if found { 1419 mdb.mu.Lock() 1420 _, found := mdb.data[bodyid] 1421 if found { 1422 delete(mdb.data, bodyid) 1423 mdb.deleteBodyID(bodyid) 1424 } 1425 mdb.mu.Unlock() 1426 } 1427 return d.deleteStoreData(ctx, keyStr) 1428 } 1429 1430 // ----- Support functions for endpoint handlers ----- 1431 1432 type writeParams struct { 1433 w http.ResponseWriter 1434 ch chan writeData 1435 fieldMap map[string]struct{} 1436 showFields Fields 1437 checkVal bool 1438 } 1439 1440 type writeData struct { 1441 bodyid uint64 1442 jsonData NeuronJSON 1443 } 1444 1445 func selectData(data writeData, params *writeParams) (key string, jsonBytes []byte, err error) { 1446 showUser, showTime := params.showFields.Bools() 1447 out := selectFields(data.jsonData, params.fieldMap, showUser, showTime) 1448 key = strconv.FormatUint(data.bodyid, 10) 1449 jsonBytes, err = json.Marshal(out) 1450 return 1451 } 1452 1453 func streamJSONtar(params *writeParams) (numKeys int, err error) { 1454 params.w.Header().Set("Content-type", "application/tar") 1455 1456 tw := tar.NewWriter(params.w) 1457 1458 var key string 1459 var jsonBytes []byte 1460 for data := range params.ch { 1461 key, jsonBytes, err = selectData(data, params) 1462 if err != nil { 1463 return 0, err 1464 } 1465 1466 hdr := &tar.Header{ 1467 Name: key, 1468 Size: int64(len(jsonBytes)), 1469 Mode: 0755, 1470 } 1471 if err = tw.WriteHeader(hdr); err != nil { 1472 return 1473 } 1474 if _, err = tw.Write(jsonBytes); err != nil { 1475 return 1476 } 1477 numKeys++ 1478 } 1479 tw.Close() 1480 return numKeys, nil 1481 } 1482 1483 func streamJSON(params *writeParams) (numKeys int, err error) { 1484 params.w.Header().Set("Content-type", "application/json") 1485 if _, err = params.w.Write([]byte("{")); err != nil { 1486 return 1487 } 1488 1489 var wroteVal bool 1490 var key string 1491 var jsonBytes []byte 1492 for data := range params.ch { 1493 key, jsonBytes, err = selectData(data, params) 1494 if err != nil { 1495 return 0, err 1496 } 1497 1498 if wroteVal { 1499 if _, err = params.w.Write([]byte(",")); err != nil { 1500 return 1501 } 1502 } 1503 if len(jsonBytes) == 0 { 1504 jsonBytes = []byte("{}") 1505 } else if params.checkVal && !json.Valid(jsonBytes) { 1506 return 0, fmt.Errorf("bad JSON for key %q", key) 1507 } 1508 out := fmt.Sprintf(`"%s":`, key) 1509 if _, err = params.w.Write([]byte(out)); err != nil { 1510 return 1511 } 1512 if _, err = params.w.Write(jsonBytes); err != nil { 1513 return 1514 } 1515 wroteVal = true 1516 } 1517 if _, err = params.w.Write([]byte("}")); err != nil { 1518 return 1519 } 1520 return numKeys, nil 1521 } 1522 1523 func streamProtobuf(params *writeParams) (numKeys int, err error) { 1524 var kvs proto.KeyValues 1525 1526 var key string 1527 var jsonBytes []byte 1528 for data := range params.ch { 1529 key, jsonBytes, err = selectData(data, params) 1530 if err != nil { 1531 return 0, err 1532 } 1533 1534 kv := &proto.KeyValue{ 1535 Key: key, 1536 Value: jsonBytes, 1537 } 1538 kvs.Kvs = append(kvs.Kvs, kv) 1539 } 1540 numKeys = len(kvs.Kvs) 1541 var serialization []byte 1542 if serialization, err = pb.Marshal(&kvs); err != nil { 1543 return 1544 } 1545 params.w.Header().Set("Content-type", "application/octet-stream") 1546 if _, err = params.w.Write(serialization); err != nil { 1547 return 1548 } 1549 1550 return numKeys, nil 1551 } 1552 1553 // writes JSON values in given format for a range of keys with given beginning & end. 1554 func (d *Data) sendJSONValuesInRange(ctx storage.VersionedCtx, w http.ResponseWriter, 1555 r *http.Request, keyBeg, keyEnd string, fieldMap map[string]struct{}, showFields Fields) (numKeys int, err error) { 1556 1557 if len(keyBeg) == 0 || len(keyEnd) == 0 { 1558 return 0, fmt.Errorf("must specify non-empty beginning and ending key") 1559 } 1560 tarOut := (r.URL.Query().Get("jsontar") == "true") || (r.URL.Query().Get("tar") == "true") 1561 jsonOut := r.URL.Query().Get("json") == "true" 1562 if tarOut && jsonOut { 1563 err = fmt.Errorf("can only specify tar or json output, not both") 1564 return 1565 } 1566 1567 // Check range keys are valid for either in-memory or on-disk store. 1568 bodyidBeg, err := parseKeyStr(keyBeg) 1569 if err != nil { 1570 return 0, err 1571 } 1572 bodyidEnd, err := parseKeyStr(keyEnd) 1573 if err != nil { 1574 return 0, err 1575 } 1576 1577 first, err := NewTKey(keyBeg) 1578 if err != nil { 1579 return 0, err 1580 } 1581 last, err := NewTKey(keyEnd) 1582 if err != nil { 1583 return 0, err 1584 } 1585 db, err := datastore.GetOrderedKeyValueDB(d) 1586 if err != nil { 1587 return 0, err 1588 } 1589 1590 // Create goroutine to read NeuronJSONs from either in-memory or on-disk store. 1591 writeCh := make(chan writeData, 1000) 1592 params := writeParams{ 1593 w: w, 1594 ch: writeCh, 1595 fieldMap: fieldMap, 1596 showFields: showFields, 1597 checkVal: r.URL.Query().Get("check") == "true", 1598 } 1599 1600 mdb, found := d.getMemDBbyVersion(ctx.VersionID()) 1601 if found { 1602 go func() { 1603 mdb.mu.RLock() 1604 begI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] >= bodyidBeg }) 1605 endI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] > bodyidEnd }) 1606 mdb.mu.RUnlock() 1607 1608 for i := begI; i < endI; i++ { 1609 mdb.mu.RLock() 1610 bodyid := mdb.ids[i] 1611 jsonData, ok := mdb.data[bodyid] 1612 mdb.mu.RUnlock() 1613 if !ok { 1614 dvid.Errorf("inconsistent neuronjson DB: bodyid %d at pos %d is not in db cache... skipping", bodyid, i) 1615 continue 1616 } 1617 writeCh <- writeData{bodyid, jsonData} 1618 } 1619 close(writeCh) 1620 }() 1621 } else { // Handle on-disk store. 1622 go func() { 1623 err = db.ProcessRange(ctx, first, last, &storage.ChunkOp{}, func(c *storage.Chunk) error { 1624 if c == nil || c.TKeyValue == nil { 1625 return nil 1626 } 1627 kv := c.TKeyValue 1628 if kv.V == nil { 1629 return nil 1630 } 1631 key, err := DecodeTKey(kv.K) 1632 if err != nil { 1633 return err 1634 } 1635 var jsonData NeuronJSON 1636 if err := json.Unmarshal(kv.V, &jsonData); err != nil { 1637 return err 1638 } 1639 out := removeReservedFields(jsonData, showFields) 1640 bodyid, err := parseKeyStr(key) 1641 if err != nil { 1642 return err 1643 } 1644 writeCh <- writeData{bodyid, out} 1645 return nil 1646 }) 1647 close(writeCh) 1648 }() 1649 } 1650 1651 // Output requested format. 1652 switch { 1653 case tarOut: 1654 numKeys, err = streamJSONtar(¶ms) 1655 case jsonOut: 1656 numKeys, err = streamJSON(¶ms) 1657 default: 1658 numKeys, err = streamProtobuf(¶ms) 1659 } 1660 1661 return 1662 } 1663 1664 // writes JSON data for the given keys 1665 func (d *Data) sendJSONKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys []string, checkVal bool, 1666 fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) { 1667 1668 w.Header().Set("Content-type", "application/json") 1669 if writtenBytes, err = w.Write([]byte("{")); err != nil { 1670 return 1671 } 1672 var n int 1673 var foundKeys bool 1674 for _, key := range keys { 1675 var val []byte 1676 var found bool 1677 if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil { 1678 return 1679 } 1680 if !found { 1681 continue 1682 } else if foundKeys { 1683 if n, err = w.Write([]byte(",")); err != nil { 1684 return 1685 } 1686 writtenBytes += n 1687 } 1688 if len(val) == 0 { 1689 val = []byte("{}") 1690 } else if checkVal && !json.Valid(val) { 1691 err = fmt.Errorf("bad JSON for key %q", key) 1692 return 1693 } 1694 out := fmt.Sprintf(`"%s":`, key) 1695 if n, err = w.Write([]byte(out)); err != nil { 1696 return 1697 } 1698 writtenBytes += n 1699 if n, err = w.Write(val); err != nil { 1700 return 1701 } 1702 writtenBytes += n 1703 foundKeys = true 1704 } 1705 _, err = w.Write([]byte("}")) 1706 return 1707 } 1708 1709 // writes tarred JSON data for the given keys 1710 func (d *Data) sendTarKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys []string, 1711 fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) { 1712 1713 var n int 1714 w.Header().Set("Content-type", "application/tar") 1715 tw := tar.NewWriter(w) 1716 for _, key := range keys { 1717 var val []byte 1718 var found bool 1719 if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil { 1720 return 1721 } 1722 if !found { 1723 val = nil 1724 } 1725 hdr := &tar.Header{ 1726 Name: key, 1727 Size: int64(len(val)), 1728 Mode: 0755, 1729 } 1730 if err = tw.WriteHeader(hdr); err != nil { 1731 return 1732 } 1733 if n, err = tw.Write(val); err != nil { 1734 return 1735 } 1736 writtenBytes += n 1737 } 1738 tw.Close() 1739 return 1740 } 1741 1742 // writes protobuf-encoded keyvalues for the given keys 1743 func (d *Data) sendProtobufKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys *proto.Keys, 1744 fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) { 1745 var kvs proto.KeyValues 1746 kvs.Kvs = make([]*proto.KeyValue, len(keys.Keys)) 1747 for i, key := range keys.Keys { 1748 var val []byte 1749 var found bool 1750 if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil { 1751 return 1752 } 1753 if !found { 1754 val = nil 1755 } 1756 kvs.Kvs[i] = &proto.KeyValue{ 1757 Key: key, 1758 Value: val, 1759 } 1760 } 1761 var serialization []byte 1762 if serialization, err = pb.Marshal(&kvs); err != nil { 1763 return 1764 } 1765 w.Header().Set("Content-type", "application/octet-stream") 1766 if writtenBytes, err = w.Write(serialization); err != nil { 1767 return 1768 } 1769 if writtenBytes != len(serialization) { 1770 err = fmt.Errorf("unable to write all %d bytes of serialized keyvalues: only %d bytes written", len(serialization), writtenBytes) 1771 } 1772 return 1773 } 1774 1775 // writes JSON values in given format for keys sent in the request body. 1776 // Uses random reads since we assume keys aren't ordered. 1777 func (d *Data) handleKeyValues(ctx storage.VersionedCtx, w http.ResponseWriter, r *http.Request, 1778 uuid dvid.UUID, fieldMap map[string]struct{}, showFields Fields) (numKeys, writtenBytes int, err error) { 1779 1780 tarOut := (r.URL.Query().Get("jsontar") == "true") || (r.URL.Query().Get("tar") == "true") 1781 jsonOut := r.URL.Query().Get("json") == "true" 1782 checkVal := r.URL.Query().Get("check") == "true" 1783 1784 if tarOut && jsonOut { 1785 err = fmt.Errorf("can only specify tar or json output, not both") 1786 return 1787 } 1788 var data []byte 1789 data, err = ioutil.ReadAll(r.Body) 1790 if err != nil { 1791 return 1792 } 1793 switch { 1794 case tarOut: 1795 var keys []string 1796 if err = json.Unmarshal(data, &keys); err != nil { 1797 return 1798 } 1799 numKeys = len(keys) 1800 writtenBytes, err = d.sendTarKV(ctx, w, keys, fieldMap, showFields) 1801 case jsonOut: 1802 var keys []string 1803 var keysInt []uint64 1804 if err = json.Unmarshal(data, &keysInt); err == nil { 1805 // convert to string keys for compatibility with keyvalue type & downstream code 1806 keys = make([]string, len(keysInt)) 1807 for i, n := range keysInt { 1808 keys[i] = strconv.FormatUint(n, 10) 1809 } 1810 } else if err = json.Unmarshal(data, &keys); err != nil { 1811 return 1812 } 1813 numKeys = len(keys) 1814 writtenBytes, err = d.sendJSONKV(ctx, w, keys, checkVal, fieldMap, showFields) 1815 default: 1816 var keys proto.Keys 1817 if err = pb.Unmarshal(data, &keys); err != nil { 1818 return 1819 } 1820 numKeys = len(keys.Keys) 1821 writtenBytes, err = d.sendProtobufKV(ctx, w, &keys, fieldMap, showFields) 1822 } 1823 return 1824 } 1825 1826 func (d *Data) handleIngest(ctx *datastore.VersionedCtx, r *http.Request, uuid dvid.UUID) error { 1827 cond_fields := r.URL.Query().Get("conditionals") 1828 conditionals := strings.Split(cond_fields, ",") 1829 replace := r.URL.Query().Get("replace") == "true" 1830 data, err := ioutil.ReadAll(r.Body) 1831 if err != nil { 1832 return err 1833 } 1834 var kvs proto.KeyValues 1835 if err := pb.Unmarshal(data, &kvs); err != nil { 1836 return err 1837 } 1838 for _, kv := range kvs.Kvs { 1839 err = d.PutData(ctx, kv.Key, kv.Value, conditionals, replace) 1840 if err != nil { 1841 return err 1842 } 1843 1844 msginfo := map[string]interface{}{ 1845 "Action": "ingestneuronjson", 1846 "Key": kv.Key, 1847 "Bytes": len(kv.Value), 1848 "UUID": string(uuid), 1849 "Timestamp": time.Now().String(), 1850 } 1851 jsonmsg, _ := json.Marshal(msginfo) 1852 if err = d.PublishKafkaMsg(jsonmsg); err != nil { 1853 dvid.Errorf("Error on sending neuronjson POST op to kafka: %v\n", err) 1854 } 1855 } 1856 return nil 1857 } 1858 1859 // --- DataService interface --- 1860 1861 func (d *Data) Help() string { 1862 return fmt.Sprint(helpMessage) 1863 } 1864 1865 // DoRPC acts as a switchboard for RPC commands. 1866 func (d *Data) DoRPC(request datastore.Request, reply *datastore.Response) error { 1867 switch request.TypeCommand() { 1868 case "put": 1869 return d.putCmd(request, reply) 1870 case "import-kv": 1871 return d.importKV(request, reply) 1872 case "version-changes": 1873 return d.versionChanges(request, reply) 1874 default: 1875 return fmt.Errorf("unknown command. Data %q [%s] does not support %q command", 1876 d.DataName(), d.TypeName(), request.TypeCommand()) 1877 } 1878 } 1879 1880 func (d *Data) handleSchema(ctx storage.VersionedCtx, w http.ResponseWriter, r *http.Request, uuid dvid.UUID, action string, meta Schema) error { 1881 switch action { 1882 case "head": 1883 found, err := d.metadataExists(ctx, meta) 1884 if err != nil { 1885 return err 1886 } 1887 if found { 1888 w.WriteHeader(http.StatusOK) 1889 } else { 1890 w.WriteHeader(http.StatusNotFound) 1891 } 1892 1893 case "get": 1894 value, err := d.getMetadata(ctx, meta) 1895 if err != nil { 1896 return err 1897 } else if value == nil { 1898 w.WriteHeader(http.StatusNotFound) 1899 return nil 1900 } 1901 if _, err := w.Write(value); err != nil { 1902 return err 1903 } 1904 w.Header().Set("Content-Type", "application/json") 1905 1906 case "delete": 1907 if err := d.deleteMetadata(ctx, meta); err != nil { 1908 return err 1909 } 1910 1911 case "post": 1912 data, err := ioutil.ReadAll(r.Body) 1913 if err != nil { 1914 return err 1915 } 1916 if err := d.putMetadata(ctx, data, meta); err != nil { 1917 return err 1918 } 1919 1920 default: 1921 return fmt.Errorf("key endpoint does not support %q HTTP verb", action) 1922 } 1923 return nil 1924 } 1925 1926 // ServeHTTP handles all incoming HTTP requests for this data. 1927 func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) (activity map[string]interface{}) { 1928 timedLog := dvid.NewTimeLog() 1929 1930 // Break URL request into arguments 1931 url := r.URL.Path[len(server.WebAPIPath):] 1932 parts := strings.Split(url, "/") 1933 if len(parts[len(parts)-1]) == 0 { 1934 parts = parts[:len(parts)-1] 1935 } 1936 1937 if len(parts) < 4 { 1938 server.BadRequest(w, r, "incomplete API specification") 1939 return 1940 } 1941 1942 var comment string 1943 action := strings.ToLower(r.Method) 1944 1945 switch parts[3] { 1946 case "help": 1947 w.Header().Set("Content-Type", "text/plain") 1948 fmt.Fprintln(w, d.Help()) 1949 return 1950 1951 case "info": 1952 jsonStr, err := d.JSONString() 1953 if err != nil { 1954 server.BadRequest(w, r, err) 1955 return 1956 } 1957 w.Header().Set("Content-Type", "application/json") 1958 fmt.Fprint(w, jsonStr) 1959 return 1960 1961 case "tags": 1962 if action == "post" { 1963 replace := r.URL.Query().Get("replace") == "true" 1964 if err := datastore.SetTagsByJSON(d, uuid, replace, r.Body); err != nil { 1965 server.BadRequest(w, r, err) 1966 return 1967 } 1968 } else { 1969 jsonBytes, err := d.MarshalJSONTags() 1970 if err != nil { 1971 server.BadRequest(w, r, err) 1972 return 1973 } 1974 w.Header().Set("Content-Type", "application/json") 1975 fmt.Fprint(w, string(jsonBytes)) 1976 } 1977 1978 case JSONSchema.String(): 1979 if err := d.handleSchema(ctx, w, r, uuid, action, JSONSchema); err != nil { 1980 server.BadRequest(w, r, err) 1981 return 1982 } 1983 1984 case NeuSchema.String(): 1985 if err := d.handleSchema(ctx, w, r, uuid, action, NeuSchema); err != nil { 1986 server.BadRequest(w, r, err) 1987 return 1988 } 1989 1990 case NeuSchemaBatch.String(): 1991 if err := d.handleSchema(ctx, w, r, uuid, action, NeuSchemaBatch); err != nil { 1992 server.BadRequest(w, r, err) 1993 return 1994 } 1995 1996 case "all": 1997 kvList, err := d.GetAll(ctx, fieldMap(r), showFields(r)) 1998 if err != nil { 1999 server.BadRequest(w, r, err) 2000 return 2001 } 2002 jsonBytes, err := json.Marshal(kvList) 2003 if err != nil { 2004 server.BadRequest(w, r, err) 2005 return 2006 } 2007 w.Header().Set("Content-Type", "application/json") 2008 fmt.Fprint(w, string(jsonBytes)) 2009 comment = "HTTP GET all" 2010 2011 case "keys": 2012 keyList, err := d.GetKeys(ctx) 2013 if err != nil { 2014 server.BadRequest(w, r, err) 2015 return 2016 } 2017 jsonBytes, err := json.Marshal(keyList) 2018 if err != nil { 2019 server.BadRequest(w, r, err) 2020 return 2021 } 2022 w.Header().Set("Content-Type", "application/json") 2023 fmt.Fprint(w, string(jsonBytes)) 2024 comment = "HTTP GET keys" 2025 2026 case "fields": 2027 fieldList, err := d.GetFields(ctx) 2028 if err != nil { 2029 server.BadRequest(w, r, err) 2030 return 2031 } 2032 jsonBytes, err := json.Marshal(fieldList) 2033 if err != nil { 2034 server.BadRequest(w, r, err) 2035 return 2036 } 2037 w.Header().Set("Content-Type", "application/json") 2038 fmt.Fprint(w, string(jsonBytes)) 2039 comment = "HTTP GET fields" 2040 2041 case "query": 2042 if action != "post" && action != "get" { 2043 server.BadRequest(w, r, fmt.Errorf("only GET or POST methods allowed for /query endpoint")) 2044 return 2045 } 2046 onlyid := r.URL.Query().Get("onlyid") == "true" 2047 err := d.Query(ctx, w, uuid, onlyid, fieldMap(r), showFields(r), r.Body) 2048 if err != nil { 2049 server.BadRequest(w, r, err) 2050 return 2051 } 2052 2053 case "keyrange": 2054 if len(parts) < 6 { 2055 server.BadRequest(w, r, "expect beginning and end keys to follow 'keyrange' endpoint") 2056 return 2057 } 2058 2059 // Return JSON list of keys 2060 keyBeg := parts[4] 2061 keyEnd := parts[5] 2062 keyList, err := d.GetKeysInRange(ctx, keyBeg, keyEnd) 2063 if err != nil { 2064 server.BadRequest(w, r, err) 2065 return 2066 } 2067 jsonBytes, err := json.Marshal(keyList) 2068 if err != nil { 2069 server.BadRequest(w, r, err) 2070 return 2071 } 2072 w.Header().Set("Content-Type", "application/json") 2073 fmt.Fprint(w, string(jsonBytes)) 2074 comment = fmt.Sprintf("HTTP GET keyrange [%q, %q]", keyBeg, keyEnd) 2075 2076 case "keyrangevalues": 2077 if len(parts) < 6 { 2078 server.BadRequest(w, r, "expect beginning and end keys to follow 'keyrangevalues' endpoint") 2079 return 2080 } 2081 2082 // Return JSON list of keys 2083 keyBeg := parts[4] 2084 keyEnd := parts[5] 2085 w.Header().Set("Content-Type", "application/json") 2086 2087 numKeys, err := d.sendJSONValuesInRange(ctx, w, r, keyBeg, keyEnd, fieldMap(r), showFields(r)) 2088 if err != nil { 2089 server.BadRequest(w, r, err) 2090 return 2091 } 2092 comment = fmt.Sprintf("HTTP GET keyrangevalues sent %d values for [%q, %q]", numKeys, keyBeg, keyEnd) 2093 2094 case "keyvalues": 2095 switch action { 2096 case "get": 2097 numKeys, writtenBytes, err := d.handleKeyValues(ctx, w, r, uuid, fieldMap(r), showFields(r)) 2098 if err != nil { 2099 server.BadRequest(w, r, "GET /keyvalues on %d keys, data %q: %v", numKeys, d.DataName(), err) 2100 return 2101 } 2102 comment = fmt.Sprintf("HTTP GET keyvalues on %d keys, %d bytes, data %q", numKeys, writtenBytes, d.DataName()) 2103 case "post": 2104 if err := d.handleIngest(ctx, r, uuid); err != nil { 2105 server.BadRequest(w, r, err) 2106 return 2107 } 2108 comment = fmt.Sprintf("HTTP POST keyvalues on data %q", d.DataName()) 2109 default: 2110 server.BadRequest(w, r, "key endpoint does not support %q HTTP verb", action) 2111 return 2112 } 2113 2114 case "key": 2115 if len(parts) < 5 { 2116 server.BadRequest(w, r, "expect key string to follow 'key' endpoint") 2117 return 2118 } 2119 keyStr := parts[4] 2120 2121 switch action { 2122 case "head": 2123 found, err := d.KeyExists(ctx, keyStr) 2124 if err != nil { 2125 server.BadRequest(w, r, err) 2126 return 2127 } 2128 if found { 2129 w.WriteHeader(http.StatusOK) 2130 } else { 2131 w.WriteHeader(http.StatusNotFound) 2132 } 2133 return 2134 2135 case "get": 2136 // Return value of single key 2137 value, found, err := d.GetData(ctx, keyStr, fieldMap(r), showFields(r)) 2138 if err != nil { 2139 server.BadRequest(w, r, err) 2140 return 2141 } 2142 if !found { 2143 http.Error(w, fmt.Sprintf("Key %q not found", keyStr), http.StatusNotFound) 2144 return 2145 } 2146 if value != nil || len(value) > 0 { 2147 _, err = w.Write(value) 2148 if err != nil { 2149 server.BadRequest(w, r, err) 2150 return 2151 } 2152 w.Header().Set("Content-Type", "application/octet-stream") 2153 } 2154 comment = fmt.Sprintf("HTTP GET key %q of neuronjson %q: %d bytes (%s)", keyStr, d.DataName(), len(value), url) 2155 2156 case "delete": 2157 if err := d.DeleteData(ctx, keyStr); err != nil { 2158 server.BadRequest(w, r, err) 2159 return 2160 } 2161 comment = fmt.Sprintf("HTTP DELETE data with key %q of neuronjson %q (%s)", keyStr, d.DataName(), url) 2162 2163 case "post": 2164 data, err := ioutil.ReadAll(r.Body) 2165 if err != nil { 2166 server.BadRequest(w, r, err) 2167 return 2168 } 2169 2170 go func() { 2171 msginfo := map[string]interface{}{ 2172 "Action": "postneuronjson", 2173 "Key": keyStr, 2174 "Bytes": len(data), 2175 "UUID": string(uuid), 2176 "Timestamp": time.Now().String(), 2177 } 2178 jsonmsg, _ := json.Marshal(msginfo) 2179 if err = d.PublishKafkaMsg(jsonmsg); err != nil { 2180 dvid.Errorf("Error on sending neuronjson POST op to kafka: %v\n", err) 2181 } 2182 }() 2183 2184 cond_fields := r.URL.Query().Get("conditionals") 2185 conditionals := strings.Split(cond_fields, ",") 2186 replace := r.URL.Query().Get("replace") == "true" 2187 2188 err = d.PutData(ctx, keyStr, data, conditionals, replace) 2189 if err != nil { 2190 server.BadRequest(w, r, err) 2191 return 2192 } 2193 comment = fmt.Sprintf("HTTP POST neuronjson '%s': %d bytes (%s)", d.DataName(), len(data), url) 2194 default: 2195 server.BadRequest(w, r, "key endpoint does not support %q HTTP verb", action) 2196 return 2197 } 2198 2199 default: 2200 server.BadAPIRequest(w, r, d) 2201 return 2202 } 2203 2204 timedLog.Infof(comment) 2205 return 2206 }