github.com/janelia-flyem/dvid@v1.0.0/datatype/neuronjson/neuronjson.go (about)

     1  /*
     2  Package neuronjson implements DVID support for neuron JSON annotations
     3  */
     4  package neuronjson
     5  
     6  import (
     7  	"archive/tar"
     8  	"bytes"
     9  	"encoding/gob"
    10  	"encoding/json"
    11  	"fmt"
    12  	"io/ioutil"
    13  	"math"
    14  	"net/http"
    15  	reflect "reflect"
    16  	"sort"
    17  	"strconv"
    18  	"strings"
    19  	"sync"
    20  	"time"
    21  
    22  	pb "google.golang.org/protobuf/proto"
    23  
    24  	"github.com/santhosh-tekuri/jsonschema/v5"
    25  
    26  	"github.com/janelia-flyem/dvid/datastore"
    27  	"github.com/janelia-flyem/dvid/datatype/common/proto"
    28  	"github.com/janelia-flyem/dvid/dvid"
    29  	"github.com/janelia-flyem/dvid/server"
    30  	"github.com/janelia-flyem/dvid/storage"
    31  )
    32  
    33  const (
    34  	Version  = "0.1"
    35  	RepoURL  = "github.com/janelia-flyem/dvid/datatype/neuronjson"
    36  	TypeName = "neuronjson"
    37  )
    38  
    39  const helpMessage = `
    40  API for 'neuronjson' datatype (github.com/janelia-flyem/dvid/datatype/neuronjson)
    41  =============================================================================
    42  
    43  The neuronjson datatype is similar supports most of the keyvalue datatype methods
    44  but extends them to include queries.  
    45  
    46  The keys are body identifier uint64 that are represented as strings for 
    47  backward-compatibility with clients that used to use the keyvalue datatype 
    48  for these neuron JSON annotations. The values are assumed to be JSON data, 
    49  and the queries are similar to how Firestore handles queries.
    50  
    51  Note: UUIDs referenced below are strings that may either be a unique prefix of a
    52  hexadecimal UUID string (e.g., 3FA22) or a branch leaf specification that adds
    53  a colon (":") followed by the case-dependent branch name.  In the case of a
    54  branch leaf specification, the unique UUID prefix just identifies the repo of
    55  the branch, and the UUID referenced is really the leaf of the branch name.
    56  For example, if we have a DAG with root A -> B -> C where C is the current
    57  HEAD or leaf of the "master" (default) branch, then asking for "B:master" is
    58  the same as asking for "C".  If we add another version so A -> B -> C -> D, then
    59  references to "B:master" now return the data from "D".
    60  
    61  Command-line:
    62  
    63  $ dvid repo <UUID> new neuronjson <data name> <settings...>
    64  
    65  	Adds newly named neuronjson data to repo with specified UUID.
    66  
    67  	Example:
    68  
    69  	$ dvid repo 3f8c new neuronjson stuff
    70  
    71  	Arguments:
    72  
    73  	UUID           Hexadecimal string with enough characters to uniquely identify a version node.
    74  	data name      Name of data to create, e.g., "myblobs"
    75  	settings       Configuration settings in "key=value" format separated by spaces.
    76  
    77  	Configuration Settings (case-insensitive keys):
    78  
    79  	Versioned      Set to "false" or "0" if the neuronjson instance is unversioned (repo-wide).
    80  				   An unversioned neuronjson will only use the UUIDs to look up the repo and
    81  				   not differentiate between versions in the same repo.  Note that unlike
    82  				   versioned data, distribution (push/pull) of unversioned data is not defined 
    83  				   at this time.
    84  
    85  $ dvid -stdin node <UUID> <data name> put <key> < data
    86  
    87  	Puts stdin data into the neuronjson data instance under the given key.
    88  
    89  $ dvid node <UUID> <dataname> import-kv <keyvalue instance name>
    90  
    91  	Imports the data from a keyvalue instance within the same repo into a
    92  	new neuronjson instance.
    93  
    94  	Example:
    95  
    96  	$ dvid repo 3f8c myNeuronJSON import-kv myOldKV
    97  
    98  	The above imports data from the keyvalue instance "myOldKV" into the neuronjson
    99  	instance "myNeuronJSON".
   100  
   101  $ dvid node <UUID> <data name> version-changes <output-dir-path>
   102  
   103  	Creates a directory at the given output-dir-path if one doesn't already exist,
   104  	then writes a file per version that has annotation changes. 
   105  
   106  	The annotation changes are a JSON object containing a list of all JSON annotations 
   107  	added/modified in that version as well as a special tombstone annotation for deleted 
   108  	annotations.
   109  
   110  	Example JSON for each "<uuid>.json" file within output directory:
   111  
   112  		[ {<annotation1>}, {<annotation2>}, {"bodyid":2000, "tombstone":true}, ...]
   113  
   114  	Note the tombstone example at end where bodyid 2000 annotation was deleted.
   115  						
   116  	------------------
   117  
   118  HTTP API (Level 2 REST):
   119  
   120  Note that browsers support HTTP PUT and DELETE via javascript but only GET/POST are
   121  included in HTML specs.  For ease of use in constructing clients, HTTP POST is used
   122  to create or modify resources in an idempotent fashion.
   123  
   124  GET  <api URL>/node/<UUID>/<data name>/help
   125  
   126  	Returns data-specific help message.
   127  
   128  
   129  GET  <api URL>/node/<UUID>/<data name>/info
   130  POST <api URL>/node/<UUID>/<data name>/info
   131  
   132  	Retrieves or puts data properties.
   133  
   134  	Example: 
   135  
   136  	GET <api URL>/node/3f8c/stuff/info
   137  
   138  	Returns JSON with configuration settings.
   139  
   140  	Arguments:
   141  
   142  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   143  	data name     Name of neuronjson data instance.
   144  
   145  GET <api URL>/node/<UUID>/<data name>/tags
   146  POST <api URL>/node/<UUID>/<data name>/tags?<options>
   147  
   148  	GET retrieves JSON of tags for this instance.
   149  	POST appends or replaces tags provided in POST body.  Expects JSON to be POSTed
   150  	with the following format:
   151  
   152  	{ "tag1": "anything you want", "tag2": "something else" }
   153  
   154  	To delete tags, pass an empty object with query string "replace=true".
   155  
   156  	POST Query-string Options:
   157  
   158  	replace   Set to "true" if you want passed tags to replace and not be appended to current tags.
   159  				Default operation is false (append).
   160  
   161  GET  <api URL>/node/<UUID>/<data name>/<schema type>
   162  POST <api URL>/node/<UUID>/<data name>/<schema type>
   163  DEL  <api URL>/node/<UUID>/<data name>/<schema type> 
   164  HEAD <api URL>/node/<UUID>/<data name>/<schema type> 
   165  
   166  	Performs operations on metadata schema depending on the HTTP verb.  
   167  	If the "json_schema" type is POSTed, it will be used to validate
   168  	future writes of neuron annotations via POST /key, /keyvalues, etc.
   169  
   170  	Example: 
   171  
   172  	GET <api URL>/node/3f8c/neuron_annotations/json_schema
   173  
   174  	Returns any JSON schema for validation stored for version node 3f8c.
   175  
   176  	The "Content-type" of the HTTP response (and usually the request) are "application/json".
   177  
   178  	Arguments:
   179  
   180  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   181  	data name     Name of keyvalue data instance.
   182  	schema type	  One of "json_schema" (validation), "schema" (neutu/neu3), "schema_batch" (neutu/neu3)
   183  				
   184  GET  <api URL>/node/<UUID>/<data name>/all[?query-options]
   185  
   186  	Returns a list of all JSON annotations
   187  
   188  	GET Query-string Options:
   189  
   190  	show		If "user", shows *_user fields.
   191  				If "time", shows *_time fields.
   192  				If "all", shows both *_user and *_time fields.
   193  				If unset (default), shows neither *_user or *_time fields.
   194  	
   195  	fields      Limit return to this list of field names separated by commas.
   196                  Example: ?fields=type,instance
   197  				Note that the above "show" query string still applies to the fields.
   198  			
   199  GET  <api URL>/node/<UUID>/<data name>/keys
   200  
   201  	Returns all keys for this data instance in JSON format:
   202  
   203  	[key1, key2, ...]
   204  
   205  GET  <api URL>/node/<UUID>/<data name>/fields
   206  
   207  	Returns all field names in annotations for the most recent version:
   208  
   209  	["field1", "field2", ...]
   210  
   211  GET  <api URL>/node/<UUID>/<data name>/keyrange/<key1>/<key2>
   212  
   213  	Returns all keys between 'key1' and 'key2' for this data instance in JSON format:
   214  
   215  	[key1, key2, ...]
   216  
   217  	Arguments:
   218  
   219  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   220  	data name     Name of neuronjson data instance.
   221  	key1          Lexicographically lowest alphanumeric key in range.
   222  	key2          Lexicographically highest alphanumeric key in range.
   223  
   224  GET  <api URL>/node/<UUID>/<data name>/keyrangevalues/<key1>/<key2>?<options>
   225  
   226  	This has the same response as the GET /keyvalues endpoint but a different way of
   227  	specifying the keys.  In this endpoint, you specify a range of keys.  In the other
   228  	endpoint, you must explicitly send the keys in a GET payload, which may not be
   229  	fully supported.
   230  
   231  	Note that this endpoint streams data to the requester, which prevents setting HTTP
   232  	status to error if the streaming has already started.  Instead, malformed output
   233  	will inform the requester of an error.
   234  
   235  	Response types:
   236  
   237  	1) json (values are expected to be valid JSON or an error is returned)
   238  
   239  		{
   240  			"key1": value1,
   241  			"key2": value2,
   242  			...
   243  		}
   244  
   245  	2) tar
   246  
   247  		A tarfile is returned with each keys specifying the file names and
   248  		values as the file bytes.
   249  
   250  	3) protobuf3
   251  	
   252  		neuronjson data needs to be serialized in a format defined by the following 
   253  		protobuf3 definitions:
   254  
   255  		message KeyValue {
   256  			string key = 1;
   257  			bytes value = 2;
   258  		}
   259  
   260  		message KeyValues {
   261  			repeated KeyValue kvs = 1;
   262  		}
   263  
   264  	Arguments:
   265  
   266  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   267  	data name     Name of neuronjson data instance.
   268  	key1          Lexicographically lowest alphanumeric key in range.
   269  	key2          Lexicographically highest alphanumeric key in range.
   270  
   271  	Query-string Options (only one of these allowed):
   272  
   273  	json        If set to "true", the response will be JSON as above and the values must
   274  				  be valid JSON or an error will be returned.
   275  	tar			If set to "true", the response will be a tarfile with keys as file names.
   276  	protobuf	Default, or can be set to "true". Response will be protobuf KeyValues response
   277  
   278  	Additional query option:
   279  
   280  	check		If json=true, setting check=false will tell server to trust that the
   281  				  values will be valid JSON instead of parsing it as a check.
   282  
   283  	show		If "user", shows *_user fields.
   284  				If "time", shows *_time fields.
   285  				If "all", shows both *_user and *_time fields.
   286  				If unset (default), shows neither *_user or *_time fields.
   287  	
   288  	fields      Limit return to this list of field names separated by commas.
   289                  Example: ?fields=type,instance
   290  				Note that the above "show" query string still applies to the fields.
   291  
   292  
   293  GET  <api URL>/node/<UUID>/<data name>/key/<key>[?query-options]
   294  
   295  	For a given neuron id key, returns a value depending on the options.  
   296  
   297  	Example: 
   298  
   299  	GET <api URL>/node/3f8c/stuff/key/myfile.dat
   300  
   301  	Returns the data associated with the key "myfile.dat" of instance "stuff" in version
   302  	node 3f8c.
   303  
   304  	The "Content-type" of the HTTP response (and usually the request) are
   305  	"application/octet-stream" for arbitrary binary data.
   306  
   307  	Arguments:
   308  
   309  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   310  	data name     Name of neuronjson data instance.
   311  	key           The uint64 of a neuron identifier
   312  
   313  	GET Query-string Options:
   314  
   315  	show		If "user", shows *_user fields.
   316  				If "time", shows *_time fields.
   317  				If "all", shows both *_user and *_time fields.
   318  				If unset (default), shows neither *_user or *_time fields.
   319  	
   320  	fields      Limit return to this list of field names separated by commas.
   321                  Example: ?fields=type,instance
   322  				Note that the above "show" query string still applies to the fields.
   323  
   324  
   325  POST <api URL>/node/<UUID>/<data name>/key/<key>
   326  
   327  	Updates a key-value pair, modifying the fields with the POSTed JSON fields.
   328  	Note that unlike POST /key in keyvalue datatype instances, this operation updates
   329  	fields by defaults (using old fields not overwritten) rather than replacing
   330  	the entire annotation. The replace behavior can be explicitly set if desired
   331  	to match old keyvalue semantics.  
   332  
   333  	For each field, a *_user and *_time field will be added to the annotation unless
   334  	one is already present.  The *_user field will be set to the user making the
   335  	request and the *_time field will be set to the current time. If the current
   336  	field value is the same as the new value, the *_user and *_time fields will
   337  	not be updated.
   338  
   339  	Example: 
   340  
   341  	POST <api URL>/node/3f8c/stuff/key/15319
   342  
   343  	Arguments:
   344  
   345  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   346  	data name     Name of neuronjson data instance.
   347  	key           The uint64 of a neuron identifier
   348  	
   349  	POSTs will be logged as a Kafka JSON message with the following format:
   350  	{ 
   351  		"Action": "postkv",
   352  		"Key": <key>,
   353  		"Bytes": <number of bytes in data>,
   354  		"UUID": <UUID on which POST was done>
   355  	}
   356  
   357  	POST Query-string Options:
   358  
   359  	conditional	List of fields separated by commas that should not be overwritten if set.
   360  
   361  	replace		If "true" will remove any fields not present
   362  
   363  
   364  DELETE <api URL>/node/<UUID>/<data name>/key/<key> 
   365  HEAD   <api URL>/node/<UUID>/<data name>/key/<key> 
   366  
   367  	Performs operations on a key-value pair depending on the HTTP verb.  
   368  
   369  	For HEAD returns:
   370  	200 (OK) if a sparse volume of the given label exists within any optional bounds.
   371  	404 (File not Found) if there is no sparse volume for the given label within any optional bounds.
   372  
   373  	Arguments:
   374  
   375  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   376  	data name     Name of neuronjson data instance.
   377  	key           The uint64 of a neuron identifier
   378  				
   379  
   380  GET <api URL>/node/<UUID>/<data name>/keyvalues[?query-options]
   381  
   382  	Allows batch query of data. 
   383  
   384  	Unless using one of the JSON query options listed below, requested keys and
   385  	returned neuronjson data is serialized in a format defined by the following 
   386  	protobuf3 definitions:
   387  
   388  		message KeyValue {
   389  			string key = 1;
   390  			bytes value = 2;
   391  		}
   392  
   393  		message Keys {
   394  			repeated string keys = 1;
   395  		}
   396  		
   397  		message KeyValues {
   398  			repeated KeyValue kvs = 1;
   399  		}
   400  	
   401  	The query body must include a Keys serialization and a KeyValues serialization is
   402  	returned.
   403  
   404  	Arguments:
   405  
   406  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   407  	data name     Name of neuronjson data instance.
   408  
   409  	Query-string Options:
   410  
   411  	show		If "user", shows *_user fields.
   412  				If "time", shows *_time fields.
   413  				If "all", shows both *_user and *_time fields.
   414  				If unset (default), shows neither *_user or *_time fields.
   415  	
   416  	fields      Limit return to this list of field names separated by commas.
   417                  Example: ?fields=type,instance
   418  				Note that the above "show" query string still applies to the fields.
   419  
   420  	Only one of the following are allowed in a single query:
   421  
   422  	json        If true (default false), query body must be JSON array of keys and returns JSON.
   423  	jsontar		If set to any value for GET, query body must be JSON array of string keys
   424  				  and the returned data will be a tarfile with keys as file names.
   425  	protobuf	If set to "true", the response will be protobuf KeyValues response
   426  
   427  	Response types:
   428  
   429  	1) json (values are expected to be valid JSON or an error is returned)
   430  
   431  		{
   432  			"key1": value1,
   433  			"key2": value2,
   434  			...
   435  		}
   436  
   437  	2) tar
   438  
   439  		A tarfile is returned with each keys specifying the file names and
   440  		values as the file bytes.
   441  
   442  	3) protobuf3
   443  	
   444  		KeyValue data needs to be serialized in a format defined by the following 
   445  		protobuf3 definitions:
   446  
   447  		message KeyValue {
   448  			string key = 1;
   449  			bytes value = 2;
   450  		}
   451  
   452  		message KeyValues {
   453  			repeated KeyValue kvs = 1;
   454  		}
   455  
   456  
   457  POST <api URL>/node/<UUID>/<data name>/keyvalues[?query-options]
   458  
   459  	Allows batch ingest of data. Each POSTed neuron annotation is handled in same
   460  	was as decribed in POST /key.
   461  	
   462  	The POST body must include a KeyValues serialization as defined by the following
   463  	protobuf3 definitions:
   464  
   465  		message KeyValue {
   466  			string key = 1;
   467  			bytes value = 2;
   468  		}
   469  
   470  		message Keys {
   471  			repeated string keys = 1;
   472  		}
   473  		
   474  		message KeyValues {
   475  			repeated KeyValue kvs = 1;
   476  		}
   477  	
   478  	
   479  	POSTs will be logged as a series of Kafka JSON messages, each with the format equivalent
   480  	to the single POST /key:
   481  	{ 
   482  		"Action": "postkv",
   483  		"Key": <key>,
   484  		"Bytes": <number of bytes in data>,
   485  		"UUID": <UUID on which POST was done>
   486  	}
   487  
   488  	Arguments:
   489  
   490  	UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   491  	data name     Name of neuronjson data instance.
   492  
   493  	Query-string Options:
   494  
   495  	replace		If "true" will remove any fields not present
   496  
   497  
   498  GET <api URL>/node/<UUID>/<data name>/query[?show=...]
   499  POST <api URL>/node/<UUID>/<data name>/query[?show=...]
   500  
   501  	Both GET and POST methods are permitted to launch queries, however the
   502  	POST method is deprecated because it will be blocked for committed versions.
   503  	The JSON query format uses field names as the keys, and desired values.
   504  	Example:
   505  	{ "bodyid": 23, "hemilineage": "0B", ... }
   506  	Each field value must be true, i.e., the conditions are ANDed together.
   507  
   508  	If a list of queries (JSON object per query) is POSTed, the results for each query are ORed
   509  	together with duplicate annotations removed.
   510  
   511  	A JSON list of objects that matches the query is returned in ascending order of body ID.
   512  
   513  	Query fields can include two special types of values:
   514  	1. Regular expressions: a string value that starts with "re/" is treated as a regex with
   515  	   the remainder of the string being the regex.  The regex is anchored to the beginning.
   516  	2. Field existence: a string value that starts with "exists/" checks if a field exists.
   517  	   If "exists/0" is specified, the field must not exist or be set to null.  If "exists/1" 
   518  	   is specified, the field must exist.
   519  
   520  	Arguments:
   521  
   522  	UUID 		Hexadecimal string with enough characters to uniquely identify a version node.
   523  	data name	Name of neuronjson data instance.
   524  
   525  	GET Query-string Options:
   526  
   527  	onlyid		If true (false by default), will only return a list of body ids that match.
   528  
   529  	show		If "user", shows *_user fields.
   530  				If "time", shows *_time fields.
   531  				If "all", shows both *_user and *_time fields.
   532  				If unset (default), shows neither *_user or *_time fields.
   533  
   534  	fields      Limit return to this list of field names separated by commas.
   535                  Example: ?fields=type,instance
   536  				Note that the above "show" query string still applies to the fields.
   537  `
   538  
   539  func init() {
   540  	datastore.Register(NewType())
   541  
   542  	// Need to register types that will be used to fulfill interfaces.
   543  	gob.Register(&Type{})
   544  	gob.Register(&Data{})
   545  	gob.Register(map[string]interface{}{})
   546  }
   547  
   548  // Schema describe various formats for neuron annotations
   549  type Schema uint8
   550  
   551  const (
   552  	// JSONSchema is validation JSON schema for annotations
   553  	JSONSchema Schema = iota
   554  
   555  	// NeuSchema is JSON for neutu/neu3 clients
   556  	NeuSchema
   557  
   558  	// NeuSchemaBatch is JSON for neutu/neu3 clients
   559  	NeuSchemaBatch
   560  )
   561  
   562  func (m Schema) String() string {
   563  	switch m {
   564  	case JSONSchema:
   565  		return "json_schema"
   566  	case NeuSchema:
   567  		return "schema"
   568  	case NeuSchemaBatch:
   569  		return "schema_batch"
   570  	default:
   571  		return "unknown metadata"
   572  	}
   573  }
   574  
   575  // Type embeds the datastore's Type to create a unique type for neuronjson functions.
   576  type Type struct {
   577  	datastore.Type
   578  }
   579  
   580  // NewType returns a pointer to a new neuronjson Type with default values set.
   581  func NewType() *Type {
   582  	dtype := new(Type)
   583  	dtype.Type = datastore.Type{
   584  		Name:         TypeName,
   585  		URL:          RepoURL,
   586  		Version:      Version,
   587  		Requirements: &storage.Requirements{},
   588  	}
   589  	return dtype
   590  }
   591  
   592  // --- TypeService interface ---
   593  
   594  // NewDataService returns a pointer to new neuronjson data with default values.
   595  func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
   596  	basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
   597  	if err != nil {
   598  		return nil, err
   599  	}
   600  	data := &Data{Data: basedata}
   601  	dvid.Infof("Creating new neuronjson %q with UUID %s\n", name, uuid)
   602  	data.Initialize()
   603  	return data, nil
   604  }
   605  
   606  func (dtype *Type) Help() string {
   607  	return fmt.Sprint(helpMessage)
   608  }
   609  
   610  // GetByUUIDName returns a pointer to labelblk data given a UUID and data name.
   611  func GetByUUIDName(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) {
   612  	source, err := datastore.GetDataByUUIDName(uuid, name)
   613  	if err != nil {
   614  		return nil, err
   615  	}
   616  	data, ok := source.(*Data)
   617  	if !ok {
   618  		return nil, fmt.Errorf("instance '%s' is not a neuronjson datatype", name)
   619  	}
   620  	return data, nil
   621  }
   622  
   623  // Parses keys as body ids, including things like 'a' that might be used in keyrange/0/a.
   624  func parseKeyStr(key string) (uint64, error) {
   625  	if len(key) == 0 {
   626  		return 0, fmt.Errorf("key string is empty")
   627  	}
   628  	if key[0] > '9' {
   629  		return math.MaxUint64, nil
   630  	} else if key[0] < '0' {
   631  		return 0, nil
   632  	}
   633  	return strconv.ParseUint(key, 10, 64)
   634  }
   635  
   636  // Get bodyid from a JSON-like map
   637  func getBodyID(data map[string]interface{}) (uint64, error) {
   638  	bodyidVal, ok := data["bodyid"]
   639  	if !ok {
   640  		return 0, fmt.Errorf("neuronjson record has no 'bodyid' field")
   641  	}
   642  	bodyid, ok := bodyidVal.(int64)
   643  	if !ok {
   644  		return 0, fmt.Errorf("neuronjson record 'bodyid' is not a uint64 value: %v", bodyidVal)
   645  	}
   646  	return uint64(bodyid), nil
   647  }
   648  
   649  type NeuronJSON map[string]interface{}
   650  
   651  func (nj NeuronJSON) copy() NeuronJSON {
   652  	dup := make(NeuronJSON, len(nj))
   653  	for k, v := range nj {
   654  		dup[k] = v
   655  	}
   656  	return dup
   657  }
   658  
   659  // UnmarshalJSON parses JSON with numbers preferentially converted to uint64
   660  // or int64 if negative.
   661  func (nj *NeuronJSON) UnmarshalJSON(jsonText []byte) error {
   662  	var raw map[string]json.RawMessage
   663  	if err := json.Unmarshal([]byte(jsonText), &raw); err != nil {
   664  		return err
   665  	}
   666  	*nj = make(NeuronJSON, len(raw))
   667  
   668  	// NOTE: An incoming JSON integer could be uint64, int64, or float64 and we
   669  	//  test in that order. We could force all integers to be int64, but then
   670  	//  any field for body IDs would be int64 instead of uint64.
   671  	//  Since we persist as JSON and only the in-memory HEAD makes these distinctions,
   672  	//  we just need to make sure queries on in-memory NeuronJSONs are consistent.
   673  	for key, val := range raw {
   674  		s := string(val)
   675  		if s == "null" {
   676  			(*nj)[key] = nil
   677  			continue
   678  		}
   679  		u, err := strconv.ParseUint(s, 10, 64)
   680  		if err == nil {
   681  			(*nj)[key] = u
   682  			continue
   683  		}
   684  		i, err := strconv.ParseInt(s, 10, 64)
   685  		if err == nil {
   686  			(*nj)[key] = i
   687  			continue
   688  		}
   689  		f, err := strconv.ParseFloat(s, 64)
   690  		if err == nil {
   691  			(*nj)[key] = f
   692  			continue
   693  		}
   694  		var int64list []int64
   695  		if err = json.Unmarshal(val, &int64list); err == nil {
   696  			(*nj)[key] = int64list
   697  			continue
   698  		}
   699  		var strlist []string
   700  		if err = json.Unmarshal(val, &strlist); err == nil {
   701  			(*nj)[key] = strlist
   702  			continue
   703  		}
   704  		var listVal interface{}
   705  		if err = json.Unmarshal(val, &listVal); err == nil {
   706  			(*nj)[key] = listVal
   707  			continue
   708  		}
   709  		return fmt.Errorf("unable to parse JSON value %q: %v", s, err)
   710  	}
   711  	return nil
   712  }
   713  
   714  type ListNeuronJSON []NeuronJSON
   715  
   716  func (lnj ListNeuronJSON) makeTimeless() ListNeuronJSON {
   717  	timelessJSON := make(ListNeuronJSON, len(lnj))
   718  	for i, data := range lnj {
   719  		out := data.copy()
   720  		for field := range data {
   721  			if strings.HasSuffix(field, "_time") {
   722  				delete(out, field)
   723  			}
   724  		}
   725  		timelessJSON[i] = out
   726  	}
   727  	return timelessJSON
   728  }
   729  
   730  // --- implement sort interface
   731  
   732  func (lnj *ListNeuronJSON) Len() int {
   733  	return len(*lnj)
   734  }
   735  
   736  func (lnj *ListNeuronJSON) Swap(i, j int) {
   737  	(*lnj)[i], (*lnj)[j] = (*lnj)[j], (*lnj)[i]
   738  	fmt.Printf("swapping %d and %d", i, j)
   739  }
   740  
   741  func (lnj *ListNeuronJSON) Less(i, j int) bool {
   742  	bodyid_i, ok := (*lnj)[i]["bodyid"].(uint64)
   743  	if !ok {
   744  		dvid.Criticalf("ListNeuronJSON bodyid not of uint64 type: %v", (*lnj)[i]["bodyid"])
   745  	}
   746  	bodyid_j, ok := (*lnj)[j]["bodyid"].(uint64)
   747  	if !ok {
   748  		dvid.Criticalf("ListNeuronJSON bodyid not of uint64 type: %v", (*lnj)[j]["bodyid"])
   749  	}
   750  	fmt.Printf("Comparing bodyid %d (%d) < %d (%d) \n", i, bodyid_i, j, bodyid_j)
   751  	return bodyid_i < bodyid_j
   752  }
   753  
   754  // Data embeds the datastore's Data and extends it with neuronjson properties.
   755  type Data struct {
   756  	*datastore.Data
   757  
   758  	// The in-memory dbs for main HEAD and any other important versions.
   759  	dbs   *memdbs
   760  	dbsMu sync.RWMutex
   761  
   762  	// The in-memory metadata for HEAD version
   763  	compiledSchema *jsonschema.Schema // cached on setting of JSONSchema value for rapid validate
   764  
   765  	metadata   map[Schema][]byte
   766  	metadataMu sync.RWMutex
   767  }
   768  
   769  // IsMutationRequest overrides the default behavior to specify POST /query as an immutable
   770  // request.
   771  func (d *Data) IsMutationRequest(action, endpoint string) bool {
   772  	lc := strings.ToLower(action)
   773  	if endpoint == "query" && lc == "post" {
   774  		return false
   775  	}
   776  	return d.Data.IsMutationRequest(action, endpoint) // default for rest.
   777  }
   778  
   779  func (d *Data) Equals(d2 *Data) bool {
   780  	return d.Data.Equals(d2.Data)
   781  }
   782  
   783  func (d *Data) MarshalJSON() ([]byte, error) {
   784  	return json.Marshal(struct {
   785  		Base     *datastore.Data
   786  		Extended struct{}
   787  	}{
   788  		d.Data,
   789  		struct{}{},
   790  	})
   791  }
   792  
   793  func (d *Data) GobDecode(b []byte) error {
   794  	buf := bytes.NewBuffer(b)
   795  	dec := gob.NewDecoder(buf)
   796  	if err := dec.Decode(&(d.Data)); err != nil {
   797  		return err
   798  	}
   799  	return nil
   800  }
   801  
   802  func (d *Data) GobEncode() ([]byte, error) {
   803  	var buf bytes.Buffer
   804  	enc := gob.NewEncoder(&buf)
   805  	if err := enc.Encode(d.Data); err != nil {
   806  		return nil, err
   807  	}
   808  	return buf.Bytes(), nil
   809  }
   810  
   811  // JSONString returns the JSON for this Data's configuration
   812  func (d *Data) JSONString() (jsonStr string, err error) {
   813  	m, err := json.Marshal(d)
   814  	if err != nil {
   815  		return "", err
   816  	}
   817  	return string(m), nil
   818  }
   819  
   820  // -----
   821  
   822  // putCmd handles a PUT command-line request.
   823  func (d *Data) putCmd(cmd datastore.Request, reply *datastore.Response) error {
   824  	if len(cmd.Command) < 5 {
   825  		return fmt.Errorf("key name must be specified after 'put'")
   826  	}
   827  	if len(cmd.Input) == 0 {
   828  		return fmt.Errorf("no data was passed into standard input")
   829  	}
   830  	var uuidStr, dataName, cmdStr, keyStr string
   831  	cmd.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &keyStr)
   832  
   833  	_, versionID, err := datastore.MatchingUUID(uuidStr)
   834  	if err != nil {
   835  		return err
   836  	}
   837  
   838  	// Store data
   839  	if !d.Versioned() {
   840  		// Map everything to root version.
   841  		versionID, err = datastore.GetRepoRootVersion(versionID)
   842  		if err != nil {
   843  			return err
   844  		}
   845  	}
   846  	ctx := datastore.NewVersionedCtx(d, versionID)
   847  	if err = d.PutData(ctx, keyStr, cmd.Input, nil, false); err != nil {
   848  		return fmt.Errorf("error on put to key %q for neuronjson %q: %v", keyStr, d.DataName(), err)
   849  	}
   850  
   851  	reply.Output = []byte(fmt.Sprintf("Put %d bytes into key %q for neuronjson %q, uuid %s\n",
   852  		len(cmd.Input), keyStr, d.DataName(), uuidStr))
   853  	return nil
   854  }
   855  
   856  ///// Persistence of neuronjson data to storage
   857  
   858  // getStoreData gets a map value using a key
   859  func (d *Data) getStoreData(ctx storage.Context, keyStr string) (value NeuronJSON, found bool, err error) {
   860  	var db storage.OrderedKeyValueDB
   861  	db, err = datastore.GetOrderedKeyValueDB(d)
   862  	if err != nil {
   863  		return
   864  	}
   865  	tk, err := NewTKey(keyStr)
   866  	if err != nil {
   867  		return
   868  	}
   869  	data, err := db.Get(ctx, tk)
   870  	if err != nil {
   871  		return nil, false, fmt.Errorf("error in retrieving key '%s': %v", keyStr, err)
   872  	}
   873  	if data == nil {
   874  		return
   875  	}
   876  	if err = json.Unmarshal(data, &value); err != nil {
   877  		return
   878  	}
   879  	return value, true, nil
   880  }
   881  
   882  // putStoreData puts a key / map value at a given uuid
   883  func (d *Data) putStoreData(ctx storage.Context, keyStr string, value NeuronJSON) error {
   884  	db, err := datastore.GetOrderedKeyValueDB(d)
   885  	if err != nil {
   886  		return err
   887  	}
   888  	data, err := json.Marshal(value)
   889  	if err != nil {
   890  		return err
   891  	}
   892  	tk, err := NewTKey(keyStr)
   893  	if err != nil {
   894  		return err
   895  	}
   896  	return db.Put(ctx, tk, data)
   897  }
   898  
   899  // deleteStoreData deletes a key-value pair
   900  func (d *Data) deleteStoreData(ctx storage.Context, keyStr string) error {
   901  	db, err := datastore.GetOrderedKeyValueDB(d)
   902  	if err != nil {
   903  		return err
   904  	}
   905  	tk, err := NewTKey(keyStr)
   906  	if err != nil {
   907  		return err
   908  	}
   909  	return db.Delete(ctx, tk)
   910  }
   911  
   912  // process a range of keys from store using supplied function.
   913  func (d *Data) processStoreAllKeys(ctx storage.Context, f func(key string)) error {
   914  	minTKey := storage.MinTKey(keyAnnotation)
   915  	maxTKey := storage.MaxTKey(keyAnnotation)
   916  	return d.processStoreKeysInRange(ctx, minTKey, maxTKey, f)
   917  }
   918  
   919  // process a range of keys using supplied function.
   920  func (d *Data) processStoreKeysInRange(ctx storage.Context, minTKey, maxTKey storage.TKey, f func(key string)) error {
   921  	db, err := datastore.GetOrderedKeyValueDB(d)
   922  	if err != nil {
   923  		return err
   924  	}
   925  	tkeys, err := db.KeysInRange(ctx, minTKey, maxTKey)
   926  	if err != nil {
   927  		return err
   928  	}
   929  	for _, tkey := range tkeys {
   930  		key, err := DecodeTKey(tkey)
   931  		if err != nil {
   932  			return err
   933  		}
   934  		f(key)
   935  	}
   936  	return nil
   937  }
   938  
   939  // process a range of key-value pairs using supplied function.
   940  func (d *Data) processStoreRange(ctx storage.Context, f func(key string, value NeuronJSON)) error {
   941  	db, err := datastore.GetOrderedKeyValueDB(d)
   942  	if err != nil {
   943  		return err
   944  	}
   945  	first := storage.MinTKey(keyAnnotation)
   946  	last := storage.MaxTKey(keyAnnotation)
   947  	err = db.ProcessRange(ctx, first, last, &storage.ChunkOp{}, func(c *storage.Chunk) error {
   948  		if c == nil || c.TKeyValue == nil {
   949  			return nil
   950  		}
   951  		kv := c.TKeyValue
   952  		if kv.V == nil {
   953  			return nil
   954  		}
   955  		key, err := DecodeTKey(kv.K)
   956  		if err != nil {
   957  			return err
   958  		}
   959  		var value NeuronJSON
   960  		if err := value.UnmarshalJSON(kv.V); err != nil {
   961  			return err
   962  		}
   963  		f(key, value)
   964  		return nil
   965  	})
   966  	return err
   967  }
   968  
   969  // Initialize loads mutable properties of the neuronjson data instance,
   970  // which in this case is the in-memory neuron json map for the specified versions.
   971  func (d *Data) Initialize() {
   972  	leafUUID, leafV, err := datastore.GetBranchHead(d.RootUUID(), "master")
   973  	if err != nil {
   974  		dvid.Infof("Can't find the leaf node of the main/master branch... skipping neuronjson initialization\n")
   975  		leafUUID = dvid.NilUUID
   976  	}
   977  
   978  	d.metadata = make(map[Schema][]byte, 3)
   979  
   980  	if leafUUID != dvid.NilUUID {
   981  		// Load the metadata
   982  		ctx := datastore.NewVersionedCtx(d, leafV)
   983  		if sch, err := d.getJSONSchema(ctx); err == nil {
   984  			if sch != nil {
   985  				d.compiledSchema = sch
   986  			}
   987  		} else {
   988  			dvid.Criticalf("Can't load JSON schema for neuronjson %q: %v\n", d.DataName(), err)
   989  		}
   990  		if value, err := d.loadMetadata(ctx, NeuSchema); err == nil {
   991  			dvid.Infof("Metadata load of neutu/neu3 JSON schema for %s: %d bytes\n", leafUUID[:6], len(value))
   992  			if value != nil {
   993  				d.metadata[NeuSchema] = value
   994  			}
   995  		} else {
   996  			dvid.Criticalf("Can't load neutu/neu3 schema for neuronjson %q: %v\n", d.DataName(), err)
   997  		}
   998  		if value, err := d.loadMetadata(ctx, NeuSchemaBatch); err == nil {
   999  			dvid.Infof("Metadata load of neutu/neu3 JSON batch schema for %s: %d bytes\n", leafUUID[:6], len(value))
  1000  			if value != nil {
  1001  				d.metadata[NeuSchemaBatch] = value
  1002  			}
  1003  		} else {
  1004  			dvid.Criticalf("Can't load neutu/neu3 batch schema for neuronjson %q: %v\n", d.DataName(), err)
  1005  		}
  1006  	}
  1007  
  1008  	// Load the in-memory databases for specified versions or branch HEADs
  1009  	store, err := storage.GetAssignedStore(d)
  1010  	if err != nil {
  1011  		dvid.Criticalf("Can't get assigned store for neuronjson %q: %v\n", d.DataName(), err)
  1012  		return
  1013  	}
  1014  
  1015  	storeConfig := store.GetStoreConfig()
  1016  	uuidList := []string{}
  1017  	uuidListI, found := storeConfig.Get("inmemory")
  1018  	if found {
  1019  		dvid.Infof("Found configuration for additional in-memory UUIDs for neuronjson %q: %v\n",
  1020  			d.DataName(), uuidListI)
  1021  		switch v := uuidListI.(type) {
  1022  		case []string:
  1023  			uuidList = v
  1024  		case []interface{}:
  1025  			for i, version := range v {
  1026  				versionStr, ok := version.(string)
  1027  				if ok {
  1028  					uuidList = append(uuidList, versionStr)
  1029  				} else {
  1030  					dvid.Criticalf("can't parse neuronjson inmemory config for %dth version: %v\n", i, version)
  1031  				}
  1032  			}
  1033  		}
  1034  	}
  1035  	if err := d.initMemoryDB(uuidList); err != nil {
  1036  		dvid.Criticalf("Can't initialize in-memory databases for neuronjson %q: %v\n", d.DataName(), err)
  1037  	}
  1038  }
  1039  
  1040  // KeyExists returns true if a key is found.
  1041  func (d *Data) KeyExists(ctx storage.VersionedCtx, keyStr string) (found bool, err error) {
  1042  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1043  	if found {
  1044  		var bodyid uint64
  1045  		bodyid, err = strconv.ParseUint(keyStr, 10, 64)
  1046  		if err != nil {
  1047  			return false, err
  1048  		}
  1049  		mdb.mu.RLock()
  1050  		_, found = mdb.data[bodyid]
  1051  		mdb.mu.RUnlock()
  1052  		return found, nil
  1053  	}
  1054  	db, err := datastore.GetKeyValueDB(d)
  1055  	if err != nil {
  1056  		return false, err
  1057  	}
  1058  	tk, err := NewTKey(keyStr)
  1059  	if err != nil {
  1060  		return false, err
  1061  	}
  1062  	return db.Exists(ctx, tk)
  1063  }
  1064  
  1065  // GetKeysInRange returns all keys in the range [keyBeg, keyEnd].  Results on HEAD are ordered
  1066  // by integer key, while results on other branches are ordered lexicographically.
  1067  func (d *Data) GetKeysInRange(ctx storage.VersionedCtx, keyBeg, keyEnd string) (keys []string, err error) {
  1068  	var bodyidBeg, bodyidEnd uint64
  1069  	if bodyidBeg, err = parseKeyStr(keyBeg); err != nil {
  1070  		return
  1071  	}
  1072  	if bodyidEnd, err = parseKeyStr(keyEnd); err != nil {
  1073  		return
  1074  	}
  1075  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1076  	if found {
  1077  		mdb.mu.RLock()
  1078  		defer mdb.mu.RUnlock()
  1079  		begI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] >= bodyidBeg })
  1080  		endI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] > bodyidEnd })
  1081  		size := endI - begI
  1082  		if size <= 0 {
  1083  			keys = []string{}
  1084  			return
  1085  		}
  1086  		keys = make([]string, size)
  1087  		pos := 0
  1088  		for i := begI; i < endI; i++ {
  1089  			bodyid := mdb.ids[i]
  1090  			keys[pos] = strconv.FormatUint(bodyid, 10)
  1091  			pos++
  1092  		}
  1093  	} else {
  1094  		var begTKey, endTKey storage.TKey
  1095  		begTKey, err = NewTKey(keyBeg)
  1096  		if err != nil {
  1097  			return nil, err
  1098  		}
  1099  		endTKey, err = NewTKey(keyEnd)
  1100  		if err != nil {
  1101  			return nil, err
  1102  		}
  1103  		process_func := func(key string) {
  1104  			bodyid, err := parseKeyStr(key)
  1105  			if err == nil && bodyid >= bodyidBeg && bodyid <= bodyidEnd {
  1106  				keys = append(keys, key)
  1107  			}
  1108  		}
  1109  		err = d.processStoreKeysInRange(ctx, begTKey, endTKey, process_func)
  1110  	}
  1111  	return
  1112  }
  1113  
  1114  func (d *Data) GetAll(ctx storage.VersionedCtx, fieldMap map[string]struct{}, showFields Fields) (ListNeuronJSON, error) {
  1115  	showUser, showTime := showFields.Bools()
  1116  
  1117  	var all ListNeuronJSON
  1118  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1119  	if found {
  1120  		mdb.mu.RLock()
  1121  		for _, value := range mdb.data {
  1122  			out := selectFields(value, fieldMap, showUser, showTime)
  1123  			if len(out) > 1 {
  1124  				all = append(all, out)
  1125  			}
  1126  		}
  1127  		mdb.mu.RUnlock()
  1128  	} else {
  1129  		process_func := func(key string, value NeuronJSON) {
  1130  			out := selectFields(value, fieldMap, showUser, showTime)
  1131  			if len(out) > 1 {
  1132  				all = append(all, out)
  1133  			}
  1134  		}
  1135  		if err := d.processStoreRange(ctx, process_func); err != nil {
  1136  			return nil, err
  1137  		}
  1138  	}
  1139  	return all, nil
  1140  }
  1141  
  1142  func (d *Data) GetKeys(ctx storage.VersionedCtx) (out []string, err error) {
  1143  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1144  	if found {
  1145  		mdb.mu.RLock()
  1146  		out = make([]string, len(mdb.ids))
  1147  		for i, bodyid := range mdb.ids {
  1148  			out[i] = strconv.FormatUint(bodyid, 10)
  1149  		}
  1150  		mdb.mu.RUnlock()
  1151  	} else {
  1152  		process_func := func(key string) {
  1153  			out = append(out, key)
  1154  		}
  1155  		if err := d.processStoreAllKeys(ctx, process_func); err != nil {
  1156  			return nil, err
  1157  		}
  1158  	}
  1159  	return out, nil
  1160  }
  1161  
  1162  func (d *Data) GetFields(ctx storage.VersionedCtx) ([]string, error) {
  1163  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1164  	if !found {
  1165  		return nil, fmt.Errorf("unable to get fields because no in-memory db for neuronjson %q, version %d", d.DataName(), ctx.VersionID())
  1166  	}
  1167  	mdb.mu.RLock()
  1168  	fields := make([]string, len(mdb.fields))
  1169  	i := 0
  1170  	for field := range mdb.fields {
  1171  		fields[i] = field
  1172  		i++
  1173  	}
  1174  	mdb.mu.RUnlock()
  1175  	return fields, nil
  1176  }
  1177  
  1178  // GetData gets a byte value using a key
  1179  func (d *Data) GetData(ctx storage.VersionedCtx, keyStr string, fieldMap map[string]struct{}, showFields Fields) ([]byte, bool, error) {
  1180  	// Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances.
  1181  	switch keyStr {
  1182  	case NeuSchema.String():
  1183  		data, err := d.getMetadata(ctx, NeuSchema)
  1184  		if err != nil {
  1185  			return nil, false, fmt.Errorf("unable to retrieve neutu/neu3 JSON schema: %v", err)
  1186  		}
  1187  		if data != nil {
  1188  			return data, true, nil
  1189  		}
  1190  		return nil, false, nil
  1191  	case NeuSchemaBatch.String():
  1192  		data, err := d.getMetadata(ctx, NeuSchemaBatch)
  1193  		if err != nil {
  1194  			return nil, false, fmt.Errorf("unable to retrieve neutu/neu3 JSON batch schema: %v", err)
  1195  		}
  1196  		if data != nil {
  1197  			return data, true, nil
  1198  		}
  1199  		return nil, false, nil
  1200  	}
  1201  	bodyid, err := strconv.ParseUint(keyStr, 10, 64)
  1202  	if err != nil {
  1203  		return nil, false, err
  1204  	}
  1205  	var value map[string]interface{}
  1206  	var found bool
  1207  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1208  	if found {
  1209  		mdb.mu.RLock()
  1210  		value, found = mdb.data[bodyid]
  1211  		mdb.mu.RUnlock()
  1212  		if !found {
  1213  			return nil, false, nil
  1214  		}
  1215  	} else {
  1216  		value, found, err = d.getStoreData(ctx, keyStr)
  1217  		if !found || err != nil {
  1218  			return nil, false, err
  1219  		}
  1220  	}
  1221  	showUser, showTime := showFields.Bools()
  1222  	out := selectFields(value, fieldMap, showUser, showTime)
  1223  	data, err := json.Marshal(out)
  1224  	return data, true, err
  1225  }
  1226  
  1227  // update _user and _time fields for any fields newly set or modified.
  1228  func updateJSON(origData, newData NeuronJSON, user string, conditionals []string, replace bool) {
  1229  	// determine if any fields are being set for the first time or modified
  1230  	newlySet := make(map[string]struct{}, len(newData))  // fields that aren't same as old data
  1231  	newFields := make(map[string]struct{}, len(newData)) // fields that are not _user or _time
  1232  	if origData == nil {
  1233  		for field := range newData {
  1234  			newlySet[field] = struct{}{}
  1235  		}
  1236  	} else {
  1237  		for field, value := range newData {
  1238  			if origValue, found := origData[field]; !found || !reflect.DeepEqual(value, origValue) {
  1239  				newlySet[field] = struct{}{}
  1240  			}
  1241  			if !strings.HasSuffix(field, "_user") && !strings.HasSuffix(field, "_time") {
  1242  				newFields[field] = struct{}{}
  1243  			}
  1244  		}
  1245  
  1246  		// carry forward any fields not being modified if replace option is not set
  1247  		if !replace {
  1248  			protectedFields := make(map[string]struct{}, len(conditionals))
  1249  			for _, field := range conditionals {
  1250  				protectedFields[field] = struct{}{}
  1251  			}
  1252  			for field, origValue := range origData {
  1253  				if _, found := newData[field]; !found {
  1254  					newData[field] = origValue
  1255  					continue
  1256  				}
  1257  				if _, found := protectedFields[field]; found {
  1258  					newData[field] = origValue
  1259  					delete(newlySet, field)
  1260  				}
  1261  			}
  1262  		}
  1263  	}
  1264  
  1265  	// add _user and _time fields for newly set and not prevented via conditionals
  1266  	t := time.Now()
  1267  	timeStr := t.Format(time.RFC3339)
  1268  	for field := range newlySet {
  1269  		if field == "bodyid" || field == "user" {
  1270  			continue // these fields shouldn't have _user or _time fields added
  1271  		}
  1272  		if strings.HasSuffix(field, "_time") || strings.HasSuffix(field, "_user") {
  1273  			continue // we will handle this with main field
  1274  		}
  1275  		if _, foundUser := newlySet[field+"_user"]; !foundUser && user != "" {
  1276  			newData[field+"_user"] = user
  1277  		}
  1278  		if _, foundTime := newlySet[field+"_time"]; !foundTime {
  1279  			newData[field+"_time"] = timeStr
  1280  		}
  1281  	}
  1282  	if replace { // keep _user and _time fields for unchanged fields
  1283  		for field := range newFields {
  1284  			if _, foundUser := newData[field+"_user"]; !foundUser {
  1285  				oldUser, found := origData[field+"_user"]
  1286  				if !found {
  1287  					oldUser = user
  1288  				}
  1289  				newData[field+"_user"] = oldUser
  1290  			}
  1291  			if _, foundTime := newData[field+"_time"]; !foundTime {
  1292  				oldTime, found := origData[field+"_time"]
  1293  				if !found {
  1294  					oldTime = timeStr
  1295  				}
  1296  				newData[field+"_time"] = oldTime
  1297  			}
  1298  		}
  1299  	}
  1300  }
  1301  
  1302  func (d *Data) storeAndUpdate(ctx *datastore.VersionedCtx, keyStr string, newData NeuronJSON, conditionals []string, replace bool) error {
  1303  	bodyid, err := strconv.ParseUint(keyStr, 10, 64)
  1304  	if err != nil {
  1305  		return err
  1306  	}
  1307  
  1308  	// get original data so we can handle default update and tell which values change for _user/_time fields.
  1309  	origData, found, err := d.getStoreData(ctx, keyStr)
  1310  	if err != nil {
  1311  		return err
  1312  	}
  1313  	if !found {
  1314  		origData = nil
  1315  	}
  1316  	updateJSON(origData, newData, ctx.User, conditionals, replace)
  1317  	dvid.Infof("neuronjson %s put by user %q, conditionals %v, replace %t:\nOrig: %v\n New: %v\n",
  1318  		d.DataName(), ctx.User, conditionals, replace, origData, newData)
  1319  
  1320  	// write result
  1321  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1322  	if found {
  1323  		mdb.mu.Lock()
  1324  		mdb.data[bodyid] = newData
  1325  		for field := range newData {
  1326  			mdb.fields[field] = struct{}{}
  1327  		}
  1328  		mdb.addBodyID(bodyid)
  1329  		mdb.mu.Unlock()
  1330  	}
  1331  	return d.putStoreData(ctx, keyStr, newData)
  1332  }
  1333  
  1334  // PutData puts a valid JSON []byte into a neuron key at a given uuid.
  1335  // If replace is true, will use given value instead of updating fields that were given.
  1336  // If field values are given but do not change, the _user and _time fields will not be updated.
  1337  func (d *Data) PutData(ctx *datastore.VersionedCtx, keyStr string, value []byte, conditionals []string, replace bool) error {
  1338  	// Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances.
  1339  	switch keyStr {
  1340  	case "0":
  1341  		return fmt.Errorf("body id 0 is reserved and so cannot be stored")
  1342  	case NeuSchema.String():
  1343  		if err := d.putMetadata(ctx, value, NeuSchema); err != nil {
  1344  			return fmt.Errorf("unable to handle POST neutu/neu3 schema metadata: %v", err)
  1345  		}
  1346  		return nil
  1347  	case NeuSchemaBatch.String():
  1348  		if err := d.putMetadata(ctx, value, NeuSchemaBatch); err != nil {
  1349  			return fmt.Errorf("unable to handle POST neutu/neu3 batch schema metadata: %v", err)
  1350  		}
  1351  		return nil
  1352  	}
  1353  
  1354  	// validate if we have a JSON schema
  1355  	if sch, err := d.getJSONSchema(ctx); err == nil {
  1356  		var v interface{}
  1357  		if err = json.Unmarshal(value, &v); err != nil {
  1358  			return err
  1359  		}
  1360  		for err = sch.Validate(v); err != nil; {
  1361  			if verr, ok := err.(*jsonschema.ValidationError); ok {
  1362  				if !strings.HasSuffix(verr.Error(), "expected integer, but got string") {
  1363  					return err
  1364  				}
  1365  				// Try to convert string to integer for fields that need conversion.
  1366  				var field string
  1367  				if _, scanerr := fmt.Sscanf(err.Error(), `jsonschema: %s does`, &field); scanerr != nil {
  1368  					return err
  1369  				}
  1370  				field = strings.Trim(field, `'/`)
  1371  				dvid.Infof("Converting string to integer for field %q\n", field)
  1372  				var newData NeuronJSON
  1373  				if err := json.Unmarshal(value, &newData); err != nil {
  1374  					return err
  1375  				}
  1376  				if newData[field], err = strconv.ParseInt(newData[field].(string), 10, 64); err != nil {
  1377  					return err
  1378  				}
  1379  				if value, err = json.Marshal(newData); err != nil {
  1380  					return err
  1381  				}
  1382  			} else {
  1383  				return err
  1384  			}
  1385  		}
  1386  	} else {
  1387  		dvid.Infof("Skipping validation of POST %q neuron annotation: %v\n", d.DataName(), err)
  1388  	}
  1389  
  1390  	var newData NeuronJSON
  1391  	if err := json.Unmarshal(value, &newData); err != nil {
  1392  		return err
  1393  	}
  1394  	return d.storeAndUpdate(ctx, keyStr, newData, conditionals, replace)
  1395  }
  1396  
  1397  // DeleteData deletes a key-value pair
  1398  func (d *Data) DeleteData(ctx storage.VersionedCtx, keyStr string) error {
  1399  	// Allow "schema" and "schema_batch" on /key endpoint for backwards compatibility with DVID keyvalue instances.
  1400  	switch keyStr {
  1401  	case NeuSchema.String():
  1402  		if err := d.deleteMetadata(ctx, NeuSchema); err != nil {
  1403  			return fmt.Errorf("unable to handle DELETE neutu/neu3 schema metadata: %v", err)
  1404  		}
  1405  		return nil
  1406  	case NeuSchemaBatch.String():
  1407  		if err := d.deleteMetadata(ctx, NeuSchemaBatch); err != nil {
  1408  			return fmt.Errorf("unable to handle DELETE neutu/neu3 batch schema metadata: %v", err)
  1409  		}
  1410  		return nil
  1411  	}
  1412  
  1413  	bodyid, err := strconv.ParseUint(keyStr, 10, 64)
  1414  	if err != nil {
  1415  		return err
  1416  	}
  1417  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1418  	if found {
  1419  		mdb.mu.Lock()
  1420  		_, found := mdb.data[bodyid]
  1421  		if found {
  1422  			delete(mdb.data, bodyid)
  1423  			mdb.deleteBodyID(bodyid)
  1424  		}
  1425  		mdb.mu.Unlock()
  1426  	}
  1427  	return d.deleteStoreData(ctx, keyStr)
  1428  }
  1429  
  1430  // ----- Support functions for endpoint handlers -----
  1431  
  1432  type writeParams struct {
  1433  	w          http.ResponseWriter
  1434  	ch         chan writeData
  1435  	fieldMap   map[string]struct{}
  1436  	showFields Fields
  1437  	checkVal   bool
  1438  }
  1439  
  1440  type writeData struct {
  1441  	bodyid   uint64
  1442  	jsonData NeuronJSON
  1443  }
  1444  
  1445  func selectData(data writeData, params *writeParams) (key string, jsonBytes []byte, err error) {
  1446  	showUser, showTime := params.showFields.Bools()
  1447  	out := selectFields(data.jsonData, params.fieldMap, showUser, showTime)
  1448  	key = strconv.FormatUint(data.bodyid, 10)
  1449  	jsonBytes, err = json.Marshal(out)
  1450  	return
  1451  }
  1452  
  1453  func streamJSONtar(params *writeParams) (numKeys int, err error) {
  1454  	params.w.Header().Set("Content-type", "application/tar")
  1455  
  1456  	tw := tar.NewWriter(params.w)
  1457  
  1458  	var key string
  1459  	var jsonBytes []byte
  1460  	for data := range params.ch {
  1461  		key, jsonBytes, err = selectData(data, params)
  1462  		if err != nil {
  1463  			return 0, err
  1464  		}
  1465  
  1466  		hdr := &tar.Header{
  1467  			Name: key,
  1468  			Size: int64(len(jsonBytes)),
  1469  			Mode: 0755,
  1470  		}
  1471  		if err = tw.WriteHeader(hdr); err != nil {
  1472  			return
  1473  		}
  1474  		if _, err = tw.Write(jsonBytes); err != nil {
  1475  			return
  1476  		}
  1477  		numKeys++
  1478  	}
  1479  	tw.Close()
  1480  	return numKeys, nil
  1481  }
  1482  
  1483  func streamJSON(params *writeParams) (numKeys int, err error) {
  1484  	params.w.Header().Set("Content-type", "application/json")
  1485  	if _, err = params.w.Write([]byte("{")); err != nil {
  1486  		return
  1487  	}
  1488  
  1489  	var wroteVal bool
  1490  	var key string
  1491  	var jsonBytes []byte
  1492  	for data := range params.ch {
  1493  		key, jsonBytes, err = selectData(data, params)
  1494  		if err != nil {
  1495  			return 0, err
  1496  		}
  1497  
  1498  		if wroteVal {
  1499  			if _, err = params.w.Write([]byte(",")); err != nil {
  1500  				return
  1501  			}
  1502  		}
  1503  		if len(jsonBytes) == 0 {
  1504  			jsonBytes = []byte("{}")
  1505  		} else if params.checkVal && !json.Valid(jsonBytes) {
  1506  			return 0, fmt.Errorf("bad JSON for key %q", key)
  1507  		}
  1508  		out := fmt.Sprintf(`"%s":`, key)
  1509  		if _, err = params.w.Write([]byte(out)); err != nil {
  1510  			return
  1511  		}
  1512  		if _, err = params.w.Write(jsonBytes); err != nil {
  1513  			return
  1514  		}
  1515  		wroteVal = true
  1516  	}
  1517  	if _, err = params.w.Write([]byte("}")); err != nil {
  1518  		return
  1519  	}
  1520  	return numKeys, nil
  1521  }
  1522  
  1523  func streamProtobuf(params *writeParams) (numKeys int, err error) {
  1524  	var kvs proto.KeyValues
  1525  
  1526  	var key string
  1527  	var jsonBytes []byte
  1528  	for data := range params.ch {
  1529  		key, jsonBytes, err = selectData(data, params)
  1530  		if err != nil {
  1531  			return 0, err
  1532  		}
  1533  
  1534  		kv := &proto.KeyValue{
  1535  			Key:   key,
  1536  			Value: jsonBytes,
  1537  		}
  1538  		kvs.Kvs = append(kvs.Kvs, kv)
  1539  	}
  1540  	numKeys = len(kvs.Kvs)
  1541  	var serialization []byte
  1542  	if serialization, err = pb.Marshal(&kvs); err != nil {
  1543  		return
  1544  	}
  1545  	params.w.Header().Set("Content-type", "application/octet-stream")
  1546  	if _, err = params.w.Write(serialization); err != nil {
  1547  		return
  1548  	}
  1549  
  1550  	return numKeys, nil
  1551  }
  1552  
  1553  // writes JSON values in given format for a range of keys with given beginning & end.
  1554  func (d *Data) sendJSONValuesInRange(ctx storage.VersionedCtx, w http.ResponseWriter,
  1555  	r *http.Request, keyBeg, keyEnd string, fieldMap map[string]struct{}, showFields Fields) (numKeys int, err error) {
  1556  
  1557  	if len(keyBeg) == 0 || len(keyEnd) == 0 {
  1558  		return 0, fmt.Errorf("must specify non-empty beginning and ending key")
  1559  	}
  1560  	tarOut := (r.URL.Query().Get("jsontar") == "true") || (r.URL.Query().Get("tar") == "true")
  1561  	jsonOut := r.URL.Query().Get("json") == "true"
  1562  	if tarOut && jsonOut {
  1563  		err = fmt.Errorf("can only specify tar or json output, not both")
  1564  		return
  1565  	}
  1566  
  1567  	// Check range keys are valid for either in-memory or on-disk store.
  1568  	bodyidBeg, err := parseKeyStr(keyBeg)
  1569  	if err != nil {
  1570  		return 0, err
  1571  	}
  1572  	bodyidEnd, err := parseKeyStr(keyEnd)
  1573  	if err != nil {
  1574  		return 0, err
  1575  	}
  1576  
  1577  	first, err := NewTKey(keyBeg)
  1578  	if err != nil {
  1579  		return 0, err
  1580  	}
  1581  	last, err := NewTKey(keyEnd)
  1582  	if err != nil {
  1583  		return 0, err
  1584  	}
  1585  	db, err := datastore.GetOrderedKeyValueDB(d)
  1586  	if err != nil {
  1587  		return 0, err
  1588  	}
  1589  
  1590  	// Create goroutine to read NeuronJSONs from either in-memory or on-disk store.
  1591  	writeCh := make(chan writeData, 1000)
  1592  	params := writeParams{
  1593  		w:          w,
  1594  		ch:         writeCh,
  1595  		fieldMap:   fieldMap,
  1596  		showFields: showFields,
  1597  		checkVal:   r.URL.Query().Get("check") == "true",
  1598  	}
  1599  
  1600  	mdb, found := d.getMemDBbyVersion(ctx.VersionID())
  1601  	if found {
  1602  		go func() {
  1603  			mdb.mu.RLock()
  1604  			begI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] >= bodyidBeg })
  1605  			endI := sort.Search(len(mdb.ids), func(i int) bool { return mdb.ids[i] > bodyidEnd })
  1606  			mdb.mu.RUnlock()
  1607  
  1608  			for i := begI; i < endI; i++ {
  1609  				mdb.mu.RLock()
  1610  				bodyid := mdb.ids[i]
  1611  				jsonData, ok := mdb.data[bodyid]
  1612  				mdb.mu.RUnlock()
  1613  				if !ok {
  1614  					dvid.Errorf("inconsistent neuronjson DB: bodyid %d at pos %d is not in db cache... skipping", bodyid, i)
  1615  					continue
  1616  				}
  1617  				writeCh <- writeData{bodyid, jsonData}
  1618  			}
  1619  			close(writeCh)
  1620  		}()
  1621  	} else { // Handle on-disk store.
  1622  		go func() {
  1623  			err = db.ProcessRange(ctx, first, last, &storage.ChunkOp{}, func(c *storage.Chunk) error {
  1624  				if c == nil || c.TKeyValue == nil {
  1625  					return nil
  1626  				}
  1627  				kv := c.TKeyValue
  1628  				if kv.V == nil {
  1629  					return nil
  1630  				}
  1631  				key, err := DecodeTKey(kv.K)
  1632  				if err != nil {
  1633  					return err
  1634  				}
  1635  				var jsonData NeuronJSON
  1636  				if err := json.Unmarshal(kv.V, &jsonData); err != nil {
  1637  					return err
  1638  				}
  1639  				out := removeReservedFields(jsonData, showFields)
  1640  				bodyid, err := parseKeyStr(key)
  1641  				if err != nil {
  1642  					return err
  1643  				}
  1644  				writeCh <- writeData{bodyid, out}
  1645  				return nil
  1646  			})
  1647  			close(writeCh)
  1648  		}()
  1649  	}
  1650  
  1651  	// Output requested format.
  1652  	switch {
  1653  	case tarOut:
  1654  		numKeys, err = streamJSONtar(&params)
  1655  	case jsonOut:
  1656  		numKeys, err = streamJSON(&params)
  1657  	default:
  1658  		numKeys, err = streamProtobuf(&params)
  1659  	}
  1660  
  1661  	return
  1662  }
  1663  
  1664  // writes JSON data for the given keys
  1665  func (d *Data) sendJSONKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys []string, checkVal bool,
  1666  	fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) {
  1667  
  1668  	w.Header().Set("Content-type", "application/json")
  1669  	if writtenBytes, err = w.Write([]byte("{")); err != nil {
  1670  		return
  1671  	}
  1672  	var n int
  1673  	var foundKeys bool
  1674  	for _, key := range keys {
  1675  		var val []byte
  1676  		var found bool
  1677  		if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil {
  1678  			return
  1679  		}
  1680  		if !found {
  1681  			continue
  1682  		} else if foundKeys {
  1683  			if n, err = w.Write([]byte(",")); err != nil {
  1684  				return
  1685  			}
  1686  			writtenBytes += n
  1687  		}
  1688  		if len(val) == 0 {
  1689  			val = []byte("{}")
  1690  		} else if checkVal && !json.Valid(val) {
  1691  			err = fmt.Errorf("bad JSON for key %q", key)
  1692  			return
  1693  		}
  1694  		out := fmt.Sprintf(`"%s":`, key)
  1695  		if n, err = w.Write([]byte(out)); err != nil {
  1696  			return
  1697  		}
  1698  		writtenBytes += n
  1699  		if n, err = w.Write(val); err != nil {
  1700  			return
  1701  		}
  1702  		writtenBytes += n
  1703  		foundKeys = true
  1704  	}
  1705  	_, err = w.Write([]byte("}"))
  1706  	return
  1707  }
  1708  
  1709  // writes tarred JSON data for the given keys
  1710  func (d *Data) sendTarKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys []string,
  1711  	fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) {
  1712  
  1713  	var n int
  1714  	w.Header().Set("Content-type", "application/tar")
  1715  	tw := tar.NewWriter(w)
  1716  	for _, key := range keys {
  1717  		var val []byte
  1718  		var found bool
  1719  		if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil {
  1720  			return
  1721  		}
  1722  		if !found {
  1723  			val = nil
  1724  		}
  1725  		hdr := &tar.Header{
  1726  			Name: key,
  1727  			Size: int64(len(val)),
  1728  			Mode: 0755,
  1729  		}
  1730  		if err = tw.WriteHeader(hdr); err != nil {
  1731  			return
  1732  		}
  1733  		if n, err = tw.Write(val); err != nil {
  1734  			return
  1735  		}
  1736  		writtenBytes += n
  1737  	}
  1738  	tw.Close()
  1739  	return
  1740  }
  1741  
  1742  // writes protobuf-encoded keyvalues for the given keys
  1743  func (d *Data) sendProtobufKV(ctx storage.VersionedCtx, w http.ResponseWriter, keys *proto.Keys,
  1744  	fieldMap map[string]struct{}, showFields Fields) (writtenBytes int, err error) {
  1745  	var kvs proto.KeyValues
  1746  	kvs.Kvs = make([]*proto.KeyValue, len(keys.Keys))
  1747  	for i, key := range keys.Keys {
  1748  		var val []byte
  1749  		var found bool
  1750  		if val, found, err = d.GetData(ctx, key, fieldMap, showFields); err != nil {
  1751  			return
  1752  		}
  1753  		if !found {
  1754  			val = nil
  1755  		}
  1756  		kvs.Kvs[i] = &proto.KeyValue{
  1757  			Key:   key,
  1758  			Value: val,
  1759  		}
  1760  	}
  1761  	var serialization []byte
  1762  	if serialization, err = pb.Marshal(&kvs); err != nil {
  1763  		return
  1764  	}
  1765  	w.Header().Set("Content-type", "application/octet-stream")
  1766  	if writtenBytes, err = w.Write(serialization); err != nil {
  1767  		return
  1768  	}
  1769  	if writtenBytes != len(serialization) {
  1770  		err = fmt.Errorf("unable to write all %d bytes of serialized keyvalues: only %d bytes written", len(serialization), writtenBytes)
  1771  	}
  1772  	return
  1773  }
  1774  
  1775  // writes JSON values in given format for keys sent in the request body.
  1776  // Uses random reads since we assume keys aren't ordered.
  1777  func (d *Data) handleKeyValues(ctx storage.VersionedCtx, w http.ResponseWriter, r *http.Request,
  1778  	uuid dvid.UUID, fieldMap map[string]struct{}, showFields Fields) (numKeys, writtenBytes int, err error) {
  1779  
  1780  	tarOut := (r.URL.Query().Get("jsontar") == "true") || (r.URL.Query().Get("tar") == "true")
  1781  	jsonOut := r.URL.Query().Get("json") == "true"
  1782  	checkVal := r.URL.Query().Get("check") == "true"
  1783  
  1784  	if tarOut && jsonOut {
  1785  		err = fmt.Errorf("can only specify tar or json output, not both")
  1786  		return
  1787  	}
  1788  	var data []byte
  1789  	data, err = ioutil.ReadAll(r.Body)
  1790  	if err != nil {
  1791  		return
  1792  	}
  1793  	switch {
  1794  	case tarOut:
  1795  		var keys []string
  1796  		if err = json.Unmarshal(data, &keys); err != nil {
  1797  			return
  1798  		}
  1799  		numKeys = len(keys)
  1800  		writtenBytes, err = d.sendTarKV(ctx, w, keys, fieldMap, showFields)
  1801  	case jsonOut:
  1802  		var keys []string
  1803  		var keysInt []uint64
  1804  		if err = json.Unmarshal(data, &keysInt); err == nil {
  1805  			// convert to string keys for compatibility with keyvalue type & downstream code
  1806  			keys = make([]string, len(keysInt))
  1807  			for i, n := range keysInt {
  1808  				keys[i] = strconv.FormatUint(n, 10)
  1809  			}
  1810  		} else if err = json.Unmarshal(data, &keys); err != nil {
  1811  			return
  1812  		}
  1813  		numKeys = len(keys)
  1814  		writtenBytes, err = d.sendJSONKV(ctx, w, keys, checkVal, fieldMap, showFields)
  1815  	default:
  1816  		var keys proto.Keys
  1817  		if err = pb.Unmarshal(data, &keys); err != nil {
  1818  			return
  1819  		}
  1820  		numKeys = len(keys.Keys)
  1821  		writtenBytes, err = d.sendProtobufKV(ctx, w, &keys, fieldMap, showFields)
  1822  	}
  1823  	return
  1824  }
  1825  
  1826  func (d *Data) handleIngest(ctx *datastore.VersionedCtx, r *http.Request, uuid dvid.UUID) error {
  1827  	cond_fields := r.URL.Query().Get("conditionals")
  1828  	conditionals := strings.Split(cond_fields, ",")
  1829  	replace := r.URL.Query().Get("replace") == "true"
  1830  	data, err := ioutil.ReadAll(r.Body)
  1831  	if err != nil {
  1832  		return err
  1833  	}
  1834  	var kvs proto.KeyValues
  1835  	if err := pb.Unmarshal(data, &kvs); err != nil {
  1836  		return err
  1837  	}
  1838  	for _, kv := range kvs.Kvs {
  1839  		err = d.PutData(ctx, kv.Key, kv.Value, conditionals, replace)
  1840  		if err != nil {
  1841  			return err
  1842  		}
  1843  
  1844  		msginfo := map[string]interface{}{
  1845  			"Action":    "ingestneuronjson",
  1846  			"Key":       kv.Key,
  1847  			"Bytes":     len(kv.Value),
  1848  			"UUID":      string(uuid),
  1849  			"Timestamp": time.Now().String(),
  1850  		}
  1851  		jsonmsg, _ := json.Marshal(msginfo)
  1852  		if err = d.PublishKafkaMsg(jsonmsg); err != nil {
  1853  			dvid.Errorf("Error on sending neuronjson POST op to kafka: %v\n", err)
  1854  		}
  1855  	}
  1856  	return nil
  1857  }
  1858  
  1859  // --- DataService interface ---
  1860  
  1861  func (d *Data) Help() string {
  1862  	return fmt.Sprint(helpMessage)
  1863  }
  1864  
  1865  // DoRPC acts as a switchboard for RPC commands.
  1866  func (d *Data) DoRPC(request datastore.Request, reply *datastore.Response) error {
  1867  	switch request.TypeCommand() {
  1868  	case "put":
  1869  		return d.putCmd(request, reply)
  1870  	case "import-kv":
  1871  		return d.importKV(request, reply)
  1872  	case "version-changes":
  1873  		return d.versionChanges(request, reply)
  1874  	default:
  1875  		return fmt.Errorf("unknown command.  Data %q [%s] does not support %q command",
  1876  			d.DataName(), d.TypeName(), request.TypeCommand())
  1877  	}
  1878  }
  1879  
  1880  func (d *Data) handleSchema(ctx storage.VersionedCtx, w http.ResponseWriter, r *http.Request, uuid dvid.UUID, action string, meta Schema) error {
  1881  	switch action {
  1882  	case "head":
  1883  		found, err := d.metadataExists(ctx, meta)
  1884  		if err != nil {
  1885  			return err
  1886  		}
  1887  		if found {
  1888  			w.WriteHeader(http.StatusOK)
  1889  		} else {
  1890  			w.WriteHeader(http.StatusNotFound)
  1891  		}
  1892  
  1893  	case "get":
  1894  		value, err := d.getMetadata(ctx, meta)
  1895  		if err != nil {
  1896  			return err
  1897  		} else if value == nil {
  1898  			w.WriteHeader(http.StatusNotFound)
  1899  			return nil
  1900  		}
  1901  		if _, err := w.Write(value); err != nil {
  1902  			return err
  1903  		}
  1904  		w.Header().Set("Content-Type", "application/json")
  1905  
  1906  	case "delete":
  1907  		if err := d.deleteMetadata(ctx, meta); err != nil {
  1908  			return err
  1909  		}
  1910  
  1911  	case "post":
  1912  		data, err := ioutil.ReadAll(r.Body)
  1913  		if err != nil {
  1914  			return err
  1915  		}
  1916  		if err := d.putMetadata(ctx, data, meta); err != nil {
  1917  			return err
  1918  		}
  1919  
  1920  	default:
  1921  		return fmt.Errorf("key endpoint does not support %q HTTP verb", action)
  1922  	}
  1923  	return nil
  1924  }
  1925  
  1926  // ServeHTTP handles all incoming HTTP requests for this data.
  1927  func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) (activity map[string]interface{}) {
  1928  	timedLog := dvid.NewTimeLog()
  1929  
  1930  	// Break URL request into arguments
  1931  	url := r.URL.Path[len(server.WebAPIPath):]
  1932  	parts := strings.Split(url, "/")
  1933  	if len(parts[len(parts)-1]) == 0 {
  1934  		parts = parts[:len(parts)-1]
  1935  	}
  1936  
  1937  	if len(parts) < 4 {
  1938  		server.BadRequest(w, r, "incomplete API specification")
  1939  		return
  1940  	}
  1941  
  1942  	var comment string
  1943  	action := strings.ToLower(r.Method)
  1944  
  1945  	switch parts[3] {
  1946  	case "help":
  1947  		w.Header().Set("Content-Type", "text/plain")
  1948  		fmt.Fprintln(w, d.Help())
  1949  		return
  1950  
  1951  	case "info":
  1952  		jsonStr, err := d.JSONString()
  1953  		if err != nil {
  1954  			server.BadRequest(w, r, err)
  1955  			return
  1956  		}
  1957  		w.Header().Set("Content-Type", "application/json")
  1958  		fmt.Fprint(w, jsonStr)
  1959  		return
  1960  
  1961  	case "tags":
  1962  		if action == "post" {
  1963  			replace := r.URL.Query().Get("replace") == "true"
  1964  			if err := datastore.SetTagsByJSON(d, uuid, replace, r.Body); err != nil {
  1965  				server.BadRequest(w, r, err)
  1966  				return
  1967  			}
  1968  		} else {
  1969  			jsonBytes, err := d.MarshalJSONTags()
  1970  			if err != nil {
  1971  				server.BadRequest(w, r, err)
  1972  				return
  1973  			}
  1974  			w.Header().Set("Content-Type", "application/json")
  1975  			fmt.Fprint(w, string(jsonBytes))
  1976  		}
  1977  
  1978  	case JSONSchema.String():
  1979  		if err := d.handleSchema(ctx, w, r, uuid, action, JSONSchema); err != nil {
  1980  			server.BadRequest(w, r, err)
  1981  			return
  1982  		}
  1983  
  1984  	case NeuSchema.String():
  1985  		if err := d.handleSchema(ctx, w, r, uuid, action, NeuSchema); err != nil {
  1986  			server.BadRequest(w, r, err)
  1987  			return
  1988  		}
  1989  
  1990  	case NeuSchemaBatch.String():
  1991  		if err := d.handleSchema(ctx, w, r, uuid, action, NeuSchemaBatch); err != nil {
  1992  			server.BadRequest(w, r, err)
  1993  			return
  1994  		}
  1995  
  1996  	case "all":
  1997  		kvList, err := d.GetAll(ctx, fieldMap(r), showFields(r))
  1998  		if err != nil {
  1999  			server.BadRequest(w, r, err)
  2000  			return
  2001  		}
  2002  		jsonBytes, err := json.Marshal(kvList)
  2003  		if err != nil {
  2004  			server.BadRequest(w, r, err)
  2005  			return
  2006  		}
  2007  		w.Header().Set("Content-Type", "application/json")
  2008  		fmt.Fprint(w, string(jsonBytes))
  2009  		comment = "HTTP GET all"
  2010  
  2011  	case "keys":
  2012  		keyList, err := d.GetKeys(ctx)
  2013  		if err != nil {
  2014  			server.BadRequest(w, r, err)
  2015  			return
  2016  		}
  2017  		jsonBytes, err := json.Marshal(keyList)
  2018  		if err != nil {
  2019  			server.BadRequest(w, r, err)
  2020  			return
  2021  		}
  2022  		w.Header().Set("Content-Type", "application/json")
  2023  		fmt.Fprint(w, string(jsonBytes))
  2024  		comment = "HTTP GET keys"
  2025  
  2026  	case "fields":
  2027  		fieldList, err := d.GetFields(ctx)
  2028  		if err != nil {
  2029  			server.BadRequest(w, r, err)
  2030  			return
  2031  		}
  2032  		jsonBytes, err := json.Marshal(fieldList)
  2033  		if err != nil {
  2034  			server.BadRequest(w, r, err)
  2035  			return
  2036  		}
  2037  		w.Header().Set("Content-Type", "application/json")
  2038  		fmt.Fprint(w, string(jsonBytes))
  2039  		comment = "HTTP GET fields"
  2040  
  2041  	case "query":
  2042  		if action != "post" && action != "get" {
  2043  			server.BadRequest(w, r, fmt.Errorf("only GET or POST methods allowed for /query endpoint"))
  2044  			return
  2045  		}
  2046  		onlyid := r.URL.Query().Get("onlyid") == "true"
  2047  		err := d.Query(ctx, w, uuid, onlyid, fieldMap(r), showFields(r), r.Body)
  2048  		if err != nil {
  2049  			server.BadRequest(w, r, err)
  2050  			return
  2051  		}
  2052  
  2053  	case "keyrange":
  2054  		if len(parts) < 6 {
  2055  			server.BadRequest(w, r, "expect beginning and end keys to follow 'keyrange' endpoint")
  2056  			return
  2057  		}
  2058  
  2059  		// Return JSON list of keys
  2060  		keyBeg := parts[4]
  2061  		keyEnd := parts[5]
  2062  		keyList, err := d.GetKeysInRange(ctx, keyBeg, keyEnd)
  2063  		if err != nil {
  2064  			server.BadRequest(w, r, err)
  2065  			return
  2066  		}
  2067  		jsonBytes, err := json.Marshal(keyList)
  2068  		if err != nil {
  2069  			server.BadRequest(w, r, err)
  2070  			return
  2071  		}
  2072  		w.Header().Set("Content-Type", "application/json")
  2073  		fmt.Fprint(w, string(jsonBytes))
  2074  		comment = fmt.Sprintf("HTTP GET keyrange [%q, %q]", keyBeg, keyEnd)
  2075  
  2076  	case "keyrangevalues":
  2077  		if len(parts) < 6 {
  2078  			server.BadRequest(w, r, "expect beginning and end keys to follow 'keyrangevalues' endpoint")
  2079  			return
  2080  		}
  2081  
  2082  		// Return JSON list of keys
  2083  		keyBeg := parts[4]
  2084  		keyEnd := parts[5]
  2085  		w.Header().Set("Content-Type", "application/json")
  2086  
  2087  		numKeys, err := d.sendJSONValuesInRange(ctx, w, r, keyBeg, keyEnd, fieldMap(r), showFields(r))
  2088  		if err != nil {
  2089  			server.BadRequest(w, r, err)
  2090  			return
  2091  		}
  2092  		comment = fmt.Sprintf("HTTP GET keyrangevalues sent %d values for [%q, %q]", numKeys, keyBeg, keyEnd)
  2093  
  2094  	case "keyvalues":
  2095  		switch action {
  2096  		case "get":
  2097  			numKeys, writtenBytes, err := d.handleKeyValues(ctx, w, r, uuid, fieldMap(r), showFields(r))
  2098  			if err != nil {
  2099  				server.BadRequest(w, r, "GET /keyvalues on %d keys, data %q: %v", numKeys, d.DataName(), err)
  2100  				return
  2101  			}
  2102  			comment = fmt.Sprintf("HTTP GET keyvalues on %d keys, %d bytes, data %q", numKeys, writtenBytes, d.DataName())
  2103  		case "post":
  2104  			if err := d.handleIngest(ctx, r, uuid); err != nil {
  2105  				server.BadRequest(w, r, err)
  2106  				return
  2107  			}
  2108  			comment = fmt.Sprintf("HTTP POST keyvalues on data %q", d.DataName())
  2109  		default:
  2110  			server.BadRequest(w, r, "key endpoint does not support %q HTTP verb", action)
  2111  			return
  2112  		}
  2113  
  2114  	case "key":
  2115  		if len(parts) < 5 {
  2116  			server.BadRequest(w, r, "expect key string to follow 'key' endpoint")
  2117  			return
  2118  		}
  2119  		keyStr := parts[4]
  2120  
  2121  		switch action {
  2122  		case "head":
  2123  			found, err := d.KeyExists(ctx, keyStr)
  2124  			if err != nil {
  2125  				server.BadRequest(w, r, err)
  2126  				return
  2127  			}
  2128  			if found {
  2129  				w.WriteHeader(http.StatusOK)
  2130  			} else {
  2131  				w.WriteHeader(http.StatusNotFound)
  2132  			}
  2133  			return
  2134  
  2135  		case "get":
  2136  			// Return value of single key
  2137  			value, found, err := d.GetData(ctx, keyStr, fieldMap(r), showFields(r))
  2138  			if err != nil {
  2139  				server.BadRequest(w, r, err)
  2140  				return
  2141  			}
  2142  			if !found {
  2143  				http.Error(w, fmt.Sprintf("Key %q not found", keyStr), http.StatusNotFound)
  2144  				return
  2145  			}
  2146  			if value != nil || len(value) > 0 {
  2147  				_, err = w.Write(value)
  2148  				if err != nil {
  2149  					server.BadRequest(w, r, err)
  2150  					return
  2151  				}
  2152  				w.Header().Set("Content-Type", "application/octet-stream")
  2153  			}
  2154  			comment = fmt.Sprintf("HTTP GET key %q of neuronjson %q: %d bytes (%s)", keyStr, d.DataName(), len(value), url)
  2155  
  2156  		case "delete":
  2157  			if err := d.DeleteData(ctx, keyStr); err != nil {
  2158  				server.BadRequest(w, r, err)
  2159  				return
  2160  			}
  2161  			comment = fmt.Sprintf("HTTP DELETE data with key %q of neuronjson %q (%s)", keyStr, d.DataName(), url)
  2162  
  2163  		case "post":
  2164  			data, err := ioutil.ReadAll(r.Body)
  2165  			if err != nil {
  2166  				server.BadRequest(w, r, err)
  2167  				return
  2168  			}
  2169  
  2170  			go func() {
  2171  				msginfo := map[string]interface{}{
  2172  					"Action":    "postneuronjson",
  2173  					"Key":       keyStr,
  2174  					"Bytes":     len(data),
  2175  					"UUID":      string(uuid),
  2176  					"Timestamp": time.Now().String(),
  2177  				}
  2178  				jsonmsg, _ := json.Marshal(msginfo)
  2179  				if err = d.PublishKafkaMsg(jsonmsg); err != nil {
  2180  					dvid.Errorf("Error on sending neuronjson POST op to kafka: %v\n", err)
  2181  				}
  2182  			}()
  2183  
  2184  			cond_fields := r.URL.Query().Get("conditionals")
  2185  			conditionals := strings.Split(cond_fields, ",")
  2186  			replace := r.URL.Query().Get("replace") == "true"
  2187  
  2188  			err = d.PutData(ctx, keyStr, data, conditionals, replace)
  2189  			if err != nil {
  2190  				server.BadRequest(w, r, err)
  2191  				return
  2192  			}
  2193  			comment = fmt.Sprintf("HTTP POST neuronjson '%s': %d bytes (%s)", d.DataName(), len(data), url)
  2194  		default:
  2195  			server.BadRequest(w, r, "key endpoint does not support %q HTTP verb", action)
  2196  			return
  2197  		}
  2198  
  2199  	default:
  2200  		server.BadAPIRequest(w, r, d)
  2201  		return
  2202  	}
  2203  
  2204  	timedLog.Infof(comment)
  2205  	return
  2206  }