github.com/janelia-flyem/dvid@v1.0.0/datatype/annotation/annotation_test.go (about)

     1  package annotation
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/binary"
     6  	"encoding/json"
     7  	"fmt"
     8  	"log"
     9  	"reflect"
    10  	"runtime"
    11  	"strings"
    12  	"sync"
    13  	"testing"
    14  
    15  	"github.com/janelia-flyem/dvid/datastore"
    16  	"github.com/janelia-flyem/dvid/dvid"
    17  	"github.com/janelia-flyem/dvid/server"
    18  )
    19  
    20  var (
    21  	syntype datastore.TypeService
    22  	testMu  sync.Mutex
    23  )
    24  
    25  // Sets package-level testRepo and TestVersionID
    26  func initTestRepo() (dvid.UUID, dvid.VersionID) {
    27  	testMu.Lock()
    28  	defer testMu.Unlock()
    29  	if syntype == nil {
    30  		var err error
    31  		syntype, err = datastore.TypeServiceByName(TypeName)
    32  		if err != nil {
    33  			log.Fatalf("Can't get synapse type: %s\n", err)
    34  		}
    35  	}
    36  	return datastore.NewTestRepo()
    37  }
    38  
    39  func TestSynapseRepoPersistence(t *testing.T) {
    40  	if err := server.OpenTest(); err != nil {
    41  		t.Fatalf("can't open test server: %v\n", err)
    42  	}
    43  	defer server.CloseTest()
    44  
    45  	uuid, _ := initTestRepo()
    46  
    47  	// Make labels and set various properties
    48  	config := dvid.NewConfig()
    49  	dataservice, err := datastore.NewData(uuid, syntype, "synapses", config)
    50  	if err != nil {
    51  		t.Errorf("Unable to create keyvalue instance: %v\n", err)
    52  	}
    53  	data, ok := dataservice.(*Data)
    54  	if !ok {
    55  		t.Errorf("Can't cast data service into synapse.Data\n")
    56  	}
    57  	oldData := *data
    58  
    59  	// Restart test datastore and see if datasets are still there.
    60  	if err = datastore.SaveDataByUUID(uuid, data); err != nil {
    61  		t.Fatalf("Unable to save repo during synapse persistence test: %v\n", err)
    62  	}
    63  	datastore.CloseReopenTest()
    64  
    65  	dataservice2, err := datastore.GetDataByUUIDName(uuid, "synapses")
    66  	if err != nil {
    67  		t.Fatalf("Can't get synapse instance from reloaded test db: %v\n", err)
    68  	}
    69  	data2, ok := dataservice2.(*Data)
    70  	if !ok {
    71  		t.Errorf("Returned new data instance 2 is not synapse.Data\n")
    72  	}
    73  	if !oldData.Equals(data2) {
    74  		t.Errorf("Expected %v, got %v\n", oldData, *data2)
    75  	}
    76  }
    77  
    78  var testData = Elements{
    79  	{
    80  		ElementNR{
    81  			Pos:  dvid.Point3d{15, 27, 35}, // Label 1
    82  			Kind: PreSyn,
    83  			Tags: []Tag{"Synapse1", "Zlt90"},
    84  			Prop: map[string]string{
    85  				"Im a T-Bar":         "yes",
    86  				"I'm not a PSD":      "sure",
    87  				"i'm really special": "",
    88  			},
    89  		},
    90  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
    91  	},
    92  	{
    93  		ElementNR{
    94  			Pos:  dvid.Point3d{20, 30, 40}, // Label 2, but can be split off
    95  			Kind: PostSyn,
    96  			Tags: []Tag{"Synapse1"},
    97  		},
    98  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
    99  	},
   100  	{
   101  		ElementNR{
   102  			Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   103  			Kind: PostSyn,
   104  			Tags: []Tag{"Synapse1", "Zlt90"},
   105  		},
   106  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   107  	},
   108  	{
   109  		ElementNR{
   110  			Pos:  dvid.Point3d{33, 30, 31},
   111  			Kind: PostSyn,
   112  			Tags: []Tag{"Synapse1", "Zlt90"},
   113  		},
   114  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   115  	},
   116  	{
   117  		ElementNR{
   118  			Pos:  dvid.Point3d{127, 63, 99}, // Label 3
   119  			Kind: PreSyn,
   120  			Tags: []Tag{"Synapse2"},
   121  			Prop: map[string]string{
   122  				"Im a T-Bar":             "no",
   123  				"I'm not a PSD":          "not really",
   124  				"i'm not really special": "at all",
   125  			},
   126  		},
   127  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   128  	},
   129  	{
   130  		ElementNR{
   131  			Pos:  dvid.Point3d{88, 47, 80}, // Label 4
   132  			Kind: PostSyn,
   133  			Tags: []Tag{"Synapse2"},
   134  		},
   135  		[]Relationship{{Rel: GroupedWith, To: dvid.Point3d{14, 25, 37}}, {Rel: PostSynTo, To: dvid.Point3d{127, 63, 99}}, {Rel: GroupedWith, To: dvid.Point3d{20, 30, 40}}},
   136  	},
   137  	{
   138  		ElementNR{
   139  			Pos:  dvid.Point3d{120, 65, 100},
   140  			Kind: PostSyn,
   141  			Tags: []Tag{"Synapse2"},
   142  		},
   143  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{127, 63, 99}}},
   144  	},
   145  	{
   146  		ElementNR{
   147  			Pos:  dvid.Point3d{126, 67, 98},
   148  			Kind: PostSyn,
   149  			Tags: []Tag{"Synapse2"},
   150  		},
   151  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{127, 63, 99}}},
   152  	},
   153  }
   154  
   155  var expectedROI = Elements{
   156  	{
   157  		ElementNR{
   158  			Pos:  dvid.Point3d{15, 27, 35}, // Label 1
   159  			Kind: PreSyn,
   160  			Tags: []Tag{"Synapse1", "Zlt90"},
   161  			Prop: map[string]string{
   162  				"Im a T-Bar":         "yes",
   163  				"I'm not a PSD":      "sure",
   164  				"i'm really special": "",
   165  			},
   166  		},
   167  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
   168  	},
   169  	{
   170  		ElementNR{
   171  			Pos:  dvid.Point3d{20, 30, 40}, // Label 2
   172  			Kind: PostSyn,
   173  			Tags: []Tag{"Synapse1"},
   174  		},
   175  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   176  	},
   177  	{
   178  		ElementNR{
   179  			Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   180  			Kind: PostSyn,
   181  			Tags: []Tag{"Synapse1", "Zlt90"},
   182  		},
   183  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   184  	},
   185  	{
   186  		ElementNR{
   187  			Pos:  dvid.Point3d{33, 30, 31},
   188  			Kind: PostSyn,
   189  			Tags: []Tag{"Synapse1", "Zlt90"},
   190  		},
   191  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   192  	},
   193  	{
   194  		ElementNR{
   195  			Pos:  dvid.Point3d{88, 47, 80}, // Label 4
   196  			Kind: PostSyn,
   197  			Tags: []Tag{"Synapse2"},
   198  		},
   199  		[]Relationship{{Rel: GroupedWith, To: dvid.Point3d{14, 25, 37}}, {Rel: PostSynTo, To: dvid.Point3d{127, 63, 99}}, {Rel: GroupedWith, To: dvid.Point3d{20, 30, 40}}},
   200  	},
   201  	{
   202  		ElementNR{
   203  			Pos:  dvid.Point3d{127, 63, 99}, // Label 3
   204  			Kind: PreSyn,
   205  			Tags: []Tag{"Synapse2"},
   206  			Prop: map[string]string{
   207  				"Im a T-Bar":             "no",
   208  				"I'm not a PSD":          "not really",
   209  				"i'm not really special": "at all",
   210  			},
   211  		},
   212  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   213  	},
   214  }
   215  
   216  var expectedLabel1 = Elements{
   217  	{
   218  		ElementNR{
   219  			Pos:  dvid.Point3d{15, 27, 35}, // Label 1
   220  			Kind: PreSyn,
   221  			Tags: []Tag{"Synapse1", "Zlt90"},
   222  			Prop: map[string]string{
   223  				"Im a T-Bar":         "yes",
   224  				"I'm not a PSD":      "sure",
   225  				"i'm really special": "",
   226  			},
   227  		},
   228  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
   229  	},
   230  }
   231  
   232  var expectedLabel2 = Elements{
   233  	{
   234  		ElementNR{
   235  			Pos:  dvid.Point3d{20, 30, 40}, // Label 2
   236  			Kind: PostSyn,
   237  			Tags: []Tag{"Synapse1"},
   238  		},
   239  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   240  	},
   241  }
   242  
   243  var expectedLabel2a = Elements{
   244  	{
   245  		ElementNR{
   246  			Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   247  			Kind: PostSyn,
   248  			Tags: []Tag{"Synapse1", "Zlt90"},
   249  		},
   250  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   251  	},
   252  }
   253  
   254  var expectedLabel2b = Elements{
   255  	{
   256  		ElementNR{
   257  			Pos:  dvid.Point3d{14, 25, 37}, // Originally Label 3
   258  			Kind: PostSyn,
   259  			Tags: []Tag{"Synapse1", "Zlt90"},
   260  		},
   261  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   262  	},
   263  	{
   264  		ElementNR{
   265  			Pos:  dvid.Point3d{20, 30, 40}, // Originally Label 2
   266  			Kind: PostSyn,
   267  			Tags: []Tag{"Synapse1"},
   268  		},
   269  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   270  	},
   271  	{
   272  		ElementNR{
   273  			Pos:  dvid.Point3d{127, 63, 99}, // Originally Label 3
   274  			Kind: PreSyn,
   275  			Tags: []Tag{"Synapse2"},
   276  			Prop: map[string]string{
   277  				"Im a T-Bar":             "no",
   278  				"I'm not a PSD":          "not really",
   279  				"i'm not really special": "at all",
   280  			},
   281  		},
   282  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   283  	},
   284  }
   285  
   286  var expectedLabel2c = Elements{
   287  	{
   288  		ElementNR{
   289  			Pos:  dvid.Point3d{127, 63, 99},
   290  			Kind: PreSyn,
   291  			Tags: []Tag{"Synapse2"},
   292  			Prop: map[string]string{
   293  				"Im a T-Bar":             "no",
   294  				"I'm not a PSD":          "not really",
   295  				"i'm not really special": "at all",
   296  			},
   297  		},
   298  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   299  	},
   300  }
   301  
   302  var expectedLabel7 = Elements{
   303  	{
   304  		ElementNR{
   305  			Pos:  dvid.Point3d{14, 25, 37},
   306  			Kind: PostSyn,
   307  			Tags: []Tag{"Synapse1", "Zlt90"},
   308  		},
   309  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   310  	},
   311  	{
   312  		ElementNR{
   313  			Pos:  dvid.Point3d{20, 30, 40},
   314  			Kind: PostSyn,
   315  			Tags: []Tag{"Synapse1"},
   316  		},
   317  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   318  	},
   319  }
   320  
   321  var afterDeleteOn7 = Elements{
   322  	{
   323  		ElementNR{
   324  			Pos:  dvid.Point3d{14, 25, 37},
   325  			Kind: PostSyn,
   326  			Tags: []Tag{"Synapse1", "Zlt90"},
   327  		},
   328  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   329  	},
   330  }
   331  
   332  var expectedLabel3 = Elements{
   333  	{
   334  		ElementNR{
   335  			Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   336  			Kind: PostSyn,
   337  			Tags: []Tag{"Synapse1", "Zlt90"},
   338  		},
   339  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   340  	},
   341  	{
   342  		ElementNR{
   343  			Pos:  dvid.Point3d{127, 63, 99}, // Label 3
   344  			Kind: PreSyn,
   345  			Tags: []Tag{"Synapse2"},
   346  			Prop: map[string]string{
   347  				"Im a T-Bar":             "no",
   348  				"I'm not a PSD":          "not really",
   349  				"i'm not really special": "at all",
   350  			},
   351  		},
   352  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   353  	},
   354  }
   355  
   356  var expectedLabel3NoRel = ElementsNR{
   357  	{
   358  		Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   359  		Kind: PostSyn,
   360  		Tags: []Tag{"Synapse1", "Zlt90"},
   361  	},
   362  	{
   363  		Pos:  dvid.Point3d{127, 63, 99}, // Label 3
   364  		Kind: PreSyn,
   365  		Tags: []Tag{"Synapse2"},
   366  		Prop: map[string]string{
   367  			"Im a T-Bar":             "no",
   368  			"I'm not a PSD":          "not really",
   369  			"i'm not really special": "at all",
   370  		},
   371  	},
   372  }
   373  
   374  var expectedLabel3a = Elements{
   375  	{
   376  		ElementNR{
   377  			Pos:  dvid.Point3d{20, 30, 40}, // Label 2
   378  			Kind: PostSyn,
   379  			Tags: []Tag{"Synapse1"},
   380  		},
   381  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   382  	},
   383  	{
   384  		ElementNR{
   385  			Pos:  dvid.Point3d{127, 63, 99}, // Label 3
   386  			Kind: PreSyn,
   387  			Tags: []Tag{"Synapse2"},
   388  			Prop: map[string]string{
   389  				"Im a T-Bar":             "no",
   390  				"I'm not a PSD":          "not really",
   391  				"i'm not really special": "at all",
   392  			},
   393  		},
   394  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   395  	},
   396  }
   397  
   398  var expectedLabel4 = Elements{
   399  	{
   400  		ElementNR{
   401  			Pos:  dvid.Point3d{88, 47, 80}, // Label 4
   402  			Kind: PostSyn,
   403  			Tags: []Tag{"Synapse2"},
   404  		},
   405  		[]Relationship{{Rel: GroupedWith, To: dvid.Point3d{14, 25, 37}}, {Rel: PostSynTo, To: dvid.Point3d{127, 63, 99}}, {Rel: GroupedWith, To: dvid.Point3d{20, 30, 40}}},
   406  	},
   407  }
   408  
   409  var expectedLabel4NoRel = ElementsNR{
   410  	{
   411  		Pos:  dvid.Point3d{88, 47, 80}, // Label 4
   412  		Kind: PostSyn,
   413  		Tags: []Tag{"Synapse2"},
   414  	},
   415  }
   416  
   417  var expected3 = Elements{
   418  	{
   419  		ElementNR{
   420  			Pos:  dvid.Point3d{127, 63, 99},
   421  			Kind: PreSyn,
   422  			Tags: []Tag{"Synapse2"},
   423  			Prop: map[string]string{
   424  				"Im a T-Bar":             "no",
   425  				"I'm not a PSD":          "not really",
   426  				"i'm not really special": "at all",
   427  			},
   428  		},
   429  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   430  	},
   431  }
   432  
   433  var afterMove = Elements{
   434  	{
   435  		ElementNR{
   436  			Pos:  dvid.Point3d{15, 27, 35},
   437  			Kind: PreSyn,
   438  			Tags: []Tag{"Synapse1", "Zlt90"},
   439  			Prop: map[string]string{
   440  				"Im a T-Bar":         "yes",
   441  				"I'm not a PSD":      "sure",
   442  				"i'm really special": "",
   443  			},
   444  		},
   445  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
   446  	},
   447  	{
   448  		ElementNR{
   449  			Pos:  dvid.Point3d{20, 30, 40},
   450  			Kind: PostSyn,
   451  			Tags: []Tag{"Synapse1"},
   452  		},
   453  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   454  	},
   455  	{
   456  		ElementNR{
   457  			Pos:  dvid.Point3d{14, 25, 37},
   458  			Kind: PostSyn,
   459  			Tags: []Tag{"Synapse1", "Zlt90"},
   460  		},
   461  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   462  	},
   463  	{
   464  		ElementNR{
   465  			Pos:  dvid.Point3d{33, 30, 31},
   466  			Kind: PostSyn,
   467  			Tags: []Tag{"Synapse1", "Zlt90"},
   468  		},
   469  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   470  	},
   471  	{
   472  		ElementNR{
   473  			Pos:  dvid.Point3d{127, 64, 100},
   474  			Kind: PreSyn,
   475  			Tags: []Tag{"Synapse2"},
   476  			Prop: map[string]string{
   477  				"Im a T-Bar":             "no",
   478  				"I'm not a PSD":          "not really",
   479  				"i'm not really special": "at all",
   480  			},
   481  		},
   482  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{88, 47, 80}}, {Rel: PreSynTo, To: dvid.Point3d{120, 65, 100}}, {Rel: PreSynTo, To: dvid.Point3d{126, 67, 98}}},
   483  	},
   484  	{
   485  		ElementNR{
   486  			Pos:  dvid.Point3d{88, 47, 80},
   487  			Kind: PostSyn,
   488  			Tags: []Tag{"Synapse2"},
   489  		},
   490  		[]Relationship{{Rel: GroupedWith, To: dvid.Point3d{14, 25, 37}}, {Rel: PostSynTo, To: dvid.Point3d{127, 64, 100}}, {Rel: GroupedWith, To: dvid.Point3d{20, 30, 40}}},
   491  	},
   492  	{
   493  		ElementNR{
   494  			Pos:  dvid.Point3d{120, 65, 100},
   495  			Kind: PostSyn,
   496  			Tags: []Tag{"Synapse2"},
   497  		},
   498  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{127, 64, 100}}},
   499  	},
   500  	{
   501  		ElementNR{
   502  			Pos:  dvid.Point3d{126, 67, 98},
   503  			Kind: PostSyn,
   504  			Tags: []Tag{"Synapse2"},
   505  		},
   506  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{127, 64, 100}}},
   507  	},
   508  }
   509  
   510  var afterDelete = Elements{
   511  	{
   512  		ElementNR{
   513  			Pos:  dvid.Point3d{15, 27, 35},
   514  			Kind: PreSyn,
   515  			Tags: []Tag{"Synapse1", "Zlt90"},
   516  			Prop: map[string]string{
   517  				"Im a T-Bar":         "yes",
   518  				"I'm not a PSD":      "sure",
   519  				"i'm really special": "",
   520  			},
   521  		},
   522  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
   523  	},
   524  	{
   525  		ElementNR{
   526  			Pos:  dvid.Point3d{20, 30, 40},
   527  			Kind: PostSyn,
   528  			Tags: []Tag{"Synapse1"},
   529  		},
   530  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   531  	},
   532  	{
   533  		ElementNR{
   534  			Pos:  dvid.Point3d{14, 25, 37},
   535  			Kind: PostSyn,
   536  			Tags: []Tag{"Synapse1", "Zlt90"},
   537  		},
   538  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   539  	},
   540  	{
   541  		ElementNR{
   542  			Pos:  dvid.Point3d{33, 30, 31},
   543  			Kind: PostSyn,
   544  			Tags: []Tag{"Synapse1", "Zlt90"},
   545  		},
   546  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   547  	},
   548  	{
   549  		ElementNR{
   550  			Pos:  dvid.Point3d{88, 47, 80},
   551  			Kind: PostSyn,
   552  			Tags: []Tag{"Synapse2"},
   553  		},
   554  		[]Relationship{{Rel: GroupedWith, To: dvid.Point3d{14, 25, 37}}, {Rel: GroupedWith, To: dvid.Point3d{20, 30, 40}}},
   555  	},
   556  	{
   557  		ElementNR{
   558  			Pos:  dvid.Point3d{120, 65, 100},
   559  			Kind: PostSyn,
   560  			Tags: []Tag{"Synapse2"},
   561  		},
   562  		[]Relationship{},
   563  	},
   564  	{
   565  		ElementNR{
   566  			Pos:  dvid.Point3d{126, 67, 98},
   567  			Kind: PostSyn,
   568  			Tags: []Tag{"Synapse2"},
   569  		},
   570  		[]Relationship{},
   571  	},
   572  }
   573  
   574  var testTagData = Elements{
   575  	{
   576  		ElementNR{
   577  			Pos:  dvid.Point3d{15, 27, 35}, // Label 1
   578  			Kind: PreSyn,
   579  			Tags: []Tag{"Synapse1", "Zlt90"},
   580  			Prop: map[string]string{
   581  				"Im a T-Bar":         "yes",
   582  				"I'm not a PSD":      "sure",
   583  				"i'm really special": "",
   584  			},
   585  		},
   586  		[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
   587  	},
   588  	{
   589  		ElementNR{
   590  			Pos:  dvid.Point3d{21, 33, 40}, // Label 2
   591  			Kind: PostSyn,
   592  			Tags: []Tag{"Synapse1"},
   593  		},
   594  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   595  	},
   596  	{
   597  		ElementNR{
   598  			Pos:  dvid.Point3d{20, 30, 40}, // Label 2
   599  			Kind: PostSyn,
   600  			Tags: []Tag{"Synapse10"},
   601  		},
   602  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   603  	},
   604  	{
   605  		ElementNR{
   606  			Pos:  dvid.Point3d{14, 25, 37}, // Label 3
   607  			Kind: PostSyn,
   608  			Tags: []Tag{"Synapse11", "Zlt90"},
   609  		},
   610  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   611  	},
   612  	{
   613  		ElementNR{
   614  			Pos:  dvid.Point3d{33, 30, 31},
   615  			Kind: PostSyn,
   616  			Tags: []Tag{"Synapse111", "Zlt90"},
   617  		},
   618  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
   619  	},
   620  }
   621  
   622  func getTag(tag Tag, elems Elements) Elements {
   623  	var result Elements
   624  	for _, elem := range elems {
   625  		for _, etag := range elem.Tags {
   626  			if etag == tag {
   627  				result = append(result, elem)
   628  				break
   629  			}
   630  		}
   631  	}
   632  	return result
   633  }
   634  
   635  var synapsesByBlocks = `{"0,0,0":[{"Pos":[15,27,35],"Kind":"PreSyn","Tags":["Synapse1","Zlt90"],"Prop":{"I'm not a PSD":"sure","Im a T-Bar":"yes","i'm really special":""},"Rels":[{"Rel":"PreSynTo","To":[20,30,40]},{"Rel":"PreSynTo","To":[14,25,37]},{"Rel":"PreSynTo","To":[33,30,31]}]},{"Pos":[20,30,40],"Kind":"PostSyn","Tags":["Synapse1"],"Prop":{},"Rels":[{"Rel":"PostSynTo","To":[15,27,35]}]},{"Pos":[14,25,37],"Kind":"PostSyn","Tags":["Synapse1","Zlt90"],"Prop":{},"Rels":[{"Rel":"PostSynTo","To":[15,27,35]}]},{"Pos":[33,30,31],"Kind":"PostSyn","Tags":["Synapse1","Zlt90"],"Prop":{},"Rels":[{"Rel":"PostSynTo","To":[15,27,35]}]}],"1,0,1":[{"Pos":[127,63,99],"Kind":"PreSyn","Tags":["Synapse2"],"Prop":{"I'm not a PSD":"not really","Im a T-Bar":"no","i'm not really special":"at all"},"Rels":[{"Rel":"PreSynTo","To":[88,47,80]},{"Rel":"PreSynTo","To":[120,65,100]},{"Rel":"PreSynTo","To":[126,67,98]}]},{"Pos":[88,47,80],"Kind":"PostSyn","Tags":["Synapse2"],"Prop":{},"Rels":[{"Rel":"GroupedWith","To":[14,25,37]},{"Rel":"PostSynTo","To":[127,63,99]},{"Rel":"GroupedWith","To":[20,30,40]}]}],"1,1,1":[{"Pos":[120,65,100],"Kind":"PostSyn","Tags":["Synapse2"],"Prop":{},"Rels":[{"Rel":"PostSynTo","To":[127,63,99]}]},{"Pos":[126,67,98],"Kind":"PostSyn","Tags":["Synapse2"],"Prop":{},"Rels":[{"Rel":"PostSynTo","To":[127,63,99]}]}]}`
   636  
   637  func testResponse(t *testing.T, expected Elements, template string, args ...interface{}) {
   638  	url := fmt.Sprintf(template, args...)
   639  	returnValue := server.TestHTTP(t, "GET", url, nil)
   640  	dvid.Infof("Returned: %s\nExpected: %s\n", string(returnValue), expected)
   641  	got := Elements{}
   642  	if err := json.Unmarshal(returnValue, &got); err != nil {
   643  		t.Fatal(err)
   644  	}
   645  	if !reflect.DeepEqual(expected.Normalize(), got.Normalize()) {
   646  		_, fn, line, _ := runtime.Caller(1)
   647  		var expectedStr, gotStr string
   648  		if jsonBytes, err := json.Marshal(expected.Normalize()); err != nil {
   649  			t.Fatalf("error converting expected to JSON: %v\n", err)
   650  		} else {
   651  			expectedStr = string(jsonBytes)
   652  		}
   653  		if jsonBytes, err := json.Marshal(got.Normalize()); err != nil {
   654  			t.Fatalf("error converting got to JSON: %v\n", err)
   655  		} else {
   656  			gotStr = string(jsonBytes)
   657  		}
   658  		t.Fatalf("Expected for %s [%s:%d]:\n%s\nGot:\n%v\n", url, fn, line, expectedStr, gotStr)
   659  	}
   660  }
   661  
   662  func testResponseLabel(t *testing.T, expected interface{}, template string, args ...interface{}) {
   663  	var useRels bool
   664  	if strings.HasSuffix(template, "?relationships=true") {
   665  		useRels = true
   666  	}
   667  	url := fmt.Sprintf(template, args...)
   668  	returnValue := server.TestHTTP(t, "GET", url, nil)
   669  
   670  	if useRels {
   671  		var elems Elements
   672  		if expected == nil {
   673  			elems = Elements{}
   674  		} else {
   675  			var ok bool
   676  			elems, ok = expected.(Elements)
   677  			if !ok {
   678  				t.Fatalf("testResponseLabel with template %q didn't get passed Elements for expected: %v\n", template, expected)
   679  			}
   680  		}
   681  		got := Elements{}
   682  		if err := json.Unmarshal(returnValue, &got); err != nil {
   683  			t.Fatal(err)
   684  		}
   685  		if !reflect.DeepEqual(elems.Normalize(), got.Normalize()) {
   686  			_, fn, line, _ := runtime.Caller(1)
   687  			t.Errorf("Expected for %s [%s:%d]:\n%v\nGot:\n%v\n", url, fn, line, elems.Normalize(), got.Normalize())
   688  		}
   689  	} else {
   690  		var elems ElementsNR
   691  		if expected == nil {
   692  			elems = ElementsNR{}
   693  		} else {
   694  			var ok bool
   695  			elems, ok = expected.(ElementsNR)
   696  			if !ok {
   697  				t.Fatalf("testResponseLabel with template %q didn't get passed ElementsNR for expected: %v\n", template, expected)
   698  			}
   699  		}
   700  		got := ElementsNR{}
   701  		if err := json.Unmarshal(returnValue, &got); err != nil {
   702  			t.Fatal(err)
   703  		}
   704  		if !reflect.DeepEqual(elems.Normalize(), got.Normalize()) {
   705  			_, fn, line, _ := runtime.Caller(1)
   706  			t.Errorf("Expected for %s [%s:%d]:\n%v\nGot:\n%v\n", url, fn, line, elems.Normalize(), got.Normalize())
   707  		}
   708  	}
   709  }
   710  
   711  type tuple [4]int32
   712  
   713  var labelsROI = []tuple{
   714  	tuple{0, 0, 0, 1}, tuple{1, 0, 1, 2},
   715  }
   716  
   717  func labelsJSON() string {
   718  	b, err := json.Marshal(labelsROI)
   719  	if err != nil {
   720  		return ""
   721  	}
   722  	return string(b)
   723  }
   724  
   725  func TestRequests(t *testing.T) {
   726  	if err := server.OpenTest(); err != nil {
   727  		t.Fatalf("can't open test server: %v\n", err)
   728  	}
   729  	defer server.CloseTest()
   730  
   731  	uuid, _ := initTestRepo()
   732  
   733  	config := dvid.NewConfig()
   734  	config.Set("BlockSize", "64,64,64")
   735  	dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
   736  	if err != nil {
   737  		t.Fatalf("Error creating new data instance: %v\n", err)
   738  	}
   739  	data, ok := dataservice.(*Data)
   740  	if !ok {
   741  		t.Fatalf("Returned new data instance is not synapse.Data\n")
   742  	}
   743  
   744  	// PUT first batch of synapses
   745  	testJSON, err := json.Marshal(testData)
   746  	if err != nil {
   747  		t.Fatal(err)
   748  	}
   749  	url1 := fmt.Sprintf("%snode/%s/%s/elements", server.WebAPIPath, uuid, data.DataName())
   750  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
   751  
   752  	// Check ROI endpoint
   753  	roiName := "myroi"
   754  	server.CreateTestInstance(t, uuid, "roi", roiName, config)
   755  	apiStr := fmt.Sprintf("%snode/%s/%s/roi", server.WebAPIPath, uuid, roiName)
   756  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(labelsJSON()))
   757  
   758  	roiSpec := fmt.Sprintf("myroi,%s", uuid)
   759  	testResponse(t, expectedROI, "%snode/%s/%s/roi/%s", server.WebAPIPath, uuid, data.DataName(), roiSpec)
   760  	testResponse(t, expectedROI, "%snode/%s/%s/roi/%s", server.WebAPIPath, uuid, data.DataName(), "myroi")
   761  
   762  	// GET synapses back within superset bounding box and make sure all data is there.
   763  	testResponse(t, testData, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
   764  
   765  	// fast GET /blocks for all synapses
   766  	blocksURL := fmt.Sprintf("%snode/%s/%s/blocks/96_63_97/31_15_0", server.WebAPIPath, uuid, data.DataName())
   767  	ret := server.TestHTTP(t, "GET", blocksURL, nil)
   768  	if string(ret) != synapsesByBlocks {
   769  		t.Fatalf("Did not get all synapse elements returned from GET /blocks:\nGot: %s\nExpected: %s\n", string(ret), synapsesByBlocks)
   770  	}
   771  
   772  	// GET /blocks using single range query
   773  	tags := map[string]string{
   774  		"ScanAllForBlocks": "true",
   775  	}
   776  	data.SetTags(tags)
   777  	blocksURL = fmt.Sprintf("%snode/%s/%s/blocks/45_23_40/66_10_70", server.WebAPIPath, uuid, data.DataName())
   778  	ret = server.TestHTTP(t, "GET", blocksURL, nil)
   779  	expectedResult := `{"1,0,1":[{"Pos":[127,63,99],"Kind":"PreSyn","Tags":["Synapse2"],"Prop":{"I'm not a PSD":"not really","Im a T-Bar":"no","i'm not really special":"at all"},"Rels":[{"Rel":"PreSynTo","To":[88,47,80]},{"Rel":"PreSynTo","To":[120,65,100]},{"Rel":"PreSynTo","To":[126,67,98]}]},{"Pos":[88,47,80],"Kind":"PostSyn","Tags":["Synapse2"],"Prop":{},"Rels":[{"Rel":"GroupedWith","To":[14,25,37]},{"Rel":"PostSynTo","To":[127,63,99]},{"Rel":"GroupedWith","To":[20,30,40]}]}]}`
   780  	if string(ret) != expectedResult {
   781  		t.Fatalf("Did not get all synapse elements returned from GET /blocks with ScanAllForBlocks:\nGot: %s\nExpected: %s\n", string(ret), expectedResult)
   782  	}
   783  
   784  	// Test subset GET
   785  	testResponse(t, expected3, "%snode/%s/%s/elements/5_5_5/126_60_97", server.WebAPIPath, uuid, data.DataName())
   786  
   787  	// Test Tag 1
   788  	tag := Tag("Synapse2")
   789  	synapse2 := getTag(tag, testData)
   790  	testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
   791  
   792  	// Test Tag 2
   793  	tag2 := Tag("Zlt90")
   794  	zlt90 := getTag(tag2, testData)
   795  	testResponse(t, zlt90, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag2)
   796  
   797  	// Test move
   798  	url5 := fmt.Sprintf("%snode/%s/%s/move/127_63_99/127_64_100", server.WebAPIPath, uuid, data.DataName())
   799  	server.TestHTTP(t, "POST", url5, nil)
   800  	testResponse(t, afterMove, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
   801  
   802  	// --- check tag
   803  	synapse2 = getTag(tag, afterMove)
   804  	testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
   805  
   806  	// Test delete
   807  	url6 := fmt.Sprintf("%snode/%s/%s/element/127_64_100", server.WebAPIPath, uuid, data.DataName())
   808  	server.TestHTTP(t, "DELETE", url6, nil)
   809  	testResponse(t, afterDelete, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
   810  
   811  	// --- check tag
   812  	synapse2 = getTag(tag, afterDelete)
   813  	testResponse(t, synapse2, "%snode/%s/%s/tag/%s?relationships=true", server.WebAPIPath, uuid, data.DataName(), tag)
   814  }
   815  
   816  var testBlocks = blockList{
   817  	"1,1,1": Elements{
   818  		{
   819  			ElementNR{
   820  				Pos:  dvid.Point3d{65, 70, 75},
   821  				Kind: PostSyn,
   822  				Tags: []Tag{"Synapse1"},
   823  			},
   824  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{129, 130, 131}}},
   825  		},
   826  	},
   827  	"2,2,2": Elements{
   828  		{
   829  			ElementNR{
   830  				Pos:  dvid.Point3d{129, 130, 131},
   831  				Kind: PreSyn,
   832  				Tags: []Tag{"Synapse1"},
   833  				Prop: map[string]string{
   834  					"Im a T-Bar":             "no",
   835  					"I'm not a PSD":          "not really",
   836  					"i'm not really special": "at all",
   837  				},
   838  			},
   839  			[]Relationship{
   840  				{Rel: PreSynTo, To: dvid.Point3d{129, 130, 131}},
   841  				{Rel: PreSynTo, To: dvid.Point3d{65, 70, 75}},
   842  			},
   843  		},
   844  		{
   845  			ElementNR{
   846  				Pos:  dvid.Point3d{130, 131, 132},
   847  				Kind: PostSyn,
   848  				Tags: []Tag{"Synapse1"},
   849  			},
   850  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{129, 130, 131}}},
   851  		},
   852  	},
   853  }
   854  
   855  var testBlocksElements = Elements{
   856  	{
   857  		ElementNR{
   858  			Pos:  dvid.Point3d{65, 70, 75},
   859  			Kind: PostSyn,
   860  			Tags: []Tag{"Synapse1"},
   861  		},
   862  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{129, 130, 131}}},
   863  	},
   864  	{
   865  		ElementNR{
   866  			Pos:  dvid.Point3d{129, 130, 131},
   867  			Kind: PreSyn,
   868  			Tags: []Tag{"Synapse1"},
   869  			Prop: map[string]string{
   870  				"Im a T-Bar":             "no",
   871  				"I'm not a PSD":          "not really",
   872  				"i'm not really special": "at all",
   873  			},
   874  		},
   875  		[]Relationship{
   876  			{Rel: PreSynTo, To: dvid.Point3d{129, 130, 131}},
   877  			{Rel: PreSynTo, To: dvid.Point3d{65, 70, 75}},
   878  		},
   879  	},
   880  	{
   881  		ElementNR{
   882  			Pos:  dvid.Point3d{130, 131, 132},
   883  			Kind: PostSyn,
   884  			Tags: []Tag{"Synapse1"},
   885  		},
   886  		[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{129, 130, 131}}},
   887  	},
   888  }
   889  
   890  var testLabelPoints = map[string]string{
   891  	"42": `[{"Pos":[31,54,17],"Kind":"PreSyn","Prop":{"Some prop":"yes"}},{"Pos":[65,70,75],"Kind":"PostSyn","Prop":{"I'm not a PSD":"sure","Im a T-Bar":"yes","i'm really special":""}}]`,
   892  	"67": `[{"Pos":[25,11,76],"Kind":"PreSyn","Prop":{"A prop":"no"}},{"Pos":[86,2,56],"Kind":"PostSyn","Prop":{"A":"B","C":"D"}}]`,
   893  }
   894  
   895  func TestPostLabels(t *testing.T) {
   896  	if err := server.OpenTest(); err != nil {
   897  		t.Fatalf("can't open test server: %v\n", err)
   898  	}
   899  	defer server.CloseTest()
   900  
   901  	uuid, _ := initTestRepo()
   902  
   903  	config := dvid.NewConfig()
   904  	dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
   905  	if err != nil {
   906  		t.Fatalf("Error creating new data instance: %v\n", err)
   907  	}
   908  	data, ok := dataservice.(*Data)
   909  	if !ok {
   910  		t.Fatalf("Returned new data instance is not synapse.Data\n")
   911  	}
   912  
   913  	// PUT first label synapses
   914  	testJSON, err := json.Marshal(testLabelPoints)
   915  	if err != nil {
   916  		t.Fatal(err)
   917  	}
   918  	url1 := fmt.Sprintf("%snode/%s/%s/labels", server.WebAPIPath, uuid, data.DataName())
   919  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
   920  
   921  	// GET for each of the labels
   922  	for label, expectedStr := range testLabelPoints {
   923  		url := fmt.Sprintf("%snode/%s/%s/label/%s", server.WebAPIPath, uuid, data.DataName(), label)
   924  		ret := server.TestHTTP(t, "GET", url, nil)
   925  		var expected, got ElementsNR
   926  		if err := json.Unmarshal([]byte(expectedStr), &expected); err != nil {
   927  			t.Fatal(err)
   928  		}
   929  		if err := json.Unmarshal(ret, &got); err != nil {
   930  			t.Fatal(err)
   931  		}
   932  		if !reflect.DeepEqual(expected, got) {
   933  			t.Errorf("Expected:\n%v\nGot:\n%v\n", expected, got)
   934  		}
   935  	}
   936  }
   937  
   938  var testBlocksReturn = `{"1,1,1":[{"Pos":[65,70,75],"Kind":"PostSyn","Tags":["Synapse1"],"Prop":null,"Rels":[{"Rel":"PostSynTo","To":[129,130,131]}]}],"2,2,2":[{"Pos":[129,130,131],"Kind":"PreSyn","Tags":["Synapse1"],"Prop":{"I'm not a PSD":"not really","Im a T-Bar":"no","i'm not really special":"at all"},"Rels":[{"Rel":"PreSynTo","To":[129,130,131]},{"Rel":"PreSynTo","To":[65,70,75]}]},{"Pos":[130,131,132],"Kind":"PostSyn","Tags":["Synapse1"],"Prop":null,"Rels":[{"Rel":"PostSynTo","To":[129,130,131]}]}]}`
   939  
   940  func TestPostBlocksAndAll(t *testing.T) {
   941  	if err := server.OpenTest(); err != nil {
   942  		t.Fatalf("can't open test server: %v\n", err)
   943  	}
   944  	defer server.CloseTest()
   945  
   946  	uuid, _ := initTestRepo()
   947  
   948  	config := dvid.NewConfig()
   949  	dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
   950  	if err != nil {
   951  		t.Fatalf("Error creating new data instance: %v\n", err)
   952  	}
   953  	data, ok := dataservice.(*Data)
   954  	if !ok {
   955  		t.Fatalf("Returned new data instance is not synapse.Data\n")
   956  	}
   957  
   958  	// PUT first batch of synapses
   959  	testJSON, err := json.Marshal(testBlocks)
   960  	if err != nil {
   961  		t.Fatal(err)
   962  	}
   963  	url1 := fmt.Sprintf("%snode/%s/%s/blocks", server.WebAPIPath, uuid, data.DataName())
   964  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
   965  
   966  	// GET synapses back within superset bounding box and make sure all data is there.
   967  	testResponse(t, testBlocksElements, "%snode/%s/%s/elements/1000_1000_1000/0_0_0", server.WebAPIPath, uuid, data.DataName())
   968  
   969  	// fast GET /all-elements for all synapses
   970  	blocksURL := fmt.Sprintf("%snode/%s/%s/all-elements", server.WebAPIPath, uuid, data.DataName())
   971  	ret := server.TestHTTP(t, "GET", blocksURL, nil)
   972  	if string(ret) != testBlocksReturn {
   973  		t.Fatalf("Did not get all elements returned from GET /all-elements:\nGot: %s\nExpected: %s\n", string(ret), testBlocksReturn)
   974  	}
   975  
   976  	// fast GET /blocks for all synapses
   977  	blocksURL = fmt.Sprintf("%snode/%s/%s/blocks/128_128_128/64_64_64", server.WebAPIPath, uuid, data.DataName())
   978  	ret = server.TestHTTP(t, "GET", blocksURL, nil)
   979  	if string(ret) != testBlocksReturn {
   980  		t.Fatalf("Did not get all elements returned from GET /blocks:\nGot: %s\nExpected: %s\n", string(ret), testBlocksReturn)
   981  	}
   982  
   983  	// Do /scan to see if blocks stats are correct.
   984  	scanURL := fmt.Sprintf("%snode/%s/%s/scan", server.WebAPIPath, uuid, data.DataName())
   985  	ret = server.TestHTTP(t, "GET", scanURL, nil)
   986  	var retJSON map[string]int
   987  	if err := json.Unmarshal(ret, &retJSON); err != nil {
   988  		t.Fatalf("Error on unmarshaling /scan response: %v\n", err)
   989  	}
   990  	if retJSON["num kv pairs"] != 2 {
   991  		t.Errorf("Expected 2 blocks in synapse data, got %d\n", retJSON["num kv pairs"])
   992  	}
   993  	if retJSON["num empty blocks"] != 0 {
   994  		t.Errorf("Expected 0 empty blocks, got %d\n", retJSON["num empty blocks"])
   995  	}
   996  
   997  	scanURL = fmt.Sprintf("%snode/%s/%s/scan?byCoord=true", server.WebAPIPath, uuid, data.DataName())
   998  	ret = server.TestHTTP(t, "GET", scanURL, nil)
   999  	if err := json.Unmarshal(ret, &retJSON); err != nil {
  1000  		t.Fatalf("Error on unmarshaling /scan response: %v\n", err)
  1001  	}
  1002  	if retJSON["num kv pairs"] != 2 {
  1003  		t.Errorf("Expected 2 blocks in synapse data, got %d\n", retJSON["num kv pairs"])
  1004  	}
  1005  	if retJSON["num empty blocks"] != 0 {
  1006  		t.Errorf("Expected 0 empty blocks, got %d\n", retJSON["num empty blocks"])
  1007  	}
  1008  
  1009  	scanURL = fmt.Sprintf("%snode/%s/%s/scan?keysOnly=true", server.WebAPIPath, uuid, data.DataName())
  1010  	ret = server.TestHTTP(t, "GET", scanURL, nil)
  1011  	if err := json.Unmarshal(ret, &retJSON); err != nil {
  1012  		t.Fatalf("Error on unmarshaling /scan response: %v\n", err)
  1013  	}
  1014  	if retJSON["num kv pairs"] != 2 {
  1015  		t.Errorf("Expected 2 blocks in synapse data, got %d\n", retJSON["num kv pairs"])
  1016  	}
  1017  
  1018  	scanURL = fmt.Sprintf("%snode/%s/%s/scan?keysOnly=true&byCoord=true", server.WebAPIPath, uuid, data.DataName())
  1019  	ret = server.TestHTTP(t, "GET", scanURL, nil)
  1020  	if err := json.Unmarshal(ret, &retJSON); err != nil {
  1021  		t.Fatalf("Error on unmarshaling /scan response: %v\n", err)
  1022  	}
  1023  	if retJSON["num kv pairs"] != 2 {
  1024  		t.Errorf("Expected 2 blocks in synapse data, got %d\n", retJSON["num kv pairs"])
  1025  	}
  1026  }
  1027  
  1028  func TestPropChange(t *testing.T) {
  1029  	if err := server.OpenTest(); err != nil {
  1030  		t.Fatalf("can't open test server: %v\n", err)
  1031  	}
  1032  	defer server.CloseTest()
  1033  
  1034  	uuid, _ := initTestRepo()
  1035  
  1036  	config := dvid.NewConfig()
  1037  	dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
  1038  	if err != nil {
  1039  		t.Fatalf("Error creating new data instance: %v\n", err)
  1040  	}
  1041  	data, ok := dataservice.(*Data)
  1042  	if !ok {
  1043  		t.Fatalf("Returned new data instance is not synapse.Data\n")
  1044  	}
  1045  
  1046  	// PUT first batch of synapses
  1047  	testJSON, err := json.Marshal(testTagData)
  1048  	if err != nil {
  1049  		t.Fatal(err)
  1050  	}
  1051  	elementsURL := fmt.Sprintf("%snode/%s/%s/elements", server.WebAPIPath, uuid, data.DataName())
  1052  	server.TestHTTP(t, "POST", elementsURL, strings.NewReader(string(testJSON)))
  1053  
  1054  	// Test Tag "Synapse1"
  1055  	expected := Elements{
  1056  		{
  1057  			ElementNR{
  1058  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1059  				Kind: PreSyn,
  1060  				Tags: []Tag{"Synapse1", "Zlt90"},
  1061  				Prop: map[string]string{
  1062  					"Im a T-Bar":         "yes",
  1063  					"I'm not a PSD":      "sure",
  1064  					"i'm really special": "",
  1065  				},
  1066  			},
  1067  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1068  		},
  1069  		{
  1070  			ElementNR{
  1071  				Pos:  dvid.Point3d{21, 33, 40}, // Label 2
  1072  				Kind: PostSyn,
  1073  				Tags: []Tag{"Synapse1"},
  1074  			},
  1075  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1076  		},
  1077  	}
  1078  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1079  
  1080  	// Change Prop and recheck Tag
  1081  	mod := Elements{
  1082  		{
  1083  			ElementNR{
  1084  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1085  				Kind: PreSyn,
  1086  				Tags: []Tag{"Synapse1", "Zlt90"},
  1087  				Prop: map[string]string{
  1088  					"Im a T-Bar":             "no",
  1089  					"I'm not a PSD":          "unsure",
  1090  					"i'm NOT really special": "",
  1091  				},
  1092  			},
  1093  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1094  		},
  1095  	}
  1096  	testJSON, err = json.Marshal(mod)
  1097  	if err != nil {
  1098  		t.Fatal(err)
  1099  	}
  1100  	server.TestHTTP(t, "POST", elementsURL, strings.NewReader(string(testJSON)))
  1101  
  1102  	expected = Elements{
  1103  		{
  1104  			ElementNR{
  1105  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1106  				Kind: PreSyn,
  1107  				Tags: []Tag{"Synapse1", "Zlt90"},
  1108  				Prop: map[string]string{
  1109  					"Im a T-Bar":             "no",
  1110  					"I'm not a PSD":          "unsure",
  1111  					"i'm NOT really special": "",
  1112  				},
  1113  			},
  1114  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1115  		},
  1116  		{
  1117  			ElementNR{
  1118  				Pos:  dvid.Point3d{21, 33, 40}, // Label 2
  1119  				Kind: PostSyn,
  1120  				Tags: []Tag{"Synapse1"},
  1121  			},
  1122  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1123  		},
  1124  	}
  1125  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1126  	removeRelationships(expected)
  1127  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1", server.WebAPIPath, uuid, data.DataName())
  1128  
  1129  	expected = Elements{
  1130  		{
  1131  			ElementNR{
  1132  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1133  				Kind: PreSyn,
  1134  				Tags: []Tag{"Synapse1", "Zlt90"},
  1135  				Prop: map[string]string{
  1136  					"Im a T-Bar":             "no",
  1137  					"I'm not a PSD":          "unsure",
  1138  					"i'm NOT really special": "",
  1139  				},
  1140  			},
  1141  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1142  		},
  1143  		{
  1144  			ElementNR{
  1145  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1146  				Kind: PostSyn,
  1147  				Tags: []Tag{"Synapse11", "Zlt90"},
  1148  			},
  1149  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1150  		},
  1151  		{
  1152  			ElementNR{
  1153  				Pos:  dvid.Point3d{33, 30, 31},
  1154  				Kind: PostSyn,
  1155  				Tags: []Tag{"Synapse111", "Zlt90"},
  1156  			},
  1157  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1158  		},
  1159  	}
  1160  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1161  	removeRelationships(expected)
  1162  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90", server.WebAPIPath, uuid, data.DataName())
  1163  }
  1164  
  1165  func removeRelationships(elems Elements) {
  1166  	for i, elem := range elems {
  1167  		elem.Rels = []Relationship{}
  1168  		elems[i] = elem
  1169  	}
  1170  }
  1171  
  1172  func TestTagRequests(t *testing.T) {
  1173  	if err := server.OpenTest(); err != nil {
  1174  		t.Fatalf("can't open test server: %v\n", err)
  1175  	}
  1176  	defer server.CloseTest()
  1177  
  1178  	uuid, _ := initTestRepo()
  1179  
  1180  	config := dvid.NewConfig()
  1181  	dataservice, err := datastore.NewData(uuid, syntype, "mysynapses", config)
  1182  	if err != nil {
  1183  		t.Fatalf("Error creating new data instance: %v\n", err)
  1184  	}
  1185  	data, ok := dataservice.(*Data)
  1186  	if !ok {
  1187  		t.Fatalf("Returned new data instance is not synapse.Data\n")
  1188  	}
  1189  
  1190  	// PUT first batch of synapses
  1191  	testJSON, err := json.Marshal(testTagData)
  1192  	if err != nil {
  1193  		t.Fatal(err)
  1194  	}
  1195  	url1 := fmt.Sprintf("%snode/%s/%s/elements", server.WebAPIPath, uuid, data.DataName())
  1196  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1197  
  1198  	// Test Tags
  1199  	expected := Elements{
  1200  		{
  1201  			ElementNR{
  1202  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1203  				Kind: PreSyn,
  1204  				Tags: []Tag{"Synapse1", "Zlt90"},
  1205  				Prop: map[string]string{
  1206  					"Im a T-Bar":         "yes",
  1207  					"I'm not a PSD":      "sure",
  1208  					"i'm really special": "",
  1209  				},
  1210  			},
  1211  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1212  		},
  1213  		{
  1214  			ElementNR{
  1215  				Pos:  dvid.Point3d{21, 33, 40}, // Label 2
  1216  				Kind: PostSyn,
  1217  				Tags: []Tag{"Synapse1"},
  1218  			},
  1219  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1220  		},
  1221  	}
  1222  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1223  	removeRelationships(expected)
  1224  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1", server.WebAPIPath, uuid, data.DataName())
  1225  
  1226  	expected = Elements{
  1227  		{
  1228  			ElementNR{
  1229  				Pos:  dvid.Point3d{20, 30, 40}, // Label 2
  1230  				Kind: PostSyn,
  1231  				Tags: []Tag{"Synapse10"},
  1232  			},
  1233  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1234  		},
  1235  	}
  1236  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse10?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1237  	removeRelationships(expected)
  1238  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse10", server.WebAPIPath, uuid, data.DataName())
  1239  
  1240  	expected = Elements{
  1241  		{
  1242  			ElementNR{
  1243  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1244  				Kind: PostSyn,
  1245  				Tags: []Tag{"Synapse11", "Zlt90"},
  1246  			},
  1247  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1248  		},
  1249  	}
  1250  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse11?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1251  	removeRelationships(expected)
  1252  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse11", server.WebAPIPath, uuid, data.DataName())
  1253  
  1254  	expected = Elements{
  1255  		{
  1256  			ElementNR{
  1257  				Pos:  dvid.Point3d{33, 30, 31},
  1258  				Kind: PostSyn,
  1259  				Tags: []Tag{"Synapse111", "Zlt90"},
  1260  			},
  1261  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1262  		},
  1263  	}
  1264  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse111?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1265  	removeRelationships(expected)
  1266  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse111", server.WebAPIPath, uuid, data.DataName())
  1267  
  1268  	expected = Elements{
  1269  		{
  1270  			ElementNR{
  1271  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1272  				Kind: PreSyn,
  1273  				Tags: []Tag{"Synapse1", "Zlt90"},
  1274  				Prop: map[string]string{
  1275  					"Im a T-Bar":         "yes",
  1276  					"I'm not a PSD":      "sure",
  1277  					"i'm really special": "",
  1278  				},
  1279  			},
  1280  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1281  		},
  1282  		{
  1283  			ElementNR{
  1284  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1285  				Kind: PostSyn,
  1286  				Tags: []Tag{"Synapse11", "Zlt90"},
  1287  			},
  1288  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1289  		},
  1290  		{
  1291  			ElementNR{
  1292  				Pos:  dvid.Point3d{33, 30, 31},
  1293  				Kind: PostSyn,
  1294  				Tags: []Tag{"Synapse111", "Zlt90"},
  1295  			},
  1296  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1297  		},
  1298  	}
  1299  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1300  	removeRelationships(expected)
  1301  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90", server.WebAPIPath, uuid, data.DataName())
  1302  
  1303  	// delete an annotation and check if its deleted in tag
  1304  	delurl := fmt.Sprintf("%snode/%s/%s/element/15_27_35", server.WebAPIPath, uuid, data.DataName())
  1305  	server.TestHTTP(t, "DELETE", delurl, nil)
  1306  
  1307  	expected = Elements{
  1308  		{
  1309  			ElementNR{
  1310  				Pos:  dvid.Point3d{21, 33, 40}, // Label 2
  1311  				Kind: PostSyn,
  1312  				Tags: []Tag{"Synapse1"},
  1313  			},
  1314  			[]Relationship{},
  1315  		},
  1316  	}
  1317  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1318  	removeRelationships(expected)
  1319  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1", server.WebAPIPath, uuid, data.DataName())
  1320  	expected = Elements{
  1321  		{
  1322  			ElementNR{
  1323  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1324  				Kind: PostSyn,
  1325  				Tags: []Tag{"Synapse11", "Zlt90"},
  1326  			},
  1327  			[]Relationship{},
  1328  		},
  1329  		{
  1330  			ElementNR{
  1331  				Pos:  dvid.Point3d{33, 30, 31},
  1332  				Kind: PostSyn,
  1333  				Tags: []Tag{"Synapse111", "Zlt90"},
  1334  			},
  1335  			[]Relationship{},
  1336  		},
  1337  	}
  1338  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1339  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90", server.WebAPIPath, uuid, data.DataName())
  1340  
  1341  	// post the same annotation but without a tag to see if we've removed the tag
  1342  	mod1 := Elements{
  1343  		{
  1344  			ElementNR{
  1345  				Pos:  dvid.Point3d{33, 30, 31},
  1346  				Kind: PostSyn,
  1347  			},
  1348  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1349  		},
  1350  	}
  1351  	testJSON, err = json.Marshal(mod1)
  1352  	if err != nil {
  1353  		t.Fatal(err)
  1354  	}
  1355  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1356  	expected = Elements{}
  1357  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse111?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1358  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse111", server.WebAPIPath, uuid, data.DataName())
  1359  	expected = Elements{
  1360  		{
  1361  			ElementNR{
  1362  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1363  				Kind: PostSyn,
  1364  				Tags: []Tag{"Synapse11", "Zlt90"},
  1365  			},
  1366  			[]Relationship{},
  1367  		},
  1368  	}
  1369  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1370  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90", server.WebAPIPath, uuid, data.DataName())
  1371  
  1372  	// post the same annotation but with a tag to see if it's added to the tag
  1373  	mod2 := Elements{
  1374  		{
  1375  			ElementNR{
  1376  				Pos:  dvid.Point3d{33, 30, 31},
  1377  				Kind: PostSyn,
  1378  				Tags: []Tag{"foobar", "foobaz"},
  1379  			},
  1380  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1381  		},
  1382  	}
  1383  	testJSON, err = json.Marshal(mod2)
  1384  	if err != nil {
  1385  		t.Fatal(err)
  1386  	}
  1387  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1388  	testResponse(t, mod2, "%snode/%s/%s/tag/foobaz?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1389  	testResponse(t, mod2, "%snode/%s/%s/tag/foobar?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1390  	removeRelationships(mod2)
  1391  	testResponse(t, mod2, "%snode/%s/%s/tag/foobaz", server.WebAPIPath, uuid, data.DataName())
  1392  	testResponse(t, mod2, "%snode/%s/%s/tag/foobar", server.WebAPIPath, uuid, data.DataName())
  1393  
  1394  	// modify the property of the annotation in a tag
  1395  	mod3 := Elements{
  1396  		{
  1397  			ElementNR{
  1398  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1399  				Kind: PreSyn,
  1400  				Tags: []Tag{"Synapse1"},
  1401  				Prop: map[string]string{
  1402  					"Im a T-Bar":             "no",
  1403  					"I'm not a PSD":          "not at all",
  1404  					"i'm really NOT special": "",
  1405  				},
  1406  			},
  1407  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1408  		},
  1409  		{
  1410  			ElementNR{
  1411  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1412  				Kind: PostSyn,
  1413  				Tags: []Tag{"Synapse11", "Zlt90"},
  1414  			},
  1415  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1416  		},
  1417  	}
  1418  	testJSON, err = json.Marshal(mod3)
  1419  	if err != nil {
  1420  		t.Fatal(err)
  1421  	}
  1422  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1423  	expected = Elements{
  1424  		{
  1425  			ElementNR{
  1426  				Pos:  dvid.Point3d{15, 27, 35}, // Label 1
  1427  				Kind: PreSyn,
  1428  				Tags: []Tag{"Synapse1"},
  1429  				Prop: map[string]string{
  1430  					"Im a T-Bar":             "no",
  1431  					"I'm not a PSD":          "not at all",
  1432  					"i'm really NOT special": "",
  1433  				},
  1434  			},
  1435  			[]Relationship{{Rel: PreSynTo, To: dvid.Point3d{21, 33, 40}}, {Rel: PreSynTo, To: dvid.Point3d{20, 30, 40}}, {Rel: PreSynTo, To: dvid.Point3d{14, 25, 37}}, {Rel: PreSynTo, To: dvid.Point3d{33, 30, 31}}},
  1436  		},
  1437  		{
  1438  			ElementNR{
  1439  				Pos:  dvid.Point3d{21, 33, 40}, // Label 2
  1440  				Kind: PostSyn,
  1441  				Tags: []Tag{"Synapse1"},
  1442  			},
  1443  			[]Relationship{},
  1444  		},
  1445  	}
  1446  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1447  	removeRelationships(expected)
  1448  	testResponse(t, expected, "%snode/%s/%s/tag/Synapse1", server.WebAPIPath, uuid, data.DataName())
  1449  
  1450  	expected = Elements{
  1451  		{
  1452  			ElementNR{
  1453  				Pos:  dvid.Point3d{14, 25, 37}, // Label 3
  1454  				Kind: PostSyn,
  1455  				Tags: []Tag{"Synapse11", "Zlt90"},
  1456  			},
  1457  			[]Relationship{{Rel: PostSynTo, To: dvid.Point3d{15, 27, 35}}},
  1458  		},
  1459  	}
  1460  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90?relationships=true", server.WebAPIPath, uuid, data.DataName())
  1461  	removeRelationships(expected)
  1462  	testResponse(t, expected, "%snode/%s/%s/tag/Zlt90", server.WebAPIPath, uuid, data.DataName())
  1463  }
  1464  
  1465  func getBytesRLE(t *testing.T, rles dvid.RLEs) *bytes.Buffer {
  1466  	n := len(rles)
  1467  	buf := new(bytes.Buffer)
  1468  	buf.WriteByte(dvid.EncodingBinary)
  1469  	binary.Write(buf, binary.LittleEndian, uint8(3))  // # of dimensions
  1470  	binary.Write(buf, binary.LittleEndian, byte(0))   // dimension of run (X = 0)
  1471  	buf.WriteByte(byte(0))                            // reserved for later
  1472  	binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels
  1473  	binary.Write(buf, binary.LittleEndian, uint32(n)) // Placeholder for # spans
  1474  	rleBytes, err := rles.MarshalBinary()
  1475  	if err != nil {
  1476  		t.Errorf("Unable to serialize RLEs: %v\n", err)
  1477  	}
  1478  	buf.Write(rleBytes)
  1479  	return buf
  1480  }
  1481  
  1482  func testLabels(t *testing.T, uuid dvid.UUID, labelblkName, labelvolName dvid.InstanceName) {
  1483  	// Make sure request of annotations for a label return [] and not nil before any
  1484  	// annotations are ingested.
  1485  	url := fmt.Sprintf("%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1486  	returnValue := server.TestHTTP(t, "GET", url, nil)
  1487  	if string(returnValue) != "[]" {
  1488  		t.Errorf("Expected [] return on querying empty label annotations, got %q\n", string(returnValue))
  1489  	}
  1490  	url = fmt.Sprintf("%snode/%s/mysynapses/label/1", server.WebAPIPath, uuid)
  1491  	returnValue = server.TestHTTP(t, "GET", url, nil)
  1492  	if string(returnValue) != "[]" {
  1493  		t.Errorf("Expected [] return on querying empty label annotations, got %q\n", string(returnValue))
  1494  	}
  1495  
  1496  	// PUT first batch of synapses
  1497  	testJSON, err := json.Marshal(testData)
  1498  	if err != nil {
  1499  		t.Fatal(err)
  1500  	}
  1501  	url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
  1502  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1503  
  1504  	// Test if labels were properly denormalized.  For the POST we have synchronized label denormalization.
  1505  	// If this were to become asynchronous, we'd want to block on updating like the labelblk<->labelvol sync.
  1506  
  1507  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1508  	testResponseLabel(t, expectedLabel2, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1509  	testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1510  	testResponseLabel(t, expectedLabel3NoRel, "%snode/%s/mysynapses/label/3", server.WebAPIPath, uuid)
  1511  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1512  
  1513  	// Make change to labelblk and make sure our label synapses have been adjusted (case A)
  1514  	_ = modifyLabelTestVolume(t, uuid, string(labelblkName))
  1515  
  1516  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1517  		t.Fatalf("Error blocking on sync of labels->annotations: %v\n", err)
  1518  	}
  1519  
  1520  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1521  	testResponseLabel(t, expectedLabel2a, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1522  	testResponseLabel(t, expectedLabel3a, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1523  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1524  	testResponseLabel(t, expectedLabel4NoRel, "%snode/%s/mysynapses/label/4", server.WebAPIPath, uuid)
  1525  
  1526  	// Make change to labelvol and make sure our label synapses have been adjusted (case B).
  1527  	// Merge 3a into 2a.
  1528  	testMerge := mergeJSON(`[2, 3]`)
  1529  	testMerge.send(t, uuid, string(labelvolName))
  1530  
  1531  	if err := datastore.BlockOnUpdating(uuid, labelblkName); err != nil {
  1532  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  1533  	}
  1534  
  1535  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1536  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1537  	}
  1538  
  1539  	if err := datastore.BlockOnUpdating(uuid, labelvolName); err != nil {
  1540  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
  1541  	}
  1542  
  1543  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1544  	testResponseLabel(t, expectedLabel2b, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1545  	testResponseLabel(t, nil, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1546  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1547  
  1548  	// Now split label 2b off and check if annotations also split
  1549  
  1550  	// Create the sparsevol encoding for split area
  1551  	numspans := len(bodysplit.voxelSpans)
  1552  	rles := make(dvid.RLEs, numspans, numspans)
  1553  	for i, span := range bodysplit.voxelSpans {
  1554  		start := dvid.Point3d{span[2], span[1], span[0]}
  1555  		length := span[3] - span[2] + 1
  1556  		rles[i] = dvid.NewRLE(start, length)
  1557  	}
  1558  	buf := getBytesRLE(t, rles)
  1559  
  1560  	// Submit the split sparsevol
  1561  	reqStr := fmt.Sprintf("%snode/%s/%s/split/2", server.WebAPIPath, uuid, labelvolName)
  1562  	r := server.TestHTTP(t, "POST", reqStr, buf)
  1563  	var jsonVal struct {
  1564  		Label uint64
  1565  	}
  1566  	if err := json.Unmarshal(r, &jsonVal); err != nil {
  1567  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
  1568  	}
  1569  	splitLabel := jsonVal.Label
  1570  
  1571  	// Verify that the annotations are correct.
  1572  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1573  		t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
  1574  	}
  1575  	testResponseLabel(t, expectedLabel2c, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1576  	url2 := fmt.Sprintf("%snode/%s/mysynapses/label/%d?relationships=true", server.WebAPIPath, uuid, splitLabel)
  1577  	testResponseLabel(t, expectedLabel7, url2)
  1578  
  1579  	// Change the name of the annotations.
  1580  	if err = datastore.RenameData(uuid, "mysynapses", labelvolName, "foobar"); err == nil {
  1581  		t.Fatalf("Should have been prevented from renaming data 'mysynapses' to existing instance!\n")
  1582  	}
  1583  	if err = datastore.RenameData(uuid, "mysynapses", "renamedData", "foobar"); err != nil {
  1584  		t.Fatalf("Error renaming annotations: %v\n", err)
  1585  	}
  1586  
  1587  	// Make sure the old name is no longer there and the new one is.
  1588  	server.TestBadHTTP(t, "GET", url2, nil)
  1589  	testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
  1590  
  1591  	// Try a coarse split.
  1592  
  1593  	// Create the encoding for split area in block coordinates.
  1594  	rles = dvid.RLEs{
  1595  		dvid.NewRLE(dvid.Point3d{3, 1, 3}, 1),
  1596  	}
  1597  	buf = getBytesRLE(t, rles)
  1598  
  1599  	// Submit the coarse split
  1600  	reqStr = fmt.Sprintf("%snode/%s/%s/split-coarse/2", server.WebAPIPath, uuid, labelvolName)
  1601  	r = server.TestHTTP(t, "POST", reqStr, buf)
  1602  	if err := json.Unmarshal(r, &jsonVal); err != nil {
  1603  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
  1604  	}
  1605  	splitLabel2 := jsonVal.Label
  1606  
  1607  	// Verify that the annotations are correct.
  1608  	if err := datastore.BlockOnUpdating(uuid, "renamedData"); err != nil {
  1609  		t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
  1610  	}
  1611  	testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/%d?relationships=true", server.WebAPIPath, uuid, splitLabel2)
  1612  	testResponseLabel(t, nil, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
  1613  
  1614  	// Delete a labeled annotation and make sure it's not in label
  1615  	delurl := fmt.Sprintf("%snode/%s/%s/element/20_30_40", server.WebAPIPath, uuid, "renamedData")
  1616  	server.TestHTTP(t, "DELETE", delurl, nil)
  1617  	testResponseLabel(t, afterDeleteOn7, "%snode/%s/%s/label/%d?relationships=true", server.WebAPIPath, uuid, "renamedData", splitLabel)
  1618  }
  1619  
  1620  func testMappedLabels(t *testing.T, uuid dvid.UUID, labelblkName, labelvolName dvid.InstanceName) {
  1621  	// Make sure request of annotations for a label return [] and not nil before any
  1622  	// annotations are ingested.
  1623  	url := fmt.Sprintf("%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1624  	returnValue := server.TestHTTP(t, "GET", url, nil)
  1625  	if string(returnValue) != "[]" {
  1626  		t.Errorf("Expected [] return on querying empty label annotations, got %q\n", string(returnValue))
  1627  	}
  1628  	url = fmt.Sprintf("%snode/%s/mysynapses/label/1", server.WebAPIPath, uuid)
  1629  	returnValue = server.TestHTTP(t, "GET", url, nil)
  1630  	if string(returnValue) != "[]" {
  1631  		t.Errorf("Expected [] return on querying empty label annotations, got %q\n", string(returnValue))
  1632  	}
  1633  
  1634  	// PUT first batch of synapses
  1635  	testJSON, err := json.Marshal(testData)
  1636  	if err != nil {
  1637  		t.Fatal(err)
  1638  	}
  1639  	url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
  1640  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1641  
  1642  	// Test if labels were properly denormalized.  For the POST we have synchronized label denormalization.
  1643  	// If this were to become asynchronous, we'd want to block on updating like the labelblk<->labelvol sync.
  1644  
  1645  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1646  	testResponseLabel(t, expectedLabel2, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1647  	testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1648  	testResponseLabel(t, expectedLabel3NoRel, "%snode/%s/mysynapses/label/3", server.WebAPIPath, uuid)
  1649  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1650  
  1651  	// Make change to labelblk and make sure our label synapses have been adjusted (case A)
  1652  	_ = modifyLabelTestVolume(t, uuid, string(labelblkName))
  1653  
  1654  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1655  		t.Fatalf("Error blocking on sync of labels->annotations: %v\n", err)
  1656  	}
  1657  
  1658  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1659  	testResponseLabel(t, expectedLabel2a, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1660  	testResponseLabel(t, expectedLabel3a, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1661  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1662  	testResponseLabel(t, expectedLabel4NoRel, "%snode/%s/mysynapses/label/4", server.WebAPIPath, uuid)
  1663  
  1664  	// Make change to labelvol and make sure our label synapses have been adjusted (case B).
  1665  	// Merge 3a into 2a.
  1666  	testMerge := mergeJSON(`[2, 3]`)
  1667  	testMerge.send(t, uuid, string(labelvolName))
  1668  
  1669  	if err := datastore.BlockOnUpdating(uuid, labelblkName); err != nil {
  1670  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  1671  	}
  1672  
  1673  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1674  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1675  	}
  1676  
  1677  	if err := datastore.BlockOnUpdating(uuid, labelvolName); err != nil {
  1678  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
  1679  	}
  1680  
  1681  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1682  	testResponseLabel(t, expectedLabel2b, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1683  	testResponseLabel(t, nil, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1684  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1685  
  1686  	// Now split label 2b off and check if annotations also split
  1687  
  1688  	// Create the sparsevol encoding for split area
  1689  	numspans := len(bodysplit.voxelSpans)
  1690  	rles := make(dvid.RLEs, numspans, numspans)
  1691  	for i, span := range bodysplit.voxelSpans {
  1692  		start := dvid.Point3d{span[2], span[1], span[0]}
  1693  		length := span[3] - span[2] + 1
  1694  		rles[i] = dvid.NewRLE(start, length)
  1695  	}
  1696  	buf := getBytesRLE(t, rles)
  1697  
  1698  	// Submit the split sparsevol
  1699  	reqStr := fmt.Sprintf("%snode/%s/%s/split/2", server.WebAPIPath, uuid, labelvolName)
  1700  	r := server.TestHTTP(t, "POST", reqStr, buf)
  1701  	var jsonVal struct {
  1702  		Label uint64
  1703  	}
  1704  	if err := json.Unmarshal(r, &jsonVal); err != nil {
  1705  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
  1706  	}
  1707  	splitLabel := jsonVal.Label
  1708  	dvid.Infof("after split/2, split label = %d\n", splitLabel)
  1709  
  1710  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1711  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1712  	}
  1713  
  1714  	reqStr = fmt.Sprintf("%snode/%s/%s/supervoxels/%d", server.WebAPIPath, uuid, labelvolName, splitLabel)
  1715  	splitSupervoxelsJSON := server.TestHTTP(t, "GET", reqStr, nil)
  1716  
  1717  	// Verify that the annotations are correct.
  1718  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1719  		t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
  1720  	}
  1721  	testResponseLabel(t, expectedLabel2c, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1722  	url2 := fmt.Sprintf("%snode/%s/mysynapses/label/%d?relationships=true", server.WebAPIPath, uuid, splitLabel)
  1723  	testResponseLabel(t, expectedLabel7, url2)
  1724  
  1725  	// Change the name of the annotations.
  1726  	if err = datastore.RenameData(uuid, "mysynapses", labelvolName, "foobar"); err == nil {
  1727  		t.Fatalf("Should have been prevented from renaming data 'mysynapses' to existing instance!\n")
  1728  	}
  1729  	if err = datastore.RenameData(uuid, "mysynapses", "renamedData", "foobar"); err != nil {
  1730  		t.Fatalf("Error renaming annotations: %v\n", err)
  1731  	}
  1732  
  1733  	// Make sure the old name is no longer there and the new one is.
  1734  	server.TestBadHTTP(t, "GET", url2, nil)
  1735  	testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
  1736  
  1737  	testMerge = mergeJSON(fmt.Sprintf(`[2, %d]`, splitLabel))
  1738  	testMerge.send(t, uuid, string(labelvolName))
  1739  	if err := datastore.BlockOnUpdating(uuid, labelblkName); err != nil {
  1740  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  1741  	}
  1742  	if err := datastore.BlockOnUpdating(uuid, "renamedData"); err != nil {
  1743  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1744  	}
  1745  	testResponseLabel(t, expectedLabel2b, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
  1746  
  1747  	// Try a cleave
  1748  	reqStr = fmt.Sprintf("%snode/%s/%s/cleave/2", server.WebAPIPath, uuid, labelvolName)
  1749  	r = server.TestHTTP(t, "POST", reqStr, bytes.NewBuffer(splitSupervoxelsJSON))
  1750  	dvid.Infof("Sending supervoxels: %s\n", string(splitSupervoxelsJSON))
  1751  	var cleaveVal struct {
  1752  		CleavedLabel uint64
  1753  	}
  1754  	if err := json.Unmarshal(r, &cleaveVal); err != nil {
  1755  		t.Errorf("Unable to get new label from cleave.  Instead got: %v\n", cleaveVal)
  1756  	}
  1757  	dvid.Infof("after cleave/2, cleaved off label = %d\n", cleaveVal.CleavedLabel)
  1758  
  1759  	// Verify that the annotations are correct.
  1760  	if err := datastore.BlockOnUpdating(uuid, "renamedData"); err != nil {
  1761  		t.Fatalf("Error blocking on sync of split->annotations: %v\n", err)
  1762  	}
  1763  	testResponseLabel(t, expectedLabel7, "%snode/%s/renamedData/label/%d?relationships=true", server.WebAPIPath, uuid, cleaveVal.CleavedLabel)
  1764  	testResponseLabel(t, expectedLabel2c, "%snode/%s/renamedData/label/2?relationships=true", server.WebAPIPath, uuid)
  1765  
  1766  	// Delete a labeled annotation and make sure it's not in label
  1767  	delurl := fmt.Sprintf("%snode/%s/%s/element/20_30_40", server.WebAPIPath, uuid, "renamedData")
  1768  	server.TestHTTP(t, "DELETE", delurl, nil)
  1769  	testResponseLabel(t, afterDeleteOn7, "%snode/%s/%s/label/%d?relationships=true", server.WebAPIPath, uuid, "renamedData", cleaveVal.CleavedLabel)
  1770  }
  1771  
  1772  func TestOldLabels(t *testing.T) {
  1773  	if err := server.OpenTest(); err != nil {
  1774  		t.Fatalf("can't open test server: %v\n", err)
  1775  	}
  1776  	defer server.CloseTest()
  1777  
  1778  	// Create testbed volume and data instances
  1779  	uuid, _ := initTestRepo()
  1780  	var config dvid.Config
  1781  	server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
  1782  	server.CreateTestInstance(t, uuid, "labelvol", "bodies", config)
  1783  
  1784  	// Establish syncs
  1785  	server.CreateTestSync(t, uuid, "labels", "bodies")
  1786  	server.CreateTestSync(t, uuid, "bodies", "labels")
  1787  
  1788  	// Populate the labels, which should automatically populate the labelvol
  1789  	_ = createLabelTestVolume(t, uuid, "labels")
  1790  
  1791  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
  1792  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  1793  	}
  1794  
  1795  	// Add annotations syncing with "labels" instance checking for deduplication.
  1796  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  1797  	server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies,labels,bodies,labels,bodies")
  1798  	dataservice, err := datastore.GetDataByUUIDName(uuid, "mysynapses")
  1799  	if err != nil {
  1800  		t.Fatal(err)
  1801  	}
  1802  	data, ok := dataservice.(*Data)
  1803  	if !ok {
  1804  		t.Fatalf("Can't convert dataservice %v into datastore.Data\n", dataservice)
  1805  	}
  1806  	if len(data.SyncedData()) != 2 {
  1807  		t.Fatalf("Expected 2 syncs (uuids for labels and bodies], got %v\n", data.SyncedData())
  1808  	}
  1809  
  1810  	testLabels(t, uuid, "labels", "bodies")
  1811  }
  1812  
  1813  func TestLabels(t *testing.T) {
  1814  	if err := server.OpenTest(); err != nil {
  1815  		t.Fatalf("can't open test server: %v\n", err)
  1816  	}
  1817  	defer server.CloseTest()
  1818  
  1819  	// Create testbed volume and data instances
  1820  	uuid, _ := initTestRepo()
  1821  	var config dvid.Config
  1822  	config.Set("BlockSize", "32,32,32")
  1823  	server.CreateTestInstance(t, uuid, "labelarray", "labels", config)
  1824  
  1825  	// Populate the labels, which should automatically populate the labelvol
  1826  	_ = createLabelTestVolume(t, uuid, "labels")
  1827  
  1828  	// Add annotations syncing with "labels" instance checking for deduplication.
  1829  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  1830  	server.CreateTestSync(t, uuid, "mysynapses", "labels,labels,labels")
  1831  	dataservice, err := datastore.GetDataByUUIDName(uuid, "mysynapses")
  1832  	if err != nil {
  1833  		t.Fatal(err)
  1834  	}
  1835  	data, ok := dataservice.(*Data)
  1836  	if !ok {
  1837  		t.Fatalf("Can't convert dataservice %v into datastore.Data\n", dataservice)
  1838  	}
  1839  	if len(data.SyncedData()) != 1 {
  1840  		t.Fatalf("Expected 1 sync (uuid for labels), got %v\n", data.SyncedData())
  1841  	}
  1842  
  1843  	testLabels(t, uuid, "labels", "labels")
  1844  }
  1845  
  1846  func TestMappedLabels(t *testing.T) {
  1847  	if err := server.OpenTest(); err != nil {
  1848  		t.Fatalf("can't open test server: %v\n", err)
  1849  	}
  1850  	defer server.CloseTest()
  1851  
  1852  	// Create testbed volume and data instances
  1853  	uuid, _ := initTestRepo()
  1854  	var config dvid.Config
  1855  	config.Set("BlockSize", "32,32,32")
  1856  	server.CreateTestInstance(t, uuid, "labelmap", "mylabelmap", config)
  1857  
  1858  	_ = createLabelTestVolume(t, uuid, "mylabelmap")
  1859  
  1860  	// Add annotations syncing with "labels" instance checking for deduplication.
  1861  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  1862  	server.CreateTestSync(t, uuid, "mysynapses", "mylabelmap,mylabelmap,mylabelmap")
  1863  	dataservice, err := datastore.GetDataByUUIDName(uuid, "mysynapses")
  1864  	if err != nil {
  1865  		t.Fatal(err)
  1866  	}
  1867  	data, ok := dataservice.(*Data)
  1868  	if !ok {
  1869  		t.Fatalf("Can't convert dataservice %v into datastore.Data\n", dataservice)
  1870  	}
  1871  	if len(data.SyncedData()) != 1 {
  1872  		t.Fatalf("Expected 1 sync (uuid for labels), got %v\n", data.SyncedData())
  1873  	}
  1874  
  1875  	testMappedLabels(t, uuid, "mylabelmap", "mylabelmap")
  1876  }
  1877  
  1878  func TestSupervoxelSplit(t *testing.T) {
  1879  	if err := server.OpenTest(); err != nil {
  1880  		t.Fatalf("can't open test server: %v\n", err)
  1881  	}
  1882  	defer server.CloseTest()
  1883  
  1884  	// Create testbed volume and data instances
  1885  	uuid, _ := initTestRepo()
  1886  	var config dvid.Config
  1887  	config.Set("BlockSize", "32,32,32")
  1888  
  1889  	labelName := "mylabelmap"
  1890  	server.CreateTestInstance(t, uuid, "labelmap", labelName, config)
  1891  
  1892  	_ = createLabelTestVolume(t, uuid, labelName)
  1893  
  1894  	// Add annotations syncing with "labels" instance checking for deduplication.
  1895  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  1896  	server.CreateTestSync(t, uuid, "mysynapses", "mylabelmap,mylabelmap,mylabelmap")
  1897  	dataservice, err := datastore.GetDataByUUIDName(uuid, "mysynapses")
  1898  	if err != nil {
  1899  		t.Fatal(err)
  1900  	}
  1901  	data, ok := dataservice.(*Data)
  1902  	if !ok {
  1903  		t.Fatalf("Can't convert dataservice %v into datastore.Data\n", dataservice)
  1904  	}
  1905  	if len(data.SyncedData()) != 1 {
  1906  		t.Fatalf("Expected 1 sync (uuid for labels), got %v\n", data.SyncedData())
  1907  	}
  1908  
  1909  	testJSON, err := json.Marshal(testData)
  1910  	if err != nil {
  1911  		t.Fatal(err)
  1912  	}
  1913  	url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
  1914  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  1915  
  1916  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1917  	testResponseLabel(t, expectedLabel2, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1918  	testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1919  	testResponseLabel(t, expectedLabel3NoRel, "%snode/%s/mysynapses/label/3", server.WebAPIPath, uuid)
  1920  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1921  
  1922  	// Create the sparsevol encoding for split area
  1923  	numspans := len(svsplit.voxelSpans)
  1924  	rles := make(dvid.RLEs, numspans, numspans)
  1925  	for i, span := range svsplit.voxelSpans {
  1926  		start := dvid.Point3d{span[2], span[1], span[0]}
  1927  		length := span[3] - span[2] + 1
  1928  		rles[i] = dvid.NewRLE(start, length)
  1929  	}
  1930  	buf := getBytesRLE(t, rles)
  1931  
  1932  	if err := datastore.BlockOnUpdating(uuid, dvid.InstanceName(labelName)); err != nil {
  1933  		t.Fatalf("Error blocking on labels updating: %v\n", err)
  1934  	}
  1935  
  1936  	// Submit the supervoxel split
  1937  	reqStr := fmt.Sprintf("%snode/%s/%s/split-supervoxel/3", server.WebAPIPath, uuid, labelName)
  1938  	r := server.TestHTTP(t, "POST", reqStr, buf)
  1939  	var jsonVal struct {
  1940  		SplitSupervoxel  uint64
  1941  		RemainSupervoxel uint64
  1942  	}
  1943  	if err := json.Unmarshal(r, &jsonVal); err != nil {
  1944  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
  1945  	}
  1946  	dvid.Infof("after supervoxel split on supervoxel 3 -> split %d, remain %d\n", jsonVal.SplitSupervoxel, jsonVal.RemainSupervoxel)
  1947  
  1948  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1949  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1950  	}
  1951  
  1952  	testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1953  }
  1954  
  1955  func testLabelsReload(t *testing.T, uuid dvid.UUID, labelblkName, labelvolName dvid.InstanceName) {
  1956  	// Test if labels were properly denormalized.  For the POST we have synchronized label denormalization.
  1957  
  1958  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1959  	testResponseLabel(t, expectedLabel2, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1960  	testResponseLabel(t, expectedLabel3, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1961  	testResponseLabel(t, expectedLabel3NoRel, "%snode/%s/mysynapses/label/3", server.WebAPIPath, uuid)
  1962  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1963  
  1964  	// Make change to labelblk and make sure our label synapses have been adjusted (case A)
  1965  	_ = modifyLabelTestVolume(t, uuid, string(labelblkName))
  1966  
  1967  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1968  		t.Fatalf("Error blocking on sync of labels->annotations: %v\n", err)
  1969  	}
  1970  
  1971  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1972  	testResponseLabel(t, expectedLabel2a, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1973  	testResponseLabel(t, expectedLabel3a, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1974  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1975  	testResponseLabel(t, expectedLabel4NoRel, "%snode/%s/mysynapses/label/4", server.WebAPIPath, uuid)
  1976  
  1977  	// Make change to labelvol and make sure our label synapses have been adjusted (case B).
  1978  	// Merge 3a into 2a.
  1979  	testMerge := mergeJSON(`[2, 3]`)
  1980  	testMerge.send(t, uuid, string(labelvolName))
  1981  
  1982  	if err := datastore.BlockOnUpdating(uuid, labelblkName); err != nil {
  1983  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  1984  	}
  1985  
  1986  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
  1987  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
  1988  	}
  1989  
  1990  	if err := datastore.BlockOnUpdating(uuid, labelvolName); err != nil {
  1991  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
  1992  	}
  1993  
  1994  	testResponseLabel(t, expectedLabel1, "%snode/%s/mysynapses/label/1?relationships=true", server.WebAPIPath, uuid)
  1995  	testResponseLabel(t, expectedLabel2b, "%snode/%s/mysynapses/label/2?relationships=true", server.WebAPIPath, uuid)
  1996  	testResponseLabel(t, nil, "%snode/%s/mysynapses/label/3?relationships=true", server.WebAPIPath, uuid)
  1997  	testResponseLabel(t, expectedLabel4, "%snode/%s/mysynapses/label/4?relationships=true", server.WebAPIPath, uuid)
  1998  }
  1999  
  2000  func TestLabelblkReload(t *testing.T) {
  2001  	if err := server.OpenTest(); err != nil {
  2002  		t.Fatalf("can't open test server: %v\n", err)
  2003  	}
  2004  	defer server.CloseTest()
  2005  
  2006  	// Create testbed volume and data instances
  2007  	uuid, v := initTestRepo()
  2008  	var config dvid.Config
  2009  	config.Set("BlockSize", "64,64,64")
  2010  	server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
  2011  	server.CreateTestInstance(t, uuid, "labelvol", "bodies", config)
  2012  
  2013  	// Establish syncs
  2014  	server.CreateTestSync(t, uuid, "labels", "bodies")
  2015  	server.CreateTestSync(t, uuid, "bodies", "labels")
  2016  
  2017  	// Populate the labels, which should automatically populate the labelvol
  2018  	_ = createLabelTestVolume(t, uuid, "labels")
  2019  
  2020  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
  2021  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  2022  	}
  2023  
  2024  	// Add annotations without syncing.
  2025  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  2026  
  2027  	// PUT first batch of synapses
  2028  	testJSON, err := json.Marshal(testData)
  2029  	if err != nil {
  2030  		t.Fatal(err)
  2031  	}
  2032  	url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
  2033  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  2034  
  2035  	// Add the sync
  2036  	server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies")
  2037  
  2038  	// Do a reload synchronously and check
  2039  	d, err := GetByUUIDName(uuid, "mysynapses")
  2040  	if err != nil {
  2041  		t.Fatal(err)
  2042  	}
  2043  	ctx := datastore.NewVersionedCtx(d, v)
  2044  	d.resyncInMemory(ctx, true)
  2045  
  2046  	testLabelsReload(t, uuid, "labels", "bodies")
  2047  }
  2048  
  2049  func testLabelReload(t *testing.T, typename string, inMemory bool) {
  2050  	if err := server.OpenTest(); err != nil {
  2051  		t.Fatalf("can't open test server: %v\n", err)
  2052  	}
  2053  	defer server.CloseTest()
  2054  
  2055  	// Create testbed volume and data instances
  2056  	uuid, v := initTestRepo()
  2057  	var config dvid.Config
  2058  	// config.Set("BlockSize", "32,32,32")
  2059  	config.Set("BlockSize", "64,64,64")
  2060  	server.CreateTestInstance(t, uuid, typename, "labels", config)
  2061  
  2062  	_ = createLabelTestVolume(t, uuid, "labels")
  2063  
  2064  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
  2065  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
  2066  	}
  2067  
  2068  	// Add annotations without syncing.
  2069  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
  2070  
  2071  	// PUT first batch of synapses
  2072  	testJSON, err := json.Marshal(testData)
  2073  	if err != nil {
  2074  		t.Fatal(err)
  2075  	}
  2076  	url1 := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
  2077  	server.TestHTTP(t, "POST", url1, strings.NewReader(string(testJSON)))
  2078  
  2079  	// Add the sync
  2080  	server.CreateTestSync(t, uuid, "mysynapses", "labels")
  2081  
  2082  	// Do a reload synchronously and check
  2083  	d, err := GetByUUIDName(uuid, "mysynapses")
  2084  	if err != nil {
  2085  		t.Fatal(err)
  2086  	}
  2087  	ctx := datastore.NewVersionedCtx(d, v)
  2088  	if inMemory {
  2089  		d.resyncInMemory(ctx, true)
  2090  	} else {
  2091  		d.resyncLowMemory(ctx)
  2092  	}
  2093  	testLabelsReload(t, uuid, "labels", "labels")
  2094  }
  2095  
  2096  func TestLabelarrayReload(t *testing.T) {
  2097  	testLabelReload(t, "labelarray", true)
  2098  	testLabelReload(t, "labelarray", false)
  2099  }
  2100  
  2101  func TestLabelmapReload(t *testing.T) {
  2102  	testLabelReload(t, "labelmap", true)
  2103  	testLabelReload(t, "labelmap", false)
  2104  }
  2105  
  2106  // A single label block within the volume
  2107  type testBody struct {
  2108  	label        uint64
  2109  	offset, size dvid.Point3d
  2110  	blockSpans   dvid.Spans
  2111  	voxelSpans   dvid.Spans
  2112  }
  2113  
  2114  // A slice of bytes representing 3d label volume
  2115  type testVolume struct {
  2116  	data []byte
  2117  	size dvid.Point3d
  2118  }
  2119  
  2120  func newTestVolume(nx, ny, nz int32) *testVolume {
  2121  	return &testVolume{
  2122  		data: make([]byte, nx*ny*nz*8),
  2123  		size: dvid.Point3d{nx, ny, nz},
  2124  	}
  2125  }
  2126  
  2127  // Sets voxels in body to given label.
  2128  func (v *testVolume) add(body testBody, label uint64) {
  2129  	nx := v.size[0]
  2130  	nxy := nx * v.size[1]
  2131  	for _, span := range body.voxelSpans {
  2132  		z, y, x0, x1 := span.Unpack()
  2133  		p := (z*nxy + y*nx) * 8
  2134  		for i := p + x0*8; i <= p+x1*8; i += 8 {
  2135  			binary.LittleEndian.PutUint64(v.data[i:i+8], label)
  2136  		}
  2137  	}
  2138  }
  2139  
  2140  // Put label data into given data instance.
  2141  func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) {
  2142  	apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/0_0_0?mutate=true", server.WebAPIPath,
  2143  		uuid, name, v.size[0], v.size[1], v.size[2])
  2144  	server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data))
  2145  }
  2146  
  2147  func createLabelTestVolume(t *testing.T, uuid dvid.UUID, name string) *testVolume {
  2148  	volume := newTestVolume(128, 128, 128)
  2149  	volume.add(body1, 1)
  2150  	volume.add(body2, 2)
  2151  	volume.add(body3, 3)
  2152  	volume.add(body4, 4)
  2153  
  2154  	// Send data over HTTP to populate a data instance
  2155  	volume.put(t, uuid, name)
  2156  	return volume
  2157  }
  2158  
  2159  func modifyLabelTestVolume(t *testing.T, uuid dvid.UUID, name string) *testVolume {
  2160  	volume := newTestVolume(128, 128, 128)
  2161  	volume.add(body1, 1)
  2162  	volume.add(body2a, 2)
  2163  	volume.add(body3a, 3)
  2164  	volume.add(body4, 4)
  2165  
  2166  	// Send data over HTTP to populate a data instance
  2167  	volume.put(t, uuid, name)
  2168  	return volume
  2169  }
  2170  
  2171  type mergeJSON string
  2172  
  2173  func (mjson mergeJSON) send(t *testing.T, uuid dvid.UUID, name string) {
  2174  	apiStr := fmt.Sprintf("%snode/%s/%s/merge", server.WebAPIPath, uuid, name)
  2175  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(string(mjson)))
  2176  }
  2177  
  2178  var (
  2179  	bodies = []testBody{
  2180  		{
  2181  			label:  1,
  2182  			offset: dvid.Point3d{10, 10, 30},
  2183  			size:   dvid.Point3d{20, 20, 10},
  2184  			blockSpans: []dvid.Span{
  2185  				{1, 0, 0, 0},
  2186  			},
  2187  			voxelSpans: []dvid.Span{
  2188  				{35, 27, 11, 28}, {36, 28, 13, 25},
  2189  			},
  2190  		}, {
  2191  			label:  2,
  2192  			offset: dvid.Point3d{10, 25, 35},
  2193  			size:   dvid.Point3d{30, 10, 10},
  2194  			blockSpans: []dvid.Span{
  2195  				{1, 0, 0, 0},
  2196  			},
  2197  			voxelSpans: []dvid.Span{
  2198  				{40, 30, 12, 20},
  2199  			},
  2200  		}, {
  2201  			label:  3,
  2202  			offset: dvid.Point3d{10, 20, 36},
  2203  			size:   dvid.Point3d{120, 45, 65},
  2204  			blockSpans: []dvid.Span{
  2205  				{1, 0, 0, 0},
  2206  				{3, 2, 4, 4},
  2207  			},
  2208  			voxelSpans: []dvid.Span{
  2209  				{37, 25, 13, 15},
  2210  				{99, 63, 126, 127},
  2211  			},
  2212  		}, {
  2213  			label:  4,
  2214  			offset: dvid.Point3d{75, 40, 75},
  2215  			size:   dvid.Point3d{20, 10, 10},
  2216  			blockSpans: []dvid.Span{
  2217  				{2, 1, 2, 2},
  2218  			},
  2219  			voxelSpans: []dvid.Span{
  2220  				{80, 47, 87, 89},
  2221  			},
  2222  		}, { // Modification to original label 2 body where we switch a span that was in label 3 and enlarge
  2223  			label:  2,
  2224  			offset: dvid.Point3d{10, 24, 35},
  2225  			size:   dvid.Point3d{30, 10, 10},
  2226  			blockSpans: []dvid.Span{
  2227  				{0, 0, 0, 0},
  2228  				{1, 0, 0, 0},
  2229  			},
  2230  			voxelSpans: []dvid.Span{
  2231  				{12, 8, 0, 10},
  2232  				{37, 25, 8, 15},
  2233  			},
  2234  		}, { // Modification to original label 3 body where we switch in a span that was in label 2
  2235  			label:  3,
  2236  			offset: dvid.Point3d{10, 20, 36},
  2237  			size:   dvid.Point3d{120, 45, 65},
  2238  			blockSpans: []dvid.Span{
  2239  				{1, 0, 0, 0},
  2240  				{3, 2, 4, 4},
  2241  			},
  2242  			voxelSpans: []dvid.Span{
  2243  				{40, 30, 12, 20}, {99, 63, 126, 127},
  2244  			},
  2245  		}, { // Split volume
  2246  			label:  7,
  2247  			offset: dvid.Point3d{10, 24, 36},
  2248  			size:   dvid.Point3d{12, 7, 5},
  2249  			blockSpans: []dvid.Span{
  2250  				{0, 0, 0, 0},
  2251  				{1, 0, 0, 0},
  2252  			},
  2253  			voxelSpans: []dvid.Span{
  2254  				{12, 8, 4, 8},
  2255  				{37, 25, 8, 10},
  2256  				{37, 25, 13, 15},
  2257  				{40, 30, 19, 21},
  2258  			},
  2259  		}, { // split supervoxel volume
  2260  			label:  7,
  2261  			offset: dvid.Point3d{10, 20, 36},
  2262  			size:   dvid.Point3d{12, 7, 5},
  2263  			blockSpans: []dvid.Span{
  2264  				{1, 0, 0, 0},
  2265  				{3, 2, 4, 4},
  2266  			},
  2267  			voxelSpans: []dvid.Span{
  2268  				{37, 25, 14, 14},
  2269  				{99, 63, 126, 126},
  2270  			},
  2271  		},
  2272  	}
  2273  	body1     = bodies[0]
  2274  	body2     = bodies[1]
  2275  	body3     = bodies[2]
  2276  	body4     = bodies[3]
  2277  	body2a    = bodies[4]
  2278  	body3a    = bodies[5]
  2279  	bodysplit = bodies[6]
  2280  	svsplit   = bodies[7]
  2281  )