github.com/janelia-flyem/dvid@v1.0.0/datatype/labelsz/labelsz_test.go (about)

     1  package labelsz
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/binary"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  	"log"
    10  	"math/rand"
    11  	"strconv"
    12  	"strings"
    13  	"testing"
    14  	"time"
    15  
    16  	"github.com/janelia-flyem/dvid/datastore"
    17  	"github.com/janelia-flyem/dvid/datatype/annotation"
    18  	"github.com/janelia-flyem/dvid/dvid"
    19  	"github.com/janelia-flyem/dvid/server"
    20  )
    21  
    22  // A slice of bytes representing 3d label volume
    23  type testVolume struct {
    24  	data []byte
    25  	size dvid.Point3d
    26  }
    27  
    28  func newTestVolume(nx, ny, nz int32) *testVolume {
    29  	return &testVolume{
    30  		data: make([]byte, nx*ny*nz*8),
    31  		size: dvid.Point3d{nx, ny, nz},
    32  	}
    33  }
    34  
    35  // Sets voxels in body to given label.
    36  func (v *testVolume) add(label uint64, ox, oy, oz int32, sx, sy, sz int32) {
    37  	nx := v.size.Value(0)
    38  	ny := v.size.Value(1)
    39  	nxy := nx * ny
    40  	for z := oz; z < oz+sz; z++ {
    41  		for y := oy; y < oy+sy; y++ {
    42  			p := (z*nxy + y*nx + ox) * 8
    43  			for x := ox; x < ox+sx; x++ {
    44  				binary.LittleEndian.PutUint64(v.data[p:p+8], label)
    45  				p += 8
    46  			}
    47  		}
    48  	}
    49  }
    50  
    51  // Put label data into given data instance.
    52  func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) {
    53  	apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/0_0_0?mutate=true", server.WebAPIPath,
    54  		uuid, name, v.size[0], v.size[1], v.size[2])
    55  	server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data))
    56  }
    57  
    58  func createLabelTestVolume(t *testing.T, uuid dvid.UUID, name string) *testVolume {
    59  	volume := newTestVolume(128, 128, 128)
    60  	volume.add(100, 0, 0, 0, 64, 128, 128)
    61  	volume.add(200, 64, 0, 0, 64, 128, 64)
    62  	volume.add(300, 64, 0, 64, 64, 128, 64)
    63  
    64  	// Send data over HTTP to populate a data instance
    65  	volume.put(t, uuid, name)
    66  	return volume
    67  }
    68  
    69  // test ROI has offset (32, 32, 32) and size (64, 64, 64)
    70  var testSpans = []dvid.Span{
    71  	dvid.Span{1, 1, 1, 2}, dvid.Span{1, 2, 1, 2},
    72  	dvid.Span{2, 1, 1, 2}, dvid.Span{2, 2, 1, 2},
    73  }
    74  
    75  func getROIReader() io.Reader {
    76  	jsonBytes, err := json.Marshal(testSpans)
    77  	if err != nil {
    78  		log.Fatalf("Can't encode spans into JSON: %v\n", err)
    79  	}
    80  	return bytes.NewReader(jsonBytes)
    81  }
    82  
    83  type mergeJSON string
    84  
    85  func (mjson mergeJSON) send(t *testing.T, uuid dvid.UUID, name string) {
    86  	apiStr := fmt.Sprintf("%snode/%s/%s/merge", server.WebAPIPath, uuid, name)
    87  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(string(mjson)))
    88  }
    89  
    90  func getBytesRLE(t *testing.T, rles dvid.RLEs) *bytes.Buffer {
    91  	n := len(rles)
    92  	buf := new(bytes.Buffer)
    93  	buf.WriteByte(dvid.EncodingBinary)
    94  	binary.Write(buf, binary.LittleEndian, uint8(3))  // # of dimensions
    95  	binary.Write(buf, binary.LittleEndian, byte(0))   // dimension of run (X = 0)
    96  	buf.WriteByte(byte(0))                            // reserved for later
    97  	binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels
    98  	binary.Write(buf, binary.LittleEndian, uint32(n)) // Placeholder for # spans
    99  	rleBytes, err := rles.MarshalBinary()
   100  	if err != nil {
   101  		t.Errorf("Unable to serialize RLEs: %v\n", err)
   102  	}
   103  	buf.Write(rleBytes)
   104  	return buf
   105  }
   106  
   107  func checkSequencing(t *testing.T, uuid dvid.UUID) {
   108  	// Check if we have correct sequencing for no ROI labelsz.
   109  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   110  		t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err)
   111  	}
   112  
   113  	url := fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid)
   114  	data := server.TestHTTP(t, "GET", url, nil)
   115  	if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192},{"Label":300,"Size":8192}]` {
   116  		t.Errorf("Got back incorrect PreSyn noroi ranking:\n%v\n", string(data))
   117  	}
   118  
   119  	url = fmt.Sprintf("%snode/%s/noroi/count/100/PreSyn", server.WebAPIPath, uuid)
   120  	data = server.TestHTTP(t, "GET", url, nil)
   121  	if string(data) != `{"Label":100,"PreSyn":16384}` {
   122  		t.Errorf("Got back incorrect PreSyn noroi count for label 100:\n%v\n", string(data))
   123  	}
   124  
   125  	url = fmt.Sprintf("%snode/%s/noroi/count/200/PreSyn", server.WebAPIPath, uuid)
   126  	data = server.TestHTTP(t, "GET", url, nil)
   127  	if string(data) != `{"Label":200,"PreSyn":8192}` {
   128  		t.Errorf("Got back incorrect PreSyn noroi count for label 200:\n%v\n", string(data))
   129  	}
   130  
   131  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid)
   132  	data = server.TestHTTP(t, "GET", url, nil)
   133  	if string(data) != `[{"Label":100,"Size":14415},{"Label":300,"Size":7936},{"Label":200,"Size":7440}]` {
   134  		t.Errorf("Got back incorrect PostSyn noroi ranking:\n%v\n", string(data))
   135  	}
   136  
   137  	url = fmt.Sprintf("%snode/%s/noroi/top/3/AllSyn", server.WebAPIPath, uuid)
   138  	data = server.TestHTTP(t, "GET", url, nil)
   139  	if string(data) != `[{"Label":100,"Size":30799},{"Label":300,"Size":16128},{"Label":200,"Size":15632}]` {
   140  		t.Errorf("Got back incorrect AllSync noroi ranking:\n%v\n", string(data))
   141  	}
   142  
   143  	url = fmt.Sprintf("%snode/%s/noroi/threshold/15633/AllSyn", server.WebAPIPath, uuid)
   144  	data = server.TestHTTP(t, "GET", url, nil)
   145  	if string(data) != `[{"Label":100,"Size":30799},{"Label":300,"Size":16128}]` {
   146  		t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data))
   147  	}
   148  
   149  	url = fmt.Sprintf("%snode/%s/noroi/threshold/1000/AllSyn?offset=1&n=2", server.WebAPIPath, uuid)
   150  	data = server.TestHTTP(t, "GET", url, nil)
   151  	if string(data) != `[{"Label":300,"Size":16128},{"Label":200,"Size":15632}]` {
   152  		t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data))
   153  	}
   154  
   155  	url = fmt.Sprintf("%snode/%s/noroi/threshold/1000/AllSyn?offset=8&n=2", server.WebAPIPath, uuid)
   156  	data = server.TestHTTP(t, "GET", url, nil)
   157  	if string(data) != `[]` {
   158  		t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data))
   159  	}
   160  
   161  	// Check if we have correct sequencing for ROI labelsz.
   162  	// ROI constitutes the inner eight 32^3 blocks.
   163  	// There are 16 PostSyn in each ROI dimension.
   164  	// There are also 16 PreSyn in each ROI dimension.
   165  	if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil {
   166  		t.Fatalf("Error blocking on sync of withroi labelsz: %v\n", err)
   167  	}
   168  
   169  	url = fmt.Sprintf("%snode/%s/withroi/top/0/AllSyn", server.WebAPIPath, uuid)
   170  	data = server.TestHTTP(t, "GET", url, nil)
   171  	if string(data) != `[]` {
   172  		t.Errorf("Incorrectly handled top n=0 case, expected [] got: %v\n", string(data))
   173  	}
   174  
   175  	url = fmt.Sprintf("%snode/%s/withroi/top/3/PreSyn", server.WebAPIPath, uuid)
   176  	data = server.TestHTTP(t, "GET", url, nil)
   177  	if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` {
   178  		t.Errorf("Got back incorrect PreSyn withroi ranking:\n%v\n", string(data))
   179  	}
   180  
   181  	url = fmt.Sprintf("%snode/%s/withroi/top/3/PostSyn", server.WebAPIPath, uuid)
   182  	data = server.TestHTTP(t, "GET", url, nil)
   183  	if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` {
   184  		t.Errorf("Got back incorrect PostSyn withroi ranking:\n%v\n", string(data))
   185  	}
   186  
   187  	// Check fewer and larger N requests.
   188  	url = fmt.Sprintf("%snode/%s/noroi/top/2/PreSyn", server.WebAPIPath, uuid)
   189  	data = server.TestHTTP(t, "GET", url, nil)
   190  	if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192}]` {
   191  		t.Errorf("Got back incorrect N=2 ranking:\n%v\n", string(data))
   192  	}
   193  
   194  	url = fmt.Sprintf("%snode/%s/noroi/top/4/PreSyn", server.WebAPIPath, uuid)
   195  	data = server.TestHTTP(t, "GET", url, nil)
   196  	if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192},{"Label":300,"Size":8192}]` {
   197  		t.Errorf("Got back incorrect N=4 ranking:\n%v\n", string(data))
   198  	}
   199  
   200  	// Test annotation move of a PostSyn from label 100->300 and also label 200->300
   201  	url = fmt.Sprintf("%snode/%s/mysynapses/move/32_32_32/75_21_69", server.WebAPIPath, uuid)
   202  	server.TestHTTP(t, "POST", url, nil)
   203  
   204  	url = fmt.Sprintf("%snode/%s/mysynapses/move/68_20_20/77_21_69", server.WebAPIPath, uuid)
   205  	server.TestHTTP(t, "POST", url, nil)
   206  
   207  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   208  		t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err)
   209  	}
   210  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid)
   211  	data = server.TestHTTP(t, "GET", url, nil)
   212  	if string(data) != `[{"Label":100,"Size":14414},{"Label":300,"Size":7938},{"Label":200,"Size":7439}]` {
   213  		t.Errorf("Got back incorrect PostSyn noroi ranking after move from label 100->300:\n%v\n", string(data))
   214  	}
   215  
   216  	// First move took synapse out of ROI so there should be one less for label 100.
   217  	if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil {
   218  		t.Fatalf("Error blocking on sync of labelsz: %v\n", err)
   219  	}
   220  
   221  	url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid)
   222  	data = server.TestHTTP(t, "GET", url, nil)
   223  	if string(data) != `[{"Label":100,"Size":2047},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` {
   224  		t.Errorf("Got back incorrect post-move PostSyn withroi ranking:\n%v\n", string(data))
   225  	}
   226  
   227  	// Test annotation deletion of moved PostSyn from label 300
   228  	url = fmt.Sprintf("%snode/%s/mysynapses/element/75_21_69", server.WebAPIPath, uuid)
   229  	server.TestHTTP(t, "DELETE", url, nil)
   230  
   231  	url = fmt.Sprintf("%snode/%s/mysynapses/element/77_21_69", server.WebAPIPath, uuid)
   232  	server.TestHTTP(t, "DELETE", url, nil)
   233  
   234  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   235  		t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err)
   236  	}
   237  
   238  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid)
   239  	data = server.TestHTTP(t, "GET", url, nil)
   240  	if string(data) != `[{"Label":100,"Size":14414},{"Label":300,"Size":7936},{"Label":200,"Size":7439}]` {
   241  		t.Errorf("Got back incorrect PostSyn noroi ranking after deletions from label 300:\n%v\n", string(data))
   242  	}
   243  
   244  	// Check sync on merge.
   245  	if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil {
   246  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
   247  	}
   248  	testMerge := mergeJSON(`[200, 300]`)
   249  	testMerge.send(t, uuid, "bodies")
   250  
   251  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   252  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   253  	}
   254  	time.Sleep(1 * time.Second)
   255  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
   256  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
   257  	}
   258  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   259  		t.Fatalf("Error blocking on sync of labelsz: %v\n", err)
   260  	}
   261  	if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil {
   262  		t.Fatalf("Error blocking on sync of labelsz: %v\n", err)
   263  	}
   264  
   265  	url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid)
   266  	data = server.TestHTTP(t, "GET", url, nil)
   267  	if string(data) != `[{"Label":200,"Size":2048},{"Label":100,"Size":2047}]` {
   268  		t.Errorf("Got back incorrect post-merge PostSyn withroi ranking:\n%v\n", string(data))
   269  	}
   270  
   271  	url = fmt.Sprintf("%snode/%s/withroi/count/100/PostSyn", server.WebAPIPath, uuid)
   272  	data = server.TestHTTP(t, "GET", url, nil)
   273  	if string(data) != `{"Label":100,"PostSyn":2047}` {
   274  		t.Errorf("Got back incorrect post-merge PostSyn withroi count of label 100:\n%v\n", string(data))
   275  	}
   276  
   277  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid)
   278  	data = server.TestHTTP(t, "GET", url, nil)
   279  	if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":16384}]` {
   280  		t.Errorf("Got back incorrect post-merge PreSyn noroi ranking:\n%v\n", string(data))
   281  	}
   282  
   283  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid)
   284  	data = server.TestHTTP(t, "GET", url, nil)
   285  	if string(data) != `[{"Label":200,"Size":15375},{"Label":100,"Size":14414}]` {
   286  		t.Errorf("Got back incorrect post-merge PostSyn noroi ranking:\n%v\n", string(data))
   287  	}
   288  
   289  	url = fmt.Sprintf("%snode/%s/noroi/top/3/AllSyn", server.WebAPIPath, uuid)
   290  	data = server.TestHTTP(t, "GET", url, nil)
   291  	if string(data) != `[{"Label":200,"Size":31759},{"Label":100,"Size":30798}]` {
   292  		t.Errorf("Got back incorrect post-merge AllSyn noroi ranking:\n%v\n", string(data))
   293  	}
   294  
   295  	// Check threshold endpoint
   296  
   297  	url = fmt.Sprintf("%snode/%s/withroi/threshold/2048/PostSyn", server.WebAPIPath, uuid)
   298  	data = server.TestHTTP(t, "GET", url, nil)
   299  	if string(data) != `[{"Label":200,"Size":2048}]` {
   300  		t.Errorf("Got back incorrect post-merge PostSyn withroi threshold:\n%v\n", string(data))
   301  	}
   302  
   303  	url = fmt.Sprintf("%snode/%s/noroi/threshold/16384/PreSyn", server.WebAPIPath, uuid)
   304  	data = server.TestHTTP(t, "GET", url, nil)
   305  	if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":16384}]` {
   306  		t.Errorf("Got back incorrect post-merge PreSyn noroi threshold:\n%v\n", string(data))
   307  	}
   308  
   309  	url = fmt.Sprintf("%snode/%s/noroi/threshold/15000/PostSyn", server.WebAPIPath, uuid)
   310  	data = server.TestHTTP(t, "GET", url, nil)
   311  	if string(data) != `[{"Label":200,"Size":15375}]` {
   312  		t.Errorf("Got back incorrect post-merge PostSyn noroi threshold:\n%v\n", string(data))
   313  	}
   314  
   315  	url = fmt.Sprintf("%snode/%s/noroi/threshold/0/PostSyn?offset=1&n=1", server.WebAPIPath, uuid)
   316  	data = server.TestHTTP(t, "GET", url, nil)
   317  	if string(data) != `[{"Label":100,"Size":14414}]` {
   318  		t.Errorf("Got back incorrect post-merge PostSyn noroi threshold with offset/n:\n%v\n", string(data))
   319  	}
   320  
   321  	// Create the sparsevol encoding for split area with label 100 -> 150.
   322  	// Split has offset (0, 0, 0) with size (19, 19, 19).
   323  	// PreSyn in split = 5 x 5 x 5 = 125
   324  	// PostSyn in split = 4 x 4 x 4 = 64
   325  	var rles dvid.RLEs
   326  	for z := int32(0); z < 19; z++ {
   327  		for y := int32(0); y < 19; y++ {
   328  			start := dvid.Point3d{0, y, z}
   329  			rles = append(rles, dvid.NewRLE(start, 19))
   330  		}
   331  	}
   332  	buf := getBytesRLE(t, rles)
   333  
   334  	// Submit the split sparsevol
   335  	url = fmt.Sprintf("%snode/%s/%s/split/%d?splitlabel=150", server.WebAPIPath, uuid, "bodies", 100)
   336  	data = server.TestHTTP(t, "POST", url, buf)
   337  	jsonVal := make(map[string]uint64)
   338  	if err := json.Unmarshal(data, &jsonVal); err != nil {
   339  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
   340  	}
   341  
   342  	// Check sync on split.
   343  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   344  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   345  	}
   346  	time.Sleep(1 * time.Second)
   347  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
   348  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
   349  	}
   350  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   351  		t.Fatalf("Error blocking on sync of labelsz: %v\n", err)
   352  	}
   353  
   354  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid)
   355  	data = server.TestHTTP(t, "GET", url, nil)
   356  	if string(data) != `[{"Label":200,"Size":16384},{"Label":100,"Size":16259},{"Label":150,"Size":125}]` {
   357  		t.Errorf("Got back incorrect post-split PreSyn noroi ranking:\n%v\n", string(data))
   358  	}
   359  
   360  	url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid)
   361  	data = server.TestHTTP(t, "GET", url, nil)
   362  	if string(data) != `[{"Label":200,"Size":15375},{"Label":100,"Size":14350},{"Label":150,"Size":64}]` {
   363  		t.Errorf("Got back incorrect post-split PostSyn noroi ranking:\n%v\n", string(data))
   364  	}
   365  
   366  	// Create the encoding for coarse split area in block coordinates from label 200.
   367  	// Split has offset (64, 96, 96) with size (64, 32, 32).
   368  	// PreSyn in split = 16 x 8 x 8 = 1024
   369  	// PostSyn in split = 16 x 8 x 8 = 1024
   370  	rles = dvid.RLEs{
   371  		dvid.NewRLE(dvid.Point3d{2, 3, 3}, 2),
   372  	}
   373  	buf = getBytesRLE(t, rles)
   374  
   375  	// Submit the coarse split of 200 -> 250
   376  	url = fmt.Sprintf("%snode/%s/%s/split-coarse/200?splitlabel=250", server.WebAPIPath, uuid, "bodies")
   377  	data = server.TestHTTP(t, "POST", url, buf)
   378  	jsonVal = make(map[string]uint64)
   379  	if err := json.Unmarshal(data, &jsonVal); err != nil {
   380  		t.Errorf("Unable to get new label from split.  Instead got: %v\n", jsonVal)
   381  	}
   382  
   383  	// Check sync on split.
   384  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   385  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   386  	}
   387  	time.Sleep(1 * time.Second)
   388  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
   389  		t.Fatalf("Error blocking on sync of synapses: %v\n", err)
   390  	}
   391  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   392  		t.Fatalf("Error blocking on sync of labelsz: %v\n", err)
   393  	}
   394  
   395  	url = fmt.Sprintf("%snode/%s/noroi/top/5/PreSyn", server.WebAPIPath, uuid)
   396  	data = server.TestHTTP(t, "GET", url, nil)
   397  	if string(data) != `[{"Label":100,"Size":16259},{"Label":200,"Size":15360},{"Label":250,"Size":1024},{"Label":150,"Size":125}]` {
   398  		t.Errorf("Got back incorrect post-coarsesplit PreSyn noroi ranking:\n%v\n", string(data))
   399  	}
   400  
   401  	url = fmt.Sprintf("%snode/%s/noroi/top/5/PostSyn", server.WebAPIPath, uuid)
   402  	data = server.TestHTTP(t, "GET", url, nil)
   403  	if string(data) != `[{"Label":200,"Size":14351},{"Label":100,"Size":14350},{"Label":250,"Size":1024},{"Label":150,"Size":64}]` {
   404  		t.Errorf("Got back incorrect post-coarsesplit PostSyn noroi ranking:\n%v\n", string(data))
   405  	}
   406  
   407  	url = fmt.Sprintf("%snode/%s/noroi/top/5/AllSyn", server.WebAPIPath, uuid)
   408  	data = server.TestHTTP(t, "GET", url, nil)
   409  	if string(data) != `[{"Label":100,"Size":30609},{"Label":200,"Size":29711},{"Label":250,"Size":2048},{"Label":150,"Size":189}]` {
   410  		t.Errorf("Got back incorrect post-coarsesplit AllSyn noroi ranking:\n%v\n", string(data))
   411  	}
   412  
   413  	url = fmt.Sprintf("%snode/%s/noroi/count/200/AllSyn", server.WebAPIPath, uuid)
   414  	data = server.TestHTTP(t, "GET", url, nil)
   415  	if string(data) != `{"Label":200,"AllSyn":29711}` {
   416  		t.Errorf("Got back incorrect post-coarsesplit AllSyn noroi count of label 200:\n%v\n", string(data))
   417  	}
   418  
   419  	// Check the ROI-restricted labelsz instance which should only be affected by merge.
   420  	url = fmt.Sprintf("%snode/%s/withroi/top/5/PreSyn", server.WebAPIPath, uuid)
   421  	data = server.TestHTTP(t, "GET", url, nil)
   422  	if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":2048}]` {
   423  		t.Errorf("Got back incorrect post-coarsesplit PreSyn withroi ranking:\n%v\n", string(data))
   424  	}
   425  
   426  	url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid)
   427  	data = server.TestHTTP(t, "GET", url, nil)
   428  	if string(data) != `[{"Label":200,"Size":2048},{"Label":100,"Size":2047}]` {
   429  		t.Errorf("Got back incorrect post-coarsesplit PostSyn withroi ranking:\n%v\n", string(data))
   430  	}
   431  
   432  	url = fmt.Sprintf("%snode/%s/withroi/top/5/AllSyn", server.WebAPIPath, uuid)
   433  	data = server.TestHTTP(t, "GET", url, nil)
   434  	if string(data) != `[{"Label":200,"Size":4096},{"Label":100,"Size":4095}]` {
   435  		t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi ranking:\n%v\n", string(data))
   436  	}
   437  
   438  	url = fmt.Sprintf("%snode/%s/withroi/count/200/AllSyn", server.WebAPIPath, uuid)
   439  	data = server.TestHTTP(t, "GET", url, nil)
   440  	if string(data) != `{"Label":200,"AllSyn":4096}` {
   441  		t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi count of label 200:\n%v\n", string(data))
   442  	}
   443  
   444  	url = fmt.Sprintf("%snode/%s/withroi/count/100/AllSyn", server.WebAPIPath, uuid)
   445  	data = server.TestHTTP(t, "GET", url, nil)
   446  	if string(data) != `{"Label":100,"AllSyn":4095}` {
   447  		t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi count of label 100:\n%v\n", string(data))
   448  	}
   449  }
   450  
   451  func TestLabels(t *testing.T) {
   452  	if err := server.OpenTest(); err != nil {
   453  		t.Fatalf("can't open test server: %v\n", err)
   454  	}
   455  	defer server.CloseTest()
   456  
   457  	// Create testbed volume and data instances
   458  	uuid, _ := datastore.NewTestRepo()
   459  	var config dvid.Config
   460  	server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
   461  	server.CreateTestInstance(t, uuid, "labelvol", "bodies", config)
   462  
   463  	// Establish syncs
   464  	server.CreateTestSync(t, uuid, "labels", "bodies")
   465  	server.CreateTestSync(t, uuid, "bodies", "labels")
   466  
   467  	// Populate the labels, which should automatically populate the labelvol
   468  	_ = createLabelTestVolume(t, uuid, "labels")
   469  
   470  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   471  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   472  	}
   473  	if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil {
   474  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
   475  	}
   476  
   477  	// Add annotations syncing with "labels" instance.
   478  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
   479  	server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies")
   480  
   481  	// Create a ROI that will be used for our labelsz.
   482  	server.CreateTestInstance(t, uuid, "roi", "myroi", config)
   483  	roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid)
   484  	server.TestHTTP(t, "POST", roiRequest, getROIReader())
   485  
   486  	// Create labelsz instances synced to the above annotations.
   487  	server.CreateTestInstance(t, uuid, "labelsz", "noroi", config)
   488  	server.CreateTestSync(t, uuid, "noroi", "mysynapses")
   489  	config.Set("ROI", fmt.Sprintf("myroi,%s", uuid))
   490  	server.CreateTestInstance(t, uuid, "labelsz", "withroi", config)
   491  	server.CreateTestSync(t, uuid, "withroi", "mysynapses")
   492  
   493  	// PUT first batch of synapses.
   494  	var synapses annotation.Elements
   495  	var x, y, z int32
   496  	// This should put 31x31x31 (29,791) PostSyn in volume with fewer in label 200 than 300.
   497  	// There will be 15 along each dimension from 0 -> 63, then 16 from 64 -> 127.
   498  	// Label 100 will have 15 x 31 x 31 = 14415
   499  	// Label 200 will have 16 x 31 x 15 = 7440
   500  	// Label 300 will have 16 x 31 x 16 = 7936
   501  	for z = 4; z < 128; z += 4 {
   502  		for y = 4; y < 128; y += 4 {
   503  			for x = 4; x < 128; x += 4 {
   504  				e := annotation.Element{
   505  					annotation.ElementNR{
   506  						Pos:  dvid.Point3d{x, y, z},
   507  						Kind: annotation.PostSyn,
   508  					},
   509  					[]annotation.Relationship{},
   510  				}
   511  				synapses = append(synapses, e)
   512  			}
   513  		}
   514  	}
   515  	// This should put 32x32x32 (32,768) PreSyn in volume split 1/2, 1/4, 1/4
   516  	for z = 2; z < 128; z += 4 {
   517  		for y = 2; y < 128; y += 4 {
   518  			for x = 2; x < 128; x += 4 {
   519  				e := annotation.Element{
   520  					annotation.ElementNR{
   521  						Pos:  dvid.Point3d{x, y, z},
   522  						Kind: annotation.PreSyn,
   523  					},
   524  					[]annotation.Relationship{},
   525  				}
   526  				synapses = append(synapses, e)
   527  			}
   528  		}
   529  	}
   530  	testJSON, err := json.Marshal(synapses)
   531  	if err != nil {
   532  		t.Fatal(err)
   533  	}
   534  	url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
   535  	server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON)))
   536  
   537  	checkSequencing(t, uuid)
   538  }
   539  
   540  func TestLabelsResync(t *testing.T) {
   541  	if err := server.OpenTest(); err != nil {
   542  		t.Fatalf("can't open test server: %v\n", err)
   543  	}
   544  	defer server.CloseTest()
   545  
   546  	// Create testbed volume and data instances
   547  	uuid, _ := datastore.NewTestRepo()
   548  	var config dvid.Config
   549  	server.CreateTestInstance(t, uuid, "labelblk", "labels", config)
   550  	server.CreateTestInstance(t, uuid, "labelvol", "bodies", config)
   551  
   552  	// Establish syncs
   553  	server.CreateTestSync(t, uuid, "labels", "bodies")
   554  	server.CreateTestSync(t, uuid, "bodies", "labels")
   555  
   556  	// Populate the labels, which should automatically populate the labelvol
   557  	_ = createLabelTestVolume(t, uuid, "labels")
   558  
   559  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   560  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   561  	}
   562  	if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil {
   563  		t.Fatalf("Error blocking on sync of bodies: %v\n", err)
   564  	}
   565  
   566  	// Add annotations syncing with "labels" instance.
   567  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
   568  	server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies")
   569  
   570  	// Create a ROI that will be used for our labelsz.
   571  	server.CreateTestInstance(t, uuid, "roi", "myroi", config)
   572  	roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid)
   573  	server.TestHTTP(t, "POST", roiRequest, getROIReader())
   574  
   575  	// PUT first batch of synapses.
   576  	var synapses annotation.Elements
   577  	var x, y, z int32
   578  	// This should put 31x31x31 (29,791) PostSyn in volume with fewer in label 200 than 300.
   579  	// There will be 15 along each dimension from 0 -> 63, then 16 from 64 -> 127.
   580  	// Label 100 will have 15 x 31 x 31 = 14415
   581  	// Label 200 will have 16 x 31 x 15 = 7440
   582  	// Label 300 will have 16 x 31 x 16 = 7936
   583  	for z = 4; z < 128; z += 4 {
   584  		for y = 4; y < 128; y += 4 {
   585  			for x = 4; x < 128; x += 4 {
   586  				e := annotation.Element{
   587  					annotation.ElementNR{
   588  						Pos:  dvid.Point3d{x, y, z},
   589  						Kind: annotation.PostSyn,
   590  					},
   591  					[]annotation.Relationship{},
   592  				}
   593  				synapses = append(synapses, e)
   594  			}
   595  		}
   596  	}
   597  	// This should put 32x32x32 (32,768) PreSyn in volume split 1/2, 1/4, 1/4
   598  	for z = 2; z < 128; z += 4 {
   599  		for y = 2; y < 128; y += 4 {
   600  			for x = 2; x < 128; x += 4 {
   601  				e := annotation.Element{
   602  					annotation.ElementNR{
   603  						Pos:  dvid.Point3d{x, y, z},
   604  						Kind: annotation.PreSyn,
   605  					},
   606  					[]annotation.Relationship{},
   607  				}
   608  				synapses = append(synapses, e)
   609  			}
   610  		}
   611  	}
   612  	testJSON, err := json.Marshal(synapses)
   613  	if err != nil {
   614  		t.Fatal(err)
   615  	}
   616  	url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
   617  	server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON)))
   618  
   619  	// Create labelsz instances synced to the above annotations AFTER population so need resync.
   620  	server.CreateTestInstance(t, uuid, "labelsz", "noroi", config)
   621  	server.CreateTestSync(t, uuid, "noroi", "mysynapses")
   622  	config.Set("ROI", fmt.Sprintf("myroi,%s", uuid))
   623  	server.CreateTestInstance(t, uuid, "labelsz", "withroi", config)
   624  	server.CreateTestSync(t, uuid, "withroi", "mysynapses")
   625  
   626  	// Do the reload.
   627  	url = fmt.Sprintf("%snode/%s/noroi/reload", server.WebAPIPath, uuid)
   628  	server.TestHTTP(t, "POST", url, nil)
   629  	url = fmt.Sprintf("%snode/%s/withroi/reload", server.WebAPIPath, uuid)
   630  	server.TestHTTP(t, "POST", url, nil)
   631  
   632  	checkSequencing(t, uuid)
   633  }
   634  
   635  func TestLabelmap(t *testing.T) {
   636  	if err := server.OpenTest(); err != nil {
   637  		t.Fatalf("can't open test server: %v\n", err)
   638  	}
   639  	defer server.CloseTest()
   640  
   641  	// Create testbed volume and data instances
   642  	uuid, _ := datastore.NewTestRepo()
   643  	var config dvid.Config
   644  	server.CreateTestInstance(t, uuid, "labelmap", "labels", config)
   645  
   646  	sz := int32(128) // size of one cube side in voxels
   647  	numVoxels := sz * sz * sz
   648  	numLabels := 100
   649  	data := make([]byte, numVoxels*8)
   650  	offset := 0
   651  	for v := 0; v < int(numVoxels); v++ {
   652  		label := uint64(rand.Int()%numLabels) + 1
   653  		binary.LittleEndian.PutUint64(data[offset:offset+8], label)
   654  		offset += 8
   655  	}
   656  	_ = createLabelTestVolume(t, uuid, "labels")
   657  	apiStr := fmt.Sprintf("%snode/%s/labels/raw/0_1_2/%d_%d_%d/0_0_0", server.WebAPIPath,
   658  		uuid, sz, sz, sz)
   659  	server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(data))
   660  
   661  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
   662  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
   663  	}
   664  
   665  	// Add annotations syncing with "labels" instance.
   666  	server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config)
   667  	server.CreateTestSync(t, uuid, "mysynapses", "labels")
   668  
   669  	// Create a ROI that will be used for our labelsz.
   670  	server.CreateTestInstance(t, uuid, "roi", "myroi", config)
   671  	roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid)
   672  	server.TestHTTP(t, "POST", roiRequest, getROIReader())
   673  
   674  	// Create labelsz instances synced to the above annotations.
   675  	server.CreateTestInstance(t, uuid, "labelsz", "noroi", config)
   676  	server.CreateTestSync(t, uuid, "noroi", "mysynapses")
   677  	config.Set("ROI", fmt.Sprintf("myroi,%s", uuid))
   678  	server.CreateTestInstance(t, uuid, "labelsz", "withroi", config)
   679  	server.CreateTestSync(t, uuid, "withroi", "mysynapses")
   680  
   681  	labels := []uint64{10, 20, 30, 40, 50}
   682  	postsynCounts := make(map[uint64]int, numLabels)
   683  	presynCounts := make(map[uint64]int, numLabels)
   684  	var synapses annotation.Elements
   685  	var x, y, z int32
   686  	// This should put 31x31x31 (29791) PostSyn
   687  	for z = 4; z < sz; z += 4 {
   688  		for y = 4; y < sz; y += 4 {
   689  			for x = 4; x < sz; x += 4 {
   690  				e := annotation.Element{
   691  					annotation.ElementNR{
   692  						Pos:  dvid.Point3d{x, y, z},
   693  						Kind: annotation.PostSyn,
   694  					},
   695  					[]annotation.Relationship{},
   696  				}
   697  				synapses = append(synapses, e)
   698  				offset := (z*sz*sz + y*sz + x) * 8
   699  				label := binary.LittleEndian.Uint64(data[offset : offset+8])
   700  				postsynCounts[label] = postsynCounts[label] + 1
   701  			}
   702  		}
   703  	}
   704  	// This should put 32x32x32 (32768) PreSyn in volume
   705  	for z = 2; z < sz; z += 4 {
   706  		for y = 2; y < sz; y += 4 {
   707  			for x = 2; x < sz; x += 4 {
   708  				e := annotation.Element{
   709  					annotation.ElementNR{
   710  						Pos:  dvid.Point3d{x, y, z},
   711  						Kind: annotation.PreSyn,
   712  					},
   713  					[]annotation.Relationship{},
   714  				}
   715  				synapses = append(synapses, e)
   716  				offset := (z*sz*sz + y*sz + x) * 8
   717  				label := binary.LittleEndian.Uint64(data[offset : offset+8])
   718  				presynCounts[label] = presynCounts[label] + 1
   719  			}
   720  		}
   721  	}
   722  	testJSON, err := json.Marshal(synapses)
   723  	if err != nil {
   724  		t.Fatal(err)
   725  	}
   726  	url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid)
   727  	server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON)))
   728  
   729  	// Need to pause to prevent race condition for when the labelsz is synced.
   730  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
   731  		t.Fatalf("Error blocking on sync of elements to labelsz: %v\n", err)
   732  	}
   733  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   734  		t.Fatalf("Error blocking on sync of elements to labelsz: %v\n", err)
   735  	}
   736  
   737  	postStr := "["
   738  	for i, label := range labels {
   739  		postStr += strconv.Itoa(int(label))
   740  		if i != len(labels)-1 {
   741  			postStr += ","
   742  		}
   743  	}
   744  	postStr += "]"
   745  	url = fmt.Sprintf("%snode/%s/noroi/counts/PreSyn", server.WebAPIPath, uuid)
   746  	retData := server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr)))
   747  	var retVal []struct {
   748  		Label  uint64
   749  		PreSyn int
   750  	}
   751  	if err := json.Unmarshal(retData, &retVal); err != nil {
   752  		t.Errorf("Unable to decode return value: %v\n", retData)
   753  	}
   754  	for _, val := range retVal {
   755  		expected, found := presynCounts[val.Label]
   756  		if !found {
   757  			t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData))
   758  		}
   759  		if expected != val.PreSyn {
   760  			t.Fatalf("Expected label %d presyn to have %d, got %d\n", val.Label, expected, val.PreSyn)
   761  		}
   762  	}
   763  	url = fmt.Sprintf("%snode/%s/noroi/counts/PostSyn", server.WebAPIPath, uuid)
   764  	retData = server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr)))
   765  	var retVal2 []struct {
   766  		Label   uint64
   767  		PostSyn int
   768  	}
   769  	if err := json.Unmarshal(retData, &retVal2); err != nil {
   770  		t.Errorf("Unable to decode return value: %v\n", retData)
   771  	}
   772  	for _, val := range retVal2 {
   773  		expected, found := postsynCounts[val.Label]
   774  		if !found {
   775  			t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData))
   776  		}
   777  		if expected != val.PostSyn {
   778  			t.Fatalf("Expected label %d postsyn to have %d, got %d\n", val.Label, expected, val.PostSyn)
   779  		}
   780  	}
   781  	url = fmt.Sprintf("%snode/%s/noroi/counts/AllSyn", server.WebAPIPath, uuid)
   782  	retData = server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr)))
   783  	var retVal3 []struct {
   784  		Label  uint64
   785  		AllSyn int
   786  	}
   787  	if err := json.Unmarshal(retData, &retVal3); err != nil {
   788  		t.Errorf("Unable to decode return value: %v\n", retData)
   789  	}
   790  	for _, val := range retVal3 {
   791  		expectedPre, found := presynCounts[val.Label]
   792  		if !found {
   793  			t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData))
   794  		}
   795  		expectedPost, found := postsynCounts[val.Label]
   796  		if !found {
   797  			t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData))
   798  		}
   799  		if expectedPre+expectedPost != val.AllSyn {
   800  			t.Fatalf("Expected label %d allsyn to have %d+%d, got %d\n", val.Label, expectedPre, expectedPost, val.AllSyn)
   801  		}
   802  	}
   803  
   804  	// merge 10+20 into 30 and recheck
   805  	url = fmt.Sprintf("%snode/%s/labels/merge", server.WebAPIPath, uuid)
   806  	server.TestHTTP(t, "POST", url, bytes.NewBufferString("[30,10,20]"))
   807  
   808  	if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil {
   809  		t.Fatalf("Error blocking on sync of annotations: %v\n", err)
   810  	}
   811  	if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil {
   812  		t.Fatalf("Error blocking on sync of annotations: %v\n", err)
   813  	}
   814  
   815  	url = fmt.Sprintf("%snode/%s/noroi/count/30/PreSyn", server.WebAPIPath, uuid)
   816  	retData = server.TestHTTP(t, "GET", url, nil)
   817  	presynMerged := presynCounts[10] + presynCounts[20] + presynCounts[30]
   818  	if string(retData) != fmt.Sprintf(`{"Label":30,"PreSyn":%d}`, presynMerged) {
   819  		t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 30: %s\nlabel 10+20+30 = %d+%d+%d = %d\n",
   820  			string(retData), presynCounts[10], presynCounts[20], presynCounts[30], presynMerged)
   821  	}
   822  
   823  	url = fmt.Sprintf("%snode/%s/noroi/count/10/PreSyn", server.WebAPIPath, uuid)
   824  	retData = server.TestHTTP(t, "GET", url, nil)
   825  	if string(retData) != `{"Label":10,"PreSyn":0}` {
   826  		t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 10: %s\n", string(retData))
   827  	}
   828  	url = fmt.Sprintf("%snode/%s/noroi/count/20/PreSyn", server.WebAPIPath, uuid)
   829  	retData = server.TestHTTP(t, "GET", url, nil)
   830  	if string(retData) != `{"Label":20,"PreSyn":0}` {
   831  		t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 20: %s\n", string(retData))
   832  	}
   833  }