github.com/janelia-flyem/dvid@v1.0.0/datatype/labelsz/labelsz_test.go (about) 1 package labelsz 2 3 import ( 4 "bytes" 5 "encoding/binary" 6 "encoding/json" 7 "fmt" 8 "io" 9 "log" 10 "math/rand" 11 "strconv" 12 "strings" 13 "testing" 14 "time" 15 16 "github.com/janelia-flyem/dvid/datastore" 17 "github.com/janelia-flyem/dvid/datatype/annotation" 18 "github.com/janelia-flyem/dvid/dvid" 19 "github.com/janelia-flyem/dvid/server" 20 ) 21 22 // A slice of bytes representing 3d label volume 23 type testVolume struct { 24 data []byte 25 size dvid.Point3d 26 } 27 28 func newTestVolume(nx, ny, nz int32) *testVolume { 29 return &testVolume{ 30 data: make([]byte, nx*ny*nz*8), 31 size: dvid.Point3d{nx, ny, nz}, 32 } 33 } 34 35 // Sets voxels in body to given label. 36 func (v *testVolume) add(label uint64, ox, oy, oz int32, sx, sy, sz int32) { 37 nx := v.size.Value(0) 38 ny := v.size.Value(1) 39 nxy := nx * ny 40 for z := oz; z < oz+sz; z++ { 41 for y := oy; y < oy+sy; y++ { 42 p := (z*nxy + y*nx + ox) * 8 43 for x := ox; x < ox+sx; x++ { 44 binary.LittleEndian.PutUint64(v.data[p:p+8], label) 45 p += 8 46 } 47 } 48 } 49 } 50 51 // Put label data into given data instance. 52 func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) { 53 apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/0_0_0?mutate=true", server.WebAPIPath, 54 uuid, name, v.size[0], v.size[1], v.size[2]) 55 server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data)) 56 } 57 58 func createLabelTestVolume(t *testing.T, uuid dvid.UUID, name string) *testVolume { 59 volume := newTestVolume(128, 128, 128) 60 volume.add(100, 0, 0, 0, 64, 128, 128) 61 volume.add(200, 64, 0, 0, 64, 128, 64) 62 volume.add(300, 64, 0, 64, 64, 128, 64) 63 64 // Send data over HTTP to populate a data instance 65 volume.put(t, uuid, name) 66 return volume 67 } 68 69 // test ROI has offset (32, 32, 32) and size (64, 64, 64) 70 var testSpans = []dvid.Span{ 71 dvid.Span{1, 1, 1, 2}, dvid.Span{1, 2, 1, 2}, 72 dvid.Span{2, 1, 1, 2}, dvid.Span{2, 2, 1, 2}, 73 } 74 75 func getROIReader() io.Reader { 76 jsonBytes, err := json.Marshal(testSpans) 77 if err != nil { 78 log.Fatalf("Can't encode spans into JSON: %v\n", err) 79 } 80 return bytes.NewReader(jsonBytes) 81 } 82 83 type mergeJSON string 84 85 func (mjson mergeJSON) send(t *testing.T, uuid dvid.UUID, name string) { 86 apiStr := fmt.Sprintf("%snode/%s/%s/merge", server.WebAPIPath, uuid, name) 87 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(string(mjson))) 88 } 89 90 func getBytesRLE(t *testing.T, rles dvid.RLEs) *bytes.Buffer { 91 n := len(rles) 92 buf := new(bytes.Buffer) 93 buf.WriteByte(dvid.EncodingBinary) 94 binary.Write(buf, binary.LittleEndian, uint8(3)) // # of dimensions 95 binary.Write(buf, binary.LittleEndian, byte(0)) // dimension of run (X = 0) 96 buf.WriteByte(byte(0)) // reserved for later 97 binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels 98 binary.Write(buf, binary.LittleEndian, uint32(n)) // Placeholder for # spans 99 rleBytes, err := rles.MarshalBinary() 100 if err != nil { 101 t.Errorf("Unable to serialize RLEs: %v\n", err) 102 } 103 buf.Write(rleBytes) 104 return buf 105 } 106 107 func checkSequencing(t *testing.T, uuid dvid.UUID) { 108 // Check if we have correct sequencing for no ROI labelsz. 109 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 110 t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err) 111 } 112 113 url := fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid) 114 data := server.TestHTTP(t, "GET", url, nil) 115 if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192},{"Label":300,"Size":8192}]` { 116 t.Errorf("Got back incorrect PreSyn noroi ranking:\n%v\n", string(data)) 117 } 118 119 url = fmt.Sprintf("%snode/%s/noroi/count/100/PreSyn", server.WebAPIPath, uuid) 120 data = server.TestHTTP(t, "GET", url, nil) 121 if string(data) != `{"Label":100,"PreSyn":16384}` { 122 t.Errorf("Got back incorrect PreSyn noroi count for label 100:\n%v\n", string(data)) 123 } 124 125 url = fmt.Sprintf("%snode/%s/noroi/count/200/PreSyn", server.WebAPIPath, uuid) 126 data = server.TestHTTP(t, "GET", url, nil) 127 if string(data) != `{"Label":200,"PreSyn":8192}` { 128 t.Errorf("Got back incorrect PreSyn noroi count for label 200:\n%v\n", string(data)) 129 } 130 131 url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid) 132 data = server.TestHTTP(t, "GET", url, nil) 133 if string(data) != `[{"Label":100,"Size":14415},{"Label":300,"Size":7936},{"Label":200,"Size":7440}]` { 134 t.Errorf("Got back incorrect PostSyn noroi ranking:\n%v\n", string(data)) 135 } 136 137 url = fmt.Sprintf("%snode/%s/noroi/top/3/AllSyn", server.WebAPIPath, uuid) 138 data = server.TestHTTP(t, "GET", url, nil) 139 if string(data) != `[{"Label":100,"Size":30799},{"Label":300,"Size":16128},{"Label":200,"Size":15632}]` { 140 t.Errorf("Got back incorrect AllSync noroi ranking:\n%v\n", string(data)) 141 } 142 143 url = fmt.Sprintf("%snode/%s/noroi/threshold/15633/AllSyn", server.WebAPIPath, uuid) 144 data = server.TestHTTP(t, "GET", url, nil) 145 if string(data) != `[{"Label":100,"Size":30799},{"Label":300,"Size":16128}]` { 146 t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data)) 147 } 148 149 url = fmt.Sprintf("%snode/%s/noroi/threshold/1000/AllSyn?offset=1&n=2", server.WebAPIPath, uuid) 150 data = server.TestHTTP(t, "GET", url, nil) 151 if string(data) != `[{"Label":300,"Size":16128},{"Label":200,"Size":15632}]` { 152 t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data)) 153 } 154 155 url = fmt.Sprintf("%snode/%s/noroi/threshold/1000/AllSyn?offset=8&n=2", server.WebAPIPath, uuid) 156 data = server.TestHTTP(t, "GET", url, nil) 157 if string(data) != `[]` { 158 t.Errorf("Got back incorrect AllSyn noroi threshold:\n%v\n", string(data)) 159 } 160 161 // Check if we have correct sequencing for ROI labelsz. 162 // ROI constitutes the inner eight 32^3 blocks. 163 // There are 16 PostSyn in each ROI dimension. 164 // There are also 16 PreSyn in each ROI dimension. 165 if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil { 166 t.Fatalf("Error blocking on sync of withroi labelsz: %v\n", err) 167 } 168 169 url = fmt.Sprintf("%snode/%s/withroi/top/0/AllSyn", server.WebAPIPath, uuid) 170 data = server.TestHTTP(t, "GET", url, nil) 171 if string(data) != `[]` { 172 t.Errorf("Incorrectly handled top n=0 case, expected [] got: %v\n", string(data)) 173 } 174 175 url = fmt.Sprintf("%snode/%s/withroi/top/3/PreSyn", server.WebAPIPath, uuid) 176 data = server.TestHTTP(t, "GET", url, nil) 177 if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` { 178 t.Errorf("Got back incorrect PreSyn withroi ranking:\n%v\n", string(data)) 179 } 180 181 url = fmt.Sprintf("%snode/%s/withroi/top/3/PostSyn", server.WebAPIPath, uuid) 182 data = server.TestHTTP(t, "GET", url, nil) 183 if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` { 184 t.Errorf("Got back incorrect PostSyn withroi ranking:\n%v\n", string(data)) 185 } 186 187 // Check fewer and larger N requests. 188 url = fmt.Sprintf("%snode/%s/noroi/top/2/PreSyn", server.WebAPIPath, uuid) 189 data = server.TestHTTP(t, "GET", url, nil) 190 if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192}]` { 191 t.Errorf("Got back incorrect N=2 ranking:\n%v\n", string(data)) 192 } 193 194 url = fmt.Sprintf("%snode/%s/noroi/top/4/PreSyn", server.WebAPIPath, uuid) 195 data = server.TestHTTP(t, "GET", url, nil) 196 if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":8192},{"Label":300,"Size":8192}]` { 197 t.Errorf("Got back incorrect N=4 ranking:\n%v\n", string(data)) 198 } 199 200 // Test annotation move of a PostSyn from label 100->300 and also label 200->300 201 url = fmt.Sprintf("%snode/%s/mysynapses/move/32_32_32/75_21_69", server.WebAPIPath, uuid) 202 server.TestHTTP(t, "POST", url, nil) 203 204 url = fmt.Sprintf("%snode/%s/mysynapses/move/68_20_20/77_21_69", server.WebAPIPath, uuid) 205 server.TestHTTP(t, "POST", url, nil) 206 207 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 208 t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err) 209 } 210 url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid) 211 data = server.TestHTTP(t, "GET", url, nil) 212 if string(data) != `[{"Label":100,"Size":14414},{"Label":300,"Size":7938},{"Label":200,"Size":7439}]` { 213 t.Errorf("Got back incorrect PostSyn noroi ranking after move from label 100->300:\n%v\n", string(data)) 214 } 215 216 // First move took synapse out of ROI so there should be one less for label 100. 217 if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil { 218 t.Fatalf("Error blocking on sync of labelsz: %v\n", err) 219 } 220 221 url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid) 222 data = server.TestHTTP(t, "GET", url, nil) 223 if string(data) != `[{"Label":100,"Size":2047},{"Label":200,"Size":1024},{"Label":300,"Size":1024}]` { 224 t.Errorf("Got back incorrect post-move PostSyn withroi ranking:\n%v\n", string(data)) 225 } 226 227 // Test annotation deletion of moved PostSyn from label 300 228 url = fmt.Sprintf("%snode/%s/mysynapses/element/75_21_69", server.WebAPIPath, uuid) 229 server.TestHTTP(t, "DELETE", url, nil) 230 231 url = fmt.Sprintf("%snode/%s/mysynapses/element/77_21_69", server.WebAPIPath, uuid) 232 server.TestHTTP(t, "DELETE", url, nil) 233 234 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 235 t.Fatalf("Error blocking on sync of noroi labelsz: %v\n", err) 236 } 237 238 url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid) 239 data = server.TestHTTP(t, "GET", url, nil) 240 if string(data) != `[{"Label":100,"Size":14414},{"Label":300,"Size":7936},{"Label":200,"Size":7439}]` { 241 t.Errorf("Got back incorrect PostSyn noroi ranking after deletions from label 300:\n%v\n", string(data)) 242 } 243 244 // Check sync on merge. 245 if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil { 246 t.Fatalf("Error blocking on sync of bodies: %v\n", err) 247 } 248 testMerge := mergeJSON(`[200, 300]`) 249 testMerge.send(t, uuid, "bodies") 250 251 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 252 t.Fatalf("Error blocking on sync of labels: %v\n", err) 253 } 254 time.Sleep(1 * time.Second) 255 if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil { 256 t.Fatalf("Error blocking on sync of synapses: %v\n", err) 257 } 258 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 259 t.Fatalf("Error blocking on sync of labelsz: %v\n", err) 260 } 261 if err := datastore.BlockOnUpdating(uuid, "withroi"); err != nil { 262 t.Fatalf("Error blocking on sync of labelsz: %v\n", err) 263 } 264 265 url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid) 266 data = server.TestHTTP(t, "GET", url, nil) 267 if string(data) != `[{"Label":200,"Size":2048},{"Label":100,"Size":2047}]` { 268 t.Errorf("Got back incorrect post-merge PostSyn withroi ranking:\n%v\n", string(data)) 269 } 270 271 url = fmt.Sprintf("%snode/%s/withroi/count/100/PostSyn", server.WebAPIPath, uuid) 272 data = server.TestHTTP(t, "GET", url, nil) 273 if string(data) != `{"Label":100,"PostSyn":2047}` { 274 t.Errorf("Got back incorrect post-merge PostSyn withroi count of label 100:\n%v\n", string(data)) 275 } 276 277 url = fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid) 278 data = server.TestHTTP(t, "GET", url, nil) 279 if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":16384}]` { 280 t.Errorf("Got back incorrect post-merge PreSyn noroi ranking:\n%v\n", string(data)) 281 } 282 283 url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid) 284 data = server.TestHTTP(t, "GET", url, nil) 285 if string(data) != `[{"Label":200,"Size":15375},{"Label":100,"Size":14414}]` { 286 t.Errorf("Got back incorrect post-merge PostSyn noroi ranking:\n%v\n", string(data)) 287 } 288 289 url = fmt.Sprintf("%snode/%s/noroi/top/3/AllSyn", server.WebAPIPath, uuid) 290 data = server.TestHTTP(t, "GET", url, nil) 291 if string(data) != `[{"Label":200,"Size":31759},{"Label":100,"Size":30798}]` { 292 t.Errorf("Got back incorrect post-merge AllSyn noroi ranking:\n%v\n", string(data)) 293 } 294 295 // Check threshold endpoint 296 297 url = fmt.Sprintf("%snode/%s/withroi/threshold/2048/PostSyn", server.WebAPIPath, uuid) 298 data = server.TestHTTP(t, "GET", url, nil) 299 if string(data) != `[{"Label":200,"Size":2048}]` { 300 t.Errorf("Got back incorrect post-merge PostSyn withroi threshold:\n%v\n", string(data)) 301 } 302 303 url = fmt.Sprintf("%snode/%s/noroi/threshold/16384/PreSyn", server.WebAPIPath, uuid) 304 data = server.TestHTTP(t, "GET", url, nil) 305 if string(data) != `[{"Label":100,"Size":16384},{"Label":200,"Size":16384}]` { 306 t.Errorf("Got back incorrect post-merge PreSyn noroi threshold:\n%v\n", string(data)) 307 } 308 309 url = fmt.Sprintf("%snode/%s/noroi/threshold/15000/PostSyn", server.WebAPIPath, uuid) 310 data = server.TestHTTP(t, "GET", url, nil) 311 if string(data) != `[{"Label":200,"Size":15375}]` { 312 t.Errorf("Got back incorrect post-merge PostSyn noroi threshold:\n%v\n", string(data)) 313 } 314 315 url = fmt.Sprintf("%snode/%s/noroi/threshold/0/PostSyn?offset=1&n=1", server.WebAPIPath, uuid) 316 data = server.TestHTTP(t, "GET", url, nil) 317 if string(data) != `[{"Label":100,"Size":14414}]` { 318 t.Errorf("Got back incorrect post-merge PostSyn noroi threshold with offset/n:\n%v\n", string(data)) 319 } 320 321 // Create the sparsevol encoding for split area with label 100 -> 150. 322 // Split has offset (0, 0, 0) with size (19, 19, 19). 323 // PreSyn in split = 5 x 5 x 5 = 125 324 // PostSyn in split = 4 x 4 x 4 = 64 325 var rles dvid.RLEs 326 for z := int32(0); z < 19; z++ { 327 for y := int32(0); y < 19; y++ { 328 start := dvid.Point3d{0, y, z} 329 rles = append(rles, dvid.NewRLE(start, 19)) 330 } 331 } 332 buf := getBytesRLE(t, rles) 333 334 // Submit the split sparsevol 335 url = fmt.Sprintf("%snode/%s/%s/split/%d?splitlabel=150", server.WebAPIPath, uuid, "bodies", 100) 336 data = server.TestHTTP(t, "POST", url, buf) 337 jsonVal := make(map[string]uint64) 338 if err := json.Unmarshal(data, &jsonVal); err != nil { 339 t.Errorf("Unable to get new label from split. Instead got: %v\n", jsonVal) 340 } 341 342 // Check sync on split. 343 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 344 t.Fatalf("Error blocking on sync of labels: %v\n", err) 345 } 346 time.Sleep(1 * time.Second) 347 if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil { 348 t.Fatalf("Error blocking on sync of synapses: %v\n", err) 349 } 350 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 351 t.Fatalf("Error blocking on sync of labelsz: %v\n", err) 352 } 353 354 url = fmt.Sprintf("%snode/%s/noroi/top/3/PreSyn", server.WebAPIPath, uuid) 355 data = server.TestHTTP(t, "GET", url, nil) 356 if string(data) != `[{"Label":200,"Size":16384},{"Label":100,"Size":16259},{"Label":150,"Size":125}]` { 357 t.Errorf("Got back incorrect post-split PreSyn noroi ranking:\n%v\n", string(data)) 358 } 359 360 url = fmt.Sprintf("%snode/%s/noroi/top/3/PostSyn", server.WebAPIPath, uuid) 361 data = server.TestHTTP(t, "GET", url, nil) 362 if string(data) != `[{"Label":200,"Size":15375},{"Label":100,"Size":14350},{"Label":150,"Size":64}]` { 363 t.Errorf("Got back incorrect post-split PostSyn noroi ranking:\n%v\n", string(data)) 364 } 365 366 // Create the encoding for coarse split area in block coordinates from label 200. 367 // Split has offset (64, 96, 96) with size (64, 32, 32). 368 // PreSyn in split = 16 x 8 x 8 = 1024 369 // PostSyn in split = 16 x 8 x 8 = 1024 370 rles = dvid.RLEs{ 371 dvid.NewRLE(dvid.Point3d{2, 3, 3}, 2), 372 } 373 buf = getBytesRLE(t, rles) 374 375 // Submit the coarse split of 200 -> 250 376 url = fmt.Sprintf("%snode/%s/%s/split-coarse/200?splitlabel=250", server.WebAPIPath, uuid, "bodies") 377 data = server.TestHTTP(t, "POST", url, buf) 378 jsonVal = make(map[string]uint64) 379 if err := json.Unmarshal(data, &jsonVal); err != nil { 380 t.Errorf("Unable to get new label from split. Instead got: %v\n", jsonVal) 381 } 382 383 // Check sync on split. 384 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 385 t.Fatalf("Error blocking on sync of labels: %v\n", err) 386 } 387 time.Sleep(1 * time.Second) 388 if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil { 389 t.Fatalf("Error blocking on sync of synapses: %v\n", err) 390 } 391 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 392 t.Fatalf("Error blocking on sync of labelsz: %v\n", err) 393 } 394 395 url = fmt.Sprintf("%snode/%s/noroi/top/5/PreSyn", server.WebAPIPath, uuid) 396 data = server.TestHTTP(t, "GET", url, nil) 397 if string(data) != `[{"Label":100,"Size":16259},{"Label":200,"Size":15360},{"Label":250,"Size":1024},{"Label":150,"Size":125}]` { 398 t.Errorf("Got back incorrect post-coarsesplit PreSyn noroi ranking:\n%v\n", string(data)) 399 } 400 401 url = fmt.Sprintf("%snode/%s/noroi/top/5/PostSyn", server.WebAPIPath, uuid) 402 data = server.TestHTTP(t, "GET", url, nil) 403 if string(data) != `[{"Label":200,"Size":14351},{"Label":100,"Size":14350},{"Label":250,"Size":1024},{"Label":150,"Size":64}]` { 404 t.Errorf("Got back incorrect post-coarsesplit PostSyn noroi ranking:\n%v\n", string(data)) 405 } 406 407 url = fmt.Sprintf("%snode/%s/noroi/top/5/AllSyn", server.WebAPIPath, uuid) 408 data = server.TestHTTP(t, "GET", url, nil) 409 if string(data) != `[{"Label":100,"Size":30609},{"Label":200,"Size":29711},{"Label":250,"Size":2048},{"Label":150,"Size":189}]` { 410 t.Errorf("Got back incorrect post-coarsesplit AllSyn noroi ranking:\n%v\n", string(data)) 411 } 412 413 url = fmt.Sprintf("%snode/%s/noroi/count/200/AllSyn", server.WebAPIPath, uuid) 414 data = server.TestHTTP(t, "GET", url, nil) 415 if string(data) != `{"Label":200,"AllSyn":29711}` { 416 t.Errorf("Got back incorrect post-coarsesplit AllSyn noroi count of label 200:\n%v\n", string(data)) 417 } 418 419 // Check the ROI-restricted labelsz instance which should only be affected by merge. 420 url = fmt.Sprintf("%snode/%s/withroi/top/5/PreSyn", server.WebAPIPath, uuid) 421 data = server.TestHTTP(t, "GET", url, nil) 422 if string(data) != `[{"Label":100,"Size":2048},{"Label":200,"Size":2048}]` { 423 t.Errorf("Got back incorrect post-coarsesplit PreSyn withroi ranking:\n%v\n", string(data)) 424 } 425 426 url = fmt.Sprintf("%snode/%s/withroi/top/5/PostSyn", server.WebAPIPath, uuid) 427 data = server.TestHTTP(t, "GET", url, nil) 428 if string(data) != `[{"Label":200,"Size":2048},{"Label":100,"Size":2047}]` { 429 t.Errorf("Got back incorrect post-coarsesplit PostSyn withroi ranking:\n%v\n", string(data)) 430 } 431 432 url = fmt.Sprintf("%snode/%s/withroi/top/5/AllSyn", server.WebAPIPath, uuid) 433 data = server.TestHTTP(t, "GET", url, nil) 434 if string(data) != `[{"Label":200,"Size":4096},{"Label":100,"Size":4095}]` { 435 t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi ranking:\n%v\n", string(data)) 436 } 437 438 url = fmt.Sprintf("%snode/%s/withroi/count/200/AllSyn", server.WebAPIPath, uuid) 439 data = server.TestHTTP(t, "GET", url, nil) 440 if string(data) != `{"Label":200,"AllSyn":4096}` { 441 t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi count of label 200:\n%v\n", string(data)) 442 } 443 444 url = fmt.Sprintf("%snode/%s/withroi/count/100/AllSyn", server.WebAPIPath, uuid) 445 data = server.TestHTTP(t, "GET", url, nil) 446 if string(data) != `{"Label":100,"AllSyn":4095}` { 447 t.Errorf("Got back incorrect post-coarsesplit AllSyn withroi count of label 100:\n%v\n", string(data)) 448 } 449 } 450 451 func TestLabels(t *testing.T) { 452 if err := server.OpenTest(); err != nil { 453 t.Fatalf("can't open test server: %v\n", err) 454 } 455 defer server.CloseTest() 456 457 // Create testbed volume and data instances 458 uuid, _ := datastore.NewTestRepo() 459 var config dvid.Config 460 server.CreateTestInstance(t, uuid, "labelblk", "labels", config) 461 server.CreateTestInstance(t, uuid, "labelvol", "bodies", config) 462 463 // Establish syncs 464 server.CreateTestSync(t, uuid, "labels", "bodies") 465 server.CreateTestSync(t, uuid, "bodies", "labels") 466 467 // Populate the labels, which should automatically populate the labelvol 468 _ = createLabelTestVolume(t, uuid, "labels") 469 470 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 471 t.Fatalf("Error blocking on sync of labels: %v\n", err) 472 } 473 if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil { 474 t.Fatalf("Error blocking on sync of bodies: %v\n", err) 475 } 476 477 // Add annotations syncing with "labels" instance. 478 server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config) 479 server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies") 480 481 // Create a ROI that will be used for our labelsz. 482 server.CreateTestInstance(t, uuid, "roi", "myroi", config) 483 roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid) 484 server.TestHTTP(t, "POST", roiRequest, getROIReader()) 485 486 // Create labelsz instances synced to the above annotations. 487 server.CreateTestInstance(t, uuid, "labelsz", "noroi", config) 488 server.CreateTestSync(t, uuid, "noroi", "mysynapses") 489 config.Set("ROI", fmt.Sprintf("myroi,%s", uuid)) 490 server.CreateTestInstance(t, uuid, "labelsz", "withroi", config) 491 server.CreateTestSync(t, uuid, "withroi", "mysynapses") 492 493 // PUT first batch of synapses. 494 var synapses annotation.Elements 495 var x, y, z int32 496 // This should put 31x31x31 (29,791) PostSyn in volume with fewer in label 200 than 300. 497 // There will be 15 along each dimension from 0 -> 63, then 16 from 64 -> 127. 498 // Label 100 will have 15 x 31 x 31 = 14415 499 // Label 200 will have 16 x 31 x 15 = 7440 500 // Label 300 will have 16 x 31 x 16 = 7936 501 for z = 4; z < 128; z += 4 { 502 for y = 4; y < 128; y += 4 { 503 for x = 4; x < 128; x += 4 { 504 e := annotation.Element{ 505 annotation.ElementNR{ 506 Pos: dvid.Point3d{x, y, z}, 507 Kind: annotation.PostSyn, 508 }, 509 []annotation.Relationship{}, 510 } 511 synapses = append(synapses, e) 512 } 513 } 514 } 515 // This should put 32x32x32 (32,768) PreSyn in volume split 1/2, 1/4, 1/4 516 for z = 2; z < 128; z += 4 { 517 for y = 2; y < 128; y += 4 { 518 for x = 2; x < 128; x += 4 { 519 e := annotation.Element{ 520 annotation.ElementNR{ 521 Pos: dvid.Point3d{x, y, z}, 522 Kind: annotation.PreSyn, 523 }, 524 []annotation.Relationship{}, 525 } 526 synapses = append(synapses, e) 527 } 528 } 529 } 530 testJSON, err := json.Marshal(synapses) 531 if err != nil { 532 t.Fatal(err) 533 } 534 url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid) 535 server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON))) 536 537 checkSequencing(t, uuid) 538 } 539 540 func TestLabelsResync(t *testing.T) { 541 if err := server.OpenTest(); err != nil { 542 t.Fatalf("can't open test server: %v\n", err) 543 } 544 defer server.CloseTest() 545 546 // Create testbed volume and data instances 547 uuid, _ := datastore.NewTestRepo() 548 var config dvid.Config 549 server.CreateTestInstance(t, uuid, "labelblk", "labels", config) 550 server.CreateTestInstance(t, uuid, "labelvol", "bodies", config) 551 552 // Establish syncs 553 server.CreateTestSync(t, uuid, "labels", "bodies") 554 server.CreateTestSync(t, uuid, "bodies", "labels") 555 556 // Populate the labels, which should automatically populate the labelvol 557 _ = createLabelTestVolume(t, uuid, "labels") 558 559 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 560 t.Fatalf("Error blocking on sync of labels: %v\n", err) 561 } 562 if err := datastore.BlockOnUpdating(uuid, "bodies"); err != nil { 563 t.Fatalf("Error blocking on sync of bodies: %v\n", err) 564 } 565 566 // Add annotations syncing with "labels" instance. 567 server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config) 568 server.CreateTestSync(t, uuid, "mysynapses", "labels,bodies") 569 570 // Create a ROI that will be used for our labelsz. 571 server.CreateTestInstance(t, uuid, "roi", "myroi", config) 572 roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid) 573 server.TestHTTP(t, "POST", roiRequest, getROIReader()) 574 575 // PUT first batch of synapses. 576 var synapses annotation.Elements 577 var x, y, z int32 578 // This should put 31x31x31 (29,791) PostSyn in volume with fewer in label 200 than 300. 579 // There will be 15 along each dimension from 0 -> 63, then 16 from 64 -> 127. 580 // Label 100 will have 15 x 31 x 31 = 14415 581 // Label 200 will have 16 x 31 x 15 = 7440 582 // Label 300 will have 16 x 31 x 16 = 7936 583 for z = 4; z < 128; z += 4 { 584 for y = 4; y < 128; y += 4 { 585 for x = 4; x < 128; x += 4 { 586 e := annotation.Element{ 587 annotation.ElementNR{ 588 Pos: dvid.Point3d{x, y, z}, 589 Kind: annotation.PostSyn, 590 }, 591 []annotation.Relationship{}, 592 } 593 synapses = append(synapses, e) 594 } 595 } 596 } 597 // This should put 32x32x32 (32,768) PreSyn in volume split 1/2, 1/4, 1/4 598 for z = 2; z < 128; z += 4 { 599 for y = 2; y < 128; y += 4 { 600 for x = 2; x < 128; x += 4 { 601 e := annotation.Element{ 602 annotation.ElementNR{ 603 Pos: dvid.Point3d{x, y, z}, 604 Kind: annotation.PreSyn, 605 }, 606 []annotation.Relationship{}, 607 } 608 synapses = append(synapses, e) 609 } 610 } 611 } 612 testJSON, err := json.Marshal(synapses) 613 if err != nil { 614 t.Fatal(err) 615 } 616 url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid) 617 server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON))) 618 619 // Create labelsz instances synced to the above annotations AFTER population so need resync. 620 server.CreateTestInstance(t, uuid, "labelsz", "noroi", config) 621 server.CreateTestSync(t, uuid, "noroi", "mysynapses") 622 config.Set("ROI", fmt.Sprintf("myroi,%s", uuid)) 623 server.CreateTestInstance(t, uuid, "labelsz", "withroi", config) 624 server.CreateTestSync(t, uuid, "withroi", "mysynapses") 625 626 // Do the reload. 627 url = fmt.Sprintf("%snode/%s/noroi/reload", server.WebAPIPath, uuid) 628 server.TestHTTP(t, "POST", url, nil) 629 url = fmt.Sprintf("%snode/%s/withroi/reload", server.WebAPIPath, uuid) 630 server.TestHTTP(t, "POST", url, nil) 631 632 checkSequencing(t, uuid) 633 } 634 635 func TestLabelmap(t *testing.T) { 636 if err := server.OpenTest(); err != nil { 637 t.Fatalf("can't open test server: %v\n", err) 638 } 639 defer server.CloseTest() 640 641 // Create testbed volume and data instances 642 uuid, _ := datastore.NewTestRepo() 643 var config dvid.Config 644 server.CreateTestInstance(t, uuid, "labelmap", "labels", config) 645 646 sz := int32(128) // size of one cube side in voxels 647 numVoxels := sz * sz * sz 648 numLabels := 100 649 data := make([]byte, numVoxels*8) 650 offset := 0 651 for v := 0; v < int(numVoxels); v++ { 652 label := uint64(rand.Int()%numLabels) + 1 653 binary.LittleEndian.PutUint64(data[offset:offset+8], label) 654 offset += 8 655 } 656 _ = createLabelTestVolume(t, uuid, "labels") 657 apiStr := fmt.Sprintf("%snode/%s/labels/raw/0_1_2/%d_%d_%d/0_0_0", server.WebAPIPath, 658 uuid, sz, sz, sz) 659 server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(data)) 660 661 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 662 t.Fatalf("Error blocking on sync of labels: %v\n", err) 663 } 664 665 // Add annotations syncing with "labels" instance. 666 server.CreateTestInstance(t, uuid, "annotation", "mysynapses", config) 667 server.CreateTestSync(t, uuid, "mysynapses", "labels") 668 669 // Create a ROI that will be used for our labelsz. 670 server.CreateTestInstance(t, uuid, "roi", "myroi", config) 671 roiRequest := fmt.Sprintf("%snode/%s/myroi/roi", server.WebAPIPath, uuid) 672 server.TestHTTP(t, "POST", roiRequest, getROIReader()) 673 674 // Create labelsz instances synced to the above annotations. 675 server.CreateTestInstance(t, uuid, "labelsz", "noroi", config) 676 server.CreateTestSync(t, uuid, "noroi", "mysynapses") 677 config.Set("ROI", fmt.Sprintf("myroi,%s", uuid)) 678 server.CreateTestInstance(t, uuid, "labelsz", "withroi", config) 679 server.CreateTestSync(t, uuid, "withroi", "mysynapses") 680 681 labels := []uint64{10, 20, 30, 40, 50} 682 postsynCounts := make(map[uint64]int, numLabels) 683 presynCounts := make(map[uint64]int, numLabels) 684 var synapses annotation.Elements 685 var x, y, z int32 686 // This should put 31x31x31 (29791) PostSyn 687 for z = 4; z < sz; z += 4 { 688 for y = 4; y < sz; y += 4 { 689 for x = 4; x < sz; x += 4 { 690 e := annotation.Element{ 691 annotation.ElementNR{ 692 Pos: dvid.Point3d{x, y, z}, 693 Kind: annotation.PostSyn, 694 }, 695 []annotation.Relationship{}, 696 } 697 synapses = append(synapses, e) 698 offset := (z*sz*sz + y*sz + x) * 8 699 label := binary.LittleEndian.Uint64(data[offset : offset+8]) 700 postsynCounts[label] = postsynCounts[label] + 1 701 } 702 } 703 } 704 // This should put 32x32x32 (32768) PreSyn in volume 705 for z = 2; z < sz; z += 4 { 706 for y = 2; y < sz; y += 4 { 707 for x = 2; x < sz; x += 4 { 708 e := annotation.Element{ 709 annotation.ElementNR{ 710 Pos: dvid.Point3d{x, y, z}, 711 Kind: annotation.PreSyn, 712 }, 713 []annotation.Relationship{}, 714 } 715 synapses = append(synapses, e) 716 offset := (z*sz*sz + y*sz + x) * 8 717 label := binary.LittleEndian.Uint64(data[offset : offset+8]) 718 presynCounts[label] = presynCounts[label] + 1 719 } 720 } 721 } 722 testJSON, err := json.Marshal(synapses) 723 if err != nil { 724 t.Fatal(err) 725 } 726 url := fmt.Sprintf("%snode/%s/mysynapses/elements", server.WebAPIPath, uuid) 727 server.TestHTTP(t, "POST", url, strings.NewReader(string(testJSON))) 728 729 // Need to pause to prevent race condition for when the labelsz is synced. 730 if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil { 731 t.Fatalf("Error blocking on sync of elements to labelsz: %v\n", err) 732 } 733 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 734 t.Fatalf("Error blocking on sync of elements to labelsz: %v\n", err) 735 } 736 737 postStr := "[" 738 for i, label := range labels { 739 postStr += strconv.Itoa(int(label)) 740 if i != len(labels)-1 { 741 postStr += "," 742 } 743 } 744 postStr += "]" 745 url = fmt.Sprintf("%snode/%s/noroi/counts/PreSyn", server.WebAPIPath, uuid) 746 retData := server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr))) 747 var retVal []struct { 748 Label uint64 749 PreSyn int 750 } 751 if err := json.Unmarshal(retData, &retVal); err != nil { 752 t.Errorf("Unable to decode return value: %v\n", retData) 753 } 754 for _, val := range retVal { 755 expected, found := presynCounts[val.Label] 756 if !found { 757 t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData)) 758 } 759 if expected != val.PreSyn { 760 t.Fatalf("Expected label %d presyn to have %d, got %d\n", val.Label, expected, val.PreSyn) 761 } 762 } 763 url = fmt.Sprintf("%snode/%s/noroi/counts/PostSyn", server.WebAPIPath, uuid) 764 retData = server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr))) 765 var retVal2 []struct { 766 Label uint64 767 PostSyn int 768 } 769 if err := json.Unmarshal(retData, &retVal2); err != nil { 770 t.Errorf("Unable to decode return value: %v\n", retData) 771 } 772 for _, val := range retVal2 { 773 expected, found := postsynCounts[val.Label] 774 if !found { 775 t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData)) 776 } 777 if expected != val.PostSyn { 778 t.Fatalf("Expected label %d postsyn to have %d, got %d\n", val.Label, expected, val.PostSyn) 779 } 780 } 781 url = fmt.Sprintf("%snode/%s/noroi/counts/AllSyn", server.WebAPIPath, uuid) 782 retData = server.TestHTTP(t, "GET", url, bytes.NewBuffer([]byte(postStr))) 783 var retVal3 []struct { 784 Label uint64 785 AllSyn int 786 } 787 if err := json.Unmarshal(retData, &retVal3); err != nil { 788 t.Errorf("Unable to decode return value: %v\n", retData) 789 } 790 for _, val := range retVal3 { 791 expectedPre, found := presynCounts[val.Label] 792 if !found { 793 t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData)) 794 } 795 expectedPost, found := postsynCounts[val.Label] 796 if !found { 797 t.Fatalf("Bad label %d returned: %s\n", val.Label, string(retData)) 798 } 799 if expectedPre+expectedPost != val.AllSyn { 800 t.Fatalf("Expected label %d allsyn to have %d+%d, got %d\n", val.Label, expectedPre, expectedPost, val.AllSyn) 801 } 802 } 803 804 // merge 10+20 into 30 and recheck 805 url = fmt.Sprintf("%snode/%s/labels/merge", server.WebAPIPath, uuid) 806 server.TestHTTP(t, "POST", url, bytes.NewBufferString("[30,10,20]")) 807 808 if err := datastore.BlockOnUpdating(uuid, "mysynapses"); err != nil { 809 t.Fatalf("Error blocking on sync of annotations: %v\n", err) 810 } 811 if err := datastore.BlockOnUpdating(uuid, "noroi"); err != nil { 812 t.Fatalf("Error blocking on sync of annotations: %v\n", err) 813 } 814 815 url = fmt.Sprintf("%snode/%s/noroi/count/30/PreSyn", server.WebAPIPath, uuid) 816 retData = server.TestHTTP(t, "GET", url, nil) 817 presynMerged := presynCounts[10] + presynCounts[20] + presynCounts[30] 818 if string(retData) != fmt.Sprintf(`{"Label":30,"PreSyn":%d}`, presynMerged) { 819 t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 30: %s\nlabel 10+20+30 = %d+%d+%d = %d\n", 820 string(retData), presynCounts[10], presynCounts[20], presynCounts[30], presynMerged) 821 } 822 823 url = fmt.Sprintf("%snode/%s/noroi/count/10/PreSyn", server.WebAPIPath, uuid) 824 retData = server.TestHTTP(t, "GET", url, nil) 825 if string(retData) != `{"Label":10,"PreSyn":0}` { 826 t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 10: %s\n", string(retData)) 827 } 828 url = fmt.Sprintf("%snode/%s/noroi/count/20/PreSyn", server.WebAPIPath, uuid) 829 retData = server.TestHTTP(t, "GET", url, nil) 830 if string(retData) != `{"Label":20,"PreSyn":0}` { 831 t.Errorf("Got back incorrect post-merge PreSyn noroi count of label 20: %s\n", string(retData)) 832 } 833 }