github.com/janelia-flyem/dvid@v1.0.0/datatype/keyvalue/keyvalue_test.go (about) 1 package keyvalue 2 3 import ( 4 "archive/tar" 5 "bytes" 6 "crypto/rand" 7 "encoding/json" 8 "fmt" 9 "io" 10 "log" 11 "net/http" 12 "strings" 13 "sync" 14 "testing" 15 16 pb "google.golang.org/protobuf/proto" 17 18 "github.com/janelia-flyem/dvid/datastore" 19 "github.com/janelia-flyem/dvid/datatype/common/proto" 20 "github.com/janelia-flyem/dvid/dvid" 21 "github.com/janelia-flyem/dvid/server" 22 ) 23 24 var ( 25 kvtype datastore.TypeService 26 testMu sync.Mutex 27 ) 28 29 // Sets package-level testRepo and TestVersionID 30 func initTestRepo() (dvid.UUID, dvid.VersionID) { 31 testMu.Lock() 32 defer testMu.Unlock() 33 if kvtype == nil { 34 var err error 35 kvtype, err = datastore.TypeServiceByName(TypeName) 36 if err != nil { 37 log.Fatalf("Can't get keyvalue type: %s\n", err) 38 } 39 } 40 return datastore.NewTestRepo() 41 } 42 43 // Make sure new keyvalue data have different IDs. 44 func TestNewKeyvalueDifferent(t *testing.T) { 45 if err := server.OpenTest(); err != nil { 46 t.Fatalf("can't open test server: %v\n", err) 47 } 48 defer server.CloseTest() 49 50 uuid, _ := initTestRepo() 51 52 // Add data 53 config := dvid.NewConfig() 54 dataservice1, err := datastore.NewData(uuid, kvtype, "instance1", config) 55 if err != nil { 56 t.Errorf("Error creating new keyvalue instance: %v\n", err) 57 } 58 kv1, ok := dataservice1.(*Data) 59 if !ok { 60 t.Errorf("Returned new data instance 1 is not keyvalue.Data\n") 61 } 62 if kv1.DataName() != "instance1" { 63 t.Errorf("New keyvalue data instance name set incorrectly: %q != %q\n", 64 kv1.DataName(), "instance1") 65 } 66 67 dataservice2, err := datastore.NewData(uuid, kvtype, "instance2", config) 68 if err != nil { 69 t.Errorf("Error creating new keyvalue instance: %v\n", err) 70 } 71 kv2, ok := dataservice2.(*Data) 72 if !ok { 73 t.Errorf("Returned new data instance 2 is not keyvalue.Data\n") 74 } 75 76 if kv1.InstanceID() == kv2.InstanceID() { 77 t.Errorf("Instance IDs should be different: %d == %d\n", 78 kv1.InstanceID(), kv2.InstanceID()) 79 } 80 } 81 82 func TestKeyvalueRoundTrip(t *testing.T) { 83 if err := server.OpenTest(); err != nil { 84 t.Fatalf("can't open test server: %v\n", err) 85 } 86 defer server.CloseTest() 87 88 uuid, versionID := initTestRepo() 89 90 // Add data 91 config := dvid.NewConfig() 92 dataservice, err := datastore.NewData(uuid, kvtype, "roundtripper", config) 93 if err != nil { 94 t.Errorf("Error creating new keyvalue instance: %v\n", err) 95 } 96 kvdata, ok := dataservice.(*Data) 97 if !ok { 98 t.Errorf("Returned new data instance is not keyvalue.Data\n") 99 } 100 101 ctx := datastore.NewVersionedCtx(dataservice, versionID) 102 103 keyStr := "testkey.-{}03`~| %@\x01" 104 value := []byte("I like Japan and this is some unicode: \u65e5\u672c\u8a9e") 105 106 if err = kvdata.PutData(ctx, keyStr, value); err != nil { 107 t.Errorf("Could not put keyvalue data: %v\n", err) 108 } 109 110 retrieved, found, err := kvdata.GetData(ctx, keyStr) 111 if err != nil { 112 t.Fatalf("Could not get keyvalue data: %v\n", err) 113 } 114 if !found { 115 t.Fatalf("Could not find put keyvalue\n") 116 } 117 if bytes.Compare(value, retrieved) != 0 { 118 t.Errorf("keyvalue retrieved %q != put %q\n", string(retrieved), string(value)) 119 } 120 } 121 122 func TestKeyvalueRepoPersistence(t *testing.T) { 123 if err := server.OpenTest(); err != nil { 124 t.Fatalf("can't open test server: %v\n", err) 125 } 126 defer server.CloseTest() 127 128 uuid, _ := initTestRepo() 129 130 // Make labels and set various properties 131 config := dvid.NewConfig() 132 dataservice, err := datastore.NewData(uuid, kvtype, "mykv", config) 133 if err != nil { 134 t.Errorf("Unable to create keyvalue instance: %v\n", err) 135 } 136 kvdata, ok := dataservice.(*Data) 137 if !ok { 138 t.Errorf("Can't cast keyvalue data service into keyvalue.Data\n") 139 } 140 oldData := *kvdata 141 142 // Restart test datastore and see if datasets are still there. 143 if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil { 144 t.Fatalf("Unable to save repo during keyvalue persistence test: %v\n", err) 145 } 146 datastore.CloseReopenTest() 147 148 dataservice2, err := datastore.GetDataByUUIDName(uuid, "mykv") 149 if err != nil { 150 t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err) 151 } 152 kvdata2, ok := dataservice2.(*Data) 153 if !ok { 154 t.Errorf("Returned new data instance 2 is not keyvalue.Data\n") 155 } 156 if !oldData.Equals(kvdata2) { 157 t.Errorf("Expected %v, got %v\n", oldData, *kvdata2) 158 } 159 } 160 161 func testRequest(t *testing.T, uuid dvid.UUID, versionID dvid.VersionID, name dvid.InstanceName) { 162 config := dvid.NewConfig() 163 dataservice, err := datastore.NewData(uuid, kvtype, name, config) 164 if err != nil { 165 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 166 } 167 data, ok := dataservice.(*Data) 168 if !ok { 169 t.Fatalf("Returned new data instance is not keyvalue.Data\n") 170 } 171 172 key1 := "mykey" 173 key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1) 174 resp := server.TestHTTPResponse(t, "HEAD", key1req, nil) 175 if resp.Code != http.StatusNotFound { 176 t.Errorf("HEAD on %s did not return 404 (File not found). Status = %d\n", key1req, resp.Code) 177 } 178 179 // PUT a value 180 value1 := "some stuff" 181 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 182 183 resp = server.TestHTTPResponse(t, "HEAD", key1req, nil) 184 if resp.Code != http.StatusOK { 185 t.Errorf("HEAD on %s did not return 200 (OK). Status = %d\n", key1req, resp.Code) 186 } 187 188 // Get back k/v 189 returnValue := server.TestHTTP(t, "GET", key1req, nil) 190 if string(returnValue) != value1 { 191 t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue)) 192 } 193 194 // Expect error if no key used. 195 badrequest := fmt.Sprintf("%snode/%s/%s/key/", server.WebAPIPath, uuid, data.DataName()) 196 server.TestBadHTTP(t, "GET", badrequest, nil) 197 198 // Add 2nd k/v 199 key2 := "my2ndkey" 200 key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2) 201 202 resp = server.TestHTTPResponse(t, "HEAD", key2req, nil) 203 if resp.Code != http.StatusNotFound { 204 t.Errorf("HEAD on %s did not return 404 (File Not Found). Status = %d\n", key2req, resp.Code) 205 } 206 207 value2 := "more good stuff" 208 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 209 210 resp = server.TestHTTPResponse(t, "HEAD", key2req, nil) 211 if resp.Code != http.StatusOK { 212 t.Errorf("HEAD on %s did not return 200 (OK). Status = %d\n", key2req, resp.Code) 213 } 214 215 // Add 3rd k/v 216 key3 := "heresanotherkey" 217 value3 := "my 3rd value" 218 key3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key3) 219 server.TestHTTP(t, "POST", key3req, strings.NewReader(value3)) 220 221 // Check return of first two keys in range. 222 rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(), 223 "my", "zebra") 224 returnValue = server.TestHTTP(t, "GET", rangereq, nil) 225 226 var retrievedKeys []string 227 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 228 t.Errorf("Bad key range request unmarshal: %v\n", err) 229 } 230 if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" { 231 t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n", 232 key2, key1, string(returnValue)) 233 } 234 235 // Check return of all keys 236 allkeyreq := fmt.Sprintf("%snode/%s/%s/keys", server.WebAPIPath, uuid, data.DataName()) 237 returnValue = server.TestHTTP(t, "GET", allkeyreq, nil) 238 239 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 240 t.Errorf("Bad key range request unmarshal: %v\n", err) 241 } 242 if len(retrievedKeys) != 3 || retrievedKeys[0] != "heresanotherkey" && retrievedKeys[1] != "my2ndKey" && retrievedKeys[2] != "mykey" { 243 t.Errorf("Bad all key request return. Expected: [%q,%q,%q]. Got: %s\n", 244 key3, key2, key1, string(returnValue)) 245 } 246 } 247 248 func TestKeyvalueRequests(t *testing.T) { 249 if err := server.OpenTest(); err != nil { 250 t.Fatalf("can't open test server: %v\n", err) 251 } 252 defer server.CloseTest() 253 254 uuid, versionID := initTestRepo() 255 256 testRequest(t, uuid, versionID, "mykeyvalue") 257 } 258 259 type resolveResp struct { 260 Child dvid.UUID `json:"child"` 261 } 262 263 func TestKeyvalueRange(t *testing.T) { 264 if err := server.OpenTest(); err != nil { 265 t.Fatalf("can't open test server: %v\n", err) 266 } 267 defer server.CloseTest() 268 269 uuid, _ := initTestRepo() 270 271 config := dvid.NewConfig() 272 config.Set("versioned", "false") 273 _, err := datastore.NewData(uuid, kvtype, "unversiontest", config) 274 if err != nil { 275 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 276 } 277 278 // PUT a value 279 key1 := "key1" 280 value1 := `[1, 2, 3]` 281 key1req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, key1) 282 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 283 284 returnValue := server.TestHTTP(t, "GET", key1req, nil) 285 if string(returnValue) != value1 { 286 t.Errorf("Error on key %q: expected %s, got %s\n", key1, value1, string(returnValue)) 287 } 288 289 // Add 2nd k/v 290 key2 := "key2" 291 value2 := `{"foo":"a string", "bar":"another string", "baz":[1, 2, 3]}` 292 key2req := fmt.Sprintf("%snode/%s/unversiontest/key/%s", server.WebAPIPath, uuid, key2) 293 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 294 295 // Test 296 rangeReq := fmt.Sprintf("%snode/%s/unversiontest/keyrangevalues/a/zz", server.WebAPIPath, uuid) 297 expectedJSON := `{"key1":[1, 2, 3],"key2":` + value2 + "}" 298 299 returnValue = server.TestHTTP(t, "GET", rangeReq+"?json=true", nil) 300 if string(returnValue) != expectedJSON { 301 t.Errorf("Error on keyrangevalues: got %s, expected %s\n", string(returnValue), expectedJSON) 302 } 303 } 304 305 func TestKeyvalueUnversioned(t *testing.T) { 306 if err := server.OpenTest(); err != nil { 307 t.Fatalf("can't open test server: %v\n", err) 308 } 309 defer server.CloseTest() 310 311 uuid, _ := initTestRepo() 312 313 config := dvid.NewConfig() 314 config.Set("versioned", "false") 315 dataservice, err := datastore.NewData(uuid, kvtype, "unversiontest", config) 316 if err != nil { 317 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 318 } 319 data, ok := dataservice.(*Data) 320 if !ok { 321 t.Fatalf("Returned new data instance is not keyvalue.Data\n") 322 } 323 324 // PUT a value 325 key1 := "mykey" 326 value1 := "some stuff" 327 key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1) 328 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 329 330 // Add 2nd k/v 331 key2 := "my2ndkey" 332 value2 := "more good stuff" 333 key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2) 334 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 335 336 // Create a new version in repo 337 if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil { 338 t.Errorf("Unable to lock root node %s: %v\n", uuid, err) 339 } 340 uuid2, err := datastore.NewVersion(uuid, "some child", "", nil) 341 if err != nil { 342 t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err) 343 } 344 _, err = datastore.VersionFromUUID(uuid2) 345 if err != nil { 346 t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err) 347 } 348 349 // Change the 2nd k/v 350 uuid2val := "this is completely different" 351 uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2) 352 server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val)) 353 354 // Now the first version value should equal the new value 355 returnValue := server.TestHTTP(t, "GET", key2req, nil) 356 if string(returnValue) != uuid2val { 357 t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 358 } 359 360 // Get the second version value 361 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 362 if string(returnValue) != uuid2val { 363 t.Errorf("Error on unversioned key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 364 } 365 366 // Check return of first two keys in range. 367 rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(), 368 "my", "zebra") 369 returnValue = server.TestHTTP(t, "GET", rangereq, nil) 370 371 var retrievedKeys []string 372 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 373 t.Errorf("Bad key range request unmarshal: %v\n", err) 374 } 375 if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" { 376 t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n", 377 key1, key2, string(returnValue)) 378 } 379 380 // Commit the repo 381 if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil { 382 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 383 } 384 385 // Make grandchild of root 386 uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil) 387 if err != nil { 388 t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err) 389 } 390 391 // Delete the 2nd k/v 392 uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2) 393 server.TestHTTP(t, "DELETE", uuid3req, nil) 394 395 server.TestBadHTTP(t, "GET", uuid3req, nil) 396 397 // Make sure the 2nd k/v is now missing for previous versions. 398 server.TestBadHTTP(t, "GET", key2req, nil) 399 server.TestBadHTTP(t, "GET", uuid2req, nil) 400 401 // Make a child 402 if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil { 403 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 404 } 405 uuid4, err := datastore.NewVersion(uuid3, "some child", "", nil) 406 if err != nil { 407 t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err) 408 } 409 410 // Change the 2nd k/v 411 uuid4val := "we are reintroducing this k/v" 412 uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2) 413 server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val)) 414 415 if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil { 416 t.Errorf("Unable to commit node %s: %v\n", uuid4, err) 417 } 418 419 // Make sure the 2nd k/v is correct for each of previous versions. 420 returnValue = server.TestHTTP(t, "GET", key2req, nil) 421 if string(returnValue) != uuid4val { 422 t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue)) 423 } 424 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 425 if string(returnValue) != uuid4val { 426 t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue)) 427 } 428 returnValue = server.TestHTTP(t, "GET", uuid3req, nil) 429 if string(returnValue) != uuid4val { 430 t.Errorf("Error on third version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue)) 431 } 432 returnValue = server.TestHTTP(t, "GET", uuid4req, nil) 433 if string(returnValue) != uuid4val { 434 t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue)) 435 } 436 } 437 438 func TestKeyvalueVersioning(t *testing.T) { 439 if err := server.OpenTest(); err != nil { 440 t.Fatalf("can't open test server: %v\n", err) 441 } 442 defer server.CloseTest() 443 444 uuid, _ := initTestRepo() 445 446 config := dvid.NewConfig() 447 dataservice, err := datastore.NewData(uuid, kvtype, "versiontest", config) 448 if err != nil { 449 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 450 } 451 data, ok := dataservice.(*Data) 452 if !ok { 453 t.Fatalf("Returned new data instance is not keyvalue.Data\n") 454 } 455 456 // PUT a value 457 key1 := "mykey" 458 value1 := `"some stuff"` 459 key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1) 460 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 461 462 // Add 2nd k/v 463 key2 := "my2ndkey" 464 value2 := `"more good stuff"` 465 key2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key2) 466 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 467 468 // Use the batch POST 469 payloadSize := 1000 470 var kvs proto.KeyValues 471 kvs.Kvs = make([]*proto.KeyValue, 5) 472 var payload [5][]byte 473 for i := 0; i < 5; i++ { 474 payload[i] = make([]byte, i*payloadSize+10) 475 n, err := rand.Read(payload[i]) 476 if n != i*payloadSize+10 { 477 t.Fatalf("couldn't create payload %d\n", i) 478 } 479 if err != nil { 480 t.Fatalf("couldn't create payload %d: %v\n", i, err) 481 } 482 kvs.Kvs[i] = &proto.KeyValue{ 483 Key: fmt.Sprintf("batchkey-%d", i), 484 Value: payload[i], 485 } 486 } 487 serialization, err := pb.Marshal(&kvs) 488 if err != nil { 489 t.Fatalf("couldn't serialize keyvalues: %v\n", err) 490 } 491 kvsPostReq := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName()) 492 server.TestHTTP(t, "POST", kvsPostReq, bytes.NewReader(serialization)) 493 494 // Create a new version in repo 495 if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil { 496 t.Errorf("Unable to lock root node %s: %v\n", uuid, err) 497 } 498 uuid2, err := datastore.NewVersion(uuid, "some child", "", nil) 499 if err != nil { 500 t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err) 501 } 502 _, err = datastore.VersionFromUUID(uuid2) 503 if err != nil { 504 t.Fatalf("Unable to get version ID from new uuid %s: %v\n", uuid2, err) 505 } 506 507 // Change the 2nd k/v 508 uuid2val := "this is completely different" 509 uuid2req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid2, data.DataName(), key2) 510 server.TestHTTP(t, "POST", uuid2req, strings.NewReader(uuid2val)) 511 512 // Get the first version value 513 returnValue := server.TestHTTP(t, "GET", key2req, nil) 514 if string(returnValue) != value2 { 515 t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue)) 516 } 517 518 // Get the second version value 519 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 520 if string(returnValue) != uuid2val { 521 t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 522 } 523 524 // Check return of first two keys in range. 525 rangereq := fmt.Sprintf("%snode/%s/%s/keyrange/%s/%s", server.WebAPIPath, uuid, data.DataName(), 526 "my", "zebra") 527 returnValue = server.TestHTTP(t, "GET", rangereq, nil) 528 529 var retrievedKeys []string 530 if err = json.Unmarshal(returnValue, &retrievedKeys); err != nil { 531 t.Errorf("Bad key range request unmarshal: %v\n", err) 532 } 533 if len(retrievedKeys) != 2 || retrievedKeys[1] != "mykey" && retrievedKeys[0] != "my2ndKey" { 534 t.Errorf("Bad key range request return. Expected: [%q,%q]. Got: %s\n", 535 key1, key2, string(returnValue)) 536 } 537 538 // Check some values using GET /keyvalues (json) 539 getreq0 := fmt.Sprintf("%snode/%s/%s/keyvalues?json=true", server.WebAPIPath, uuid, data.DataName()) 540 expectedKeys := []string{key1, key2, "missing-key"} 541 542 jsonKeys, err := json.Marshal(expectedKeys) 543 if err != nil { 544 t.Fatalf("couldn't parse expectedKeys\n") 545 } 546 547 returnValue = server.TestHTTP(t, "GET", getreq0, bytes.NewBuffer(jsonKeys)) 548 expectedJSON := fmt.Sprintf(`{%q:%s,%q:%s,"missing-key":{}}`, key1, value1, key2, value2) 549 if string(returnValue) != expectedJSON { 550 t.Errorf("Error on keyvalues JSON return: got %s, expected %s\n", string(returnValue), expectedJSON) 551 } 552 553 // Check values from batch POST using individual key gets 554 for i := 0; i < 5; i++ { 555 k := fmt.Sprintf("batchkey-%d", i) 556 keyreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), k) 557 returnValue := server.TestHTTP(t, "GET", keyreq, nil) 558 if len(returnValue) != i*payloadSize+10 { 559 t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", k, i+10, len(returnValue)) 560 } 561 if !bytes.Equal(payload[i], returnValue) { 562 t.Fatalf("bad response for key %q\n", k) 563 } 564 } 565 566 // Check some values from batch POST using GET /keyvalues?jsontar=true 567 getreq1 := fmt.Sprintf("%snode/%s/%s/keyvalues?jsontar=true", server.WebAPIPath, uuid, data.DataName()) 568 tardata := server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["batchkey-0","batchkey-1","batchkey-4","batchkey-1000"]`)) 569 tarbuf := bytes.NewBuffer(tardata) 570 tr := tar.NewReader(tarbuf) 571 expectedKeys = []string{"batchkey-0", "batchkey-1", "batchkey-4", "batchkey-1000"} 572 keyNum := 0 573 for { 574 hdr, err := tr.Next() 575 if err == io.EOF { 576 break 577 } 578 if err != nil { 579 t.Fatalf("error parsing tar: %v\n", err) 580 } 581 if hdr.Name != expectedKeys[keyNum] { 582 t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name) 583 } 584 var i int 585 if _, err = fmt.Sscanf(hdr.Name, "batchkey-%d", &i); err != nil { 586 t.Fatalf("error parsing tar file hdr %q: %v\n", hdr.Name, err) 587 } 588 var val bytes.Buffer 589 if _, err := io.Copy(&val, tr); err != nil { 590 t.Fatalf("error reading tar data: %v\n", err) 591 } 592 returnValue := val.Bytes() 593 if i != 1000 { 594 if len(returnValue) != i*payloadSize+10 { 595 t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", hdr.Name, i*payloadSize+10, len(returnValue)) 596 } 597 if !bytes.Equal(payload[i], returnValue) { 598 t.Fatalf("bad response for key %q\n", hdr.Name) 599 } 600 } else { 601 if len(returnValue) != 0 { 602 t.Fatalf("expected 0 byte response for key %q, got %d bytes\n", hdr.Name, len(returnValue)) 603 } 604 } 605 keyNum++ 606 } 607 if keyNum != 4 { 608 t.Fatalf("Got %d keys when there should have been 4\n", keyNum) 609 } 610 tardata = server.TestHTTP(t, "GET", getreq1, bytes.NewBufferString(`["batchkey-3"]`)) 611 tarbuf = bytes.NewBuffer(tardata) 612 tr = tar.NewReader(tarbuf) 613 expectedKeys = []string{"batchkey-3"} 614 keyNum = 0 615 for { 616 hdr, err := tr.Next() 617 if err == io.EOF { 618 break 619 } 620 if err != nil { 621 t.Fatalf("error parsing tar: %v\n", err) 622 } 623 if hdr.Name != expectedKeys[keyNum] { 624 t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], hdr.Name) 625 } 626 var i int 627 if _, err = fmt.Sscanf(hdr.Name, "batchkey-%d", &i); err != nil { 628 t.Fatalf("error parsing tar file hdr %q: %v\n", hdr.Name, err) 629 } 630 var val bytes.Buffer 631 if _, err := io.Copy(&val, tr); err != nil { 632 t.Fatalf("error reading tar data: %v\n", err) 633 } 634 returnValue := val.Bytes() 635 if len(returnValue) != i*payloadSize+10 { 636 t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", hdr.Name, i*payloadSize+10, len(returnValue)) 637 } 638 if !bytes.Equal(payload[i], returnValue) { 639 t.Fatalf("bad response for key %q\n", hdr.Name) 640 } 641 keyNum++ 642 } 643 if keyNum != 1 { 644 t.Fatalf("Got %d keys when there should have been 1\n", keyNum) 645 } 646 647 // Check some values from batch POST using GET /keyvalues (protobuf3) 648 getreq2 := fmt.Sprintf("%snode/%s/%s/keyvalues", server.WebAPIPath, uuid, data.DataName()) 649 expectedKeys = []string{"batchkey-1", "batchkey-2", "batchkey-3", "batchkey-1000"} 650 pbufKeys := proto.Keys{ 651 Keys: expectedKeys, 652 } 653 keysSerialization, err := pb.Marshal(&pbufKeys) 654 if err != nil { 655 t.Fatalf("couldn't serialized protobuf keys: %v\n", err) 656 } 657 keyvaluesSerialization := server.TestHTTP(t, "GET", getreq2, bytes.NewBuffer(keysSerialization)) 658 var pbKVs proto.KeyValues 659 if err := pb.Unmarshal(keyvaluesSerialization, &pbKVs); err != nil { 660 t.Fatalf("couldn't unmarshal keyvalues protobuf: %v\n", err) 661 } 662 if len(pbKVs.Kvs) != 4 { 663 t.Fatalf("expected 4 kv pairs returned, got %d\n", len(pbKVs.Kvs)) 664 } 665 for keyNum, kv := range pbKVs.Kvs { 666 if kv.Key != expectedKeys[keyNum] { 667 t.Fatalf("expected for key %d %q, got %q", keyNum, expectedKeys[keyNum], kv.Key) 668 } 669 var i int 670 if _, err = fmt.Sscanf(kv.Key, "batchkey-%d", &i); err != nil { 671 t.Fatalf("error parsing key %d %q: %v\n", keyNum, kv.Key, err) 672 } 673 if i != 1000 { 674 if len(kv.Value) != i*payloadSize+10 { 675 t.Errorf("Expected batch POST key %q to have value with %d bytes, got %d instead\n", kv.Key, i*payloadSize+10, len(kv.Value)) 676 } 677 if !bytes.Equal(payload[i], kv.Value) { 678 t.Fatalf("bad response for key %q\n", kv.Key) 679 } 680 } else { 681 if len(kv.Value) != 0 { 682 t.Fatalf("expected 0 byte value for key %q but got %d bytes instead\n", kv.Key, len(kv.Value)) 683 } 684 } 685 } 686 687 // Commit the repo 688 if err = datastore.Commit(uuid2, "my 2nd commit msg", []string{"changed 2nd k/v"}); err != nil { 689 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 690 } 691 692 // Make grandchild of root 693 uuid3, err := datastore.NewVersion(uuid2, "some child", "", nil) 694 if err != nil { 695 t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err) 696 } 697 698 // Delete the 2nd k/v 699 uuid3req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid3, data.DataName(), key2) 700 server.TestHTTP(t, "DELETE", uuid3req, nil) 701 702 server.TestBadHTTP(t, "GET", uuid3req, nil) 703 704 // Make sure the 2nd k/v is correct for each of previous versions. 705 returnValue = server.TestHTTP(t, "GET", key2req, nil) 706 if string(returnValue) != value2 { 707 t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue)) 708 } 709 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 710 if string(returnValue) != uuid2val { 711 t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 712 } 713 714 // Make a child 715 if err = datastore.Commit(uuid3, "my 3rd commit msg", []string{"deleted 2nd k/v"}); err != nil { 716 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 717 } 718 uuid4, err := datastore.NewVersion(uuid3, "some child", "", nil) 719 if err != nil { 720 t.Fatalf("Unable to create new version off node %s: %v\n", uuid3, err) 721 } 722 723 // Change the 2nd k/v 724 uuid4val := "we are reintroducing this k/v" 725 uuid4req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid4, data.DataName(), key2) 726 server.TestHTTP(t, "POST", uuid4req, strings.NewReader(uuid4val)) 727 728 if err = datastore.Commit(uuid4, "commit node 4", []string{"we modified stuff"}); err != nil { 729 t.Errorf("Unable to commit node %s: %v\n", uuid4, err) 730 } 731 732 // Make sure the 2nd k/v is correct for each of previous versions. 733 returnValue = server.TestHTTP(t, "GET", key2req, nil) 734 if string(returnValue) != value2 { 735 t.Errorf("Error on first version, key %q: expected %s, got %s\n", key2, value2, string(returnValue)) 736 } 737 returnValue = server.TestHTTP(t, "GET", uuid2req, nil) 738 if string(returnValue) != uuid2val { 739 t.Errorf("Error on second version, key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 740 } 741 server.TestBadHTTP(t, "GET", uuid3req, nil) 742 returnValue = server.TestHTTP(t, "GET", uuid4req, nil) 743 if string(returnValue) != uuid4val { 744 t.Errorf("Error on fourth version, key %q: expected %s, got %s\n", key2, uuid4val, string(returnValue)) 745 } 746 747 // Let's try a merge! 748 749 // Make a child off the 2nd version from root. 750 uuid5, err := datastore.NewVersion(uuid2, "some child", "some child", nil) 751 if err != nil { 752 t.Fatalf("Unable to create new version off node %s: %v\n", uuid2, err) 753 } 754 755 // Store new stuff in 2nd k/v 756 uuid5val := "this is forked value" 757 uuid5req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid5, data.DataName(), key2) 758 server.TestHTTP(t, "POST", uuid5req, strings.NewReader(uuid5val)) 759 760 returnValue = server.TestHTTP(t, "GET", uuid5req, nil) 761 if string(returnValue) != uuid5val { 762 t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue)) 763 } 764 765 // Commit node 766 if err = datastore.Commit(uuid5, "forked node", []string{"we modified stuff"}); err != nil { 767 t.Errorf("Unable to commit node %s: %v\n", uuid5, err) 768 } 769 770 // Should be able to merge using conflict-free (disjoint at key level) merge even though 771 // its conflicted. Will get lazy error on request. 772 badChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid5}, "some child", datastore.MergeConflictFree) 773 if err != nil { 774 t.Errorf("Error doing merge: %v\n", err) 775 } 776 childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, badChild, data.DataName(), key2) 777 server.TestBadHTTP(t, "GET", childreq, nil) 778 779 // Manually fix conflict: Branch, and then delete 2nd k/v and commit. 780 uuid6, err := datastore.NewVersion(uuid5, "some child", "", nil) 781 if err != nil { 782 t.Fatalf("Unable to create new version off node %s: %v\n", uuid5, err) 783 } 784 785 uuid6req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid6, data.DataName(), key2) 786 server.TestHTTP(t, "DELETE", uuid6req, nil) 787 server.TestBadHTTP(t, "GET", uuid6req, nil) 788 789 if err = datastore.Commit(uuid6, "deleted forked node 2nd k/v", []string{"we modified stuff"}); err != nil { 790 t.Errorf("Unable to commit node %s: %s\n", uuid6, err) 791 } 792 793 // Should now be able to correctly merge the two branches. 794 goodChild, err := datastore.Merge([]dvid.UUID{uuid4, uuid6}, "merging stuff", datastore.MergeConflictFree) 795 if err != nil { 796 t.Errorf("Error doing merge: %v\n", err) 797 } 798 799 // We should be able to see just the original uuid4 value of the 2nd k/v 800 childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, goodChild, data.DataName(), key2) 801 returnValue = server.TestHTTP(t, "GET", childreq, nil) 802 if string(returnValue) != uuid4val { 803 t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue)) 804 } 805 806 // Apply the automatic conflict resolution using ordering. 807 resolveNote := fmt.Sprintf(`{"data":["versiontest"],"parents":[%q,%q],"note":"automatic resolved merge"}`, uuid5, uuid4) 808 resolveReq := fmt.Sprintf("%srepo/%s/resolve", server.WebAPIPath, uuid4) 809 returnValue = server.TestHTTP(t, "POST", resolveReq, bytes.NewBufferString(resolveNote)) 810 resolveResp := struct { 811 Child dvid.UUID `json:"child"` 812 }{} 813 if err := json.Unmarshal(returnValue, &resolveResp); err != nil { 814 t.Fatalf("Can't parse return of resolve request: %s\n", string(returnValue)) 815 } 816 817 // We should now see the uuid5 version of the 2nd k/v in the returned merged node. 818 childreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, resolveResp.Child, data.DataName(), key2) 819 returnValue = server.TestHTTP(t, "GET", childreq, nil) 820 if string(returnValue) != uuid5val { 821 t.Errorf("Error on auto merged child, key %q: expected %q, got %q\n", key2, uuid5val, string(returnValue)) 822 } 823 824 // Introduce a child off root but don't add 2nd k/v to it. 825 uuid7, err := datastore.NewVersion(uuid, "2nd child off root", "2nd child off root", nil) 826 if err != nil { 827 t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err) 828 } 829 if err = datastore.Commit(uuid7, "useless node", []string{"we modified nothing!"}); err != nil { 830 t.Errorf("Unable to commit node %s: %v\n", uuid7, err) 831 } 832 833 // Now merge the previously merged node with the newly created "blank" child off root. 834 if err = datastore.Commit(goodChild, "this was a good merge", []string{}); err != nil { 835 t.Errorf("Unable to commit node %s: %v\n", goodChild, err) 836 } 837 merge2, err := datastore.Merge([]dvid.UUID{goodChild, uuid7}, "merging a useless path", datastore.MergeConflictFree) 838 if err != nil { 839 t.Errorf("Error doing merge: %v\n", err) 840 } 841 merge3, err := datastore.Merge([]dvid.UUID{uuid7, goodChild}, "merging a useless path in reverse order", datastore.MergeConflictFree) 842 if err != nil { 843 t.Errorf("Error doing merge: %v\n", err) 844 } 845 846 // We should still be conflict free since 2nd key in left parent path will take precedent over shared 2nd key 847 // in root. This tests our invalidation of ancestors. 848 toughreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge2, data.DataName(), key2) 849 returnValue = server.TestHTTP(t, "GET", toughreq, nil) 850 if string(returnValue) != uuid4val { 851 t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue)) 852 } 853 toughreq = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, merge3, data.DataName(), key2) 854 returnValue = server.TestHTTP(t, "GET", toughreq, nil) 855 if string(returnValue) != uuid4val { 856 t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key2, uuid4val, string(returnValue)) 857 } 858 859 // Create a new keyvalue data instance at an interior non-root node. 860 uuid8, err := datastore.NewVersion(uuid7, "open leaf child", "", nil) 861 if err != nil { 862 t.Fatalf("Unable to create new version off %s: %v\n", uuid7, err) 863 } 864 dataservice, err = datastore.NewData(uuid8, kvtype, "leafdata", config) 865 if err != nil { 866 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 867 } 868 leafdata, ok := dataservice.(*Data) 869 if !ok { 870 t.Fatalf("Returned new data instance is not keyvalue.Data\n") 871 } 872 873 // PUT a value 874 key1req = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid8, leafdata.DataName(), key1) 875 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 876 877 // Add 2nd k/v 878 key2req = fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid8, leafdata.DataName(), key2) 879 server.TestHTTP(t, "POST", key2req, strings.NewReader(value2)) 880 881 // Change the 2nd k/v 882 server.TestHTTP(t, "POST", key2req, strings.NewReader(uuid2val)) 883 884 // Check the values 885 returnValue = server.TestHTTP(t, "GET", key1req, nil) 886 if string(returnValue) != value1 { 887 t.Errorf("Key %q: expected %s, got %s\n", key1, value1, string(returnValue)) 888 } 889 returnValue = server.TestHTTP(t, "GET", key2req, nil) 890 if string(returnValue) != uuid2val { 891 t.Errorf("Key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 892 } 893 894 // Change the name of the interior data. 895 if err = datastore.RenameData(uuid8, "leafdata", "versiontest", "foobar"); err == nil { 896 t.Fatalf("Should have been prevented from renaming data 'leafdata' to existing data 'versiontest'!\n") 897 } 898 if err = datastore.RenameData(uuid8, "leafdata", "renamedData", "foobar"); err != nil { 899 t.Fatalf("Error renaming leafdata: %v\n", err) 900 } 901 902 // Check the values 903 key1req = fmt.Sprintf("%snode/%s/renamedData/key/%s", server.WebAPIPath, uuid8, key1) 904 key2req = fmt.Sprintf("%snode/%s/renamedData/key/%s", server.WebAPIPath, uuid8, key2) 905 returnValue = server.TestHTTP(t, "GET", key1req, nil) 906 if string(returnValue) != value1 { 907 t.Errorf("Key %q: expected %s, got %s\n", key1, value1, string(returnValue)) 908 } 909 returnValue = server.TestHTTP(t, "GET", key2req, nil) 910 if string(returnValue) != uuid2val { 911 t.Errorf("Key %q: expected %s, got %s\n", key2, uuid2val, string(returnValue)) 912 } 913 } 914 915 // Test added after error in getting two paths to the same ancestor k/v after merge. 916 func TestDiamondGetOnMerge(t *testing.T) { 917 if err := server.OpenTest(); err != nil { 918 t.Fatalf("can't open test server: %v\n", err) 919 } 920 defer server.CloseTest() 921 922 uuid, _ := initTestRepo() 923 924 config := dvid.NewConfig() 925 dataservice, err := datastore.NewData(uuid, kvtype, "mergetest", config) 926 if err != nil { 927 t.Fatalf("Error creating new keyvalue instance: %v\n", err) 928 } 929 data, ok := dataservice.(*Data) 930 if !ok { 931 t.Fatalf("Returned new data instance is not keyvalue.Data\n") 932 } 933 934 // PUT a value 935 key1 := "mykey" 936 value1 := "some stuff" 937 key1req := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, uuid, data.DataName(), key1) 938 server.TestHTTP(t, "POST", key1req, strings.NewReader(value1)) 939 940 if err = datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil { 941 t.Errorf("Unable to lock root node %s: %v\n", uuid, err) 942 } 943 uuid2, err := datastore.NewVersion(uuid, "first child", "", nil) 944 if err != nil { 945 t.Fatalf("Unable to create 1st child off root %s: %v\n", uuid, err) 946 } 947 if err = datastore.Commit(uuid2, "first child", nil); err != nil { 948 t.Errorf("Unable to commit node %s: %v\n", uuid2, err) 949 } 950 uuid3, err := datastore.NewVersion(uuid, "second child", "newbranch", nil) 951 if err != nil { 952 t.Fatalf("Unable to create 2nd child off root %s: %v\n", uuid, err) 953 } 954 if err = datastore.Commit(uuid3, "second child", nil); err != nil { 955 t.Errorf("Unable to commit node %s: %v\n", uuid3, err) 956 } 957 958 child, err := datastore.Merge([]dvid.UUID{uuid2, uuid3}, "merging stuff", datastore.MergeConflictFree) 959 if err != nil { 960 t.Errorf("Error doing merge: %v\n", err) 961 } 962 963 // We should be able to see just the original uuid value of the k/v 964 childreq := fmt.Sprintf("%snode/%s/%s/key/%s", server.WebAPIPath, child, data.DataName(), key1) 965 returnValue := server.TestHTTP(t, "GET", childreq, nil) 966 if string(returnValue) != value1 { 967 t.Errorf("Error on merged child, key %q: expected %q, got %q\n", key1, value1, string(returnValue)) 968 } 969 } 970 971 /* 972 TODO -- Complete when mutation log access added, so we can check mutation is logged and test blobstore 973 fetch with reference. 974 975 func TestBlobstoreMutationLog(t *testing.T) { 976 if err := server.OpenTest(); err != nil { 977 t.Fatalf("can't open test server: %v\n", err) 978 } 979 defer server.CloseTest() 980 981 uuid, _ := initTestRepo() 982 var config dvid.Config 983 server.CreateTestInstance(t, uuid, "keyvalue", "mykv", config) 984 985 key := "mykey" 986 value := "some stuff" 987 url := fmt.Sprintf("%snode/%s/mykv/key/%s", server.WebAPIPath, uuid, key) 988 server.TestHTTP(t, "POST", url, strings.NewReader(value)) 989 } 990 991 */