github.com/holochain/holochain-proto@v0.1.0-alpha-26.0.20200915073418-5c83169c9b5b/buntdbht.go (about) 1 // Copyright (C) 2013-2018, The MetaCurrency Project (Eric Harris-Braun, Arthur Brock, et. al.) 2 // Use of this source code is governed by GPLv3 found in the LICENSE file 3 //---------------------------------------------------------------------------------------- 4 5 // implements a buntdb based instance of HashTable 6 7 package holochain 8 9 import ( 10 "bytes" 11 "encoding/json" 12 "fmt" 13 "strconv" 14 "strings" 15 16 . "github.com/holochain/holochain-proto/hash" 17 peer "github.com/libp2p/go-libp2p-peer" 18 "github.com/tidwall/buntdb" 19 ) 20 21 type BuntHT struct { 22 db *buntdb.DB 23 } 24 25 // linkEvent represents the value stored in buntDB associated with a 26 // link key for one source having stored one LinkingEntry 27 // (The Link struct defined in entry.go is encoded in the key used for buntDB) 28 type linkEvent struct { 29 Status int 30 Source string 31 LinksEntry string 32 } 33 34 func (ht *BuntHT) Open(options interface{}) (err error) { 35 file := options.(string) 36 db, err := buntdb.Open(file) 37 if err != nil { 38 panic(err) 39 } 40 db.CreateIndex("link", "link:*", buntdb.IndexString) 41 db.CreateIndex("idx", "idx:*", buntdb.IndexInt) 42 db.CreateIndex("peer", "peer:*", buntdb.IndexString) 43 db.CreateIndex("list", "list:*", buntdb.IndexString) 44 db.CreateIndex("entry", "entry:*", buntdb.IndexString) 45 46 ht.db = db 47 return 48 } 49 50 // Put stores a value to the DHT store 51 // N.B. This call assumes that the value has already been validated 52 func (ht *BuntHT) Put(m *Message, entryType string, key Hash, src peer.ID, value []byte, status int) (err error) { 53 k := key.String() 54 err = ht.db.Update(func(tx *buntdb.Tx) error { 55 _, err := incIdx(tx, m) 56 if err != nil { 57 return err 58 } 59 _, _, err = tx.Set("entry:"+k, string(value), nil) 60 if err != nil { 61 return err 62 } 63 _, _, err = tx.Set("type:"+k, entryType, nil) 64 if err != nil { 65 return err 66 } 67 _, _, err = tx.Set("src:"+k, peer.IDB58Encode(src), nil) 68 if err != nil { 69 return err 70 } 71 _, _, err = tx.Set("status:"+k, fmt.Sprintf("%d", status), nil) 72 if err != nil { 73 return err 74 } 75 return err 76 }) 77 return 78 } 79 80 // Del moves the given hash to the StatusDeleted status 81 // N.B. this functions assumes that the validity of this action has been confirmed 82 func (ht *BuntHT) Del(m *Message, key Hash) (err error) { 83 k := key.String() 84 err = ht.db.Update(func(tx *buntdb.Tx) error { 85 err = _setStatus(tx, m, k, StatusDeleted) 86 return err 87 }) 88 return 89 } 90 91 func _setStatus(tx *buntdb.Tx, m *Message, key string, status int) (err error) { 92 93 _, err = tx.Get("entry:" + key) 94 if err != nil { 95 if err == buntdb.ErrNotFound { 96 err = ErrHashNotFound 97 } 98 return 99 } 100 101 _, err = incIdx(tx, m) 102 if err != nil { 103 return 104 } 105 106 _, _, err = tx.Set("status:"+key, fmt.Sprintf("%d", status), nil) 107 if err != nil { 108 return 109 } 110 return 111 } 112 113 // Mod moves the given hash to the StatusModified status 114 // N.B. this functions assumes that the validity of this action has been confirmed 115 func (ht *BuntHT) Mod(m *Message, key Hash, newkey Hash) (err error) { 116 k := key.String() 117 err = ht.db.Update(func(tx *buntdb.Tx) error { 118 err = _setStatus(tx, m, k, StatusModified) 119 if err == nil { 120 link := newkey.String() 121 err = _link(tx, k, link, SysTagReplacedBy, m.From, StatusLive, newkey) 122 if err == nil { 123 _, _, err = tx.Set("replacedBy:"+k, link, nil) 124 if err != nil { 125 return err 126 } 127 } 128 } 129 return err 130 }) 131 return 132 } 133 134 func _get(tx *buntdb.Tx, k string, statusMask int) (string, error) { 135 val, err := tx.Get("entry:" + k) 136 if err == buntdb.ErrNotFound { 137 err = ErrHashNotFound 138 return val, err 139 } 140 var statusVal string 141 statusVal, err = tx.Get("status:" + k) 142 if err == nil { 143 144 if statusMask == StatusDefault { 145 // if the status mask is not given (i.e. Default) then 146 // we return information about the status if it's other than live 147 switch statusVal { 148 case StatusDeletedVal: 149 err = ErrHashDeleted 150 case StatusModifiedVal: 151 val, err = tx.Get("replacedBy:" + k) 152 if err != nil { 153 panic("missing expected replacedBy record") 154 } 155 err = ErrHashModified 156 case StatusRejectedVal: 157 err = ErrHashRejected 158 case StatusLiveVal: 159 default: 160 panic("unknown status!") 161 } 162 } else { 163 // otherwise we return the value only if the status is in the mask 164 var status int 165 status, err = strconv.Atoi(statusVal) 166 if err == nil { 167 if (status & statusMask) == 0 { 168 err = ErrHashNotFound 169 } 170 } 171 } 172 } 173 return val, err 174 } 175 176 // Exists checks for the existence of the hash in the store 177 func (ht *BuntHT) Exists(key Hash, statusMask int) (err error) { 178 err = ht.db.View(func(tx *buntdb.Tx) error { 179 _, err := _get(tx, key.String(), statusMask) 180 return err 181 }) 182 return 183 } 184 185 // Source returns the source node address of a given hash 186 func (ht *BuntHT) Source(key Hash) (id peer.ID, err error) { 187 err = ht.db.View(func(tx *buntdb.Tx) error { 188 val, err := tx.Get("src:" + key.String()) 189 if err == buntdb.ErrNotFound { 190 err = ErrHashNotFound 191 } 192 if err == nil { 193 id, err = peer.IDB58Decode(val) 194 } 195 return err 196 }) 197 return 198 } 199 200 // Get retrieves a value from the DHT store 201 func (ht *BuntHT) Get(key Hash, statusMask int, getMask int) (data []byte, entryType string, sources []string, status int, err error) { 202 if getMask == GetMaskDefault { 203 getMask = GetMaskEntry 204 } 205 err = ht.db.View(func(tx *buntdb.Tx) error { 206 k := key.String() 207 val, err := _get(tx, k, statusMask) 208 if err != nil { 209 data = []byte(val) // gotta do this because value is valid if ErrHashModified 210 return err 211 } 212 data = []byte(val) 213 214 if (getMask & GetMaskEntryType) != 0 { 215 entryType, err = tx.Get("type:" + k) 216 if err != nil { 217 return err 218 } 219 } 220 if (getMask & GetMaskSources) != 0 { 221 val, err = tx.Get("src:" + k) 222 if err == buntdb.ErrNotFound { 223 err = ErrHashNotFound 224 } 225 if err == nil { 226 sources = append(sources, val) 227 } 228 if err != nil { 229 return err 230 } 231 } 232 233 val, err = tx.Get("status:" + k) 234 if err != nil { 235 return err 236 } 237 status, err = strconv.Atoi(val) 238 if err != nil { 239 return err 240 } 241 242 return err 243 }) 244 return 245 } 246 247 // _link is a low level routine to add a link, also used by delLink 248 // this ensure monotonic recording of linking attempts 249 func _link(tx *buntdb.Tx, base string, link string, tag string, src peer.ID, status int, linkingEntryHash Hash) (err error) { 250 key := "link:" + base + ":" + link + ":" + tag 251 var val string 252 val, err = tx.Get(key) 253 source := peer.IDB58Encode(src) 254 lehStr := linkingEntryHash.String() 255 var records []linkEvent 256 if err == nil { 257 // load the previous value so we can append to it. 258 json.Unmarshal([]byte(val), &records) 259 260 // TODO: if the link exists, then load the statuses and see 261 // what we should do about this situation 262 /* 263 // search for the source and linking entry in the status 264 for _, s := range records { 265 if s.Source == source && s.LinksEntry == lehStr { 266 if status == StatusLive && s.Status != status { 267 err = ErrPutLinkOverDeleted 268 return 269 } 270 // return silently because this is just a duplicate putLink 271 break 272 } 273 } // fall through and add this linking event. 274 */ 275 276 } else if err == buntdb.ErrNotFound { 277 // when deleting the key must exist 278 if status == StatusDeleted { 279 err = ErrLinkNotFound 280 return 281 } 282 err = nil 283 } else { 284 return 285 } 286 records = append(records, linkEvent{status, source, lehStr}) 287 var b []byte 288 b, err = json.Marshal(records) 289 if err != nil { 290 return 291 } 292 _, _, err = tx.Set(key, string(b), nil) 293 if err != nil { 294 return 295 } 296 return 297 } 298 299 func (ht *BuntHT) link(m *Message, base string, link string, tag string, status int) (err error) { 300 err = ht.db.Update(func(tx *buntdb.Tx) error { 301 _, err := _get(tx, base, StatusLive) 302 if err != nil { 303 return err 304 } 305 err = _link(tx, base, link, tag, m.From, status, m.Body.(HoldReq).EntryHash) 306 if err != nil { 307 return err 308 } 309 310 //var index string 311 _, err = incIdx(tx, m) 312 313 if err != nil { 314 return err 315 } 316 return nil 317 }) 318 return 319 } 320 321 // PutLink associates a link with a stored hash 322 // N.B. this function assumes that the data associated has been properly retrieved 323 // and validated from the cource chain 324 func (ht *BuntHT) PutLink(m *Message, base string, link string, tag string) (err error) { 325 err = ht.link(m, base, link, tag, StatusLive) 326 return 327 } 328 329 // DelLink removes a link and tag associated with a stored hash 330 // N.B. this function assumes that the action has been properly validated 331 func (ht *BuntHT) DelLink(m *Message, base string, link string, tag string) (err error) { 332 err = ht.link(m, base, link, tag, StatusDeleted) 333 return 334 } 335 336 // GetLinks retrieves meta value associated with a base 337 func (ht *BuntHT) GetLinks(base Hash, tag string, statusMask int) (results []TaggedHash, err error) { 338 b := base.String() 339 err = ht.db.View(func(tx *buntdb.Tx) error { 340 _, err := _get(tx, b, StatusLive+StatusModified) //only get links on live and modified bases 341 if err != nil { 342 return err 343 } 344 345 if statusMask == StatusDefault { 346 statusMask = StatusLive 347 } 348 349 results = make([]TaggedHash, 0) 350 err = tx.Ascend("link", func(key, value string) bool { 351 x := strings.Split(key, ":") 352 t := string(x[3]) 353 if string(x[1]) == b && (tag == "" || tag == t) { 354 var records []linkEvent 355 json.Unmarshal([]byte(value), &records) 356 l := len(records) 357 //TODO: this is totally bogus currently simply 358 // looking at the last item we ever got 359 if l > 0 { 360 entry := records[l-1] 361 if err == nil && (entry.Status&statusMask) > 0 { 362 th := TaggedHash{H: string(x[2]), Source: entry.Source} 363 if tag == "" { 364 th.T = t 365 } 366 results = append(results, th) 367 } 368 } 369 } 370 371 return true 372 }) 373 374 return err 375 }) 376 return 377 } 378 379 // Close cleans up any resources used by the table 380 func (ht *BuntHT) Close() { 381 ht.db.Close() 382 ht.db = nil 383 } 384 385 // incIdx adds a new index record to dht for gossiping later 386 func incIdx(tx *buntdb.Tx, m *Message) (index string, err error) { 387 // if message is nil we can't record this for gossiping 388 // this should only be the case for the DNA 389 if m == nil { 390 return 391 } 392 393 var idx int 394 idx, err = getIntVal("_idx", tx) 395 if err != nil { 396 return 397 } 398 idx++ 399 index = fmt.Sprintf("%d", idx) 400 _, _, err = tx.Set("_idx", index, nil) 401 if err != nil { 402 return 403 } 404 405 var msg string 406 407 if m != nil { 408 var b []byte 409 b, err = ByteEncoder(m) 410 if err != nil { 411 return 412 } 413 msg = string(b) 414 } 415 _, _, err = tx.Set("idx:"+index, msg, nil) 416 if err != nil { 417 return 418 } 419 420 f, err := m.Fingerprint() 421 if err != nil { 422 return 423 } 424 _, _, err = tx.Set("f:"+f.String(), index, nil) 425 if err != nil { 426 return 427 } 428 429 return 430 } 431 432 // getIntVal returns an integer value at a given key, and assumes the value 0 if the key doesn't exist 433 func getIntVal(key string, tx *buntdb.Tx) (idx int, err error) { 434 var val string 435 val, err = tx.Get(key) 436 if err == buntdb.ErrNotFound { 437 err = nil 438 } else if err != nil { 439 return 440 } else { 441 idx, err = strconv.Atoi(val) 442 if err != nil { 443 return 444 } 445 } 446 return 447 } 448 449 // GetIdx returns the current index of changes to the HashTable 450 func (ht *BuntHT) GetIdx() (idx int, err error) { 451 err = ht.db.View(func(tx *buntdb.Tx) error { 452 var e error 453 idx, e = getIntVal("_idx", tx) 454 if e != nil { 455 return e 456 } 457 return nil 458 }) 459 return 460 } 461 462 // GetIdxMessage returns the messages that causes the change at a given index 463 func (ht *BuntHT) GetIdxMessage(idx int) (msg Message, err error) { 464 err = ht.db.View(func(tx *buntdb.Tx) error { 465 msgStr, e := tx.Get(fmt.Sprintf("idx:%d", idx)) 466 if e == buntdb.ErrNotFound { 467 return ErrNoSuchIdx 468 } 469 if e != nil { 470 return e 471 } 472 e = ByteDecoder([]byte(msgStr), &msg) 473 if err != nil { 474 return e 475 } 476 return nil 477 }) 478 return 479 } 480 481 // DumpIdx converts message and data of a DHT change request to a string for human consumption 482 func (ht *BuntHT) dumpIdx(idx int) (str string, err error) { 483 var msg Message 484 msg, err = ht.GetIdxMessage(idx) 485 if err != nil { 486 return 487 } 488 f, _ := msg.Fingerprint() 489 str = fmt.Sprintf("MSG (fingerprint %v):\n %v\n", f, msg) 490 switch msg.Type { 491 case PUT_REQUEST: 492 key := msg.Body.(HoldReq).EntryHash 493 entry, entryType, _, _, e := ht.Get(key, StatusDefault, GetMaskAll) 494 if e != nil { 495 err = fmt.Errorf("couldn't get %v err:%v ", key, e) 496 return 497 } else { 498 str += fmt.Sprintf("DATA: type:%s entry: %v\n", entryType, entry) 499 } 500 } 501 return 502 } 503 504 func statusValueToString(val string) string { 505 //TODO 506 return val 507 } 508 509 // String converts the table into a human readable string 510 func (ht *BuntHT) String() (result string) { 511 idx, err := ht.GetIdx() 512 if err != nil { 513 return err.Error() 514 } 515 result += fmt.Sprintf("DHT changes: %d\n", idx) 516 for i := 1; i <= idx; i++ { 517 str, err := ht.dumpIdx(i) 518 if err != nil { 519 result += fmt.Sprintf("%d Error:%v\n", i, err) 520 } else { 521 result += fmt.Sprintf("%d\n%v\n", i, str) 522 } 523 } 524 525 result += fmt.Sprintf("DHT entries:\n") 526 err = ht.db.View(func(tx *buntdb.Tx) error { 527 err = tx.Ascend("entry", func(key, value string) bool { 528 x := strings.Split(key, ":") 529 k := string(x[1]) 530 var status string 531 statusVal, err := tx.Get("status:" + k) 532 if err != nil { 533 status = fmt.Sprintf("<err getting status:%v>", err) 534 } else { 535 status = statusValueToString(statusVal) 536 } 537 538 var sources string 539 sources, err = tx.Get("src:" + k) 540 if err != nil { 541 sources = fmt.Sprintf("<err getting sources:%v>", err) 542 } 543 var links string 544 err = tx.Ascend("link", func(key, value string) bool { 545 x := strings.Split(key, ":") 546 base := x[1] 547 link := x[2] 548 tag := x[3] 549 if base == k { 550 links += fmt.Sprintf("Linked to: %s with tag %s\n", link, tag) 551 links += value + "\n" 552 } 553 return true 554 }) 555 result += fmt.Sprintf("Hash--%s (status %s):\nValue: %s\nSources: %s\n%s\n", k, status, value, sources, links) 556 return true 557 }) 558 return nil 559 }) 560 561 return 562 } 563 564 // DumpIdxJSON converts message and data of a DHT change request to a JSON string representation. 565 func (ht *BuntHT) dumpIdxJSON(idx int) (str string, err error) { 566 var msg Message 567 var buffer bytes.Buffer 568 var msgField, dataField string 569 msg, err = ht.GetIdxMessage(idx) 570 571 if err != nil { 572 return "", err 573 } 574 575 f, _ := msg.Fingerprint() 576 buffer.WriteString(fmt.Sprintf("{ \"index\": %d,", idx)) 577 msgField = fmt.Sprintf("\"message\": { \"fingerprint\": \"%v\", \"content\": \"%v\" },", f, msg) 578 579 switch msg.Type { 580 case PUT_REQUEST: 581 key := msg.Body.(HoldReq).EntryHash 582 entry, entryType, _, _, e := ht.Get(key, StatusAny, GetMaskAll) 583 if e != nil { 584 err = fmt.Errorf("couldn't get %v err:%v ", key, e) 585 return 586 } 587 dataField = fmt.Sprintf("\"data\": { \"type\": \"%s\", \"entry\": \"%v\" }", entryType, entry) 588 } 589 590 if len(dataField) > 0 { 591 buffer.WriteString(msgField) 592 buffer.WriteString(dataField) 593 } else { 594 buffer.WriteString(strings.TrimSuffix(msgField, ",")) 595 } 596 buffer.WriteString("}") 597 return PrettyPrintJSON(buffer.Bytes()) 598 } 599 600 // JSON converts the table into a JSON string representation. 601 func (ht *BuntHT) JSON() (result string, err error) { 602 var buffer, entries bytes.Buffer 603 idx, err := ht.GetIdx() 604 if err != nil { 605 return "", err 606 } 607 buffer.WriteString("{ \"dht_changes\": [") 608 for i := 1; i <= idx; i++ { 609 json, err := ht.dumpIdxJSON(i) 610 if err != nil { 611 return "", fmt.Errorf("DHT Change %d, Error: %v", i, err) 612 } 613 buffer.WriteString(json) 614 if i < idx { 615 buffer.WriteString(",") 616 } 617 } 618 buffer.WriteString("], \"dht_entries\": [") 619 err = ht.db.View(func(tx *buntdb.Tx) error { 620 err = tx.Ascend("entry", func(key, value string) bool { 621 x := strings.Split(key, ":") 622 k := string(x[1]) 623 var status string 624 statusVal, err := tx.Get("status:" + k) 625 if err != nil { 626 status = fmt.Sprintf("<err getting status:%v>", err) 627 } else { 628 status = statusValueToString(statusVal) 629 } 630 631 var sources string 632 sources, err = tx.Get("src:" + k) 633 if err != nil { 634 sources = fmt.Sprintf("<err getting sources:%v>", err) 635 } 636 var links bytes.Buffer 637 err = tx.Ascend("link", func(key, value string) bool { 638 x := strings.Split(key, ":") 639 base := x[1] 640 link := x[2] 641 tag := x[3] 642 if base == k { 643 links.WriteString(fmt.Sprintf("{ \"linkTo\": \"%s\",", link)) 644 links.WriteString(fmt.Sprintf("\"tag\": \"%s\",", tag)) 645 links.WriteString(fmt.Sprintf("\"value\": \"%s\" },", EscapeJSONValue(value))) 646 } 647 return true 648 }) 649 entries.WriteString(fmt.Sprintf("{ \"hash\": \"%s\",", k)) 650 entries.WriteString(fmt.Sprintf("\"status\": \"%s\",", status)) 651 entries.WriteString(fmt.Sprintf("\"value\": \"%s\",", EscapeJSONValue(value))) 652 entries.WriteString(fmt.Sprintf("\"sources\": \"%s\"", sources)) 653 if links.Len() > 0 { 654 entries.WriteString(fmt.Sprintf(",\"links\": [%s]", strings.TrimSuffix(links.String(), ","))) 655 } 656 entries.WriteString("},") 657 return true 658 }) 659 return nil 660 }) 661 buffer.WriteString(strings.TrimSuffix(entries.String(), ",")) 662 buffer.WriteString("]}") 663 return PrettyPrintJSON(buffer.Bytes()) 664 } 665 666 func (ht *BuntHT) Iterate(fn HashTableIterateFn) { 667 ht.db.View(func(tx *buntdb.Tx) error { 668 err := tx.Ascend("entry", func(key, value string) bool { 669 x := strings.Split(key, ":") 670 k := string(x[1]) 671 hash, err := NewHash(k) 672 if err != nil { 673 return false 674 } 675 return fn(hash) 676 }) 677 return err 678 }) 679 }