github.com/runner-mei/ql@v1.1.0/file.go (about) 1 // Copyright 2014 The ql Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Well known handles 6 // 1: root 7 // 2: id 8 9 package ql 10 11 import ( 12 "crypto/sha1" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "math/big" 17 "os" 18 "path/filepath" 19 "sync" 20 "time" 21 22 "github.com/cznic/lldb" 23 "github.com/cznic/mathutil" 24 "github.com/cznic/ql/vendored/github.com/camlistore/go4/lock" 25 ) 26 27 const ( 28 magic = "\x60\xdbql" 29 ) 30 31 var ( 32 _ btreeIndex = (*fileIndex)(nil) 33 _ btreeIterator = (*fileBTreeIterator)(nil) 34 _ indexIterator = (*fileIndexIterator)(nil) 35 _ storage = (*file)(nil) 36 _ temp = (*fileTemp)(nil) 37 ) 38 39 type chunk struct { // expanded to blob types lazily 40 f *file 41 b []byte 42 } 43 44 func (c chunk) expand() (v interface{}, err error) { 45 return c.f.loadChunks(c.b) 46 } 47 48 func expand1(data interface{}, e error) (v interface{}, err error) { 49 if e != nil { 50 return nil, e 51 } 52 53 c, ok := data.(chunk) 54 if !ok { 55 return data, nil 56 } 57 58 return c.expand() 59 } 60 61 func expand(data []interface{}) (err error) { 62 for i, v := range data { 63 if data[i], err = expand1(v, nil); err != nil { 64 return 65 } 66 } 67 return 68 } 69 70 // OpenFile returns a DB backed by a named file. The back end limits the size 71 // of a record to about 64 kB. 72 func OpenFile(name string, opt *Options) (db *DB, err error) { 73 var f lldb.OSFile 74 if f = opt.OSFile; f == nil { 75 f, err = os.OpenFile(name, os.O_RDWR, 0666) 76 if err != nil { 77 if !os.IsNotExist(err) { 78 return nil, err 79 } 80 81 if !opt.CanCreate { 82 return nil, err 83 } 84 85 f, err = os.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) 86 if err != nil { 87 return nil, err 88 } 89 } 90 } 91 92 fi, err := newFileFromOSFile(f, opt.Headroom) // always ACID 93 if err != nil { 94 return 95 } 96 97 if fi.tempFile = opt.TempFile; fi.tempFile == nil { 98 fi.tempFile = func(dir, prefix string) (f lldb.OSFile, err error) { 99 f0, err := ioutil.TempFile(dir, prefix) 100 return f0, err 101 } 102 } 103 104 fi.removeEmptyWAL = opt.RemoveEmptyWAL 105 106 return newDB(fi) 107 } 108 109 // Options amend the behavior of OpenFile. 110 // 111 // CanCreate 112 // 113 // The CanCreate option enables OpenFile to create the DB file if it does not 114 // exists. 115 // 116 // OSFile 117 // 118 // OSFile allows to pass an os.File like back end providing, for example, 119 // encrypted storage. If this field is nil then OpenFile uses the file named by 120 // the 'name' parameter instead. 121 // 122 // TempFile 123 // 124 // TempFile provides a temporary file used for evaluating the GROUP BY, ORDER 125 // BY, ... clauses. The hook is intended to be used by encrypted DB back ends 126 // to avoid leaks of unecrypted data to such temp files by providing temp files 127 // which are encrypted as well. Note that *os.File satisfies the lldb.OSFile 128 // interface. 129 // 130 // If TempFile is nil it defaults to ioutil.TempFile. 131 // 132 // Headroom 133 // 134 // Headroom selects the minimum size a WAL file will have. The "extra" 135 // allocated file space serves as a headroom. Commits that fit into the 136 // headroom should not fail due to 'not enough space on the volume' errors. The 137 // headroom parameter is first rounded-up to a non negative multiple of the 138 // size of the lldb.Allocator atom. 139 // 140 // RemoveEmptyWAL 141 // 142 // RemoveEmptyWAL controls whether empty WAL files should be deleted on 143 // clean exit. 144 type Options struct { 145 CanCreate bool 146 OSFile lldb.OSFile 147 TempFile func(dir, prefix string) (f lldb.OSFile, err error) 148 Headroom int64 149 RemoveEmptyWAL bool 150 } 151 152 type fileBTreeIterator struct { 153 en *lldb.BTreeEnumerator 154 t *fileTemp 155 } 156 157 func (it *fileBTreeIterator) Next() (k, v []interface{}, err error) { 158 bk, bv, err := it.en.Next() 159 if err != nil { 160 return 161 } 162 163 if k, err = lldb.DecodeScalars(bk); err != nil { 164 return 165 } 166 167 for i, val := range k { 168 b, ok := val.([]byte) 169 if !ok { 170 continue 171 } 172 173 c := chunk{it.t.file, b} 174 if k[i], err = c.expand(); err != nil { 175 return nil, nil, err 176 } 177 } 178 179 if err = enforce(k, it.t.colsK); err != nil { 180 return 181 } 182 183 if v, err = lldb.DecodeScalars(bv); err != nil { 184 return 185 } 186 187 for i, val := range v { 188 b, ok := val.([]byte) 189 if !ok { 190 continue 191 } 192 193 c := chunk{it.t.file, b} 194 if v[i], err = c.expand(); err != nil { 195 return nil, nil, err 196 } 197 } 198 199 err = enforce(v, it.t.colsV) 200 return 201 } 202 203 func enforce(val []interface{}, cols []*col) (err error) { 204 for i, v := range val { 205 if val[i], err = convert(v, cols[i].typ); err != nil { 206 return 207 } 208 } 209 return 210 } 211 212 //NTYPE 213 func infer(from []interface{}, to *[]*col) { 214 if len(*to) == 0 { 215 *to = make([]*col, len(from)) 216 for i := range *to { 217 (*to)[i] = &col{} 218 } 219 } 220 for i, c := range *to { 221 if f := from[i]; f != nil { 222 switch x := f.(type) { 223 //case nil: 224 case idealComplex: 225 c.typ = qComplex128 226 from[i] = complex128(x) 227 case idealFloat: 228 c.typ = qFloat64 229 from[i] = float64(x) 230 case idealInt: 231 c.typ = qInt64 232 from[i] = int64(x) 233 case idealRune: 234 c.typ = qInt32 235 from[i] = int32(x) 236 case idealUint: 237 c.typ = qUint64 238 from[i] = uint64(x) 239 case bool: 240 c.typ = qBool 241 case complex128: 242 c.typ = qComplex128 243 case complex64: 244 c.typ = qComplex64 245 case float64: 246 c.typ = qFloat64 247 case float32: 248 c.typ = qFloat32 249 case int8: 250 c.typ = qInt8 251 case int16: 252 c.typ = qInt16 253 case int32: 254 c.typ = qInt32 255 case int64: 256 c.typ = qInt64 257 case string: 258 c.typ = qString 259 case uint8: 260 c.typ = qUint8 261 case uint16: 262 c.typ = qUint16 263 case uint32: 264 c.typ = qUint32 265 case uint64: 266 c.typ = qUint64 267 case []byte: 268 c.typ = qBlob 269 case *big.Int: 270 c.typ = qBigInt 271 case *big.Rat: 272 c.typ = qBigRat 273 case time.Time: 274 c.typ = qTime 275 case time.Duration: 276 c.typ = qDuration 277 case chunk: 278 vals, err := lldb.DecodeScalars([]byte(x.b)) 279 if err != nil { 280 panic(err) 281 } 282 283 if len(vals) == 0 { 284 panic("internal error 040") 285 } 286 287 i, ok := vals[0].(int64) 288 if !ok { 289 panic("internal error 041") 290 } 291 292 c.typ = int(i) 293 case map[string]interface{}: // map of ids of a cross join 294 default: 295 panic("internal error 042") 296 } 297 } 298 } 299 } 300 301 type fileTemp struct { 302 *file 303 colsK []*col 304 colsV []*col 305 t *lldb.BTree 306 } 307 308 func (t *fileTemp) BeginTransaction() error { 309 return nil 310 } 311 312 func (t *fileTemp) Get(k []interface{}) (v []interface{}, err error) { 313 if err = expand(k); err != nil { 314 return 315 } 316 317 if err = t.flatten(k); err != nil { 318 return nil, err 319 } 320 321 bk, err := lldb.EncodeScalars(k...) 322 if err != nil { 323 return 324 } 325 326 bv, err := t.t.Get(nil, bk) 327 if err != nil { 328 return 329 } 330 331 return lldb.DecodeScalars(bv) 332 } 333 334 func (t *fileTemp) Drop() (err error) { 335 if t.f0 == nil { 336 return 337 } 338 339 fn := t.f0.Name() 340 if err = t.f0.Close(); err != nil { 341 return 342 } 343 344 if fn == "" { 345 return 346 } 347 348 return os.Remove(fn) 349 } 350 351 func (t *fileTemp) SeekFirst() (it btreeIterator, err error) { 352 en, err := t.t.SeekFirst() 353 if err != nil { 354 return 355 } 356 357 return &fileBTreeIterator{t: t, en: en}, nil 358 } 359 360 func (t *fileTemp) Set(k, v []interface{}) (err error) { 361 if err = expand(k); err != nil { 362 return 363 } 364 365 if err = expand(v); err != nil { 366 return 367 } 368 369 infer(k, &t.colsK) 370 infer(v, &t.colsV) 371 372 if err = t.flatten(k); err != nil { 373 return 374 } 375 376 bk, err := lldb.EncodeScalars(k...) 377 if err != nil { 378 return 379 } 380 381 if err = t.flatten(v); err != nil { 382 return 383 } 384 385 bv, err := lldb.EncodeScalars(v...) 386 if err != nil { 387 return 388 } 389 390 return t.t.Set(bk, bv) 391 } 392 393 type file struct { 394 a *lldb.Allocator 395 codec *gobCoder 396 f lldb.Filer 397 f0 lldb.OSFile 398 id int64 399 lck io.Closer 400 mu sync.Mutex 401 name string 402 tempFile func(dir, prefix string) (f lldb.OSFile, err error) 403 wal *os.File 404 removeEmptyWAL bool // Whether empty WAL files should be removed on close 405 } 406 407 func newFileFromOSFile(f lldb.OSFile, headroom int64) (fi *file, err error) { 408 nm := lockName(f.Name()) 409 lck, err := lock.Lock(nm) 410 if err != nil { 411 if lck != nil { 412 lck.Close() 413 } 414 return nil, err 415 } 416 417 close := true 418 defer func() { 419 if close && lck != nil { 420 lck.Close() 421 } 422 }() 423 424 var w *os.File 425 closew := false 426 wn := walName(f.Name()) 427 w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) 428 closew = true 429 defer func() { 430 if w != nil && closew { 431 nm := w.Name() 432 w.Close() 433 os.Remove(nm) 434 w = nil 435 } 436 }() 437 438 if err != nil { 439 if !os.IsExist(err) { 440 return nil, err 441 } 442 443 closew = false 444 w, err = os.OpenFile(wn, os.O_RDWR, 0666) 445 if err != nil { 446 return nil, err 447 } 448 449 closew = true 450 st, err := w.Stat() 451 if err != nil { 452 return nil, err 453 } 454 455 closew = st.Size() == 0 456 } 457 458 info, err := f.Stat() 459 if err != nil { 460 return nil, err 461 } 462 463 switch sz := info.Size(); { 464 case sz == 0: 465 b := make([]byte, 16) 466 copy(b, []byte(magic)) 467 if _, err := f.Write(b); err != nil { 468 return nil, err 469 } 470 471 filer := lldb.Filer(lldb.NewOSFiler(f)) 472 filer = lldb.NewInnerFiler(filer, 16) 473 if filer, err = lldb.NewACIDFiler(filer, w, lldb.MinWAL(headroom)); err != nil { 474 return nil, err 475 } 476 477 a, err := lldb.NewAllocator(filer, &lldb.Options{}) 478 if err != nil { 479 return nil, err 480 } 481 482 a.Compress = true 483 s := &file{ 484 a: a, 485 codec: newGobCoder(), 486 f0: f, 487 f: filer, 488 lck: lck, 489 name: f.Name(), 490 wal: w, 491 } 492 if err = s.BeginTransaction(); err != nil { 493 return nil, err 494 } 495 496 h, err := s.Create() 497 if err != nil { 498 return nil, err 499 } 500 501 if h != 1 { // root 502 panic("internal error 043") 503 } 504 505 if h, err = s.a.Alloc(make([]byte, 8)); err != nil { 506 return nil, err 507 } 508 509 if h != 2 { // id 510 panic("internal error 044") 511 } 512 513 close, closew = false, false 514 return s, s.Commit() 515 default: 516 b := make([]byte, 16) 517 if _, err := f.Read(b); err != nil { 518 return nil, err 519 } 520 521 if string(b[:len(magic)]) != magic { 522 return nil, fmt.Errorf("(file-002) unknown file format") 523 } 524 525 filer := lldb.Filer(lldb.NewOSFiler(f)) 526 filer = lldb.NewInnerFiler(filer, 16) 527 if filer, err = lldb.NewACIDFiler(filer, w, lldb.MinWAL(headroom)); err != nil { 528 return nil, err 529 } 530 531 a, err := lldb.NewAllocator(filer, &lldb.Options{}) 532 if err != nil { 533 return nil, err 534 } 535 536 bid, err := a.Get(nil, 2) // id 537 if err != nil { 538 return nil, err 539 } 540 541 if len(bid) != 8 { 542 return nil, fmt.Errorf("(file-003) corrupted DB: id |% x|", bid) 543 } 544 545 id := int64(0) 546 for _, v := range bid { 547 id = (id << 8) | int64(v) 548 } 549 550 a.Compress = true 551 s := &file{ 552 a: a, 553 codec: newGobCoder(), 554 f0: f, 555 f: filer, 556 id: id, 557 lck: lck, 558 name: f.Name(), 559 wal: w, 560 } 561 562 close, closew = false, false 563 return s, nil 564 } 565 } 566 567 func (s *file) OpenIndex(unique bool, handle int64) (btreeIndex, error) { 568 t, err := lldb.OpenBTree(s.a, s.collate, handle) 569 if err != nil { 570 return nil, err 571 } 572 573 return &fileIndex{s, handle, t, unique, newGobCoder()}, nil 574 } 575 576 func (s *file) CreateIndex(unique bool) ( /* handle */ int64, btreeIndex, error) { 577 t, h, err := lldb.CreateBTree(s.a, s.collate) 578 if err != nil { 579 return -1, nil, err 580 } 581 582 return h, &fileIndex{s, h, t, unique, newGobCoder()}, nil 583 } 584 585 func (s *file) Acid() bool { return s.wal != nil } 586 587 func errSet(p *error, errs ...error) (err error) { 588 err = *p 589 for _, e := range errs { 590 if err != nil { 591 return 592 } 593 *p, err = e, e 594 } 595 return 596 } 597 598 func (s *file) lock() func() { 599 s.mu.Lock() 600 return s.mu.Unlock 601 } 602 603 func (s *file) Close() (err error) { 604 defer s.lock()() 605 606 es := s.f0.Sync() 607 ef := s.f0.Close() 608 var ew, estat, eremove error 609 if s.wal != nil { 610 remove := false 611 wn := s.wal.Name() 612 if s.removeEmptyWAL { 613 var stat os.FileInfo 614 stat, estat = s.wal.Stat() 615 remove = stat.Size() == 0 616 } 617 ew = s.wal.Close() 618 if remove { 619 eremove = os.Remove(wn) 620 } 621 } 622 el := s.lck.Close() 623 return errSet(&err, es, ef, ew, el, estat, eremove) 624 } 625 626 func (s *file) Name() string { return s.name } 627 628 func (s *file) Verify() (allocs int64, err error) { 629 defer s.lock()() 630 var stat lldb.AllocStats 631 if err = s.a.Verify(lldb.NewMemFiler(), nil, &stat); err != nil { 632 return 633 } 634 635 allocs = stat.AllocAtoms 636 return 637 } 638 639 func (s *file) expandBytes(d []interface{}) (err error) { 640 for i, v := range d { 641 b, ok := v.([]byte) 642 if !ok { 643 continue 644 } 645 646 d[i], err = s.loadChunks(b) 647 if err != nil { 648 return 649 } 650 } 651 return 652 } 653 654 func (s *file) collate(a, b []byte) int { //TODO w/ error return 655 da, err := lldb.DecodeScalars(a) 656 if err != nil { 657 panic(err) 658 } 659 660 if err = s.expandBytes(da); err != nil { 661 panic(err) 662 } 663 664 db, err := lldb.DecodeScalars(b) 665 if err != nil { 666 panic(err) 667 } 668 669 if err = s.expandBytes(db); err != nil { 670 panic(err) 671 } 672 673 //dbg("da: %v, db: %v", da, db) 674 return collate(da, db) 675 } 676 677 func (s *file) CreateTemp(asc bool) (bt temp, err error) { 678 f, err := s.tempFile("", "ql-tmp-") 679 if err != nil { 680 return nil, err 681 } 682 683 fn := f.Name() 684 filer := lldb.NewOSFiler(f) 685 a, err := lldb.NewAllocator(filer, &lldb.Options{}) 686 if err != nil { 687 f.Close() 688 os.Remove(fn) 689 return nil, err 690 } 691 692 k := 1 693 if !asc { 694 k = -1 695 } 696 697 t, _, err := lldb.CreateBTree(a, func(a, b []byte) int { //TODO w/ error return 698 return k * s.collate(a, b) 699 }) 700 if err != nil { 701 f.Close() 702 if fn != "" { 703 os.Remove(fn) 704 } 705 return nil, err 706 } 707 708 x := &fileTemp{file: &file{ 709 a: a, 710 codec: newGobCoder(), 711 f0: f, 712 }, 713 t: t} 714 return x, nil 715 } 716 717 func (s *file) BeginTransaction() (err error) { 718 defer s.lock()() 719 return s.f.BeginUpdate() 720 } 721 722 func (s *file) Rollback() (err error) { 723 defer s.lock()() 724 return s.f.Rollback() 725 } 726 727 func (s *file) Commit() (err error) { 728 defer s.lock()() 729 return s.f.EndUpdate() 730 } 731 732 func (s *file) Create(data ...interface{}) (h int64, err error) { 733 if err = expand(data); err != nil { 734 return 735 } 736 737 if err = s.flatten(data); err != nil { 738 return 739 } 740 741 b, err := lldb.EncodeScalars(data...) 742 if err != nil { 743 return 744 } 745 746 defer s.lock()() 747 return s.a.Alloc(b) 748 } 749 750 func (s *file) Delete(h int64, blobCols ...*col) (err error) { 751 switch len(blobCols) { 752 case 0: 753 defer s.lock()() 754 return s.a.Free(h) 755 default: 756 return s.free(h, blobCols) 757 } 758 } 759 760 func (s *file) ResetID() (err error) { 761 s.id = 0 762 return 763 } 764 765 func (s *file) ID() (int64, error) { 766 defer s.lock()() 767 768 s.id++ 769 b := make([]byte, 8) 770 id := s.id 771 for i := 7; i >= 0; i-- { 772 b[i] = byte(id) 773 id >>= 8 774 } 775 776 return s.id, s.a.Realloc(2, b) 777 } 778 779 func (s *file) free(h int64, blobCols []*col) (err error) { 780 b, err := s.a.Get(nil, h) //LATER +bufs 781 if err != nil { 782 return 783 } 784 785 rec, err := lldb.DecodeScalars(b) 786 if err != nil { 787 return 788 } 789 790 for _, col := range blobCols { 791 if col.index >= len(rec) { 792 return fmt.Errorf("(file-004) file.free: corrupted DB (record len)") 793 } 794 if col.index+2 >= len(rec) { 795 continue 796 } 797 798 switch x := rec[col.index+2].(type) { 799 case nil: 800 // nop 801 case []byte: 802 if err = s.freeChunks(x); err != nil { 803 return 804 } 805 } 806 } 807 defer s.lock()() 808 return s.a.Free(h) 809 } 810 811 func (s *file) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { //NTYPE 812 b, err := s.a.Get(nil, h) //LATER +bufs 813 if err != nil { 814 return 815 } 816 817 rec, err := lldb.DecodeScalars(b) 818 if err != nil { 819 return 820 } 821 822 for _, col := range cols { 823 i := col.index + 2 824 if i >= len(rec) || rec[i] == nil { 825 continue 826 } 827 828 switch col.typ { 829 case 0: 830 case qBool: 831 case qComplex64: 832 rec[i] = complex64(rec[i].(complex128)) 833 case qComplex128: 834 case qFloat32: 835 rec[i] = float32(rec[i].(float64)) 836 case qFloat64: 837 case qInt8: 838 rec[i] = int8(rec[i].(int64)) 839 case qInt16: 840 rec[i] = int16(rec[i].(int64)) 841 case qInt32: 842 rec[i] = int32(rec[i].(int64)) 843 case qInt64: 844 case qString: 845 case qUint8: 846 rec[i] = uint8(rec[i].(uint64)) 847 case qUint16: 848 rec[i] = uint16(rec[i].(uint64)) 849 case qUint32: 850 rec[i] = uint32(rec[i].(uint64)) 851 case qUint64: 852 case qBlob, qBigInt, qBigRat, qTime, qDuration: 853 switch x := rec[i].(type) { 854 case []byte: 855 rec[i] = chunk{f: s, b: x} 856 default: 857 return nil, fmt.Errorf("(file-006) corrupted DB: non nil chunk type is not []byte") 858 } 859 default: 860 panic("internal error 045") 861 } 862 } 863 864 if cols != nil { 865 for n, dn := len(cols)+2, len(rec); dn < n; dn++ { 866 rec = append(rec, nil) 867 } 868 } 869 870 return rec, nil 871 } 872 873 func (s *file) freeChunks(enc []byte) (err error) { 874 items, err := lldb.DecodeScalars(enc) 875 if err != nil { 876 return 877 } 878 879 var ok bool 880 var next int64 881 switch len(items) { 882 case 2: 883 return 884 case 3: 885 if next, ok = items[1].(int64); !ok || next == 0 { 886 return fmt.Errorf("(file-007) corrupted DB: first chunk link") 887 } 888 default: 889 return fmt.Errorf("(file-008) corrupted DB: first chunk") 890 } 891 892 for next != 0 { 893 b, err := s.a.Get(nil, next) 894 if err != nil { 895 return err 896 } 897 898 if items, err = lldb.DecodeScalars(b); err != nil { 899 return err 900 } 901 902 var h int64 903 switch len(items) { 904 case 1: 905 // nop 906 case 2: 907 if h, ok = items[0].(int64); !ok { 908 return fmt.Errorf("(file-009) corrupted DB: chunk link") 909 } 910 911 default: 912 return fmt.Errorf("(file-010) corrupted DB: chunk items %d (%v)", len(items), items) 913 } 914 915 s.mu.Lock() 916 if err = s.a.Free(next); err != nil { 917 s.mu.Unlock() 918 return err 919 } 920 921 s.mu.Unlock() 922 next = h 923 } 924 return 925 } 926 927 func (s *file) loadChunks(enc []byte) (v interface{}, err error) { 928 items, err := lldb.DecodeScalars(enc) 929 if err != nil { 930 return 931 } 932 933 var ok bool 934 var next int64 935 switch len(items) { 936 case 2: 937 // nop 938 case 3: 939 if next, ok = items[1].(int64); !ok || next == 0 { 940 return nil, fmt.Errorf("(file-011) corrupted DB: first chunk link") 941 } 942 default: 943 //fmt.Printf("%d: %#v\n", len(items), items) 944 return nil, fmt.Errorf("(file-012) corrupted DB: first chunk") 945 } 946 947 typ, ok := items[0].(int64) 948 if !ok { 949 return nil, fmt.Errorf("(file-013) corrupted DB: first chunk tag") 950 } 951 952 buf, ok := items[len(items)-1].([]byte) 953 if !ok { 954 return nil, fmt.Errorf("(file-014) corrupted DB: first chunk data") 955 } 956 957 for next != 0 { 958 b, err := s.a.Get(nil, next) 959 if err != nil { 960 return nil, err 961 } 962 963 if items, err = lldb.DecodeScalars(b); err != nil { 964 return nil, err 965 } 966 967 switch len(items) { 968 case 1: 969 next = 0 970 case 2: 971 if next, ok = items[0].(int64); !ok { 972 return nil, fmt.Errorf("(file-015) corrupted DB: chunk link") 973 } 974 975 items = items[1:] 976 default: 977 return nil, fmt.Errorf("(file-016) corrupted DB: chunk items %d (%v)", len(items), items) 978 } 979 980 if b, ok = items[0].([]byte); !ok { 981 return nil, fmt.Errorf("(file-017) corrupted DB: chunk data") 982 } 983 984 buf = append(buf, b...) 985 } 986 return s.codec.decode(buf, int(typ)) 987 } 988 989 func (s *file) Update(h int64, data ...interface{}) (err error) { 990 b, err := lldb.EncodeScalars(data...) 991 if err != nil { 992 return 993 } 994 995 defer s.lock()() 996 return s.a.Realloc(h, b) 997 } 998 999 func (s *file) UpdateRow(h int64, blobCols []*col, data ...interface{}) (err error) { 1000 if len(blobCols) == 0 { 1001 return s.Update(h, data...) 1002 } 1003 1004 if err = expand(data); err != nil { 1005 return 1006 } 1007 1008 data0, err := s.Read(nil, h, blobCols...) 1009 if err != nil { 1010 return 1011 } 1012 1013 for _, c := range blobCols { 1014 if c.index+2 >= len(data0) { 1015 continue 1016 } 1017 1018 if x := data0[c.index+2]; x != nil { 1019 if err = s.freeChunks(x.(chunk).b); err != nil { 1020 return 1021 } 1022 } 1023 } 1024 1025 if err = s.flatten(data); err != nil { 1026 return 1027 } 1028 1029 return s.Update(h, data...) 1030 } 1031 1032 // []interface{}{qltype, ...}->[]interface{}{lldb scalar type, ...} 1033 // + long blobs are (pre)written to a chain of chunks. 1034 func (s *file) flatten(data []interface{}) (err error) { 1035 for i, v := range data { 1036 tag := 0 1037 var b []byte 1038 switch x := v.(type) { 1039 case []byte: 1040 tag = qBlob 1041 b = x 1042 case *big.Int: 1043 tag = qBigInt 1044 b, err = s.codec.encode(x) 1045 case *big.Rat: 1046 tag = qBigRat 1047 b, err = s.codec.encode(x) 1048 case time.Time: 1049 tag = qTime 1050 b, err = s.codec.encode(x) 1051 case time.Duration: 1052 tag = qDuration 1053 b, err = s.codec.encode(x) 1054 default: 1055 continue 1056 } 1057 if err != nil { 1058 return 1059 } 1060 1061 const chunk = 1 << 16 1062 chunks := 0 1063 var next int64 1064 var buf []byte 1065 for rem := len(b); rem > shortBlob; { 1066 n := mathutil.Min(rem, chunk) 1067 part := b[rem-n:] 1068 b = b[:rem-n] 1069 rem -= n 1070 switch next { 1071 case 0: // last chunk 1072 buf, err = lldb.EncodeScalars([]interface{}{part}...) 1073 default: // middle chunk 1074 buf, err = lldb.EncodeScalars([]interface{}{next, part}...) 1075 } 1076 if err != nil { 1077 return 1078 } 1079 1080 s.mu.Lock() 1081 h, err := s.a.Alloc(buf) 1082 s.mu.Unlock() 1083 if err != nil { 1084 return err 1085 } 1086 1087 next = h 1088 chunks++ 1089 } 1090 1091 switch next { 1092 case 0: // single chunk 1093 buf, err = lldb.EncodeScalars([]interface{}{tag, b}...) 1094 default: // multi chunks 1095 buf, err = lldb.EncodeScalars([]interface{}{tag, next, b}...) 1096 } 1097 if err != nil { 1098 return 1099 } 1100 1101 data[i] = buf 1102 } 1103 return 1104 } 1105 1106 func lockName(dbname string) string { 1107 base := filepath.Base(filepath.Clean(dbname)) + "lockfile" 1108 h := sha1.New() 1109 io.WriteString(h, base) 1110 return filepath.Join(filepath.Dir(dbname), fmt.Sprintf(".%x", h.Sum(nil))) 1111 } 1112 1113 func walName(dbname string) (r string) { 1114 base := filepath.Base(filepath.Clean(dbname)) 1115 h := sha1.New() 1116 io.WriteString(h, base) 1117 return filepath.Join(filepath.Dir(dbname), fmt.Sprintf(".%x", h.Sum(nil))) 1118 } 1119 1120 type fileIndex struct { 1121 f *file 1122 h int64 1123 t *lldb.BTree 1124 unique bool 1125 codec *gobCoder 1126 } 1127 1128 func (x *fileIndex) Clear() error { 1129 return x.t.Clear() 1130 } 1131 1132 var gbZeroInt64 []byte 1133 1134 func init() { 1135 var err error 1136 if gbZeroInt64, err = lldb.EncodeScalars(int64(0)); err != nil { 1137 panic(err) 1138 } 1139 } 1140 1141 func isIndexNull(data []interface{}) bool { 1142 for _, v := range data { 1143 if v != nil { 1144 return false 1145 } 1146 } 1147 return true 1148 } 1149 1150 // The []byte version of the key in the BTree shares chunks, if any, with 1151 // the value stored in the record. 1152 func (x *fileIndex) Create(indexedValues []interface{}, h int64) error { 1153 for i, indexedValue := range indexedValues { 1154 chunk, ok := indexedValue.(chunk) 1155 if ok { 1156 indexedValues[i] = chunk.b 1157 } 1158 } 1159 1160 t := x.t 1161 switch { 1162 case !x.unique: 1163 k, err := lldb.EncodeScalars(append(indexedValues, h)...) 1164 if err != nil { 1165 return err 1166 } 1167 1168 return t.Set(k, gbZeroInt64) 1169 case isIndexNull(indexedValues): // unique, NULL 1170 k, err := lldb.EncodeScalars(nil, h) 1171 if err != nil { 1172 return err 1173 } 1174 1175 return t.Set(k, gbZeroInt64) 1176 default: // unique, non NULL 1177 k, err := lldb.EncodeScalars(append(indexedValues, int64(0))...) 1178 if err != nil { 1179 return err 1180 } 1181 1182 v, err := lldb.EncodeScalars(h) 1183 if err != nil { 1184 return err 1185 } 1186 1187 _, _, err = t.Put(nil, k, func(key, old []byte) (new []byte, write bool, err error) { 1188 if old == nil { 1189 return v, true, nil 1190 } 1191 1192 return nil, false, fmt.Errorf("(file-018) cannot insert into unique index: duplicate value(s): %v", indexedValues) 1193 }) 1194 return err 1195 } 1196 } 1197 1198 func (x *fileIndex) Delete(indexedValues []interface{}, h int64) error { 1199 for i, indexedValue := range indexedValues { 1200 chunk, ok := indexedValue.(chunk) 1201 if ok { 1202 indexedValues[i] = chunk.b 1203 } 1204 } 1205 1206 t := x.t 1207 var k []byte 1208 var err error 1209 switch { 1210 case !x.unique: 1211 k, err = lldb.EncodeScalars(append(indexedValues, h)...) 1212 case isIndexNull(indexedValues): // unique, NULL 1213 k, err = lldb.EncodeScalars(nil, h) 1214 default: // unique, non NULL 1215 k, err = lldb.EncodeScalars(append(indexedValues, int64(0))...) 1216 } 1217 if err != nil { 1218 return err 1219 } 1220 1221 return t.Delete(k) 1222 } 1223 1224 func (x *fileIndex) Drop() error { 1225 if err := x.Clear(); err != nil { 1226 return err 1227 } 1228 1229 return x.f.a.Free(x.h) 1230 } 1231 1232 // []interface{}{qltype, ...}->[]interface{}{lldb scalar type, ...} 1233 func (x *fileIndex) flatten(data []interface{}) (err error) { 1234 for i, v := range data { 1235 tag := 0 1236 var b []byte 1237 switch xx := v.(type) { 1238 case []byte: 1239 tag = qBlob 1240 b = xx 1241 case *big.Int: 1242 tag = qBigInt 1243 b, err = x.codec.encode(xx) 1244 case *big.Rat: 1245 tag = qBigRat 1246 b, err = x.codec.encode(xx) 1247 case time.Time: 1248 tag = qTime 1249 b, err = x.codec.encode(xx) 1250 case time.Duration: 1251 tag = qDuration 1252 b, err = x.codec.encode(xx) 1253 default: 1254 continue 1255 } 1256 if err != nil { 1257 return 1258 } 1259 1260 var buf []byte 1261 if buf, err = lldb.EncodeScalars([]interface{}{tag, b}...); err != nil { 1262 return 1263 } 1264 1265 data[i] = buf 1266 } 1267 return 1268 } 1269 1270 func (x *fileIndex) Seek(indexedValues []interface{}) (indexIterator, bool, error) { 1271 data := append(indexedValues, 0) 1272 if err := x.flatten(data); err != nil { 1273 return nil, false, err 1274 } 1275 1276 k, err := lldb.EncodeScalars(data...) 1277 if err != nil { 1278 return nil, false, err 1279 } 1280 1281 en, hit, err := x.t.Seek(k) 1282 if err != nil { 1283 return nil, false, err 1284 } 1285 1286 return &fileIndexIterator{x.f, en, x.unique}, hit, nil 1287 } 1288 1289 func (x *fileIndex) SeekFirst() (iter indexIterator, err error) { 1290 en, err := x.t.SeekFirst() 1291 return &fileIndexIterator{x.f, en, x.unique}, err 1292 } 1293 1294 func (x *fileIndex) SeekLast() (iter indexIterator, err error) { 1295 en, err := x.t.SeekLast() 1296 return &fileIndexIterator{x.f, en, x.unique}, err 1297 } 1298 1299 type fileIndexIterator struct { 1300 f *file 1301 en *lldb.BTreeEnumerator 1302 unique bool 1303 } 1304 1305 func (i *fileIndexIterator) nextPrev(f func() ([]byte, []byte, error)) ([]interface{}, int64, error) { //TODO(indices) blobs: +test 1306 bk, bv, err := f() 1307 if err != nil { 1308 return nil, -1, err 1309 } 1310 1311 dk, err := lldb.DecodeScalars(bk) 1312 if err != nil { 1313 return nil, -1, err 1314 } 1315 1316 b, ok := dk[0].([]byte) 1317 if ok { 1318 dk[0] = chunk{i.f, b} 1319 if expand(dk[:1]); err != nil { 1320 return nil, -1, err 1321 } 1322 } 1323 1324 var k indexKey 1325 k.value = dk[:len(dk)-1] 1326 switch i.unique { 1327 case true: 1328 if isIndexNull(k.value) { 1329 return nil, dk[len(dk)-1].(int64), nil 1330 } 1331 1332 dv, err := lldb.DecodeScalars(bv) 1333 if err != nil { 1334 return nil, -1, err 1335 } 1336 1337 return k.value, dv[0].(int64), nil 1338 default: 1339 return k.value, dk[len(dk)-1].(int64), nil 1340 } 1341 } 1342 1343 func (i *fileIndexIterator) Next() ([]interface{}, int64, error) { //TODO(indices) blobs: +test 1344 return i.nextPrev(i.en.Next) 1345 } 1346 1347 func (i *fileIndexIterator) Prev() ([]interface{}, int64, error) { //TODO(indices) blobs: +test 1348 return i.nextPrev(i.en.Prev) 1349 }