github.com/pygolin/runtime@v0.0.0-20201208210830-a62e3cd39798/dict.go (about) 1 // Copyright 2016 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package runtime 16 17 import ( 18 "bytes" 19 "fmt" 20 "reflect" 21 "sync/atomic" 22 "unsafe" 23 ) 24 25 var ( 26 // DictType is the object representing the Python 'dict' type. 27 DictType = newBasisType("dict", reflect.TypeOf(Dict{}), toDictUnsafe, ObjectType) 28 dictItemIteratorType = newBasisType("dictionary-itemiterator", reflect.TypeOf(dictItemIterator{}), toDictItemIteratorUnsafe, ObjectType) 29 dictKeyIteratorType = newBasisType("dictionary-keyiterator", reflect.TypeOf(dictKeyIterator{}), toDictKeyIteratorUnsafe, ObjectType) 30 dictValueIteratorType = newBasisType("dictionary-valueiterator", reflect.TypeOf(dictValueIterator{}), toDictValueIteratorUnsafe, ObjectType) 31 ) 32 33 const ( 34 // maxDictSize is the largest number of entries a dictionary can hold. 35 // Dict sizes must be a power of two and this is the largest such 36 // number multiplied by 2 is representable as uint32. 37 maxDictSize = 1 << 30 38 minDictSize = 4 39 maxDictWithoutHashSize = 8 40 ) 41 42 // dictEntry represents an element in array of entries of the dictTable of a Dict. 43 // 44 // hash and key field are immutable once set, therefore, they could be checked 45 // almost without synchronization (synchronization is made at dictTable level). 46 // They not overwritten even when entry were deleted. Deleted entry is detected 47 // by nil value. 48 // 49 // value is mutable, and therefore should be written and read with atomic 50 // (except insertAbsentEntry, since it is synced on Dict level). 51 // value is changed in a way nil->non-nil->non-nil->...->non-nil->nil. Once 52 // it is set to nil, it should not be overwritten again. It is not hard demand, 53 // but it simplifies a bit, and allows to preserve ordering, ie if dict had 54 // no key (since it were deleted) then after insertion it should be last. 55 type dictEntry struct { 56 hash int 57 key *Object 58 value *Object 59 } 60 61 // dictTable is the hash table underlying Dict. 62 // It preserves insertion order of key by using array of entries and indices 63 // into array in a hash part. 64 // For concurrency, indices and entries pointers are never overwritten. 65 // When entries array is full, new dictTable is created. 66 // Hash part is twice larger than entries array, therefore, there is no need 67 // to trace its fill factor separately. 68 // Index to array is 1 based, and zero means never-written empty element. 69 // Index is not cleared when entry deleted, but could be overwritten when 70 // same key inserted again. 71 // Index should be written with atomic after new entry is written (except insertAbsentEntry), 72 // therefore lookupEntry could rely on it for synchronization. 73 // fill field also plays important role in synchronization: it should be incremented 74 // with atomic after new entry is written (except insertAbsentEntry). Using this 75 // fact, and entry.key immutability, iteration could access key without synchronization, 76 // though it still should check entry.value using atomic. 77 type dictTable struct { 78 // used is real number of alive values in a table 79 used uint32 80 // fill is the number of slots that are used or once were used but have 81 // since been cleared. Thus used <= fill <= len(entries). 82 // it is like len(entries) if entries were slice 83 fill uint32 84 // capa is a real capacity of entries, ie cap(entries) if entries were slice 85 capa uint32 86 // mask is len(indicies)-1 , and as we use power-of-two tables, it is 87 // used for index calculation: hash&mask == hash%len(indices) 88 mask uint32 89 // indices is a hash part of dictTable. It contains indices into entries array. 90 indices *[maxDictSize * 2]uint32 91 // entries is a an array of entries. It is used to keep insertion order of 92 // entries. 93 entries *[maxDictSize]dictEntry 94 } 95 96 // newDictTable allocates a table where at least minCapacity entries can be 97 // accommodated. minCapacity must be <= maxDictSize. 98 func newDictTable(numEntries uint32) *dictTable { 99 // It rounds required to nearest power of two using bit trick if neccessary. 100 // It doesn't increase capacity if it is already power of two, unless it is 101 // less than minDictSize. 102 // For tables smaller than maxDictWithoutHashSize indices array is not allocated. 103 if numEntries <= minDictSize { 104 numEntries = minDictSize 105 } else if (numEntries-1)&numEntries != 0 { 106 numEntries |= numEntries >> 1 107 numEntries |= numEntries >> 2 108 numEntries |= numEntries >> 4 109 numEntries |= numEntries >> 8 110 numEntries |= numEntries >> 16 111 numEntries++ 112 } 113 t := &dictTable{ 114 capa: numEntries, 115 entries: (*[maxDictSize]dictEntry)(unsafe.Pointer(&make([]dictEntry, numEntries)[0])), 116 } 117 if numEntries > maxDictWithoutHashSize { 118 t.mask = numEntries*2 - 1 119 t.indices = (*[maxDictSize * 2]uint32)(unsafe.Pointer(&make([]uint32, numEntries*2)[0])) 120 } 121 return t 122 } 123 124 func (t *dictTable) loadIndex(i uint32) uint32 { 125 return atomic.LoadUint32(&t.indices[i]) 126 } 127 128 func (t *dictTable) storeIndex(i, idx uint32) { 129 atomic.StoreUint32(&t.indices[i], idx) 130 } 131 132 func (t *dictTable) loadUsed() int { 133 if t == nil { 134 return 0 135 } 136 return int(atomic.LoadUint32(&t.used)) 137 } 138 139 func (t *dictTable) incUsed(n int) { 140 atomic.AddUint32(&t.used, uint32(n)) 141 } 142 143 func (t *dictTable) loadFill() uint32 { 144 if t == nil { 145 return 0 146 } 147 return atomic.LoadUint32(&t.fill) 148 } 149 150 func (t *dictTable) incFill(n int) { 151 atomic.AddUint32(&t.fill, uint32(n)) 152 } 153 154 func (t *dictEntry) loadValue() *Object { 155 p := (*unsafe.Pointer)(unsafe.Pointer(&t.value)) 156 return (*Object)(atomic.LoadPointer(p)) 157 } 158 159 func (t *dictEntry) storeValue(o *Object) { 160 p := (*unsafe.Pointer)(unsafe.Pointer(&t.value)) 161 atomic.StorePointer(p, unsafe.Pointer(o)) 162 } 163 164 func (t *dictEntry) swapValue(o *Object) *Object { 165 p := (*unsafe.Pointer)(unsafe.Pointer(&t.value)) 166 return (*Object)(atomic.SwapPointer(p, unsafe.Pointer(o))) 167 } 168 169 // insertAbsentEntry adds the populated entry to t assuming that the key 170 // specified in entry is absent from t. Since the key is absent, no key 171 // comparisons are necessary to perform the insert. 172 // It doesn't use atomic instructions and there fore it should operate only on 173 // non-yet reachable table. Dict.storeTable is used as synchronization point. 174 func (t *dictTable) insertAbsentEntry(entry *dictEntry) { 175 if t.fill == t.capa { 176 panic("overrun") 177 } 178 if mask := t.mask; mask != 0 { 179 i := uint32(entry.hash) & mask 180 perturb := uint(entry.hash) 181 index := i 182 // The key we're trying to insert is known to be absent from the dict 183 // so probe for the first zero entry. 184 for ; t.indices[index] != 0; index = i & mask { 185 i, perturb = dictNextIndex(i, perturb) 186 } 187 t.indices[index] = t.fill + 1 188 } 189 t.entries[t.fill] = *entry 190 t.used++ 191 t.fill++ 192 } 193 194 // lookupEntry returns the index to indices table and entry in entries with the given hash and key. 195 // Non-nil entry could be returned if entry were deleted, therefore entry.value should be checked after. 196 // When t.indices is allocated, it uses synchronization on t.fill, otherwise, it uses synchronization 197 // on index (since it is written atomically after entry is written). 198 // Therefore it is not necessary to lock the dict to do entry lookups in a consistent way. 199 // When entry is not found, returned index points into empty place in t.indices. 200 func (t *dictTable) lookupEntry(f *Frame, hash int, key *Object) (uint32, *dictEntry, *BaseException) { 201 if t == nil { 202 return 0, nil, nil 203 } 204 mask := t.mask 205 if mask == 0 { 206 // need to iterate in reverse order to find inserted-after-deleted entries 207 for eidx := t.loadFill(); eidx > 0; eidx-- { 208 entry := &t.entries[eidx-1] 209 if entry.hash == hash { 210 o, raised := Eq(f, entry.key, key) 211 if raised != nil { 212 return 0, nil, raised 213 } 214 eq, raised := IsTrue(f, o) 215 if raised != nil { 216 return 0, nil, raised 217 } 218 if eq { 219 return 0, entry, nil 220 } 221 } 222 } 223 return 0, nil, nil 224 } 225 i, perturb := uint32(hash)&mask, uint(hash) 226 index := i & mask 227 for { 228 idx := t.loadIndex(index) 229 if idx == 0 { 230 return index, nil, nil 231 } 232 eidx := idx - 1 233 entry := &t.entries[eidx] 234 if entry.hash == hash { 235 o, raised := Eq(f, entry.key, key) 236 if raised != nil { 237 return 0, nil, raised 238 } 239 eq, raised := IsTrue(f, o) 240 if raised != nil { 241 return 0, nil, raised 242 } 243 if eq { 244 return index, entry, nil 245 } 246 } 247 i, perturb = dictNextIndex(i, perturb) 248 index = i & mask 249 } 250 } 251 252 // writeNewEntry writes entry at the end of entries array, and its index 253 // into position, returned by previously called lookupEntry (so, index 254 // points into empty position or position bounded with the same key). 255 // It differs from insertAbsentEntry because a) index position is known, 256 // b) it has to use atomics to store index and increment t.fill. 257 func (t *dictTable) writeNewEntry(index uint32, nentry *dictEntry) { 258 eidx := t.fill 259 t.entries[eidx] = *nentry 260 if t.mask != 0 { 261 // store index atomically after entry is stored to synchronize 262 // with lookupEntry 263 t.storeIndex(index, eidx+1) 264 } 265 t.incUsed(1) 266 // increment fill atomically after entry is stored to synchronize 267 // with iteration. 268 t.incFill(1) 269 } 270 271 // writeValue rewrites value in non-empty entry. 272 // Old value should be non-nil and therefore it is not checked. 273 // While it is not hard demand, but it is part of insertion-order keeping. 274 func (t *dictTable) writeValue(entry *dictEntry, value *Object) { 275 entry.storeValue(value) 276 if value == nil { 277 t.incUsed(-1) 278 } 279 } 280 281 // growTable allocates table at least twice larger than already used space. 282 // It should be called when t.fill == t.capa . 283 func (t *dictTable) growTable() (*dictTable, bool) { 284 if t == nil { 285 // allocate minimal table 286 return newDictTable(0), true 287 } 288 var n uint32 289 if t.used < t.capa/2 { 290 n = t.used * 2 291 } else if t.capa <= maxDictSize/2 { 292 n = t.capa * 2 293 } else { 294 return nil, false 295 } 296 newTable := newDictTable(n) 297 for i := uint32(0); i < t.capa; i++ { 298 oldEntry := &t.entries[i] 299 if oldEntry.value != nil { 300 newTable.insertAbsentEntry(oldEntry) 301 } 302 } 303 return newTable, true 304 } 305 306 // dictEntryIterator is used to iterate over the entries in a dictTable in an 307 // arbitrary order. 308 type dictEntryIterator struct { 309 index uint32 310 table *dictTable 311 } 312 313 // newDictEntryIterator creates a dictEntryIterator object for d. It assumes 314 // that d.mutex is held by the caller. 315 func newDictEntryIterator(d *Dict) dictEntryIterator { 316 return dictEntryIterator{table: d.loadTable()} 317 } 318 319 // next advances this iterator to the next occupied entry and returns it. 320 // it returns nil, nil when there is no more elements. 321 func (iter *dictEntryIterator) next() (*Object, *Object) { 322 filled := iter.table.loadFill() 323 for { 324 index := atomic.AddUint32(&iter.index, 1) - 1 325 if index >= filled { 326 atomic.AddUint32(&iter.index, ^uint32(0)) 327 return nil, nil 328 } 329 entry := &iter.table.entries[index] 330 if value := entry.loadValue(); value != nil { 331 return entry.key, value 332 } 333 } 334 } 335 336 // dictVersionGuard is used to detect when a dict has been modified. 337 type dictVersionGuard struct { 338 dict *Dict 339 version int64 340 } 341 342 func newDictVersionGuard(d *Dict) dictVersionGuard { 343 return dictVersionGuard{d, d.loadVersion()} 344 } 345 346 // check returns false if the dict held by g has changed since g was created, 347 // true otherwise. 348 func (g *dictVersionGuard) check() bool { 349 return g.dict.loadVersion() == g.version 350 } 351 352 // Dict represents Python 'dict' objects. The public methods of *Dict are 353 // thread safe. 354 type Dict struct { 355 Object 356 table *dictTable 357 // We use a recursive mutex for synchronization because the hash and 358 // key comparison operations may re-enter DelItem/SetItem. 359 mutex recursiveMutex 360 // version is incremented whenever the Dict is modified. See: 361 // https://www.python.org/dev/peps/pep-0509/ 362 version int64 363 } 364 365 // NewDict returns an empty Dict. 366 func NewDict() *Dict { 367 return &Dict{Object: Object{typ: DictType}} 368 } 369 370 func newStringDict(items map[string]*Object) *Dict { 371 if len(items) > maxDictSize { 372 panic(fmt.Sprintf("dictionary too big: %d", len(items))) 373 } 374 table := newDictTable(uint32(len(items))) 375 for key, value := range items { 376 table.insertAbsentEntry(&dictEntry{hashString(key), NewStr(key).ToObject(), value}) 377 } 378 return &Dict{Object: Object{typ: DictType}, table: table} 379 } 380 381 func toDictUnsafe(o *Object) *Dict { 382 return (*Dict)(o.toPointer()) 383 } 384 385 // loadTable atomically loads and returns d's underlying dictTable. 386 func (d *Dict) loadTable() *dictTable { 387 p := (*unsafe.Pointer)(unsafe.Pointer(&d.table)) 388 return (*dictTable)(atomic.LoadPointer(p)) 389 } 390 391 // storeTable atomically updates d's underlying dictTable to the one given. 392 func (d *Dict) storeTable(table *dictTable) { 393 p := (*unsafe.Pointer)(unsafe.Pointer(&d.table)) 394 atomic.StorePointer(p, unsafe.Pointer(table)) 395 } 396 397 // loadVersion atomically loads and returns d's version. 398 func (d *Dict) loadVersion() int64 { 399 // 64bit atomic ops need to be 8 byte aligned. This compile time check 400 // verifies alignment by creating a negative constant for an unsigned type. 401 // See sync/atomic docs for details. 402 const blank = -(unsafe.Offsetof(d.version) % 8) 403 return atomic.LoadInt64(&d.version) 404 } 405 406 // incVersion atomically increments d's version. 407 func (d *Dict) incVersion() { 408 // 64bit atomic ops need to be 8 byte aligned. This compile time check 409 // verifies alignment by creating a negative constant for an unsigned type. 410 // See sync/atomic docs for details. 411 const blank = -(unsafe.Offsetof(d.version) % 8) 412 atomic.AddInt64(&d.version, 1) 413 } 414 415 // DelItem removes the entry associated with key from d. It returns true if an 416 // item was removed, or false if it did not exist in d. 417 func (d *Dict) DelItem(f *Frame, key *Object) (bool, *BaseException) { 418 originValue, raised := d.putItem(f, key, nil, true) 419 if raised != nil { 420 return false, raised 421 } 422 return originValue != nil, nil 423 } 424 425 // DelItemString removes the entry associated with key from d. It returns true 426 // if an item was removed, or false if it did not exist in d. 427 func (d *Dict) DelItemString(f *Frame, key string) (bool, *BaseException) { 428 return d.DelItem(f, NewStr(key).ToObject()) 429 } 430 431 // GetItem looks up key in d, returning the associated value or nil if key is 432 // not present in d. 433 func (d *Dict) GetItem(f *Frame, key *Object) (*Object, *BaseException) { 434 hash, raised := Hash(f, key) 435 if raised != nil { 436 return nil, raised 437 } 438 _, entry, raised := d.loadTable().lookupEntry(f, hash.Value(), key) 439 if raised != nil { 440 return nil, raised 441 } 442 if entry != nil { 443 return entry.loadValue(), nil 444 } 445 return nil, nil 446 } 447 448 // GetItemString looks up key in d, returning the associated value or nil if 449 // key is not present in d. 450 func (d *Dict) GetItemString(f *Frame, key string) (*Object, *BaseException) { 451 return d.GetItem(f, NewStr(key).ToObject()) 452 } 453 454 // Pop looks up key in d, returning and removing the associalted value if exist, 455 // or nil if key is not present in d. 456 func (d *Dict) Pop(f *Frame, key *Object) (*Object, *BaseException) { 457 return d.putItem(f, key, nil, true) 458 } 459 460 // Keys returns a list containing all the keys in d. 461 func (d *Dict) Keys(f *Frame) *List { 462 table := d.loadTable() 463 fill := int(table.loadFill()) 464 used := table.loadUsed() 465 // since `used` is loaded after `fill`, then number of alive values 466 // in t.entries[:fill] could not be larger than `used` 467 keys := make([]*Object, used) 468 i := 0 469 for k := 0; k < fill; k++ { 470 entry := &table.entries[k] 471 if value := entry.loadValue(); value != nil { 472 keys[i] = entry.key 473 i++ 474 } 475 } 476 return NewList(keys[:i]...) 477 } 478 479 // Len returns the number of entries in d. 480 func (d *Dict) Len() int { 481 return d.loadTable().loadUsed() 482 } 483 484 // putItem associates value with key in d, returning the old associated value if 485 // the key was added, or nil if it was not already present in d. 486 func (d *Dict) putItem(f *Frame, key, value *Object, overwrite bool) (*Object, *BaseException) { 487 hash, raised := Hash(f, key) 488 if raised != nil { 489 return nil, raised 490 } 491 hashv := hash.Value() 492 d.mutex.Lock(f) 493 // we do not use `defer d.mutex.Unlock(f)` here because defer is not free: it slows putItem by 30% . 494 // Since putItem is a hot place, lets Unlock manually. 495 t := d.table 496 v := d.version 497 index, entry, raised := t.lookupEntry(f, hashv, key) 498 if raised != nil { 499 d.mutex.Unlock(f) 500 return nil, raised 501 } 502 if v != d.version { 503 // Dictionary was recursively modified. Blow up instead 504 // of trying to recover. 505 d.mutex.Unlock(f) 506 return nil, f.RaiseType(RuntimeErrorType, "dictionary changed during write") 507 } 508 var originValue *Object 509 if entry == nil || entry.value == nil { 510 // either key were never inserted, or it was deleted 511 if value != nil { 512 if t == nil || t.fill == d.table.capa { 513 if newTable, ok := d.table.growTable(); ok { 514 newTable.insertAbsentEntry(&dictEntry{ 515 hash: hashv, 516 key: key, 517 value: value, 518 }) 519 // synchronization point 520 d.storeTable(newTable) 521 } else { 522 d.mutex.Unlock(f) 523 return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) 524 } 525 } else { 526 t.writeNewEntry(index, &dictEntry{ 527 hash: hashv, 528 key: key, 529 value: value, 530 }) 531 } 532 d.incVersion() 533 } 534 } else { 535 originValue = entry.value 536 if overwrite { 537 t.writeValue(entry, value) 538 d.incVersion() 539 if value == nil && t.used < t.capa/8 && t.fill > t.capa/8*5 { 540 if newTable, ok := t.growTable(); ok { 541 d.storeTable(newTable) 542 // doesn't increment version here, because we didn't change content in growTable. 543 } else { 544 d.mutex.Unlock(f) 545 panic("some unknown error on downsizing dictionary") 546 } 547 } 548 } 549 } 550 d.mutex.Unlock(f) 551 return originValue, raised 552 } 553 554 // SetItem associates value with key in d. 555 func (d *Dict) SetItem(f *Frame, key, value *Object) *BaseException { 556 _, raised := d.putItem(f, key, value, true) 557 return raised 558 } 559 560 // SetItemString associates value with key in d. 561 func (d *Dict) SetItemString(f *Frame, key string, value *Object) *BaseException { 562 return d.SetItem(f, NewStr(key).ToObject(), value) 563 } 564 565 // ToObject upcasts d to an Object. 566 func (d *Dict) ToObject() *Object { 567 return &d.Object 568 } 569 570 // Update copies the items from the mapping or sequence of 2-tuples o into d. 571 func (d *Dict) Update(f *Frame, o *Object) (raised *BaseException) { 572 var iter *Object 573 if o.isInstance(DictType) { 574 d2 := toDictUnsafe(o) 575 d2.mutex.Lock(f) 576 // Concurrent modifications to d2 will cause Update to raise 577 // "dictionary changed during iteration". 578 iter = newDictItemIterator(d2).ToObject() 579 d2.mutex.Unlock(f) 580 } else { 581 iter, raised = Iter(f, o) 582 } 583 if raised != nil { 584 return raised 585 } 586 return seqForEach(f, iter, func(item *Object) *BaseException { 587 return seqApply(f, item, func(elems []*Object, _ bool) *BaseException { 588 if numElems := len(elems); numElems != 2 { 589 format := "dictionary update sequence element has length %d; 2 is required" 590 return f.RaiseType(ValueErrorType, fmt.Sprintf(format, numElems)) 591 } 592 return d.SetItem(f, elems[0], elems[1]) 593 }) 594 }) 595 } 596 597 // dictsAreEqual returns true if d1 and d2 have the same keys and values, false 598 // otherwise. If either d1 or d2 are concurrently modified then RuntimeError is 599 // raised. 600 func dictsAreEqual(f *Frame, d1, d2 *Dict) (bool, *BaseException) { 601 if d1 == d2 { 602 return true, nil 603 } 604 // Do not hold both locks at the same time to avoid deadlock. 605 d1.mutex.Lock(f) 606 iter := newDictEntryIterator(d1) 607 g1 := newDictVersionGuard(d1) 608 len1 := d1.Len() 609 d1.mutex.Unlock(f) 610 d2.mutex.Lock(f) 611 g2 := newDictVersionGuard(d2) 612 len2 := d2.Len() 613 d2.mutex.Unlock(f) 614 if len1 != len2 { 615 return false, nil 616 } 617 result := true 618 for key, value := iter.next(); key != nil && result; key, value = iter.next() { 619 if v, raised := d2.GetItem(f, key); raised != nil { 620 return false, raised 621 } else if v == nil { 622 result = false 623 } else { 624 eq, raised := Eq(f, value, v) 625 if raised != nil { 626 return false, raised 627 } 628 result, raised = IsTrue(f, eq) 629 if raised != nil { 630 return false, raised 631 } 632 } 633 } 634 if !g1.check() || !g2.check() { 635 return false, f.RaiseType(RuntimeErrorType, "dictionary changed during iteration") 636 } 637 return result, nil 638 } 639 640 func dictClear(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { 641 if raised := checkMethodArgs(f, "clear", args, DictType); raised != nil { 642 return nil, raised 643 } 644 d := toDictUnsafe(args[0]) 645 d.mutex.Lock(f) 646 d.table = newDictTable(0) 647 d.incVersion() 648 d.mutex.Unlock(f) 649 return None, nil 650 } 651 652 func dictContains(f *Frame, seq, value *Object) (*Object, *BaseException) { 653 item, raised := toDictUnsafe(seq).GetItem(f, value) 654 if raised != nil { 655 return nil, raised 656 } 657 return GetBool(item != nil).ToObject(), nil 658 } 659 660 func dictCopy(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { 661 if raised := checkMethodArgs(f, "copy", args, DictType); raised != nil { 662 return nil, raised 663 } 664 return DictType.Call(f, args, nil) 665 } 666 667 func dictDelItem(f *Frame, o, key *Object) *BaseException { 668 deleted, raised := toDictUnsafe(o).DelItem(f, key) 669 if raised != nil { 670 return raised 671 } 672 if !deleted { 673 return raiseKeyError(f, key) 674 } 675 return nil 676 } 677 678 func dictEq(f *Frame, v, w *Object) (*Object, *BaseException) { 679 if !w.isInstance(DictType) { 680 return NotImplemented, nil 681 } 682 eq, raised := dictsAreEqual(f, toDictUnsafe(v), toDictUnsafe(w)) 683 if raised != nil { 684 return nil, raised 685 } 686 return GetBool(eq).ToObject(), nil 687 } 688 689 func dictGet(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 690 expectedTypes := []*Type{DictType, ObjectType, ObjectType} 691 argc := len(args) 692 if argc == 2 { 693 expectedTypes = expectedTypes[:2] 694 } 695 if raised := checkMethodArgs(f, "get", args, expectedTypes...); raised != nil { 696 return nil, raised 697 } 698 item, raised := toDictUnsafe(args[0]).GetItem(f, args[1]) 699 if raised == nil && item == nil { 700 item = None 701 if argc > 2 { 702 item = args[2] 703 } 704 } 705 return item, raised 706 } 707 708 func dictHasKey(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { 709 if raised := checkMethodArgs(f, "has_key", args, DictType, ObjectType); raised != nil { 710 return nil, raised 711 } 712 return dictContains(f, args[0], args[1]) 713 } 714 715 func dictItems(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 716 if raised := checkMethodArgs(f, "items", args, DictType); raised != nil { 717 return nil, raised 718 } 719 d := toDictUnsafe(args[0]) 720 d.mutex.Lock(f) 721 iter := newDictItemIterator(d).ToObject() 722 d.mutex.Unlock(f) 723 return ListType.Call(f, Args{iter}, nil) 724 } 725 726 func dictIterItems(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 727 if raised := checkMethodArgs(f, "iteritems", args, DictType); raised != nil { 728 return nil, raised 729 } 730 d := toDictUnsafe(args[0]) 731 d.mutex.Lock(f) 732 iter := newDictItemIterator(d).ToObject() 733 d.mutex.Unlock(f) 734 return iter, nil 735 } 736 737 func dictIterKeys(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 738 if raised := checkMethodArgs(f, "iterkeys", args, DictType); raised != nil { 739 return nil, raised 740 } 741 return dictIter(f, args[0]) 742 } 743 744 func dictIterValues(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 745 if raised := checkMethodArgs(f, "itervalues", args, DictType); raised != nil { 746 return nil, raised 747 } 748 d := toDictUnsafe(args[0]) 749 d.mutex.Lock(f) 750 iter := newDictValueIterator(d).ToObject() 751 d.mutex.Unlock(f) 752 return iter, nil 753 } 754 755 func dictKeys(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 756 if raised := checkMethodArgs(f, "keys", args, DictType); raised != nil { 757 return nil, raised 758 } 759 return toDictUnsafe(args[0]).Keys(f).ToObject(), nil 760 } 761 762 func dictGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { 763 item, raised := toDictUnsafe(o).GetItem(f, key) 764 if raised != nil { 765 return nil, raised 766 } 767 if item == nil { 768 return nil, raiseKeyError(f, key) 769 } 770 return item, nil 771 } 772 773 func dictInit(f *Frame, o *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { 774 var expectedTypes []*Type 775 argc := len(args) 776 if argc > 0 { 777 expectedTypes = []*Type{ObjectType} 778 } 779 if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil { 780 return nil, raised 781 } 782 d := toDictUnsafe(o) 783 if argc > 0 { 784 if raised := d.Update(f, args[0]); raised != nil { 785 return nil, raised 786 } 787 } 788 for _, kwarg := range kwargs { 789 if raised := d.SetItemString(f, kwarg.Name, kwarg.Value); raised != nil { 790 return nil, raised 791 } 792 } 793 return None, nil 794 } 795 796 func dictIter(f *Frame, o *Object) (*Object, *BaseException) { 797 d := toDictUnsafe(o) 798 d.mutex.Lock(f) 799 iter := newDictKeyIterator(d).ToObject() 800 d.mutex.Unlock(f) 801 return iter, nil 802 } 803 804 func dictLen(f *Frame, o *Object) (*Object, *BaseException) { 805 d := toDictUnsafe(o) 806 ret := NewInt(d.Len()).ToObject() 807 return ret, nil 808 } 809 810 func dictNE(f *Frame, v, w *Object) (*Object, *BaseException) { 811 if !w.isInstance(DictType) { 812 return NotImplemented, nil 813 } 814 eq, raised := dictsAreEqual(f, toDictUnsafe(v), toDictUnsafe(w)) 815 if raised != nil { 816 return nil, raised 817 } 818 return GetBool(!eq).ToObject(), nil 819 } 820 821 func dictNew(f *Frame, t *Type, _ Args, _ KWArgs) (*Object, *BaseException) { 822 d := toDictUnsafe(newObject(t)) 823 d.table = newDictTable(0) 824 return d.ToObject(), nil 825 } 826 827 func dictPop(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { 828 expectedTypes := []*Type{DictType, ObjectType, ObjectType} 829 argc := len(args) 830 if argc == 2 { 831 expectedTypes = expectedTypes[:2] 832 } 833 if raised := checkMethodArgs(f, "pop", args, expectedTypes...); raised != nil { 834 return nil, raised 835 } 836 key := args[1] 837 d := toDictUnsafe(args[0]) 838 item, raised := d.Pop(f, key) 839 if raised == nil && item == nil { 840 if argc > 2 { 841 item = args[2] 842 } else { 843 raised = raiseKeyError(f, key) 844 } 845 } 846 return item, raised 847 } 848 849 func dictPopItem(f *Frame, args Args, _ KWArgs) (item *Object, raised *BaseException) { 850 if raised := checkMethodArgs(f, "popitem", args, DictType); raised != nil { 851 return nil, raised 852 } 853 d := toDictUnsafe(args[0]) 854 d.mutex.Lock(f) 855 defer d.mutex.Unlock(f) 856 if d.table.used == 0 { 857 return nil, f.RaiseType(KeyErrorType, "popitem(): dictionary is empty") 858 } 859 // unfortunately, 3.7 standardized popping last key-value 860 for i := int(d.table.fill) - 1; i >= 0; i-- { 861 entry := &d.table.entries[i] 862 if entry.value != nil { 863 item = NewTuple(entry.key, entry.value).ToObject() 864 entry.storeValue(nil) 865 d.table.incUsed(-1) 866 d.incVersion() 867 return item, nil 868 } 869 } 870 panic("there shall be at least one item") 871 } 872 873 func dictRepr(f *Frame, o *Object) (*Object, *BaseException) { 874 d := toDictUnsafe(o) 875 if f.reprEnter(d.ToObject()) { 876 return NewStr("{...}").ToObject(), nil 877 } 878 defer f.reprLeave(d.ToObject()) 879 // Lock d so that we get a consistent view of it. Otherwise we may 880 // return a state that d was never actually in. 881 d.mutex.Lock(f) 882 defer d.mutex.Unlock(f) 883 var buf bytes.Buffer 884 buf.WriteString("{") 885 iter := newDictEntryIterator(d) 886 i := 0 887 for key, value := iter.next(); key != nil; key, value = iter.next() { 888 if i > 0 { 889 buf.WriteString(", ") 890 } 891 s, raised := Repr(f, key) 892 if raised != nil { 893 return nil, raised 894 } 895 buf.WriteString(s.Value()) 896 buf.WriteString(": ") 897 if s, raised = Repr(f, value); raised != nil { 898 return nil, raised 899 } 900 buf.WriteString(s.Value()) 901 i++ 902 } 903 buf.WriteString("}") 904 return NewStr(buf.String()).ToObject(), nil 905 } 906 907 func dictSetDefault(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { 908 argc := len(args) 909 if argc == 1 { 910 return nil, f.RaiseType(TypeErrorType, "setdefault expected at least 1 arguments, got 0") 911 } 912 if argc > 3 { 913 return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("setdefault expected at most 2 arguments, got %v", argc-1)) 914 } 915 expectedTypes := []*Type{DictType, ObjectType, ObjectType} 916 if argc == 2 { 917 expectedTypes = expectedTypes[:2] 918 } 919 if raised := checkMethodArgs(f, "setdefault", args, expectedTypes...); raised != nil { 920 return nil, raised 921 } 922 d := toDictUnsafe(args[0]) 923 key := args[1] 924 var value *Object 925 if argc > 2 { 926 value = args[2] 927 } else { 928 value = None 929 } 930 originValue, raised := d.putItem(f, key, value, false) 931 if originValue != nil { 932 return originValue, raised 933 } 934 return value, raised 935 } 936 937 func dictSetItem(f *Frame, o, key, value *Object) *BaseException { 938 return toDictUnsafe(o).SetItem(f, key, value) 939 } 940 941 func dictUpdate(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 942 expectedTypes := []*Type{DictType, ObjectType} 943 argc := len(args) 944 if argc == 1 { 945 expectedTypes = expectedTypes[:1] 946 } 947 if raised := checkMethodArgs(f, "update", args, expectedTypes...); raised != nil { 948 return nil, raised 949 } 950 d := toDictUnsafe(args[0]) 951 if argc > 1 { 952 if raised := d.Update(f, args[1]); raised != nil { 953 return nil, raised 954 } 955 } 956 for _, kwarg := range kwargs { 957 if raised := d.SetItemString(f, kwarg.Name, kwarg.Value); raised != nil { 958 return nil, raised 959 } 960 } 961 return None, nil 962 } 963 964 func dictValues(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { 965 if raised := checkMethodArgs(f, "values", args, DictType); raised != nil { 966 return nil, raised 967 } 968 iter, raised := dictIterValues(f, args, nil) 969 if raised != nil { 970 return nil, raised 971 } 972 return ListType.Call(f, Args{iter}, nil) 973 } 974 975 func initDictType(dict map[string]*Object) { 976 dict["clear"] = newBuiltinFunction("clear", dictClear).ToObject() 977 dict["copy"] = newBuiltinFunction("copy", dictCopy).ToObject() 978 dict["get"] = newBuiltinFunction("get", dictGet).ToObject() 979 dict["has_key"] = newBuiltinFunction("has_key", dictHasKey).ToObject() 980 dict["items"] = newBuiltinFunction("items", dictItems).ToObject() 981 dict["iteritems"] = newBuiltinFunction("iteritems", dictIterItems).ToObject() 982 dict["iterkeys"] = newBuiltinFunction("iterkeys", dictIterKeys).ToObject() 983 dict["itervalues"] = newBuiltinFunction("itervalues", dictIterValues).ToObject() 984 dict["keys"] = newBuiltinFunction("keys", dictKeys).ToObject() 985 dict["pop"] = newBuiltinFunction("pop", dictPop).ToObject() 986 dict["popitem"] = newBuiltinFunction("popitem", dictPopItem).ToObject() 987 dict["setdefault"] = newBuiltinFunction("setdefault", dictSetDefault).ToObject() 988 dict["update"] = newBuiltinFunction("update", dictUpdate).ToObject() 989 dict["values"] = newBuiltinFunction("values", dictValues).ToObject() 990 DictType.slots.Contains = &binaryOpSlot{dictContains} 991 DictType.slots.DelItem = &delItemSlot{dictDelItem} 992 DictType.slots.Eq = &binaryOpSlot{dictEq} 993 DictType.slots.GetItem = &binaryOpSlot{dictGetItem} 994 DictType.slots.Hash = &unaryOpSlot{hashNotImplemented} 995 DictType.slots.Init = &initSlot{dictInit} 996 DictType.slots.Iter = &unaryOpSlot{dictIter} 997 DictType.slots.Len = &unaryOpSlot{dictLen} 998 DictType.slots.NE = &binaryOpSlot{dictNE} 999 DictType.slots.New = &newSlot{dictNew} 1000 DictType.slots.Repr = &unaryOpSlot{dictRepr} 1001 DictType.slots.SetItem = &setItemSlot{dictSetItem} 1002 } 1003 1004 type dictItemIterator struct { 1005 Object 1006 iter dictEntryIterator 1007 guard dictVersionGuard 1008 } 1009 1010 // newDictItemIterator creates a dictItemIterator object for d. It assumes that 1011 // d.mutex is held by the caller. 1012 func newDictItemIterator(d *Dict) *dictItemIterator { 1013 return &dictItemIterator{ 1014 Object: Object{typ: dictItemIteratorType}, 1015 iter: newDictEntryIterator(d), 1016 guard: newDictVersionGuard(d), 1017 } 1018 } 1019 1020 func toDictItemIteratorUnsafe(o *Object) *dictItemIterator { 1021 return (*dictItemIterator)(o.toPointer()) 1022 } 1023 1024 func (iter *dictItemIterator) ToObject() *Object { 1025 return &iter.Object 1026 } 1027 1028 func dictItemIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { 1029 return o, nil 1030 } 1031 1032 func dictItemIteratorNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { 1033 iter := toDictItemIteratorUnsafe(o) 1034 key, value, raised := dictIteratorNext(f, &iter.iter, &iter.guard) 1035 if raised != nil { 1036 return nil, raised 1037 } 1038 return NewTuple2(key, value).ToObject(), nil 1039 } 1040 1041 func initDictItemIteratorType(map[string]*Object) { 1042 dictItemIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable 1043 dictItemIteratorType.slots.Iter = &unaryOpSlot{dictItemIteratorIter} 1044 dictItemIteratorType.slots.Next = &unaryOpSlot{dictItemIteratorNext} 1045 } 1046 1047 type dictKeyIterator struct { 1048 Object 1049 iter dictEntryIterator 1050 guard dictVersionGuard 1051 } 1052 1053 // newDictKeyIterator creates a dictKeyIterator object for d. It assumes that 1054 // d.mutex is held by the caller. 1055 func newDictKeyIterator(d *Dict) *dictKeyIterator { 1056 return &dictKeyIterator{ 1057 Object: Object{typ: dictKeyIteratorType}, 1058 iter: newDictEntryIterator(d), 1059 guard: newDictVersionGuard(d), 1060 } 1061 } 1062 1063 func toDictKeyIteratorUnsafe(o *Object) *dictKeyIterator { 1064 return (*dictKeyIterator)(o.toPointer()) 1065 } 1066 1067 func (iter *dictKeyIterator) ToObject() *Object { 1068 return &iter.Object 1069 } 1070 1071 func dictKeyIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { 1072 return o, nil 1073 } 1074 1075 func dictKeyIteratorNext(f *Frame, o *Object) (*Object, *BaseException) { 1076 iter := toDictKeyIteratorUnsafe(o) 1077 key, _, raised := dictIteratorNext(f, &iter.iter, &iter.guard) 1078 if raised != nil { 1079 return nil, raised 1080 } 1081 return key, nil 1082 } 1083 1084 func initDictKeyIteratorType(map[string]*Object) { 1085 dictKeyIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable 1086 dictKeyIteratorType.slots.Iter = &unaryOpSlot{dictKeyIteratorIter} 1087 dictKeyIteratorType.slots.Next = &unaryOpSlot{dictKeyIteratorNext} 1088 } 1089 1090 type dictValueIterator struct { 1091 Object 1092 iter dictEntryIterator 1093 guard dictVersionGuard 1094 } 1095 1096 // newDictValueIterator creates a dictValueIterator object for d. It assumes 1097 // that d.mutex is held by the caller. 1098 func newDictValueIterator(d *Dict) *dictValueIterator { 1099 return &dictValueIterator{ 1100 Object: Object{typ: dictValueIteratorType}, 1101 iter: newDictEntryIterator(d), 1102 guard: newDictVersionGuard(d), 1103 } 1104 } 1105 1106 func toDictValueIteratorUnsafe(o *Object) *dictValueIterator { 1107 return (*dictValueIterator)(o.toPointer()) 1108 } 1109 1110 func (iter *dictValueIterator) ToObject() *Object { 1111 return &iter.Object 1112 } 1113 1114 func dictValueIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { 1115 return o, nil 1116 } 1117 1118 func dictValueIteratorNext(f *Frame, o *Object) (*Object, *BaseException) { 1119 iter := toDictValueIteratorUnsafe(o) 1120 _, value, raised := dictIteratorNext(f, &iter.iter, &iter.guard) 1121 if raised != nil { 1122 return nil, raised 1123 } 1124 return value, nil 1125 } 1126 1127 func initDictValueIteratorType(map[string]*Object) { 1128 dictValueIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable 1129 dictValueIteratorType.slots.Iter = &unaryOpSlot{dictValueIteratorIter} 1130 dictValueIteratorType.slots.Next = &unaryOpSlot{dictValueIteratorNext} 1131 } 1132 1133 func raiseKeyError(f *Frame, key *Object) *BaseException { 1134 s, raised := ToStr(f, key) 1135 if raised == nil { 1136 raised = f.RaiseType(KeyErrorType, s.Value()) 1137 } 1138 return raised 1139 } 1140 1141 func dictNextIndex(i uint32, perturb uint) (uint32, uint) { 1142 return (i << 2) + i + uint32(perturb) + 1, perturb >> 5 1143 } 1144 1145 func dictIteratorNext(f *Frame, iter *dictEntryIterator, guard *dictVersionGuard) (*Object, *Object, *BaseException) { 1146 // NOTE: The behavior here diverges from CPython where an iterator that 1147 // is exhausted will always return StopIteration regardless whether the 1148 // underlying dict is subsequently modified. In Grumpy, an iterator for 1149 // a dict that has been modified will always raise RuntimeError even if 1150 // the iterator was exhausted before the modification. 1151 key, value := iter.next() 1152 if !guard.check() { 1153 return nil, nil, f.RaiseType(RuntimeErrorType, "dictionary changed during iteration") 1154 } 1155 if key == nil { 1156 return nil, nil, f.Raise(StopIterationType.ToObject(), nil, nil) 1157 } 1158 return key, value, nil 1159 }