github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/index/receive.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package index 18 19 import ( 20 "bytes" 21 "crypto/sha1" 22 "errors" 23 "fmt" 24 _ "image/gif" 25 _ "image/jpeg" 26 _ "image/png" 27 "io" 28 "log" 29 "os" 30 "sort" 31 "strings" 32 "sync" 33 "time" 34 35 "camlistore.org/pkg/blob" 36 "camlistore.org/pkg/blobserver" 37 "camlistore.org/pkg/images" 38 "camlistore.org/pkg/jsonsign" 39 "camlistore.org/pkg/magic" 40 "camlistore.org/pkg/media" 41 "camlistore.org/pkg/schema" 42 "camlistore.org/pkg/types" 43 44 "camlistore.org/third_party/github.com/camlistore/goexif/exif" 45 "camlistore.org/third_party/github.com/camlistore/goexif/tiff" 46 "camlistore.org/third_party/github.com/hjfreyer/taglib-go/taglib" 47 ) 48 49 func (ix *Index) outOfOrderIndexerLoop() { 50 if ix.BlobSource == nil { 51 // Bail early. Nothing will work later anyway. (for tests) 52 return 53 } 54 WaitTickle: 55 for _ = range ix.tickleOoo { 56 for { 57 ix.mu.Lock() 58 if len(ix.readyReindex) == 0 { 59 ix.mu.Unlock() 60 continue WaitTickle 61 } 62 var br blob.Ref 63 for br = range ix.readyReindex { 64 break 65 } 66 delete(ix.readyReindex, br) 67 ix.mu.Unlock() 68 69 err := ix.indexBlob(br) 70 if err != nil { 71 log.Printf("out-of-order indexBlob(%v) = %v", br, err) 72 ix.mu.Lock() 73 if len(ix.needs[br]) == 0 { 74 ix.readyReindex[br] = true 75 } 76 ix.mu.Unlock() 77 } 78 } 79 } 80 } 81 82 func (ix *Index) indexBlob(br blob.Ref) error { 83 bs := ix.BlobSource 84 if bs == nil { 85 return fmt.Errorf("index: can't re-index %v: no BlobSource", br) 86 } 87 rc, _, err := bs.Fetch(br) 88 if err != nil { 89 return fmt.Errorf("index: failed to fetch %v for reindexing: %v", br, err) 90 } 91 defer rc.Close() 92 if _, err := blobserver.Receive(ix, br, rc); err != nil { 93 return err 94 } 95 return nil 96 } 97 98 type mutationMap struct { 99 kv map[string]string // the keys and values we populate 100 101 // We record if we get a delete claim, so we can update 102 // the deletes cache right after committing the mutation. 103 // 104 // TODO(mpl): we only need to keep track of one claim so far, 105 // but I chose a slice for when we need to do multi-claims? 106 deletes []schema.Claim 107 } 108 109 func (mm *mutationMap) Set(k, v string) { 110 if mm.kv == nil { 111 mm.kv = make(map[string]string) 112 } 113 mm.kv[k] = v 114 } 115 116 func (mm *mutationMap) noteDelete(deleteClaim schema.Claim) { 117 mm.deletes = append(mm.deletes, deleteClaim) 118 } 119 120 func blobsFilteringOut(v []blob.Ref, x blob.Ref) []blob.Ref { 121 switch len(v) { 122 case 0: 123 return nil 124 case 1: 125 if v[0] == x { 126 return nil 127 } 128 return v 129 } 130 nl := v[:0] 131 for _, vb := range v { 132 if vb != x { 133 nl = append(nl, vb) 134 } 135 } 136 return nl 137 } 138 139 func (ix *Index) noteBlobIndexed(br blob.Ref) { 140 ix.mu.Lock() 141 defer ix.mu.Unlock() 142 for _, needer := range ix.neededBy[br] { 143 newNeeds := blobsFilteringOut(ix.needs[needer], br) 144 if len(newNeeds) == 0 { 145 ix.readyReindex[needer] = true 146 delete(ix.needs, needer) 147 select { 148 case ix.tickleOoo <- true: 149 default: 150 } 151 } else { 152 ix.needs[needer] = newNeeds 153 } 154 } 155 delete(ix.neededBy, br) 156 } 157 158 func (ix *Index) removeAllMissingEdges(br blob.Ref) { 159 var toDelete []string 160 it := ix.queryPrefix(keyMissing, br) 161 for it.Next() { 162 toDelete = append(toDelete, it.Key()) 163 } 164 if err := it.Close(); err != nil { 165 // TODO: Care? Can lazily clean up later. 166 log.Printf("Iterator close error: %v", err) 167 } 168 for _, k := range toDelete { 169 if err := ix.s.Delete(k); err != nil { 170 log.Printf("Error deleting key %s: %v", k, err) 171 } 172 } 173 } 174 175 func (ix *Index) ReceiveBlob(blobRef blob.Ref, source io.Reader) (retsb blob.SizedRef, err error) { 176 missingDeps := false 177 defer func() { 178 if err == nil { 179 ix.noteBlobIndexed(blobRef) 180 if !missingDeps { 181 ix.removeAllMissingEdges(blobRef) 182 } 183 } 184 }() 185 sniffer := NewBlobSniffer(blobRef) 186 written, err := io.Copy(sniffer, source) 187 if err != nil { 188 return 189 } 190 if _, haveErr := ix.s.Get("have:" + blobRef.String()); haveErr == nil { 191 return blob.SizedRef{blobRef, uint32(written)}, nil 192 } 193 194 sniffer.Parse() 195 196 fetcher := &missTrackFetcher{ 197 fetcher: ix.BlobSource, 198 } 199 200 mm, err := ix.populateMutationMap(fetcher, blobRef, sniffer) 201 if err != nil { 202 fetcher.mu.Lock() 203 defer fetcher.mu.Unlock() 204 if len(fetcher.missing) == 0 { 205 return 206 } 207 missingDeps = true 208 allRecorded := true 209 for _, missing := range fetcher.missing { 210 if err := ix.noteNeeded(blobRef, missing); err != nil { 211 allRecorded = false 212 } 213 } 214 if allRecorded { 215 // Lie and say things are good. We've 216 // successfully recorded that the blob isn't 217 // indexed, but we'll reindex it later once 218 // the dependent blobs arrive. 219 return blob.SizedRef{blobRef, uint32(written)}, nil 220 } 221 return 222 } 223 224 if err := ix.commit(mm); err != nil { 225 return retsb, err 226 } 227 228 if c := ix.corpus; c != nil { 229 if err = c.addBlob(blobRef, mm); err != nil { 230 return 231 } 232 } 233 234 // TODO(bradfitz): log levels? These are generally noisy 235 // (especially in tests, like search/handler_test), but I 236 // could see it being useful in production. For now, disabled: 237 // 238 // mimeType := sniffer.MIMEType() 239 // log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated()) 240 241 return blob.SizedRef{blobRef, uint32(written)}, nil 242 } 243 244 // commit writes the contents of the mutationMap on a batch 245 // mutation and commits that batch. It also updates the deletes 246 // cache. 247 func (ix *Index) commit(mm *mutationMap) error { 248 // We want the update of the deletes cache to be atomic 249 // with the transaction commit, so we lock here instead 250 // of within updateDeletesCache. 251 ix.deletes.Lock() 252 defer ix.deletes.Unlock() 253 bm := ix.s.BeginBatch() 254 for k, v := range mm.kv { 255 bm.Set(k, v) 256 } 257 err := ix.s.CommitBatch(bm) 258 if err != nil { 259 return err 260 } 261 for _, cl := range mm.deletes { 262 if err := ix.updateDeletesCache(cl); err != nil { 263 return fmt.Errorf("Could not update the deletes cache after deletion from %v: %v", cl, err) 264 } 265 } 266 return nil 267 } 268 269 // populateMutationMap populates keys & values that will be committed 270 // into the returned map. 271 // 272 // the blobref can be trusted at this point (it's been fully consumed 273 // and verified to match), and the sniffer has been populated. 274 func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) { 275 // TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when 276 // e.g in populateClaim we detect a bogus claim (which does not yield an error)? 277 mm := &mutationMap{ 278 kv: map[string]string{ 279 "have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()), 280 "meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()), 281 }, 282 } 283 284 if blob, ok := sniffer.SchemaBlob(); ok { 285 switch blob.Type() { 286 case "claim": 287 if err := ix.populateClaim(fetcher, blob, mm); err != nil { 288 return nil, err 289 } 290 case "file": 291 if err := ix.populateFile(fetcher, blob, mm); err != nil { 292 return nil, err 293 } 294 case "directory": 295 if err := ix.populateDir(fetcher, blob, mm); err != nil { 296 return nil, err 297 } 298 } 299 } 300 301 return mm, nil 302 } 303 304 // keepFirstN keeps the first N bytes written to it in Bytes. 305 type keepFirstN struct { 306 N int 307 Bytes []byte 308 } 309 310 func (w *keepFirstN) Write(p []byte) (n int, err error) { 311 if n := w.N - len(w.Bytes); n > 0 { 312 if n > len(p) { 313 n = len(p) 314 } 315 w.Bytes = append(w.Bytes, p[:n]...) 316 } 317 return len(p), nil 318 } 319 320 // missTrackFetcher is a blob.Fetcher that records which blob(s) it 321 // failed to load from src. 322 type missTrackFetcher struct { 323 fetcher blob.Fetcher 324 325 mu sync.Mutex // guards missing 326 missing []blob.Ref 327 } 328 329 func (f *missTrackFetcher) Fetch(br blob.Ref) (blob io.ReadCloser, size uint32, err error) { 330 blob, size, err = f.fetcher.Fetch(br) 331 if err == os.ErrNotExist { 332 f.mu.Lock() 333 defer f.mu.Unlock() 334 f.missing = append(f.missing, br) 335 } 336 return 337 } 338 339 // b: the parsed file schema blob 340 // mm: keys to populate 341 func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) (err error) { 342 var times []time.Time // all creation or mod times seen; may be zero 343 times = append(times, b.ModTime()) 344 345 blobRef := b.BlobRef() 346 fr, err := b.NewFileReader(fetcher) 347 if err != nil { 348 return err 349 } 350 defer fr.Close() 351 mime, reader := magic.MIMETypeFromReader(fr) 352 353 sha1 := sha1.New() 354 var copyDest io.Writer = sha1 355 var imageBuf *keepFirstN // or nil 356 if strings.HasPrefix(mime, "image/") { 357 // Emperically derived 1MiB assuming CR2 images require more than any 358 // other filetype we support: 359 // https://gist.github.com/wathiede/7982372 360 imageBuf = &keepFirstN{N: 1 << 20} 361 copyDest = io.MultiWriter(copyDest, imageBuf) 362 } 363 size, err := io.Copy(copyDest, reader) 364 if err != nil { 365 return err 366 } 367 wholeRef := blob.RefFromHash(sha1) 368 369 if imageBuf != nil { 370 if conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes)); err == nil { 371 mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height))) 372 } 373 if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil { 374 log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err) 375 times = append(times, ft) 376 } else { 377 log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err) 378 } 379 380 indexEXIF(wholeRef, imageBuf.Bytes, mm) 381 } 382 383 var sortTimes []time.Time 384 for _, t := range times { 385 if !t.IsZero() { 386 sortTimes = append(sortTimes, t) 387 } 388 } 389 sort.Sort(types.ByTime(sortTimes)) 390 var time3339s string 391 switch { 392 case len(sortTimes) == 1: 393 time3339s = types.Time3339(sortTimes[0]).String() 394 case len(sortTimes) >= 2: 395 oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1] 396 time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String() 397 } 398 399 mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1") 400 mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime)) 401 mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s)) 402 403 if strings.HasPrefix(mime, "audio/") { 404 indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm) 405 } 406 407 return nil 408 } 409 410 func tagFormatString(tag *tiff.Tag) string { 411 switch tag.Format() { 412 case tiff.IntVal: 413 return "int" 414 case tiff.RatVal: 415 return "rat" 416 case tiff.FloatVal: 417 return "float" 418 case tiff.StringVal: 419 return "string" 420 } 421 return "" 422 } 423 424 type exifWalkFunc func(name exif.FieldName, tag *tiff.Tag) error 425 426 func (f exifWalkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error { return f(name, tag) } 427 428 func indexEXIF(wholeRef blob.Ref, header []byte, mm *mutationMap) { 429 ex, err := exif.Decode(bytes.NewReader(header)) 430 if err != nil { 431 return 432 } 433 defer func() { 434 // The EXIF library panics if you access a field past 435 // what the file contains. Be paranoid and just 436 // recover here, instead of crashing on an invalid 437 // EXIF file. 438 if e := recover(); e != nil { 439 log.Printf("Ignoring invalid EXIF file. Caught panic: %v", e) 440 } 441 }() 442 443 ex.Walk(exifWalkFunc(func(name exif.FieldName, tag *tiff.Tag) error { 444 tagFmt := tagFormatString(tag) 445 if tagFmt == "" { 446 return nil 447 } 448 key := keyEXIFTag.Key(wholeRef, fmt.Sprintf("%04x", tag.Id)) 449 numComp := int(tag.Ncomp) 450 if tag.Format() == tiff.StringVal { 451 numComp = 1 452 } 453 var val bytes.Buffer 454 val.WriteString(keyEXIFTag.Val(tagFmt, numComp, "")) 455 if tag.Format() == tiff.StringVal { 456 str := tag.StringVal() 457 if containsUnsafeRawStrByte(str) { 458 val.WriteString(urle(str)) 459 } else { 460 val.WriteString(str) 461 } 462 } else { 463 for i := 0; i < int(tag.Ncomp); i++ { 464 if i > 0 { 465 val.WriteByte('|') 466 } 467 switch tagFmt { 468 case "int": 469 fmt.Fprintf(&val, "%d", tag.Int(i)) 470 case "rat": 471 n, d := tag.Rat2(i) 472 fmt.Fprintf(&val, "%d/%d", n, d) 473 case "float": 474 fmt.Fprintf(&val, "%v", tag.Float(i)) 475 default: 476 panic("shouldn't get here") 477 } 478 } 479 } 480 valStr := val.String() 481 mm.Set(key, valStr) 482 return nil 483 })) 484 485 longTag, err := ex.Get(exif.FieldName("GPSLongitude")) 486 if err != nil { 487 return 488 } 489 ewTag, err := ex.Get(exif.FieldName("GPSLongitudeRef")) 490 if err != nil { 491 return 492 } 493 latTag, err := ex.Get(exif.FieldName("GPSLatitude")) 494 if err != nil { 495 return 496 } 497 nsTag, err := ex.Get(exif.FieldName("GPSLatitudeRef")) 498 if err != nil { 499 return 500 } 501 long := tagDegrees(longTag) 502 lat := tagDegrees(latTag) 503 if ewTag.StringVal() == "W" { 504 long *= -1.0 505 } 506 if nsTag.StringVal() == "S" { 507 lat *= -1.0 508 } 509 mm.Set(keyEXIFGPS.Key(wholeRef), keyEXIFGPS.Val(fmt.Sprint(lat), fmt.Sprint(long))) 510 } 511 512 func ratFloat(num, dem int64) float64 { 513 return float64(num) / float64(dem) 514 } 515 516 func tagDegrees(tag *tiff.Tag) float64 { 517 return ratFloat(tag.Rat2(0)) + ratFloat(tag.Rat2(1))/60 + ratFloat(tag.Rat2(2))/3600 518 } 519 520 // indexMusic adds mutations to index the wholeRef by attached metadata and other properties. 521 func indexMusic(r types.SizeReaderAt, wholeRef blob.Ref, mm *mutationMap) { 522 tag, err := taglib.Decode(r, r.Size()) 523 if err != nil { 524 log.Print("index: error parsing tag: ", err) 525 return 526 } 527 528 var footerLength int64 = 0 529 if hasTag, err := media.HasID3v1Tag(r); err != nil { 530 log.Print("index: unable to check for ID3v1 tag: ", err) 531 return 532 } else if hasTag { 533 footerLength = media.ID3v1TagLength 534 } 535 536 // Generate a hash of the audio portion of the file (i.e. excluding ID3v1 and v2 tags). 537 audioStart := int64(tag.TagSize()) 538 audioSize := r.Size() - audioStart - footerLength 539 hash := sha1.New() 540 if _, err := io.Copy(hash, io.NewSectionReader(r, audioStart, audioSize)); err != nil { 541 log.Print("index: error generating SHA1 from audio data: ", err) 542 return 543 } 544 mediaRef := blob.RefFromHash(hash) 545 546 duration, err := media.GetMPEGAudioDuration(io.NewSectionReader(r, audioStart, audioSize)) 547 if err != nil { 548 log.Print("index: unable to calculate audio duration: ", err) 549 duration = 0 550 } 551 552 var yearStr, trackStr, discStr, durationStr string 553 if !tag.Year().IsZero() { 554 const justYearLayout = "2006" 555 yearStr = tag.Year().Format(justYearLayout) 556 } 557 if tag.Track() != 0 { 558 trackStr = fmt.Sprintf("%d", tag.Track()) 559 } 560 if tag.Disc() != 0 { 561 discStr = fmt.Sprintf("%d", tag.Disc()) 562 } 563 if duration != 0 { 564 durationStr = fmt.Sprintf("%d", duration/time.Millisecond) 565 } 566 567 // Note: if you add to this map, please update 568 // pkg/search/query.go's MediaTagConstraint Tag docs. 569 tags := map[string]string{ 570 "title": tag.Title(), 571 "artist": tag.Artist(), 572 "album": tag.Album(), 573 "genre": tag.Genre(), 574 "musicbrainzalbumid": tag.CustomFrames()["MusicBrainz Album Id"], 575 "year": yearStr, 576 "track": trackStr, 577 "disc": discStr, 578 "mediaref": mediaRef.String(), 579 "durationms": durationStr, 580 } 581 582 for tag, value := range tags { 583 if value != "" { 584 mm.Set(keyMediaTag.Key(wholeRef, tag), keyMediaTag.Val(value)) 585 } 586 } 587 } 588 589 // b: the parsed file schema blob 590 // mm: keys to populate 591 func (ix *Index) populateDir(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) error { 592 blobRef := b.BlobRef() 593 // TODO(bradfitz): move the NewDirReader and FileName method off *schema.Blob and onto 594 // StaticFile/StaticDirectory or something. 595 596 dr, err := b.NewDirReader(fetcher) 597 if err != nil { 598 // TODO(bradfitz): propagate up a transient failure 599 // error type, so we can retry indexing files in the 600 // future if blobs are only temporarily unavailable. 601 log.Printf("index: error indexing directory, creating NewDirReader %s: %v", blobRef, err) 602 return nil 603 } 604 sts, err := dr.StaticSet() 605 if err != nil { 606 log.Printf("index: error indexing directory: can't get StaticSet: %v\n", err) 607 return nil 608 } 609 610 mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), "")) 611 for _, br := range sts { 612 mm.Set(keyStaticDirChild.Key(blobRef, br.String()), "1") 613 } 614 return nil 615 } 616 617 // populateDeleteClaim adds to mm the entries resulting from the delete claim cl. 618 // It is assumed cl is a valid claim, and vr has already been verified. 619 func (ix *Index) populateDeleteClaim(cl schema.Claim, vr *jsonsign.VerifyRequest, mm *mutationMap) { 620 br := cl.Blob().BlobRef() 621 target := cl.Target() 622 if !target.Valid() { 623 log.Print(fmt.Errorf("no valid target for delete claim %v", br)) 624 return 625 } 626 meta, err := ix.GetBlobMeta(target) 627 if err != nil { 628 if err == os.ErrNotExist { 629 // TODO: return a dependency error type, to schedule re-indexing in the future 630 } 631 log.Print(fmt.Errorf("Could not get mime type of target blob %v: %v", target, err)) 632 return 633 } 634 // TODO(mpl): create consts somewhere for "claim" and "permanode" as camliTypes, and use them, 635 // instead of hardcoding. Unless they already exist ? (didn't find them). 636 if meta.CamliType != "permanode" && meta.CamliType != "claim" { 637 log.Print(fmt.Errorf("delete claim target in %v is neither a permanode nor a claim: %v", br, meta.CamliType)) 638 return 639 } 640 mm.Set(keyDeleted.Key(target, cl.ClaimDateString(), br), "") 641 if meta.CamliType == "claim" { 642 return 643 } 644 recentKey := keyRecentPermanode.Key(vr.SignerKeyId, cl.ClaimDateString(), br) 645 mm.Set(recentKey, target.String()) 646 attr, value := cl.Attribute(), cl.Value() 647 claimKey := keyPermanodeClaim.Key(target, vr.SignerKeyId, cl.ClaimDateString(), br) 648 mm.Set(claimKey, keyPermanodeClaim.Val(cl.ClaimType(), attr, value, vr.CamliSigner)) 649 } 650 651 func (ix *Index) populateClaim(fetcher *missTrackFetcher, b *schema.Blob, mm *mutationMap) error { 652 br := b.BlobRef() 653 654 claim, ok := b.AsClaim() 655 if !ok { 656 // Skip bogus claim with malformed permanode. 657 return nil 658 } 659 660 vr := jsonsign.NewVerificationRequest(b.JSON(), blob.NewSerialFetcher(ix.KeyFetcher, fetcher)) 661 if !vr.Verify() { 662 // TODO(bradfitz): ask if the vr.Err.(jsonsign.Error).IsPermanent() and retry 663 // later if it's not permanent? or maybe do this up a level? 664 if vr.Err != nil { 665 return vr.Err 666 } 667 return errors.New("index: populateClaim verification failure") 668 } 669 verifiedKeyId := vr.SignerKeyId 670 mm.Set("signerkeyid:"+vr.CamliSigner.String(), verifiedKeyId) 671 672 if claim.ClaimType() == string(schema.DeleteClaim) { 673 ix.populateDeleteClaim(claim, vr, mm) 674 mm.noteDelete(claim) 675 return nil 676 } 677 678 pnbr := claim.ModifiedPermanode() 679 if !pnbr.Valid() { 680 // A different type of claim; not modifying a permanode. 681 return nil 682 } 683 684 attr, value := claim.Attribute(), claim.Value() 685 recentKey := keyRecentPermanode.Key(verifiedKeyId, claim.ClaimDateString(), br) 686 mm.Set(recentKey, pnbr.String()) 687 claimKey := keyPermanodeClaim.Key(pnbr, verifiedKeyId, claim.ClaimDateString(), br) 688 mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value, vr.CamliSigner)) 689 690 if strings.HasPrefix(attr, "camliPath:") { 691 targetRef, ok := blob.Parse(value) 692 if ok { 693 // TODO: deal with set-attribute vs. del-attribute 694 // properly? I think we get it for free when 695 // del-attribute has no Value, but we need to deal 696 // with the case where they explicitly delete the 697 // current value. 698 suffix := attr[len("camliPath:"):] 699 active := "Y" 700 if claim.ClaimType() == "del-attribute" { 701 active = "N" 702 } 703 baseRef := pnbr 704 claimRef := br 705 706 key := keyPathBackward.Key(verifiedKeyId, targetRef, claimRef) 707 val := keyPathBackward.Val(claim.ClaimDateString(), baseRef, active, suffix) 708 mm.Set(key, val) 709 710 key = keyPathForward.Key(verifiedKeyId, baseRef, suffix, claim.ClaimDateString(), claimRef) 711 val = keyPathForward.Val(active, targetRef) 712 mm.Set(key, val) 713 } 714 } 715 716 if claim.ClaimType() != string(schema.DelAttributeClaim) && IsIndexedAttribute(attr) { 717 key := keySignerAttrValue.Key(verifiedKeyId, attr, value, claim.ClaimDateString(), br) 718 mm.Set(key, keySignerAttrValue.Val(pnbr)) 719 } 720 721 if IsBlobReferenceAttribute(attr) { 722 targetRef, ok := blob.Parse(value) 723 if ok { 724 key := keyEdgeBackward.Key(targetRef, pnbr, br) 725 mm.Set(key, keyEdgeBackward.Val("permanode", "")) 726 } 727 } 728 729 return nil 730 } 731 732 // updateDeletesCache updates the index deletes cache with the cl delete claim. 733 // deleteClaim is trusted to be a valid delete Claim. 734 func (x *Index) updateDeletesCache(deleteClaim schema.Claim) error { 735 target := deleteClaim.Target() 736 deleter := deleteClaim.Blob() 737 when, err := deleter.ClaimDate() 738 if err != nil { 739 return fmt.Errorf("Could not get date of delete claim %v: %v", deleteClaim, err) 740 } 741 targetDeletions := append(x.deletes.m[target], 742 deletion{ 743 deleter: deleter.BlobRef(), 744 when: when, 745 }) 746 sort.Sort(sort.Reverse(byDeletionDate(targetDeletions))) 747 x.deletes.m[target] = targetDeletions 748 return nil 749 }