github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/index/receive.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package index
    18  
    19  import (
    20  	"bytes"
    21  	"crypto/sha1"
    22  	"errors"
    23  	"fmt"
    24  	_ "image/gif"
    25  	_ "image/jpeg"
    26  	_ "image/png"
    27  	"io"
    28  	"log"
    29  	"os"
    30  	"sort"
    31  	"strings"
    32  	"sync"
    33  	"time"
    34  
    35  	"camlistore.org/pkg/blob"
    36  	"camlistore.org/pkg/blobserver"
    37  	"camlistore.org/pkg/images"
    38  	"camlistore.org/pkg/jsonsign"
    39  	"camlistore.org/pkg/magic"
    40  	"camlistore.org/pkg/media"
    41  	"camlistore.org/pkg/schema"
    42  	"camlistore.org/pkg/types"
    43  
    44  	"camlistore.org/third_party/github.com/camlistore/goexif/exif"
    45  	"camlistore.org/third_party/github.com/camlistore/goexif/tiff"
    46  	"camlistore.org/third_party/github.com/hjfreyer/taglib-go/taglib"
    47  )
    48  
    49  func (ix *Index) outOfOrderIndexerLoop() {
    50  	if ix.BlobSource == nil {
    51  		// Bail early. Nothing will work later anyway. (for tests)
    52  		return
    53  	}
    54  WaitTickle:
    55  	for _ = range ix.tickleOoo {
    56  		for {
    57  			ix.mu.Lock()
    58  			if len(ix.readyReindex) == 0 {
    59  				ix.mu.Unlock()
    60  				continue WaitTickle
    61  			}
    62  			var br blob.Ref
    63  			for br = range ix.readyReindex {
    64  				break
    65  			}
    66  			delete(ix.readyReindex, br)
    67  			ix.mu.Unlock()
    68  
    69  			err := ix.indexBlob(br)
    70  			if err != nil {
    71  				log.Printf("out-of-order indexBlob(%v) = %v", br, err)
    72  				ix.mu.Lock()
    73  				if len(ix.needs[br]) == 0 {
    74  					ix.readyReindex[br] = true
    75  				}
    76  				ix.mu.Unlock()
    77  			}
    78  		}
    79  	}
    80  }
    81  
    82  func (ix *Index) indexBlob(br blob.Ref) error {
    83  	bs := ix.BlobSource
    84  	if bs == nil {
    85  		return fmt.Errorf("index: can't re-index %v: no BlobSource", br)
    86  	}
    87  	rc, _, err := bs.Fetch(br)
    88  	if err != nil {
    89  		return fmt.Errorf("index: failed to fetch %v for reindexing: %v", br, err)
    90  	}
    91  	defer rc.Close()
    92  	if _, err := blobserver.Receive(ix, br, rc); err != nil {
    93  		return err
    94  	}
    95  	return nil
    96  }
    97  
    98  type mutationMap struct {
    99  	kv map[string]string // the keys and values we populate
   100  
   101  	// We record if we get a delete claim, so we can update
   102  	// the deletes cache right after committing the mutation.
   103  	//
   104  	// TODO(mpl): we only need to keep track of one claim so far,
   105  	// but I chose a slice for when we need to do multi-claims?
   106  	deletes []schema.Claim
   107  }
   108  
   109  func (mm *mutationMap) Set(k, v string) {
   110  	if mm.kv == nil {
   111  		mm.kv = make(map[string]string)
   112  	}
   113  	mm.kv[k] = v
   114  }
   115  
   116  func (mm *mutationMap) noteDelete(deleteClaim schema.Claim) {
   117  	mm.deletes = append(mm.deletes, deleteClaim)
   118  }
   119  
   120  func blobsFilteringOut(v []blob.Ref, x blob.Ref) []blob.Ref {
   121  	switch len(v) {
   122  	case 0:
   123  		return nil
   124  	case 1:
   125  		if v[0] == x {
   126  			return nil
   127  		}
   128  		return v
   129  	}
   130  	nl := v[:0]
   131  	for _, vb := range v {
   132  		if vb != x {
   133  			nl = append(nl, vb)
   134  		}
   135  	}
   136  	return nl
   137  }
   138  
   139  func (ix *Index) noteBlobIndexed(br blob.Ref) {
   140  	ix.mu.Lock()
   141  	defer ix.mu.Unlock()
   142  	for _, needer := range ix.neededBy[br] {
   143  		newNeeds := blobsFilteringOut(ix.needs[needer], br)
   144  		if len(newNeeds) == 0 {
   145  			ix.readyReindex[needer] = true
   146  			delete(ix.needs, needer)
   147  			select {
   148  			case ix.tickleOoo <- true:
   149  			default:
   150  			}
   151  		} else {
   152  			ix.needs[needer] = newNeeds
   153  		}
   154  	}
   155  	delete(ix.neededBy, br)
   156  }
   157  
   158  func (ix *Index) removeAllMissingEdges(br blob.Ref) {
   159  	var toDelete []string
   160  	it := ix.queryPrefix(keyMissing, br)
   161  	for it.Next() {
   162  		toDelete = append(toDelete, it.Key())
   163  	}
   164  	if err := it.Close(); err != nil {
   165  		// TODO: Care? Can lazily clean up later.
   166  		log.Printf("Iterator close error: %v", err)
   167  	}
   168  	for _, k := range toDelete {
   169  		if err := ix.s.Delete(k); err != nil {
   170  			log.Printf("Error deleting key %s: %v", k, err)
   171  		}
   172  	}
   173  }
   174  
   175  func (ix *Index) ReceiveBlob(blobRef blob.Ref, source io.Reader) (retsb blob.SizedRef, err error) {
   176  	missingDeps := false
   177  	defer func() {
   178  		if err == nil {
   179  			ix.noteBlobIndexed(blobRef)
   180  			if !missingDeps {
   181  				ix.removeAllMissingEdges(blobRef)
   182  			}
   183  		}
   184  	}()
   185  	sniffer := NewBlobSniffer(blobRef)
   186  	written, err := io.Copy(sniffer, source)
   187  	if err != nil {
   188  		return
   189  	}
   190  	if _, haveErr := ix.s.Get("have:" + blobRef.String()); haveErr == nil {
   191  		return blob.SizedRef{blobRef, uint32(written)}, nil
   192  	}
   193  
   194  	sniffer.Parse()
   195  
   196  	fetcher := &missTrackFetcher{
   197  		fetcher: ix.BlobSource,
   198  	}
   199  
   200  	mm, err := ix.populateMutationMap(fetcher, blobRef, sniffer)
   201  	if err != nil {
   202  		fetcher.mu.Lock()
   203  		defer fetcher.mu.Unlock()
   204  		if len(fetcher.missing) == 0 {
   205  			return
   206  		}
   207  		missingDeps = true
   208  		allRecorded := true
   209  		for _, missing := range fetcher.missing {
   210  			if err := ix.noteNeeded(blobRef, missing); err != nil {
   211  				allRecorded = false
   212  			}
   213  		}
   214  		if allRecorded {
   215  			// Lie and say things are good. We've
   216  			// successfully recorded that the blob isn't
   217  			// indexed, but we'll reindex it later once
   218  			// the dependent blobs arrive.
   219  			return blob.SizedRef{blobRef, uint32(written)}, nil
   220  		}
   221  		return
   222  	}
   223  
   224  	if err := ix.commit(mm); err != nil {
   225  		return retsb, err
   226  	}
   227  
   228  	if c := ix.corpus; c != nil {
   229  		if err = c.addBlob(blobRef, mm); err != nil {
   230  			return
   231  		}
   232  	}
   233  
   234  	// TODO(bradfitz): log levels? These are generally noisy
   235  	// (especially in tests, like search/handler_test), but I
   236  	// could see it being useful in production. For now, disabled:
   237  	//
   238  	// mimeType := sniffer.MIMEType()
   239  	// log.Printf("indexer: received %s; type=%v; truncated=%v", blobRef, mimeType, sniffer.IsTruncated())
   240  
   241  	return blob.SizedRef{blobRef, uint32(written)}, nil
   242  }
   243  
   244  // commit writes the contents of the mutationMap on a batch
   245  // mutation and commits that batch. It also updates the deletes
   246  // cache.
   247  func (ix *Index) commit(mm *mutationMap) error {
   248  	// We want the update of the deletes cache to be atomic
   249  	// with the transaction commit, so we lock here instead
   250  	// of within updateDeletesCache.
   251  	ix.deletes.Lock()
   252  	defer ix.deletes.Unlock()
   253  	bm := ix.s.BeginBatch()
   254  	for k, v := range mm.kv {
   255  		bm.Set(k, v)
   256  	}
   257  	err := ix.s.CommitBatch(bm)
   258  	if err != nil {
   259  		return err
   260  	}
   261  	for _, cl := range mm.deletes {
   262  		if err := ix.updateDeletesCache(cl); err != nil {
   263  			return fmt.Errorf("Could not update the deletes cache after deletion from %v: %v", cl, err)
   264  		}
   265  	}
   266  	return nil
   267  }
   268  
   269  // populateMutationMap populates keys & values that will be committed
   270  // into the returned map.
   271  //
   272  // the blobref can be trusted at this point (it's been fully consumed
   273  // and verified to match), and the sniffer has been populated.
   274  func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) {
   275  	// TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when
   276  	// e.g in populateClaim we detect a bogus claim (which does not yield an error)?
   277  	mm := &mutationMap{
   278  		kv: map[string]string{
   279  			"have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()),
   280  			"meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()),
   281  		},
   282  	}
   283  
   284  	if blob, ok := sniffer.SchemaBlob(); ok {
   285  		switch blob.Type() {
   286  		case "claim":
   287  			if err := ix.populateClaim(fetcher, blob, mm); err != nil {
   288  				return nil, err
   289  			}
   290  		case "file":
   291  			if err := ix.populateFile(fetcher, blob, mm); err != nil {
   292  				return nil, err
   293  			}
   294  		case "directory":
   295  			if err := ix.populateDir(fetcher, blob, mm); err != nil {
   296  				return nil, err
   297  			}
   298  		}
   299  	}
   300  
   301  	return mm, nil
   302  }
   303  
   304  // keepFirstN keeps the first N bytes written to it in Bytes.
   305  type keepFirstN struct {
   306  	N     int
   307  	Bytes []byte
   308  }
   309  
   310  func (w *keepFirstN) Write(p []byte) (n int, err error) {
   311  	if n := w.N - len(w.Bytes); n > 0 {
   312  		if n > len(p) {
   313  			n = len(p)
   314  		}
   315  		w.Bytes = append(w.Bytes, p[:n]...)
   316  	}
   317  	return len(p), nil
   318  }
   319  
   320  // missTrackFetcher is a blob.Fetcher that records which blob(s) it
   321  // failed to load from src.
   322  type missTrackFetcher struct {
   323  	fetcher blob.Fetcher
   324  
   325  	mu      sync.Mutex // guards missing
   326  	missing []blob.Ref
   327  }
   328  
   329  func (f *missTrackFetcher) Fetch(br blob.Ref) (blob io.ReadCloser, size uint32, err error) {
   330  	blob, size, err = f.fetcher.Fetch(br)
   331  	if err == os.ErrNotExist {
   332  		f.mu.Lock()
   333  		defer f.mu.Unlock()
   334  		f.missing = append(f.missing, br)
   335  	}
   336  	return
   337  }
   338  
   339  // b: the parsed file schema blob
   340  // mm: keys to populate
   341  func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) (err error) {
   342  	var times []time.Time // all creation or mod times seen; may be zero
   343  	times = append(times, b.ModTime())
   344  
   345  	blobRef := b.BlobRef()
   346  	fr, err := b.NewFileReader(fetcher)
   347  	if err != nil {
   348  		return err
   349  	}
   350  	defer fr.Close()
   351  	mime, reader := magic.MIMETypeFromReader(fr)
   352  
   353  	sha1 := sha1.New()
   354  	var copyDest io.Writer = sha1
   355  	var imageBuf *keepFirstN // or nil
   356  	if strings.HasPrefix(mime, "image/") {
   357  		// Emperically derived 1MiB assuming CR2 images require more than any
   358  		// other filetype we support:
   359  		//   https://gist.github.com/wathiede/7982372
   360  		imageBuf = &keepFirstN{N: 1 << 20}
   361  		copyDest = io.MultiWriter(copyDest, imageBuf)
   362  	}
   363  	size, err := io.Copy(copyDest, reader)
   364  	if err != nil {
   365  		return err
   366  	}
   367  	wholeRef := blob.RefFromHash(sha1)
   368  
   369  	if imageBuf != nil {
   370  		if conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes)); err == nil {
   371  			mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
   372  		}
   373  		if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil {
   374  			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
   375  			times = append(times, ft)
   376  		} else {
   377  			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
   378  		}
   379  
   380  		indexEXIF(wholeRef, imageBuf.Bytes, mm)
   381  	}
   382  
   383  	var sortTimes []time.Time
   384  	for _, t := range times {
   385  		if !t.IsZero() {
   386  			sortTimes = append(sortTimes, t)
   387  		}
   388  	}
   389  	sort.Sort(types.ByTime(sortTimes))
   390  	var time3339s string
   391  	switch {
   392  	case len(sortTimes) == 1:
   393  		time3339s = types.Time3339(sortTimes[0]).String()
   394  	case len(sortTimes) >= 2:
   395  		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
   396  		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
   397  	}
   398  
   399  	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
   400  	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime))
   401  	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))
   402  
   403  	if strings.HasPrefix(mime, "audio/") {
   404  		indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm)
   405  	}
   406  
   407  	return nil
   408  }
   409  
   410  func tagFormatString(tag *tiff.Tag) string {
   411  	switch tag.Format() {
   412  	case tiff.IntVal:
   413  		return "int"
   414  	case tiff.RatVal:
   415  		return "rat"
   416  	case tiff.FloatVal:
   417  		return "float"
   418  	case tiff.StringVal:
   419  		return "string"
   420  	}
   421  	return ""
   422  }
   423  
   424  type exifWalkFunc func(name exif.FieldName, tag *tiff.Tag) error
   425  
   426  func (f exifWalkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error { return f(name, tag) }
   427  
   428  func indexEXIF(wholeRef blob.Ref, header []byte, mm *mutationMap) {
   429  	ex, err := exif.Decode(bytes.NewReader(header))
   430  	if err != nil {
   431  		return
   432  	}
   433  	defer func() {
   434  		// The EXIF library panics if you access a field past
   435  		// what the file contains.  Be paranoid and just
   436  		// recover here, instead of crashing on an invalid
   437  		// EXIF file.
   438  		if e := recover(); e != nil {
   439  			log.Printf("Ignoring invalid EXIF file. Caught panic: %v", e)
   440  		}
   441  	}()
   442  
   443  	ex.Walk(exifWalkFunc(func(name exif.FieldName, tag *tiff.Tag) error {
   444  		tagFmt := tagFormatString(tag)
   445  		if tagFmt == "" {
   446  			return nil
   447  		}
   448  		key := keyEXIFTag.Key(wholeRef, fmt.Sprintf("%04x", tag.Id))
   449  		numComp := int(tag.Ncomp)
   450  		if tag.Format() == tiff.StringVal {
   451  			numComp = 1
   452  		}
   453  		var val bytes.Buffer
   454  		val.WriteString(keyEXIFTag.Val(tagFmt, numComp, ""))
   455  		if tag.Format() == tiff.StringVal {
   456  			str := tag.StringVal()
   457  			if containsUnsafeRawStrByte(str) {
   458  				val.WriteString(urle(str))
   459  			} else {
   460  				val.WriteString(str)
   461  			}
   462  		} else {
   463  			for i := 0; i < int(tag.Ncomp); i++ {
   464  				if i > 0 {
   465  					val.WriteByte('|')
   466  				}
   467  				switch tagFmt {
   468  				case "int":
   469  					fmt.Fprintf(&val, "%d", tag.Int(i))
   470  				case "rat":
   471  					n, d := tag.Rat2(i)
   472  					fmt.Fprintf(&val, "%d/%d", n, d)
   473  				case "float":
   474  					fmt.Fprintf(&val, "%v", tag.Float(i))
   475  				default:
   476  					panic("shouldn't get here")
   477  				}
   478  			}
   479  		}
   480  		valStr := val.String()
   481  		mm.Set(key, valStr)
   482  		return nil
   483  	}))
   484  
   485  	longTag, err := ex.Get(exif.FieldName("GPSLongitude"))
   486  	if err != nil {
   487  		return
   488  	}
   489  	ewTag, err := ex.Get(exif.FieldName("GPSLongitudeRef"))
   490  	if err != nil {
   491  		return
   492  	}
   493  	latTag, err := ex.Get(exif.FieldName("GPSLatitude"))
   494  	if err != nil {
   495  		return
   496  	}
   497  	nsTag, err := ex.Get(exif.FieldName("GPSLatitudeRef"))
   498  	if err != nil {
   499  		return
   500  	}
   501  	long := tagDegrees(longTag)
   502  	lat := tagDegrees(latTag)
   503  	if ewTag.StringVal() == "W" {
   504  		long *= -1.0
   505  	}
   506  	if nsTag.StringVal() == "S" {
   507  		lat *= -1.0
   508  	}
   509  	mm.Set(keyEXIFGPS.Key(wholeRef), keyEXIFGPS.Val(fmt.Sprint(lat), fmt.Sprint(long)))
   510  }
   511  
   512  func ratFloat(num, dem int64) float64 {
   513  	return float64(num) / float64(dem)
   514  }
   515  
   516  func tagDegrees(tag *tiff.Tag) float64 {
   517  	return ratFloat(tag.Rat2(0)) + ratFloat(tag.Rat2(1))/60 + ratFloat(tag.Rat2(2))/3600
   518  }
   519  
   520  // indexMusic adds mutations to index the wholeRef by attached metadata and other properties.
   521  func indexMusic(r types.SizeReaderAt, wholeRef blob.Ref, mm *mutationMap) {
   522  	tag, err := taglib.Decode(r, r.Size())
   523  	if err != nil {
   524  		log.Print("index: error parsing tag: ", err)
   525  		return
   526  	}
   527  
   528  	var footerLength int64 = 0
   529  	if hasTag, err := media.HasID3v1Tag(r); err != nil {
   530  		log.Print("index: unable to check for ID3v1 tag: ", err)
   531  		return
   532  	} else if hasTag {
   533  		footerLength = media.ID3v1TagLength
   534  	}
   535  
   536  	// Generate a hash of the audio portion of the file (i.e. excluding ID3v1 and v2 tags).
   537  	audioStart := int64(tag.TagSize())
   538  	audioSize := r.Size() - audioStart - footerLength
   539  	hash := sha1.New()
   540  	if _, err := io.Copy(hash, io.NewSectionReader(r, audioStart, audioSize)); err != nil {
   541  		log.Print("index: error generating SHA1 from audio data: ", err)
   542  		return
   543  	}
   544  	mediaRef := blob.RefFromHash(hash)
   545  
   546  	duration, err := media.GetMPEGAudioDuration(io.NewSectionReader(r, audioStart, audioSize))
   547  	if err != nil {
   548  		log.Print("index: unable to calculate audio duration: ", err)
   549  		duration = 0
   550  	}
   551  
   552  	var yearStr, trackStr, discStr, durationStr string
   553  	if !tag.Year().IsZero() {
   554  		const justYearLayout = "2006"
   555  		yearStr = tag.Year().Format(justYearLayout)
   556  	}
   557  	if tag.Track() != 0 {
   558  		trackStr = fmt.Sprintf("%d", tag.Track())
   559  	}
   560  	if tag.Disc() != 0 {
   561  		discStr = fmt.Sprintf("%d", tag.Disc())
   562  	}
   563  	if duration != 0 {
   564  		durationStr = fmt.Sprintf("%d", duration/time.Millisecond)
   565  	}
   566  
   567  	// Note: if you add to this map, please update
   568  	// pkg/search/query.go's MediaTagConstraint Tag docs.
   569  	tags := map[string]string{
   570  		"title":              tag.Title(),
   571  		"artist":             tag.Artist(),
   572  		"album":              tag.Album(),
   573  		"genre":              tag.Genre(),
   574  		"musicbrainzalbumid": tag.CustomFrames()["MusicBrainz Album Id"],
   575  		"year":               yearStr,
   576  		"track":              trackStr,
   577  		"disc":               discStr,
   578  		"mediaref":           mediaRef.String(),
   579  		"durationms":         durationStr,
   580  	}
   581  
   582  	for tag, value := range tags {
   583  		if value != "" {
   584  			mm.Set(keyMediaTag.Key(wholeRef, tag), keyMediaTag.Val(value))
   585  		}
   586  	}
   587  }
   588  
   589  // b: the parsed file schema blob
   590  // mm: keys to populate
   591  func (ix *Index) populateDir(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) error {
   592  	blobRef := b.BlobRef()
   593  	// TODO(bradfitz): move the NewDirReader and FileName method off *schema.Blob and onto
   594  	// StaticFile/StaticDirectory or something.
   595  
   596  	dr, err := b.NewDirReader(fetcher)
   597  	if err != nil {
   598  		// TODO(bradfitz): propagate up a transient failure
   599  		// error type, so we can retry indexing files in the
   600  		// future if blobs are only temporarily unavailable.
   601  		log.Printf("index: error indexing directory, creating NewDirReader %s: %v", blobRef, err)
   602  		return nil
   603  	}
   604  	sts, err := dr.StaticSet()
   605  	if err != nil {
   606  		log.Printf("index: error indexing directory: can't get StaticSet: %v\n", err)
   607  		return nil
   608  	}
   609  
   610  	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), ""))
   611  	for _, br := range sts {
   612  		mm.Set(keyStaticDirChild.Key(blobRef, br.String()), "1")
   613  	}
   614  	return nil
   615  }
   616  
   617  // populateDeleteClaim adds to mm the entries resulting from the delete claim cl.
   618  // It is assumed cl is a valid claim, and vr has already been verified.
   619  func (ix *Index) populateDeleteClaim(cl schema.Claim, vr *jsonsign.VerifyRequest, mm *mutationMap) {
   620  	br := cl.Blob().BlobRef()
   621  	target := cl.Target()
   622  	if !target.Valid() {
   623  		log.Print(fmt.Errorf("no valid target for delete claim %v", br))
   624  		return
   625  	}
   626  	meta, err := ix.GetBlobMeta(target)
   627  	if err != nil {
   628  		if err == os.ErrNotExist {
   629  			// TODO: return a dependency error type, to schedule re-indexing in the future
   630  		}
   631  		log.Print(fmt.Errorf("Could not get mime type of target blob %v: %v", target, err))
   632  		return
   633  	}
   634  	// TODO(mpl): create consts somewhere for "claim" and "permanode" as camliTypes, and use them,
   635  	// instead of hardcoding. Unless they already exist ? (didn't find them).
   636  	if meta.CamliType != "permanode" && meta.CamliType != "claim" {
   637  		log.Print(fmt.Errorf("delete claim target in %v is neither a permanode nor a claim: %v", br, meta.CamliType))
   638  		return
   639  	}
   640  	mm.Set(keyDeleted.Key(target, cl.ClaimDateString(), br), "")
   641  	if meta.CamliType == "claim" {
   642  		return
   643  	}
   644  	recentKey := keyRecentPermanode.Key(vr.SignerKeyId, cl.ClaimDateString(), br)
   645  	mm.Set(recentKey, target.String())
   646  	attr, value := cl.Attribute(), cl.Value()
   647  	claimKey := keyPermanodeClaim.Key(target, vr.SignerKeyId, cl.ClaimDateString(), br)
   648  	mm.Set(claimKey, keyPermanodeClaim.Val(cl.ClaimType(), attr, value, vr.CamliSigner))
   649  }
   650  
   651  func (ix *Index) populateClaim(fetcher *missTrackFetcher, b *schema.Blob, mm *mutationMap) error {
   652  	br := b.BlobRef()
   653  
   654  	claim, ok := b.AsClaim()
   655  	if !ok {
   656  		// Skip bogus claim with malformed permanode.
   657  		return nil
   658  	}
   659  
   660  	vr := jsonsign.NewVerificationRequest(b.JSON(), blob.NewSerialFetcher(ix.KeyFetcher, fetcher))
   661  	if !vr.Verify() {
   662  		// TODO(bradfitz): ask if the vr.Err.(jsonsign.Error).IsPermanent() and retry
   663  		// later if it's not permanent? or maybe do this up a level?
   664  		if vr.Err != nil {
   665  			return vr.Err
   666  		}
   667  		return errors.New("index: populateClaim verification failure")
   668  	}
   669  	verifiedKeyId := vr.SignerKeyId
   670  	mm.Set("signerkeyid:"+vr.CamliSigner.String(), verifiedKeyId)
   671  
   672  	if claim.ClaimType() == string(schema.DeleteClaim) {
   673  		ix.populateDeleteClaim(claim, vr, mm)
   674  		mm.noteDelete(claim)
   675  		return nil
   676  	}
   677  
   678  	pnbr := claim.ModifiedPermanode()
   679  	if !pnbr.Valid() {
   680  		// A different type of claim; not modifying a permanode.
   681  		return nil
   682  	}
   683  
   684  	attr, value := claim.Attribute(), claim.Value()
   685  	recentKey := keyRecentPermanode.Key(verifiedKeyId, claim.ClaimDateString(), br)
   686  	mm.Set(recentKey, pnbr.String())
   687  	claimKey := keyPermanodeClaim.Key(pnbr, verifiedKeyId, claim.ClaimDateString(), br)
   688  	mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value, vr.CamliSigner))
   689  
   690  	if strings.HasPrefix(attr, "camliPath:") {
   691  		targetRef, ok := blob.Parse(value)
   692  		if ok {
   693  			// TODO: deal with set-attribute vs. del-attribute
   694  			// properly? I think we get it for free when
   695  			// del-attribute has no Value, but we need to deal
   696  			// with the case where they explicitly delete the
   697  			// current value.
   698  			suffix := attr[len("camliPath:"):]
   699  			active := "Y"
   700  			if claim.ClaimType() == "del-attribute" {
   701  				active = "N"
   702  			}
   703  			baseRef := pnbr
   704  			claimRef := br
   705  
   706  			key := keyPathBackward.Key(verifiedKeyId, targetRef, claimRef)
   707  			val := keyPathBackward.Val(claim.ClaimDateString(), baseRef, active, suffix)
   708  			mm.Set(key, val)
   709  
   710  			key = keyPathForward.Key(verifiedKeyId, baseRef, suffix, claim.ClaimDateString(), claimRef)
   711  			val = keyPathForward.Val(active, targetRef)
   712  			mm.Set(key, val)
   713  		}
   714  	}
   715  
   716  	if claim.ClaimType() != string(schema.DelAttributeClaim) && IsIndexedAttribute(attr) {
   717  		key := keySignerAttrValue.Key(verifiedKeyId, attr, value, claim.ClaimDateString(), br)
   718  		mm.Set(key, keySignerAttrValue.Val(pnbr))
   719  	}
   720  
   721  	if IsBlobReferenceAttribute(attr) {
   722  		targetRef, ok := blob.Parse(value)
   723  		if ok {
   724  			key := keyEdgeBackward.Key(targetRef, pnbr, br)
   725  			mm.Set(key, keyEdgeBackward.Val("permanode", ""))
   726  		}
   727  	}
   728  
   729  	return nil
   730  }
   731  
   732  // updateDeletesCache updates the index deletes cache with the cl delete claim.
   733  // deleteClaim is trusted to be a valid delete Claim.
   734  func (x *Index) updateDeletesCache(deleteClaim schema.Claim) error {
   735  	target := deleteClaim.Target()
   736  	deleter := deleteClaim.Blob()
   737  	when, err := deleter.ClaimDate()
   738  	if err != nil {
   739  		return fmt.Errorf("Could not get date of delete claim %v: %v", deleteClaim, err)
   740  	}
   741  	targetDeletions := append(x.deletes.m[target],
   742  		deletion{
   743  			deleter: deleter.BlobRef(),
   744  			when:    when,
   745  		})
   746  	sort.Sort(sort.Reverse(byDeletionDate(targetDeletions)))
   747  	x.deletes.m[target] = targetDeletions
   748  	return nil
   749  }