github.com/devfans/go-ethereum@v1.5.10-0.20170326212234-7419d0c38291/swarm/api/manifest.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package api
    18  
    19  import (
    20  	"bytes"
    21  	"encoding/json"
    22  	"fmt"
    23  	"sync"
    24  
    25  	"github.com/ethereum/go-ethereum/common"
    26  	"github.com/ethereum/go-ethereum/log"
    27  	"github.com/ethereum/go-ethereum/swarm/storage"
    28  )
    29  
    30  const (
    31  	manifestType = "application/bzz-manifest+json"
    32  )
    33  
    34  type manifestTrie struct {
    35  	dpa     *storage.DPA
    36  	entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
    37  	hash    storage.Key             // if hash != nil, it is stored
    38  }
    39  
    40  type manifestJSON struct {
    41  	Entries []*manifestTrieEntry `json:"entries"`
    42  }
    43  
    44  type manifestTrieEntry struct {
    45  	Path        string `json:"path"`
    46  	Hash        string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
    47  	ContentType string `json:"contentType"`
    48  	Status      int    `json:"status"`
    49  	subtrie     *manifestTrie
    50  }
    51  
    52  func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
    53  
    54  	log.Trace(fmt.Sprintf("manifest lookup key: '%v'.", hash.Log()))
    55  	// retrieve manifest via DPA
    56  	manifestReader := dpa.Retrieve(hash)
    57  	return readManifest(manifestReader, hash, dpa, quitC)
    58  }
    59  
    60  func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
    61  
    62  	// TODO check size for oversized manifests
    63  	size, err := manifestReader.Size(quitC)
    64  	if err != nil { // size == 0
    65  		// can't determine size means we don't have the root chunk
    66  		err = fmt.Errorf("Manifest not Found")
    67  		return
    68  	}
    69  	manifestData := make([]byte, size)
    70  	read, err := manifestReader.Read(manifestData)
    71  	if int64(read) < size {
    72  		log.Trace(fmt.Sprintf("Manifest %v not found.", hash.Log()))
    73  		if err == nil {
    74  			err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
    75  		}
    76  		return
    77  	}
    78  
    79  	log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
    80  	man := manifestJSON{}
    81  	err = json.Unmarshal(manifestData, &man)
    82  	if err != nil {
    83  		err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
    84  		log.Trace(fmt.Sprintf("%v", err))
    85  		return
    86  	}
    87  
    88  	log.Trace(fmt.Sprintf("Manifest %v has %d entries.", hash.Log(), len(man.Entries)))
    89  
    90  	trie = &manifestTrie{
    91  		dpa: dpa,
    92  	}
    93  	for _, entry := range man.Entries {
    94  		trie.addEntry(entry, quitC)
    95  	}
    96  	return
    97  }
    98  
    99  func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
   100  	self.hash = nil // trie modified, hash needs to be re-calculated on demand
   101  
   102  	if len(entry.Path) == 0 {
   103  		self.entries[256] = entry
   104  		return
   105  	}
   106  
   107  	b := byte(entry.Path[0])
   108  	if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) {
   109  		self.entries[b] = entry
   110  		return
   111  	}
   112  
   113  	oldentry := self.entries[b]
   114  	cpl := 0
   115  	for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
   116  		cpl++
   117  	}
   118  
   119  	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
   120  		if self.loadSubTrie(oldentry, quitC) != nil {
   121  			return
   122  		}
   123  		entry.Path = entry.Path[cpl:]
   124  		oldentry.subtrie.addEntry(entry, quitC)
   125  		oldentry.Hash = ""
   126  		return
   127  	}
   128  
   129  	commonPrefix := entry.Path[:cpl]
   130  
   131  	subtrie := &manifestTrie{
   132  		dpa: self.dpa,
   133  	}
   134  	entry.Path = entry.Path[cpl:]
   135  	oldentry.Path = oldentry.Path[cpl:]
   136  	subtrie.addEntry(entry, quitC)
   137  	subtrie.addEntry(oldentry, quitC)
   138  
   139  	self.entries[b] = &manifestTrieEntry{
   140  		Path:        commonPrefix,
   141  		Hash:        "",
   142  		ContentType: manifestType,
   143  		subtrie:     subtrie,
   144  	}
   145  }
   146  
   147  func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
   148  	for _, e := range self.entries {
   149  		if e != nil {
   150  			cnt++
   151  			entry = e
   152  		}
   153  	}
   154  	return
   155  }
   156  
   157  func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
   158  	self.hash = nil // trie modified, hash needs to be re-calculated on demand
   159  
   160  	if len(path) == 0 {
   161  		self.entries[256] = nil
   162  		return
   163  	}
   164  
   165  	b := byte(path[0])
   166  	entry := self.entries[b]
   167  	if entry == nil {
   168  		return
   169  	}
   170  	if entry.Path == path {
   171  		self.entries[b] = nil
   172  		return
   173  	}
   174  
   175  	epl := len(entry.Path)
   176  	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
   177  		if self.loadSubTrie(entry, quitC) != nil {
   178  			return
   179  		}
   180  		entry.subtrie.deleteEntry(path[epl:], quitC)
   181  		entry.Hash = ""
   182  		// remove subtree if it has less than 2 elements
   183  		cnt, lastentry := entry.subtrie.getCountLast()
   184  		if cnt < 2 {
   185  			if lastentry != nil {
   186  				lastentry.Path = entry.Path + lastentry.Path
   187  			}
   188  			self.entries[b] = lastentry
   189  		}
   190  	}
   191  }
   192  
   193  func (self *manifestTrie) recalcAndStore() error {
   194  	if self.hash != nil {
   195  		return nil
   196  	}
   197  
   198  	var buffer bytes.Buffer
   199  	buffer.WriteString(`{"entries":[`)
   200  
   201  	list := &manifestJSON{}
   202  	for _, entry := range self.entries {
   203  		if entry != nil {
   204  			if entry.Hash == "" { // TODO: paralellize
   205  				err := entry.subtrie.recalcAndStore()
   206  				if err != nil {
   207  					return err
   208  				}
   209  				entry.Hash = entry.subtrie.hash.String()
   210  			}
   211  			list.Entries = append(list.Entries, entry)
   212  		}
   213  	}
   214  
   215  	manifest, err := json.Marshal(list)
   216  	if err != nil {
   217  		return err
   218  	}
   219  
   220  	sr := bytes.NewReader(manifest)
   221  	wg := &sync.WaitGroup{}
   222  	key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil)
   223  	wg.Wait()
   224  	self.hash = key
   225  	return err2
   226  }
   227  
   228  func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
   229  	if entry.subtrie == nil {
   230  		hash := common.Hex2Bytes(entry.Hash)
   231  		entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
   232  		entry.Hash = "" // might not match, should be recalculated
   233  	}
   234  	return
   235  }
   236  
   237  func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
   238  	plen := len(prefix)
   239  	var start, stop int
   240  	if plen == 0 {
   241  		start = 0
   242  		stop = 256
   243  	} else {
   244  		start = int(prefix[0])
   245  		stop = start
   246  	}
   247  
   248  	for i := start; i <= stop; i++ {
   249  		select {
   250  		case <-quitC:
   251  			return fmt.Errorf("aborted")
   252  		default:
   253  		}
   254  		entry := self.entries[i]
   255  		if entry != nil {
   256  			epl := len(entry.Path)
   257  			if entry.ContentType == manifestType {
   258  				l := plen
   259  				if epl < l {
   260  					l = epl
   261  				}
   262  				if prefix[:l] == entry.Path[:l] {
   263  					err := self.loadSubTrie(entry, quitC)
   264  					if err != nil {
   265  						return err
   266  					}
   267  					err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
   268  					if err != nil {
   269  						return err
   270  					}
   271  				}
   272  			} else {
   273  				if (epl >= plen) && (prefix == entry.Path[:plen]) {
   274  					cb(entry, rp+entry.Path[plen:])
   275  				}
   276  			}
   277  		}
   278  	}
   279  	return nil
   280  }
   281  
   282  func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
   283  	return self.listWithPrefixInt(prefix, "", quitC, cb)
   284  }
   285  
   286  func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
   287  
   288  	log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))
   289  
   290  	if len(path) == 0 {
   291  		return self.entries[256], 0
   292  	}
   293  
   294  	b := byte(path[0])
   295  	entry = self.entries[b]
   296  	if entry == nil {
   297  		return self.entries[256], 0
   298  	}
   299  	epl := len(entry.Path)
   300  	log.Trace(fmt.Sprintf("path = %v  entry.Path = %v  epl = %v", path, entry.Path, epl))
   301  	if (len(path) >= epl) && (path[:epl] == entry.Path) {
   302  		log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
   303  		if entry.ContentType == manifestType {
   304  			err := self.loadSubTrie(entry, quitC)
   305  			if err != nil {
   306  				return nil, 0
   307  			}
   308  			entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
   309  			if entry != nil {
   310  				pos += epl
   311  			}
   312  		} else {
   313  			pos = epl
   314  		}
   315  	}
   316  	return
   317  }
   318  
   319  // file system manifest always contains regularized paths
   320  // no leading or trailing slashes, only single slashes inside
   321  func RegularSlashes(path string) (res string) {
   322  	for i := 0; i < len(path); i++ {
   323  		if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
   324  			res = res + path[i:i+1]
   325  		}
   326  	}
   327  	if (len(res) > 0) && (res[len(res)-1] == '/') {
   328  		res = res[:len(res)-1]
   329  	}
   330  	return
   331  }
   332  
   333  func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
   334  	path := RegularSlashes(spath)
   335  	var pos int
   336  	quitC := make(chan bool)
   337  	entry, pos = self.findPrefixOf(path, quitC)
   338  	return entry, path[:pos]
   339  }