github.com/devfans/go-ethereum@v1.5.10-0.20170326212234-7419d0c38291/swarm/api/manifest.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package api 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "sync" 24 25 "github.com/ethereum/go-ethereum/common" 26 "github.com/ethereum/go-ethereum/log" 27 "github.com/ethereum/go-ethereum/swarm/storage" 28 ) 29 30 const ( 31 manifestType = "application/bzz-manifest+json" 32 ) 33 34 type manifestTrie struct { 35 dpa *storage.DPA 36 entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry 37 hash storage.Key // if hash != nil, it is stored 38 } 39 40 type manifestJSON struct { 41 Entries []*manifestTrieEntry `json:"entries"` 42 } 43 44 type manifestTrieEntry struct { 45 Path string `json:"path"` 46 Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated 47 ContentType string `json:"contentType"` 48 Status int `json:"status"` 49 subtrie *manifestTrie 50 } 51 52 func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand 53 54 log.Trace(fmt.Sprintf("manifest lookup key: '%v'.", hash.Log())) 55 // retrieve manifest via DPA 56 manifestReader := dpa.Retrieve(hash) 57 return readManifest(manifestReader, hash, dpa, quitC) 58 } 59 60 func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand 61 62 // TODO check size for oversized manifests 63 size, err := manifestReader.Size(quitC) 64 if err != nil { // size == 0 65 // can't determine size means we don't have the root chunk 66 err = fmt.Errorf("Manifest not Found") 67 return 68 } 69 manifestData := make([]byte, size) 70 read, err := manifestReader.Read(manifestData) 71 if int64(read) < size { 72 log.Trace(fmt.Sprintf("Manifest %v not found.", hash.Log())) 73 if err == nil { 74 err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) 75 } 76 return 77 } 78 79 log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log())) 80 man := manifestJSON{} 81 err = json.Unmarshal(manifestData, &man) 82 if err != nil { 83 err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) 84 log.Trace(fmt.Sprintf("%v", err)) 85 return 86 } 87 88 log.Trace(fmt.Sprintf("Manifest %v has %d entries.", hash.Log(), len(man.Entries))) 89 90 trie = &manifestTrie{ 91 dpa: dpa, 92 } 93 for _, entry := range man.Entries { 94 trie.addEntry(entry, quitC) 95 } 96 return 97 } 98 99 func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { 100 self.hash = nil // trie modified, hash needs to be re-calculated on demand 101 102 if len(entry.Path) == 0 { 103 self.entries[256] = entry 104 return 105 } 106 107 b := byte(entry.Path[0]) 108 if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) { 109 self.entries[b] = entry 110 return 111 } 112 113 oldentry := self.entries[b] 114 cpl := 0 115 for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) { 116 cpl++ 117 } 118 119 if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { 120 if self.loadSubTrie(oldentry, quitC) != nil { 121 return 122 } 123 entry.Path = entry.Path[cpl:] 124 oldentry.subtrie.addEntry(entry, quitC) 125 oldentry.Hash = "" 126 return 127 } 128 129 commonPrefix := entry.Path[:cpl] 130 131 subtrie := &manifestTrie{ 132 dpa: self.dpa, 133 } 134 entry.Path = entry.Path[cpl:] 135 oldentry.Path = oldentry.Path[cpl:] 136 subtrie.addEntry(entry, quitC) 137 subtrie.addEntry(oldentry, quitC) 138 139 self.entries[b] = &manifestTrieEntry{ 140 Path: commonPrefix, 141 Hash: "", 142 ContentType: manifestType, 143 subtrie: subtrie, 144 } 145 } 146 147 func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { 148 for _, e := range self.entries { 149 if e != nil { 150 cnt++ 151 entry = e 152 } 153 } 154 return 155 } 156 157 func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { 158 self.hash = nil // trie modified, hash needs to be re-calculated on demand 159 160 if len(path) == 0 { 161 self.entries[256] = nil 162 return 163 } 164 165 b := byte(path[0]) 166 entry := self.entries[b] 167 if entry == nil { 168 return 169 } 170 if entry.Path == path { 171 self.entries[b] = nil 172 return 173 } 174 175 epl := len(entry.Path) 176 if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { 177 if self.loadSubTrie(entry, quitC) != nil { 178 return 179 } 180 entry.subtrie.deleteEntry(path[epl:], quitC) 181 entry.Hash = "" 182 // remove subtree if it has less than 2 elements 183 cnt, lastentry := entry.subtrie.getCountLast() 184 if cnt < 2 { 185 if lastentry != nil { 186 lastentry.Path = entry.Path + lastentry.Path 187 } 188 self.entries[b] = lastentry 189 } 190 } 191 } 192 193 func (self *manifestTrie) recalcAndStore() error { 194 if self.hash != nil { 195 return nil 196 } 197 198 var buffer bytes.Buffer 199 buffer.WriteString(`{"entries":[`) 200 201 list := &manifestJSON{} 202 for _, entry := range self.entries { 203 if entry != nil { 204 if entry.Hash == "" { // TODO: paralellize 205 err := entry.subtrie.recalcAndStore() 206 if err != nil { 207 return err 208 } 209 entry.Hash = entry.subtrie.hash.String() 210 } 211 list.Entries = append(list.Entries, entry) 212 } 213 } 214 215 manifest, err := json.Marshal(list) 216 if err != nil { 217 return err 218 } 219 220 sr := bytes.NewReader(manifest) 221 wg := &sync.WaitGroup{} 222 key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil) 223 wg.Wait() 224 self.hash = key 225 return err2 226 } 227 228 func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { 229 if entry.subtrie == nil { 230 hash := common.Hex2Bytes(entry.Hash) 231 entry.subtrie, err = loadManifest(self.dpa, hash, quitC) 232 entry.Hash = "" // might not match, should be recalculated 233 } 234 return 235 } 236 237 func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error { 238 plen := len(prefix) 239 var start, stop int 240 if plen == 0 { 241 start = 0 242 stop = 256 243 } else { 244 start = int(prefix[0]) 245 stop = start 246 } 247 248 for i := start; i <= stop; i++ { 249 select { 250 case <-quitC: 251 return fmt.Errorf("aborted") 252 default: 253 } 254 entry := self.entries[i] 255 if entry != nil { 256 epl := len(entry.Path) 257 if entry.ContentType == manifestType { 258 l := plen 259 if epl < l { 260 l = epl 261 } 262 if prefix[:l] == entry.Path[:l] { 263 err := self.loadSubTrie(entry, quitC) 264 if err != nil { 265 return err 266 } 267 err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb) 268 if err != nil { 269 return err 270 } 271 } 272 } else { 273 if (epl >= plen) && (prefix == entry.Path[:plen]) { 274 cb(entry, rp+entry.Path[plen:]) 275 } 276 } 277 } 278 } 279 return nil 280 } 281 282 func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) { 283 return self.listWithPrefixInt(prefix, "", quitC, cb) 284 } 285 286 func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) { 287 288 log.Trace(fmt.Sprintf("findPrefixOf(%s)", path)) 289 290 if len(path) == 0 { 291 return self.entries[256], 0 292 } 293 294 b := byte(path[0]) 295 entry = self.entries[b] 296 if entry == nil { 297 return self.entries[256], 0 298 } 299 epl := len(entry.Path) 300 log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl)) 301 if (len(path) >= epl) && (path[:epl] == entry.Path) { 302 log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType)) 303 if entry.ContentType == manifestType { 304 err := self.loadSubTrie(entry, quitC) 305 if err != nil { 306 return nil, 0 307 } 308 entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) 309 if entry != nil { 310 pos += epl 311 } 312 } else { 313 pos = epl 314 } 315 } 316 return 317 } 318 319 // file system manifest always contains regularized paths 320 // no leading or trailing slashes, only single slashes inside 321 func RegularSlashes(path string) (res string) { 322 for i := 0; i < len(path); i++ { 323 if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) { 324 res = res + path[i:i+1] 325 } 326 } 327 if (len(res) > 0) && (res[len(res)-1] == '/') { 328 res = res[:len(res)-1] 329 } 330 return 331 } 332 333 func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) { 334 path := RegularSlashes(spath) 335 var pos int 336 quitC := make(chan bool) 337 entry, pos = self.findPrefixOf(path, quitC) 338 return entry, path[:pos] 339 }