github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/api/filesystem.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package api 18 19 import ( 20 "bufio" 21 "context" 22 "fmt" 23 "io" 24 "os" 25 "path" 26 "path/filepath" 27 "sync" 28 29 "github.com/ethereum/go-ethereum/common" 30 "github.com/ethereum/go-ethereum/swarm/log" 31 "github.com/ethereum/go-ethereum/swarm/storage" 32 ) 33 34 const maxParallelFiles = 5 35 36 type FileSystem struct { 37 api *API 38 } 39 40 func NewFileSystem(api *API) *FileSystem { 41 return &FileSystem{api} 42 } 43 44 // Upload replicates a local directory as a manifest file and uploads it 45 // using FileStore store 46 // This function waits the chunks to be stored. 47 // TODO: localpath should point to a manifest 48 // 49 // DEPRECATED: Use the HTTP API instead 50 func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error) { 51 var list []*manifestTrieEntry 52 localpath, err := filepath.Abs(filepath.Clean(lpath)) 53 if err != nil { 54 return "", err 55 } 56 57 f, err := os.Open(localpath) 58 if err != nil { 59 return "", err 60 } 61 stat, err := f.Stat() 62 if err != nil { 63 return "", err 64 } 65 66 var start int 67 if stat.IsDir() { 68 start = len(localpath) 69 log.Debug(fmt.Sprintf("uploading '%s'", localpath)) 70 err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error { 71 if (err == nil) && !info.IsDir() { 72 if len(path) <= start { 73 return fmt.Errorf("Path is too short") 74 } 75 if path[:start] != localpath { 76 return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath) 77 } 78 entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil) 79 list = append(list, entry) 80 } 81 return err 82 }) 83 if err != nil { 84 return "", err 85 } 86 } else { 87 dir := filepath.Dir(localpath) 88 start = len(dir) 89 if len(localpath) <= start { 90 return "", fmt.Errorf("Path is too short") 91 } 92 if localpath[:start] != dir { 93 return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir) 94 } 95 entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil) 96 list = append(list, entry) 97 } 98 99 errors := make([]error, len(list)) 100 sem := make(chan bool, maxParallelFiles) 101 defer close(sem) 102 103 for i, entry := range list { 104 sem <- true 105 go func(i int, entry *manifestTrieEntry) { 106 defer func() { <-sem }() 107 108 f, err := os.Open(entry.Path) 109 if err != nil { 110 errors[i] = err 111 return 112 } 113 defer f.Close() 114 115 stat, err := f.Stat() 116 if err != nil { 117 errors[i] = err 118 return 119 } 120 121 var hash storage.Address 122 var wait func(context.Context) error 123 ctx := context.TODO() 124 hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt) 125 if hash != nil { 126 list[i].Hash = hash.Hex() 127 } 128 if err := wait(ctx); err != nil { 129 errors[i] = err 130 return 131 } 132 133 list[i].ContentType, err = DetectContentType(f.Name(), f) 134 if err != nil { 135 errors[i] = err 136 return 137 } 138 139 }(i, entry) 140 } 141 for i := 0; i < cap(sem); i++ { 142 sem <- true 143 } 144 145 trie := &manifestTrie{ 146 fileStore: fs.api.fileStore, 147 } 148 quitC := make(chan bool) 149 for i, entry := range list { 150 if errors[i] != nil { 151 return "", errors[i] 152 } 153 entry.Path = RegularSlashes(entry.Path[start:]) 154 if entry.Path == index { 155 ientry := newManifestTrieEntry(&ManifestEntry{ 156 ContentType: entry.ContentType, 157 }, nil) 158 ientry.Hash = entry.Hash 159 trie.addEntry(ientry, quitC) 160 } 161 trie.addEntry(entry, quitC) 162 } 163 164 err2 := trie.recalcAndStore() 165 var hs string 166 if err2 == nil { 167 hs = trie.ref.Hex() 168 } 169 return hs, err2 170 } 171 172 // Download replicates the manifest basePath structure on the local filesystem 173 // under localpath 174 // 175 // DEPRECATED: Use the HTTP API instead 176 func (fs *FileSystem) Download(bzzpath, localpath string) error { 177 lpath, err := filepath.Abs(filepath.Clean(localpath)) 178 if err != nil { 179 return err 180 } 181 err = os.MkdirAll(lpath, os.ModePerm) 182 if err != nil { 183 return err 184 } 185 186 //resolving host and port 187 uri, err := Parse(path.Join("bzz:/", bzzpath)) 188 if err != nil { 189 return err 190 } 191 addr, err := fs.api.Resolve(context.TODO(), uri.Addr) 192 if err != nil { 193 return err 194 } 195 path := uri.Path 196 197 if len(path) > 0 { 198 path += "/" 199 } 200 201 quitC := make(chan bool) 202 trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt) 203 if err != nil { 204 log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err)) 205 return err 206 } 207 208 type downloadListEntry struct { 209 addr storage.Address 210 path string 211 } 212 213 var list []*downloadListEntry 214 var mde error 215 216 prevPath := lpath 217 err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) { 218 log.Trace(fmt.Sprintf("fs.Download: %#v", entry)) 219 220 addr = common.Hex2Bytes(entry.Hash) 221 path := lpath + "/" + suffix 222 dir := filepath.Dir(path) 223 if dir != prevPath { 224 mde = os.MkdirAll(dir, os.ModePerm) 225 prevPath = dir 226 } 227 if (mde == nil) && (path != dir+"/") { 228 list = append(list, &downloadListEntry{addr: addr, path: path}) 229 } 230 }) 231 if err != nil { 232 return err 233 } 234 235 wg := sync.WaitGroup{} 236 errC := make(chan error) 237 done := make(chan bool, maxParallelFiles) 238 for i, entry := range list { 239 select { 240 case done <- true: 241 wg.Add(1) 242 case <-quitC: 243 return fmt.Errorf("aborted") 244 } 245 go func(i int, entry *downloadListEntry) { 246 defer wg.Done() 247 err := retrieveToFile(quitC, fs.api.fileStore, entry.addr, entry.path) 248 if err != nil { 249 select { 250 case errC <- err: 251 case <-quitC: 252 } 253 return 254 } 255 <-done 256 }(i, entry) 257 } 258 go func() { 259 wg.Wait() 260 close(errC) 261 }() 262 select { 263 case err = <-errC: 264 return err 265 case <-quitC: 266 return fmt.Errorf("aborted") 267 } 268 } 269 270 func retrieveToFile(quitC chan bool, fileStore *storage.FileStore, addr storage.Address, path string) error { 271 f, err := os.Create(path) // TODO: basePath separators 272 if err != nil { 273 return err 274 } 275 reader, _ := fileStore.Retrieve(context.TODO(), addr) 276 writer := bufio.NewWriter(f) 277 size, err := reader.Size(context.TODO(), quitC) 278 if err != nil { 279 return err 280 } 281 if _, err = io.CopyN(writer, reader, size); err != nil { 282 return err 283 } 284 if err := writer.Flush(); err != nil { 285 return err 286 } 287 return f.Close() 288 }