github.com/codingfuture/orig-energi3@v0.8.4/cmd/swarm/swarm-smoke/feed_upload_and_sync.go (about) 1 // Copyright 2018 The Energi Core Authors 2 // Copyright 2018 The go-ethereum Authors 3 // This file is part of Energi Core. 4 // 5 // Energi Core is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // Energi Core is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU General Public License for more details. 14 // 15 // You should have received a copy of the GNU General Public License 16 // along with Energi Core. If not, see <http://www.gnu.org/licenses/>. 17 18 package main 19 20 import ( 21 "bytes" 22 "crypto/md5" 23 "fmt" 24 "io" 25 "io/ioutil" 26 "os" 27 "os/exec" 28 "strings" 29 "sync" 30 "time" 31 32 "github.com/ethereum/go-ethereum/common/hexutil" 33 "github.com/ethereum/go-ethereum/crypto" 34 "github.com/ethereum/go-ethereum/log" 35 "github.com/ethereum/go-ethereum/metrics" 36 "github.com/ethereum/go-ethereum/swarm/storage/feed" 37 "github.com/ethereum/go-ethereum/swarm/testutil" 38 "github.com/pborman/uuid" 39 cli "gopkg.in/urfave/cli.v1" 40 ) 41 42 const ( 43 feedRandomDataLength = 8 44 ) 45 46 func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error { 47 errc := make(chan error) 48 49 go func() { 50 errc <- feedUploadAndSync(ctx, tuid) 51 }() 52 53 select { 54 case err := <-errc: 55 if err != nil { 56 metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) 57 } 58 return err 59 case <-time.After(time.Duration(timeout) * time.Second): 60 metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) 61 62 return fmt.Errorf("timeout after %v sec", timeout) 63 } 64 } 65 66 func feedUploadAndSync(c *cli.Context, tuid string) error { 67 log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing") 68 69 // create a random private key to sign updates with and derive the address 70 pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") 71 if err != nil { 72 return err 73 } 74 defer pkFile.Close() 75 defer os.Remove(pkFile.Name()) 76 77 privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" 78 privKey, err := crypto.HexToECDSA(privkeyHex) 79 if err != nil { 80 return err 81 } 82 user := crypto.PubkeyToAddress(privKey.PublicKey) 83 userHex := hexutil.Encode(user.Bytes()) 84 85 // save the private key to a file 86 _, err = io.WriteString(pkFile, privkeyHex) 87 if err != nil { 88 return err 89 } 90 91 // keep hex strings for topic and subtopic 92 var topicHex string 93 var subTopicHex string 94 95 // and create combination hex topics for bzz-feed retrieval 96 // xor'ed with topic (zero-value topic if no topic) 97 var subTopicOnlyHex string 98 var mergedSubTopicHex string 99 100 // generate random topic and subtopic and put a hex on them 101 topicBytes, err := generateRandomData(feed.TopicLength) 102 topicHex = hexutil.Encode(topicBytes) 103 subTopicBytes, err := generateRandomData(8) 104 subTopicHex = hexutil.Encode(subTopicBytes) 105 if err != nil { 106 return err 107 } 108 mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) 109 if err != nil { 110 return err 111 } 112 mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) 113 subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) 114 if err != nil { 115 return err 116 } 117 subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) 118 119 // create feed manifest, topic only 120 var out bytes.Buffer 121 cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex) 122 cmd.Stdout = &out 123 log.Debug("create feed manifest topic cmd", "cmd", cmd) 124 err = cmd.Run() 125 if err != nil { 126 return err 127 } 128 manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 129 if len(manifestWithTopic) != 64 { 130 return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) 131 } 132 log.Debug("create topic feed", "manifest", manifestWithTopic) 133 out.Reset() 134 135 // create feed manifest, subtopic only 136 cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex) 137 cmd.Stdout = &out 138 log.Debug("create feed manifest subtopic cmd", "cmd", cmd) 139 err = cmd.Run() 140 if err != nil { 141 return err 142 } 143 manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 144 if len(manifestWithSubTopic) != 64 { 145 return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) 146 } 147 log.Debug("create subtopic feed", "manifest", manifestWithTopic) 148 out.Reset() 149 150 // create feed manifest, merged topic 151 cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) 152 cmd.Stdout = &out 153 log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) 154 err = cmd.Run() 155 if err != nil { 156 log.Error(err.Error()) 157 return err 158 } 159 manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 160 if len(manifestWithMergedTopic) != 64 { 161 return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) 162 } 163 log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) 164 out.Reset() 165 166 // create test data 167 data, err := generateRandomData(feedRandomDataLength) 168 if err != nil { 169 return err 170 } 171 h := md5.New() 172 h.Write(data) 173 dataHash := h.Sum(nil) 174 dataHex := hexutil.Encode(data) 175 176 // update with topic 177 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex) 178 cmd.Stdout = &out 179 log.Debug("update feed manifest topic cmd", "cmd", cmd) 180 err = cmd.Run() 181 if err != nil { 182 return err 183 } 184 log.Debug("feed update topic", "out", out) 185 out.Reset() 186 187 // update with subtopic 188 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex) 189 cmd.Stdout = &out 190 log.Debug("update feed manifest subtopic cmd", "cmd", cmd) 191 err = cmd.Run() 192 if err != nil { 193 return err 194 } 195 log.Debug("feed update subtopic", "out", out) 196 out.Reset() 197 198 // update with merged topic 199 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) 200 cmd.Stdout = &out 201 log.Debug("update feed manifest merged topic cmd", "cmd", cmd) 202 err = cmd.Run() 203 if err != nil { 204 return err 205 } 206 log.Debug("feed update mergedtopic", "out", out) 207 out.Reset() 208 209 time.Sleep(3 * time.Second) 210 211 // retrieve the data 212 wg := sync.WaitGroup{} 213 for _, host := range hosts { 214 // raw retrieve, topic only 215 for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { 216 wg.Add(1) 217 ruid := uuid.New()[:8] 218 go func(hex string, endpoint string, ruid string) { 219 for { 220 err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid) 221 if err != nil { 222 continue 223 } 224 225 wg.Done() 226 return 227 } 228 }(hex, httpEndpoint(host), ruid) 229 } 230 } 231 wg.Wait() 232 log.Info("all endpoints synced random data successfully") 233 234 // upload test file 235 log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed) 236 237 randomBytes := testutil.RandomBytes(seed, filesize*1000) 238 239 hash, err := upload(randomBytes, httpEndpoint(hosts[0])) 240 if err != nil { 241 return err 242 } 243 hashBytes, err := hexutil.Decode("0x" + hash) 244 if err != nil { 245 return err 246 } 247 multihashHex := hexutil.Encode(hashBytes) 248 fileHash := h.Sum(nil) 249 250 log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) 251 252 // update file with topic 253 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex) 254 cmd.Stdout = &out 255 err = cmd.Run() 256 if err != nil { 257 return err 258 } 259 log.Debug("feed update topic", "out", out) 260 out.Reset() 261 262 // update file with subtopic 263 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex) 264 cmd.Stdout = &out 265 err = cmd.Run() 266 if err != nil { 267 return err 268 } 269 log.Debug("feed update subtopic", "out", out) 270 out.Reset() 271 272 // update file with merged topic 273 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) 274 cmd.Stdout = &out 275 err = cmd.Run() 276 if err != nil { 277 return err 278 } 279 log.Debug("feed update mergedtopic", "out", out) 280 out.Reset() 281 282 time.Sleep(3 * time.Second) 283 284 for _, host := range hosts { 285 286 // manifest retrieve, topic only 287 for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { 288 wg.Add(1) 289 ruid := uuid.New()[:8] 290 go func(url string, endpoint string, ruid string) { 291 for { 292 err := fetch(url, endpoint, fileHash, ruid, "") 293 if err != nil { 294 continue 295 } 296 297 wg.Done() 298 return 299 } 300 }(url, httpEndpoint(host), ruid) 301 } 302 303 } 304 wg.Wait() 305 log.Info("all endpoints synced random file successfully") 306 307 return nil 308 }