github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/cmd/swarm/swarm-smoke/feed_upload_and_sync.go (about) 1 package main 2 3 import ( 4 "bytes" 5 "crypto/md5" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "net/http" 10 "os" 11 "os/exec" 12 "strings" 13 "sync" 14 "time" 15 16 "github.com/ethereum/go-ethereum/common/hexutil" 17 "github.com/ethereum/go-ethereum/crypto" 18 "github.com/ethereum/go-ethereum/log" 19 "github.com/ethereum/go-ethereum/swarm/multihash" 20 "github.com/ethereum/go-ethereum/swarm/storage/feed" 21 colorable "github.com/mattn/go-colorable" 22 "github.com/pborman/uuid" 23 cli "gopkg.in/urfave/cli.v1" 24 ) 25 26 const ( 27 feedRandomDataLength = 8 28 ) 29 30 // TODO: retrieve with manifest + extract repeating code 31 func cliFeedUploadAndSync(c *cli.Context) error { 32 33 log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) 34 35 defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) 36 37 generateEndpoints(scheme, cluster, from, to) 38 39 log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing") 40 41 // create a random private key to sign updates with and derive the address 42 pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") 43 if err != nil { 44 return err 45 } 46 defer pkFile.Close() 47 defer os.Remove(pkFile.Name()) 48 49 privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" 50 privKey, err := crypto.HexToECDSA(privkeyHex) 51 if err != nil { 52 return err 53 } 54 user := crypto.PubkeyToAddress(privKey.PublicKey) 55 userHex := hexutil.Encode(user.Bytes()) 56 57 // save the private key to a file 58 _, err = io.WriteString(pkFile, privkeyHex) 59 if err != nil { 60 return err 61 } 62 63 // keep hex strings for topic and subtopic 64 var topicHex string 65 var subTopicHex string 66 67 // and create combination hex topics for bzz-feed retrieval 68 // xor'ed with topic (zero-value topic if no topic) 69 var subTopicOnlyHex string 70 var mergedSubTopicHex string 71 72 // generate random topic and subtopic and put a hex on them 73 topicBytes, err := generateRandomData(feed.TopicLength) 74 topicHex = hexutil.Encode(topicBytes) 75 subTopicBytes, err := generateRandomData(8) 76 subTopicHex = hexutil.Encode(subTopicBytes) 77 if err != nil { 78 return err 79 } 80 mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) 81 if err != nil { 82 return err 83 } 84 mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) 85 subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) 86 if err != nil { 87 return err 88 } 89 subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) 90 91 // create feed manifest, topic only 92 var out bytes.Buffer 93 cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) 94 cmd.Stdout = &out 95 log.Debug("create feed manifest topic cmd", "cmd", cmd) 96 err = cmd.Run() 97 if err != nil { 98 return err 99 } 100 manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 101 if len(manifestWithTopic) != 64 { 102 return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) 103 } 104 log.Debug("create topic feed", "manifest", manifestWithTopic) 105 out.Reset() 106 107 // create feed manifest, subtopic only 108 cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) 109 cmd.Stdout = &out 110 log.Debug("create feed manifest subtopic cmd", "cmd", cmd) 111 err = cmd.Run() 112 if err != nil { 113 return err 114 } 115 manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 116 if len(manifestWithSubTopic) != 64 { 117 return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) 118 } 119 log.Debug("create subtopic feed", "manifest", manifestWithTopic) 120 out.Reset() 121 122 // create feed manifest, merged topic 123 cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) 124 cmd.Stdout = &out 125 log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) 126 err = cmd.Run() 127 if err != nil { 128 log.Error(err.Error()) 129 return err 130 } 131 manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 132 if len(manifestWithMergedTopic) != 64 { 133 return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) 134 } 135 log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) 136 out.Reset() 137 138 // create test data 139 data, err := generateRandomData(feedRandomDataLength) 140 if err != nil { 141 return err 142 } 143 h := md5.New() 144 h.Write(data) 145 dataHash := h.Sum(nil) 146 dataHex := hexutil.Encode(data) 147 148 // update with topic 149 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) 150 cmd.Stdout = &out 151 log.Debug("update feed manifest topic cmd", "cmd", cmd) 152 err = cmd.Run() 153 if err != nil { 154 return err 155 } 156 log.Debug("feed update topic", "out", out) 157 out.Reset() 158 159 // update with subtopic 160 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) 161 cmd.Stdout = &out 162 log.Debug("update feed manifest subtopic cmd", "cmd", cmd) 163 err = cmd.Run() 164 if err != nil { 165 return err 166 } 167 log.Debug("feed update subtopic", "out", out) 168 out.Reset() 169 170 // update with merged topic 171 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) 172 cmd.Stdout = &out 173 log.Debug("update feed manifest merged topic cmd", "cmd", cmd) 174 err = cmd.Run() 175 if err != nil { 176 return err 177 } 178 log.Debug("feed update mergedtopic", "out", out) 179 out.Reset() 180 181 time.Sleep(3 * time.Second) 182 183 // retrieve the data 184 wg := sync.WaitGroup{} 185 for _, endpoint := range endpoints { 186 // raw retrieve, topic only 187 for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { 188 wg.Add(1) 189 ruid := uuid.New()[:8] 190 go func(hex string, endpoint string, ruid string) { 191 for { 192 err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) 193 if err != nil { 194 continue 195 } 196 197 wg.Done() 198 return 199 } 200 }(hex, endpoint, ruid) 201 202 } 203 } 204 wg.Wait() 205 log.Info("all endpoints synced random data successfully") 206 207 // upload test file 208 log.Info("uploading to " + endpoints[0] + " and syncing") 209 210 f, cleanup := generateRandomFile(filesize * 1000) 211 defer cleanup() 212 213 hash, err := upload(f, endpoints[0]) 214 if err != nil { 215 return err 216 } 217 hashBytes, err := hexutil.Decode("0x" + hash) 218 if err != nil { 219 return err 220 } 221 multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes)) 222 223 fileHash, err := digest(f) 224 if err != nil { 225 return err 226 } 227 228 log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) 229 230 // update file with topic 231 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) 232 cmd.Stdout = &out 233 err = cmd.Run() 234 if err != nil { 235 return err 236 } 237 log.Debug("feed update topic", "out", out) 238 out.Reset() 239 240 // update file with subtopic 241 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) 242 cmd.Stdout = &out 243 err = cmd.Run() 244 if err != nil { 245 return err 246 } 247 log.Debug("feed update subtopic", "out", out) 248 out.Reset() 249 250 // update file with merged topic 251 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) 252 cmd.Stdout = &out 253 err = cmd.Run() 254 if err != nil { 255 return err 256 } 257 log.Debug("feed update mergedtopic", "out", out) 258 out.Reset() 259 260 time.Sleep(3 * time.Second) 261 262 for _, endpoint := range endpoints { 263 264 // manifest retrieve, topic only 265 for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { 266 wg.Add(1) 267 ruid := uuid.New()[:8] 268 go func(url string, endpoint string, ruid string) { 269 for { 270 err := fetch(url, endpoint, fileHash, ruid) 271 if err != nil { 272 continue 273 } 274 275 wg.Done() 276 return 277 } 278 }(url, endpoint, ruid) 279 } 280 281 } 282 wg.Wait() 283 log.Info("all endpoints synced random file successfully") 284 285 return nil 286 } 287 288 func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { 289 log.Trace("sleeping", "ruid", ruid) 290 time.Sleep(3 * time.Second) 291 292 log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) 293 res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) 294 if err != nil { 295 return err 296 } 297 log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) 298 299 if res.StatusCode != 200 { 300 return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) 301 } 302 303 defer res.Body.Close() 304 305 rdigest, err := digest(res.Body) 306 if err != nil { 307 log.Warn(err.Error(), "ruid", ruid) 308 return err 309 } 310 311 if !bytes.Equal(rdigest, original) { 312 err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) 313 log.Warn(err.Error(), "ruid", ruid) 314 return err 315 } 316 317 log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) 318 319 return nil 320 }