github.com/rayrapetyan/go-ethereum@v1.8.21/cmd/swarm/swarm-smoke/feed_upload_and_sync.go (about) 1 package main 2 3 import ( 4 "bytes" 5 "context" 6 "crypto/md5" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "net/http" 11 "net/http/httptrace" 12 "os" 13 "os/exec" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/ethereum/go-ethereum/common/hexutil" 19 "github.com/ethereum/go-ethereum/crypto" 20 "github.com/ethereum/go-ethereum/log" 21 "github.com/ethereum/go-ethereum/metrics" 22 "github.com/ethereum/go-ethereum/swarm/api/client" 23 "github.com/ethereum/go-ethereum/swarm/spancontext" 24 "github.com/ethereum/go-ethereum/swarm/storage/feed" 25 "github.com/ethereum/go-ethereum/swarm/testutil" 26 colorable "github.com/mattn/go-colorable" 27 opentracing "github.com/opentracing/opentracing-go" 28 "github.com/pborman/uuid" 29 cli "gopkg.in/urfave/cli.v1" 30 ) 31 32 const ( 33 feedRandomDataLength = 8 34 ) 35 36 func cliFeedUploadAndSync(c *cli.Context) error { 37 metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) 38 log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) 39 40 errc := make(chan error) 41 go func() { 42 errc <- feedUploadAndSync(c) 43 }() 44 45 select { 46 case err := <-errc: 47 if err != nil { 48 metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) 49 } 50 return err 51 case <-time.After(time.Duration(timeout) * time.Second): 52 metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) 53 return fmt.Errorf("timeout after %v sec", timeout) 54 } 55 } 56 57 // TODO: retrieve with manifest + extract repeating code 58 func feedUploadAndSync(c *cli.Context) error { 59 defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) 60 61 generateEndpoints(scheme, cluster, appName, from, to) 62 63 log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") 64 65 // create a random private key to sign updates with and derive the address 66 pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") 67 if err != nil { 68 return err 69 } 70 defer pkFile.Close() 71 defer os.Remove(pkFile.Name()) 72 73 privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" 74 privKey, err := crypto.HexToECDSA(privkeyHex) 75 if err != nil { 76 return err 77 } 78 user := crypto.PubkeyToAddress(privKey.PublicKey) 79 userHex := hexutil.Encode(user.Bytes()) 80 81 // save the private key to a file 82 _, err = io.WriteString(pkFile, privkeyHex) 83 if err != nil { 84 return err 85 } 86 87 // keep hex strings for topic and subtopic 88 var topicHex string 89 var subTopicHex string 90 91 // and create combination hex topics for bzz-feed retrieval 92 // xor'ed with topic (zero-value topic if no topic) 93 var subTopicOnlyHex string 94 var mergedSubTopicHex string 95 96 // generate random topic and subtopic and put a hex on them 97 topicBytes, err := generateRandomData(feed.TopicLength) 98 topicHex = hexutil.Encode(topicBytes) 99 subTopicBytes, err := generateRandomData(8) 100 subTopicHex = hexutil.Encode(subTopicBytes) 101 if err != nil { 102 return err 103 } 104 mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) 105 if err != nil { 106 return err 107 } 108 mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) 109 subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) 110 if err != nil { 111 return err 112 } 113 subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) 114 115 // create feed manifest, topic only 116 var out bytes.Buffer 117 cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) 118 cmd.Stdout = &out 119 log.Debug("create feed manifest topic cmd", "cmd", cmd) 120 err = cmd.Run() 121 if err != nil { 122 return err 123 } 124 manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 125 if len(manifestWithTopic) != 64 { 126 return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) 127 } 128 log.Debug("create topic feed", "manifest", manifestWithTopic) 129 out.Reset() 130 131 // create feed manifest, subtopic only 132 cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) 133 cmd.Stdout = &out 134 log.Debug("create feed manifest subtopic cmd", "cmd", cmd) 135 err = cmd.Run() 136 if err != nil { 137 return err 138 } 139 manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 140 if len(manifestWithSubTopic) != 64 { 141 return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) 142 } 143 log.Debug("create subtopic feed", "manifest", manifestWithTopic) 144 out.Reset() 145 146 // create feed manifest, merged topic 147 cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) 148 cmd.Stdout = &out 149 log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) 150 err = cmd.Run() 151 if err != nil { 152 log.Error(err.Error()) 153 return err 154 } 155 manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) 156 if len(manifestWithMergedTopic) != 64 { 157 return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) 158 } 159 log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) 160 out.Reset() 161 162 // create test data 163 data, err := generateRandomData(feedRandomDataLength) 164 if err != nil { 165 return err 166 } 167 h := md5.New() 168 h.Write(data) 169 dataHash := h.Sum(nil) 170 dataHex := hexutil.Encode(data) 171 172 // update with topic 173 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) 174 cmd.Stdout = &out 175 log.Debug("update feed manifest topic cmd", "cmd", cmd) 176 err = cmd.Run() 177 if err != nil { 178 return err 179 } 180 log.Debug("feed update topic", "out", out) 181 out.Reset() 182 183 // update with subtopic 184 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) 185 cmd.Stdout = &out 186 log.Debug("update feed manifest subtopic cmd", "cmd", cmd) 187 err = cmd.Run() 188 if err != nil { 189 return err 190 } 191 log.Debug("feed update subtopic", "out", out) 192 out.Reset() 193 194 // update with merged topic 195 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) 196 cmd.Stdout = &out 197 log.Debug("update feed manifest merged topic cmd", "cmd", cmd) 198 err = cmd.Run() 199 if err != nil { 200 return err 201 } 202 log.Debug("feed update mergedtopic", "out", out) 203 out.Reset() 204 205 time.Sleep(3 * time.Second) 206 207 // retrieve the data 208 wg := sync.WaitGroup{} 209 for _, endpoint := range endpoints { 210 // raw retrieve, topic only 211 for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { 212 wg.Add(1) 213 ruid := uuid.New()[:8] 214 go func(hex string, endpoint string, ruid string) { 215 for { 216 err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) 217 if err != nil { 218 continue 219 } 220 221 wg.Done() 222 return 223 } 224 }(hex, endpoint, ruid) 225 226 } 227 } 228 wg.Wait() 229 log.Info("all endpoints synced random data successfully") 230 231 // upload test file 232 seed := int(time.Now().UnixNano() / 1e6) 233 log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) 234 235 randomBytes := testutil.RandomBytes(seed, filesize*1000) 236 237 hash, err := upload(&randomBytes, endpoints[0]) 238 if err != nil { 239 return err 240 } 241 hashBytes, err := hexutil.Decode("0x" + hash) 242 if err != nil { 243 return err 244 } 245 multihashHex := hexutil.Encode(hashBytes) 246 fileHash, err := digest(bytes.NewReader(randomBytes)) 247 if err != nil { 248 return err 249 } 250 251 log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) 252 253 // update file with topic 254 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) 255 cmd.Stdout = &out 256 err = cmd.Run() 257 if err != nil { 258 return err 259 } 260 log.Debug("feed update topic", "out", out) 261 out.Reset() 262 263 // update file with subtopic 264 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) 265 cmd.Stdout = &out 266 err = cmd.Run() 267 if err != nil { 268 return err 269 } 270 log.Debug("feed update subtopic", "out", out) 271 out.Reset() 272 273 // update file with merged topic 274 cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) 275 cmd.Stdout = &out 276 err = cmd.Run() 277 if err != nil { 278 return err 279 } 280 log.Debug("feed update mergedtopic", "out", out) 281 out.Reset() 282 283 time.Sleep(3 * time.Second) 284 285 for _, endpoint := range endpoints { 286 287 // manifest retrieve, topic only 288 for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { 289 wg.Add(1) 290 ruid := uuid.New()[:8] 291 go func(url string, endpoint string, ruid string) { 292 for { 293 err := fetch(url, endpoint, fileHash, ruid) 294 if err != nil { 295 continue 296 } 297 298 wg.Done() 299 return 300 } 301 }(url, endpoint, ruid) 302 } 303 304 } 305 wg.Wait() 306 log.Info("all endpoints synced random file successfully") 307 308 return nil 309 } 310 311 func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { 312 ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") 313 defer sp.Finish() 314 315 log.Trace("sleeping", "ruid", ruid) 316 time.Sleep(3 * time.Second) 317 318 log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) 319 320 var tn time.Time 321 reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user 322 req, _ := http.NewRequest("GET", reqUri, nil) 323 324 opentracing.GlobalTracer().Inject( 325 sp.Context(), 326 opentracing.HTTPHeaders, 327 opentracing.HTTPHeadersCarrier(req.Header)) 328 329 trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) 330 331 req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) 332 transport := http.DefaultTransport 333 334 //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 335 336 tn = time.Now() 337 res, err := transport.RoundTrip(req) 338 if err != nil { 339 log.Error(err.Error(), "ruid", ruid) 340 return err 341 } 342 343 log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) 344 345 if res.StatusCode != 200 { 346 return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) 347 } 348 349 defer res.Body.Close() 350 351 rdigest, err := digest(res.Body) 352 if err != nil { 353 log.Warn(err.Error(), "ruid", ruid) 354 return err 355 } 356 357 if !bytes.Equal(rdigest, original) { 358 err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) 359 log.Warn(err.Error(), "ruid", ruid) 360 return err 361 } 362 363 log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) 364 365 return nil 366 }