github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/network/stream/intervals_test.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package stream 18 19 import ( 20 "context" 21 crand "crypto/rand" 22 "encoding/binary" 23 "fmt" 24 "io" 25 "os" 26 "sync" 27 "testing" 28 "time" 29 30 "github.com/ethereum/go-ethereum/log" 31 "github.com/ethereum/go-ethereum/node" 32 "github.com/ethereum/go-ethereum/p2p" 33 "github.com/ethereum/go-ethereum/p2p/enode" 34 "github.com/ethereum/go-ethereum/p2p/simulations/adapters" 35 "github.com/ethereum/go-ethereum/swarm/network" 36 "github.com/ethereum/go-ethereum/swarm/network/simulation" 37 "github.com/ethereum/go-ethereum/swarm/state" 38 "github.com/ethereum/go-ethereum/swarm/storage" 39 ) 40 41 func TestIntervalsLive(t *testing.T) { 42 testIntervals(t, true, nil, false) 43 testIntervals(t, true, nil, true) 44 } 45 46 func TestIntervalsHistory(t *testing.T) { 47 testIntervals(t, false, NewRange(9, 26), false) 48 testIntervals(t, false, NewRange(9, 26), true) 49 } 50 51 func TestIntervalsLiveAndHistory(t *testing.T) { 52 testIntervals(t, true, NewRange(9, 26), false) 53 testIntervals(t, true, NewRange(9, 26), true) 54 } 55 56 func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { 57 nodes := 2 58 chunkCount := dataChunkCount 59 externalStreamName := "externalStream" 60 externalStreamSessionAt := uint64(50) 61 externalStreamMaxKeys := uint64(100) 62 63 sim := simulation.New(map[string]simulation.ServiceFunc{ 64 "intervalsStreamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { 65 n := ctx.Config.Node() 66 addr := network.NewAddr(n) 67 store, datadir, err := createTestLocalStorageForID(n.ID(), addr) 68 if err != nil { 69 return nil, nil, err 70 } 71 bucket.Store(bucketKeyStore, store) 72 cleanup = func() { 73 store.Close() 74 os.RemoveAll(datadir) 75 } 76 localStore := store.(*storage.LocalStore) 77 netStore, err := storage.NewNetStore(localStore, nil) 78 if err != nil { 79 return nil, nil, err 80 } 81 kad := network.NewKademlia(addr.Over(), network.NewKadParams()) 82 delivery := NewDelivery(kad, netStore) 83 netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New 84 85 r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ 86 SkipCheck: skipCheck, 87 }) 88 bucket.Store(bucketKeyRegistry, r) 89 90 r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) { 91 return newTestExternalClient(netStore), nil 92 }) 93 r.RegisterServerFunc(externalStreamName, func(p *Peer, t string, live bool) (Server, error) { 94 return newTestExternalServer(t, externalStreamSessionAt, externalStreamMaxKeys, nil), nil 95 }) 96 97 fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams()) 98 bucket.Store(bucketKeyFileStore, fileStore) 99 100 return r, cleanup, nil 101 102 }, 103 }) 104 defer sim.Close() 105 106 log.Info("Adding nodes to simulation") 107 _, err := sim.AddNodesAndConnectChain(nodes) 108 if err != nil { 109 t.Fatal(err) 110 } 111 112 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) 113 defer cancel() 114 115 if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { 116 t.Fatal(err) 117 } 118 119 result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { 120 nodeIDs := sim.UpNodeIDs() 121 storer := nodeIDs[0] 122 checker := nodeIDs[1] 123 124 item, ok := sim.NodeItem(storer, bucketKeyFileStore) 125 if !ok { 126 return fmt.Errorf("No filestore") 127 } 128 fileStore := item.(*storage.FileStore) 129 130 size := chunkCount * chunkSize 131 _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) 132 if err != nil { 133 log.Error("Store error: %v", "err", err) 134 t.Fatal(err) 135 } 136 err = wait(ctx) 137 if err != nil { 138 log.Error("Wait error: %v", "err", err) 139 t.Fatal(err) 140 } 141 142 item, ok = sim.NodeItem(checker, bucketKeyRegistry) 143 if !ok { 144 return fmt.Errorf("No registry") 145 } 146 registry := item.(*Registry) 147 148 liveErrC := make(chan error) 149 historyErrC := make(chan error) 150 151 log.Debug("Watching for disconnections") 152 disconnections := sim.PeerEvents( 153 context.Background(), 154 sim.NodeIDs(), 155 simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), 156 ) 157 158 err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) 159 if err != nil { 160 return err 161 } 162 163 go func() { 164 for d := range disconnections { 165 if d.Error != nil { 166 log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) 167 t.Fatal(d.Error) 168 } 169 } 170 }() 171 172 go func() { 173 if !live { 174 close(liveErrC) 175 return 176 } 177 178 var err error 179 defer func() { 180 liveErrC <- err 181 }() 182 183 // live stream 184 var liveHashesChan chan []byte 185 liveHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", true)) 186 if err != nil { 187 log.Error("get hashes", "err", err) 188 return 189 } 190 i := externalStreamSessionAt 191 192 // we have subscribed, enable notifications 193 err = enableNotifications(registry, storer, NewStream(externalStreamName, "", true)) 194 if err != nil { 195 return 196 } 197 198 for { 199 select { 200 case hash := <-liveHashesChan: 201 h := binary.BigEndian.Uint64(hash) 202 if h != i { 203 err = fmt.Errorf("expected live hash %d, got %d", i, h) 204 return 205 } 206 i++ 207 if i > externalStreamMaxKeys { 208 return 209 } 210 case <-ctx.Done(): 211 return 212 } 213 } 214 }() 215 216 go func() { 217 if live && history == nil { 218 close(historyErrC) 219 return 220 } 221 222 var err error 223 defer func() { 224 historyErrC <- err 225 }() 226 227 // history stream 228 var historyHashesChan chan []byte 229 historyHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", false)) 230 if err != nil { 231 log.Error("get hashes", "err", err) 232 return 233 } 234 235 var i uint64 236 historyTo := externalStreamMaxKeys 237 if history != nil { 238 i = history.From 239 if history.To != 0 { 240 historyTo = history.To 241 } 242 } 243 244 // we have subscribed, enable notifications 245 err = enableNotifications(registry, storer, NewStream(externalStreamName, "", false)) 246 if err != nil { 247 return 248 } 249 250 for { 251 select { 252 case hash := <-historyHashesChan: 253 h := binary.BigEndian.Uint64(hash) 254 if h != i { 255 err = fmt.Errorf("expected history hash %d, got %d", i, h) 256 return 257 } 258 i++ 259 if i > historyTo { 260 return 261 } 262 case <-ctx.Done(): 263 return 264 } 265 } 266 }() 267 268 if err := <-liveErrC; err != nil { 269 return err 270 } 271 if err := <-historyErrC; err != nil { 272 return err 273 } 274 275 return nil 276 }) 277 278 if result.Error != nil { 279 t.Fatal(result.Error) 280 } 281 } 282 283 func getHashes(ctx context.Context, r *Registry, peerID enode.ID, s Stream) (chan []byte, error) { 284 peer := r.getPeer(peerID) 285 286 client, err := peer.getClient(ctx, s) 287 if err != nil { 288 return nil, err 289 } 290 291 c := client.Client.(*testExternalClient) 292 293 return c.hashes, nil 294 } 295 296 func enableNotifications(r *Registry, peerID enode.ID, s Stream) error { 297 peer := r.getPeer(peerID) 298 299 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 300 defer cancel() 301 302 client, err := peer.getClient(ctx, s) 303 if err != nil { 304 return err 305 } 306 307 close(client.Client.(*testExternalClient).enableNotificationsC) 308 309 return nil 310 } 311 312 type testExternalClient struct { 313 hashes chan []byte 314 store storage.SyncChunkStore 315 enableNotificationsC chan struct{} 316 } 317 318 func newTestExternalClient(store storage.SyncChunkStore) *testExternalClient { 319 return &testExternalClient{ 320 hashes: make(chan []byte), 321 store: store, 322 enableNotificationsC: make(chan struct{}), 323 } 324 } 325 326 func (c *testExternalClient) NeedData(ctx context.Context, hash []byte) func(context.Context) error { 327 wait := c.store.FetchFunc(ctx, storage.Address(hash)) 328 if wait == nil { 329 return nil 330 } 331 select { 332 case c.hashes <- hash: 333 case <-ctx.Done(): 334 log.Warn("testExternalClient NeedData context", "err", ctx.Err()) 335 return func(_ context.Context) error { 336 return ctx.Err() 337 } 338 } 339 return wait 340 } 341 342 func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) { 343 return nil 344 } 345 346 func (c *testExternalClient) Close() {} 347 348 type testExternalServer struct { 349 t string 350 keyFunc func(key []byte, index uint64) 351 sessionAt uint64 352 maxKeys uint64 353 } 354 355 func newTestExternalServer(t string, sessionAt, maxKeys uint64, keyFunc func(key []byte, index uint64)) *testExternalServer { 356 if keyFunc == nil { 357 keyFunc = binary.BigEndian.PutUint64 358 } 359 return &testExternalServer{ 360 t: t, 361 keyFunc: keyFunc, 362 sessionAt: sessionAt, 363 maxKeys: maxKeys, 364 } 365 } 366 367 func (s *testExternalServer) SessionIndex() (uint64, error) { 368 return s.sessionAt, nil 369 } 370 371 func (s *testExternalServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { 372 if to > s.maxKeys { 373 to = s.maxKeys 374 } 375 b := make([]byte, HashSize*(to-from+1)) 376 for i := from; i <= to; i++ { 377 s.keyFunc(b[(i-from)*HashSize:(i-from+1)*HashSize], i) 378 } 379 return b, from, to, nil, nil 380 } 381 382 func (s *testExternalServer) GetData(context.Context, []byte) ([]byte, error) { 383 return make([]byte, 4096), nil 384 } 385 386 func (s *testExternalServer) Close() {}