github.com/ipni/storetheindex@v0.8.30/e2e_test.go (about) 1 package main_test 2 3 //lint:file-ignore U1000 Currently skipping this test since it's slow and breaks 4 //often because it's non-reproducible. TODO fixme 5 6 import ( 7 "bytes" 8 "context" 9 "fmt" 10 "io" 11 "net/http" 12 "os" 13 "path/filepath" 14 "runtime" 15 "strings" 16 "testing" 17 "time" 18 19 findclient "github.com/ipni/go-libipni/find/client" 20 "github.com/ipni/go-libipni/find/model" 21 "github.com/ipni/storetheindex/carstore" 22 "github.com/ipni/storetheindex/config" 23 "github.com/ipni/storetheindex/filestore" 24 "github.com/ipni/storetheindex/test" 25 "github.com/multiformats/go-multihash" 26 "github.com/stretchr/testify/require" 27 ) 28 29 // This is a full end-to-end test with storetheindex as the indexer daemon, 30 // and index-provider/cmd/provider as a client. 31 // We build both programs, noting that we always build the latest provider. 32 // We initialize their setup, start the two daemons, and connect the peers. 33 // We then import a CAR file and query its CIDs. 34 35 func TestEndToEndWithAllProviderTypes(t *testing.T) { 36 if os.Getenv("CI") != "" { 37 t.Skip("Skipping e2e test in CI environment") 38 } 39 switch runtime.GOOS { 40 case "windows": 41 t.Skip("skipping test on", runtime.GOOS) 42 } 43 44 // Test with publisher running HTTP ipnisync over libp2p. 45 t.Run("Libp2pProvider", func(t *testing.T) { 46 testEndToEndWithReferenceProvider(t, "libp2p") 47 }) 48 49 // Test with publisher running plain HTTP only, not over libp2p. 50 t.Run("PlainHTTPProvider", func(t *testing.T) { 51 testEndToEndWithReferenceProvider(t, "http") 52 }) 53 54 // Test with publisher running plain HTTP only, not over libp2p. 55 t.Run("Libp2pWithHTTPProvider", func(t *testing.T) { 56 testEndToEndWithReferenceProvider(t, "libp2phttp") 57 }) 58 59 // Test with publisher running dtsync over libp2p. 60 t.Run("DTSyncProvider", func(t *testing.T) { 61 testEndToEndWithReferenceProvider(t, "dtsync") 62 }) 63 } 64 65 func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { 66 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) 67 defer cancel() 68 69 e := test.NewTestIpniRunner(t, ctx, t.TempDir()) 70 71 carPath := filepath.Join(e.Dir, "sample-wrapped-v2.car") 72 err := downloadFile("https://github.com/ipni/index-provider/raw/main/testdata/sample-wrapped-v2.car", carPath) 73 require.NoError(t, err) 74 75 // install storetheindex 76 indexer := filepath.Join(e.Dir, "storetheindex") 77 e.Run("go", "install", ".") 78 79 provider := filepath.Join(e.Dir, "provider") 80 dhstore := filepath.Join(e.Dir, "dhstore") 81 ipni := filepath.Join(e.Dir, "ipni") 82 83 cwd, err := os.Getwd() 84 require.NoError(t, err) 85 86 err = os.Chdir(e.Dir) 87 require.NoError(t, err) 88 89 // install index-provider 90 switch publisherProto { 91 case "dtsync": 92 // Install index-provider that supports dtsync. 93 e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@v0.13.6") 94 case "libp2p", "libp2phttp", "http": 95 e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@latest") 96 default: 97 panic("providerProto must be one of: libp2phttp, http, dtsync") 98 } 99 // install dhstore 100 e.Run("go", "install", "-tags", "nofdb", "github.com/ipni/dhstore/cmd/dhstore@latest") 101 102 // install ipni-cli 103 e.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") 104 105 err = os.Chdir(cwd) 106 require.NoError(t, err) 107 108 // initialize index-provider 109 switch publisherProto { 110 case "dtsync": 111 e.Run(provider, "init") 112 case "http": 113 e.Run(provider, "init", "--pubkind=http") 114 case "libp2p": 115 e.Run(provider, "init", "--pubkind=libp2p") 116 case "libp2phttp": 117 e.Run(provider, "init", "--pubkind=libp2phttp") 118 } 119 providerCfgPath := filepath.Join(e.Dir, ".index-provider", "config") 120 cfg, err := config.Load(providerCfgPath) 121 require.NoError(t, err) 122 providerID := cfg.Identity.PeerID 123 t.Logf("Initialized provider ID: %s", providerID) 124 125 // initialize indexer 126 e.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") 127 stiCfgPath := filepath.Join(e.Dir, ".storetheindex", "config") 128 cfg, err = config.Load(stiCfgPath) 129 require.NoError(t, err) 130 indexerID := cfg.Identity.PeerID 131 cfg.Ingest.AdvertisementMirror = config.Mirror{ 132 Compress: "gzip", 133 Write: true, 134 Storage: filestore.Config{ 135 Type: "local", 136 Local: filestore.LocalConfig{ 137 BasePath: e.Dir, 138 }, 139 }, 140 } 141 rdMirrorDir := e.Dir 142 cfg.Save(stiCfgPath) 143 144 // start provider 145 providerReady := test.NewStdoutWatcher(test.ProviderReadyMatch) 146 providerHasPeer := test.NewStdoutWatcher(test.ProviderHasPeerMatch) 147 cmdProvider := e.Start(test.NewExecution(provider, "daemon").WithWatcher(providerReady).WithWatcher(providerHasPeer)) 148 select { 149 case <-providerReady.Signal: 150 case <-ctx.Done(): 151 t.Fatal("timed out waiting for provider to start") 152 } 153 154 // start dhstore 155 dhstoreReady := test.NewStdoutWatcher(test.DhstoreReady) 156 cmdDhstore := e.Start(test.NewExecution(dhstore, "--storePath", e.Dir).WithWatcher(dhstoreReady)) 157 select { 158 case <-dhstoreReady.Signal: 159 case <-ctx.Done(): 160 t.Fatal("timed out waiting for dhstore to start") 161 } 162 163 // start indexer 164 indexerReady := test.NewStdoutWatcher(test.IndexerReadyMatch) 165 cmdIndexer := e.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady)) 166 select { 167 case <-indexerReady.Signal: 168 case <-ctx.Done(): 169 t.Fatal("timed out waiting for indexer to start") 170 } 171 172 // connect provider to the indexer 173 e.Run(provider, "connect", 174 "--imaddr", fmt.Sprintf("/dns/localhost/tcp/3003/p2p/%s", indexerID), 175 "--listen-admin", "http://localhost:3102", 176 ) 177 select { 178 case <-providerHasPeer.Signal: 179 case <-ctx.Done(): 180 t.Fatal("timed out waiting for provider to connect to indexer") 181 } 182 183 // Allow provider advertisements, regardless of default policy. 184 e.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", providerID) 185 186 // Import a car file into the provider. This will cause the provider to 187 // publish an advertisement that the indexer will read. The indexer will 188 // then import the advertised content. 189 outImport := e.Run(provider, "import", "car", 190 "-i", carPath, 191 "--listen-admin", "http://localhost:3102", 192 ) 193 t.Logf("import output:\n%s\n", outImport) 194 195 // Wait for the CAR to be indexed 196 require.Eventually(t, func() bool { 197 for _, mh := range []string{ 198 "2DrjgbFdhNiSJghFWcQbzw6E8y4jU1Z7ZsWo3dJbYxwGTNFmAj", 199 "2DrjgbFY1BnkgZwA3oL7ijiDn7sJMf4bhhQNTtDqgZP826vGzv", 200 } { 201 findOutput := e.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) 202 t.Logf("find output:\n%s\n", findOutput) 203 204 if bytes.Contains(findOutput, []byte("not found")) { 205 return false 206 } 207 if !bytes.Contains(findOutput, []byte("Provider:")) { 208 t.Logf("mh %s: unexpected error: %s", mh, findOutput) 209 return false 210 } 211 } 212 return true 213 }, 10*time.Second, time.Second) 214 215 e.Run("sync") 216 217 // Check that ad was saved as CAR file. 218 dir, err := os.Open(e.Dir) 219 require.NoError(t, err) 220 names, err := dir.Readdirnames(-1) 221 dir.Close() 222 require.NoError(t, err) 223 var carCount, headCount int 224 225 carSuffix := carstore.CarFileSuffix + carstore.GzipFileSuffix 226 for _, name := range names { 227 if strings.HasSuffix(name, carSuffix) && strings.HasPrefix(name, "baguqeera") { 228 carCount++ 229 } else if strings.HasSuffix(name, carstore.HeadFileSuffix) { 230 headCount++ 231 } 232 } 233 require.Equal(t, 1, carCount) 234 require.Equal(t, 1, headCount) 235 236 outRates := e.Run(indexer, "admin", "telemetry", "-i", "http://localhost:3002") 237 require.Contains(t, string(outRates), "1043 multihashes from 1 ads") 238 t.Logf("Telemetry:\n%s", outRates) 239 240 root2 := filepath.Join(e.Dir, ".storetheindex2") 241 e.Env = append(e.Env, fmt.Sprintf("%s=%s", config.EnvDir, root2)) 242 e.Run(indexer, "init", "--store", "dhstore", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap", "--dhstore", "http://127.0.0.1:40080", 243 "--listen-admin", "/ip4/127.0.0.1/tcp/3202", "--listen-finder", "/ip4/127.0.0.1/tcp/3200", "--listen-ingest", "/ip4/127.0.0.1/tcp/3201", 244 "--listen-p2p", "/ip4/127.0.0.1/tcp/3203") 245 246 sti2CfgPath := filepath.Join(root2, "config") 247 cfg, err = config.Load(sti2CfgPath) 248 require.NoError(t, err) 249 indexer2ID := cfg.Identity.PeerID 250 cfg.Ingest.AdvertisementMirror = config.Mirror{ 251 Compress: "gzip", 252 Read: true, 253 Write: true, 254 Retrieval: filestore.Config{ 255 Type: "local", 256 Local: filestore.LocalConfig{ 257 BasePath: rdMirrorDir, 258 }, 259 }, 260 Storage: filestore.Config{ 261 Type: "local", 262 Local: filestore.LocalConfig{ 263 BasePath: e.Dir, 264 }, 265 }, 266 } 267 cfg.Save(sti2CfgPath) 268 wrMirrorDir := e.Dir 269 270 indexerReady2 := test.NewStdoutWatcher(test.IndexerReadyMatch) 271 cmdIndexer2 := e.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady2)) 272 select { 273 case <-indexerReady2.Signal: 274 case <-ctx.Done(): 275 t.Fatal("timed out waiting for indexer2 to start") 276 } 277 278 outProviders := e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200") 279 require.Contains(t, string(outProviders), "No providers registered with indexer", 280 "expected no providers message") 281 282 // import providers from first indexer. 283 e.Run(indexer, "admin", "import-providers", "--indexer", "http://localhost:3202", "--from", "localhost:3000") 284 285 // Check that provider ID now appears in providers output. 286 outProviders = e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200", "--id-only") 287 require.Contains(t, string(outProviders), providerID, "expected provider id in providers output after import-providers") 288 289 // Connect provider to the 2nd indexer. 290 e.Run(provider, "connect", 291 "--imaddr", fmt.Sprintf("/dns/localhost/tcp/3203/p2p/%s", indexer2ID), 292 "--listen-admin", "http://localhost:3102", 293 ) 294 select { 295 case <-providerHasPeer.Signal: 296 case <-ctx.Done(): 297 t.Fatal("timed out waiting for provider to connect to indexer") 298 } 299 300 // Tell provider to send direct announce to 2nd indexer. 301 out := e.Run(provider, "announce-http", 302 "-i", "http://localhost:3201", 303 "--listen-admin", "http://localhost:3102", 304 ) 305 t.Logf("announce output:\n%s\n", out) 306 307 // Create double hashed client and verify that 2nd indexer wrote 308 // multihashes to dhstore. 309 client, err := findclient.NewDHashClient(findclient.WithProvidersURL("http://127.0.0.1:3000"), findclient.WithDHStoreURL("http://127.0.0.1:40080")) 310 require.NoError(t, err) 311 312 mh, err := multihash.FromB58String("2DrjgbFdhNiSJghFWcQbzw6E8y4jU1Z7ZsWo3dJbYxwGTNFmAj") 313 require.NoError(t, err) 314 315 var dhResp *model.FindResponse 316 require.Eventually(t, func() bool { 317 dhResp, err = client.Find(e.Ctx, mh) 318 return err == nil && len(dhResp.MultihashResults) != 0 319 }, 10*time.Second, time.Second) 320 321 require.Equal(t, 1, len(dhResp.MultihashResults)) 322 require.Equal(t, dhResp.MultihashResults[0].Multihash, mh) 323 require.Equal(t, 1, len(dhResp.MultihashResults[0].ProviderResults)) 324 require.Equal(t, providerID, dhResp.MultihashResults[0].ProviderResults[0].Provider.ID.String()) 325 326 // Get the CAR file from the read mirror. 327 rdCarFile, err := carFromMirror(e.Ctx, rdMirrorDir) 328 require.NoError(t, err) 329 require.NotZero(t, rdCarFile.Size) 330 331 // Get the CAR file from the write mirror and compare size. 332 wrCarFS, err := filestore.NewLocal(wrMirrorDir) 333 require.NoError(t, err) 334 wrCarFile, err := wrCarFS.Head(e.Ctx, rdCarFile.Path) 335 require.NoError(t, err) 336 require.Equal(t, rdCarFile.Size, wrCarFile.Size) 337 t.Logf("CAR file %q is same size in read and write mirror: %d bytes", wrCarFile.Path, wrCarFile.Size) 338 339 // Remove a car file from the provider. This will cause the provider to 340 // publish an advertisement that tells the indexer to remove the car file 341 // content by contextID. The indexer will then import the advertisement and 342 // remove content. 343 outRemove := e.Run(provider, "remove", "car", 344 "-i", carPath, 345 "--listen-admin", "http://localhost:3102", 346 ) 347 t.Logf("remove output:\n%s\n", outRemove) 348 349 // Wait for the CAR indexes to be removed 350 require.Eventually(t, func() bool { 351 for _, mh := range []string{ 352 "2DrjgbFdhNiSJghFWcQbzw6E8y4jU1Z7ZsWo3dJbYxwGTNFmAj", 353 "2DrjgbFY1BnkgZwA3oL7ijiDn7sJMf4bhhQNTtDqgZP826vGzv", 354 } { 355 findOutput := e.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) 356 t.Logf("find output:\n%s\n", findOutput) 357 if !bytes.Contains(findOutput, []byte("not found")) { 358 return false 359 } 360 } 361 return true 362 }, 10*time.Second, time.Second) 363 364 // Check that status is not frozen. 365 outStatus := e.Run(indexer, "admin", "status", "--indexer", "http://localhost:3202") 366 require.Contains(t, string(outStatus), "Frozen: false", "expected indexer to be frozen") 367 368 e.Run(indexer, "admin", "freeze", "--indexer", "http://localhost:3202") 369 outProviders = e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200") 370 371 // Check that provider ID now appears as frozen in providers output. 372 require.Contains(t, string(outProviders), "FrozenAtTime", "expected provider to be frozen") 373 374 // Check that status is frozen. 375 outStatus = e.Run(indexer, "admin", "status", "--indexer", "http://localhost:3202") 376 require.Contains(t, string(outStatus), "Frozen: true", "expected indexer to be frozen") 377 378 logLevel := "info" 379 if testing.Verbose() { 380 logLevel = "debug" 381 } 382 outgc := string(e.Run(indexer, "gc", "provider", "-pid", providerID, "-ll", logLevel, 383 "-i", "http://localhost:3200", 384 "-i", "http://localhost:3000", 385 "-sync-segment-size", "2", 386 )) 387 t.Logf("GC Results:\n%s\n", outgc) 388 require.Contains(t, outgc, `"count": 1043, "total": 1043, "source": "CAR"`) 389 390 e.Stop(cmdIndexer2, time.Second) 391 392 e.Stop(cmdIndexer, time.Second) 393 e.Stop(cmdProvider, time.Second) 394 e.Stop(cmdDhstore, time.Second) 395 } 396 397 func downloadFile(fileURL, filePath string) error { 398 rsp, err := http.Get(fileURL) 399 if err != nil { 400 return err 401 } 402 defer rsp.Body.Close() 403 404 if rsp.StatusCode != 200 { 405 return fmt.Errorf("error response getting file: %d", rsp.StatusCode) 406 } 407 408 file, err := os.Create(filePath) 409 if err != nil { 410 return err 411 } 412 defer file.Close() 413 414 _, err = io.Copy(file, rsp.Body) 415 return err 416 } 417 418 func carFromMirror(ctx context.Context, mirrorDir string) (*filestore.File, error) { 419 listCtx, cancel := context.WithCancel(ctx) 420 defer cancel() 421 mirrorFS, err := filestore.NewLocal(mirrorDir) 422 if err != nil { 423 return nil, err 424 } 425 files, errs := mirrorFS.List(listCtx, "/", false) 426 for f := range files { 427 if strings.HasSuffix(f.Path, ".car.gz") { 428 return f, nil 429 } 430 } 431 return nil, <-errs 432 }