github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/network/alsp/manager/manager_test.go (about) 1 package alspmgr_test 2 3 import ( 4 "context" 5 "math" 6 "math/rand" 7 "sync" 8 "testing" 9 "time" 10 11 "github.com/rs/zerolog" 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/mock" 14 "github.com/stretchr/testify/require" 15 16 "github.com/onflow/flow-go/config" 17 "github.com/onflow/flow-go/model/flow" 18 "github.com/onflow/flow-go/module" 19 "github.com/onflow/flow-go/module/id" 20 "github.com/onflow/flow-go/module/irrecoverable" 21 "github.com/onflow/flow-go/module/metrics" 22 mockmodule "github.com/onflow/flow-go/module/mock" 23 "github.com/onflow/flow-go/network" 24 "github.com/onflow/flow-go/network/alsp" 25 "github.com/onflow/flow-go/network/alsp/internal" 26 alspmgr "github.com/onflow/flow-go/network/alsp/manager" 27 mockalsp "github.com/onflow/flow-go/network/alsp/mock" 28 "github.com/onflow/flow-go/network/alsp/model" 29 "github.com/onflow/flow-go/network/channels" 30 "github.com/onflow/flow-go/network/internal/testutils" 31 "github.com/onflow/flow-go/network/mocknetwork" 32 "github.com/onflow/flow-go/network/p2p" 33 p2ptest "github.com/onflow/flow-go/network/p2p/test" 34 "github.com/onflow/flow-go/network/slashing" 35 "github.com/onflow/flow-go/network/underlay" 36 "github.com/onflow/flow-go/utils/unittest" 37 ) 38 39 // TestHandleReportedMisbehavior tests the handling of reported misbehavior by the network. 40 // 41 // The test sets up a mock MisbehaviorReportManager and a conduitFactory with this manager. 42 // It generates a single node network with the conduitFactory and starts it. 43 // It then uses a mock engine to register a channel with the network. 44 // It prepares a set of misbehavior reports and reports them to the conduit on the test channel. 45 // The test ensures that the MisbehaviorReportManager receives and handles all reported misbehavior 46 // without any duplicate reports and within a specified time. 47 func TestNetworkPassesReportedMisbehavior(t *testing.T) { 48 misbehaviorReportManger := mocknetwork.NewMisbehaviorReportManager(t) 49 misbehaviorReportManger.On("Start", mock.Anything).Return().Once() 50 51 readyDoneChan := func() <-chan struct{} { 52 ch := make(chan struct{}) 53 close(ch) 54 return ch 55 }() 56 57 sporkId := unittest.IdentifierFixture() 58 misbehaviorReportManger.On("Ready").Return(readyDoneChan).Once() 59 misbehaviorReportManger.On("Done").Return(readyDoneChan).Once() 60 ids, nodes := testutils.LibP2PNodeForNetworkFixture(t, sporkId, 1) 61 idProvider := id.NewFixedIdentityProvider(ids) 62 networkCfg := testutils.NetworkConfigFixture(t, *ids[0], idProvider, sporkId, nodes[0]) 63 net, err := underlay.NewNetwork(networkCfg, underlay.WithAlspManager(misbehaviorReportManger)) 64 require.NoError(t, err) 65 66 ctx, cancel := context.WithCancel(context.Background()) 67 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 68 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{net}) 69 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 70 defer cancel() 71 72 e := mocknetwork.NewEngine(t) 73 con, err := net.Register(channels.TestNetworkChannel, e) 74 require.NoError(t, err) 75 76 reports := testutils.MisbehaviorReportsFixture(t, 10) 77 allReportsManaged := sync.WaitGroup{} 78 allReportsManaged.Add(len(reports)) 79 var seenReports []network.MisbehaviorReport 80 misbehaviorReportManger.On("HandleMisbehaviorReport", channels.TestNetworkChannel, mock.Anything).Run(func(args mock.Arguments) { 81 report := args.Get(1).(network.MisbehaviorReport) 82 require.Contains(t, reports, report) // ensures that the report is one of the reports we expect. 83 require.NotContainsf(t, seenReports, report, "duplicate report: %v", report) // ensures that we have not seen this report before. 84 seenReports = append(seenReports, report) // adds the report to the list of seen reports. 85 allReportsManaged.Done() 86 }).Return(nil) 87 88 for _, report := range reports { 89 con.ReportMisbehavior(report) // reports the misbehavior 90 } 91 92 unittest.RequireReturnsBefore(t, allReportsManaged.Wait, 100*time.Millisecond, "did not receive all reports") 93 } 94 95 // TestHandleReportedMisbehavior tests the handling of reported misbehavior by the network. 96 // 97 // The test sets up a mock MisbehaviorReportManager and a conduitFactory with this manager. 98 // It generates a single node network with the conduitFactory and starts it. 99 // It then uses a mock engine to register a channel with the network. 100 // It prepares a set of misbehavior reports and reports them to the conduit on the test channel. 101 // The test ensures that the MisbehaviorReportManager receives and handles all reported misbehavior 102 // without any duplicate reports and within a specified time. 103 func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { 104 cfg := managerCfgFixture(t) 105 106 // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute 107 // of the network, we need to configure the ALSP manager via the network configuration, and let the network create 108 // the ALSP manager. 109 var cache alsp.SpamRecordCache 110 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 111 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 112 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 113 return cache 114 }), 115 } 116 117 sporkId := unittest.IdentifierFixture() 118 ids, nodes := testutils.LibP2PNodeForNetworkFixture(t, sporkId, 1) 119 idProvider := id.NewFixedIdentityProvider(ids) 120 networkCfg := testutils.NetworkConfigFixture(t, *ids[0], idProvider, sporkId, nodes[0], underlay.WithAlspConfig(cfg)) 121 net, err := underlay.NewNetwork(networkCfg) 122 require.NoError(t, err) 123 124 ctx, cancel := context.WithCancel(context.Background()) 125 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 126 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{net}) 127 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 128 defer cancel() 129 130 e := mocknetwork.NewEngine(t) 131 con, err := net.Register(channels.TestNetworkChannel, e) 132 require.NoError(t, err) 133 134 // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) 135 numPeers := 10 136 numReportsPerPeer := 5 137 peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) 138 139 for i := 0; i < numPeers; i++ { 140 originID := unittest.IdentifierFixture() 141 reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) 142 peersReports[originID] = reports 143 } 144 145 wg := sync.WaitGroup{} 146 for _, reports := range peersReports { 147 wg.Add(len(reports)) 148 // reports the misbehavior 149 for _, report := range reports { 150 r := report // capture range variable 151 go func() { 152 defer wg.Done() 153 154 con.ReportMisbehavior(r) 155 }() 156 } 157 } 158 159 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 160 161 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 162 require.Eventually(t, func() bool { 163 for originID, reports := range peersReports { 164 totalPenalty := float64(0) 165 for _, report := range reports { 166 totalPenalty += report.Penalty() 167 } 168 169 record, ok := cache.Get(originID) 170 if !ok { 171 return false 172 } 173 require.NotNil(t, record) 174 175 require.Equal(t, totalPenalty, record.Penalty) 176 // with just reporting a single misbehavior report, the cutoff counter should not be incremented. 177 require.Equal(t, uint64(0), record.CutoffCounter) 178 // with just reporting a single misbehavior report, the node should not be disallowed. 179 require.False(t, record.DisallowListed) 180 // the decay should be the default decay value. 181 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 182 } 183 184 return true 185 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 186 } 187 188 // TestHandleReportedMisbehavior_And_DisallowListing_Integration implements an end-to-end integration test for the 189 // handling of reported misbehavior and disallow listing. 190 // 191 // The test sets up 3 nodes, one victim, one honest, and one (alleged) spammer. 192 // Initially, the test ensures that all nodes are connected to each other. 193 // Then, test imitates that victim node reports the spammer node for spamming. 194 // The test generates enough spam reports to trigger the disallow-listing of the victim node. 195 // The test ensures that the victim node is disconnected from the spammer node. 196 // The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and 197 // the disallow-listed spammer node are established. 198 func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) { 199 cfg := managerCfgFixture(t) 200 201 // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute 202 // of the network, we need to configure the ALSP manager via the network configuration, and let the network create 203 // the ALSP manager. 204 var victimSpamRecordCache alsp.SpamRecordCache 205 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 206 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 207 victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 208 return victimSpamRecordCache 209 }), 210 } 211 212 sporkId := unittest.IdentifierFixture() 213 ids, nodes := testutils.LibP2PNodeForNetworkFixture( 214 t, 215 sporkId, 216 3, 217 p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) 218 219 idProvider := id.NewFixedIdentityProvider(ids) 220 networkCfg := testutils.NetworkConfigFixture(t, *ids[0], idProvider, sporkId, nodes[0], underlay.WithAlspConfig(cfg)) 221 victimNetwork, err := underlay.NewNetwork(networkCfg) 222 require.NoError(t, err) 223 224 // index of the victim node in the nodes slice. 225 victimIndex := 0 226 // index of the spammer node in the nodes slice (the node that will be reported for misbehavior and disallow-listed by victim). 227 spammerIndex := 1 228 // other node (not victim and not spammer) that we have to ensure is not affected by the disallow-listing of the spammer. 229 honestIndex := 2 230 231 ctx, cancel := context.WithCancel(context.Background()) 232 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 233 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{victimNetwork}) 234 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 235 defer cancel() 236 237 p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) 238 // initially victim and spammer should be able to connect to each other. 239 p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) 240 241 e := mocknetwork.NewEngine(t) 242 con, err := victimNetwork.Register(channels.TestNetworkChannel, e) 243 require.NoError(t, err) 244 245 // creates a misbehavior report for the spammer 246 report := misbehaviorReportFixtureWithPenalty(t, ids[spammerIndex].NodeID, model.DefaultPenaltyValue) 247 248 // simulates the victim node reporting the spammer node misbehavior 120 times 249 // to the network. As each report has the default penalty, ideally the spammer should be disallow-listed after 250 // 100 reports (each having 0.01 * disallow-listing penalty). But we take 120 as a safe number to ensure that 251 // the spammer is definitely disallow-listed. 252 reportCount := 120 253 wg := sync.WaitGroup{} 254 for i := 0; i < reportCount; i++ { 255 wg.Add(1) 256 // reports the misbehavior 257 r := report // capture range variable 258 go func() { 259 defer wg.Done() 260 261 con.ReportMisbehavior(r) 262 }() 263 } 264 265 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 266 267 // ensures that the spammer is disallow-listed by the victim 268 p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}, 100*time.Millisecond, 5*time.Second) 269 270 // despite disallow-listing spammer, ensure that (victim and honest) and (honest and spammer) are still connected. 271 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[spammerIndex], nodes[honestIndex]}, 1*time.Millisecond, 100*time.Millisecond) 272 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) 273 274 // while the spammer node is disallow-listed, it cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the spammer node, unless 275 // it is allow-listed again. 276 p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}, 100*time.Millisecond, 2*time.Second) 277 } 278 279 // TestHandleReportedMisbehavior_And_DisallowListing_RepeatOffender_Integration implements an end-to-end integration test for the 280 // handling of repeated reported misbehavior and disallow listing. 281 func TestHandleReportedMisbehavior_And_DisallowListing_RepeatOffender_Integration(t *testing.T) { 282 cfg := managerCfgFixture(t) 283 sporkId := unittest.IdentifierFixture() 284 fastDecay := false 285 fastDecayFunc := func(record model.ProtocolSpamRecord) float64 { 286 t.Logf("decayFuc called with record: %+v", record) 287 if fastDecay { 288 // decay to zero in a single heart beat 289 t.Log("fastDecay is true, so decay to zero") 290 return 0 291 } else { 292 // decay as usual 293 t.Log("fastDecay is false, so decay as usual") 294 return math.Min(record.Penalty+record.Decay, 0) 295 } 296 } 297 298 // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute 299 // of the network, we need to configure the ALSP manager via the network configuration, and let the network create 300 // the ALSP manager. 301 var victimSpamRecordCache alsp.SpamRecordCache 302 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 303 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 304 victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 305 return victimSpamRecordCache 306 }), 307 alspmgr.WithDecayFunc(fastDecayFunc), 308 } 309 310 ids, nodes := testutils.LibP2PNodeForNetworkFixture(t, sporkId, 3, 311 p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(p2ptest.WithZeroJitterAndZeroBackoff(t)), nil)) 312 idProvider := unittest.NewUpdatableIDProvider(ids) 313 networkCfg := testutils.NetworkConfigFixture(t, *ids[0], idProvider, sporkId, nodes[0], underlay.WithAlspConfig(cfg)) 314 315 victimNetwork, err := underlay.NewNetwork(networkCfg) 316 require.NoError(t, err) 317 318 // index of the victim node in the nodes slice. 319 victimIndex := 0 320 // index of the spammer node in the nodes slice (the node that will be reported for misbehavior and disallow-listed by victim). 321 spammerIndex := 1 322 // other node (not victim and not spammer) that we have to ensure is not affected by the disallow-listing of the spammer. 323 honestIndex := 2 324 325 ctx, cancel := context.WithCancel(context.Background()) 326 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 327 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{victimNetwork}) 328 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 329 defer cancel() 330 331 p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) 332 // initially victim and spammer should be able to connect to each other. 333 p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) 334 335 e := mocknetwork.NewEngine(t) 336 con, err := victimNetwork.Register(channels.TestNetworkChannel, e) 337 require.NoError(t, err) 338 339 // creates a misbehavior report for the spammer 340 report := misbehaviorReportFixtureWithPenalty(t, ids[spammerIndex].NodeID, model.DefaultPenaltyValue) 341 342 expectedDecays := []float64{1000, 100, 10, 1, 1, 1} // list of expected decay values after each disallow listing 343 344 t.Log("resetting cutoff counter") 345 expectedCutoffCounter := uint64(0) 346 347 // keep misbehaving until the spammer is disallow-listed and check that the decay is as expected 348 for expectedDecayIndex := range expectedDecays { 349 t.Logf("starting iteration %d with expected decay index %f", expectedDecayIndex, expectedDecays[expectedDecayIndex]) 350 351 // reset the decay function to the default 352 fastDecay = false 353 354 // simulates the victim node reporting the spammer node misbehavior 120 times 355 // as each report has the default penalty, ideally the spammer should be disallow-listed after 356 // 100 reports (each having 0.01 * disallow-listing penalty). But we take 120 as a safe number to ensure that 357 // the spammer is definitely disallow-listed. 358 reportCount := 120 359 wg := sync.WaitGroup{} 360 for reportCounter := 0; reportCounter < reportCount; reportCounter++ { 361 wg.Add(1) 362 // reports the misbehavior 363 r := report // capture range variable 364 go func() { 365 defer wg.Done() 366 367 con.ReportMisbehavior(r) 368 }() 369 } 370 371 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 372 373 expectedCutoffCounter++ // cutoff counter is expected to be incremented after each disallow listing 374 375 // ensures that the spammer is disallow-listed by the victim 376 // while the spammer node is disallow-listed, it cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the spammer node, unless 377 // it is allow-listed again. 378 p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}, 100*time.Millisecond, 3*time.Second) 379 380 // ensures that the spammer is not disallow-listed by the honest node 381 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestIndex], nodes[spammerIndex]}, 1*time.Millisecond, 100*time.Millisecond) 382 383 // ensures that the spammer is disallow-listed for the expected amount of time 384 record, ok := victimSpamRecordCache.Get(ids[spammerIndex].NodeID) 385 require.True(t, ok) 386 require.NotNil(t, record) 387 388 // check the penalty of the spammer node, which should be below the disallow-listing threshold. 389 // i.e. spammer penalty should be more negative than the disallow-listing threshold, hence disallow-listed. 390 require.Less(t, record.Penalty, float64(model.DisallowListingThreshold)) 391 require.Equal(t, expectedDecays[expectedDecayIndex], record.Decay) 392 393 require.Equal(t, expectedDecays[expectedDecayIndex], record.Decay) 394 // when a node is disallow-listed, it remains disallow-listed until its penalty decays back to zero. 395 require.Equal(t, true, record.DisallowListed) 396 require.Equal(t, expectedCutoffCounter, record.CutoffCounter) 397 398 penalty1 := record.Penalty 399 400 // wait for one heartbeat to be processed. 401 time.Sleep(1 * time.Second) 402 403 record, ok = victimSpamRecordCache.Get(ids[spammerIndex].NodeID) 404 require.True(t, ok) 405 require.NotNil(t, record) 406 407 // check the penalty of the spammer node, which should be below the disallow-listing threshold. 408 // i.e. spammer penalty should be more negative than the disallow-listing threshold, hence disallow-listed. 409 require.Less(t, record.Penalty, float64(model.DisallowListingThreshold)) 410 require.Equal(t, expectedDecays[expectedDecayIndex], record.Decay) 411 412 require.Equal(t, expectedDecays[expectedDecayIndex], record.Decay) 413 // when a node is disallow-listed, it remains disallow-listed until its penalty decays back to zero. 414 require.Equal(t, true, record.DisallowListed) 415 require.Equal(t, expectedCutoffCounter, record.CutoffCounter) 416 penalty2 := record.Penalty 417 418 // check that the penalty has decayed by the expected amount in one heartbeat 419 require.Equal(t, expectedDecays[expectedDecayIndex], penalty2-penalty1) 420 421 // decay the disallow-listing penalty of the spammer node to zero. 422 t.Log("about to decay the disallow-listing penalty of the spammer node to zero") 423 fastDecay = true 424 t.Log("decayed the disallow-listing penalty of the spammer node to zero") 425 426 // after serving the disallow-listing period, the spammer should be able to connect to the victim node again. 427 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[spammerIndex], nodes[victimIndex]}, 1*time.Millisecond, 3*time.Second) 428 t.Log("spammer node is able to connect to the victim node again") 429 430 // all the nodes should be able to connect to each other again. 431 p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) 432 433 record, ok = victimSpamRecordCache.Get(ids[spammerIndex].NodeID) 434 require.True(t, ok) 435 require.NotNil(t, record) 436 437 require.Equal(t, float64(0), record.Penalty) 438 require.Equal(t, expectedDecays[expectedDecayIndex], record.Decay) 439 require.Equal(t, false, record.DisallowListed) 440 require.Equal(t, expectedCutoffCounter, record.CutoffCounter) 441 442 // go back to regular decay to prepare for the next set of misbehavior reports. 443 fastDecay = false 444 t.Log("about to report misbehavior again") 445 } 446 } 447 448 // TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration implements an end-to-end integration test for the 449 // handling of reported misbehavior from the slashing.ViolationsConsumer. 450 // 451 // The test sets up one victim, one honest, and one (alleged) spammer for each of the current slashing violations. 452 // Initially, the test ensures that all nodes are connected to each other. 453 // Then, test imitates the slashing violations consumer on the victim node reporting misbehavior's for each slashing violation. 454 // The test generates enough slashing violations to trigger the connection to each of the spamming nodes to be eventually pruned. 455 // The test ensures that the victim node is disconnected from all spammer nodes. 456 // The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and 457 // the pruned spammer nodes are established. 458 func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { 459 sporkId := unittest.IdentifierFixture() 460 461 // create 1 victim node, 1 honest node and a node for each slashing violation 462 ids, nodes := testutils.LibP2PNodeForNetworkFixture(t, sporkId, 7) // creates 7 nodes (1 victim, 1 honest, 5 spammer nodes one for each slashing violation). 463 idProvider := id.NewFixedIdentityProvider(ids) 464 465 // also a placeholder for the slashing violations consumer. 466 var violationsConsumer network.ViolationsConsumer 467 networkCfg := testutils.NetworkConfigFixture( 468 t, 469 *ids[0], 470 idProvider, 471 sporkId, 472 nodes[0], 473 underlay.WithAlspConfig(managerCfgFixture(t)), 474 underlay.WithSlashingViolationConsumerFactory(func(adapter network.ConduitAdapter) network.ViolationsConsumer { 475 violationsConsumer = slashing.NewSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), adapter) 476 return violationsConsumer 477 })) 478 victimNetwork, err := underlay.NewNetwork(networkCfg) 479 require.NoError(t, err) 480 481 ctx, cancel := context.WithCancel(context.Background()) 482 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 483 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{victimNetwork}) 484 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 485 defer cancel() 486 487 p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) 488 // initially victim and misbehaving nodes should be able to connect to each other. 489 p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) 490 491 // each slashing violation func is mapped to a violation with the identity of one of the misbehaving nodes 492 // index of the victim node in the nodes slice. 493 victimIndex := 0 494 honestNodeIndex := 1 495 invalidMessageIndex := 2 496 senderEjectedIndex := 3 497 unauthorizedUnicastOnChannelIndex := 4 498 unauthorizedPublishOnChannelIndex := 5 499 unknownMsgTypeIndex := 6 500 slashingViolationTestCases := []struct { 501 violationsConsumerFunc func(violation *network.Violation) 502 violation *network.Violation 503 }{ 504 {violationsConsumer.OnUnAuthorizedSenderError, &network.Violation{Identity: ids[invalidMessageIndex]}}, 505 {violationsConsumer.OnSenderEjectedError, &network.Violation{Identity: ids[senderEjectedIndex]}}, 506 {violationsConsumer.OnUnauthorizedUnicastOnChannel, &network.Violation{Identity: ids[unauthorizedUnicastOnChannelIndex]}}, 507 {violationsConsumer.OnUnauthorizedPublishOnChannel, &network.Violation{Identity: ids[unauthorizedPublishOnChannelIndex]}}, 508 {violationsConsumer.OnUnknownMsgTypeError, &network.Violation{Identity: ids[unknownMsgTypeIndex]}}, 509 } 510 511 violationsWg := sync.WaitGroup{} 512 violationCount := 120 513 for _, testCase := range slashingViolationTestCases { 514 for i := 0; i < violationCount; i++ { 515 testCase := testCase 516 violationsWg.Add(1) 517 go func() { 518 defer violationsWg.Done() 519 testCase.violationsConsumerFunc(testCase.violation) 520 }() 521 } 522 } 523 unittest.RequireReturnsBefore(t, violationsWg.Wait, 100*time.Millisecond, "slashing violations not reported in time") 524 525 forEachMisbehavingNode := func(f func(i int)) { 526 for misbehavingNodeIndex := 2; misbehavingNodeIndex <= len(nodes)-1; misbehavingNodeIndex++ { 527 f(misbehavingNodeIndex) 528 } 529 } 530 531 // ensures all misbehaving nodes are disconnected from the victim node 532 forEachMisbehavingNode(func(misbehavingNodeIndex int) { 533 p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}, 100*time.Millisecond, 2*time.Second) 534 }) 535 536 // despite being disconnected from the victim node, misbehaving nodes and the honest node are still connected. 537 forEachMisbehavingNode(func(misbehavingNodeIndex int) { 538 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[misbehavingNodeIndex]}, 1*time.Millisecond, 100*time.Millisecond) 539 }) 540 541 // despite disconnecting misbehaving nodes, ensure that (victim and honest) are still connected. 542 p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) 543 544 // while misbehaving nodes are disconnected, they cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the misbehaving nodes until each node's peer score decays. 545 forEachMisbehavingNode(func(misbehavingNodeIndex int) { 546 p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}) 547 }) 548 } 549 550 // TestMisbehaviorReportMetrics tests the recording of misbehavior report metrics. 551 // It checks that when a misbehavior report is received by the ALSP manager, the metrics are recorded. 552 // It fails the test if the metrics are not recorded or if they are recorded incorrectly. 553 func TestMisbehaviorReportMetrics(t *testing.T) { 554 cfg := managerCfgFixture(t) 555 556 // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute 557 // of the network, we need to configure the ALSP manager via the network configuration, and let the network create 558 // the ALSP manager. 559 alspMetrics := mockmodule.NewAlspMetrics(t) 560 cfg.AlspMetrics = alspMetrics 561 562 sporkId := unittest.IdentifierFixture() 563 ids, nodes := testutils.LibP2PNodeForNetworkFixture(t, sporkId, 1) 564 idProvider := id.NewFixedIdentityProvider(ids) 565 566 networkCfg := testutils.NetworkConfigFixture(t, *ids[0], idProvider, sporkId, nodes[0], underlay.WithAlspConfig(cfg)) 567 net, err := underlay.NewNetwork(networkCfg) 568 require.NoError(t, err) 569 570 ctx, cancel := context.WithCancel(context.Background()) 571 572 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 573 testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.EngineRegistry{net}) 574 defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) 575 defer cancel() 576 577 e := mocknetwork.NewEngine(t) 578 con, err := net.Register(channels.TestNetworkChannel, e) 579 require.NoError(t, err) 580 581 report := testutils.MisbehaviorReportFixture(t) 582 583 // this channel is used to signal that the metrics have been recorded by the ALSP manager correctly. 584 reported := make(chan struct{}) 585 586 // ensures that the metrics are recorded when a misbehavior report is received. 587 alspMetrics.On("OnMisbehaviorReported", channels.TestNetworkChannel.String(), report.Reason().String()).Run(func(args mock.Arguments) { 588 close(reported) 589 }).Once() 590 591 con.ReportMisbehavior(report) // reports the misbehavior 592 593 unittest.RequireCloseBefore(t, reported, 100*time.Millisecond, "metrics for the misbehavior report were not recorded") 594 } 595 596 // The TestReportCreation tests the creation of misbehavior reports using the alsp.NewMisbehaviorReport function. 597 // The function tests the creation of both valid and invalid misbehavior reports by setting different penalty amplification values. 598 func TestReportCreation(t *testing.T) { 599 600 // creates a valid misbehavior report (i.e., amplification between 1 and 100) 601 report, err := alsp.NewMisbehaviorReport( 602 unittest.IdentifierFixture(), 603 testutils.MisbehaviorTypeFixture(t), 604 alsp.WithPenaltyAmplification(10)) 605 require.NoError(t, err) 606 require.NotNil(t, report) 607 608 // creates a valid misbehavior report with default amplification. 609 report, err = alsp.NewMisbehaviorReport( 610 unittest.IdentifierFixture(), 611 testutils.MisbehaviorTypeFixture(t)) 612 require.NoError(t, err) 613 require.NotNil(t, report) 614 615 // creates an in valid misbehavior report (i.e., amplification greater than 100 and less than 1) 616 report, err = alsp.NewMisbehaviorReport( 617 unittest.IdentifierFixture(), 618 testutils.MisbehaviorTypeFixture(t), 619 alsp.WithPenaltyAmplification(100*rand.Float64()-101)) 620 require.Error(t, err) 621 require.Nil(t, report) 622 623 report, err = alsp.NewMisbehaviorReport( 624 unittest.IdentifierFixture(), 625 testutils.MisbehaviorTypeFixture(t), 626 alsp.WithPenaltyAmplification(100*rand.Float64()+101)) 627 require.Error(t, err) 628 require.Nil(t, report) 629 630 // 0 is not a valid amplification 631 report, err = alsp.NewMisbehaviorReport( 632 unittest.IdentifierFixture(), 633 testutils.MisbehaviorTypeFixture(t), 634 alsp.WithPenaltyAmplification(0)) 635 require.Error(t, err) 636 require.Nil(t, report) 637 } 638 639 // TestNewMisbehaviorReportManager tests the creation of a new ALSP manager. 640 // It is a minimum viable test that ensures that a non-nil ALSP manager is created with expected set of inputs. 641 // In other words, variation of input values do not cause a nil ALSP manager to be created or a panic. 642 func TestNewMisbehaviorReportManager(t *testing.T) { 643 cfg := managerCfgFixture(t) 644 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 645 var cache alsp.SpamRecordCache 646 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 647 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 648 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 649 return cache 650 }), 651 } 652 653 t.Run("with default values", func(t *testing.T) { 654 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 655 require.NoError(t, err) 656 assert.NotNil(t, m) 657 }) 658 659 t.Run("with a custom spam record cache", func(t *testing.T) { 660 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 661 require.NoError(t, err) 662 assert.NotNil(t, m) 663 }) 664 665 t.Run("with ALSP module enabled", func(t *testing.T) { 666 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 667 require.NoError(t, err) 668 assert.NotNil(t, m) 669 }) 670 671 t.Run("with ALSP module disabled", func(t *testing.T) { 672 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 673 require.NoError(t, err) 674 assert.NotNil(t, m) 675 }) 676 } 677 678 // TestMisbehaviorReportManager_InitializationError tests the creation of a new ALSP manager with invalid inputs. 679 // It is a minimum viable test that ensures that a nil ALSP manager is created with invalid set of inputs. 680 func TestMisbehaviorReportManager_InitializationError(t *testing.T) { 681 cfg := managerCfgFixture(t) 682 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 683 684 t.Run("missing spam report queue size", func(t *testing.T) { 685 cfg.SpamReportQueueSize = 0 686 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 687 require.Error(t, err) 688 require.ErrorIs(t, err, alspmgr.ErrSpamReportQueueSizeNotSet) 689 assert.Nil(t, m) 690 }) 691 692 t.Run("missing spam record cache size", func(t *testing.T) { 693 cfg.SpamRecordCacheSize = 0 694 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 695 require.Error(t, err) 696 require.ErrorIs(t, err, alspmgr.ErrSpamRecordCacheSizeNotSet) 697 assert.Nil(t, m) 698 }) 699 700 t.Run("missing heartbeat intervals", func(t *testing.T) { 701 cfg.HeartBeatInterval = 0 702 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 703 require.Error(t, err) 704 require.ErrorIs(t, err, alspmgr.ErrSpamRecordCacheSizeNotSet) 705 assert.Nil(t, m) 706 }) 707 } 708 709 // TestHandleMisbehaviorReport_SinglePenaltyReport tests the handling of a single misbehavior report. 710 // The test ensures that the misbehavior report is handled correctly and the penalty is applied to the peer in the cache. 711 func TestHandleMisbehaviorReport_SinglePenaltyReport(t *testing.T) { 712 cfg := managerCfgFixture(t) 713 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 714 715 // create a new MisbehaviorReportManager 716 var cache alsp.SpamRecordCache 717 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 718 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 719 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 720 return cache 721 }), 722 } 723 724 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 725 require.NoError(t, err) 726 727 // start the ALSP manager 728 ctx, cancel := context.WithCancel(context.Background()) 729 defer func() { 730 cancel() 731 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 732 }() 733 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 734 m.Start(signalerCtx) 735 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 736 737 // create a mock misbehavior report with a negative penalty value 738 penalty := float64(-5) 739 report := mocknetwork.NewMisbehaviorReport(t) 740 report.On("OriginId").Return(unittest.IdentifierFixture()) 741 report.On("Reason").Return(alsp.InvalidMessage) 742 report.On("Penalty").Return(penalty) 743 744 channel := channels.Channel("test-channel") 745 746 // handle the misbehavior report 747 m.HandleMisbehaviorReport(channel, report) 748 749 require.Eventually(t, func() bool { 750 // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache 751 record, ok := cache.Get(report.OriginId()) 752 if !ok { 753 return false 754 } 755 require.NotNil(t, record) 756 require.Equal(t, penalty, record.Penalty) 757 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet 758 require.Equal(t, uint64(0), record.CutoffCounter) // with just reporting a misbehavior, the cutoff counter should not be incremented. 759 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) // the decay should be the default decay value. 760 761 return true 762 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 763 } 764 765 // TestHandleMisbehaviorReport_SinglePenaltyReport_PenaltyDisable tests the handling of a single misbehavior report when the penalty is disabled. 766 // The test ensures that the misbehavior is reported on metrics but the penalty is not applied to the peer in the cache. 767 func TestHandleMisbehaviorReport_SinglePenaltyReport_PenaltyDisable(t *testing.T) { 768 cfg := managerCfgFixture(t) 769 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 770 771 cfg.DisablePenalty = true // disable penalty for misbehavior reports 772 alspMetrics := mockmodule.NewAlspMetrics(t) 773 cfg.AlspMetrics = alspMetrics 774 775 // we use a mock cache but we do not expect any calls to the cache, since the penalty is disabled. 776 var cache *mockalsp.SpamRecordCache 777 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 778 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 779 cache = mockalsp.NewSpamRecordCache(t) 780 return cache 781 }), 782 } 783 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 784 require.NoError(t, err) 785 786 // start the ALSP manager 787 ctx, cancel := context.WithCancel(context.Background()) 788 defer func() { 789 cancel() 790 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 791 }() 792 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 793 m.Start(signalerCtx) 794 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 795 796 // create a mock misbehavior report with a negative penalty value 797 penalty := float64(-5) 798 report := mocknetwork.NewMisbehaviorReport(t) 799 report.On("OriginId").Return(unittest.IdentifierFixture()) 800 report.On("Reason").Return(alsp.InvalidMessage) 801 report.On("Penalty").Return(penalty) 802 803 channel := channels.Channel("test-channel") 804 805 // this channel is used to signal that the metrics have been recorded by the ALSP manager correctly. 806 // even in case of a disabled penalty, the metrics should be recorded. 807 reported := make(chan struct{}) 808 809 // ensures that the metrics are recorded when a misbehavior report is received. 810 alspMetrics.On("OnMisbehaviorReported", channel.String(), report.Reason().String()).Run(func(args mock.Arguments) { 811 close(reported) 812 }).Once() 813 814 // handle the misbehavior report 815 m.HandleMisbehaviorReport(channel, report) 816 817 unittest.RequireCloseBefore(t, reported, 100*time.Millisecond, "metrics for the misbehavior report were not recorded") 818 819 // since the penalty is disabled, we do not expect any calls to the cache. 820 cache.AssertNotCalled(t, "Adjust", mock.Anything, mock.Anything) 821 } 822 823 // TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequentially tests the handling of multiple misbehavior reports for a single peer. 824 // Reports are coming in sequentially. 825 // The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache. 826 func TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequentially(t *testing.T) { 827 cfg := managerCfgFixture(t) 828 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 829 830 // create a new MisbehaviorReportManager 831 var cache alsp.SpamRecordCache 832 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 833 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 834 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 835 return cache 836 }), 837 } 838 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 839 require.NoError(t, err) 840 841 // start the ALSP manager 842 ctx, cancel := context.WithCancel(context.Background()) 843 defer func() { 844 cancel() 845 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 846 }() 847 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 848 m.Start(signalerCtx) 849 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 850 851 // creates a list of mock misbehavior reports with negative penalty values for a single peer 852 originId := unittest.IdentifierFixture() 853 reports := createRandomMisbehaviorReportsForOriginId(t, originId, 5) 854 855 channel := channels.Channel("test-channel") 856 857 // handle the misbehavior reports 858 totalPenalty := float64(0) 859 for _, report := range reports { 860 totalPenalty += report.Penalty() 861 m.HandleMisbehaviorReport(channel, report) 862 } 863 864 require.Eventually(t, func() bool { 865 // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache 866 record, ok := cache.Get(originId) 867 if !ok { 868 return false 869 } 870 require.NotNil(t, record) 871 872 if totalPenalty != record.Penalty { 873 // all the misbehavior reports should be processed by now, so the penalty should be equal to the total penalty 874 return false 875 } 876 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 877 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 878 require.Equal(t, uint64(0), record.CutoffCounter) 879 // the decay should be the default decay value. 880 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 881 882 return true 883 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 884 } 885 886 // TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequential tests the handling of multiple misbehavior reports for a single peer. 887 // Reports are coming in concurrently. 888 // The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache. 889 func TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Concurrently(t *testing.T) { 890 cfg := managerCfgFixture(t) 891 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 892 893 var cache alsp.SpamRecordCache 894 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 895 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 896 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 897 return cache 898 }), 899 } 900 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 901 require.NoError(t, err) 902 903 // start the ALSP manager 904 ctx, cancel := context.WithCancel(context.Background()) 905 defer func() { 906 cancel() 907 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 908 }() 909 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 910 m.Start(signalerCtx) 911 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 912 913 // creates a list of mock misbehavior reports with negative penalty values for a single peer 914 originId := unittest.IdentifierFixture() 915 reports := createRandomMisbehaviorReportsForOriginId(t, originId, 5) 916 917 channel := channels.Channel("test-channel") 918 919 wg := sync.WaitGroup{} 920 wg.Add(len(reports)) 921 // handle the misbehavior reports 922 totalPenalty := float64(0) 923 for _, report := range reports { 924 r := report // capture range variable 925 totalPenalty += report.Penalty() 926 go func() { 927 defer wg.Done() 928 929 m.HandleMisbehaviorReport(channel, r) 930 }() 931 } 932 933 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 934 935 require.Eventually(t, func() bool { 936 // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache 937 record, ok := cache.Get(originId) 938 if !ok { 939 return false 940 } 941 require.NotNil(t, record) 942 943 if totalPenalty != record.Penalty { 944 // all the misbehavior reports should be processed by now, so the penalty should be equal to the total penalty 945 return false 946 } 947 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 948 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 949 require.Equal(t, uint64(0), record.CutoffCounter) 950 // the decay should be the default decay value. 951 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 952 953 return true 954 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 955 } 956 957 // TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Sequentially tests the handling of single misbehavior reports for multiple peers. 958 // Reports are coming in sequentially. 959 // The test ensures that each misbehavior report is handled correctly and the penalties are applied to the corresponding peers in the cache. 960 func TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Sequentially(t *testing.T) { 961 cfg := managerCfgFixture(t) 962 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 963 964 var cache alsp.SpamRecordCache 965 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 966 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 967 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 968 return cache 969 }), 970 } 971 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 972 require.NoError(t, err) 973 974 // start the ALSP manager 975 ctx, cancel := context.WithCancel(context.Background()) 976 defer func() { 977 cancel() 978 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 979 }() 980 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 981 m.Start(signalerCtx) 982 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 983 984 // creates a list of single misbehavior reports for multiple peers (10 peers) 985 numPeers := 10 986 reports := createRandomMisbehaviorReports(t, numPeers) 987 988 channel := channels.Channel("test-channel") 989 990 // handle the misbehavior reports 991 for _, report := range reports { 992 m.HandleMisbehaviorReport(channel, report) 993 } 994 995 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 996 require.Eventually(t, func() bool { 997 for _, report := range reports { 998 originID := report.OriginId() 999 record, ok := cache.Get(originID) 1000 if !ok { 1001 return false 1002 } 1003 require.NotNil(t, record) 1004 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1005 require.Equal(t, report.Penalty(), record.Penalty) 1006 // with just reporting a single misbehavior report, the cutoff counter should not be incremented. 1007 require.Equal(t, uint64(0), record.CutoffCounter) 1008 // the decay should be the default decay value. 1009 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1010 } 1011 1012 return true 1013 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1014 1015 } 1016 1017 // TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Concurrently tests the handling of single misbehavior reports for multiple peers. 1018 // Reports are coming in concurrently. 1019 // The test ensures that each misbehavior report is handled correctly and the penalties are applied to the corresponding peers in the cache. 1020 func TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Concurrently(t *testing.T) { 1021 cfg := managerCfgFixture(t) 1022 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1023 1024 var cache alsp.SpamRecordCache 1025 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1026 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1027 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1028 return cache 1029 }), 1030 } 1031 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1032 require.NoError(t, err) 1033 1034 // start the ALSP manager 1035 ctx, cancel := context.WithCancel(context.Background()) 1036 defer func() { 1037 cancel() 1038 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1039 }() 1040 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1041 m.Start(signalerCtx) 1042 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1043 1044 // creates a list of single misbehavior reports for multiple peers (10 peers) 1045 numPeers := 10 1046 reports := createRandomMisbehaviorReports(t, numPeers) 1047 1048 channel := channels.Channel("test-channel") 1049 1050 wg := sync.WaitGroup{} 1051 wg.Add(len(reports)) 1052 // handle the misbehavior reports 1053 totalPenalty := float64(0) 1054 for _, report := range reports { 1055 r := report // capture range variable 1056 totalPenalty += report.Penalty() 1057 go func() { 1058 defer wg.Done() 1059 1060 m.HandleMisbehaviorReport(channel, r) 1061 }() 1062 } 1063 1064 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1065 1066 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1067 require.Eventually(t, func() bool { 1068 for _, report := range reports { 1069 originID := report.OriginId() 1070 record, ok := cache.Get(originID) 1071 if !ok { 1072 return false 1073 } 1074 require.NotNil(t, record) 1075 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1076 require.Equal(t, report.Penalty(), record.Penalty) 1077 // with just reporting a single misbehavior report, the cutoff counter should not be incremented. 1078 require.Equal(t, uint64(0), record.CutoffCounter) 1079 // the decay should be the default decay value. 1080 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1081 } 1082 1083 return true 1084 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1085 } 1086 1087 // TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially tests the handling of multiple misbehavior reports for multiple peers. 1088 // Reports are coming in sequentially. 1089 // The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the corresponding peers in the cache. 1090 func TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially(t *testing.T) { 1091 cfg := managerCfgFixture(t) 1092 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1093 1094 var cache alsp.SpamRecordCache 1095 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1096 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1097 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1098 return cache 1099 }), 1100 } 1101 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1102 require.NoError(t, err) 1103 1104 // start the ALSP manager 1105 ctx, cancel := context.WithCancel(context.Background()) 1106 defer func() { 1107 cancel() 1108 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1109 }() 1110 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1111 m.Start(signalerCtx) 1112 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1113 1114 // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) 1115 numPeers := 10 1116 numReportsPerPeer := 5 1117 peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) 1118 1119 for i := 0; i < numPeers; i++ { 1120 originID := unittest.IdentifierFixture() 1121 reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) 1122 peersReports[originID] = reports 1123 } 1124 1125 channel := channels.Channel("test-channel") 1126 1127 wg := sync.WaitGroup{} 1128 // handle the misbehavior reports 1129 for _, reports := range peersReports { 1130 wg.Add(len(reports)) 1131 for _, report := range reports { 1132 r := report // capture range variable 1133 go func() { 1134 defer wg.Done() 1135 1136 m.HandleMisbehaviorReport(channel, r) 1137 }() 1138 } 1139 } 1140 1141 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1142 1143 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1144 require.Eventually(t, func() bool { 1145 for originID, reports := range peersReports { 1146 totalPenalty := float64(0) 1147 for _, report := range reports { 1148 totalPenalty += report.Penalty() 1149 } 1150 1151 record, ok := cache.Get(originID) 1152 if !ok { 1153 return false 1154 } 1155 require.NotNil(t, record) 1156 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1157 require.Equal(t, totalPenalty, record.Penalty) 1158 // with just reporting a single misbehavior report, the cutoff counter should not be incremented. 1159 require.Equal(t, uint64(0), record.CutoffCounter) 1160 // the decay should be the default decay value. 1161 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1162 } 1163 1164 return true 1165 }, 2*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1166 } 1167 1168 // TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially tests the handling of multiple misbehavior reports for multiple peers. 1169 // Reports are coming in concurrently. 1170 // The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the corresponding peers in the cache. 1171 func TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Concurrently(t *testing.T) { 1172 cfg := managerCfgFixture(t) 1173 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1174 1175 var cache alsp.SpamRecordCache 1176 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1177 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1178 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1179 return cache 1180 }), 1181 } 1182 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1183 require.NoError(t, err) 1184 1185 // start the ALSP manager 1186 ctx, cancel := context.WithCancel(context.Background()) 1187 defer func() { 1188 cancel() 1189 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1190 }() 1191 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1192 m.Start(signalerCtx) 1193 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1194 1195 // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) 1196 numPeers := 10 1197 numReportsPerPeer := 5 1198 peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) 1199 1200 for i := 0; i < numPeers; i++ { 1201 originID := unittest.IdentifierFixture() 1202 reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) 1203 peersReports[originID] = reports 1204 } 1205 1206 channel := channels.Channel("test-channel") 1207 1208 // handle the misbehavior reports 1209 for _, reports := range peersReports { 1210 for _, report := range reports { 1211 m.HandleMisbehaviorReport(channel, report) 1212 } 1213 } 1214 1215 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1216 require.Eventually(t, func() bool { 1217 for originID, reports := range peersReports { 1218 totalPenalty := float64(0) 1219 for _, report := range reports { 1220 totalPenalty += report.Penalty() 1221 } 1222 1223 record, ok := cache.Get(originID) 1224 if !ok { 1225 return false 1226 } 1227 require.NotNil(t, record) 1228 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1229 require.Equal(t, totalPenalty, record.Penalty) 1230 // with just reporting a single misbehavior report, the cutoff counter should not be incremented. 1231 require.Equal(t, uint64(0), record.CutoffCounter) 1232 // the decay should be the default decay value. 1233 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1234 } 1235 1236 return true 1237 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1238 } 1239 1240 // TestHandleMisbehaviorReport_DuplicateReportsForSinglePeer_Concurrently tests the handling of duplicate misbehavior reports for a single peer. 1241 // Reports are coming in concurrently. 1242 // The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache, in 1243 // other words, the duplicate reports are not ignored. This is important because the misbehavior reports are assumed each uniquely reporting 1244 // a different misbehavior even though they are coming with the same description. This is similar to the traffic tickets, where each ticket 1245 // is uniquely identifying a traffic violation, even though the description of the violation is the same. 1246 func TestHandleMisbehaviorReport_DuplicateReportsForSinglePeer_Concurrently(t *testing.T) { 1247 cfg := managerCfgFixture(t) 1248 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1249 1250 var cache alsp.SpamRecordCache 1251 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1252 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1253 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1254 return cache 1255 }), 1256 } 1257 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1258 require.NoError(t, err) 1259 1260 // start the ALSP manager 1261 ctx, cancel := context.WithCancel(context.Background()) 1262 defer func() { 1263 cancel() 1264 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1265 }() 1266 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1267 m.Start(signalerCtx) 1268 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1269 1270 // creates a single misbehavior report 1271 originId := unittest.IdentifierFixture() 1272 report := misbehaviorReportFixture(t, originId) 1273 1274 channel := channels.Channel("test-channel") 1275 1276 times := 100 // number of times the duplicate misbehavior report is reported concurrently 1277 wg := sync.WaitGroup{} 1278 wg.Add(times) 1279 1280 // concurrently reports the same misbehavior report twice 1281 for i := 0; i < times; i++ { 1282 go func() { 1283 defer wg.Done() 1284 1285 m.HandleMisbehaviorReport(channel, report) 1286 }() 1287 } 1288 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1289 1290 require.Eventually(t, func() bool { 1291 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1292 record, ok := cache.Get(originId) 1293 if !ok { 1294 return false 1295 } 1296 require.NotNil(t, record) 1297 1298 // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. 1299 if record.Penalty != report.Penalty()*float64(times) { 1300 return false 1301 } 1302 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1303 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1304 require.Equal(t, uint64(0), record.CutoffCounter) 1305 // the decay should be the default decay value. 1306 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1307 1308 return true 1309 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1310 } 1311 1312 // TestDecayMisbehaviorPenalty_SingleHeartbeat tests the decay of the misbehavior penalty. The test ensures that the misbehavior penalty 1313 // is decayed after a single heartbeat. The test guarantees waiting for at least one heartbeat by waiting for the first decay to happen. 1314 func TestDecayMisbehaviorPenalty_SingleHeartbeat(t *testing.T) { 1315 cfg := managerCfgFixture(t) 1316 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1317 1318 var cache alsp.SpamRecordCache 1319 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1320 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1321 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1322 return cache 1323 }), 1324 } 1325 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1326 require.NoError(t, err) 1327 1328 // start the ALSP manager 1329 ctx, cancel := context.WithCancel(context.Background()) 1330 defer func() { 1331 cancel() 1332 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1333 }() 1334 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1335 m.Start(signalerCtx) 1336 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1337 1338 // creates a single misbehavior report 1339 originId := unittest.IdentifierFixture() 1340 report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) 1341 require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative 1342 1343 channel := channels.Channel("test-channel") 1344 1345 // number of times the duplicate misbehavior report is reported concurrently 1346 times := 10 1347 wg := sync.WaitGroup{} 1348 wg.Add(times) 1349 1350 // concurrently reports the same misbehavior report twice 1351 for i := 0; i < times; i++ { 1352 go func() { 1353 defer wg.Done() 1354 1355 m.HandleMisbehaviorReport(channel, report) 1356 }() 1357 } 1358 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1359 1360 // phase-1: eventually all the misbehavior reports should be processed. 1361 penaltyBeforeDecay := float64(0) 1362 require.Eventually(t, func() bool { 1363 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1364 record, ok := cache.Get(originId) 1365 if !ok { 1366 return false 1367 } 1368 require.NotNil(t, record) 1369 1370 // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. 1371 if record.Penalty != report.Penalty()*float64(times) { 1372 return false 1373 } 1374 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1375 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1376 require.Equal(t, uint64(0), record.CutoffCounter) 1377 // the decay should be the default decay value. 1378 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1379 1380 penaltyBeforeDecay = record.Penalty 1381 return true 1382 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1383 1384 // phase-2: wait enough for at least one heartbeat to be processed. 1385 time.Sleep(1 * time.Second) 1386 1387 // phase-3: check if the penalty was decayed for at least one heartbeat. 1388 record, ok := cache.Get(originId) 1389 require.True(t, ok) // the record should be in the cache 1390 require.NotNil(t, record) 1391 1392 // with at least a single heartbeat, the penalty should be greater than the penalty before the decay. 1393 require.Greater(t, record.Penalty, penaltyBeforeDecay) 1394 // we waited for at most one heartbeat, so the decayed penalty should be still less than the value after 2 heartbeats. 1395 require.Less(t, record.Penalty, penaltyBeforeDecay+2*record.Decay) 1396 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1397 require.Equal(t, uint64(0), record.CutoffCounter) 1398 // the decay should be the default decay value. 1399 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1400 } 1401 1402 // TestDecayMisbehaviorPenalty_MultipleHeartbeat tests the decay of the misbehavior penalty under multiple heartbeats. 1403 // The test ensures that the misbehavior penalty is decayed with a linear progression within multiple heartbeats. 1404 func TestDecayMisbehaviorPenalty_MultipleHeartbeats(t *testing.T) { 1405 cfg := managerCfgFixture(t) 1406 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1407 1408 var cache alsp.SpamRecordCache 1409 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1410 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1411 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1412 return cache 1413 }), 1414 } 1415 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1416 require.NoError(t, err) 1417 1418 // start the ALSP manager 1419 ctx, cancel := context.WithCancel(context.Background()) 1420 defer func() { 1421 cancel() 1422 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1423 }() 1424 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1425 m.Start(signalerCtx) 1426 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1427 1428 // creates a single misbehavior report 1429 originId := unittest.IdentifierFixture() 1430 report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) 1431 require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative 1432 1433 channel := channels.Channel("test-channel") 1434 1435 // number of times the duplicate misbehavior report is reported concurrently 1436 times := 10 1437 wg := sync.WaitGroup{} 1438 wg.Add(times) 1439 1440 // concurrently reports the same misbehavior report twice 1441 for i := 0; i < times; i++ { 1442 go func() { 1443 defer wg.Done() 1444 1445 m.HandleMisbehaviorReport(channel, report) 1446 }() 1447 } 1448 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1449 1450 // phase-1: eventually all the misbehavior reports should be processed. 1451 penaltyBeforeDecay := float64(0) 1452 require.Eventually(t, func() bool { 1453 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1454 record, ok := cache.Get(originId) 1455 if !ok { 1456 return false 1457 } 1458 require.NotNil(t, record) 1459 1460 // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. 1461 if record.Penalty != report.Penalty()*float64(times) { 1462 return false 1463 } 1464 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1465 require.Equal(t, uint64(0), record.CutoffCounter) 1466 // the decay should be the default decay value. 1467 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1468 1469 penaltyBeforeDecay = record.Penalty 1470 return true 1471 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1472 1473 // phase-2: wait for 3 heartbeats to be processed. 1474 time.Sleep(3 * time.Second) 1475 1476 // phase-3: check if the penalty was decayed in a linear progression. 1477 record, ok := cache.Get(originId) 1478 require.True(t, ok) // the record should be in the cache 1479 require.NotNil(t, record) 1480 1481 // with 3 heartbeats processed, the penalty should be greater than the penalty before the decay. 1482 require.Greater(t, record.Penalty, penaltyBeforeDecay) 1483 // with 3 heartbeats processed, the decayed penalty should be less than the value after 4 heartbeats. 1484 require.Less(t, record.Penalty, penaltyBeforeDecay+4*record.Decay) 1485 require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. 1486 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1487 require.Equal(t, uint64(0), record.CutoffCounter) 1488 // the decay should be the default decay value. 1489 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1490 } 1491 1492 // TestDecayMisbehaviorPenalty_MultipleHeartbeat tests the decay of the misbehavior penalty under multiple heartbeats. 1493 // The test ensures that the misbehavior penalty is decayed with a linear progression within multiple heartbeats. 1494 func TestDecayMisbehaviorPenalty_DecayToZero(t *testing.T) { 1495 cfg := managerCfgFixture(t) 1496 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1497 1498 var cache alsp.SpamRecordCache 1499 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1500 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1501 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1502 return cache 1503 }), 1504 } 1505 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1506 require.NoError(t, err) 1507 1508 // start the ALSP manager 1509 ctx, cancel := context.WithCancel(context.Background()) 1510 defer func() { 1511 cancel() 1512 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1513 }() 1514 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1515 m.Start(signalerCtx) 1516 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1517 1518 // creates a single misbehavior report 1519 originId := unittest.IdentifierFixture() 1520 report := misbehaviorReportFixture(t, originId) // penalties are between -1 and -10 1521 require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative 1522 1523 channel := channels.Channel("test-channel") 1524 1525 // number of times the duplicate misbehavior report is reported concurrently 1526 times := 10 1527 wg := sync.WaitGroup{} 1528 wg.Add(times) 1529 1530 // concurrently reports the same misbehavior report twice 1531 for i := 0; i < times; i++ { 1532 go func() { 1533 defer wg.Done() 1534 1535 m.HandleMisbehaviorReport(channel, report) 1536 }() 1537 } 1538 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1539 1540 // phase-1: eventually all the misbehavior reports should be processed. 1541 require.Eventually(t, func() bool { 1542 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1543 record, ok := cache.Get(originId) 1544 if !ok { 1545 return false 1546 } 1547 require.NotNil(t, record) 1548 1549 // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. 1550 if record.Penalty != report.Penalty()*float64(times) { 1551 return false 1552 } 1553 // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. 1554 require.Equal(t, uint64(0), record.CutoffCounter) 1555 // the decay should be the default decay value. 1556 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1557 1558 return true 1559 }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1560 1561 // phase-2: default decay speed is 1000 and with 10 penalties in range of [-1, -10], the penalty should be decayed to zero in 1562 // a single heartbeat. 1563 time.Sleep(1 * time.Second) 1564 1565 // phase-3: check if the penalty was decayed to zero. 1566 record, ok := cache.Get(originId) 1567 require.True(t, ok) // the record should be in the cache 1568 require.NotNil(t, record) 1569 1570 require.False(t, record.DisallowListed) // the peer should not be disallow listed. 1571 // with a single heartbeat and decay speed of 1000, the penalty should be decayed to zero. 1572 require.Equal(t, float64(0), record.Penalty) 1573 // the decay should be the default decay value. 1574 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1575 } 1576 1577 // TestDecayMisbehaviorPenalty_DecayToZero_AllowListing tests that when the misbehavior penalty of an already disallow-listed 1578 // peer is decayed to zero, the peer is allow-listed back in the network, and its spam record cache is updated accordingly. 1579 func TestDecayMisbehaviorPenalty_DecayToZero_AllowListing(t *testing.T) { 1580 cfg := managerCfgFixture(t) 1581 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1582 1583 var cache alsp.SpamRecordCache 1584 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1585 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1586 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1587 return cache 1588 }), 1589 } 1590 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1591 require.NoError(t, err) 1592 1593 // start the ALSP manager 1594 ctx, cancel := context.WithCancel(context.Background()) 1595 defer func() { 1596 cancel() 1597 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1598 }() 1599 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1600 m.Start(signalerCtx) 1601 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1602 1603 // simulates a disallow-listed peer in cache. 1604 originId := unittest.IdentifierFixture() 1605 penalty, err := cache.AdjustWithInit(originId, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { 1606 record.Penalty = -10 // set the penalty to -10 to simulate that the penalty has already been decayed for a while. 1607 record.CutoffCounter = 1 1608 record.DisallowListed = true 1609 record.OriginId = originId 1610 record.Decay = model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay 1611 return record, nil 1612 }) 1613 require.NoError(t, err) 1614 require.Equal(t, float64(-10), penalty) 1615 1616 // sanity check 1617 record, ok := cache.Get(originId) 1618 require.True(t, ok) // the record should be in the cache 1619 require.NotNil(t, record) 1620 require.Equal(t, float64(-10), record.Penalty) 1621 require.True(t, record.DisallowListed) 1622 require.Equal(t, uint64(1), record.CutoffCounter) 1623 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1624 1625 // eventually, we expect the ALSP manager to emit an allow list notification to the network layer when the penalty is decayed to zero. 1626 consumer.On("OnAllowListNotification", &network.AllowListingUpdate{ 1627 FlowIds: flow.IdentifierList{originId}, 1628 Cause: network.DisallowListedCauseAlsp, 1629 }).Return(nil).Once() 1630 1631 // wait for at most two heartbeats; default decay speed is 1000 and with a penalty of -10, the penalty should be decayed to zero in a single heartbeat. 1632 require.Eventually(t, func() bool { 1633 record, ok = cache.Get(originId) 1634 if !ok { 1635 t.Log("spam record not found in cache") 1636 return false 1637 } 1638 if record.DisallowListed { 1639 t.Logf("peer %s is still disallow-listed", originId) 1640 return false // the peer should not be allow-listed yet. 1641 } 1642 if record.Penalty != float64(0) { 1643 t.Log("penalty is not decayed to zero") 1644 return false // the penalty should be decayed to zero. 1645 } 1646 if record.CutoffCounter != 1 { 1647 t.Logf("cutoff counter is %d, expected 1", record.CutoffCounter) 1648 return false // the cutoff counter should be incremented. 1649 } 1650 if record.Decay != model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay { 1651 t.Logf("decay is %f, expected %f", record.Decay, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay) 1652 return false // the decay should be the default decay value. 1653 } 1654 1655 return true 1656 1657 }, 2*time.Second, 10*time.Millisecond, "penalty was not decayed to zero") 1658 } 1659 1660 // TestDisallowListNotification tests the emission of the allow list notification to the network layer when the misbehavior 1661 // penalty of a node is dropped below the disallow-listing threshold. The test ensures that the disallow list notification is 1662 // emitted to the network layer when the misbehavior penalty is dropped below the disallow-listing threshold and that the 1663 // cutoff counter of the spam record for the misbehaving node is incremented indicating that the node is disallow-listed once. 1664 func TestDisallowListNotification(t *testing.T) { 1665 cfg := managerCfgFixture(t) 1666 consumer := mocknetwork.NewDisallowListNotificationConsumer(t) 1667 1668 var cache alsp.SpamRecordCache 1669 cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ 1670 alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { 1671 cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) 1672 return cache 1673 }), 1674 } 1675 m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) 1676 require.NoError(t, err) 1677 1678 // start the ALSP manager 1679 ctx, cancel := context.WithCancel(context.Background()) 1680 defer func() { 1681 cancel() 1682 unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") 1683 }() 1684 signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) 1685 m.Start(signalerCtx) 1686 unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") 1687 1688 // creates a single misbehavior report 1689 originId := unittest.IdentifierFixture() 1690 report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) 1691 require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative 1692 1693 channel := channels.Channel("test-channel") 1694 1695 // reporting the same misbehavior 120 times, should result in a single disallow list notification, since each 1696 // misbehavior report is reported with the same penalty 0.01 * diallowlisting-threshold. We go over the threshold 1697 // to ensure that the disallow list notification is emitted only once. 1698 times := 120 1699 wg := sync.WaitGroup{} 1700 wg.Add(times) 1701 1702 // concurrently reports the same misbehavior report twice 1703 for i := 0; i < times; i++ { 1704 go func() { 1705 defer wg.Done() 1706 1707 m.HandleMisbehaviorReport(channel, report) 1708 }() 1709 } 1710 1711 // at this point, we expect a single disallow list notification to be emitted to the network layer when all the misbehavior 1712 // reports are processed by the ALSP manager (the notification is emitted when at the next heartbeat). 1713 consumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ 1714 FlowIds: flow.IdentifierList{report.OriginId()}, 1715 Cause: network.DisallowListedCauseAlsp, 1716 }).Return().Once() 1717 1718 unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") 1719 1720 require.Eventually(t, func() bool { 1721 // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache 1722 record, ok := cache.Get(originId) 1723 if !ok { 1724 return false 1725 } 1726 require.NotNil(t, record) 1727 1728 // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports (with the default decay). 1729 // the decay is added to the penalty as we allow for a single heartbeat before the disallow list notification is emitted. 1730 if record.Penalty != report.Penalty()*float64(times)+record.Decay { 1731 return false 1732 } 1733 require.True(t, record.DisallowListed) // the peer should be disallow-listed. 1734 // cutoff counter should be incremented since the penalty is above the disallow-listing threshold. 1735 require.Equal(t, uint64(1), record.CutoffCounter) 1736 // the decay should be the default decay value. 1737 require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) 1738 1739 return true 1740 }, 2*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") 1741 } 1742 1743 // //////////////////////////// TEST HELPERS /////////////////////////////////////////////////////////////////////////////// 1744 // The following functions are helpers for the tests. It wasn't feasible to put them in a helper file in the alspmgr_test 1745 // package because that would break encapsulation of the ALSP manager and require making some fields exportable. 1746 // Putting them in alspmgr package would cause a circular import cycle. Therefore, they are put in the internal test package here. 1747 1748 // createRandomMisbehaviorReportsForOriginId creates a slice of random misbehavior reports for a single origin id. 1749 // Args: 1750 // - t: the testing.T instance 1751 // - originID: the origin id of the misbehavior reports 1752 // - numReports: the number of misbehavior reports to create 1753 // Returns: 1754 // - []network.MisbehaviorReport: the slice of misbehavior reports 1755 // Note: the penalty of the misbehavior reports is randomly chosen between -1 and -10. 1756 func createRandomMisbehaviorReportsForOriginId(t *testing.T, originID flow.Identifier, numReports int) []network.MisbehaviorReport { 1757 reports := make([]network.MisbehaviorReport, numReports) 1758 1759 for i := 0; i < numReports; i++ { 1760 reports[i] = misbehaviorReportFixture(t, originID) 1761 } 1762 1763 return reports 1764 } 1765 1766 // createRandomMisbehaviorReports creates a slice of random misbehavior reports. 1767 // Args: 1768 // - t: the testing.T instance 1769 // - numReports: the number of misbehavior reports to create 1770 // Returns: 1771 // - []network.MisbehaviorReport: the slice of misbehavior reports 1772 // Note: the penalty of the misbehavior reports is randomly chosen between -1 and -10. 1773 func createRandomMisbehaviorReports(t *testing.T, numReports int) []network.MisbehaviorReport { 1774 reports := make([]network.MisbehaviorReport, numReports) 1775 1776 for i := 0; i < numReports; i++ { 1777 reports[i] = misbehaviorReportFixture(t, unittest.IdentifierFixture()) 1778 } 1779 1780 return reports 1781 } 1782 1783 // managerCfgFixture creates a new MisbehaviorReportManagerConfig with default values for testing. 1784 func managerCfgFixture(t *testing.T) *alspmgr.MisbehaviorReportManagerConfig { 1785 c, err := config.DefaultConfig() 1786 require.NoError(t, err) 1787 return &alspmgr.MisbehaviorReportManagerConfig{ 1788 Logger: unittest.Logger(), 1789 SpamRecordCacheSize: c.NetworkConfig.AlspConfig.SpamRecordCacheSize, 1790 SpamReportQueueSize: c.NetworkConfig.AlspConfig.SpamReportQueueSize, 1791 HeartBeatInterval: c.NetworkConfig.AlspConfig.HearBeatInterval, 1792 AlspMetrics: metrics.NewNoopCollector(), 1793 HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), 1794 } 1795 } 1796 1797 // misbehaviorReportFixture creates a mock misbehavior report for a single origin id. 1798 // Args: 1799 // - t: the testing.T instance 1800 // - originID: the origin id of the misbehavior report 1801 // Returns: 1802 // - network.MisbehaviorReport: the misbehavior report 1803 // Note: the penalty of the misbehavior report is randomly chosen between -1 and -10. 1804 func misbehaviorReportFixture(t *testing.T, originID flow.Identifier) network.MisbehaviorReport { 1805 return misbehaviorReportFixtureWithPenalty(t, originID, math.Min(-1, float64(-1-rand.Intn(10)))) 1806 } 1807 1808 func misbehaviorReportFixtureWithDefaultPenalty(t *testing.T, originID flow.Identifier) network.MisbehaviorReport { 1809 return misbehaviorReportFixtureWithPenalty(t, originID, model.DefaultPenaltyValue) 1810 } 1811 1812 func misbehaviorReportFixtureWithPenalty(t *testing.T, originID flow.Identifier, penalty float64) network.MisbehaviorReport { 1813 report := mocknetwork.NewMisbehaviorReport(t) 1814 report.On("OriginId").Return(originID) 1815 report.On("Reason").Return(alsp.AllMisbehaviorTypes()[rand.Intn(len(alsp.AllMisbehaviorTypes()))]) 1816 report.On("Penalty").Return(penalty) 1817 1818 return report 1819 }