github.com/badrootd/celestia-core@v0.0.0-20240305091328-aa4207a4b25d/p2p/conn/connection_bench_test.go (about)

     1  package conn
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"math"
     7  	"net"
     8  	"sync"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/badrootd/celestia-core/libs/log"
    13  	"github.com/stretchr/testify/require"
    14  )
    15  
    16  const (
    17  	kibibyte = 1024
    18  	mebibyte = 1024 * 1024
    19  )
    20  
    21  // generateAndSendMessages sends a sequence of messages to the specified multiplex connection `mc`.
    22  // Each message has the given size and is sent at the specified rate
    23  // `messagingRate`. This process continues for the duration `totalDuration` or
    24  // until `totalNum` messages are sent. If `totalNum` is negative,
    25  // messaging persists for the entire `totalDuration`.
    26  func generateAndSendMessages(mc *MConnection,
    27  	messagingRate time.Duration,
    28  	totalDuration time.Duration, totalNum int, msgSize int,
    29  	msgContnet []byte, chID byte) {
    30  	var msg []byte
    31  	if msgContnet == nil {
    32  		msg = bytes.Repeat([]byte{'x'}, msgSize)
    33  	} else {
    34  		msg = msgContnet
    35  	}
    36  
    37  	// message generation interval ticker
    38  	ticker := time.NewTicker(messagingRate)
    39  	defer ticker.Stop()
    40  
    41  	// timer for the total duration
    42  	timer := time.NewTimer(totalDuration)
    43  	defer timer.Stop()
    44  
    45  	sentNum := 0
    46  	// generating messages
    47  	for {
    48  		select {
    49  		case <-ticker.C:
    50  			// generate message
    51  			if mc.Send(chID, msg) {
    52  				sentNum++
    53  				if totalNum > 0 && sentNum >= totalNum {
    54  					log.TestingLogger().Info("Completed the message generation as the" +
    55  						" total number of messages is reached")
    56  					return
    57  				}
    58  			}
    59  		case <-timer.C:
    60  			// time's up
    61  			log.TestingLogger().Info("Completed the message generation as the total " +
    62  				"duration is reached")
    63  			return
    64  		}
    65  	}
    66  }
    67  
    68  // sendMessages sends the supplied messages `msgs` to the specified multiplex
    69  // connection in order and according to the specified rate `messagingRate`.
    70  // chIDs is the list of channel IDs to which the messages are sent.
    71  // This process terminates after the duration `timeout` or when all
    72  // messages are sent.
    73  func sendMessages(mc *MConnection,
    74  	messagingRate time.Duration,
    75  	timeout time.Duration,
    76  	msgs [][]byte, chIDs []byte) {
    77  
    78  	var i = 0
    79  	total := len(msgs)
    80  	// message generation interval ticker
    81  	ticker := time.NewTicker(messagingRate)
    82  	defer ticker.Stop()
    83  
    84  	// timer for the total duration
    85  	timer := time.NewTimer(timeout)
    86  	defer timer.Stop()
    87  
    88  	// generating messages
    89  	for {
    90  		select {
    91  		case <-ticker.C:
    92  			// generate message
    93  			if mc.Send(chIDs[i], msgs[i]) {
    94  				log.TestingLogger().Info("Sent message ", i, " on channel ",
    95  					chIDs[i])
    96  				i++
    97  				if i >= total {
    98  					log.TestingLogger().Info("Completed the message generation as the" +
    99  						" total number of messages is reached")
   100  					return
   101  				}
   102  			}
   103  		case <-timer.C:
   104  			// time's up
   105  			log.TestingLogger().Info("Completed the message generation as the total " +
   106  				"duration is reached")
   107  			return
   108  		}
   109  	}
   110  }
   111  
   112  func BenchmarkMConnection(b *testing.B) {
   113  	chID := byte(0x01)
   114  
   115  	// Testcases 1-3 evaluate the effect of send queue capacity on message transmission delay.
   116  
   117  	// Testcases 3-5 assess the full utilization of maximum bandwidth by
   118  	// incrementally increasing the total load while keeping send and
   119  	// receive rates constant.
   120  	// The transmission time should be ~ totalMsg*msgSize/sendRate,
   121  	// indicating that the actual sendRate is in effect and has been
   122  	// utilized.
   123  
   124  	// Testcases 5-7 assess if surpassing available bandwidth maintains
   125  	// consistent transmission delay without congestion or performance
   126  	// degradation. A uniform delay across these testcases is expected.
   127  
   128  	tests := []struct {
   129  		name              string
   130  		msgSize           int           // size of each message in bytes
   131  		totalMsg          int           // total number of messages to be sent
   132  		messagingRate     time.Duration // rate at which messages are sent
   133  		totalDuration     time.Duration // total duration for which messages are sent
   134  		sendQueueCapacity int           // send queue capacity i.e., the number of messages that can be buffered
   135  		sendRate          int64         // send rate in bytes per second
   136  		recRate           int64         // receive rate in bytes per second
   137  	}{
   138  		{
   139  			// testcase 1
   140  			// Sends one 1KB message every 20ms, totaling 50 messages (50KB/s) per second.
   141  			// Ideal transmission time for 50 messages is ~1 second.
   142  			// SendQueueCapacity should not affect transmission delay.
   143  			name: "queue capacity = 1, " +
   144  				"total load = 50 KB, " +
   145  				"msg rate = send rate",
   146  			msgSize:           1 * kibibyte,
   147  			totalMsg:          1 * 50,
   148  			sendQueueCapacity: 1,
   149  			messagingRate:     20 * time.Millisecond,
   150  			totalDuration:     1 * time.Minute,
   151  			sendRate:          50 * kibibyte,
   152  			recRate:           50 * kibibyte,
   153  		},
   154  		{
   155  			// testcase 2
   156  			// Sends one 1KB message every 20ms, totaling 50 messages (50KB/s) per second.
   157  			// Ideal transmission time for 50 messages is ~1 second.
   158  			// Increasing SendQueueCapacity should not affect transmission
   159  			// delay.
   160  			name: "queue capacity = 50, " +
   161  				"total load = 50 KB, " +
   162  				"traffic rate = send rate",
   163  			msgSize:           1 * kibibyte,
   164  			totalMsg:          1 * 50,
   165  			sendQueueCapacity: 50,
   166  			messagingRate:     20 * time.Millisecond,
   167  			totalDuration:     1 * time.Minute,
   168  			sendRate:          50 * kibibyte,
   169  			recRate:           50 * kibibyte,
   170  		},
   171  		{
   172  			// testcase 3
   173  			// Sends one 1KB message every 20ms, totaling 50 messages (50KB/s) per second.
   174  			// Ideal transmission time for 50 messages is ~1 second.
   175  			// Increasing SendQueueCapacity should not affect transmission
   176  			// delay.
   177  			name: "queue capacity = 100, " +
   178  				"total load = 50 KB, " +
   179  				"traffic rate = send rate",
   180  			msgSize:           1 * kibibyte,
   181  			totalMsg:          1 * 50,
   182  			sendQueueCapacity: 100,
   183  			messagingRate:     20 * time.Millisecond,
   184  			totalDuration:     1 * time.Minute,
   185  			sendRate:          50 * kibibyte,
   186  			recRate:           50 * kibibyte,
   187  		},
   188  		{
   189  			// testcase 4
   190  			// Sends one 1KB message every 20ms, totaling 50 messages (50KB/s) per second.
   191  			// The test runs for 100 messages, expecting a total transmission time of ~2 seconds.
   192  			name: "queue capacity = 100, " +
   193  				"total load = 2 * 50 KB, " +
   194  				"traffic rate = send rate",
   195  			msgSize:           1 * kibibyte,
   196  			totalMsg:          2 * 50,
   197  			sendQueueCapacity: 100,
   198  			messagingRate:     20 * time.Millisecond,
   199  			totalDuration:     1 * time.Minute,
   200  			sendRate:          50 * kibibyte,
   201  			recRate:           50 * kibibyte,
   202  		},
   203  		{
   204  			// testcase 5
   205  			// Sends one 1KB message every 20ms, totaling 50 messages (50KB/s) per second.
   206  			// The test runs for 400 messages,
   207  			// expecting a total transmission time of ~8 seconds.
   208  			name: "queue capacity = 100, " +
   209  				"total load = 8 * 50 KB, " +
   210  				"traffic rate = send rate",
   211  			msgSize:           1 * kibibyte,
   212  			totalMsg:          8 * 50,
   213  			sendQueueCapacity: 100,
   214  			messagingRate:     20 * time.Millisecond,
   215  			totalDuration:     1 * time.Minute,
   216  			sendRate:          50 * kibibyte,
   217  			recRate:           50 * kibibyte,
   218  		},
   219  		{
   220  			// testcase 6
   221  			// Sends one 1KB message every 10ms, totaling 100 messages (100KB/s) per second.
   222  			// The test runs for 400 messages,
   223  			// expecting a total transmission time of ~8 seconds.
   224  			name: "queue capacity = 100, " +
   225  				"total load = 8 * 50 KB, " +
   226  				"traffic rate = 2 * send rate",
   227  			msgSize:           1 * kibibyte,
   228  			totalMsg:          8 * 50,
   229  			sendQueueCapacity: 100,
   230  			messagingRate:     10 * time.Millisecond,
   231  			totalDuration:     1 * time.Minute,
   232  			sendRate:          50 * kibibyte,
   233  			recRate:           50 * kibibyte,
   234  		},
   235  		{
   236  			// testcase 7
   237  			// Sends one 1KB message every 2ms, totaling 500 messages (500KB/s) per second.
   238  			// The test runs for 400 messages,
   239  			// expecting a total transmission time of ~8 seconds.
   240  			name: "queue capacity = 100, " +
   241  				"total load = 8 * 50 KB, " +
   242  				"traffic rate = 10 * send rate",
   243  			msgSize:           1 * kibibyte,
   244  			totalMsg:          8 * 50,
   245  			sendQueueCapacity: 100,
   246  			messagingRate:     2 * time.Millisecond,
   247  			totalDuration:     1 * time.Minute,
   248  			sendRate:          50 * kibibyte,
   249  			recRate:           50 * kibibyte,
   250  		},
   251  	}
   252  
   253  	for _, tt := range tests {
   254  		b.Run(tt.name, func(b *testing.B) {
   255  			for n := 0; n < b.N; n++ {
   256  				// set up two networked connections
   257  				// server, client := NetPipe() // can alternatively use this and comment out the line below
   258  				server, client := tcpNetPipe()
   259  				defer server.Close()
   260  				defer client.Close()
   261  
   262  				// prepare callback to receive messages
   263  				allReceived := make(chan bool)
   264  				receivedLoad := 0 // number of messages received
   265  				onReceive := func(chID byte, msgBytes []byte) {
   266  					receivedLoad++
   267  					if receivedLoad >= tt.totalMsg && tt.totalMsg > 0 {
   268  						allReceived <- true
   269  					}
   270  				}
   271  
   272  				cnfg := DefaultMConnConfig()
   273  				cnfg.SendRate = tt.sendRate
   274  				cnfg.RecvRate = tt.recRate
   275  				chDescs := []*ChannelDescriptor{{ID: chID, Priority: 1,
   276  					SendQueueCapacity: tt.sendQueueCapacity}}
   277  				clientMconn := NewMConnectionWithConfig(client, chDescs,
   278  					func(chID byte, msgBytes []byte) {},
   279  					func(r interface{}) {},
   280  					cnfg)
   281  				serverChDescs := []*ChannelDescriptor{{ID: chID, Priority: 1,
   282  					SendQueueCapacity: tt.sendQueueCapacity}}
   283  				serverMconn := NewMConnectionWithConfig(server, serverChDescs,
   284  					onReceive,
   285  					func(r interface{}) {},
   286  					cnfg)
   287  				clientMconn.SetLogger(log.TestingLogger())
   288  				serverMconn.SetLogger(log.TestingLogger())
   289  
   290  				err := clientMconn.Start()
   291  				require.Nil(b, err)
   292  				defer func() {
   293  					_ = clientMconn.Stop()
   294  				}()
   295  				err = serverMconn.Start()
   296  				require.Nil(b, err)
   297  				defer func() {
   298  					_ = serverMconn.Stop()
   299  				}()
   300  
   301  				// start measuring the time from here to exclude the time
   302  				// taken to set up the connections
   303  				b.StartTimer()
   304  				// start generating messages, it is a blocking call
   305  				generateAndSendMessages(clientMconn,
   306  					tt.messagingRate,
   307  					tt.totalDuration,
   308  					tt.totalMsg,
   309  					tt.msgSize,
   310  					nil, chID)
   311  
   312  				// wait for all messages to be received
   313  				<-allReceived
   314  				b.StopTimer()
   315  			}
   316  		})
   317  
   318  	}
   319  
   320  }
   321  
   322  // tcpNetPipe creates a pair of connected net.Conn objects that can be used in tests.
   323  func tcpNetPipe() (net.Conn, net.Conn) {
   324  	ln, _ := net.Listen("tcp", "127.0.0.1:0")
   325  	var conn1 net.Conn
   326  	var wg sync.WaitGroup
   327  	wg.Add(1)
   328  	go func(c *net.Conn) {
   329  		*c, _ = ln.Accept()
   330  		wg.Done()
   331  	}(&conn1)
   332  
   333  	addr := ln.Addr().String()
   334  	conn2, _ := net.Dial("tcp", addr)
   335  
   336  	wg.Wait()
   337  
   338  	return conn2, conn1
   339  }
   340  
   341  // generateExponentialSizedMessages creates and returns a series of messages
   342  // with sizes (in the specified unit) increasing exponentially.
   343  // The size of each message doubles, starting from 1 up to maxSizeBytes.
   344  // unit is expected to be a power of 2.
   345  func generateExponentialSizedMessages(maxSizeBytes int, unit int) [][]byte {
   346  	maxSizeToUnit := maxSizeBytes / unit
   347  	msgs := make([][]byte, 0)
   348  
   349  	for size := 1; size <= maxSizeToUnit; size *= 2 {
   350  		msgs = append(msgs, bytes.Repeat([]byte{'x'}, size*unit)) // create a message of the calculated size
   351  	}
   352  	return msgs
   353  }
   354  
   355  type testCase struct {
   356  	name              string
   357  	msgSize           int           // size of each message in bytes
   358  	msg               []byte        // message to be sent
   359  	totalMsg          int           // total number of messages to be sent
   360  	messagingRate     time.Duration // rate at which messages are sent
   361  	totalDuration     time.Duration // total duration for which messages are sent
   362  	sendQueueCapacity int           // send queue capacity i.e., the number of messages that can be buffered
   363  	sendRate          int64         // send rate in bytes per second
   364  	recRate           int64         // receive rate in bytes per second
   365  	chID              byte          // channel ID
   366  }
   367  
   368  func runBenchmarkTest(b *testing.B, tt testCase) {
   369  	b.Run(tt.name, func(b *testing.B) {
   370  		for n := 0; n < b.N; n++ {
   371  			// set up two networked connections
   372  			// server, client := NetPipe() // can alternatively use this and comment out the line below
   373  			server, client := tcpNetPipe()
   374  			defer server.Close()
   375  			defer client.Close()
   376  
   377  			// prepare callback to receive messages
   378  			allReceived := make(chan bool)
   379  			receivedLoad := 0 // number of messages received
   380  			onReceive := func(chID byte, msgBytes []byte) {
   381  				receivedLoad++
   382  				if receivedLoad >= tt.totalMsg && tt.totalMsg > 0 {
   383  					allReceived <- true
   384  				}
   385  			}
   386  
   387  			cnfg := DefaultMConnConfig()
   388  			cnfg.SendRate = tt.sendRate
   389  			cnfg.RecvRate = tt.recRate
   390  			chDescs := []*ChannelDescriptor{{ID: tt.chID, Priority: 1,
   391  				SendQueueCapacity: tt.sendQueueCapacity}}
   392  			clientMconn := NewMConnectionWithConfig(client, chDescs,
   393  				func(chID byte, msgBytes []byte) {},
   394  				func(r interface{}) {},
   395  				cnfg)
   396  			serverChDescs := []*ChannelDescriptor{{ID: tt.chID, Priority: 1,
   397  				SendQueueCapacity: tt.sendQueueCapacity}}
   398  			serverMconn := NewMConnectionWithConfig(server, serverChDescs,
   399  				onReceive,
   400  				func(r interface{}) {},
   401  				cnfg)
   402  			clientMconn.SetLogger(log.TestingLogger())
   403  			serverMconn.SetLogger(log.TestingLogger())
   404  
   405  			err := clientMconn.Start()
   406  			require.Nil(b, err)
   407  			defer func() {
   408  				_ = clientMconn.Stop()
   409  			}()
   410  			err = serverMconn.Start()
   411  			require.Nil(b, err)
   412  			defer func() {
   413  				_ = serverMconn.Stop()
   414  			}()
   415  
   416  			// start measuring the time from here to exclude the time
   417  			// taken to set up the connections
   418  			b.StartTimer()
   419  			// start generating messages, it is a blocking call
   420  			generateAndSendMessages(clientMconn,
   421  				tt.messagingRate,
   422  				tt.totalDuration,
   423  				tt.totalMsg,
   424  				tt.msgSize,
   425  				tt.msg,
   426  				tt.chID)
   427  
   428  			// wait for all messages to be received
   429  			<-allReceived
   430  			b.StopTimer()
   431  		}
   432  	})
   433  }
   434  
   435  func BenchmarkMConnection_ScalingPayloadSizes_HighSendRate(b *testing.B) {
   436  	// One aspect that could impact the performance of MConnection and the
   437  	// transmission rate is the size of the messages sent over the network,
   438  	// especially when they exceed the MConnection.MaxPacketMsgPayloadSize (
   439  	// messages are sent in packets of maximum size MConnection.
   440  	// MaxPacketMsgPayloadSize).
   441  	// The test cases in this benchmark involve sending messages with sizes
   442  	// ranging exponentially from 1KB to 8192KB (
   443  	// the max value of 8192KB is inspired by the largest possible PFB in a
   444  	// Celestia block with 128*128 number of 512-byte shares)
   445  	// The bandwidth is set significantly higher than the message load to ensure
   446  	// it does not become a limiting factor.
   447  	// All test cases are expected to complete in less than one second,
   448  	// indicating a healthy performance.
   449  
   450  	squareSize := 128                              // number of shares in a row/column
   451  	shareSize := 512                               // bytes
   452  	maxSize := squareSize * squareSize * shareSize // bytes
   453  	msgs := generateExponentialSizedMessages(maxSize, kibibyte)
   454  	chID := byte(0x01)
   455  
   456  	// create test cases for each message size
   457  	var testCases = make([]testCase, len(msgs))
   458  	for i, msg := range msgs {
   459  		testCases[i] = testCase{
   460  			name:              fmt.Sprintf("msgSize = %d KB", len(msg)/kibibyte),
   461  			msgSize:           len(msg),
   462  			msg:               msg,
   463  			totalMsg:          10,
   464  			messagingRate:     time.Millisecond,
   465  			totalDuration:     1 * time.Minute,
   466  			sendQueueCapacity: 100,
   467  			sendRate:          512 * mebibyte,
   468  			recRate:           512 * mebibyte,
   469  			chID:              chID,
   470  		}
   471  	}
   472  
   473  	for _, tt := range testCases {
   474  		runBenchmarkTest(b, tt)
   475  	}
   476  }
   477  
   478  func BenchmarkMConnection_ScalingPayloadSizes_LowSendRate(b *testing.B) {
   479  	// This benchmark test builds upon the previous one i.e.,
   480  	// BenchmarkMConnection_ScalingPayloadSizes_HighSendRate
   481  	// by setting the send/and receive rates lower than the message load.
   482  	// Test cases involve sending the same load of messages but with different message sizes.
   483  	// Since the message load and bandwidth are consistent across all test cases,
   484  	// they are expected to complete in the same amount of time. i.e.,
   485  	//totalLoad/sendRate.
   486  
   487  	maxSize := 32 * kibibyte // 32KB
   488  	msgs := generateExponentialSizedMessages(maxSize, kibibyte)
   489  	totalLoad := float64(maxSize)
   490  	chID := byte(0x01)
   491  	// create test cases for each message size
   492  	var testCases = make([]testCase, len(msgs))
   493  	for i, msg := range msgs {
   494  		msgSize := len(msg)
   495  		totalMsg := int(math.Ceil(totalLoad / float64(msgSize)))
   496  		testCases[i] = testCase{
   497  			name:              fmt.Sprintf("msgSize = %d KB", msgSize/kibibyte),
   498  			msgSize:           msgSize,
   499  			msg:               msg,
   500  			totalMsg:          totalMsg,
   501  			messagingRate:     time.Millisecond,
   502  			totalDuration:     1 * time.Minute,
   503  			sendQueueCapacity: 100,
   504  			sendRate:          4 * kibibyte,
   505  			recRate:           4 * kibibyte,
   506  			chID:              chID,
   507  		}
   508  	}
   509  
   510  	for _, tt := range testCases {
   511  		runBenchmarkTest(b, tt)
   512  	}
   513  }
   514  
   515  // BenchmarkMConnection_Multiple_ChannelID assesses the max bw/send rate
   516  // utilization of MConnection when configured with multiple channel IDs.
   517  func BenchmarkMConnection_Multiple_ChannelID(b *testing.B) {
   518  	// These tests create two connections with two channels each, one channel having higher priority.
   519  	// Traffic is sent from the client to the server, split between the channels with varying proportions in each test case.
   520  	// All tests should complete in 2 seconds (calculated as totalTraffic* msgSize / sendRate),
   521  	// demonstrating that channel count doesn't affect max bandwidth utilization.
   522  	totalTraffic := 100
   523  	msgSize := 1 * kibibyte
   524  	sendRate := 50 * kibibyte
   525  	recRate := 50 * kibibyte
   526  	chDescs := []*ChannelDescriptor{
   527  		{ID: 0x01, Priority: 1, SendQueueCapacity: 50},
   528  		{ID: 0x02, Priority: 2, SendQueueCapacity: 50},
   529  	}
   530  	type testCase struct {
   531  		name       string
   532  		trafficMap map[byte]int // channel ID -> number of messages to be sent
   533  	}
   534  	var tests []testCase
   535  	for i := 0.0; i < 1; i += 0.1 {
   536  		tests = append(tests, testCase{
   537  			name: fmt.Sprintf("2 channel IDs with traffic proportion %f %f",
   538  				i, 1-i),
   539  			trafficMap: map[byte]int{ // channel ID -> number of messages to be sent
   540  				0x01: int(i * float64(totalTraffic)),
   541  				0x02: totalTraffic - int(i*float64(totalTraffic)), // the rest of the traffic
   542  			},
   543  		})
   544  	}
   545  	for _, tt := range tests {
   546  		b.Run(tt.name, func(b *testing.B) {
   547  			for n := 0; n < b.N; n++ {
   548  				// set up two networked connections
   549  				// server, client := NetPipe() // can alternatively use this and comment out the line below
   550  				server, client := tcpNetPipe()
   551  				defer server.Close()
   552  				defer client.Close()
   553  
   554  				// prepare callback to receive messages
   555  				allReceived := make(chan bool)
   556  				receivedLoad := 0 // number of messages received
   557  				onReceive := func(chID byte, msgBytes []byte) {
   558  					receivedLoad++
   559  					if receivedLoad >= totalTraffic {
   560  						allReceived <- true
   561  					}
   562  				}
   563  
   564  				cnfg := DefaultMConnConfig()
   565  				cnfg.SendRate = int64(sendRate)
   566  				cnfg.RecvRate = int64(recRate)
   567  
   568  				// mount the channel descriptors to the connections
   569  				clientMconn := NewMConnectionWithConfig(client, chDescs,
   570  					func(chID byte, msgBytes []byte) {},
   571  					func(r interface{}) {},
   572  					cnfg)
   573  				serverMconn := NewMConnectionWithConfig(server, chDescs,
   574  					onReceive,
   575  					func(r interface{}) {},
   576  					cnfg)
   577  				clientMconn.SetLogger(log.TestingLogger())
   578  				serverMconn.SetLogger(log.TestingLogger())
   579  
   580  				err := clientMconn.Start()
   581  				require.Nil(b, err)
   582  				defer func() {
   583  					_ = clientMconn.Stop()
   584  				}()
   585  				err = serverMconn.Start()
   586  				require.Nil(b, err)
   587  				defer func() {
   588  					_ = serverMconn.Stop()
   589  				}()
   590  
   591  				// start measuring the time from here to exclude the time
   592  				// taken to set up the connections
   593  				b.StartTimer()
   594  				// start generating messages over the two channels,
   595  				// concurrently, with the specified proportions
   596  				for chID, trafficPortion := range tt.trafficMap {
   597  					go generateAndSendMessages(clientMconn,
   598  						time.Millisecond,
   599  						1*time.Minute,
   600  						trafficPortion,
   601  						msgSize,
   602  						nil,
   603  						chID)
   604  				}
   605  
   606  				// wait for all messages to be received
   607  				<-allReceived
   608  				b.StopTimer()
   609  			}
   610  		})
   611  	}
   612  }
   613  
   614  func TestMConnection_Message_Order_ChannelID(t *testing.T) {
   615  	// This test involves two connections, each with two channels:
   616  	// channel ID 1 (high priority) and channel ID 2 (low priority).
   617  	// It sends 11 messages from client to server, with the first 10 on
   618  	// channel ID 2 and the final one on channel ID 1.
   619  	// The aim is to show that message order at the receiver is based solely
   620  	// on the send order, as the receiver does not prioritize channels.
   621  	// To enforce a specific send order,
   622  	// channel ID 2's send queue capacity is limited to 1;
   623  	// preventing message queuing on channel ID 2 that could otherwise give
   624  	// priority to channel ID 1's message (despite being the last message),
   625  	// disrupting the send order.
   626  	totalMsgs := 11
   627  	msgSize := 1 * kibibyte
   628  	sendRate := 50 * kibibyte
   629  	recRate := 50 * kibibyte
   630  	clientChDesc := []*ChannelDescriptor{
   631  		{ID: 0x01, Priority: 1, SendQueueCapacity: 10,
   632  			RecvMessageCapacity: defaultRecvMessageCapacity,
   633  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   634  		{ID: 0x02, Priority: 2,
   635  			// channel ID 2's send queue capacity is limited to 1;
   636  			// to enforce a specific send order.
   637  			SendQueueCapacity:   1,
   638  			RecvMessageCapacity: defaultRecvMessageCapacity,
   639  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   640  	}
   641  	serverChDesc := []*ChannelDescriptor{
   642  		{ID: 0x01, Priority: 1, SendQueueCapacity: 50,
   643  			RecvMessageCapacity: defaultRecvMessageCapacity,
   644  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   645  		{ID: 0x02, Priority: 2, SendQueueCapacity: 50,
   646  			RecvMessageCapacity: defaultRecvMessageCapacity,
   647  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   648  	}
   649  
   650  	// prepare messages and channel IDs
   651  	// 10 messages on channel ID 2 and 1 message on channel ID 1
   652  	msgs := make([][]byte, totalMsgs)
   653  	chIDs := make([]byte, totalMsgs)
   654  	for i := 0; i < totalMsgs-1; i++ {
   655  		msg := bytes.Repeat([]byte{'x'}, msgSize)
   656  		msgs[i] = msg
   657  		chIDs[i] = 0x02
   658  	}
   659  	msgs[totalMsgs-1] = bytes.Repeat([]byte{'y'}, msgSize)
   660  	chIDs[totalMsgs-1] = 0x01
   661  
   662  	// set up two networked connections
   663  	// server, client := NetPipe() // can alternatively use this and comment out the line below
   664  	server, client := tcpNetPipe()
   665  	defer server.Close()
   666  	defer client.Close()
   667  
   668  	// prepare callback to receive messages
   669  	allReceived := make(chan bool)
   670  	received := 0                        // number of messages received
   671  	recvChIds := make([]byte, totalMsgs) // keep track of the order of channel IDs of received messages
   672  	onReceive := func(chID byte, msgBytes []byte) {
   673  		// wait for 100ms to simulate processing time
   674  		// Also, the added delay allows the receiver to buffer all 11 messages,
   675  		// testing if the message on channel ID 1 (high priority) is received last or
   676  		// prioritized among the 10 messages on channel ID 2.
   677  		time.Sleep(100 * time.Millisecond)
   678  		recvChIds[received] = chID
   679  		received++
   680  		if received >= totalMsgs {
   681  			allReceived <- true
   682  		}
   683  	}
   684  
   685  	cnfg := DefaultMConnConfig()
   686  	cnfg.SendRate = int64(sendRate)
   687  	cnfg.RecvRate = int64(recRate)
   688  
   689  	// mount the channel descriptors to the connections
   690  	clientMconn := NewMConnectionWithConfig(client, clientChDesc,
   691  		func(chID byte, msgBytes []byte) {},
   692  		func(r interface{}) {},
   693  		cnfg)
   694  	serverMconn := NewMConnectionWithConfig(server, serverChDesc,
   695  		onReceive,
   696  		func(r interface{}) {},
   697  		cnfg)
   698  	clientMconn.SetLogger(log.TestingLogger())
   699  	serverMconn.SetLogger(log.TestingLogger())
   700  
   701  	err := clientMconn.Start()
   702  	require.Nil(t, err)
   703  	defer func() {
   704  		_ = clientMconn.Stop()
   705  	}()
   706  	err = serverMconn.Start()
   707  	require.Nil(t, err)
   708  	defer func() {
   709  		_ = serverMconn.Stop()
   710  	}()
   711  
   712  	go sendMessages(clientMconn,
   713  		time.Millisecond,
   714  		1*time.Minute,
   715  		msgs, chIDs)
   716  
   717  	// wait for all messages to be received
   718  	<-allReceived
   719  
   720  	require.Equal(t, chIDs, recvChIds) // assert that the order of received messages is the same as the order of sent messages
   721  }
   722  
   723  func TestMConnection_Failing_Large_Messages(t *testing.T) {
   724  	// This test evaluates how MConnection handles messages exceeding channel
   725  	// ID's receive message capacity i.e., `RecvMessageCapacity`.
   726  	// It involves two connections, each with two channels: Channel ID 1 (
   727  	// capacity 1024 bytes) and Channel ID 2 (capacity 1023 bytes).
   728  	// All the other channel ID's and MConnection's configurations are set high
   729  	// enough to not be a limiting factor.
   730  	// A 1KB message is sent over the first and second channels in succession.
   731  	// Message on Channel ID 1 (capacity equal to message size) is received,
   732  	// while the message on Channel ID 2 (capacity less than message size) is dropped.
   733  
   734  	totalMsgs := 2
   735  	msgSize := 1 * kibibyte
   736  	sendRate := 50 * kibibyte
   737  	recRate := 50 * kibibyte
   738  	chDesc := []*ChannelDescriptor{
   739  		{ID: 0x01, Priority: 1, SendQueueCapacity: 50,
   740  			RecvMessageCapacity: msgSize,
   741  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   742  		{ID: 0x02, Priority: 1, SendQueueCapacity: 50,
   743  			RecvMessageCapacity: msgSize - 1,
   744  			RecvBufferCapacity:  defaultRecvBufferCapacity},
   745  	}
   746  
   747  	// prepare messages and channel IDs
   748  	// 1 message on channel ID 1 and 1 message on channel ID 2
   749  	msgs := make([][]byte, totalMsgs)
   750  	chIDs := make([]byte, totalMsgs)
   751  	msgs[0] = bytes.Repeat([]byte{'x'}, msgSize)
   752  	chIDs[0] = 0x01
   753  	msgs[1] = bytes.Repeat([]byte{'y'}, msgSize)
   754  	chIDs[1] = 0x02
   755  
   756  	// set up two networked connections
   757  	// server, client := NetPipe() // can alternatively use this and comment out the line below
   758  	server, client := tcpNetPipe()
   759  	defer server.Close()
   760  	defer client.Close()
   761  
   762  	// prepare callback to receive messages
   763  	allReceived := make(chan bool)
   764  	recvChIds := make(chan byte, totalMsgs)
   765  	onReceive := func(chID byte, msgBytes []byte) {
   766  		recvChIds <- chID
   767  		if len(recvChIds) >= totalMsgs {
   768  			allReceived <- true
   769  		}
   770  	}
   771  
   772  	cnfg := DefaultMConnConfig()
   773  	cnfg.SendRate = int64(sendRate)
   774  	cnfg.RecvRate = int64(recRate)
   775  
   776  	// mount the channel descriptors to the connections
   777  	clientMconn := NewMConnectionWithConfig(client, chDesc,
   778  		func(chID byte, msgBytes []byte) {},
   779  		func(r interface{}) {},
   780  		cnfg)
   781  	serverMconn := NewMConnectionWithConfig(server, chDesc,
   782  		onReceive,
   783  		func(r interface{}) {},
   784  		cnfg)
   785  	clientMconn.SetLogger(log.TestingLogger())
   786  	serverMconn.SetLogger(log.TestingLogger())
   787  
   788  	err := clientMconn.Start()
   789  	require.Nil(t, err)
   790  	defer func() {
   791  		_ = clientMconn.Stop()
   792  	}()
   793  	err = serverMconn.Start()
   794  	require.Nil(t, err)
   795  	defer func() {
   796  		_ = serverMconn.Stop()
   797  	}()
   798  
   799  	// start sending messages
   800  	go sendMessages(clientMconn,
   801  		time.Millisecond,
   802  		1*time.Second,
   803  		msgs, chIDs)
   804  
   805  	// wait for messages to be received
   806  	select {
   807  	case <-allReceived:
   808  		require.Fail(t, "All messages should not have been received") // the message sent
   809  		// on channel ID 2 should have been dropped
   810  	case <-time.After(500 * time.Millisecond):
   811  		require.Equal(t, 1, len(recvChIds))
   812  		require.Equal(t, chIDs[0], <-recvChIds)   // the first message should be received
   813  		require.True(t, !serverMconn.IsRunning()) // the serverMconn should have stopped due to the error
   814  	}
   815  }