storj.io/uplink@v1.13.0/private/storage/streams/splitter/base_splitter_test.go (about)

     1  // Copyright (C) 2023 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package splitter
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"io"
    10  	"testing"
    11  
    12  	"github.com/stretchr/testify/require"
    13  	"github.com/zeebo/errs"
    14  
    15  	"storj.io/common/memory"
    16  	"storj.io/uplink/private/storage/streams/buffer"
    17  )
    18  
    19  func TestBaseSplitter(t *testing.T) {
    20  	ctx := context.Background()
    21  
    22  	const (
    23  		split   = 20
    24  		minimum = 10
    25  	)
    26  
    27  	canceled := errs.New("canceled")
    28  
    29  	type result struct {
    30  		kind   string
    31  		amount int64
    32  		err    error
    33  	}
    34  
    35  	readResult := func(splitter *baseSplitter) (res result, ok bool) {
    36  		buf := buffer.New(buffer.NewMemoryBackend(splitter.split), 10)
    37  
    38  		inline, eof, err := splitter.Next(ctx, buf)
    39  		if err != nil {
    40  			return result{"error", 0, err}, false
    41  		} else if eof {
    42  			return result{"done", 0, nil}, false
    43  		}
    44  
    45  		if inline != nil {
    46  			return result{"inline", int64(len(inline)), nil}, true
    47  		}
    48  
    49  		amount, err := io.Copy(io.Discard, buf.Reader())
    50  		buf.DoneReading(nil)
    51  
    52  		// we get a nondeterministic number of bytes read if there was an error so
    53  		// zero it so that the tests never fail
    54  		if err != nil {
    55  			amount = 0
    56  		}
    57  
    58  		return result{"buffer", amount, err}, err == nil
    59  	}
    60  
    61  	type test struct {
    62  		name    string
    63  		write   int64
    64  		finish  error
    65  		results []result
    66  	}
    67  
    68  	var cases = []test{
    69  		{"Basic", 45, nil, []result{
    70  			{"buffer", 20, nil},
    71  			{"buffer", 20, nil},
    72  			{"inline", 5, nil},
    73  			{"done", 0, nil},
    74  		}},
    75  
    76  		{"Aligned", 40, nil, []result{
    77  			{"buffer", 20, nil},
    78  			{"buffer", 20, nil},
    79  			{"done", 0, nil},
    80  		}},
    81  
    82  		{"Inline", 5, nil, []result{
    83  			{"inline", 5, nil},
    84  			{"done", 0, nil},
    85  		}},
    86  
    87  		{"Inline_Aligned", 10, nil, []result{
    88  			{"inline", 10, nil},
    89  			{"done", 0, nil},
    90  		}},
    91  
    92  		{"Zero", 0, nil, []result{
    93  			{"inline", 0, nil},
    94  			{"done", 0, nil},
    95  		}},
    96  
    97  		{"Error_Inline", 45, canceled, []result{
    98  			{"buffer", 20, nil},
    99  			{"buffer", 20, nil},
   100  			{"error", 0, canceled},
   101  		}},
   102  
   103  		{"Error_Buffer", 55, canceled, []result{
   104  			{"buffer", 20, nil},
   105  			{"buffer", 20, nil},
   106  			{"buffer", 0, canceled},
   107  		}},
   108  
   109  		{"Error_Aligned", 30, canceled, []result{
   110  			{"buffer", 20, nil},
   111  			{"error", 0, canceled},
   112  		}},
   113  	}
   114  
   115  	for _, tc := range cases {
   116  		tc := tc
   117  		t.Run(tc.name, func(t *testing.T) {
   118  			splitter := newBaseSplitter(split, minimum)
   119  			go func() {
   120  				n, err := io.CopyN(randomWriter{splitter}, emptyReader{}, tc.write)
   121  				splitter.Finish(tc.finish)
   122  				if n != tc.write || err != nil {
   123  					panic(fmt.Sprintln("not enough bytes written or error:", n, tc.write, err))
   124  				}
   125  			}()
   126  
   127  			var results []result
   128  			for {
   129  				res, ok := readResult(splitter)
   130  				results = append(results, res)
   131  				if !ok {
   132  					break
   133  				}
   134  			}
   135  			require.Equal(t, tc.results, results)
   136  		})
   137  	}
   138  }
   139  
   140  func BenchmarkBaseSplitter(b *testing.B) {
   141  	ctx := context.Background()
   142  
   143  	const (
   144  		minimum = 4 << 10
   145  		split   = 64 << 20
   146  	)
   147  
   148  	run := func(b *testing.B, size int) {
   149  		b.SetBytes(int64(size))
   150  		b.ReportAllocs()
   151  		b.ResetTimer()
   152  
   153  		for i := 0; i < b.N; i++ {
   154  			splitter := newBaseSplitter(split, minimum)
   155  
   156  			go func() {
   157  				_, _ = io.Copy(splitter, &emptyLimitReader{size})
   158  				splitter.Finish(nil)
   159  			}()
   160  
   161  			for {
   162  				buf := buffer.New(buffer.NewMemoryBackend(minimum), minimum)
   163  				inline, eof, err := splitter.Next(ctx, buf)
   164  				require.NoError(b, err)
   165  				if eof {
   166  					buf.DoneReading(nil)
   167  					buf.DoneWriting(nil)
   168  					break
   169  				}
   170  				if inline == nil {
   171  					_, _ = io.Copy(io.Discard, buf.Reader())
   172  					buf.DoneReading(nil)
   173  				}
   174  			}
   175  		}
   176  	}
   177  
   178  	sizes := []memory.Size{
   179  		1 * memory.KiB,
   180  		4 * memory.KiB,
   181  		256 * memory.KiB,
   182  		1 * memory.MiB,
   183  		4 * memory.MiB,
   184  		16 * memory.MiB,
   185  		64 * memory.MiB,
   186  		256 * memory.MiB,
   187  		512 * memory.MiB,
   188  		1 * memory.GiB,
   189  	}
   190  
   191  	for _, size := range sizes {
   192  		b.Run(size.String(), func(b *testing.B) {
   193  			run(b, size.Int())
   194  		})
   195  	}
   196  }