github.com/grailbio/base@v0.0.11/file/internal/readmatcher/readmatcher_test.go (about)

     1  package readmatcher_test
     2  
     3  import (
     4  	"bytes"
     5  	"flag"
     6  	"io"
     7  	"math/rand"
     8  	"os"
     9  	"path"
    10  	"runtime"
    11  	"testing"
    12  
    13  	"github.com/grailbio/base/file/fsnode"
    14  	"github.com/grailbio/base/file/fsnodefuse"
    15  	"github.com/grailbio/base/file/internal/readmatcher"
    16  	"github.com/grailbio/base/file/internal/readmatcher/readmatchertest"
    17  	"github.com/grailbio/base/ioctx"
    18  	"github.com/grailbio/base/log"
    19  	"github.com/grailbio/base/must"
    20  	"github.com/grailbio/testutil"
    21  	"github.com/grailbio/testutil/assert"
    22  	"github.com/hanwen/go-fuse/v2/fs"
    23  	"github.com/hanwen/go-fuse/v2/fuse"
    24  	"github.com/stretchr/testify/require"
    25  )
    26  
    27  var (
    28  	dataBytes         = flag.Int("data-bytes", 1<<27, "read corpus size")
    29  	stressParallelism = flag.Int("stress-parallelism", runtime.NumCPU(),
    30  		"number of parallel readers during stress test")
    31  	fuseFlag = flag.Bool("fuse", false, "create a temporary FUSE mount and test through that")
    32  )
    33  
    34  func TestStress(t *testing.T) {
    35  	var data = make([]byte, *dataBytes)
    36  	_, _ = rand.New(rand.NewSource(1)).Read(data)
    37  	offsetReader := func(start int64) ioctx.ReadCloser {
    38  		return ioctx.FromStdReadCloser(io.NopCloser(bytes.NewReader(data[start:])))
    39  	}
    40  	type fuseCase struct {
    41  		name string
    42  		test func(*testing.T, ioctx.ReaderAt)
    43  	}
    44  	fuseCases := []fuseCase{
    45  		{
    46  			"nofuse",
    47  			func(t *testing.T, r ioctx.ReaderAt) {
    48  				readmatchertest.Stress(data, r, *stressParallelism)
    49  			},
    50  		},
    51  	}
    52  	if *fuseFlag {
    53  		fuseCases = append(fuseCases, fuseCase{
    54  			"fuse",
    55  			func(t *testing.T, rAt ioctx.ReaderAt) {
    56  				mountPoint, cleanUpMountPoint := testutil.TempDir(t, "", "readmatcher_test")
    57  				defer cleanUpMountPoint()
    58  				const filename = "data"
    59  				server, err := fs.Mount(
    60  					mountPoint,
    61  					fsnodefuse.NewRoot(fsnode.NewParent(
    62  						fsnode.NewDirInfo("root"),
    63  						fsnode.ConstChildren(
    64  							fsnode.ReaderAtLeaf(
    65  								fsnode.NewRegInfo(filename).WithSize(int64(len(data))),
    66  								rAt,
    67  							),
    68  						),
    69  					)),
    70  					&fs.Options{
    71  						MountOptions: func() fuse.MountOptions {
    72  							opts := fuse.MountOptions{FsName: "test", Debug: log.At(log.Debug)}
    73  							fsnodefuse.ConfigureRequiredMountOptions(&opts)
    74  							fsnodefuse.ConfigureDefaultMountOptions(&opts)
    75  							return opts
    76  						}(),
    77  					},
    78  				)
    79  				require.NoError(t, err, "mounting %q", mountPoint)
    80  				defer func() {
    81  					log.Printf("unmounting %q", mountPoint)
    82  					assert.NoError(t, server.Unmount(),
    83  						"unmount of FUSE mounted at %q failed; may need manual cleanup",
    84  						mountPoint,
    85  					)
    86  					log.Printf("unmounted %q", mountPoint)
    87  				}()
    88  				f, err := os.Open(path.Join(mountPoint, filename))
    89  				require.NoError(t, err)
    90  				defer func() { require.NoError(t, f.Close()) }()
    91  				readmatchertest.Stress(data, ioctx.FromStdReaderAt(f), *stressParallelism)
    92  			},
    93  		})
    94  	}
    95  	for _, c := range fuseCases {
    96  		t.Run(c.name, func(t *testing.T) {
    97  			t.Run("less parallelism", func(t *testing.T) {
    98  				readerParallelism := *stressParallelism / 2
    99  				must.True(readerParallelism > 0)
   100  				m := readmatcher.New(offsetReader, readmatcher.SoftMaxReaders(readerParallelism))
   101  				c.test(t, m)
   102  			})
   103  			t.Run("more parallelism", func(t *testing.T) {
   104  				readerParallelism := 2 * *stressParallelism
   105  				m := readmatcher.New(offsetReader, readmatcher.SoftMaxReaders(readerParallelism))
   106  				c.test(t, m)
   107  			})
   108  		})
   109  	}
   110  }