github.com/ipni/storetheindex@v0.8.30/filestore/filestore_test.go (about)

     1  package filestore_test
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"io/fs"
     8  	"os"
     9  	"path/filepath"
    10  	"strings"
    11  	"testing"
    12  
    13  	"github.com/ipni/storetheindex/filestore"
    14  	"github.com/ipni/storetheindex/fsutil"
    15  	"github.com/stretchr/testify/require"
    16  
    17  	"github.com/orlangure/gnomock"
    18  	"github.com/orlangure/gnomock/preset/localstack"
    19  )
    20  
    21  const (
    22  	fileName  = "testfile.txt"
    23  	fileName1 = "testfile1.txt"
    24  	fileName2 = "testfile2.txt"
    25  	fileName3 = "abc/testfile3.txt"
    26  	subdir    = "abc"
    27  
    28  	data  = "hello world"
    29  	data1 = "foo"
    30  	data2 = "bar"
    31  	data3 = "baz"
    32  )
    33  
    34  func TestS3(t *testing.T) {
    35  	const bucketName = "testbucket"
    36  
    37  	tempDir := t.TempDir()
    38  	err := os.MkdirAll(fmt.Sprintf("%s/%s", tempDir, bucketName), 0755)
    39  	require.NoError(t, err)
    40  
    41  	p := localstack.Preset(
    42  		localstack.WithServices(localstack.S3),
    43  		localstack.WithS3Files(tempDir),
    44  	)
    45  	localS3, err := gnomock.Start(p)
    46  	if err != nil {
    47  		if strings.HasPrefix(err.Error(), "can't start container") {
    48  			t.Skip("Docker required for s3 tests")
    49  		}
    50  	}
    51  	require.NoError(t, err)
    52  	defer func() { _ = gnomock.Stop(localS3) }()
    53  
    54  	fileStore, err := filestore.NewS3(bucketName,
    55  		filestore.WithEndpoint(fmt.Sprintf("http://%s/", localS3.Address(localstack.APIPort))),
    56  		filestore.WithKeys("abcd1234", "1qaz2wsx"))
    57  	require.NoError(t, err)
    58  	require.Equal(t, "s3", fileStore.Type())
    59  
    60  	t.Run("test-S3-Put", func(t *testing.T) {
    61  		testPut(t, fileStore)
    62  	})
    63  
    64  	t.Run("test-S3-Head", func(t *testing.T) {
    65  		testHead(t, fileStore)
    66  	})
    67  
    68  	t.Run("test-S3-Get", func(t *testing.T) {
    69  		testGet(t, fileStore)
    70  	})
    71  
    72  	t.Run("test-S3-List", func(t *testing.T) {
    73  		testList(t, fileStore)
    74  	})
    75  
    76  	t.Run("test-S3-Delete", func(t *testing.T) {
    77  		testDelete(t, fileStore)
    78  	})
    79  }
    80  
    81  func TestLocal(t *testing.T) {
    82  	carDir := t.TempDir()
    83  
    84  	fileStore, err := filestore.NewLocal(carDir)
    85  	require.NoError(t, err)
    86  	require.Equal(t, "local", fileStore.Type())
    87  
    88  	t.Run("test-Local-Put", func(t *testing.T) {
    89  		testPut(t, fileStore)
    90  	})
    91  
    92  	require.True(t, fsutil.FileExists(filepath.Join(carDir, fileName)))
    93  
    94  	t.Run("test-Local-Head", func(t *testing.T) {
    95  		testHead(t, fileStore)
    96  	})
    97  
    98  	t.Run("test-Local-Get", func(t *testing.T) {
    99  		testGet(t, fileStore)
   100  	})
   101  
   102  	t.Run("test-Local-List", func(t *testing.T) {
   103  		testList(t, fileStore)
   104  	})
   105  
   106  	t.Run("test-Local-Delete", func(t *testing.T) {
   107  		testDelete(t, fileStore)
   108  	})
   109  }
   110  
   111  func TestMakeFilestore(t *testing.T) {
   112  	cfg := filestore.Config{
   113  		Type: "none",
   114  	}
   115  	fs, err := filestore.MakeFilestore(cfg)
   116  	require.NoError(t, err)
   117  	require.Nil(t, fs)
   118  
   119  	cfg.Type = "unknown"
   120  	_, err = filestore.MakeFilestore(cfg)
   121  	require.ErrorContains(t, err, "unsupported")
   122  
   123  	cfg.Type = ""
   124  	_, err = filestore.MakeFilestore(cfg)
   125  	require.ErrorContains(t, err, "not defined")
   126  
   127  	cfg.Type = "local"
   128  	_, err = filestore.MakeFilestore(cfg)
   129  	require.ErrorContains(t, err, "base path")
   130  
   131  	cfg.Local.BasePath = t.TempDir()
   132  	fs, err = filestore.MakeFilestore(cfg)
   133  	require.NoError(t, err)
   134  	require.NotNil(t, fs)
   135  }
   136  
   137  func testPut(t *testing.T, fileStore filestore.Interface) {
   138  	fileInfo, err := fileStore.Put(context.Background(), fileName, strings.NewReader(data))
   139  	require.NoError(t, err)
   140  	require.Equal(t, fileName, fileInfo.Path)
   141  	require.Equal(t, int64(len(data)), fileInfo.Size)
   142  }
   143  
   144  func testHead(t *testing.T, fileStore filestore.Interface) {
   145  	// Get file that does not exist.
   146  	fileInfo, err := fileStore.Head(context.Background(), "not-here")
   147  	require.ErrorIs(t, err, fs.ErrNotExist)
   148  	require.Nil(t, fileInfo)
   149  
   150  	_, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data))
   151  	require.NoError(t, err)
   152  
   153  	fileInfo, err = fileStore.Head(context.Background(), fileName3)
   154  	require.NoError(t, err)
   155  	require.Equal(t, fileName3, fileInfo.Path)
   156  	require.Equal(t, int64(len(data)), fileInfo.Size)
   157  	require.False(t, fileInfo.Modified.IsZero())
   158  
   159  	// Should get fs.ErrNotExist when looking for subdirectory.
   160  	_, err = fileStore.Head(context.Background(), subdir)
   161  	require.ErrorIs(t, err, fs.ErrNotExist)
   162  }
   163  
   164  func testGet(t *testing.T, fileStore filestore.Interface) {
   165  	// Get file that does not exist.
   166  	fileInfo, _, err := fileStore.Get(context.Background(), "not-here")
   167  	require.ErrorIs(t, err, fs.ErrNotExist)
   168  	require.Nil(t, fileInfo)
   169  
   170  	_, err = fileStore.Put(context.Background(), fileName, strings.NewReader(data))
   171  	require.NoError(t, err)
   172  
   173  	fileInfo, r, err := fileStore.Get(context.Background(), fileName)
   174  	require.NoError(t, err)
   175  	require.Equal(t, fileName, fileInfo.Path)
   176  	require.Equal(t, int64(len(data)), fileInfo.Size)
   177  	require.False(t, fileInfo.Modified.IsZero())
   178  
   179  	data2 := make([]byte, len(data))
   180  	n, err := r.Read(data2)
   181  	require.NoError(t, err)
   182  	require.Equal(t, len(data), n)
   183  	require.Equal(t, []byte(data), data2)
   184  	_, err = r.Read(data2)
   185  	require.ErrorIs(t, err, io.EOF)
   186  	require.NoError(t, r.Close())
   187  
   188  	_, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data3))
   189  	require.NoError(t, err)
   190  
   191  	_, _, err = fileStore.Get(context.Background(), subdir)
   192  	require.ErrorIs(t, err, fs.ErrNotExist)
   193  
   194  	fileInfo, r, err = fileStore.Get(context.Background(), fileName3)
   195  	require.NoError(t, err)
   196  	require.NoError(t, r.Close())
   197  	require.Equal(t, int64(len(data3)), fileInfo.Size)
   198  }
   199  
   200  func testList(t *testing.T, fileStore filestore.Interface) {
   201  	// List file that does not exist.
   202  	fileCh, errCh := fileStore.List(context.Background(), "not-here/", false)
   203  	fileInfo, ok := <-fileCh
   204  	require.Nil(t, fileInfo)
   205  	require.False(t, ok)
   206  	err := <-errCh
   207  	require.NoError(t, err)
   208  
   209  	_, err = fileStore.Put(context.Background(), fileName1, strings.NewReader(data1))
   210  	require.NoError(t, err)
   211  
   212  	_, err = fileStore.Put(context.Background(), fileName2, strings.NewReader(data2))
   213  	require.NoError(t, err)
   214  
   215  	_, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data3))
   216  	require.NoError(t, err)
   217  
   218  	fileCh, errCh = fileStore.List(context.Background(), "", false)
   219  	infos := make([]*filestore.File, 0, 3)
   220  	for fileInfo := range fileCh {
   221  		infos = append(infos, fileInfo)
   222  	}
   223  	err = <-errCh
   224  	require.NoError(t, err)
   225  	require.Equal(t, 3, len(infos))
   226  	expectNames := []string{fileName, fileName1, fileName2}
   227  	expectSizes := []int64{int64(len(data)), int64(len(data1)), int64(len(data2))}
   228  	for i := range infos {
   229  		require.Equal(t, expectNames[i], infos[i].Path)
   230  		require.Equal(t, expectSizes[i], infos[i].Size)
   231  		require.False(t, infos[0].Modified.IsZero())
   232  	}
   233  
   234  	fileCh, errCh = fileStore.List(context.Background(), "", true)
   235  	infos = infos[:0]
   236  	for fileInfo := range fileCh {
   237  		infos = append(infos, fileInfo)
   238  	}
   239  	err = <-errCh
   240  	require.NoError(t, err)
   241  	require.Equal(t, 4, len(infos))
   242  	require.Equal(t, fileName3, infos[0].Path)
   243  	require.Equal(t, int64(len(data3)), infos[0].Size)
   244  
   245  	// File specific file.
   246  	fileCh, errCh = fileStore.List(context.Background(), fileName1, false)
   247  	infos = infos[:0]
   248  	for fileInfo := range fileCh {
   249  		infos = append(infos, fileInfo)
   250  	}
   251  	err = <-errCh
   252  	require.NoError(t, err)
   253  	require.Equal(t, 1, len(infos))
   254  	require.Equal(t, fileName1, infos[0].Path)
   255  
   256  	// File specific file.
   257  	fileCh, errCh = fileStore.List(context.Background(), fileName3, false)
   258  	infos = infos[:0]
   259  	for fileInfo := range fileCh {
   260  		infos = append(infos, fileInfo)
   261  	}
   262  	err = <-errCh
   263  	require.NoError(t, err)
   264  	require.Equal(t, 1, len(infos))
   265  	require.Equal(t, fileName3, infos[0].Path)
   266  }
   267  
   268  func testDelete(t *testing.T, fileStore filestore.Interface) {
   269  	ctx := context.Background()
   270  
   271  	_, err := fileStore.Put(ctx, fileName1, strings.NewReader(data1))
   272  	require.NoError(t, err)
   273  
   274  	_, err = fileStore.Put(ctx, fileName2, strings.NewReader(data2))
   275  	require.NoError(t, err)
   276  
   277  	_, err = fileStore.Put(ctx, fileName3, strings.NewReader(data3))
   278  	require.NoError(t, err)
   279  
   280  	// File exists before delete.
   281  	_, err = fileStore.Head(ctx, fileName1)
   282  	require.NoError(t, err)
   283  
   284  	err = fileStore.Delete(ctx, fileName1)
   285  	require.NoError(t, err)
   286  
   287  	// File gone after delete.
   288  	_, err = fileStore.Head(ctx, fileName1)
   289  	require.ErrorIs(t, err, fs.ErrNotExist)
   290  
   291  	// Delete non-existant file should be OK.
   292  	err = fileStore.Delete(ctx, fileName1)
   293  	require.NoError(t, err)
   294  
   295  	err = fileStore.Delete(ctx, fileName2)
   296  	require.NoError(t, err)
   297  
   298  	err = fileStore.Delete(ctx, fileName3)
   299  	require.NoError(t, err)
   300  }