github.com/ipni/storetheindex@v0.8.30/filestore/filestore_test.go (about) 1 package filestore_test 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/fs" 8 "os" 9 "path/filepath" 10 "strings" 11 "testing" 12 13 "github.com/ipni/storetheindex/filestore" 14 "github.com/ipni/storetheindex/fsutil" 15 "github.com/stretchr/testify/require" 16 17 "github.com/orlangure/gnomock" 18 "github.com/orlangure/gnomock/preset/localstack" 19 ) 20 21 const ( 22 fileName = "testfile.txt" 23 fileName1 = "testfile1.txt" 24 fileName2 = "testfile2.txt" 25 fileName3 = "abc/testfile3.txt" 26 subdir = "abc" 27 28 data = "hello world" 29 data1 = "foo" 30 data2 = "bar" 31 data3 = "baz" 32 ) 33 34 func TestS3(t *testing.T) { 35 const bucketName = "testbucket" 36 37 tempDir := t.TempDir() 38 err := os.MkdirAll(fmt.Sprintf("%s/%s", tempDir, bucketName), 0755) 39 require.NoError(t, err) 40 41 p := localstack.Preset( 42 localstack.WithServices(localstack.S3), 43 localstack.WithS3Files(tempDir), 44 ) 45 localS3, err := gnomock.Start(p) 46 if err != nil { 47 if strings.HasPrefix(err.Error(), "can't start container") { 48 t.Skip("Docker required for s3 tests") 49 } 50 } 51 require.NoError(t, err) 52 defer func() { _ = gnomock.Stop(localS3) }() 53 54 fileStore, err := filestore.NewS3(bucketName, 55 filestore.WithEndpoint(fmt.Sprintf("http://%s/", localS3.Address(localstack.APIPort))), 56 filestore.WithKeys("abcd1234", "1qaz2wsx")) 57 require.NoError(t, err) 58 require.Equal(t, "s3", fileStore.Type()) 59 60 t.Run("test-S3-Put", func(t *testing.T) { 61 testPut(t, fileStore) 62 }) 63 64 t.Run("test-S3-Head", func(t *testing.T) { 65 testHead(t, fileStore) 66 }) 67 68 t.Run("test-S3-Get", func(t *testing.T) { 69 testGet(t, fileStore) 70 }) 71 72 t.Run("test-S3-List", func(t *testing.T) { 73 testList(t, fileStore) 74 }) 75 76 t.Run("test-S3-Delete", func(t *testing.T) { 77 testDelete(t, fileStore) 78 }) 79 } 80 81 func TestLocal(t *testing.T) { 82 carDir := t.TempDir() 83 84 fileStore, err := filestore.NewLocal(carDir) 85 require.NoError(t, err) 86 require.Equal(t, "local", fileStore.Type()) 87 88 t.Run("test-Local-Put", func(t *testing.T) { 89 testPut(t, fileStore) 90 }) 91 92 require.True(t, fsutil.FileExists(filepath.Join(carDir, fileName))) 93 94 t.Run("test-Local-Head", func(t *testing.T) { 95 testHead(t, fileStore) 96 }) 97 98 t.Run("test-Local-Get", func(t *testing.T) { 99 testGet(t, fileStore) 100 }) 101 102 t.Run("test-Local-List", func(t *testing.T) { 103 testList(t, fileStore) 104 }) 105 106 t.Run("test-Local-Delete", func(t *testing.T) { 107 testDelete(t, fileStore) 108 }) 109 } 110 111 func TestMakeFilestore(t *testing.T) { 112 cfg := filestore.Config{ 113 Type: "none", 114 } 115 fs, err := filestore.MakeFilestore(cfg) 116 require.NoError(t, err) 117 require.Nil(t, fs) 118 119 cfg.Type = "unknown" 120 _, err = filestore.MakeFilestore(cfg) 121 require.ErrorContains(t, err, "unsupported") 122 123 cfg.Type = "" 124 _, err = filestore.MakeFilestore(cfg) 125 require.ErrorContains(t, err, "not defined") 126 127 cfg.Type = "local" 128 _, err = filestore.MakeFilestore(cfg) 129 require.ErrorContains(t, err, "base path") 130 131 cfg.Local.BasePath = t.TempDir() 132 fs, err = filestore.MakeFilestore(cfg) 133 require.NoError(t, err) 134 require.NotNil(t, fs) 135 } 136 137 func testPut(t *testing.T, fileStore filestore.Interface) { 138 fileInfo, err := fileStore.Put(context.Background(), fileName, strings.NewReader(data)) 139 require.NoError(t, err) 140 require.Equal(t, fileName, fileInfo.Path) 141 require.Equal(t, int64(len(data)), fileInfo.Size) 142 } 143 144 func testHead(t *testing.T, fileStore filestore.Interface) { 145 // Get file that does not exist. 146 fileInfo, err := fileStore.Head(context.Background(), "not-here") 147 require.ErrorIs(t, err, fs.ErrNotExist) 148 require.Nil(t, fileInfo) 149 150 _, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data)) 151 require.NoError(t, err) 152 153 fileInfo, err = fileStore.Head(context.Background(), fileName3) 154 require.NoError(t, err) 155 require.Equal(t, fileName3, fileInfo.Path) 156 require.Equal(t, int64(len(data)), fileInfo.Size) 157 require.False(t, fileInfo.Modified.IsZero()) 158 159 // Should get fs.ErrNotExist when looking for subdirectory. 160 _, err = fileStore.Head(context.Background(), subdir) 161 require.ErrorIs(t, err, fs.ErrNotExist) 162 } 163 164 func testGet(t *testing.T, fileStore filestore.Interface) { 165 // Get file that does not exist. 166 fileInfo, _, err := fileStore.Get(context.Background(), "not-here") 167 require.ErrorIs(t, err, fs.ErrNotExist) 168 require.Nil(t, fileInfo) 169 170 _, err = fileStore.Put(context.Background(), fileName, strings.NewReader(data)) 171 require.NoError(t, err) 172 173 fileInfo, r, err := fileStore.Get(context.Background(), fileName) 174 require.NoError(t, err) 175 require.Equal(t, fileName, fileInfo.Path) 176 require.Equal(t, int64(len(data)), fileInfo.Size) 177 require.False(t, fileInfo.Modified.IsZero()) 178 179 data2 := make([]byte, len(data)) 180 n, err := r.Read(data2) 181 require.NoError(t, err) 182 require.Equal(t, len(data), n) 183 require.Equal(t, []byte(data), data2) 184 _, err = r.Read(data2) 185 require.ErrorIs(t, err, io.EOF) 186 require.NoError(t, r.Close()) 187 188 _, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data3)) 189 require.NoError(t, err) 190 191 _, _, err = fileStore.Get(context.Background(), subdir) 192 require.ErrorIs(t, err, fs.ErrNotExist) 193 194 fileInfo, r, err = fileStore.Get(context.Background(), fileName3) 195 require.NoError(t, err) 196 require.NoError(t, r.Close()) 197 require.Equal(t, int64(len(data3)), fileInfo.Size) 198 } 199 200 func testList(t *testing.T, fileStore filestore.Interface) { 201 // List file that does not exist. 202 fileCh, errCh := fileStore.List(context.Background(), "not-here/", false) 203 fileInfo, ok := <-fileCh 204 require.Nil(t, fileInfo) 205 require.False(t, ok) 206 err := <-errCh 207 require.NoError(t, err) 208 209 _, err = fileStore.Put(context.Background(), fileName1, strings.NewReader(data1)) 210 require.NoError(t, err) 211 212 _, err = fileStore.Put(context.Background(), fileName2, strings.NewReader(data2)) 213 require.NoError(t, err) 214 215 _, err = fileStore.Put(context.Background(), fileName3, strings.NewReader(data3)) 216 require.NoError(t, err) 217 218 fileCh, errCh = fileStore.List(context.Background(), "", false) 219 infos := make([]*filestore.File, 0, 3) 220 for fileInfo := range fileCh { 221 infos = append(infos, fileInfo) 222 } 223 err = <-errCh 224 require.NoError(t, err) 225 require.Equal(t, 3, len(infos)) 226 expectNames := []string{fileName, fileName1, fileName2} 227 expectSizes := []int64{int64(len(data)), int64(len(data1)), int64(len(data2))} 228 for i := range infos { 229 require.Equal(t, expectNames[i], infos[i].Path) 230 require.Equal(t, expectSizes[i], infos[i].Size) 231 require.False(t, infos[0].Modified.IsZero()) 232 } 233 234 fileCh, errCh = fileStore.List(context.Background(), "", true) 235 infos = infos[:0] 236 for fileInfo := range fileCh { 237 infos = append(infos, fileInfo) 238 } 239 err = <-errCh 240 require.NoError(t, err) 241 require.Equal(t, 4, len(infos)) 242 require.Equal(t, fileName3, infos[0].Path) 243 require.Equal(t, int64(len(data3)), infos[0].Size) 244 245 // File specific file. 246 fileCh, errCh = fileStore.List(context.Background(), fileName1, false) 247 infos = infos[:0] 248 for fileInfo := range fileCh { 249 infos = append(infos, fileInfo) 250 } 251 err = <-errCh 252 require.NoError(t, err) 253 require.Equal(t, 1, len(infos)) 254 require.Equal(t, fileName1, infos[0].Path) 255 256 // File specific file. 257 fileCh, errCh = fileStore.List(context.Background(), fileName3, false) 258 infos = infos[:0] 259 for fileInfo := range fileCh { 260 infos = append(infos, fileInfo) 261 } 262 err = <-errCh 263 require.NoError(t, err) 264 require.Equal(t, 1, len(infos)) 265 require.Equal(t, fileName3, infos[0].Path) 266 } 267 268 func testDelete(t *testing.T, fileStore filestore.Interface) { 269 ctx := context.Background() 270 271 _, err := fileStore.Put(ctx, fileName1, strings.NewReader(data1)) 272 require.NoError(t, err) 273 274 _, err = fileStore.Put(ctx, fileName2, strings.NewReader(data2)) 275 require.NoError(t, err) 276 277 _, err = fileStore.Put(ctx, fileName3, strings.NewReader(data3)) 278 require.NoError(t, err) 279 280 // File exists before delete. 281 _, err = fileStore.Head(ctx, fileName1) 282 require.NoError(t, err) 283 284 err = fileStore.Delete(ctx, fileName1) 285 require.NoError(t, err) 286 287 // File gone after delete. 288 _, err = fileStore.Head(ctx, fileName1) 289 require.ErrorIs(t, err, fs.ErrNotExist) 290 291 // Delete non-existant file should be OK. 292 err = fileStore.Delete(ctx, fileName1) 293 require.NoError(t, err) 294 295 err = fileStore.Delete(ctx, fileName2) 296 require.NoError(t, err) 297 298 err = fileStore.Delete(ctx, fileName3) 299 require.NoError(t, err) 300 }