github.com/pachyderm/pachyderm@v1.13.4/src/server/pfs/s3/util_test.go (about)

     1  package s3
     2  
     3  import (
     4  	"context"
     5  	"crypto/md5"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"net"
    10  	"os"
    11  	"path/filepath"
    12  	"sort"
    13  	"strings"
    14  	"testing"
    15  	"time"
    16  
    17  	minio "github.com/minio/minio-go/v6"
    18  	"github.com/pachyderm/pachyderm/src/client"
    19  	"github.com/pachyderm/pachyderm/src/client/pkg/require"
    20  	"github.com/pachyderm/pachyderm/src/server/pkg/serviceenv"
    21  )
    22  
    23  func getObject(t *testing.T, minioClient *minio.Client, bucket, file string) (string, error) {
    24  	t.Helper()
    25  
    26  	obj, err := minioClient.GetObject(bucket, file, minio.GetObjectOptions{})
    27  	if err != nil {
    28  		return "", err
    29  	}
    30  	defer func() { err = obj.Close() }()
    31  	bytes, err := ioutil.ReadAll(obj)
    32  	if err != nil {
    33  		return "", err
    34  	}
    35  	return string(bytes), err
    36  }
    37  
    38  func checkListObjects(t *testing.T, ch <-chan minio.ObjectInfo, startTime *time.Time, endTime *time.Time, expectedFiles []string, expectedDirs []string) {
    39  	t.Helper()
    40  
    41  	// sort expected files/dirs, as the S3 gateway should always return
    42  	// results in sorted order
    43  	sort.Strings(expectedFiles)
    44  	sort.Strings(expectedDirs)
    45  
    46  	actualFiles := []minio.ObjectInfo{}
    47  	actualDirs := []minio.ObjectInfo{}
    48  	for obj := range ch {
    49  		require.NoError(t, obj.Err)
    50  
    51  		if strings.HasSuffix(obj.Key, "/") {
    52  			actualDirs = append(actualDirs, obj)
    53  		} else {
    54  			actualFiles = append(actualFiles, obj)
    55  		}
    56  	}
    57  
    58  	require.Equal(t, len(expectedFiles), len(actualFiles), "unexpected number of files")
    59  	require.Equal(t, len(expectedDirs), len(actualDirs), "unexpected number of dirs")
    60  
    61  	for i, expectedFilename := range expectedFiles {
    62  		actualFile := actualFiles[i]
    63  		require.Equal(t, expectedFilename, actualFile.Key)
    64  		require.True(t, len(actualFile.ETag) > 0, fmt.Sprintf("unexpected empty etag for %s", expectedFilename))
    65  		expectedLen := int64(len(filepath.Base(expectedFilename)) + 1)
    66  		require.Equal(t, expectedLen, actualFile.Size, fmt.Sprintf("unexpected file length for %s", expectedFilename))
    67  
    68  		if startTime != nil {
    69  			require.True(t, startTime.Before(actualFile.LastModified), fmt.Sprintf("unexpected last modified for %s", expectedFilename))
    70  		}
    71  
    72  		if endTime != nil {
    73  			require.True(t, endTime.After(actualFile.LastModified), fmt.Sprintf("unexpected last modified for %s", expectedFilename))
    74  		}
    75  	}
    76  
    77  	for i, expectedDirname := range expectedDirs {
    78  		actualDir := actualDirs[i]
    79  		require.Equal(t, expectedDirname, actualDir.Key)
    80  		require.True(t, len(actualDir.ETag) == 0, fmt.Sprintf("unexpected etag for %s: %s", expectedDirname, actualDir.ETag))
    81  		require.Equal(t, int64(0), actualDir.Size)
    82  		require.True(t, actualDir.LastModified.IsZero(), fmt.Sprintf("unexpected last modified for %s", expectedDirname))
    83  	}
    84  }
    85  
    86  func putListFileTestObject(t *testing.T, pachClient *client.APIClient, repo string, commitID string, dir string, i int) {
    87  	t.Helper()
    88  	_, err := pachClient.PutFile(
    89  		repo,
    90  		commitID,
    91  		fmt.Sprintf("%s%d", dir, i),
    92  		strings.NewReader(fmt.Sprintf("%d\n", i)),
    93  	)
    94  	require.NoError(t, err)
    95  }
    96  
    97  func bucketNotFoundError(t *testing.T, err error) {
    98  	t.Helper()
    99  	require.YesError(t, err)
   100  	require.Equal(t, "The specified bucket does not exist.", err.Error())
   101  }
   102  
   103  func keyNotFoundError(t *testing.T, err error) {
   104  	t.Helper()
   105  	require.YesError(t, err)
   106  	require.Equal(t, "The specified key does not exist.", err.Error())
   107  }
   108  
   109  func notImplementedError(t *testing.T, err error) {
   110  	t.Helper()
   111  	require.YesError(t, err)
   112  	require.Equal(t, "This functionality is not implemented.", err.Error())
   113  }
   114  
   115  func fileHash(t *testing.T, name string) (int64, []byte) {
   116  	t.Helper()
   117  
   118  	f, err := os.Open(name)
   119  	require.NoError(t, err)
   120  
   121  	fi, err := f.Stat()
   122  	require.NoError(t, err)
   123  
   124  	hash := md5.New()
   125  	_, err = io.Copy(hash, f)
   126  	require.NoError(t, err)
   127  
   128  	// make sure it's not the hash of an empty string
   129  	hashSum := hash.Sum(nil)
   130  	require.NotEqual(t, hashSum, []byte{0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x0, 0xb2, 0x4, 0xe9, 0x80, 0x9, 0x98, 0xec, 0xf8, 0x42, 0x7e})
   131  
   132  	return fi.Size(), hashSum
   133  }
   134  
   135  func testRunner(t *testing.T, group string, driver Driver, runner func(t *testing.T, pachClient *client.APIClient, minioClient *minio.Client)) {
   136  	env := serviceenv.InitPachOnlyTestEnv(t, &serviceenv.Configuration{
   137  		GlobalConfiguration: &serviceenv.GlobalConfiguration{
   138  			S3GatewayPort: 0,
   139  		}})
   140  	server, err := Server(env, driver)
   141  	require.NoError(t, err)
   142  	listener, err := net.Listen("tcp", ":0")
   143  	require.NoError(t, err)
   144  
   145  	go func() {
   146  		server.Serve(listener)
   147  	}()
   148  
   149  	port := listener.Addr().(*net.TCPAddr).Port
   150  
   151  	pachClient, err := client.NewForTest()
   152  	require.NoError(t, err)
   153  
   154  	minioClient, err := minio.NewV4(fmt.Sprintf("127.0.0.1:%d", port), "", "", false)
   155  	require.NoError(t, err)
   156  
   157  	t.Run(group, func(t *testing.T) {
   158  		runner(t, pachClient, minioClient)
   159  	})
   160  
   161  	require.NoError(t, server.Shutdown(context.Background()))
   162  }