github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/cmd/serve/s3/s3_test.go (about) 1 // Serve s3 tests set up a server and run the integration tests 2 // for the s3 remote against it. 3 4 package s3 5 6 import ( 7 "bytes" 8 "context" 9 "fmt" 10 "io" 11 "net/url" 12 "os" 13 "os/exec" 14 "path" 15 "strings" 16 "testing" 17 "time" 18 19 "github.com/minio/minio-go/v7" 20 "github.com/minio/minio-go/v7/pkg/credentials" 21 "github.com/rclone/rclone/fs/object" 22 23 _ "github.com/rclone/rclone/backend/local" 24 "github.com/rclone/rclone/cmd/serve/servetest" 25 "github.com/rclone/rclone/fs" 26 "github.com/rclone/rclone/fs/config/configmap" 27 "github.com/rclone/rclone/fs/hash" 28 "github.com/rclone/rclone/fstest" 29 httplib "github.com/rclone/rclone/lib/http" 30 "github.com/rclone/rclone/lib/random" 31 "github.com/stretchr/testify/assert" 32 "github.com/stretchr/testify/require" 33 ) 34 35 const ( 36 endpoint = "localhost:0" 37 ) 38 39 // Configure and serve the server 40 func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) { 41 keyid = random.String(16) 42 keysec = random.String(16) 43 serveropt := &Options{ 44 HTTP: httplib.DefaultCfg(), 45 pathBucketMode: true, 46 hashName: "", 47 hashType: hash.None, 48 authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)}, 49 } 50 51 serveropt.HTTP.ListenAddr = []string{endpoint} 52 w, _ := newServer(context.Background(), f, serveropt) 53 router := w.Router() 54 55 w.Bind(router) 56 w.Serve() 57 testURL = w.Server.URLs()[0] 58 59 return 60 } 61 62 // TestS3 runs the s3 server then runs the unit tests for the 63 // s3 remote against it. 64 func TestS3(t *testing.T) { 65 start := func(f fs.Fs) (configmap.Simple, func()) { 66 testURL, keyid, keysec := serveS3(f) 67 // Config for the backend we'll use to connect to the server 68 config := configmap.Simple{ 69 "type": "s3", 70 "provider": "Rclone", 71 "endpoint": testURL, 72 "access_key_id": keyid, 73 "secret_access_key": keysec, 74 } 75 76 return config, func() {} 77 } 78 79 RunS3UnitTests(t, "s3", start) 80 } 81 82 func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) { 83 fstest.Initialise() 84 ci := fs.GetConfig(context.Background()) 85 ci.DisableFeatures = append(ci.DisableFeatures, "Metadata") 86 87 fremote, _, clean, err := fstest.RandomRemote() 88 assert.NoError(t, err) 89 defer clean() 90 91 err = fremote.Mkdir(context.Background(), "") 92 assert.NoError(t, err) 93 94 f := fremote 95 config, cleanup := start(f) 96 defer cleanup() 97 98 // Change directory to run the tests 99 cwd, err := os.Getwd() 100 require.NoError(t, err) 101 err = os.Chdir("../../../backend/" + name) 102 require.NoError(t, err, "failed to cd to "+name+" backend") 103 defer func() { 104 // Change back to the old directory 105 require.NoError(t, os.Chdir(cwd)) 106 }() 107 108 // RunS3UnitTests the backend tests with an on the fly remote 109 args := []string{"test"} 110 if testing.Verbose() { 111 args = append(args, "-v") 112 } 113 if *fstest.Verbose { 114 args = append(args, "-verbose") 115 } 116 remoteName := "serve" + name + ":" 117 args = append(args, "-remote", remoteName) 118 args = append(args, "-run", "^TestIntegration$") 119 args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries)) 120 cmd := exec.Command("go", args...) 121 122 // Configure the backend with environment variables 123 cmd.Env = os.Environ() 124 prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_" 125 for k, v := range config { 126 cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v) 127 } 128 129 // RunS3UnitTests the test 130 out, err := cmd.CombinedOutput() 131 if len(out) != 0 { 132 t.Logf("\n----------\n%s----------\n", string(out)) 133 } 134 assert.NoError(t, err, "Running "+name+" integration tests") 135 } 136 137 // tests using the minio client 138 func TestEncodingWithMinioClient(t *testing.T) { 139 cases := []struct { 140 description string 141 bucket string 142 path string 143 filename string 144 expected string 145 }{ 146 { 147 description: "weird file in bucket root", 148 bucket: "mybucket", 149 path: "", 150 filename: " file with w€r^d ch@r \\#~+§4%&'. txt ", 151 }, 152 { 153 description: "weird file inside a weird folder", 154 bucket: "mybucket", 155 path: "ä#/नेपाल&/?/", 156 filename: " file with w€r^d ch@r \\#~+§4%&'. txt ", 157 }, 158 } 159 160 for _, tt := range cases { 161 t.Run(tt.description, func(t *testing.T) { 162 fstest.Initialise() 163 f, _, clean, err := fstest.RandomRemote() 164 assert.NoError(t, err) 165 defer clean() 166 err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path)) 167 assert.NoError(t, err) 168 169 buf := bytes.NewBufferString("contents") 170 uploadHash := hash.NewMultiHasher() 171 in := io.TeeReader(buf, uploadHash) 172 173 obji := object.NewStaticObjectInfo( 174 path.Join(tt.bucket, tt.path, tt.filename), 175 time.Now(), 176 int64(buf.Len()), 177 true, 178 nil, 179 nil, 180 ) 181 _, err = f.Put(context.Background(), in, obji) 182 assert.NoError(t, err) 183 184 endpoint, keyid, keysec := serveS3(f) 185 testURL, _ := url.Parse(endpoint) 186 minioClient, err := minio.New(testURL.Host, &minio.Options{ 187 Creds: credentials.NewStaticV4(keyid, keysec, ""), 188 Secure: false, 189 }) 190 assert.NoError(t, err) 191 192 buckets, err := minioClient.ListBuckets(context.Background()) 193 assert.NoError(t, err) 194 assert.Equal(t, buckets[0].Name, tt.bucket) 195 objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{ 196 Recursive: true, 197 }) 198 for object := range objects { 199 assert.Equal(t, path.Join(tt.path, tt.filename), object.Key) 200 } 201 }) 202 } 203 204 }