github.com/wgh-/mattermost-server@v4.8.0-rc2+incompatible/utils/file_backend_s3.go (about) 1 // Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package utils 5 6 import ( 7 "bytes" 8 "io/ioutil" 9 "net/http" 10 "os" 11 "path/filepath" 12 "strings" 13 14 l4g "github.com/alecthomas/log4go" 15 s3 "github.com/minio/minio-go" 16 "github.com/minio/minio-go/pkg/credentials" 17 18 "github.com/mattermost/mattermost-server/model" 19 ) 20 21 type S3FileBackend struct { 22 endpoint string 23 accessKey string 24 secretKey string 25 secure bool 26 signV2 bool 27 region string 28 bucket string 29 encrypt bool 30 trace bool 31 } 32 33 // Similar to s3.New() but allows initialization of signature v2 or signature v4 client. 34 // If signV2 input is false, function always returns signature v4. 35 // 36 // Additionally this function also takes a user defined region, if set 37 // disables automatic region lookup. 38 func (b *S3FileBackend) s3New() (*s3.Client, error) { 39 var creds *credentials.Credentials 40 if b.signV2 { 41 creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) 42 } else { 43 creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) 44 } 45 46 s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) 47 if err != nil { 48 return nil, err 49 } 50 51 if b.trace { 52 s3Clnt.TraceOn(os.Stdout) 53 } 54 55 return s3Clnt, nil 56 } 57 58 func (b *S3FileBackend) TestConnection() *model.AppError { 59 s3Clnt, err := b.s3New() 60 if err != nil { 61 return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) 62 } 63 64 exists, err := s3Clnt.BucketExists(b.bucket) 65 if err != nil { 66 return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) 67 } 68 69 if !exists { 70 l4g.Warn("Bucket specified does not exist. Attempting to create...") 71 err := s3Clnt.MakeBucket(b.bucket, b.region) 72 if err != nil { 73 l4g.Error("Unable to create bucket.") 74 return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) 75 } 76 } 77 l4g.Info("Connection to S3 or minio is good. Bucket exists.") 78 return nil 79 } 80 81 func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { 82 s3Clnt, err := b.s3New() 83 if err != nil { 84 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 85 } 86 minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) 87 if err != nil { 88 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 89 } 90 defer minioObject.Close() 91 if f, err := ioutil.ReadAll(minioObject); err != nil { 92 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 93 } else { 94 return f, nil 95 } 96 } 97 98 func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { 99 s3Clnt, err := b.s3New() 100 if err != nil { 101 return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 102 } 103 104 source := s3.NewSourceInfo(b.bucket, oldPath, nil) 105 destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) 106 if err != nil { 107 return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 108 } 109 if err = s3Clnt.CopyObject(destination, source); err != nil { 110 return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 111 } 112 return nil 113 } 114 115 func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { 116 s3Clnt, err := b.s3New() 117 if err != nil { 118 return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 119 } 120 121 source := s3.NewSourceInfo(b.bucket, oldPath, nil) 122 destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) 123 if err != nil { 124 return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 125 } 126 if err = s3Clnt.CopyObject(destination, source); err != nil { 127 return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 128 } 129 if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { 130 return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 131 } 132 return nil 133 } 134 135 func (b *S3FileBackend) WriteFile(f []byte, path string) *model.AppError { 136 s3Clnt, err := b.s3New() 137 if err != nil { 138 return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 139 } 140 141 var contentType string 142 if ext := filepath.Ext(path); model.IsFileExtImage(ext) { 143 contentType = model.GetImageMimeType(ext) 144 } else { 145 contentType = "binary/octet-stream" 146 } 147 148 options := s3PutOptions(b.encrypt, contentType) 149 150 if _, err = s3Clnt.PutObject(b.bucket, path, bytes.NewReader(f), -1, options); err != nil { 151 return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 152 } 153 154 return nil 155 } 156 157 func (b *S3FileBackend) RemoveFile(path string) *model.AppError { 158 s3Clnt, err := b.s3New() 159 if err != nil { 160 return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 161 } 162 163 if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { 164 return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 165 } 166 167 return nil 168 } 169 170 func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { 171 out := make(chan string, 1) 172 173 go func() { 174 defer close(out) 175 176 for { 177 info, done := <-in 178 179 if !done { 180 break 181 } 182 183 out <- info.Key 184 } 185 }() 186 187 return out 188 } 189 190 func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { 191 var paths []string 192 193 s3Clnt, err := b.s3New() 194 if err != nil { 195 return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 196 } 197 198 doneCh := make(chan struct{}) 199 200 defer close(doneCh) 201 202 for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { 203 if object.Err != nil { 204 return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) 205 } 206 paths = append(paths, strings.Trim(object.Key, "/")) 207 } 208 209 return &paths, nil 210 } 211 212 func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { 213 s3Clnt, err := b.s3New() 214 if err != nil { 215 return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 216 } 217 218 doneCh := make(chan struct{}) 219 220 for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { 221 if err.Err != nil { 222 doneCh <- struct{}{} 223 return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) 224 } 225 } 226 227 close(doneCh) 228 return nil 229 } 230 231 func s3PutOptions(encrypt bool, contentType string) s3.PutObjectOptions { 232 options := s3.PutObjectOptions{} 233 if encrypt { 234 options.UserMetadata = make(map[string]string) 235 options.UserMetadata["x-amz-server-side-encryption"] = "AES256" 236 } 237 options.ContentType = contentType 238 239 return options 240 } 241 242 func s3CopyMetadata(encrypt bool) map[string]string { 243 metaData := make(map[string]string) 244 metaData["x-amz-server-side-encryption"] = "AES256" 245 return metaData 246 }