github.com/psyb0t/mattermost-server@v4.6.1-0.20180125161845-5503a1351abf+incompatible/utils/file_backend_s3.go (about) 1 // Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package utils 5 6 import ( 7 "bytes" 8 "io/ioutil" 9 "net/http" 10 "os" 11 "path/filepath" 12 "strings" 13 14 l4g "github.com/alecthomas/log4go" 15 s3 "github.com/minio/minio-go" 16 "github.com/minio/minio-go/pkg/credentials" 17 18 "github.com/mattermost/mattermost-server/model" 19 ) 20 21 type S3FileBackend struct { 22 endpoint string 23 accessKey string 24 secretKey string 25 secure bool 26 signV2 bool 27 region string 28 bucket string 29 encrypt bool 30 trace bool 31 } 32 33 // Similar to s3.New() but allows initialization of signature v2 or signature v4 client. 34 // If signV2 input is false, function always returns signature v4. 35 // 36 // Additionally this function also takes a user defined region, if set 37 // disables automatic region lookup. 38 func (b *S3FileBackend) s3New() (*s3.Client, error) { 39 var creds *credentials.Credentials 40 if b.signV2 { 41 creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) 42 } else { 43 creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) 44 } 45 46 s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) 47 if err != nil { 48 return nil, err 49 } 50 51 if b.trace { 52 s3Clnt.TraceOn(os.Stdout) 53 } 54 55 return s3Clnt, nil 56 } 57 58 func (b *S3FileBackend) TestConnection() *model.AppError { 59 s3Clnt, err := b.s3New() 60 if err != nil { 61 return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) 62 } 63 64 exists, err := s3Clnt.BucketExists(b.bucket) 65 if err != nil { 66 return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) 67 } 68 69 if !exists { 70 l4g.Warn("Bucket specified does not exist. Attempting to create...") 71 err := s3Clnt.MakeBucket(b.bucket, b.region) 72 if err != nil { 73 l4g.Error("Unable to create bucket.") 74 return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) 75 } 76 } 77 l4g.Info("Connection to S3 or minio is good. Bucket exists.") 78 return nil 79 } 80 81 func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { 82 s3Clnt, err := b.s3New() 83 if err != nil { 84 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 85 } 86 minioObject, err := s3Clnt.GetObject(b.bucket, path) 87 if err != nil { 88 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 89 } 90 defer minioObject.Close() 91 if f, err := ioutil.ReadAll(minioObject); err != nil { 92 return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 93 } else { 94 return f, nil 95 } 96 } 97 98 func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { 99 s3Clnt, err := b.s3New() 100 if err != nil { 101 return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 102 } 103 104 source := s3.NewSourceInfo(b.bucket, oldPath, nil) 105 destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) 106 if err != nil { 107 return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 108 } 109 if err = s3Clnt.CopyObject(destination, source); err != nil { 110 return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 111 } 112 return nil 113 } 114 115 func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { 116 s3Clnt, err := b.s3New() 117 if err != nil { 118 return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 119 } 120 121 source := s3.NewSourceInfo(b.bucket, oldPath, nil) 122 destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) 123 if err != nil { 124 return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 125 } 126 if err = s3Clnt.CopyObject(destination, source); err != nil { 127 return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 128 } 129 if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { 130 return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) 131 } 132 return nil 133 } 134 135 func (b *S3FileBackend) WriteFile(f []byte, path string) *model.AppError { 136 s3Clnt, err := b.s3New() 137 if err != nil { 138 return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 139 } 140 141 ext := filepath.Ext(path) 142 metaData := s3Metadata(b.encrypt, "binary/octet-stream") 143 if model.IsFileExtImage(ext) { 144 metaData = s3Metadata(b.encrypt, model.GetImageMimeType(ext)) 145 } 146 147 if _, err = s3Clnt.PutObjectWithMetadata(b.bucket, path, bytes.NewReader(f), metaData, nil); err != nil { 148 return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 149 } 150 151 return nil 152 } 153 154 func (b *S3FileBackend) RemoveFile(path string) *model.AppError { 155 s3Clnt, err := b.s3New() 156 if err != nil { 157 return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 158 } 159 160 if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { 161 return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 162 } 163 164 return nil 165 } 166 167 func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { 168 out := make(chan string, 1) 169 170 go func() { 171 defer close(out) 172 173 for { 174 info, done := <-in 175 176 if !done { 177 break 178 } 179 180 out <- info.Key 181 } 182 }() 183 184 return out 185 } 186 187 func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { 188 var paths []string 189 190 s3Clnt, err := b.s3New() 191 if err != nil { 192 return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 193 } 194 195 doneCh := make(chan struct{}) 196 197 defer close(doneCh) 198 199 for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { 200 if object.Err != nil { 201 return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) 202 } 203 paths = append(paths, strings.Trim(object.Key, "/")) 204 } 205 206 return &paths, nil 207 } 208 209 func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { 210 s3Clnt, err := b.s3New() 211 if err != nil { 212 return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) 213 } 214 215 doneCh := make(chan struct{}) 216 217 for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { 218 if err.Err != nil { 219 doneCh <- struct{}{} 220 return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) 221 } 222 } 223 224 close(doneCh) 225 return nil 226 } 227 228 func s3Metadata(encrypt bool, contentType string) map[string][]string { 229 metaData := make(map[string][]string) 230 if contentType != "" { 231 metaData["Content-Type"] = []string{"contentType"} 232 } 233 if encrypt { 234 metaData["x-amz-server-side-encryption"] = []string{"AES256"} 235 } 236 return metaData 237 } 238 239 func s3CopyMetadata(encrypt bool) map[string]string { 240 metaData := make(map[string]string) 241 metaData["x-amz-server-side-encryption"] = "AES256" 242 return metaData 243 }