github.com/spline-fu/mattermost-server@v4.10.10+incompatible/app/file.go (about) 1 // Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "bytes" 8 "crypto/sha256" 9 "encoding/base64" 10 "fmt" 11 "image" 12 "image/color" 13 "image/draw" 14 _ "image/gif" 15 "image/jpeg" 16 "io" 17 "mime/multipart" 18 "net/http" 19 "net/url" 20 "path/filepath" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/disintegration/imaging" 26 "github.com/rwcarlsen/goexif/exif" 27 _ "golang.org/x/image/bmp" 28 29 "github.com/mattermost/mattermost-server/mlog" 30 "github.com/mattermost/mattermost-server/model" 31 "github.com/mattermost/mattermost-server/utils" 32 ) 33 34 const ( 35 /* 36 EXIF Image Orientations 37 1 2 3 4 5 6 7 8 38 39 888888 888888 88 88 8888888888 88 88 8888888888 40 88 88 88 88 88 88 88 88 88 88 88 88 41 8888 8888 8888 8888 88 8888888888 8888888888 88 42 88 88 88 88 43 88 88 888888 888888 44 */ 45 Upright = 1 46 UprightMirrored = 2 47 UpsideDown = 3 48 UpsideDownMirrored = 4 49 RotatedCWMirrored = 5 50 RotatedCCW = 6 51 RotatedCCWMirrored = 7 52 RotatedCW = 8 53 54 MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image 55 IMAGE_THUMBNAIL_PIXEL_WIDTH = 120 56 IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100 57 IMAGE_PREVIEW_PIXEL_WIDTH = 1024 58 ) 59 60 func (a *App) FileBackend() (utils.FileBackend, *model.AppError) { 61 license := a.License() 62 return utils.NewFileBackend(&a.Config().FileSettings, license != nil && *license.Features.Compliance) 63 } 64 65 func (a *App) ReadFile(path string) ([]byte, *model.AppError) { 66 backend, err := a.FileBackend() 67 if err != nil { 68 return nil, err 69 } 70 return backend.ReadFile(path) 71 } 72 73 func (a *App) MoveFile(oldPath, newPath string) *model.AppError { 74 backend, err := a.FileBackend() 75 if err != nil { 76 return err 77 } 78 return backend.MoveFile(oldPath, newPath) 79 } 80 81 func (a *App) WriteFile(f []byte, path string) *model.AppError { 82 backend, err := a.FileBackend() 83 if err != nil { 84 return err 85 } 86 return backend.WriteFile(f, path) 87 } 88 89 func (a *App) RemoveFile(path string) *model.AppError { 90 backend, err := a.FileBackend() 91 if err != nil { 92 return err 93 } 94 return backend.RemoveFile(path) 95 } 96 97 func (a *App) GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { 98 // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} 99 split := strings.SplitN(filename, "/", 5) 100 if len(split) < 5 { 101 mlog.Error("Unable to decipher filename when migrating post to use FileInfos", mlog.String("post_id", post.Id), mlog.String("filename", filename)) 102 return nil 103 } 104 105 channelId := split[1] 106 userId := split[2] 107 oldId := split[3] 108 name, _ := url.QueryUnescape(split[4]) 109 110 if split[0] != "" || split[1] != post.ChannelId || split[2] != post.UserId || strings.Contains(split[4], "/") { 111 mlog.Warn( 112 "Found an unusual filename when migrating post to use FileInfos", 113 mlog.String("post_id", post.Id), 114 mlog.String("channel_id", post.ChannelId), 115 mlog.String("user_id", post.UserId), 116 mlog.String("filename", filename), 117 ) 118 } 119 120 pathPrefix := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/", teamId, channelId, userId, oldId) 121 path := pathPrefix + name 122 123 // Open the file and populate the fields of the FileInfo 124 var info *model.FileInfo 125 if data, err := a.ReadFile(path); err != nil { 126 mlog.Error( 127 fmt.Sprintf("File not found when migrating post to use FileInfos, err=%v", err), 128 mlog.String("post_id", post.Id), 129 mlog.String("filename", filename), 130 mlog.String("path", path), 131 ) 132 return nil 133 } else { 134 var err *model.AppError 135 info, err = model.GetInfoForBytes(name, data) 136 if err != nil { 137 mlog.Warn( 138 fmt.Sprintf("Unable to fully decode file info when migrating post to use FileInfos, err=%v", err), 139 mlog.String("post_id", post.Id), 140 mlog.String("filename", filename), 141 ) 142 } 143 } 144 145 // Generate a new ID because with the old system, you could very rarely get multiple posts referencing the same file 146 info.Id = model.NewId() 147 info.CreatorId = post.UserId 148 info.PostId = post.Id 149 info.CreateAt = post.CreateAt 150 info.UpdateAt = post.UpdateAt 151 info.Path = path 152 153 if info.IsImage() { 154 nameWithoutExtension := name[:strings.LastIndex(name, ".")] 155 info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" 156 info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" 157 } 158 159 return info 160 } 161 162 func (a *App) FindTeamIdForFilename(post *model.Post, filename string) string { 163 split := strings.SplitN(filename, "/", 5) 164 id := split[3] 165 name, _ := url.QueryUnescape(split[4]) 166 167 // This post is in a direct channel so we need to figure out what team the files are stored under. 168 if result := <-a.Srv.Store.Team().GetTeamsByUserId(post.UserId); result.Err != nil { 169 mlog.Error(fmt.Sprintf("Unable to get teams when migrating post to use FileInfo, err=%v", result.Err), mlog.String("post_id", post.Id)) 170 } else if teams := result.Data.([]*model.Team); len(teams) == 1 { 171 // The user has only one team so the post must've been sent from it 172 return teams[0].Id 173 } else { 174 for _, team := range teams { 175 path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) 176 if _, err := a.ReadFile(path); err == nil { 177 // Found the team that this file was posted from 178 return team.Id 179 } 180 } 181 } 182 183 return "" 184 } 185 186 var fileMigrationLock sync.Mutex 187 188 // Creates and stores FileInfos for a post created before the FileInfos table existed. 189 func (a *App) MigrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo { 190 if len(post.Filenames) == 0 { 191 mlog.Warn("Unable to migrate post to use FileInfos with an empty Filenames field", mlog.String("post_id", post.Id)) 192 return []*model.FileInfo{} 193 } 194 195 cchan := a.Srv.Store.Channel().Get(post.ChannelId, true) 196 197 // There's a weird bug that rarely happens where a post ends up with duplicate Filenames so remove those 198 filenames := utils.RemoveDuplicatesFromStringArray(post.Filenames) 199 200 var channel *model.Channel 201 if result := <-cchan; result.Err != nil { 202 mlog.Error( 203 fmt.Sprintf("Unable to get channel when migrating post to use FileInfos, err=%v", result.Err), 204 mlog.String("post_id", post.Id), 205 mlog.String("channel_id", post.ChannelId), 206 ) 207 return []*model.FileInfo{} 208 } else { 209 channel = result.Data.(*model.Channel) 210 } 211 212 // Find the team that was used to make this post since its part of the file path that isn't saved in the Filename 213 var teamId string 214 if channel.TeamId == "" { 215 // This post was made in a cross-team DM channel so we need to find where its files were saved 216 teamId = a.FindTeamIdForFilename(post, filenames[0]) 217 } else { 218 teamId = channel.TeamId 219 } 220 221 // Create FileInfo objects for this post 222 infos := make([]*model.FileInfo, 0, len(filenames)) 223 if teamId == "" { 224 mlog.Error( 225 fmt.Sprintf("Unable to find team id for files when migrating post to use FileInfos, filenames=%v", filenames), 226 mlog.String("post_id", post.Id), 227 ) 228 } else { 229 for _, filename := range filenames { 230 info := a.GetInfoForFilename(post, teamId, filename) 231 if info == nil { 232 continue 233 } 234 235 infos = append(infos, info) 236 } 237 } 238 239 // Lock to prevent only one migration thread from trying to update the post at once, preventing duplicate FileInfos from being created 240 fileMigrationLock.Lock() 241 defer fileMigrationLock.Unlock() 242 243 if result := <-a.Srv.Store.Post().Get(post.Id); result.Err != nil { 244 mlog.Error(fmt.Sprintf("Unable to get post when migrating post to use FileInfos, err=%v", result.Err), mlog.String("post_id", post.Id)) 245 return []*model.FileInfo{} 246 } else if newPost := result.Data.(*model.PostList).Posts[post.Id]; len(newPost.Filenames) != len(post.Filenames) { 247 // Another thread has already created FileInfos for this post, so just return those 248 if result := <-a.Srv.Store.FileInfo().GetForPost(post.Id, true, false); result.Err != nil { 249 mlog.Error(fmt.Sprintf("Unable to get FileInfos for migrated post, err=%v", result.Err), mlog.String("post_id", post.Id)) 250 return []*model.FileInfo{} 251 } else { 252 mlog.Debug("Post already migrated to use FileInfos", mlog.String("post_id", post.Id)) 253 return result.Data.([]*model.FileInfo) 254 } 255 } 256 257 mlog.Debug("Migrating post to use FileInfos", mlog.String("post_id", post.Id)) 258 259 savedInfos := make([]*model.FileInfo, 0, len(infos)) 260 fileIds := make([]string, 0, len(filenames)) 261 for _, info := range infos { 262 if result := <-a.Srv.Store.FileInfo().Save(info); result.Err != nil { 263 mlog.Error( 264 fmt.Sprintf("Unable to save file info when migrating post to use FileInfos, err=%v", result.Err), 265 mlog.String("post_id", post.Id), 266 mlog.String("file_info_id", info.Id), 267 mlog.String("file_info_path", info.Path), 268 ) 269 continue 270 } 271 272 savedInfos = append(savedInfos, info) 273 fileIds = append(fileIds, info.Id) 274 } 275 276 // Copy and save the updated post 277 newPost := &model.Post{} 278 *newPost = *post 279 280 newPost.Filenames = []string{} 281 newPost.FileIds = fileIds 282 283 // Update Posts to clear Filenames and set FileIds 284 if result := <-a.Srv.Store.Post().Update(newPost, post); result.Err != nil { 285 mlog.Error(fmt.Sprintf("Unable to save migrated post when migrating to use FileInfos, new_file_ids=%v, old_filenames=%v, err=%v", newPost.FileIds, post.Filenames, result.Err), mlog.String("post_id", post.Id)) 286 return []*model.FileInfo{} 287 } else { 288 return savedInfos 289 } 290 } 291 292 func (a *App) GeneratePublicLink(siteURL string, info *model.FileInfo) string { 293 hash := GeneratePublicLinkHash(info.Id, *a.Config().FileSettings.PublicLinkSalt) 294 return fmt.Sprintf("%s/files/%v/public?h=%s", siteURL, info.Id, hash) 295 } 296 297 func (a *App) GeneratePublicLinkV3(siteURL string, info *model.FileInfo) string { 298 hash := GeneratePublicLinkHash(info.Id, *a.Config().FileSettings.PublicLinkSalt) 299 return fmt.Sprintf("%s%s/public/files/%v/get?h=%s", siteURL, model.API_URL_SUFFIX_V3, info.Id, hash) 300 } 301 302 func GeneratePublicLinkHash(fileId, salt string) string { 303 hash := sha256.New() 304 hash.Write([]byte(salt)) 305 hash.Write([]byte(fileId)) 306 307 return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) 308 } 309 310 func (a *App) UploadMultipartFiles(teamId string, channelId string, userId string, fileHeaders []*multipart.FileHeader, clientIds []string) (*model.FileUploadResponse, *model.AppError) { 311 files := make([]io.ReadCloser, len(fileHeaders)) 312 filenames := make([]string, len(fileHeaders)) 313 314 for i, fileHeader := range fileHeaders { 315 file, fileErr := fileHeader.Open() 316 if fileErr != nil { 317 return nil, model.NewAppError("UploadFiles", "api.file.upload_file.bad_parse.app_error", nil, fileErr.Error(), http.StatusBadRequest) 318 } 319 320 // Will be closed after UploadFiles returns 321 defer file.Close() 322 323 files[i] = file 324 filenames[i] = fileHeader.Filename 325 } 326 327 return a.UploadFiles(teamId, channelId, userId, files, filenames, clientIds) 328 } 329 330 // Uploads some files to the given team and channel as the given user. files and filenames should have 331 // the same length. clientIds should either not be provided or have the same length as files and filenames. 332 // The provided files should be closed by the caller so that they are not leaked. 333 func (a *App) UploadFiles(teamId string, channelId string, userId string, files []io.ReadCloser, filenames []string, clientIds []string) (*model.FileUploadResponse, *model.AppError) { 334 if len(*a.Config().FileSettings.DriverName) == 0 { 335 return nil, model.NewAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "", http.StatusNotImplemented) 336 } 337 338 if len(filenames) != len(files) || (len(clientIds) > 0 && len(clientIds) != len(files)) { 339 return nil, model.NewAppError("UploadFiles", "api.file.upload_file.incorrect_number_of_files.app_error", nil, "", http.StatusBadRequest) 340 } 341 342 resStruct := &model.FileUploadResponse{ 343 FileInfos: []*model.FileInfo{}, 344 ClientIds: []string{}, 345 } 346 347 previewPathList := []string{} 348 thumbnailPathList := []string{} 349 imageDataList := [][]byte{} 350 351 for i, file := range files { 352 buf := bytes.NewBuffer(nil) 353 io.Copy(buf, file) 354 data := buf.Bytes() 355 356 info, err := a.DoUploadFile(time.Now(), teamId, channelId, userId, filenames[i], data) 357 if err != nil { 358 return nil, err 359 } 360 361 if info.PreviewPath != "" || info.ThumbnailPath != "" { 362 previewPathList = append(previewPathList, info.PreviewPath) 363 thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath) 364 imageDataList = append(imageDataList, data) 365 } 366 367 resStruct.FileInfos = append(resStruct.FileInfos, info) 368 369 if len(clientIds) > 0 { 370 resStruct.ClientIds = append(resStruct.ClientIds, clientIds[i]) 371 } 372 } 373 374 a.HandleImages(previewPathList, thumbnailPathList, imageDataList) 375 376 return resStruct, nil 377 } 378 379 func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { 380 filename := filepath.Base(rawFilename) 381 teamId := filepath.Base(rawTeamId) 382 channelId := filepath.Base(rawChannelId) 383 userId := filepath.Base(rawUserId) 384 385 info, err := model.GetInfoForBytes(filename, data) 386 if err != nil { 387 err.StatusCode = http.StatusBadRequest 388 return nil, err 389 } 390 391 if orientation, err := getImageOrientation(bytes.NewReader(data)); err == nil && 392 (orientation == RotatedCWMirrored || 393 orientation == RotatedCCW || 394 orientation == RotatedCCWMirrored || 395 orientation == RotatedCW) { 396 info.Width, info.Height = info.Height, info.Width 397 } 398 399 info.Id = model.NewId() 400 info.CreatorId = userId 401 402 pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" 403 info.Path = pathPrefix + filename 404 405 if info.IsImage() { 406 // Check dimensions before loading the whole thing into memory later on 407 if info.Width*info.Height > MaxImageSize { 408 err := model.NewAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "", http.StatusBadRequest) 409 return nil, err 410 } 411 412 nameWithoutExtension := filename[:strings.LastIndex(filename, ".")] 413 info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" 414 info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" 415 } 416 417 if err := a.WriteFile(data, info.Path); err != nil { 418 return nil, err 419 } 420 421 if result := <-a.Srv.Store.FileInfo().Save(info); result.Err != nil { 422 return nil, result.Err 423 } 424 425 return info, nil 426 } 427 428 func (a *App) HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { 429 wg := new(sync.WaitGroup) 430 431 for i := range fileData { 432 img, width, height := prepareImage(fileData[i]) 433 if img != nil { 434 wg.Add(2) 435 go func(img *image.Image, path string, width int, height int) { 436 defer wg.Done() 437 a.generateThumbnailImage(*img, path, width, height) 438 }(img, thumbnailPathList[i], width, height) 439 440 go func(img *image.Image, path string, width int) { 441 defer wg.Done() 442 a.generatePreviewImage(*img, path, width) 443 }(img, previewPathList[i], width) 444 } 445 } 446 wg.Wait() 447 } 448 449 func prepareImage(fileData []byte) (*image.Image, int, int) { 450 // Decode image bytes into Image object 451 img, imgType, err := image.Decode(bytes.NewReader(fileData)) 452 if err != nil { 453 mlog.Error(fmt.Sprintf("Unable to decode image err=%v", err)) 454 return nil, 0, 0 455 } 456 457 width := img.Bounds().Dx() 458 height := img.Bounds().Dy() 459 460 // Fill in the background of a potentially-transparent png file as white 461 if imgType == "png" { 462 dst := image.NewRGBA(img.Bounds()) 463 draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) 464 draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over) 465 img = dst 466 } 467 468 // Flip the image to be upright 469 orientation, _ := getImageOrientation(bytes.NewReader(fileData)) 470 img = makeImageUpright(img, orientation) 471 472 return &img, width, height 473 } 474 475 func makeImageUpright(img image.Image, orientation int) image.Image { 476 switch orientation { 477 case UprightMirrored: 478 return imaging.FlipH(img) 479 case UpsideDown: 480 return imaging.Rotate180(img) 481 case UpsideDownMirrored: 482 return imaging.FlipV(img) 483 case RotatedCWMirrored: 484 return imaging.Transpose(img) 485 case RotatedCCW: 486 return imaging.Rotate270(img) 487 case RotatedCCWMirrored: 488 return imaging.Transverse(img) 489 case RotatedCW: 490 return imaging.Rotate90(img) 491 default: 492 return img 493 } 494 } 495 496 func getImageOrientation(input io.Reader) (int, error) { 497 if exifData, err := exif.Decode(input); err != nil { 498 return Upright, err 499 } else { 500 if tag, err := exifData.Get("Orientation"); err != nil { 501 return Upright, err 502 } else { 503 orientation, err := tag.Int(0) 504 if err != nil { 505 return Upright, err 506 } else { 507 return orientation, nil 508 } 509 } 510 } 511 } 512 513 func (a *App) generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { 514 thumbWidth := float64(IMAGE_THUMBNAIL_PIXEL_WIDTH) 515 thumbHeight := float64(IMAGE_THUMBNAIL_PIXEL_HEIGHT) 516 imgWidth := float64(width) 517 imgHeight := float64(height) 518 519 var thumbnail image.Image 520 if imgHeight < IMAGE_THUMBNAIL_PIXEL_HEIGHT && imgWidth < thumbWidth { 521 thumbnail = img 522 } else if imgHeight/imgWidth < thumbHeight/thumbWidth { 523 thumbnail = imaging.Resize(img, 0, IMAGE_THUMBNAIL_PIXEL_HEIGHT, imaging.Lanczos) 524 } else { 525 thumbnail = imaging.Resize(img, IMAGE_THUMBNAIL_PIXEL_WIDTH, 0, imaging.Lanczos) 526 } 527 528 buf := new(bytes.Buffer) 529 if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil { 530 mlog.Error(fmt.Sprintf("Unable to encode image as jpeg path=%v err=%v", thumbnailPath, err)) 531 return 532 } 533 534 if err := a.WriteFile(buf.Bytes(), thumbnailPath); err != nil { 535 mlog.Error(fmt.Sprintf("Unable to upload thumbnail path=%v err=%v", thumbnailPath, err)) 536 return 537 } 538 } 539 540 func (a *App) generatePreviewImage(img image.Image, previewPath string, width int) { 541 var preview image.Image 542 543 if width > IMAGE_PREVIEW_PIXEL_WIDTH { 544 preview = imaging.Resize(img, IMAGE_PREVIEW_PIXEL_WIDTH, 0, imaging.Lanczos) 545 } else { 546 preview = img 547 } 548 549 buf := new(bytes.Buffer) 550 551 if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil { 552 mlog.Error(fmt.Sprintf("Unable to encode image as preview jpg err=%v", err), mlog.String("path", previewPath)) 553 return 554 } 555 556 if err := a.WriteFile(buf.Bytes(), previewPath); err != nil { 557 mlog.Error(fmt.Sprintf("Unable to upload preview err=%v", err), mlog.String("path", previewPath)) 558 return 559 } 560 } 561 562 func (a *App) GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) { 563 if result := <-a.Srv.Store.FileInfo().Get(fileId); result.Err != nil { 564 return nil, result.Err 565 } else { 566 return result.Data.(*model.FileInfo), nil 567 } 568 }