github.com/xzl8028/xenia-server@v0.0.0-20190809101854-18450a97da63/app/file.go (about) 1 // Copyright (c) 2017-present Xenia, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "bytes" 8 "crypto/sha256" 9 "encoding/base64" 10 "fmt" 11 "image" 12 "image/color" 13 "image/draw" 14 "image/gif" 15 "image/jpeg" 16 "io" 17 "mime/multipart" 18 "net/http" 19 "net/url" 20 "path/filepath" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/disintegration/imaging" 26 "github.com/rwcarlsen/goexif/exif" 27 _ "golang.org/x/image/bmp" 28 29 "github.com/xzl8028/xenia-server/mlog" 30 "github.com/xzl8028/xenia-server/model" 31 "github.com/xzl8028/xenia-server/plugin" 32 "github.com/xzl8028/xenia-server/services/filesstore" 33 "github.com/xzl8028/xenia-server/utils" 34 ) 35 36 const ( 37 /* 38 EXIF Image Orientations 39 1 2 3 4 5 6 7 8 40 41 888888 888888 88 88 8888888888 88 88 8888888888 42 88 88 88 88 88 88 88 88 88 88 88 88 43 8888 8888 8888 8888 88 8888888888 8888888888 88 44 88 88 88 88 45 88 88 888888 888888 46 */ 47 Upright = 1 48 UprightMirrored = 2 49 UpsideDown = 3 50 UpsideDownMirrored = 4 51 RotatedCWMirrored = 5 52 RotatedCCW = 6 53 RotatedCCWMirrored = 7 54 RotatedCW = 8 55 56 MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image 57 ImageThumbnailWidth = 120 58 ImageThumbnailHeight = 100 59 ImageThumbnailRatio = float64(ImageThumbnailHeight) / float64(ImageThumbnailWidth) 60 ImagePreviewWidth = 1920 61 62 UploadFileInitialBufferSize = 2 * 1024 * 1024 // 2Mb 63 64 // Deprecated 65 IMAGE_THUMBNAIL_PIXEL_WIDTH = 120 66 IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100 67 IMAGE_PREVIEW_PIXEL_WIDTH = 1920 68 ) 69 70 func (a *App) FileBackend() (filesstore.FileBackend, *model.AppError) { 71 license := a.License() 72 return filesstore.NewFileBackend(&a.Config().FileSettings, license != nil && *license.Features.Compliance) 73 } 74 75 func (a *App) ReadFile(path string) ([]byte, *model.AppError) { 76 backend, err := a.FileBackend() 77 if err != nil { 78 return nil, err 79 } 80 return backend.ReadFile(path) 81 } 82 83 // Caller must close the first return value 84 func (a *App) FileReader(path string) (filesstore.ReadCloseSeeker, *model.AppError) { 85 backend, err := a.FileBackend() 86 if err != nil { 87 return nil, err 88 } 89 return backend.Reader(path) 90 } 91 92 func (a *App) FileExists(path string) (bool, *model.AppError) { 93 backend, err := a.FileBackend() 94 if err != nil { 95 return false, err 96 } 97 return backend.FileExists(path) 98 } 99 100 func (a *App) MoveFile(oldPath, newPath string) *model.AppError { 101 backend, err := a.FileBackend() 102 if err != nil { 103 return err 104 } 105 return backend.MoveFile(oldPath, newPath) 106 } 107 108 func (a *App) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { 109 backend, err := a.FileBackend() 110 if err != nil { 111 return 0, err 112 } 113 114 return backend.WriteFile(fr, path) 115 } 116 117 func (a *App) RemoveFile(path string) *model.AppError { 118 backend, err := a.FileBackend() 119 if err != nil { 120 return err 121 } 122 return backend.RemoveFile(path) 123 } 124 125 func (a *App) GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { 126 // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} 127 split := strings.SplitN(filename, "/", 5) 128 if len(split) < 5 { 129 mlog.Error("Unable to decipher filename when migrating post to use FileInfos", mlog.String("post_id", post.Id), mlog.String("filename", filename)) 130 return nil 131 } 132 133 channelId := split[1] 134 userId := split[2] 135 oldId := split[3] 136 name, _ := url.QueryUnescape(split[4]) 137 138 if split[0] != "" || split[1] != post.ChannelId || split[2] != post.UserId || strings.Contains(split[4], "/") { 139 mlog.Warn( 140 "Found an unusual filename when migrating post to use FileInfos", 141 mlog.String("post_id", post.Id), 142 mlog.String("channel_id", post.ChannelId), 143 mlog.String("user_id", post.UserId), 144 mlog.String("filename", filename), 145 ) 146 } 147 148 pathPrefix := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/", teamId, channelId, userId, oldId) 149 path := pathPrefix + name 150 151 // Open the file and populate the fields of the FileInfo 152 data, err := a.ReadFile(path) 153 if err != nil { 154 mlog.Error( 155 fmt.Sprintf("File not found when migrating post to use FileInfos, err=%v", err), 156 mlog.String("post_id", post.Id), 157 mlog.String("filename", filename), 158 mlog.String("path", path), 159 ) 160 return nil 161 } 162 163 info, err := model.GetInfoForBytes(name, data) 164 if err != nil { 165 mlog.Warn( 166 fmt.Sprintf("Unable to fully decode file info when migrating post to use FileInfos, err=%v", err), 167 mlog.String("post_id", post.Id), 168 mlog.String("filename", filename), 169 ) 170 } 171 172 // Generate a new ID because with the old system, you could very rarely get multiple posts referencing the same file 173 info.Id = model.NewId() 174 info.CreatorId = post.UserId 175 info.PostId = post.Id 176 info.CreateAt = post.CreateAt 177 info.UpdateAt = post.UpdateAt 178 info.Path = path 179 180 if info.IsImage() { 181 nameWithoutExtension := name[:strings.LastIndex(name, ".")] 182 info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" 183 info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" 184 } 185 186 return info 187 } 188 189 func (a *App) FindTeamIdForFilename(post *model.Post, filename string) string { 190 split := strings.SplitN(filename, "/", 5) 191 id := split[3] 192 name, _ := url.QueryUnescape(split[4]) 193 194 // This post is in a direct channel so we need to figure out what team the files are stored under. 195 result := <-a.Srv.Store.Team().GetTeamsByUserId(post.UserId) 196 if result.Err != nil { 197 mlog.Error(fmt.Sprintf("Unable to get teams when migrating post to use FileInfo, err=%v", result.Err), mlog.String("post_id", post.Id)) 198 return "" 199 } 200 201 teams := result.Data.([]*model.Team) 202 if len(teams) == 1 { 203 // The user has only one team so the post must've been sent from it 204 return teams[0].Id 205 } 206 207 for _, team := range teams { 208 path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) 209 if _, err := a.ReadFile(path); err == nil { 210 // Found the team that this file was posted from 211 return team.Id 212 } 213 } 214 215 return "" 216 } 217 218 var fileMigrationLock sync.Mutex 219 220 // Creates and stores FileInfos for a post created before the FileInfos table existed. 221 func (a *App) MigrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo { 222 if len(post.Filenames) == 0 { 223 mlog.Warn("Unable to migrate post to use FileInfos with an empty Filenames field", mlog.String("post_id", post.Id)) 224 return []*model.FileInfo{} 225 } 226 227 channel, errCh := a.Srv.Store.Channel().Get(post.ChannelId, true) 228 // There's a weird bug that rarely happens where a post ends up with duplicate Filenames so remove those 229 filenames := utils.RemoveDuplicatesFromStringArray(post.Filenames) 230 if errCh != nil { 231 mlog.Error( 232 fmt.Sprintf("Unable to get channel when migrating post to use FileInfos, err=%v", errCh), 233 mlog.String("post_id", post.Id), 234 mlog.String("channel_id", post.ChannelId), 235 ) 236 return []*model.FileInfo{} 237 } 238 239 // Find the team that was used to make this post since its part of the file path that isn't saved in the Filename 240 var teamId string 241 if channel.TeamId == "" { 242 // This post was made in a cross-team DM channel, so we need to find where its files were saved 243 teamId = a.FindTeamIdForFilename(post, filenames[0]) 244 } else { 245 teamId = channel.TeamId 246 } 247 248 // Create FileInfo objects for this post 249 infos := make([]*model.FileInfo, 0, len(filenames)) 250 if teamId == "" { 251 mlog.Error( 252 fmt.Sprintf("Unable to find team id for files when migrating post to use FileInfos, filenames=%v", filenames), 253 mlog.String("post_id", post.Id), 254 ) 255 } else { 256 for _, filename := range filenames { 257 info := a.GetInfoForFilename(post, teamId, filename) 258 if info == nil { 259 continue 260 } 261 262 infos = append(infos, info) 263 } 264 } 265 266 // Lock to prevent only one migration thread from trying to update the post at once, preventing duplicate FileInfos from being created 267 fileMigrationLock.Lock() 268 defer fileMigrationLock.Unlock() 269 270 result, err := a.Srv.Store.Post().Get(post.Id) 271 if err != nil { 272 mlog.Error(fmt.Sprintf("Unable to get post when migrating post to use FileInfos, err=%v", err), mlog.String("post_id", post.Id)) 273 return []*model.FileInfo{} 274 } 275 276 if newPost := result.Posts[post.Id]; len(newPost.Filenames) != len(post.Filenames) { 277 // Another thread has already created FileInfos for this post, so just return those 278 var fileInfos []*model.FileInfo 279 fileInfos, err = a.Srv.Store.FileInfo().GetForPost(post.Id, true, false) 280 if err != nil { 281 mlog.Error(fmt.Sprintf("Unable to get FileInfos for migrated post, err=%v", err), mlog.String("post_id", post.Id)) 282 return []*model.FileInfo{} 283 } 284 285 mlog.Debug("Post already migrated to use FileInfos", mlog.String("post_id", post.Id)) 286 return fileInfos 287 } 288 289 mlog.Debug("Migrating post to use FileInfos", mlog.String("post_id", post.Id)) 290 291 savedInfos := make([]*model.FileInfo, 0, len(infos)) 292 fileIds := make([]string, 0, len(filenames)) 293 for _, info := range infos { 294 if _, err = a.Srv.Store.FileInfo().Save(info); err != nil { 295 mlog.Error( 296 fmt.Sprintf("Unable to save file info when migrating post to use FileInfos, err=%v", err), 297 mlog.String("post_id", post.Id), 298 mlog.String("file_info_id", info.Id), 299 mlog.String("file_info_path", info.Path), 300 ) 301 continue 302 } 303 304 savedInfos = append(savedInfos, info) 305 fileIds = append(fileIds, info.Id) 306 } 307 308 // Copy and save the updated post 309 newPost := &model.Post{} 310 *newPost = *post 311 312 newPost.Filenames = []string{} 313 newPost.FileIds = fileIds 314 315 // Update Posts to clear Filenames and set FileIds 316 if _, err = a.Srv.Store.Post().Update(newPost, post); err != nil { 317 mlog.Error(fmt.Sprintf("Unable to save migrated post when migrating to use FileInfos, new_file_ids=%v, old_filenames=%v, err=%v", newPost.FileIds, post.Filenames, err), mlog.String("post_id", post.Id)) 318 return []*model.FileInfo{} 319 } 320 return savedInfos 321 } 322 323 func (a *App) GeneratePublicLink(siteURL string, info *model.FileInfo) string { 324 hash := GeneratePublicLinkHash(info.Id, *a.Config().FileSettings.PublicLinkSalt) 325 return fmt.Sprintf("%s/files/%v/public?h=%s", siteURL, info.Id, hash) 326 } 327 328 func GeneratePublicLinkHash(fileId, salt string) string { 329 hash := sha256.New() 330 hash.Write([]byte(salt)) 331 hash.Write([]byte(fileId)) 332 333 return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) 334 } 335 336 func (a *App) UploadMultipartFiles(teamId string, channelId string, userId string, fileHeaders []*multipart.FileHeader, clientIds []string, now time.Time) (*model.FileUploadResponse, *model.AppError) { 337 files := make([]io.ReadCloser, len(fileHeaders)) 338 filenames := make([]string, len(fileHeaders)) 339 340 for i, fileHeader := range fileHeaders { 341 file, fileErr := fileHeader.Open() 342 if fileErr != nil { 343 return nil, model.NewAppError("UploadFiles", "api.file.upload_file.read_request.app_error", 344 map[string]interface{}{"Filename": fileHeader.Filename}, fileErr.Error(), http.StatusBadRequest) 345 } 346 347 // Will be closed after UploadFiles returns 348 defer file.Close() 349 350 files[i] = file 351 filenames[i] = fileHeader.Filename 352 } 353 354 return a.UploadFiles(teamId, channelId, userId, files, filenames, clientIds, now) 355 } 356 357 // Uploads some files to the given team and channel as the given user. files and filenames should have 358 // the same length. clientIds should either not be provided or have the same length as files and filenames. 359 // The provided files should be closed by the caller so that they are not leaked. 360 func (a *App) UploadFiles(teamId string, channelId string, userId string, files []io.ReadCloser, filenames []string, clientIds []string, now time.Time) (*model.FileUploadResponse, *model.AppError) { 361 if len(*a.Config().FileSettings.DriverName) == 0 { 362 return nil, model.NewAppError("UploadFiles", "api.file.upload_file.storage.app_error", nil, "", http.StatusNotImplemented) 363 } 364 365 if len(filenames) != len(files) || (len(clientIds) > 0 && len(clientIds) != len(files)) { 366 return nil, model.NewAppError("UploadFiles", "api.file.upload_file.incorrect_number_of_files.app_error", nil, "", http.StatusBadRequest) 367 } 368 369 resStruct := &model.FileUploadResponse{ 370 FileInfos: []*model.FileInfo{}, 371 ClientIds: []string{}, 372 } 373 374 previewPathList := []string{} 375 thumbnailPathList := []string{} 376 imageDataList := [][]byte{} 377 378 for i, file := range files { 379 buf := bytes.NewBuffer(nil) 380 io.Copy(buf, file) 381 data := buf.Bytes() 382 383 info, data, err := a.DoUploadFileExpectModification(now, teamId, channelId, userId, filenames[i], data) 384 if err != nil { 385 return nil, err 386 } 387 388 if info.PreviewPath != "" || info.ThumbnailPath != "" { 389 previewPathList = append(previewPathList, info.PreviewPath) 390 thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath) 391 imageDataList = append(imageDataList, data) 392 } 393 394 resStruct.FileInfos = append(resStruct.FileInfos, info) 395 396 if len(clientIds) > 0 { 397 resStruct.ClientIds = append(resStruct.ClientIds, clientIds[i]) 398 } 399 } 400 401 a.HandleImages(previewPathList, thumbnailPathList, imageDataList) 402 403 return resStruct, nil 404 } 405 406 // UploadFile uploads a single file in form of a completely constructed byte array for a channel. 407 func (a *App) UploadFile(data []byte, channelId string, filename string) (*model.FileInfo, *model.AppError) { 408 info, _, appError := a.DoUploadFileExpectModification(time.Now(), "noteam", channelId, "nouser", filename, data) 409 410 if appError != nil { 411 return nil, appError 412 } 413 414 if info.PreviewPath != "" || info.ThumbnailPath != "" { 415 previewPathList := []string{info.PreviewPath} 416 thumbnailPathList := []string{info.ThumbnailPath} 417 imageDataList := [][]byte{data} 418 419 a.HandleImages(previewPathList, thumbnailPathList, imageDataList) 420 } 421 422 return info, nil 423 } 424 425 func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { 426 info, _, err := a.DoUploadFileExpectModification(now, rawTeamId, rawChannelId, rawUserId, rawFilename, data) 427 return info, err 428 } 429 430 func UploadFileSetTeamId(teamId string) func(t *uploadFileTask) { 431 return func(t *uploadFileTask) { 432 t.TeamId = filepath.Base(teamId) 433 } 434 } 435 436 func UploadFileSetUserId(userId string) func(t *uploadFileTask) { 437 return func(t *uploadFileTask) { 438 t.UserId = filepath.Base(userId) 439 } 440 } 441 442 func UploadFileSetTimestamp(timestamp time.Time) func(t *uploadFileTask) { 443 return func(t *uploadFileTask) { 444 t.Timestamp = timestamp 445 } 446 } 447 448 func UploadFileSetContentLength(contentLength int64) func(t *uploadFileTask) { 449 return func(t *uploadFileTask) { 450 t.ContentLength = contentLength 451 } 452 } 453 454 func UploadFileSetClientId(clientId string) func(t *uploadFileTask) { 455 return func(t *uploadFileTask) { 456 t.ClientId = clientId 457 } 458 } 459 460 func UploadFileSetRaw() func(t *uploadFileTask) { 461 return func(t *uploadFileTask) { 462 t.Raw = true 463 } 464 } 465 466 type uploadFileTask struct { 467 // File name. 468 Name string 469 470 ChannelId string 471 TeamId string 472 UserId string 473 474 // Time stamp to use when creating the file. 475 Timestamp time.Time 476 477 // The value of the Content-Length http header, when available. 478 ContentLength int64 479 480 // The file data stream. 481 Input io.Reader 482 483 // An optional, client-assigned Id field. 484 ClientId string 485 486 // If Raw, do not execute special processing for images, just upload 487 // the file. Plugins are still invoked. 488 Raw bool 489 490 //============================================================= 491 // Internal state 492 493 buf *bytes.Buffer 494 limit int64 495 limitedInput io.Reader 496 teeInput io.Reader 497 fileinfo *model.FileInfo 498 maxFileSize int64 499 500 // Cached image data that (may) get initialized in preprocessImage and 501 // is used in postprocessImage 502 decoded image.Image 503 imageType string 504 imageOrientation int 505 506 // Testing: overrideable dependency functions 507 pluginsEnvironment *plugin.Environment 508 writeFile func(io.Reader, string) (int64, *model.AppError) 509 saveToDatabase func(*model.FileInfo) (*model.FileInfo, *model.AppError) 510 } 511 512 func (t *uploadFileTask) init(a *App) { 513 t.buf = &bytes.Buffer{} 514 t.maxFileSize = *a.Config().FileSettings.MaxFileSize 515 t.limit = *a.Config().FileSettings.MaxFileSize 516 517 t.fileinfo = model.NewInfo(filepath.Base(t.Name)) 518 t.fileinfo.Id = model.NewId() 519 t.fileinfo.CreatorId = t.UserId 520 t.fileinfo.CreateAt = t.Timestamp.UnixNano() / int64(time.Millisecond) 521 t.fileinfo.Path = t.pathPrefix() + t.Name 522 523 // Prepare to read ContentLength if it is known, otherwise limit 524 // ourselves to MaxFileSize. Add an extra byte to check and fail if the 525 // client sent too many bytes. 526 if t.ContentLength > 0 { 527 t.limit = t.ContentLength 528 // Over-Grow the buffer to prevent bytes.ReadFrom from doing it 529 // at the very end. 530 t.buf.Grow(int(t.limit + 1 + bytes.MinRead)) 531 } else { 532 // If we don't know the upload size, grow the buffer somewhat 533 // anyway to avoid extra reslicing. 534 t.buf.Grow(UploadFileInitialBufferSize) 535 } 536 t.limitedInput = &io.LimitedReader{ 537 R: t.Input, 538 N: t.limit + 1, 539 } 540 t.teeInput = io.TeeReader(t.limitedInput, t.buf) 541 542 t.pluginsEnvironment = a.GetPluginsEnvironment() 543 t.writeFile = a.WriteFile 544 t.saveToDatabase = a.Srv.Store.FileInfo().Save 545 } 546 547 // UploadFileX uploads a single file as specified in t. It applies the upload 548 // constraints, executes plugins and image processing logic as needed. It 549 // returns a filled-out FileInfo and an optional error. A plugin may reject the 550 // upload, returning a rejection error. In this case FileInfo would have 551 // contained the last "good" FileInfo before the execution of that plugin. 552 func (a *App) UploadFileX(channelId, name string, input io.Reader, 553 opts ...func(*uploadFileTask)) (*model.FileInfo, *model.AppError) { 554 555 t := &uploadFileTask{ 556 ChannelId: filepath.Base(channelId), 557 Name: filepath.Base(name), 558 Input: input, 559 } 560 for _, o := range opts { 561 o(t) 562 } 563 t.init(a) 564 565 if len(*a.Config().FileSettings.DriverName) == 0 { 566 return nil, t.newAppError("api.file.upload_file.storage.app_error", 567 "", http.StatusNotImplemented) 568 } 569 if t.ContentLength > t.maxFileSize { 570 return nil, t.newAppError("api.file.upload_file.too_large_detailed.app_error", 571 "", http.StatusRequestEntityTooLarge, "Length", t.ContentLength, "Limit", t.maxFileSize) 572 } 573 574 var aerr *model.AppError 575 if !t.Raw && t.fileinfo.IsImage() { 576 aerr = t.preprocessImage() 577 if aerr != nil { 578 return t.fileinfo, aerr 579 } 580 } 581 582 aerr = t.readAll() 583 if aerr != nil { 584 return t.fileinfo, aerr 585 } 586 587 aerr = t.runPlugins() 588 if aerr != nil { 589 return t.fileinfo, aerr 590 } 591 592 // Concurrently upload and update DB, and post-process the image. 593 wg := sync.WaitGroup{} 594 595 if !t.Raw && t.fileinfo.IsImage() { 596 wg.Add(1) 597 go func() { 598 t.postprocessImage() 599 wg.Done() 600 }() 601 } 602 603 _, aerr = t.writeFile(t.newReader(), t.fileinfo.Path) 604 if aerr != nil { 605 return nil, aerr 606 } 607 608 if _, err := t.saveToDatabase(t.fileinfo); err != nil { 609 return nil, err 610 } 611 612 wg.Wait() 613 614 return t.fileinfo, nil 615 } 616 617 func (t *uploadFileTask) readAll() *model.AppError { 618 _, err := t.buf.ReadFrom(t.limitedInput) 619 if err != nil { 620 return t.newAppError("api.file.upload_file.read_request.app_error", 621 err.Error(), http.StatusBadRequest) 622 } 623 if int64(t.buf.Len()) > t.limit { 624 return t.newAppError("api.file.upload_file.too_large_detailed.app_error", 625 "", http.StatusRequestEntityTooLarge, "Length", t.buf.Len(), "Limit", t.limit) 626 } 627 t.fileinfo.Size = int64(t.buf.Len()) 628 629 t.limitedInput = nil 630 t.teeInput = nil 631 return nil 632 } 633 634 func (t *uploadFileTask) runPlugins() *model.AppError { 635 if t.pluginsEnvironment == nil { 636 return nil 637 } 638 639 pluginContext := &plugin.Context{} 640 var rejectionError *model.AppError 641 642 t.pluginsEnvironment.RunMultiPluginHook(func(hooks plugin.Hooks) bool { 643 buf := &bytes.Buffer{} 644 replacementInfo, rejectionReason := hooks.FileWillBeUploaded(pluginContext, 645 t.fileinfo, t.newReader(), buf) 646 if rejectionReason != "" { 647 rejectionError = t.newAppError("api.file.upload_file.rejected_by_plugin.app_error", 648 rejectionReason, http.StatusForbidden, "Reason", rejectionReason) 649 return false 650 } 651 if replacementInfo != nil { 652 t.fileinfo = replacementInfo 653 } 654 if buf.Len() != 0 { 655 t.buf = buf 656 t.teeInput = nil 657 t.limitedInput = nil 658 t.fileinfo.Size = int64(buf.Len()) 659 } 660 661 return true 662 }, plugin.FileWillBeUploadedId) 663 664 if rejectionError != nil { 665 return rejectionError 666 } 667 668 return nil 669 } 670 671 func (t *uploadFileTask) preprocessImage() *model.AppError { 672 // If SVG, attempt to extract dimensions and then return 673 if t.fileinfo.MimeType == "image/svg+xml" { 674 svgInfo, err := parseSVG(t.newReader()) 675 if err != nil { 676 mlog.Error("Failed to parse SVG", mlog.Err(err)) 677 } 678 if svgInfo.Width > 0 && svgInfo.Height > 0 { 679 t.fileinfo.Width = svgInfo.Width 680 t.fileinfo.Height = svgInfo.Height 681 } 682 t.fileinfo.HasPreviewImage = false 683 return nil 684 } 685 686 // If we fail to decode, return "as is". 687 config, _, err := image.DecodeConfig(t.newReader()) 688 if err != nil { 689 return nil 690 } 691 692 t.fileinfo.Width = config.Width 693 t.fileinfo.Height = config.Height 694 695 // Check dimensions before loading the whole thing into memory later on. 696 if t.fileinfo.Width*t.fileinfo.Height > MaxImageSize { 697 return t.newAppError("api.file.upload_file.large_image_detailed.app_error", 698 "", http.StatusBadRequest) 699 } 700 t.fileinfo.HasPreviewImage = true 701 nameWithoutExtension := t.Name[:strings.LastIndex(t.Name, ".")] 702 t.fileinfo.PreviewPath = t.pathPrefix() + nameWithoutExtension + "_preview.jpg" 703 t.fileinfo.ThumbnailPath = t.pathPrefix() + nameWithoutExtension + "_thumb.jpg" 704 705 // check the image orientation with goexif; consume the bytes we 706 // already have first, then keep Tee-ing from input. 707 // TODO: try to reuse exif's .Raw buffer rather than Tee-ing 708 if t.imageOrientation, err = getImageOrientation(t.newReader()); err == nil && 709 (t.imageOrientation == RotatedCWMirrored || 710 t.imageOrientation == RotatedCCW || 711 t.imageOrientation == RotatedCCWMirrored || 712 t.imageOrientation == RotatedCW) { 713 t.fileinfo.Width, t.fileinfo.Height = t.fileinfo.Height, t.fileinfo.Width 714 } 715 716 // For animated GIFs disable the preview; since we have to Decode gifs 717 // anyway, cache the decoded image for later. 718 if t.fileinfo.MimeType == "image/gif" { 719 gifConfig, err := gif.DecodeAll(t.newReader()) 720 if err == nil { 721 if len(gifConfig.Image) >= 1 { 722 t.fileinfo.HasPreviewImage = false 723 724 } 725 if len(gifConfig.Image) > 0 { 726 t.decoded = gifConfig.Image[0] 727 t.imageType = "gif" 728 } 729 } 730 } 731 732 return nil 733 } 734 735 func (t *uploadFileTask) postprocessImage() { 736 // don't try to process SVG files 737 if t.fileinfo.MimeType == "image/svg+xml" { 738 return 739 } 740 741 decoded, typ := t.decoded, t.imageType 742 if decoded == nil { 743 var err error 744 decoded, typ, err = image.Decode(t.newReader()) 745 if err != nil { 746 mlog.Error(fmt.Sprintf("Unable to decode image err=%v", err)) 747 return 748 } 749 } 750 751 // Fill in the background of a potentially-transparent png file as 752 // white. 753 if typ == "png" { 754 dst := image.NewRGBA(decoded.Bounds()) 755 draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) 756 draw.Draw(dst, dst.Bounds(), decoded, decoded.Bounds().Min, draw.Over) 757 decoded = dst 758 } 759 760 decoded = makeImageUpright(decoded, t.imageOrientation) 761 if decoded == nil { 762 return 763 } 764 765 writeJPEG := func(img image.Image, path string) { 766 r, w := io.Pipe() 767 go func() { 768 _, aerr := t.writeFile(r, path) 769 if aerr != nil { 770 mlog.Error(fmt.Sprintf("Unable to upload path=%v err=%v", path, aerr)) 771 return 772 } 773 }() 774 775 err := jpeg.Encode(w, img, &jpeg.Options{Quality: 90}) 776 if err != nil { 777 mlog.Error(fmt.Sprintf("Unable to encode image as jpeg path=%v err=%v", path, err)) 778 w.CloseWithError(err) 779 } else { 780 w.Close() 781 } 782 } 783 784 w := decoded.Bounds().Dx() 785 h := decoded.Bounds().Dy() 786 787 wg := &sync.WaitGroup{} 788 wg.Add(2) 789 go func() { 790 defer wg.Done() 791 thumb := decoded 792 if h > ImageThumbnailHeight || w > ImageThumbnailWidth { 793 if float64(h)/float64(w) < ImageThumbnailRatio { 794 thumb = imaging.Resize(decoded, 0, ImageThumbnailHeight, imaging.Lanczos) 795 } else { 796 thumb = imaging.Resize(decoded, ImageThumbnailWidth, 0, imaging.Lanczos) 797 } 798 } 799 writeJPEG(thumb, t.fileinfo.ThumbnailPath) 800 }() 801 802 go func() { 803 defer wg.Done() 804 preview := decoded 805 if w > ImagePreviewWidth { 806 preview = imaging.Resize(decoded, ImagePreviewWidth, 0, imaging.Lanczos) 807 } 808 writeJPEG(preview, t.fileinfo.PreviewPath) 809 }() 810 wg.Wait() 811 } 812 813 func (t uploadFileTask) newReader() io.Reader { 814 if t.teeInput != nil { 815 return io.MultiReader(bytes.NewReader(t.buf.Bytes()), t.teeInput) 816 } else { 817 return bytes.NewReader(t.buf.Bytes()) 818 } 819 } 820 821 func (t uploadFileTask) pathPrefix() string { 822 return t.Timestamp.Format("20060102") + 823 "/teams/" + t.TeamId + 824 "/channels/" + t.ChannelId + 825 "/users/" + t.UserId + 826 "/" + t.fileinfo.Id + "/" 827 } 828 829 func (t uploadFileTask) newAppError(id string, details interface{}, httpStatus int, extra ...interface{}) *model.AppError { 830 params := map[string]interface{}{ 831 "Name": t.Name, 832 "Filename": t.Name, 833 "ChannelId": t.ChannelId, 834 "TeamId": t.TeamId, 835 "UserId": t.UserId, 836 "ContentLength": t.ContentLength, 837 "ClientId": t.ClientId, 838 } 839 if t.fileinfo != nil { 840 params["Width"] = t.fileinfo.Width 841 params["Height"] = t.fileinfo.Height 842 } 843 for i := 0; i+1 < len(extra); i += 2 { 844 params[fmt.Sprintf("%v", extra[i])] = extra[i+1] 845 } 846 847 return model.NewAppError("uploadFileTask", id, params, fmt.Sprintf("%v", details), httpStatus) 848 } 849 850 func (a *App) DoUploadFileExpectModification(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError) { 851 filename := filepath.Base(rawFilename) 852 teamId := filepath.Base(rawTeamId) 853 channelId := filepath.Base(rawChannelId) 854 userId := filepath.Base(rawUserId) 855 856 info, err := model.GetInfoForBytes(filename, data) 857 if err != nil { 858 err.StatusCode = http.StatusBadRequest 859 return nil, data, err 860 } 861 862 if orientation, err := getImageOrientation(bytes.NewReader(data)); err == nil && 863 (orientation == RotatedCWMirrored || 864 orientation == RotatedCCW || 865 orientation == RotatedCCWMirrored || 866 orientation == RotatedCW) { 867 info.Width, info.Height = info.Height, info.Width 868 } 869 870 info.Id = model.NewId() 871 info.CreatorId = userId 872 info.CreateAt = now.UnixNano() / int64(time.Millisecond) 873 874 pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" 875 info.Path = pathPrefix + filename 876 877 if info.IsImage() { 878 // Check dimensions before loading the whole thing into memory later on 879 if info.Width*info.Height > MaxImageSize { 880 err := model.NewAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "", http.StatusBadRequest) 881 return nil, data, err 882 } 883 884 nameWithoutExtension := filename[:strings.LastIndex(filename, ".")] 885 info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" 886 info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" 887 } 888 889 if pluginsEnvironment := a.GetPluginsEnvironment(); pluginsEnvironment != nil { 890 var rejectionError *model.AppError 891 pluginContext := a.PluginContext() 892 pluginsEnvironment.RunMultiPluginHook(func(hooks plugin.Hooks) bool { 893 var newBytes bytes.Buffer 894 replacementInfo, rejectionReason := hooks.FileWillBeUploaded(pluginContext, info, bytes.NewReader(data), &newBytes) 895 if rejectionReason != "" { 896 rejectionError = model.NewAppError("DoUploadFile", "File rejected by plugin. "+rejectionReason, nil, "", http.StatusBadRequest) 897 return false 898 } 899 if replacementInfo != nil { 900 info = replacementInfo 901 } 902 if newBytes.Len() != 0 { 903 data = newBytes.Bytes() 904 info.Size = int64(len(data)) 905 } 906 907 return true 908 }, plugin.FileWillBeUploadedId) 909 if rejectionError != nil { 910 return nil, data, rejectionError 911 } 912 } 913 914 if _, err := a.WriteFile(bytes.NewReader(data), info.Path); err != nil { 915 return nil, data, err 916 } 917 918 if _, err := a.Srv.Store.FileInfo().Save(info); err != nil { 919 return nil, data, err 920 } 921 922 return info, data, nil 923 } 924 925 func (a *App) HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { 926 wg := new(sync.WaitGroup) 927 928 for i := range fileData { 929 img, width, height := prepareImage(fileData[i]) 930 if img != nil { 931 wg.Add(2) 932 go func(img image.Image, path string, width int, height int) { 933 defer wg.Done() 934 a.generateThumbnailImage(img, path, width, height) 935 }(img, thumbnailPathList[i], width, height) 936 937 go func(img image.Image, path string, width int) { 938 defer wg.Done() 939 a.generatePreviewImage(img, path, width) 940 }(img, previewPathList[i], width) 941 } 942 } 943 wg.Wait() 944 } 945 946 func prepareImage(fileData []byte) (image.Image, int, int) { 947 // Decode image bytes into Image object 948 img, imgType, err := image.Decode(bytes.NewReader(fileData)) 949 if err != nil { 950 mlog.Error(fmt.Sprintf("Unable to decode image err=%v", err)) 951 return nil, 0, 0 952 } 953 954 width := img.Bounds().Dx() 955 height := img.Bounds().Dy() 956 957 // Fill in the background of a potentially-transparent png file as white 958 if imgType == "png" { 959 dst := image.NewRGBA(img.Bounds()) 960 draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) 961 draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over) 962 img = dst 963 } 964 965 // Flip the image to be upright 966 orientation, _ := getImageOrientation(bytes.NewReader(fileData)) 967 img = makeImageUpright(img, orientation) 968 969 return img, width, height 970 } 971 972 func makeImageUpright(img image.Image, orientation int) image.Image { 973 switch orientation { 974 case UprightMirrored: 975 return imaging.FlipH(img) 976 case UpsideDown: 977 return imaging.Rotate180(img) 978 case UpsideDownMirrored: 979 return imaging.FlipV(img) 980 case RotatedCWMirrored: 981 return imaging.Transpose(img) 982 case RotatedCCW: 983 return imaging.Rotate270(img) 984 case RotatedCCWMirrored: 985 return imaging.Transverse(img) 986 case RotatedCW: 987 return imaging.Rotate90(img) 988 default: 989 return img 990 } 991 } 992 993 func getImageOrientation(input io.Reader) (int, error) { 994 exifData, err := exif.Decode(input) 995 if err != nil { 996 return Upright, err 997 } 998 999 tag, err := exifData.Get("Orientation") 1000 if err != nil { 1001 return Upright, err 1002 } 1003 1004 orientation, err := tag.Int(0) 1005 if err != nil { 1006 return Upright, err 1007 } 1008 1009 return orientation, nil 1010 } 1011 1012 func (a *App) generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { 1013 thumbWidth := float64(IMAGE_THUMBNAIL_PIXEL_WIDTH) 1014 thumbHeight := float64(IMAGE_THUMBNAIL_PIXEL_HEIGHT) 1015 imgWidth := float64(width) 1016 imgHeight := float64(height) 1017 1018 var thumbnail image.Image 1019 if imgHeight < IMAGE_THUMBNAIL_PIXEL_HEIGHT && imgWidth < thumbWidth { 1020 thumbnail = img 1021 } else if imgHeight/imgWidth < thumbHeight/thumbWidth { 1022 thumbnail = imaging.Resize(img, 0, IMAGE_THUMBNAIL_PIXEL_HEIGHT, imaging.Lanczos) 1023 } else { 1024 thumbnail = imaging.Resize(img, IMAGE_THUMBNAIL_PIXEL_WIDTH, 0, imaging.Lanczos) 1025 } 1026 1027 buf := new(bytes.Buffer) 1028 if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil { 1029 mlog.Error(fmt.Sprintf("Unable to encode image as jpeg path=%v err=%v", thumbnailPath, err)) 1030 return 1031 } 1032 1033 if _, err := a.WriteFile(buf, thumbnailPath); err != nil { 1034 mlog.Error(fmt.Sprintf("Unable to upload thumbnail path=%v err=%v", thumbnailPath, err)) 1035 return 1036 } 1037 } 1038 1039 func (a *App) generatePreviewImage(img image.Image, previewPath string, width int) { 1040 var preview image.Image 1041 1042 if width > IMAGE_PREVIEW_PIXEL_WIDTH { 1043 preview = imaging.Resize(img, IMAGE_PREVIEW_PIXEL_WIDTH, 0, imaging.Lanczos) 1044 } else { 1045 preview = img 1046 } 1047 1048 buf := new(bytes.Buffer) 1049 1050 if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil { 1051 mlog.Error(fmt.Sprintf("Unable to encode image as preview jpg err=%v", err), mlog.String("path", previewPath)) 1052 return 1053 } 1054 1055 if _, err := a.WriteFile(buf, previewPath); err != nil { 1056 mlog.Error(fmt.Sprintf("Unable to upload preview err=%v", err), mlog.String("path", previewPath)) 1057 return 1058 } 1059 } 1060 1061 func (a *App) GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) { 1062 return a.Srv.Store.FileInfo().Get(fileId) 1063 } 1064 1065 func (a *App) GetFile(fileId string) ([]byte, *model.AppError) { 1066 info, err := a.GetFileInfo(fileId) 1067 if err != nil { 1068 return nil, err 1069 } 1070 1071 data, err := a.ReadFile(info.Path) 1072 if err != nil { 1073 return nil, err 1074 } 1075 1076 return data, nil 1077 } 1078 1079 func (a *App) CopyFileInfos(userId string, fileIds []string) ([]string, *model.AppError) { 1080 var newFileIds []string 1081 1082 now := model.GetMillis() 1083 1084 for _, fileId := range fileIds { 1085 fileInfo, err := a.Srv.Store.FileInfo().Get(fileId) 1086 if err != nil { 1087 return nil, err 1088 } 1089 1090 fileInfo.Id = model.NewId() 1091 fileInfo.CreatorId = userId 1092 fileInfo.CreateAt = now 1093 fileInfo.UpdateAt = now 1094 fileInfo.PostId = "" 1095 1096 if _, err := a.Srv.Store.FileInfo().Save(fileInfo); err != nil { 1097 return newFileIds, err 1098 } 1099 1100 newFileIds = append(newFileIds, fileInfo.Id) 1101 } 1102 1103 return newFileIds, nil 1104 }