github.com/upyun/upx@v0.4.7-0.20240419023638-b184a7cb7c10/session.go (about) 1 package upx 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io/fs" 8 "io/ioutil" 9 "log" 10 "math/rand" 11 "net/http" 12 "net/url" 13 "os" 14 "os/signal" 15 "path" 16 "path/filepath" 17 "strings" 18 "sync" 19 "time" 20 21 "github.com/fatih/color" 22 "github.com/upyun/go-sdk/v3/upyun" 23 "github.com/upyun/upx/fsutil" 24 "github.com/upyun/upx/partial" 25 "github.com/upyun/upx/processbar" 26 "github.com/vbauerster/mpb/v8" 27 ) 28 29 const ( 30 SYNC_EXISTS = iota 31 SYNC_OK 32 SYNC_FAIL 33 SYNC_NOT_FOUND 34 DELETE_OK 35 DELETE_FAIL 36 37 MinResumePutFileSize = 100 * 1024 * 1024 38 DefaultBlockSize = 10 * 1024 * 1024 39 DefaultResumeRetry = 10 40 ) 41 42 type Session struct { 43 Bucket string `json:"bucket"` 44 Operator string `json:"username"` 45 Password string `json:"password"` 46 CWD string `json:"cwd"` 47 48 updriver *upyun.UpYun 49 color bool 50 51 scores map[int]int 52 smu sync.RWMutex 53 54 taskChan chan interface{} 55 } 56 57 type syncTask struct { 58 src, dest string 59 isdir bool 60 } 61 62 type delTask struct { 63 src, dest string 64 isdir bool 65 } 66 67 type UploadedFile struct { 68 barId int 69 LocalPath string 70 UpPath string 71 LocalInfo os.FileInfo 72 } 73 74 var ( 75 session *Session 76 ) 77 78 func (sess *Session) update(key int) { 79 sess.smu.Lock() 80 sess.scores[key]++ 81 sess.smu.Unlock() 82 } 83 84 func (sess *Session) dump() string { 85 s := make(map[string]string) 86 titles := []string{"SYNC_EXISTS", "SYNC_OK", "SYNC_FAIL", "SYNC_NOT_FOUND", "DELETE_OK", "DELETE_FAIL"} 87 for i, title := range titles { 88 v := fmt.Sprint(sess.scores[i]) 89 if len(v) > len(title) { 90 title = strings.Repeat(" ", len(v)-len(title)) + title 91 } else { 92 v = strings.Repeat(" ", len(title)-len(v)) + v 93 } 94 s[title] = v 95 } 96 header := "+" 97 for _, title := range titles { 98 header += strings.Repeat("=", len(s[title])+2) + "+" 99 } 100 header += "\n" 101 footer := strings.Replace(header, "=", "-", -1) 102 103 ret := "\n\n" + header 104 ret += "|" 105 for _, title := range titles { 106 ret += " " + title + " |" 107 } 108 ret += "\n" + footer 109 110 ret += "|" 111 for _, title := range titles { 112 ret += " " + s[title] + " |" 113 } 114 return ret + "\n" + footer 115 } 116 117 func (sess *Session) AbsPath(upPath string) (ret string) { 118 if strings.HasPrefix(upPath, "/") { 119 ret = path.Join(upPath) 120 } else { 121 ret = path.Join(sess.CWD, upPath) 122 } 123 124 if strings.HasSuffix(upPath, "/") && ret != "/" { 125 ret += "/" 126 } 127 return 128 } 129 130 func (sess *Session) IsUpYunDir(upPath string) (isDir bool, exist bool) { 131 upInfo, err := sess.updriver.GetInfo(sess.AbsPath(upPath)) 132 if err != nil { 133 return false, false 134 } 135 return upInfo.IsDir, true 136 } 137 138 func (sess *Session) IsLocalDir(localPath string) (isDir bool, exist bool) { 139 fInfo, err := os.Stat(localPath) 140 if err != nil { 141 return false, false 142 } 143 return fInfo.IsDir(), true 144 } 145 146 func (sess *Session) FormatUpInfo(upInfo *upyun.FileInfo) string { 147 s := "drwxrwxrwx" 148 if !upInfo.IsDir { 149 s = "-rw-rw-rw-" 150 } 151 s += fmt.Sprintf(" 1 %s %s %12d", sess.Operator, sess.Bucket, upInfo.Size) 152 if upInfo.Time.Year() != time.Now().Year() { 153 s += " " + upInfo.Time.Format("Jan 02 2006") 154 } else { 155 s += " " + upInfo.Time.Format("Jan 02 03:04") 156 } 157 if upInfo.IsDir && sess.color { 158 s += " " + color.BlueString(upInfo.Name) 159 } else { 160 s += " " + upInfo.Name 161 } 162 return s 163 } 164 165 func (sess *Session) Init() error { 166 sess.scores = make(map[int]int) 167 sess.updriver = upyun.NewUpYun(&upyun.UpYunConfig{ 168 Bucket: sess.Bucket, 169 Operator: sess.Operator, 170 Password: sess.Password, 171 UserAgent: fmt.Sprintf("upx/%s", VERSION), 172 }) 173 _, err := sess.updriver.Usage() 174 return err 175 } 176 177 func (sess *Session) Info() { 178 n, err := sess.updriver.Usage() 179 if err != nil { 180 PrintErrorAndExit("usage: %v", err) 181 } 182 183 tmp := []string{ 184 fmt.Sprintf("ServiceName: %s", sess.Bucket), 185 fmt.Sprintf("Operator: %s", sess.Operator), 186 fmt.Sprintf("CurrentDir: %s", sess.CWD), 187 fmt.Sprintf("Usage: %s", humanizeSize(n)), 188 } 189 190 Print(strings.Join(tmp, "\n")) 191 } 192 193 func (sess *Session) Pwd() { 194 Print("%s", sess.CWD) 195 } 196 197 func (sess *Session) Mkdir(upPaths ...string) { 198 for _, upPath := range upPaths { 199 fpath := sess.AbsPath(upPath) 200 for fpath != "/" { 201 if err := sess.updriver.Mkdir(fpath); err != nil { 202 PrintErrorAndExit("mkdir %s: %v", fpath, err) 203 } 204 fpath = path.Dir(fpath) 205 } 206 } 207 } 208 209 func (sess *Session) Cd(upPath string) { 210 fpath := sess.AbsPath(upPath) 211 if isDir, _ := sess.IsUpYunDir(fpath); isDir { 212 sess.CWD = fpath 213 Print(sess.CWD) 214 } else { 215 PrintErrorAndExit("cd: %s: Not a directory", fpath) 216 } 217 } 218 219 func (sess *Session) Ls(upPath string, match *MatchConfig, maxItems int, isDesc bool) { 220 fpath := sess.AbsPath(upPath) 221 isDir, exist := sess.IsUpYunDir(fpath) 222 if !exist { 223 PrintErrorAndExit("ls: cannot access %s: No such file or directory", fpath) 224 } 225 226 if !isDir { 227 fInfo, err := sess.updriver.GetInfo(fpath) 228 if err != nil { 229 PrintErrorAndExit("ls %s: %v", fpath, err) 230 } 231 if IsMatched(fInfo, match) { 232 Print(sess.FormatUpInfo(fInfo)) 233 } else { 234 PrintErrorAndExit("ls: cannot access %s: No such file or directory", fpath) 235 } 236 return 237 } 238 239 fInfoChan := make(chan *upyun.FileInfo, 50) 240 go func() { 241 err := sess.updriver.List(&upyun.GetObjectsConfig{ 242 Path: fpath, 243 ObjectsChan: fInfoChan, 244 DescOrder: isDesc, 245 }) 246 if err != nil { 247 PrintErrorAndExit("ls %s: %v", fpath, err) 248 } 249 }() 250 251 objs := 0 252 for fInfo := range fInfoChan { 253 if IsMatched(fInfo, match) { 254 Print(sess.FormatUpInfo(fInfo)) 255 objs++ 256 } 257 if maxItems > 0 && objs >= maxItems { 258 break 259 } 260 } 261 if objs == 0 && (match.Wildcard != "" || match.TimeType != TIME_NOT_SET) { 262 msg := fpath 263 if match.Wildcard != "" { 264 msg = fpath + "/" + match.Wildcard 265 } 266 if match.TimeType != TIME_NOT_SET { 267 msg += " timestamp@" 268 if match.TimeType == TIME_AFTER || match.TimeType == TIME_INTERVAL { 269 msg += "[" + match.After.Format("2006-01-02 15:04:05") + "," 270 } else { 271 msg += "[-oo," 272 } 273 if match.TimeType == TIME_BEFORE || match.TimeType == TIME_INTERVAL { 274 msg += match.Before.Format("2006-01-02 15:04:05") + "]" 275 } else { 276 msg += "+oo]" 277 } 278 } 279 PrintErrorAndExit("ls: cannot access %s: No such file or directory", msg) 280 } 281 } 282 283 func (sess *Session) getDir(upPath, localPath string, match *MatchConfig, workers int, resume bool) error { 284 if err := os.MkdirAll(localPath, 0755); err != nil { 285 return err 286 } 287 288 var wg sync.WaitGroup 289 290 fInfoChan := make(chan *upyun.FileInfo, workers*2) 291 wg.Add(workers) 292 for w := 0; w < workers; w++ { 293 go func() { 294 defer wg.Done() 295 var e error 296 for fInfo := range fInfoChan { 297 if IsMatched(fInfo, match) { 298 fpath := path.Join(upPath, fInfo.Name) 299 lpath := filepath.Join(localPath, filepath.FromSlash(fInfo.Name)) 300 if fInfo.IsDir { 301 os.MkdirAll(lpath, 0755) 302 } else { 303 isContinue := resume 304 305 // 判断本地文件是否存在 306 // 如果存在,大小一致 并且本地文件的最后修改时间大于云端文件的最后修改时间 则跳过该下载 307 // 如果云端文件最后的修改时间大于本地文件的创建时间,则强制重新下载 308 stat, err := os.Stat(lpath) 309 if err == nil { 310 if stat.Size() == fInfo.Size && stat.ModTime().After(fInfo.Time) { 311 continue 312 } 313 if stat.Size() > fInfo.Size { 314 isContinue = false 315 } 316 if fInfo.Time.After(stat.ModTime()) { 317 isContinue = false 318 } 319 } 320 321 for i := 1; i <= MaxRetry; i++ { 322 e = sess.getFileWithProgress(fpath, lpath, fInfo, 1, isContinue) 323 if e == nil { 324 break 325 } 326 if upyun.IsNotExist(e) { 327 e = nil 328 break 329 } 330 331 time.Sleep(time.Duration(i*(rand.Intn(MaxJitter-MinJitter)+MinJitter)) * time.Second) 332 } 333 } 334 if e != nil { 335 return 336 } 337 } 338 } 339 }() 340 } 341 342 err := sess.updriver.List(&upyun.GetObjectsConfig{ 343 Path: upPath, 344 ObjectsChan: fInfoChan, 345 MaxListTries: 3, 346 MaxListLevel: -1, 347 }) 348 wg.Wait() 349 return err 350 } 351 352 func (sess *Session) getFileWithProgress(upPath, localPath string, upInfo *upyun.FileInfo, works int, resume bool) error { 353 var err error 354 355 var bar *mpb.Bar 356 if upInfo.Size > 0 { 357 bar = processbar.ProcessBar.AddBar(localPath, upInfo.Size) 358 } 359 360 dir := filepath.Dir(localPath) 361 if err = os.MkdirAll(dir, 0755); err != nil { 362 return err 363 } 364 365 w, err := NewFileWrappedWriter(localPath, bar, resume) 366 if err != nil { 367 return err 368 } 369 defer w.Close() 370 371 downloader := partial.NewMultiPartialDownloader( 372 localPath, 373 upInfo.Size, 374 partial.DefaultChunkSize, 375 w, 376 works, 377 func(start, end int64) ([]byte, error) { 378 var buffer bytes.Buffer 379 _, err = sess.updriver.Get(&upyun.GetObjectConfig{ 380 Path: sess.AbsPath(upPath), 381 Writer: &buffer, 382 Headers: map[string]string{ 383 "Range": fmt.Sprintf("bytes=%d-%d", start, end), 384 }, 385 }) 386 return buffer.Bytes(), err 387 }, 388 ) 389 err = downloader.Download() 390 if bar != nil { 391 bar.EnableTriggerComplete() 392 if err != nil { 393 bar.Abort(false) 394 } 395 } 396 return err 397 } 398 399 func (sess *Session) Get(upPath, localPath string, match *MatchConfig, workers int, resume bool) { 400 upPath = sess.AbsPath(upPath) 401 upInfo, err := sess.updriver.GetInfo(upPath) 402 if err != nil { 403 PrintErrorAndExit("getinfo %s: %v", upPath, err) 404 } 405 406 exist, isDir := false, false 407 if localInfo, _ := os.Stat(localPath); localInfo != nil { 408 exist = true 409 isDir = localInfo.IsDir() 410 } else { 411 if strings.HasSuffix(localPath, "/") { 412 isDir = true 413 } 414 } 415 416 if upInfo.IsDir { 417 if exist { 418 if !isDir { 419 PrintErrorAndExit("get: %s Not a directory", localPath) 420 } else { 421 if match.Wildcard == "" { 422 localPath = filepath.Join(localPath, path.Base(upPath)) 423 } 424 } 425 } 426 if err := sess.getDir(upPath, localPath, match, workers, resume); err != nil { 427 PrintErrorAndExit(err.Error()) 428 } 429 } else { 430 if isDir { 431 localPath = filepath.Join(localPath, path.Base(upPath)) 432 } 433 434 // 小于 100M 不开启多线程 435 if upInfo.Size < 1024*1024*100 { 436 workers = 1 437 } 438 err := sess.getFileWithProgress(upPath, localPath, upInfo, workers, resume) 439 if err != nil { 440 PrintErrorAndExit(err.Error()) 441 } 442 } 443 } 444 445 func (sess *Session) GetStartBetweenEndFiles(upPath, localPath string, match *MatchConfig, workers int) { 446 fpath := sess.AbsPath(upPath) 447 isDir, exist := sess.IsUpYunDir(fpath) 448 if !exist { 449 if match.ItemType == DIR { 450 isDir = true 451 } else { 452 PrintErrorAndExit("get: cannot down %s:No such file or directory", fpath) 453 } 454 } 455 if isDir && match != nil && match.Wildcard == "" { 456 if match.ItemType == FILE { 457 PrintErrorAndExit("get: cannot down %s: Is a directory", fpath) 458 } 459 } 460 461 fInfoChan := make(chan *upyun.FileInfo, 1) 462 objectsConfig := &upyun.GetObjectsConfig{ 463 Path: fpath, 464 ObjectsChan: fInfoChan, 465 QuitChan: make(chan bool, 1), 466 } 467 go func() { 468 err := sess.updriver.List(objectsConfig) 469 if err != nil { 470 PrintErrorAndExit("ls %s: %v", fpath, err) 471 } 472 }() 473 474 startList := match.Start 475 if startList != "" && startList[0] != '/' { 476 startList = filepath.Join(fpath, startList) 477 } 478 endList := match.End 479 if endList != "" && endList[0] != '/' { 480 endList = filepath.Join(fpath, endList) 481 } 482 483 for fInfo := range fInfoChan { 484 fp := filepath.Join(fpath, fInfo.Name) 485 if (fp >= startList || startList == "") && (fp < endList || endList == "") { 486 sess.Get(fp, localPath, match, workers, false) 487 } else if strings.HasPrefix(startList, fp) { 488 //前缀相同进入下一级文件夹,继续递归判断 489 if fInfo.IsDir { 490 sess.GetStartBetweenEndFiles(fp, localPath+fInfo.Name+"/", match, workers) 491 } 492 } 493 if fp >= endList && endList != "" && fInfo.IsDir { 494 close(objectsConfig.QuitChan) 495 break 496 } 497 } 498 } 499 500 func (sess *Session) putFileWithProgress(localPath, upPath string, localInfo os.FileInfo) error { 501 var err error 502 fd, err := os.Open(localPath) 503 if err != nil { 504 return err 505 } 506 defer fd.Close() 507 cfg := &upyun.PutObjectConfig{ 508 Path: upPath, 509 Headers: map[string]string{ 510 "Content-Length": fmt.Sprint(localInfo.Size()), 511 }, 512 Reader: fd, 513 } 514 515 var bar *mpb.Bar 516 if IsVerbose { 517 if localInfo.Size() > 0 { 518 bar = processbar.ProcessBar.AddBar(upPath, localInfo.Size()) 519 cfg.Reader = NewFileWrappedReader(bar, fd) 520 } 521 } else { 522 log.Printf("file: %s, Start\n", upPath) 523 if localInfo.Size() >= MinResumePutFileSize { 524 cfg.UseResumeUpload = true 525 cfg.ResumePartSize = DefaultBlockSize 526 cfg.MaxResumePutTries = DefaultResumeRetry 527 } 528 } 529 err = sess.updriver.Put(cfg) 530 if bar != nil { 531 bar.EnableTriggerComplete() 532 if err != nil { 533 bar.Abort(false) 534 } 535 } 536 if !IsVerbose { 537 log.Printf("file: %s, Done\n", upPath) 538 } 539 return err 540 } 541 542 func (sess *Session) putRemoteFileWithProgress(rawURL, upPath string) error { 543 var size int64 544 545 // 如果可以的话,先从 Head 请求中获取文件长度 546 resp, err := http.Head(rawURL) 547 if err == nil && resp.ContentLength > 0 { 548 size = resp.ContentLength 549 } 550 resp.Body.Close() 551 552 // 通过get方法获取文件,如果get头中包含Content-Length,则使用get头中的Content-Length 553 resp, err = http.Get(rawURL) 554 if err != nil { 555 return fmt.Errorf("http Get %s error: %v", rawURL, err) 556 } 557 defer resp.Body.Close() 558 559 if resp.ContentLength > 0 { 560 size = resp.ContentLength 561 } 562 563 // 如果无法获取 Content-Length 则报错 564 if size == 0 { 565 return fmt.Errorf("get http file Content-Length error: response headers not has Content-Length") 566 } 567 568 // 创建进度条 569 bar := processbar.ProcessBar.AddBar(upPath, size) 570 reader := NewFileWrappedReader(bar, resp.Body) 571 572 // 上传文件 573 err = sess.updriver.Put(&upyun.PutObjectConfig{ 574 Path: upPath, 575 Reader: reader, 576 UseMD5: false, 577 Headers: map[string]string{ 578 "Content-Length": fmt.Sprint(size), 579 }, 580 }) 581 if bar != nil { 582 bar.EnableTriggerComplete() 583 if err != nil { 584 bar.Abort(false) 585 } 586 } 587 if err != nil { 588 PrintErrorAndExit("put file error: %v", err) 589 } 590 591 return nil 592 } 593 594 func (sess *Session) putFilesWitchProgress(localFiles []*UploadedFile, workers int) { 595 var wg sync.WaitGroup 596 597 tasks := make(chan *UploadedFile, workers*2) 598 for w := 0; w < workers; w++ { 599 wg.Add(1) 600 go func() { 601 defer wg.Done() 602 for task := range tasks { 603 err := sess.putFileWithProgress( 604 task.LocalPath, 605 task.UpPath, 606 task.LocalInfo, 607 ) 608 if err != nil { 609 fmt.Println("putFileWithProgress error: ", err.Error()) 610 return 611 } 612 } 613 }() 614 } 615 616 for _, f := range localFiles { 617 tasks <- f 618 } 619 620 close(tasks) 621 wg.Wait() 622 } 623 624 func (sess *Session) putDir(localPath, upPath string, workers int, withIgnore bool) { 625 localAbsPath, err := filepath.Abs(localPath) 626 if err != nil { 627 PrintErrorAndExit(err.Error()) 628 } 629 // 如果上传的是目录,并且是隐藏的目录,则触发提示 630 rootDirInfo, err := os.Stat(localAbsPath) 631 if err != nil { 632 PrintErrorAndExit(err.Error()) 633 } 634 if !withIgnore && fsutil.IsIgnoreFile(localAbsPath, rootDirInfo) { 635 PrintErrorAndExit("%s is a ignore dir, use `-all` to force put all files", localAbsPath) 636 } 637 638 type FileInfo struct { 639 fpath string 640 fInfo os.FileInfo 641 } 642 localFiles := make(chan *FileInfo, workers*2) 643 var wg sync.WaitGroup 644 wg.Add(workers) 645 for w := 0; w < workers; w++ { 646 go func() { 647 defer wg.Done() 648 for info := range localFiles { 649 rel, _ := filepath.Rel(localAbsPath, info.fpath) 650 desPath := path.Join(upPath, filepath.ToSlash(rel)) 651 fInfo, err := os.Stat(info.fpath) 652 if err == nil && fInfo.IsDir() { 653 err = sess.updriver.Mkdir(desPath) 654 } else { 655 err = sess.putFileWithProgress(info.fpath, desPath, info.fInfo) 656 } 657 if err != nil { 658 log.Printf("put %s to %s error: %s", info.fpath, desPath, err) 659 if upyun.IsTooManyRequests(err) { 660 time.Sleep(time.Second) 661 continue 662 } 663 return 664 } 665 } 666 }() 667 } 668 669 filepath.Walk(localAbsPath, func(path string, info fs.FileInfo, err error) error { 670 if err != nil { 671 return err 672 } 673 if !withIgnore && fsutil.IsIgnoreFile(path, info) { 674 if info.IsDir() { 675 return filepath.SkipDir 676 } 677 } else { 678 localFiles <- &FileInfo{ 679 fpath: path, 680 fInfo: info, 681 } 682 } 683 return nil 684 }) 685 686 close(localFiles) 687 wg.Wait() 688 } 689 690 // / Put 上传单文件或单目录 691 func (sess *Session) Put(localPath, upPath string, workers int, withIgnore bool) { 692 upPath = sess.AbsPath(upPath) 693 694 exist, isDir := false, false 695 if upInfo, _ := sess.updriver.GetInfo(upPath); upInfo != nil { 696 exist = true 697 isDir = upInfo.IsDir 698 } 699 // 如果指定了是远程的目录 但是实际在远程的目录是文件类型则报错 700 if exist && !isDir && strings.HasSuffix(upPath, "/") { 701 PrintErrorAndExit("cant put to %s: path is not a directory, maybe a file", upPath) 702 } 703 if !exist && strings.HasSuffix(upPath, "/") { 704 isDir = true 705 } 706 707 // 如果需要上传的文件是URL链接 708 fileURL, _ := url.ParseRequestURI(localPath) 709 if fileURL != nil && fileURL.Scheme != "" && fileURL.Host != "" { 710 if !contains([]string{"http", "https"}, fileURL.Scheme) { 711 PrintErrorAndExit("Invalid URL %s", localPath) 712 } 713 714 // 如果指定的远程路径 upPath 是目录 715 // 则从 url 中获取文件名,获取文件名失败则报错 716 if isDir { 717 if spaces := strings.Split(fileURL.Path, "/"); len(spaces) > 0 { 718 upPath = path.Join(upPath, spaces[len(spaces)-1]) 719 } else { 720 PrintErrorAndExit("missing file name in the url, must has remote path name") 721 } 722 } 723 err := sess.putRemoteFileWithProgress(localPath, upPath) 724 if err != nil { 725 PrintErrorAndExit(err.Error()) 726 } 727 return 728 } 729 730 localInfo, err := os.Stat(localPath) 731 if err != nil { 732 PrintErrorAndExit("stat %s: %v", localPath, err) 733 } 734 735 if localInfo.IsDir() { 736 if exist { 737 if !isDir { 738 PrintErrorAndExit("put: %s: Not a directory", upPath) 739 } else { 740 upPath = path.Join(upPath, filepath.Base(localPath)) 741 } 742 } 743 sess.putDir(localPath, upPath, workers, withIgnore) 744 } else { 745 if isDir { 746 upPath = path.Join(upPath, filepath.Base(localPath)) 747 } 748 sess.putFileWithProgress(localPath, upPath, localInfo) 749 } 750 } 751 752 // put 的升级版命令, 支持多文件上传 753 func (sess *Session) Upload(filenames []string, upPath string, workers int, withIgnore bool) { 754 upPath = sess.AbsPath(upPath) 755 756 // 检测云端的目的地目录 757 upPathExist, upPathIsDir := false, false 758 if upInfo, _ := sess.updriver.GetInfo(upPath); upInfo != nil { 759 upPathExist = true 760 upPathIsDir = upInfo.IsDir 761 } 762 // 多文件上传 upPath 如果存在则只能是目录 763 if upPathExist && !upPathIsDir { 764 PrintErrorAndExit("upload: %s: Not a directory", upPath) 765 } 766 767 var ( 768 dirs []string 769 uploadedFile []*UploadedFile 770 ) 771 for _, filename := range filenames { 772 localInfo, err := os.Stat(filename) 773 if err != nil { 774 PrintErrorAndExit(err.Error()) 775 } 776 777 if localInfo.IsDir() { 778 dirs = append(dirs, filename) 779 } else { 780 uploadedFile = append(uploadedFile, &UploadedFile{ 781 barId: -1, 782 LocalPath: filename, 783 UpPath: path.Join(upPath, filepath.Base(filename)), 784 LocalInfo: localInfo, 785 }) 786 } 787 } 788 789 // 上传目录 790 for _, localPath := range dirs { 791 sess.putDir( 792 localPath, 793 path.Join(upPath, filepath.Base(localPath)), 794 workers, 795 withIgnore, 796 ) 797 } 798 799 // 上传文件 800 sess.putFilesWitchProgress(uploadedFile, workers) 801 } 802 803 func (sess *Session) rm(fpath string, isAsync bool, isFolder bool) { 804 err := sess.updriver.Delete(&upyun.DeleteObjectConfig{ 805 Path: fpath, 806 Async: isAsync, 807 Folder: isFolder, 808 }) 809 if err == nil || upyun.IsNotExist(err) { 810 sess.update(DELETE_OK) 811 PrintOnlyVerbose("DELETE %s OK", fpath) 812 } else { 813 sess.update(DELETE_FAIL) 814 PrintError("DELETE %s FAIL %v", fpath, err) 815 } 816 } 817 func (sess *Session) rmFile(fpath string, isAsync bool) { 818 sess.rm(fpath, isAsync, false) 819 } 820 821 func (sess *Session) rmEmptyDir(fpath string, isAsync bool) { 822 sess.rm(fpath, isAsync, true) 823 } 824 825 func (sess *Session) rmDir(fpath string, isAsync bool) { 826 fInfoChan := make(chan *upyun.FileInfo, 50) 827 go func() { 828 err := sess.updriver.List(&upyun.GetObjectsConfig{ 829 Path: fpath, 830 ObjectsChan: fInfoChan, 831 }) 832 if err != nil { 833 if upyun.IsNotExist(err) { 834 return 835 } else { 836 PrintErrorAndExit("ls %s: %v", fpath, err) 837 } 838 } 839 }() 840 841 for fInfo := range fInfoChan { 842 fp := path.Join(fpath, fInfo.Name) 843 if fInfo.IsDir { 844 sess.rmDir(fp, isAsync) 845 } else { 846 sess.rmFile(fp, isAsync) 847 } 848 } 849 sess.rmEmptyDir(fpath, isAsync) 850 } 851 852 func (sess *Session) Rm(upPath string, match *MatchConfig, isAsync bool) { 853 fpath := sess.AbsPath(upPath) 854 isDir, exist := sess.IsUpYunDir(fpath) 855 if !exist { 856 if match.ItemType == DIR { 857 isDir = true 858 } else { 859 PrintErrorAndExit("rm: cannot remove %s: No such file or directory", fpath) 860 } 861 } 862 863 if isDir && match != nil && match.Wildcard == "" { 864 if match.ItemType == FILE { 865 PrintErrorAndExit("rm: cannot remove %s: Is a directory, add -d/-a flag", fpath) 866 } 867 sess.rmDir(fpath, isAsync) 868 return 869 } 870 871 if !isDir { 872 fInfo, err := sess.updriver.GetInfo(fpath) 873 if err != nil { 874 PrintErrorAndExit("getinfo %s: %v", fpath, err) 875 } 876 if IsMatched(fInfo, match) { 877 sess.rmFile(fpath, isAsync) 878 } 879 return 880 } 881 882 fInfoChan := make(chan *upyun.FileInfo, 50) 883 go func() { 884 err := sess.updriver.List(&upyun.GetObjectsConfig{ 885 Path: fpath, 886 ObjectsChan: fInfoChan, 887 }) 888 if err != nil { 889 PrintErrorAndExit("ls %s: %v", fpath, err) 890 } 891 }() 892 893 for fInfo := range fInfoChan { 894 fp := path.Join(fpath, fInfo.Name) 895 if IsMatched(fInfo, match) { 896 if fInfo.IsDir { 897 sess.rmDir(fp, isAsync) 898 } else { 899 sess.rmFile(fp, isAsync) 900 } 901 } 902 } 903 } 904 905 func (sess *Session) tree(upPath, prefix string, output chan string) (folders, files int, err error) { 906 upInfos := make(chan *upyun.FileInfo, 50) 907 fpath := sess.AbsPath(upPath) 908 wg := sync.WaitGroup{} 909 wg.Add(1) 910 911 go func() { 912 defer wg.Done() 913 prevInfo := <-upInfos 914 for fInfo := range upInfos { 915 p := prefix + "|-- " 916 if prevInfo.IsDir { 917 if sess.color { 918 output <- p + color.BlueString("%s", prevInfo.Name) 919 } else { 920 output <- p + prevInfo.Name 921 } 922 folders++ 923 d, f, _ := sess.tree(path.Join(fpath, prevInfo.Name), prefix+"! ", output) 924 folders += d 925 files += f 926 } else { 927 output <- p + prevInfo.Name 928 files++ 929 } 930 prevInfo = fInfo 931 } 932 if prevInfo == nil { 933 return 934 } 935 p := prefix + "`-- " 936 if prevInfo.IsDir { 937 if sess.color { 938 output <- p + color.BlueString("%s", prevInfo.Name) 939 } else { 940 output <- p + prevInfo.Name 941 } 942 folders++ 943 d, f, _ := sess.tree(path.Join(fpath, prevInfo.Name), prefix+" ", output) 944 folders += d 945 files += f 946 } else { 947 output <- p + prevInfo.Name 948 files++ 949 } 950 }() 951 952 err = sess.updriver.List(&upyun.GetObjectsConfig{ 953 Path: fpath, 954 ObjectsChan: upInfos, 955 }) 956 wg.Wait() 957 return 958 } 959 960 func (sess *Session) Tree(upPath string) { 961 fpath := sess.AbsPath(upPath) 962 files, folders := 0, 0 963 defer func() { 964 Print("\n%d directories, %d files", folders, files) 965 }() 966 967 if isDir, _ := sess.IsUpYunDir(fpath); !isDir { 968 PrintErrorAndExit("%s [error opening dir]", fpath) 969 } 970 Print("%s", fpath) 971 972 output := make(chan string, 50) 973 go func() { 974 folders, files, _ = sess.tree(fpath, "", output) 975 close(output) 976 }() 977 978 for s := range output { 979 Print(s) 980 } 981 return 982 } 983 984 func (sess *Session) syncFile(localPath, upPath string, strongCheck bool) (status int, err error) { 985 curMeta, err := makeDBValue(localPath, false) 986 if err != nil { 987 if os.IsNotExist(err) { 988 return SYNC_NOT_FOUND, err 989 } 990 return SYNC_FAIL, err 991 } 992 if curMeta.IsDir == "true" { 993 return SYNC_FAIL, fmt.Errorf("file type changed") 994 } 995 996 if strongCheck { 997 upInfo, _ := sess.updriver.GetInfo(upPath) 998 if upInfo != nil { 999 curMeta.Md5, _ = md5File(localPath) 1000 if curMeta.Md5 == upInfo.MD5 { 1001 setDBValue(localPath, upPath, curMeta) 1002 return SYNC_EXISTS, nil 1003 } 1004 } 1005 } else { 1006 prevMeta, err := getDBValue(localPath, upPath) 1007 if err != nil { 1008 return SYNC_FAIL, err 1009 } 1010 1011 if prevMeta != nil { 1012 if curMeta.ModifyTime == prevMeta.ModifyTime { 1013 return SYNC_EXISTS, nil 1014 } 1015 curMeta.Md5, _ = md5File(localPath) 1016 if curMeta.Md5 == prevMeta.Md5 { 1017 setDBValue(localPath, upPath, curMeta) 1018 return SYNC_EXISTS, nil 1019 } 1020 } 1021 } 1022 1023 err = sess.updriver.Put(&upyun.PutObjectConfig{Path: upPath, LocalPath: localPath}) 1024 if err != nil { 1025 return SYNC_FAIL, err 1026 } 1027 setDBValue(localPath, upPath, curMeta) 1028 return SYNC_OK, nil 1029 } 1030 1031 func (sess *Session) syncObject(localPath, upPath string, isDir bool) { 1032 if isDir { 1033 status, err := sess.syncDirectory(localPath, upPath) 1034 switch status { 1035 case SYNC_OK: 1036 PrintOnlyVerbose("sync %s to %s OK", localPath, upPath) 1037 case SYNC_EXISTS: 1038 PrintOnlyVerbose("sync %s to %s EXISTS", localPath, upPath) 1039 case SYNC_FAIL, SYNC_NOT_FOUND: 1040 PrintError("sync %s to %s FAIL %v", localPath, upPath, err) 1041 } 1042 sess.update(status) 1043 } else { 1044 sess.taskChan <- &syncTask{src: localPath, dest: upPath} 1045 } 1046 } 1047 1048 func (sess *Session) syncDirectory(localPath, upPath string) (int, error) { 1049 delFunc := func(prevMeta *fileMeta) { 1050 sess.taskChan <- &delTask{ 1051 src: filepath.Join(localPath, prevMeta.Name), 1052 dest: path.Join(upPath, prevMeta.Name), 1053 isdir: prevMeta.IsDir, 1054 } 1055 } 1056 syncFunc := func(curMeta *fileMeta) { 1057 src := filepath.Join(localPath, curMeta.Name) 1058 dest := path.Join(upPath, curMeta.Name) 1059 sess.syncObject(src, dest, curMeta.IsDir) 1060 } 1061 1062 dbVal, err := getDBValue(localPath, upPath) 1063 if err != nil { 1064 return SYNC_FAIL, err 1065 } 1066 1067 curMetas, err := makeFileMetas(localPath) 1068 if err != nil { 1069 // if not exist, should sync next time 1070 if os.IsNotExist(err) { 1071 return SYNC_NOT_FOUND, err 1072 } 1073 return SYNC_FAIL, err 1074 } 1075 1076 status := SYNC_EXISTS 1077 var prevMetas []*fileMeta 1078 if dbVal != nil && dbVal.IsDir == "true" { 1079 prevMetas = dbVal.Items 1080 } else { 1081 if err = sess.updriver.Mkdir(upPath); err != nil { 1082 return SYNC_FAIL, err 1083 } 1084 status = SYNC_OK 1085 } 1086 1087 cur, curSize, prev, prevSize := 0, len(curMetas), 0, len(prevMetas) 1088 for cur < curSize && prev < prevSize { 1089 curMeta, prevMeta := curMetas[cur], prevMetas[prev] 1090 if curMeta.Name == prevMeta.Name { 1091 if curMeta.IsDir != prevMeta.IsDir { 1092 delFunc(prevMeta) 1093 } 1094 syncFunc(curMeta) 1095 prev++ 1096 cur++ 1097 } else if curMeta.Name > prevMeta.Name { 1098 delFunc(prevMeta) 1099 prev++ 1100 } else { 1101 syncFunc(curMeta) 1102 cur++ 1103 } 1104 } 1105 for ; cur < curSize; cur++ { 1106 syncFunc(curMetas[cur]) 1107 } 1108 for ; prev < prevSize; prev++ { 1109 delFunc(prevMetas[prev]) 1110 } 1111 1112 setDBValue(localPath, upPath, &dbValue{IsDir: "true", Items: curMetas}) 1113 return status, nil 1114 } 1115 1116 func (sess *Session) Sync(localPath, upPath string, workers int, delete, strong bool) { 1117 var wg sync.WaitGroup 1118 sess.taskChan = make(chan interface{}, workers*2) 1119 stopChan := make(chan bool, 1) 1120 sigChan := make(chan os.Signal, 1) 1121 signal.Notify(sigChan, os.Interrupt) 1122 1123 upPath = sess.AbsPath(upPath) 1124 localPath, _ = filepath.Abs(localPath) 1125 1126 if err := initDB(); err != nil { 1127 PrintErrorAndExit("sync: init database: %v", err) 1128 } 1129 1130 var delLock sync.Mutex 1131 for w := 0; w < workers; w++ { 1132 wg.Add(1) 1133 go func() { 1134 defer wg.Done() 1135 for task := range sess.taskChan { 1136 switch v := task.(type) { 1137 case *syncTask: 1138 stat, err := sess.syncFile(v.src, v.dest, strong) 1139 switch stat { 1140 case SYNC_OK: 1141 PrintOnlyVerbose("sync %s to %s OK", v.src, v.dest) 1142 case SYNC_EXISTS: 1143 PrintOnlyVerbose("sync %s to %s EXISTS", v.src, v.dest) 1144 case SYNC_FAIL, SYNC_NOT_FOUND: 1145 PrintError("sync %s to %s FAIL %v", v.src, v.dest, err) 1146 } 1147 sess.update(stat) 1148 case *delTask: 1149 if delete { 1150 delDBValue(v.src, v.dest) 1151 delLock.Lock() 1152 if v.isdir { 1153 sess.rmDir(v.dest, false) 1154 } else { 1155 sess.rmFile(v.dest, false) 1156 } 1157 delLock.Unlock() 1158 } 1159 } 1160 } 1161 }() 1162 } 1163 1164 go func() { 1165 wg.Wait() 1166 close(stopChan) 1167 }() 1168 1169 go func() { 1170 isDir, _ := sess.IsLocalDir(localPath) 1171 sess.syncObject(localPath, upPath, isDir) 1172 close(sess.taskChan) 1173 }() 1174 1175 select { 1176 case <-sigChan: 1177 PrintErrorAndExit("%s", sess.dump()) 1178 case <-stopChan: 1179 if sess.scores[SYNC_FAIL] > 0 || sess.scores[DELETE_FAIL] > 0 { 1180 PrintErrorAndExit("%s", sess.dump()) 1181 } else { 1182 Print("%s", sess.dump()) 1183 } 1184 } 1185 } 1186 func (sess *Session) PostTask(app, notify, taskFile string) { 1187 fd, err := os.Open(taskFile) 1188 if err != nil { 1189 PrintErrorAndExit("open %s: %v", taskFile, err) 1190 } 1191 1192 body, err := ioutil.ReadAll(fd) 1193 fd.Close() 1194 if err != nil { 1195 PrintErrorAndExit("read %s: %v", taskFile, err) 1196 } 1197 1198 var tasks []interface{} 1199 if err = json.Unmarshal(body, &tasks); err != nil { 1200 PrintErrorAndExit("json Unmarshal: %v", err) 1201 } 1202 1203 if notify == "" { 1204 notify = "https://httpbin.org/post" 1205 } 1206 ids, err := sess.updriver.CommitTasks(&upyun.CommitTasksConfig{ 1207 AppName: app, 1208 NotifyUrl: notify, 1209 Tasks: tasks, 1210 }) 1211 if err != nil { 1212 PrintErrorAndExit("commit tasks: %v", err) 1213 } 1214 Print("%v", ids) 1215 } 1216 1217 func (sess *Session) Purge(urls []string, file string) { 1218 if urls == nil { 1219 urls = make([]string, 0) 1220 } 1221 if file != "" { 1222 fd, err := os.Open(file) 1223 if err != nil { 1224 PrintErrorAndExit("open %s: %v", file, err) 1225 } 1226 body, err := ioutil.ReadAll(fd) 1227 fd.Close() 1228 if err != nil { 1229 PrintErrorAndExit("read %s: %v", file, err) 1230 } 1231 for _, line := range strings.Split(string(body), "\n") { 1232 if line == "" { 1233 continue 1234 } 1235 urls = append(urls, line) 1236 } 1237 } 1238 for idx := range urls { 1239 if !strings.HasPrefix(urls[idx], "http") { 1240 urls[idx] = "http://" + urls[idx] 1241 } 1242 } 1243 if len(urls) == 0 { 1244 return 1245 } 1246 1247 fails, err := sess.updriver.Purge(urls) 1248 if fails != nil && len(fails) != 0 { 1249 PrintError("Purge failed urls:") 1250 for _, url := range fails { 1251 PrintError("%s", url) 1252 } 1253 PrintErrorAndExit("too many fails") 1254 } 1255 if err != nil { 1256 PrintErrorAndExit("purge error: %v", err) 1257 } 1258 } 1259 1260 func (sess *Session) Copy(srcPath, destPath string, force bool) error { 1261 return sess.copyMove(srcPath, destPath, "copy", force) 1262 } 1263 1264 func (sess *Session) Move(srcPath, destPath string, force bool) error { 1265 return sess.copyMove(srcPath, destPath, "move", force) 1266 } 1267 1268 // 移动或者复制 1269 // method: "move" | "copy" 1270 // force: 是否覆盖目标文件 1271 func (sess *Session) copyMove(srcPath, destPath, method string, force bool) error { 1272 // 将源文件路径转化为绝对路径 1273 srcPath = sess.AbsPath(srcPath) 1274 1275 // 检测源文件 1276 sourceFileInfo, err := sess.updriver.GetInfo(srcPath) 1277 if err != nil { 1278 if upyun.IsNotExist(err) { 1279 return fmt.Errorf("source file %s is not exist", srcPath) 1280 } 1281 return err 1282 } 1283 if sourceFileInfo.IsDir { 1284 return fmt.Errorf("not support dir, %s is dir", srcPath) 1285 } 1286 1287 // 将目标路径转化为绝对路径 1288 destPath = sess.AbsPath(destPath) 1289 1290 destFileInfo, err := sess.updriver.GetInfo(destPath) 1291 // 如果返回的错误不是文件不存在错误,则返回错误 1292 if err != nil && !upyun.IsNotExist(err) { 1293 return err 1294 } 1295 // 如果没有错误,表示文件存在,则检测文件类型,并判断是否允许覆盖 1296 if err == nil { 1297 if !destFileInfo.IsDir { 1298 // 如果目标文件是文件类型,则需要使用强制覆盖 1299 if !force { 1300 return fmt.Errorf( 1301 "target path %s already exists use -f to force overwrite", 1302 destPath, 1303 ) 1304 } 1305 } else { 1306 // 补全文件名后,再次检测文件存不存在 1307 destPath = path.Join(destPath, path.Base(srcPath)) 1308 destFileInfo, err := sess.updriver.GetInfo(destPath) 1309 if err == nil { 1310 if destFileInfo.IsDir { 1311 return fmt.Errorf( 1312 "target file %s already exists and is dir", 1313 destPath, 1314 ) 1315 } 1316 if !force { 1317 return fmt.Errorf( 1318 "target file %s already exists use -f to force overwrite", 1319 destPath, 1320 ) 1321 } 1322 } 1323 } 1324 } 1325 1326 if srcPath == destPath { 1327 return fmt.Errorf( 1328 "source and target are the same %s => %s", 1329 srcPath, 1330 destPath, 1331 ) 1332 } 1333 1334 switch method { 1335 case "copy": 1336 return sess.updriver.Copy(&upyun.CopyObjectConfig{ 1337 SrcPath: srcPath, 1338 DestPath: destPath, 1339 }) 1340 case "move": 1341 return sess.updriver.Move(&upyun.MoveObjectConfig{ 1342 SrcPath: srcPath, 1343 DestPath: destPath, 1344 }) 1345 default: 1346 return fmt.Errorf("not support method") 1347 } 1348 }