github.com/qjfoidnh/BaiduPCS-Go@v0.0.0-20231011165705-caa18a3765f3/baidupcs/extends.go (about)

     1  package baidupcs
     2  
     3  import (
     4  	"crypto/md5"
     5  	"encoding/hex"
     6  	"errors"
     7  	"github.com/qjfoidnh/BaiduPCS-Go/baidupcs/pcserror"
     8  	"github.com/qjfoidnh/BaiduPCS-Go/pcsutil/cachepool"
     9  	"github.com/qjfoidnh/BaiduPCS-Go/pcsutil/converter"
    10  	"github.com/qjfoidnh/BaiduPCS-Go/pcsutil/escaper"
    11  	"github.com/qjfoidnh/BaiduPCS-Go/requester/downloader"
    12  	"io"
    13  	"mime"
    14  	"net/http"
    15  	"path"
    16  	"strconv"
    17  	"strings"
    18  )
    19  
    20  const (
    21  	// ShellPatternCharacters 通配符字符串
    22  	ShellPatternCharacters = "*?[]"
    23  )
    24  
    25  var (
    26  	// ErrFixMD5Isdir 目录不需要修复md5
    27  	ErrFixMD5Isdir = errors.New("directory not support fix md5")
    28  	// ErrFixMD5Failed 修复MD5失败, 可能服务器未刷新
    29  	ErrFixMD5Failed = errors.New("fix md5 failed")
    30  	// ErrFixMD5FileInfoNil 文件信息对象为空
    31  	ErrFixMD5FileInfoNil = errors.New("file info is nil")
    32  	// ErrMatchPathByShellPatternNotAbsPath 不是绝对路径
    33  	ErrMatchPathByShellPatternNotAbsPath = errors.New("not absolute path")
    34  
    35  	ErrContentRangeNotFound               = errors.New("Content-Range not found")
    36  	ErrGetRapidUploadInfoLengthNotFound   = errors.New("Content-Length not found")
    37  	ErrGetRapidUploadInfoMD5NotFound      = errors.New("Content-MD5 not found")
    38  	ErrGetRapidUploadInfoCrc32NotFound    = errors.New("x-bs-meta-crc32 not found")
    39  	ErrGetRapidUploadInfoFilenameNotEqual = errors.New("文件名不匹配")
    40  	ErrGetRapidUploadInfoLengthNotEqual   = errors.New("Content-Length 不匹配")
    41  	ErrGetRapidUploadInfoMD5NotEqual      = errors.New("Content-MD5 不匹配")
    42  	ErrGetRapidUploadInfoCrc32NotEqual    = errors.New("x-bs-meta-crc32 不匹配")
    43  	ErrGetRapidUploadInfoSliceMD5NotEqual = errors.New("slice-md5 不匹配")
    44  
    45  	ErrFileTooLarge = errors.New("文件大于20GB, 无法秒传")
    46  )
    47  
    48  func (pcs *BaiduPCS) getLocateDownloadLink(pcspath string) (link string, pcsError pcserror.Error) {
    49  	info, pcsError := pcs.LocateDownload(pcspath)
    50  	if pcsError != nil {
    51  		return
    52  	}
    53  
    54  	u := info.SingleURL(pcs.isHTTPS)
    55  	if u == nil {
    56  		return "", &pcserror.PCSErrInfo{
    57  			Operation: OperationLocateDownload,
    58  			ErrType:   pcserror.ErrTypeOthers,
    59  			Err:       ErrLocateDownloadURLNotFound,
    60  		}
    61  	}
    62  	return u.String(), nil
    63  }
    64  
    65  // ExportByFileInfo 通过文件信息对象, 导出文件信息
    66  func (pcs *BaiduPCS) ExportByFileInfo(finfo *FileDirectory) (rinfo *RapidUploadInfo, pcsError pcserror.Error) {
    67  	errInfo := pcserror.NewPCSErrorInfo(OperationExportFileInfo)
    68  	errInfo.ErrType = pcserror.ErrTypeOthers
    69  	if finfo.Size > MaxRapidUploadSize {
    70  		errInfo.Err = ErrFileTooLarge
    71  		return nil, errInfo
    72  	}
    73  
    74  	rinfo, pcsError = pcs.GetRapidUploadInfoByFileInfo(finfo)
    75  	if pcsError != nil {
    76  		return nil, pcsError
    77  	}
    78  	if rinfo.Filename != finfo.Filename {
    79  		baiduPCSVerbose.Infof("%s filename not equal, local: %s, remote link: %s\n", OperationExportFileInfo, finfo.Filename, rinfo.Filename)
    80  		rinfo.Filename = finfo.Filename
    81  	}
    82  	return rinfo, nil
    83  }
    84  
    85  // GetRapidUploadInfoByFileInfo 通过文件信息对象, 获取秒传信息
    86  func (pcs *BaiduPCS) GetRapidUploadInfoByFileInfo(finfo *FileDirectory) (rinfo *RapidUploadInfo, pcsError pcserror.Error) {
    87  	if finfo.Size <= SliceMD5Size && len(finfo.BlockList) == 1 && finfo.BlockList[0] == finfo.MD5 {
    88  		// 可直接秒传
    89  		return &RapidUploadInfo{
    90  			Filename:      finfo.Filename,
    91  			ContentLength: finfo.Size,
    92  			ContentMD5:    finfo.MD5,
    93  			SliceMD5:      finfo.MD5,
    94  			ContentCrc32:  "0",
    95  		}, nil
    96  	}
    97  
    98  	link, pcsError := pcs.getLocateDownloadLink(finfo.Path)
    99  	if pcsError != nil {
   100  		return nil, pcsError
   101  	}
   102  
   103  	// 只有ContentLength可以比较
   104  	// finfo记录的ContentMD5不一定是正确的
   105  	// finfo记录的Filename不一定与获取到的一致
   106  	rinfo, pcsError = pcs.GetRapidUploadInfoByLink(link, &RapidUploadInfo{
   107  		ContentLength: finfo.Size,
   108  	})
   109  
   110  	// 如果是没获取到MD5, 可尝试新接口(测试中), 新接口调用频率有限制且文件大小不能超过约3.9G
   111  	if pcsError != nil && pcsError.GetError() == ErrGetRapidUploadInfoMD5NotFound && finfo.Size < 4 * converter.GB {
   112  		link, pcsError = pcs.GetDirectDownloadLink(finfo.Path)
   113  		rinfo, pcsError = pcs.GetRapidUploadInfoByLink(link, &RapidUploadInfo{
   114  			ContentLength: finfo.Size,
   115  		})
   116  	}
   117  	return rinfo, pcsError
   118  }
   119  
   120  // GetDirectDownloadLink 根据method=download方法获取下载链接, 能获取到新上传文件的信息, 作为常规秒传的备选方案
   121  func (pcs *BaiduPCS) GetDirectDownloadLink(path string) (link string, pcsError pcserror.Error) {
   122  	var header = pcs.getPanUAHeader()
   123  	header["Range"] = "bytes=0-" + strconv.FormatInt(SliceMD5Size-1, 10)
   124  	RawQuery := map[string] string{"path": path}
   125  	pcsURL := pcs.generatePCSURL("file", "download", RawQuery)
   126  	return pcsURL.String(), nil
   127  }
   128  
   129  
   130  // GetRapidUploadInfoByLink 通过下载链接, 获取文件秒传信息
   131  func (pcs *BaiduPCS) GetRapidUploadInfoByLink(link string, compareRInfo *RapidUploadInfo) (rinfo *RapidUploadInfo, pcsError pcserror.Error) {
   132  	errInfo := pcserror.NewPCSErrorInfo(OperationGetRapidUploadInfo)
   133  	errInfo.ErrType = pcserror.ErrTypeOthers
   134  
   135  	var (
   136  		header     = pcs.getPanUAHeader()
   137  		isSetRange = compareRInfo != nil && compareRInfo.ContentLength > SliceMD5Size // 是否设置Range
   138  	)
   139  	if isSetRange {
   140  		header["Range"] = "bytes=0-" + strconv.FormatInt(SliceMD5Size-1, 10)
   141  	}
   142  
   143  	resp, err := pcs.client.Req(http.MethodGet, link, nil, header)
   144  	if resp != nil {
   145  		defer resp.Body.Close()
   146  	}
   147  	if err != nil {
   148  		errInfo.SetNetError(err)
   149  		return nil, errInfo
   150  	}
   151  
   152  	// 检测响应状态码
   153  	if resp.StatusCode/100 != 2 {
   154  		errInfo.SetNetError(errors.New(resp.Status))
   155  		return nil, errInfo
   156  	}
   157  
   158  	// 检测是否存在MD5
   159  	md5Str := resp.Header.Get("Content-MD5")
   160  	if md5Str == "" { // 未找到md5值, 可能是服务器未刷新
   161  		errInfo.Err = ErrGetRapidUploadInfoMD5NotFound
   162  		return nil, errInfo
   163  	}
   164  	if compareRInfo != nil && compareRInfo.ContentMD5 != "" && compareRInfo.ContentMD5 != md5Str {
   165  		errInfo.Err = ErrGetRapidUploadInfoMD5NotEqual
   166  		return nil, errInfo
   167  	}
   168  
   169  	// 获取文件名
   170  	_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
   171  	if err != nil {
   172  		errInfo.Err = err
   173  		return nil, errInfo
   174  	}
   175  	filename := params["filename"]
   176  
   177  	if compareRInfo != nil && compareRInfo.Filename != "" && compareRInfo.Filename != filename {
   178  		errInfo.Err = ErrGetRapidUploadInfoFilenameNotEqual
   179  		return nil, errInfo
   180  	}
   181  
   182  	var (
   183  		contentLength int64
   184  	)
   185  	if isSetRange {
   186  		// 检测Content-Range
   187  		contentRange := resp.Header.Get("Content-Range")
   188  		if contentRange == "" {
   189  			errInfo.Err = ErrContentRangeNotFound
   190  			return nil, errInfo
   191  		}
   192  		contentLength = downloader.ParseContentRange(contentRange)
   193  	} else {
   194  		contentLength = resp.ContentLength
   195  	}
   196  
   197  	// 检测Content-Length
   198  	switch contentLength {
   199  	case -1:
   200  		errInfo.Err = ErrGetRapidUploadInfoLengthNotFound
   201  		return nil, errInfo
   202  	case 0:
   203  		return &RapidUploadInfo{
   204  			Filename:      filename,
   205  			ContentLength: contentLength,
   206  			ContentMD5:    EmptyContentMD5,
   207  			SliceMD5:      EmptyContentMD5,
   208  			ContentCrc32:  "0",
   209  		}, nil
   210  	default:
   211  		if compareRInfo != nil && compareRInfo.ContentLength > 0 && compareRInfo.ContentLength != contentLength {
   212  			errInfo.Err = ErrGetRapidUploadInfoLengthNotEqual
   213  			return nil, errInfo
   214  		}
   215  	}
   216  
   217  	// 检测是否存在crc32 值, 一般都会存在的
   218  	crc32Str := resp.Header.Get("x-bs-meta-crc32")
   219  	if crc32Str == "" || crc32Str == "0" {
   220  		errInfo.Err = ErrGetRapidUploadInfoCrc32NotFound
   221  		return nil, errInfo
   222  	}
   223  	if compareRInfo != nil && compareRInfo.ContentCrc32 != "" && compareRInfo.ContentCrc32 != crc32Str {
   224  		errInfo.Err = ErrGetRapidUploadInfoCrc32NotEqual
   225  		return nil, errInfo
   226  	}
   227  
   228  	// 获取slice-md5
   229  	// 忽略比较slice-md5
   230  	if contentLength <= SliceMD5Size {
   231  		return &RapidUploadInfo{
   232  			Filename:      filename,
   233  			ContentLength: contentLength,
   234  			ContentMD5:    md5Str,
   235  			SliceMD5:      md5Str,
   236  			ContentCrc32:  crc32Str,
   237  		}, nil
   238  	}
   239  
   240  	buf := cachepool.RawMallocByteSlice(int(SliceMD5Size))
   241  	_, err = io.ReadFull(resp.Body, buf)
   242  	if err != nil {
   243  		errInfo.SetNetError(err)
   244  		return nil, errInfo
   245  	}
   246  
   247  	// 计算slice-md5
   248  	m := md5.New()
   249  	_, err = m.Write(buf)
   250  	if err != nil {
   251  		panic(err)
   252  	}
   253  
   254  	sliceMD5Str := hex.EncodeToString(m.Sum(nil))
   255  
   256  	// 检测slice-md5, 不必要的
   257  	if compareRInfo != nil && compareRInfo.SliceMD5 != "" && compareRInfo.SliceMD5 != sliceMD5Str {
   258  		errInfo.Err = ErrGetRapidUploadInfoSliceMD5NotEqual
   259  		return nil, errInfo
   260  	}
   261  
   262  	return &RapidUploadInfo{
   263  		Filename:      filename,
   264  		ContentLength: contentLength,
   265  		ContentMD5:    md5Str,
   266  		SliceMD5:      sliceMD5Str,
   267  		ContentCrc32:  crc32Str,
   268  	}, nil
   269  }
   270  
   271  // FixMD5ByFileInfo 尝试修复文件的md5, 通过文件信息对象
   272  func (pcs *BaiduPCS) FixMD5ByFileInfo(finfo *FileDirectory) (pcsError pcserror.Error) {
   273  	errInfo := pcserror.NewPCSErrorInfo(OperationFixMD5)
   274  	errInfo.ErrType = pcserror.ErrTypeOthers
   275  	if finfo == nil {
   276  		errInfo.Err = ErrFixMD5FileInfoNil
   277  		return errInfo
   278  	}
   279  
   280  	if finfo.Size > MaxRapidUploadSize { // 文件大于20GB
   281  		errInfo.Err = ErrFileTooLarge
   282  		return errInfo
   283  	}
   284  
   285  	// 忽略目录
   286  	if finfo.Isdir {
   287  		errInfo.Err = ErrFixMD5Isdir
   288  		return errInfo
   289  	}
   290  
   291  	if len(finfo.BlockList) == 1 && finfo.BlockList[0] == finfo.MD5 {
   292  		// 不需要修复
   293  		return nil
   294  	}
   295  
   296  	link, pcsError := pcs.getLocateDownloadLink(finfo.Path)
   297  	if pcsError != nil {
   298  		return pcsError
   299  	}
   300  
   301  	var (
   302  		cmpInfo = &RapidUploadInfo{
   303  			Filename:      finfo.Filename,
   304  			ContentLength: finfo.Size,
   305  		}
   306  	)
   307  	rinfo, pcsError := pcs.GetRapidUploadInfoByLink(link, cmpInfo)
   308  	if pcsError != nil {
   309  		switch pcsError.GetError() {
   310  		case ErrGetRapidUploadInfoMD5NotFound, ErrGetRapidUploadInfoCrc32NotFound:
   311  			errInfo.Err = ErrFixMD5Failed
   312  		default:
   313  			errInfo.Err = pcsError
   314  		}
   315  		return errInfo
   316  	}
   317  
   318  	// 开始修复
   319  	return pcs.RapidUploadNoCheckDir(finfo.Path, rinfo.ContentMD5, rinfo.SliceMD5, rinfo.ContentCrc32, rinfo.ContentLength)
   320  }
   321  
   322  // FixMD5 尝试修复文件的md5
   323  func (pcs *BaiduPCS) FixMD5(pcspath string) (pcsError pcserror.Error) {
   324  	finfo, pcsError := pcs.FilesDirectoriesMeta(pcspath)
   325  	if pcsError != nil {
   326  		return
   327  	}
   328  
   329  	return pcs.FixMD5ByFileInfo(finfo)
   330  }
   331  
   332  func (pcs *BaiduPCS) recurseMatchPathByShellPattern(index int, patternSlice *[]string, ps *[]string, pcspaths *[]string) {
   333  	if index == len(*patternSlice) {
   334  		*pcspaths = append(*pcspaths, strings.Join(*ps, PathSeparator))
   335  		return
   336  	}
   337  
   338  	if !strings.ContainsAny((*patternSlice)[index], ShellPatternCharacters) {
   339  		(*ps)[index] = (*patternSlice)[index]
   340  		pcs.recurseMatchPathByShellPattern(index+1, patternSlice, ps, pcspaths)
   341  		return
   342  	}
   343  
   344  	fds, pcsError := pcs.FilesDirectoriesList(strings.Join((*ps)[:index], PathSeparator), DefaultOrderOptions)
   345  	if pcsError != nil {
   346  		panic(pcsError) // 抛出异常
   347  	}
   348  
   349  	for k := range fds {
   350  		if matched, _ := path.Match((*patternSlice)[index], fds[k].Filename); matched {
   351  			(*ps)[index] = fds[k].Filename
   352  			pcs.recurseMatchPathByShellPattern(index+1, patternSlice, ps, pcspaths)
   353  		}
   354  	}
   355  	return
   356  }
   357  
   358  // MatchPathByShellPattern 通配符匹配文件路径, pattern 为绝对路径
   359  func (pcs *BaiduPCS) MatchPathByShellPattern(pattern string) (pcspaths []string, pcsError pcserror.Error) {
   360  	errInfo := pcserror.NewPCSErrorInfo(OperrationMatchPathByShellPattern)
   361  	errInfo.ErrType = pcserror.ErrTypeOthers
   362  
   363  	patternSlice := strings.Split(escaper.Escape(path.Clean(pattern), []rune{'['}), PathSeparator) // 转义中括号
   364  	if patternSlice[0] != "" {
   365  		errInfo.Err = ErrMatchPathByShellPatternNotAbsPath
   366  		return nil, errInfo
   367  	}
   368  
   369  	ps := make([]string, len(patternSlice))
   370  	defer func() { // 捕获异常
   371  		if err := recover(); err != nil {
   372  			pcspaths = nil
   373  			pcsError = err.(pcserror.Error)
   374  		}
   375  	}()
   376  	pcs.recurseMatchPathByShellPattern(1, &patternSlice, &ps, &pcspaths)
   377  	return pcspaths, nil
   378  }