github.com/qiniu/go-sdk/v7@v7.27.0/storagev2/downloader/download_manager_test.go (about)

     1  //go:build unit
     2  // +build unit
     3  
     4  package downloader_test
     5  
     6  import (
     7  	"context"
     8  	"encoding/json"
     9  	"io"
    10  	"math/rand"
    11  	"net/http"
    12  	"net/http/httptest"
    13  	"os"
    14  	"path/filepath"
    15  	"strconv"
    16  	"strings"
    17  	"testing"
    18  	"time"
    19  
    20  	"github.com/qiniu/go-sdk/v7/storagev2/apis/get_objects"
    21  	"github.com/qiniu/go-sdk/v7/storagev2/credentials"
    22  	"github.com/qiniu/go-sdk/v7/storagev2/downloader"
    23  	"github.com/qiniu/go-sdk/v7/storagev2/http_client"
    24  	"github.com/qiniu/go-sdk/v7/storagev2/region"
    25  )
    26  
    27  func TestDownloadManagerDownloadDirectory(t *testing.T) {
    28  	rsfMux := http.NewServeMux()
    29  	rsfMux.HandleFunc("/list", func(w http.ResponseWriter, r *http.Request) {
    30  		if r.Method != http.MethodGet {
    31  			t.Fatalf("unexpected method")
    32  		}
    33  		w.Header().Set("Content-Type", "application/json")
    34  		query := r.URL.Query()
    35  		if query.Get("bucket") != "bucket1" {
    36  			t.Fatalf("unexpected bucket")
    37  		}
    38  		if query.Get("prefix") != "" {
    39  			t.Fatalf("unexpected prefix")
    40  		}
    41  		if query.Get("limit") != "" {
    42  			t.Fatalf("unexpected limit")
    43  		}
    44  		jsonData, err := json.Marshal(&get_objects.Response{
    45  			Items: []get_objects.ListedObjectEntry{{
    46  				Key:      "test1/file1",
    47  				PutTime:  time.Now().UnixNano() / 100,
    48  				Hash:     "testhash1",
    49  				Size:     4 * 1024 * 1024,
    50  				MimeType: "application/json",
    51  			}, {
    52  				Key:      "test2/file2",
    53  				PutTime:  time.Now().UnixNano() / 100,
    54  				Hash:     "testhash2",
    55  				Size:     4 * 1024 * 1024,
    56  				MimeType: "application/json",
    57  			}},
    58  		})
    59  		if err != nil {
    60  			t.Fatal(err)
    61  		}
    62  		w.Header().Add("X-ReqId", "fakereqid")
    63  		w.Write(jsonData)
    64  	})
    65  	rsfServer := httptest.NewServer(rsfMux)
    66  	defer rsfServer.Close()
    67  
    68  	ioMux := http.NewServeMux()
    69  	ioMux.HandleFunc("/test1/file1", func(w http.ResponseWriter, r *http.Request) {
    70  		rander := rand.New(rand.NewSource(time.Now().UnixNano()))
    71  		w.Header().Set("Accept-Ranges", "bytes")
    72  		w.Header().Add("X-ReqId", "fakereqid")
    73  		switch r.Method {
    74  		case http.MethodHead:
    75  			w.Header().Set("Content-Type", "application/json")
    76  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
    77  			w.Header().Set("ETag", "testetag1")
    78  		case http.MethodGet:
    79  			w.Header().Set("Content-Type", "application/json")
    80  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
    81  			w.Header().Set("ETag", "testetag1")
    82  			io.CopyN(w, rander, 4*1024*1024)
    83  		default:
    84  			t.Fatalf("unexpected method")
    85  		}
    86  	})
    87  	ioMux.HandleFunc("/test2/file2", func(w http.ResponseWriter, r *http.Request) {
    88  		rander := rand.New(rand.NewSource(time.Now().UnixNano()))
    89  		w.Header().Set("Accept-Ranges", "bytes")
    90  		w.Header().Add("X-ReqId", "fakereqid")
    91  		switch r.Method {
    92  		case http.MethodHead:
    93  			w.Header().Set("Content-Type", "application/json")
    94  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
    95  			w.Header().Set("ETag", "testetag1")
    96  		case http.MethodGet:
    97  			w.Header().Set("Content-Type", "application/json")
    98  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
    99  			w.Header().Set("ETag", "testetag1")
   100  			io.CopyN(w, rander, 4*1024*1024)
   101  		default:
   102  			t.Fatalf("unexpected method")
   103  		}
   104  	})
   105  	ioServer := httptest.NewServer(ioMux)
   106  	defer ioServer.Close()
   107  
   108  	tmpDir, err := os.MkdirTemp("", "")
   109  	if err != nil {
   110  		t.Fatal(err)
   111  	}
   112  	defer os.RemoveAll(tmpDir)
   113  
   114  	downloadManager := downloader.NewDownloadManager(&downloader.DownloadManagerOptions{
   115  		Options: http_client.Options{
   116  			Regions: &region.Region{
   117  				Rsf: region.Endpoints{Preferred: []string{rsfServer.URL}},
   118  			},
   119  			Credentials:         credentials.NewCredentials("testaccesskey", "testsecretkey"),
   120  			UseInsecureProtocol: true,
   121  		},
   122  		DestinationDownloader: downloader.NewConcurrentDownloader(&downloader.ConcurrentDownloaderOptions{
   123  			Concurrency: 1,
   124  			PartSize:    10 * 1024 * 1024,
   125  		}),
   126  	})
   127  	if err = downloadManager.DownloadDirectory(context.Background(), tmpDir, &downloader.DirectoryOptions{
   128  		UseInsecureProtocol:  true,
   129  		BucketName:           "bucket1",
   130  		DownloadURLsProvider: downloader.NewStaticDomainBasedURLsProvider([]string{ioServer.URL}),
   131  	}); err != nil {
   132  		t.Fatal(err)
   133  	}
   134  	if fileInfo, err := os.Stat(filepath.Join(tmpDir, "test1", "file1")); err != nil {
   135  		t.Fatal(err)
   136  	} else if fileInfo.Size() != 4*1024*1024 {
   137  		t.Fatalf("unexpected file size: test1/file1")
   138  	}
   139  	if fileInfo, err := os.Stat(filepath.Join(tmpDir, "test2", "file2")); err != nil {
   140  		t.Fatal(err)
   141  	} else if fileInfo.Size() != 4*1024*1024 {
   142  		t.Fatalf("unexpected file size: test2/file2")
   143  	}
   144  }
   145  
   146  func TestDownloadManagerDownloadDirectoryWithPrefix(t *testing.T) {
   147  	rsfMux := http.NewServeMux()
   148  	rsfMux.HandleFunc("/list", func(w http.ResponseWriter, r *http.Request) {
   149  		if r.Method != http.MethodGet {
   150  			t.Fatalf("unexpected method")
   151  		}
   152  		w.Header().Set("Content-Type", "application/json")
   153  		query := r.URL.Query()
   154  		if query.Get("bucket") != "bucket1" {
   155  			t.Fatalf("unexpected bucket")
   156  		}
   157  		if query.Get("prefix") != "test1/" {
   158  			t.Fatalf("unexpected prefix")
   159  		}
   160  		if query.Get("limit") != "" {
   161  			t.Fatalf("unexpected limit")
   162  		}
   163  		jsonData, err := json.Marshal(&get_objects.Response{
   164  			Items: []get_objects.ListedObjectEntry{{
   165  				Key:      "test1/",
   166  				PutTime:  time.Now().UnixNano() / 100,
   167  				Hash:     "testhash1",
   168  				Size:     0,
   169  				MimeType: "application/octet-stream",
   170  			}, {
   171  				Key:      "test1/file1",
   172  				PutTime:  time.Now().UnixNano() / 100,
   173  				Hash:     "testhash1",
   174  				Size:     4 * 1024 * 1024,
   175  				MimeType: "application/json",
   176  			}},
   177  		})
   178  		if err != nil {
   179  			t.Fatal(err)
   180  		}
   181  		w.Header().Add("X-ReqId", "fakereqid")
   182  		w.Write(jsonData)
   183  	})
   184  	rsfServer := httptest.NewServer(rsfMux)
   185  	defer rsfServer.Close()
   186  
   187  	ioMux := http.NewServeMux()
   188  	ioMux.HandleFunc("/test1/file1", func(w http.ResponseWriter, r *http.Request) {
   189  		rander := rand.New(rand.NewSource(time.Now().UnixNano()))
   190  		w.Header().Set("Accept-Ranges", "bytes")
   191  		w.Header().Add("X-ReqId", "fakereqid")
   192  		switch r.Method {
   193  		case http.MethodHead:
   194  			w.Header().Set("Content-Type", "application/json")
   195  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
   196  			w.Header().Set("ETag", "testetag1")
   197  		case http.MethodGet:
   198  			w.Header().Set("Content-Type", "application/json")
   199  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
   200  			w.Header().Set("ETag", "testetag1")
   201  			io.CopyN(w, rander, 4*1024*1024)
   202  		default:
   203  			t.Fatalf("unexpected method")
   204  		}
   205  	})
   206  	ioServer := httptest.NewServer(ioMux)
   207  	defer ioServer.Close()
   208  
   209  	tmpDir, err := os.MkdirTemp("", "")
   210  	if err != nil {
   211  		t.Fatal(err)
   212  	}
   213  	defer os.RemoveAll(tmpDir)
   214  
   215  	downloadManager := downloader.NewDownloadManager(&downloader.DownloadManagerOptions{
   216  		Options: http_client.Options{
   217  			Regions: &region.Region{
   218  				Rsf: region.Endpoints{Preferred: []string{rsfServer.URL}},
   219  			},
   220  			Credentials:         credentials.NewCredentials("testaccesskey", "testsecretkey"),
   221  			UseInsecureProtocol: true,
   222  		},
   223  		DestinationDownloader: downloader.NewConcurrentDownloader(&downloader.ConcurrentDownloaderOptions{
   224  			Concurrency: 1,
   225  			PartSize:    10 * 1024 * 1024,
   226  		}),
   227  	})
   228  	if err = downloadManager.DownloadDirectory(context.Background(), tmpDir, &downloader.DirectoryOptions{
   229  		UseInsecureProtocol:  true,
   230  		BucketName:           "bucket1",
   231  		ObjectPrefix:         "test1/",
   232  		DownloadURLsProvider: downloader.NewStaticDomainBasedURLsProvider([]string{ioServer.URL}),
   233  	}); err != nil {
   234  		t.Fatal(err)
   235  	}
   236  	if fileInfo, err := os.Stat(filepath.Join(tmpDir, "file1")); err != nil {
   237  		t.Fatal(err)
   238  	} else if fileInfo.Size() != 4*1024*1024 {
   239  		t.Fatalf("unexpected file size: file1")
   240  	}
   241  }
   242  
   243  func TestDownloadManagerDownloadDirectoryWithFilter(t *testing.T) {
   244  	rsfMux := http.NewServeMux()
   245  	rsfMux.HandleFunc("/list", func(w http.ResponseWriter, r *http.Request) {
   246  		if r.Method != http.MethodGet {
   247  			t.Fatalf("unexpected method")
   248  		}
   249  		w.Header().Set("Content-Type", "application/json")
   250  		query := r.URL.Query()
   251  		if query.Get("bucket") != "bucket1" {
   252  			t.Fatalf("unexpected bucket")
   253  		}
   254  		if query.Get("prefix") != "" {
   255  			t.Fatalf("unexpected prefix")
   256  		}
   257  		if query.Get("limit") != "" {
   258  			t.Fatalf("unexpected limit")
   259  		}
   260  		jsonData, err := json.Marshal(&get_objects.Response{
   261  			Items: []get_objects.ListedObjectEntry{{
   262  				Key:      "test1/",
   263  				PutTime:  time.Now().UnixNano() / 100,
   264  				Hash:     "testhash1",
   265  				Size:     0,
   266  				MimeType: "application/octet-stream",
   267  			}, {
   268  				Key:      "test1/file1",
   269  				PutTime:  time.Now().UnixNano() / 100,
   270  				Hash:     "testhash1",
   271  				Size:     4 * 1024 * 1024,
   272  				MimeType: "application/json",
   273  			}, {
   274  				Key:      "test2/",
   275  				PutTime:  time.Now().UnixNano() / 100,
   276  				Hash:     "testhash2",
   277  				Size:     0,
   278  				MimeType: "application/octet-stream",
   279  			}, {
   280  				Key:      "test2/file2",
   281  				PutTime:  time.Now().UnixNano() / 100,
   282  				Hash:     "testhash2",
   283  				Size:     4 * 1024 * 1024,
   284  				MimeType: "application/json",
   285  			}},
   286  		})
   287  		if err != nil {
   288  			t.Fatal(err)
   289  		}
   290  		w.Header().Add("X-ReqId", "fakereqid")
   291  		w.Write(jsonData)
   292  	})
   293  	rsfServer := httptest.NewServer(rsfMux)
   294  	defer rsfServer.Close()
   295  
   296  	ioMux := http.NewServeMux()
   297  	ioMux.HandleFunc("/test1/file1", func(w http.ResponseWriter, r *http.Request) {
   298  		rander := rand.New(rand.NewSource(time.Now().UnixNano()))
   299  		w.Header().Set("Accept-Ranges", "bytes")
   300  		w.Header().Add("X-ReqId", "fakereqid")
   301  		switch r.Method {
   302  		case http.MethodHead:
   303  			w.Header().Set("Content-Type", "application/json")
   304  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
   305  			w.Header().Set("ETag", "testetag1")
   306  		case http.MethodGet:
   307  			w.Header().Set("Content-Type", "application/json")
   308  			w.Header().Set("Content-Length", strconv.Itoa(4*1024*1024))
   309  			w.Header().Set("ETag", "testetag1")
   310  			io.CopyN(w, rander, 4*1024*1024)
   311  		default:
   312  			t.Fatalf("unexpected method")
   313  		}
   314  	})
   315  	ioServer := httptest.NewServer(ioMux)
   316  	defer ioServer.Close()
   317  
   318  	tmpDir, err := os.MkdirTemp("", "")
   319  	if err != nil {
   320  		t.Fatal(err)
   321  	}
   322  	defer os.RemoveAll(tmpDir)
   323  
   324  	downloadManager := downloader.NewDownloadManager(&downloader.DownloadManagerOptions{
   325  		Options: http_client.Options{
   326  			Regions: &region.Region{
   327  				Rsf: region.Endpoints{Preferred: []string{rsfServer.URL}},
   328  			},
   329  			Credentials:         credentials.NewCredentials("testaccesskey", "testsecretkey"),
   330  			UseInsecureProtocol: true,
   331  		},
   332  		DestinationDownloader: downloader.NewConcurrentDownloader(&downloader.ConcurrentDownloaderOptions{
   333  			Concurrency: 1,
   334  			PartSize:    10 * 1024 * 1024,
   335  		}),
   336  	})
   337  	if err = downloadManager.DownloadDirectory(context.Background(), tmpDir, &downloader.DirectoryOptions{
   338  		UseInsecureProtocol:  true,
   339  		BucketName:           "bucket1",
   340  		DownloadURLsProvider: downloader.NewStaticDomainBasedURLsProvider([]string{ioServer.URL}),
   341  		ShouldDownloadObject: func(objectName string) bool {
   342  			return strings.HasPrefix(objectName, "test1/")
   343  		},
   344  	}); err != nil {
   345  		t.Fatal(err)
   346  	}
   347  	if fileInfo, err := os.Stat(filepath.Join(tmpDir, "test1", "file1")); err != nil {
   348  		t.Fatal(err)
   349  	} else if fileInfo.Size() != 4*1024*1024 {
   350  		t.Fatalf("unexpected file size: test1/file1")
   351  	}
   352  	if _, err = os.Stat(filepath.Join(tmpDir, "test2")); err == nil {
   353  		t.Fatalf("unexpected test2/")
   354  	}
   355  	if _, err = os.Stat(filepath.Join(tmpDir, "test2", "file2")); err == nil {
   356  		t.Fatalf("unexpected test2/file2")
   357  	}
   358  }