github.com/isi-lincoln/grab@v2.0.1-0.20200331080741-9f014744ee41+incompatible/grab.go (about)

     1  package grab
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  )
     7  
     8  // Get sends a HTTP request and downloads the content of the requested URL to
     9  // the given destination file path. The caller is blocked until the download is
    10  // completed, successfully or otherwise.
    11  //
    12  // An error is returned if caused by client policy (such as CheckRedirect), or
    13  // if there was an HTTP protocol or IO error.
    14  //
    15  // For non-blocking calls or control over HTTP client headers, redirect policy,
    16  // and other settings, create a Client instead.
    17  func Get(dst, urlStr string) (*Response, error) {
    18  	req, err := NewRequest(dst, urlStr)
    19  	if err != nil {
    20  		return nil, err
    21  	}
    22  
    23  	resp := DefaultClient.Do(req)
    24  	return resp, resp.Err()
    25  }
    26  
    27  // GetBatch sends multiple HTTP requests and downloads the content of the
    28  // requested URLs to the given destination directory using the given number of
    29  // concurrent worker goroutines.
    30  //
    31  // The Response for each requested URL is sent through the returned Response
    32  // channel, as soon as a worker receives a response from the remote server. The
    33  // Response can then be used to track the progress of the download while it is
    34  // in progress.
    35  //
    36  // The returned Response channel will be closed by Grab, only once all downloads
    37  // have completed or failed.
    38  //
    39  // If an error occurs during any download, it will be available via call to the
    40  // associated Response.Err.
    41  //
    42  // For control over HTTP client headers, redirect policy, and other settings,
    43  // create a Client instead.
    44  func GetBatch(workers int, dst string, urlStrs ...string) (<-chan *Response, error) {
    45  	fi, err := os.Stat(dst)
    46  	if err != nil {
    47  		return nil, err
    48  	}
    49  	if !fi.IsDir() {
    50  		return nil, fmt.Errorf("destination is not a directory")
    51  	}
    52  
    53  	reqs := make([]*Request, len(urlStrs))
    54  	for i := 0; i < len(urlStrs); i++ {
    55  		req, err := NewRequest(dst, urlStrs[i])
    56  		if err != nil {
    57  			return nil, err
    58  		}
    59  		reqs[i] = req
    60  	}
    61  
    62  	ch := DefaultClient.DoBatch(workers, reqs...)
    63  	return ch, nil
    64  }