github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/downloader/http/downloader.go (about)

     1  package http
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"io"
     7  	"net/http"
     8  	"os"
     9  	"time"
    10  
    11  	"github.com/filecoin-project/bacalhau/pkg/model"
    12  	"github.com/filecoin-project/bacalhau/pkg/system"
    13  	"github.com/filecoin-project/bacalhau/pkg/util/closer"
    14  	"github.com/rs/zerolog/log"
    15  )
    16  
    17  type Downloader struct {
    18  	Settings *model.DownloaderSettings
    19  }
    20  
    21  func NewHTTPDownloader(settings *model.DownloaderSettings) *Downloader {
    22  	return &Downloader{
    23  		Settings: settings,
    24  	}
    25  }
    26  
    27  func (httpDownloader *Downloader) IsInstalled(context.Context) (bool, error) {
    28  	return true, nil
    29  }
    30  
    31  func (httpDownloader *Downloader) FetchResult(ctx context.Context, result model.PublishedResult, downloadPath string) error {
    32  	ctx, span := system.NewSpan(ctx, system.GetTracer(), "pkg/downloader/http.Downloader.FetchResults")
    33  	defer span.End()
    34  
    35  	err := func() error {
    36  		log.Ctx(ctx).Debug().Msgf(
    37  			"Downloading result URL %s '%s' to '%s'...",
    38  			result.Data.Name,
    39  			result.Data.URL, downloadPath,
    40  		)
    41  
    42  		innerCtx, cancel := context.WithDeadline(ctx, time.Now().Add(httpDownloader.Settings.Timeout))
    43  		defer cancel()
    44  
    45  		return fetch(innerCtx, result.Data.URL, downloadPath)
    46  	}()
    47  
    48  	if err != nil {
    49  		if errors.Is(err, context.DeadlineExceeded) {
    50  			log.Ctx(ctx).Error().Msg("Timed out while downloading result.")
    51  		}
    52  
    53  		return err
    54  	}
    55  	return nil
    56  }
    57  
    58  func fetch(ctx context.Context, url string, filepath string) error {
    59  	// Create a new file at the specified filepath
    60  	out, err := os.OpenFile(filepath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, model.DownloadFilePerm)
    61  	if err != nil {
    62  		return err
    63  	}
    64  	defer closer.CloseWithLogOnError("file", out)
    65  
    66  	// Make an HTTP GET request to the URL
    67  	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
    68  	if err != nil {
    69  		return err
    70  	}
    71  
    72  	response, err := http.DefaultClient.Do(req) //nolint
    73  	if err != nil {
    74  		return err
    75  	}
    76  
    77  	defer closer.DrainAndCloseWithLogOnError(ctx, "http response", response.Body)
    78  
    79  	// Write the contents of the response body to the file
    80  	_, err = io.Copy(out, response.Body)
    81  	if err != nil {
    82  		return err
    83  	}
    84  
    85  	return nil
    86  }