github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/publisher/estuary/publisher.go (about)

     1  package estuary
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"net/http"
     8  	"os"
     9  	"path/filepath"
    10  	"time"
    11  
    12  	"github.com/filecoin-project/bacalhau/pkg/job"
    13  	"github.com/filecoin-project/bacalhau/pkg/util/closer"
    14  
    15  	"github.com/antihax/optional"
    16  	estuary_client "github.com/application-research/estuary-clients/go"
    17  	"github.com/filecoin-project/bacalhau/pkg/ipfs/car"
    18  	"github.com/filecoin-project/bacalhau/pkg/model"
    19  	"github.com/filecoin-project/bacalhau/pkg/publisher"
    20  	"github.com/pkg/errors"
    21  	"github.com/rs/zerolog/log"
    22  )
    23  
    24  type estuaryPublisher struct {
    25  	client *estuary_client.APIClient
    26  }
    27  
    28  const publisherTimeout = 5 * time.Minute
    29  
    30  func NewEstuaryPublisher(config EstuaryPublisherConfig) publisher.Publisher {
    31  	return &estuaryPublisher{
    32  		client: GetClient(config.APIKey),
    33  	}
    34  }
    35  
    36  // IsInstalled implements publisher.Publisher
    37  func (e *estuaryPublisher) IsInstalled(ctx context.Context) (bool, error) {
    38  	_, response, err := e.client.CollectionsApi.CollectionsGet(ctx) //nolint:bodyclose // golangcilint is dumb - this is closed
    39  	if response != nil {
    40  		defer closer.DrainAndCloseWithLogOnError(ctx, "estuary-response", response.Body)
    41  		return response.StatusCode == http.StatusOK, nil
    42  	} else {
    43  		return false, err
    44  	}
    45  }
    46  
    47  // PublishShardResult implements publisher.Publisher
    48  func (e *estuaryPublisher) PublishShardResult(
    49  	ctx context.Context,
    50  	shard model.JobShard,
    51  	hostID string,
    52  	shardResultPath string,
    53  ) (model.StorageSpec, error) {
    54  	tempDir, err := os.MkdirTemp(os.TempDir(), "bacalhau-estuary-publisher")
    55  	if err != nil {
    56  		return model.StorageSpec{}, err
    57  	}
    58  	defer os.RemoveAll(tempDir)
    59  
    60  	carFile := filepath.Join(tempDir, "results.car")
    61  	_, err = car.CreateCar(ctx, shardResultPath, carFile, 1)
    62  	if err != nil {
    63  		return model.StorageSpec{}, err
    64  	}
    65  
    66  	carReader, err := os.Open(carFile)
    67  	if err != nil {
    68  		return model.StorageSpec{}, errors.Wrap(err, "error opening CAR file")
    69  	}
    70  
    71  	carContent, err := io.ReadAll(carReader)
    72  	if err != nil {
    73  		return model.StorageSpec{}, errors.Wrap(err, "error reading CAR data")
    74  	}
    75  
    76  	timeout, cancel := context.WithTimeout(ctx, publisherTimeout)
    77  	defer cancel()
    78  
    79  	addCarResponse, httpResponse, err := e.client.ContentApi.ContentAddCarPost( //nolint:bodyclose // golangcilint is dumb - this is closed
    80  		timeout,
    81  		string(carContent),
    82  		&estuary_client.ContentApiContentAddCarPostOpts{
    83  			Filename: optional.NewString(shard.ID()),
    84  		},
    85  	)
    86  	if err != nil && err != io.EOF {
    87  		return model.StorageSpec{}, err
    88  	} else if httpResponse.StatusCode != http.StatusOK {
    89  		return model.StorageSpec{}, fmt.Errorf("upload to Estuary failed")
    90  	}
    91  	log.Ctx(ctx).Debug().Interface("Response", addCarResponse).Int("StatusCode", httpResponse.StatusCode).Msg("Estuary response")
    92  	defer closer.DrainAndCloseWithLogOnError(ctx, "estuary-response", httpResponse.Body)
    93  
    94  	spec := job.GetPublishedStorageSpec(shard, model.StorageSourceEstuary, hostID, addCarResponse.Cid)
    95  	spec.URL = addCarResponse.EstuaryRetrievalUrl
    96  
    97  	return spec, nil
    98  }
    99  
   100  var _ publisher.Publisher = (*estuaryPublisher)(nil)