github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/executor/util/utils.go (about)

     1  package util
     2  
     3  import (
     4  	"context"
     5  
     6  	"github.com/filecoin-project/bacalhau/pkg/executor"
     7  	"github.com/filecoin-project/bacalhau/pkg/executor/docker"
     8  	"github.com/filecoin-project/bacalhau/pkg/executor/language"
     9  	noop_executor "github.com/filecoin-project/bacalhau/pkg/executor/noop"
    10  	pythonwasm "github.com/filecoin-project/bacalhau/pkg/executor/python_wasm"
    11  	"github.com/filecoin-project/bacalhau/pkg/executor/wasm"
    12  	"github.com/filecoin-project/bacalhau/pkg/ipfs"
    13  	"github.com/filecoin-project/bacalhau/pkg/model"
    14  	"github.com/filecoin-project/bacalhau/pkg/storage"
    15  	"github.com/filecoin-project/bacalhau/pkg/storage/combo"
    16  	filecoinunsealed "github.com/filecoin-project/bacalhau/pkg/storage/filecoin_unsealed"
    17  	"github.com/filecoin-project/bacalhau/pkg/storage/inline"
    18  	ipfs_storage "github.com/filecoin-project/bacalhau/pkg/storage/ipfs"
    19  	noop_storage "github.com/filecoin-project/bacalhau/pkg/storage/noop"
    20  	"github.com/filecoin-project/bacalhau/pkg/storage/tracing"
    21  	"github.com/filecoin-project/bacalhau/pkg/storage/url/urldownload"
    22  	"github.com/filecoin-project/bacalhau/pkg/system"
    23  )
    24  
    25  type StandardStorageProviderOptions struct {
    26  	API                  ipfs.Client
    27  	FilecoinUnsealedPath string
    28  	DownloadPath         string
    29  }
    30  
    31  type StandardExecutorOptions struct {
    32  	DockerID string
    33  	Storage  StandardStorageProviderOptions
    34  }
    35  
    36  func NewStandardStorageProvider(
    37  	_ context.Context,
    38  	cm *system.CleanupManager,
    39  	options StandardStorageProviderOptions,
    40  ) (storage.StorageProvider, error) {
    41  	ipfsAPICopyStorage, err := ipfs_storage.NewStorage(cm, options.API)
    42  	if err != nil {
    43  		return nil, err
    44  	}
    45  
    46  	urlDownloadStorage, err := urldownload.NewStorage(cm)
    47  	if err != nil {
    48  		return nil, err
    49  	}
    50  
    51  	filecoinUnsealedStorage, err := filecoinunsealed.NewStorage(cm, options.FilecoinUnsealedPath)
    52  	if err != nil {
    53  		return nil, err
    54  	}
    55  
    56  	inlineStorage := inline.NewStorage()
    57  
    58  	var useIPFSDriver storage.Storage = ipfsAPICopyStorage
    59  
    60  	// if we are using a FilecoinUnsealedPath then construct a combo
    61  	// driver that will give preference to the filecoin unsealed driver
    62  	// if the cid is deemed to be local
    63  	if options.FilecoinUnsealedPath != "" {
    64  		comboDriver, err := combo.NewStorage(
    65  			cm,
    66  			func(ctx context.Context) ([]storage.Storage, error) {
    67  				return []storage.Storage{
    68  					filecoinUnsealedStorage,
    69  					ipfsAPICopyStorage,
    70  				}, nil
    71  			},
    72  			func(ctx context.Context, spec model.StorageSpec) (storage.Storage, error) {
    73  				filecoinUnsealedHasCid, err := filecoinUnsealedStorage.HasStorageLocally(ctx, spec)
    74  				if err != nil {
    75  					return ipfsAPICopyStorage, err
    76  				}
    77  				if filecoinUnsealedHasCid {
    78  					return filecoinUnsealedStorage, nil
    79  				} else {
    80  					return ipfsAPICopyStorage, nil
    81  				}
    82  			},
    83  			func(ctx context.Context) (storage.Storage, error) {
    84  				return ipfsAPICopyStorage, nil
    85  			},
    86  		)
    87  
    88  		if err != nil {
    89  			return nil, err
    90  		}
    91  
    92  		useIPFSDriver = comboDriver
    93  	}
    94  
    95  	return model.NewMappedProvider(map[model.StorageSourceType]storage.Storage{
    96  		model.StorageSourceIPFS:             tracing.Wrap(useIPFSDriver),
    97  		model.StorageSourceURLDownload:      tracing.Wrap(urlDownloadStorage),
    98  		model.StorageSourceFilecoinUnsealed: tracing.Wrap(filecoinUnsealedStorage),
    99  		model.StorageSourceInline:           tracing.Wrap(inlineStorage),
   100  	}), nil
   101  }
   102  
   103  func NewNoopStorageProvider(
   104  	ctx context.Context,
   105  	cm *system.CleanupManager,
   106  	config noop_storage.StorageConfig,
   107  ) (storage.StorageProvider, error) {
   108  	noopStorage, err := noop_storage.NewNoopStorage(ctx, cm, config)
   109  	if err != nil {
   110  		return nil, err
   111  	}
   112  	return model.NewNoopProvider[model.StorageSourceType, storage.Storage](noopStorage), nil
   113  }
   114  
   115  func NewStandardExecutorProvider(
   116  	ctx context.Context,
   117  	cm *system.CleanupManager,
   118  	executorOptions StandardExecutorOptions,
   119  ) (executor.ExecutorProvider, error) {
   120  	storageProvider, err := NewStandardStorageProvider(ctx, cm, executorOptions.Storage)
   121  	if err != nil {
   122  		return nil, err
   123  	}
   124  
   125  	dockerExecutor, err := docker.NewExecutor(ctx, cm, executorOptions.DockerID, storageProvider)
   126  	if err != nil {
   127  		return nil, err
   128  	}
   129  
   130  	wasmExecutor, err := wasm.NewExecutor(ctx, storageProvider)
   131  	if err != nil {
   132  		return nil, err
   133  	}
   134  
   135  	executors := model.NewMappedProvider(map[model.Engine]executor.Executor{
   136  		model.EngineDocker: dockerExecutor,
   137  		model.EngineWasm:   wasmExecutor,
   138  	})
   139  
   140  	// language executors wrap other executors, so pass them a reference to all
   141  	// the executors so they can look up the ones they need
   142  	exLang, err := language.NewExecutor(ctx, cm, executors)
   143  	if err != nil {
   144  		return nil, err
   145  	}
   146  	executors.Add(model.EngineLanguage, exLang)
   147  
   148  	exPythonWasm, err := pythonwasm.NewExecutor(executors)
   149  	if err != nil {
   150  		return nil, err
   151  	}
   152  	executors.Add(model.EnginePythonWasm, exPythonWasm)
   153  
   154  	return executors, nil
   155  }
   156  
   157  // return noop executors for all engines
   158  func NewNoopExecutors(config noop_executor.ExecutorConfig) executor.ExecutorProvider {
   159  	noopExecutor := noop_executor.NewNoopExecutorWithConfig(config)
   160  	return model.NewNoopProvider[model.Engine, executor.Executor](noopExecutor)
   161  }