github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/executor/results.go (about)

     1  package executor
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"strings"
    10  
    11  	"github.com/c2h5oh/datasize"
    12  	"github.com/filecoin-project/bacalhau/pkg/model"
    13  	"github.com/filecoin-project/bacalhau/pkg/system"
    14  	"github.com/filecoin-project/bacalhau/pkg/util/closer"
    15  	"go.ptx.dk/multierrgroup"
    16  	"go.uber.org/multierr"
    17  )
    18  
    19  type outputResult struct {
    20  	contents     io.Reader
    21  	filename     string
    22  	fileLimit    datasize.ByteSize
    23  	summary      *string
    24  	summaryLimit datasize.ByteSize
    25  	truncated    *bool
    26  }
    27  
    28  func writeOutputResult(resultsDir string, output outputResult) error {
    29  	if output.contents == nil {
    30  		// contents may be nil if something went wrong while trying to get the logs
    31  		output.contents = bytes.NewReader(nil)
    32  	}
    33  
    34  	var err error
    35  
    36  	// Consume the passed buffers up to the limit of the maximum bytes. The
    37  	// buffers will then contain whatever is left that overflows, and we can
    38  	// write that directly to disk rather than needing to hold it all in memory.
    39  	summary := make([]byte, output.summaryLimit+1)
    40  	summaryRead, err := output.contents.Read(summary)
    41  	if err != nil && err != io.EOF {
    42  		return err
    43  	}
    44  
    45  	available := system.Min(summaryRead, int(output.summaryLimit))
    46  
    47  	if output.summary != nil {
    48  		*(output.summary) = string(summary[:available])
    49  	}
    50  	if output.truncated != nil {
    51  		*(output.truncated) = summaryRead > int(output.summaryLimit)
    52  	}
    53  	if err != nil && err != io.EOF {
    54  		return err
    55  	}
    56  
    57  	file, err := os.Create(filepath.Join(resultsDir, output.filename))
    58  	if err != nil {
    59  		return err
    60  	}
    61  	defer closer.CloseWithLogOnError("file", file)
    62  
    63  	// First write the bytes we have already read, and then write whatever
    64  	// is left in the buffer, but only up to the maximum file limit.
    65  	available = system.Min(summaryRead, int(output.fileLimit))
    66  	fileWritten, err := file.Write(summary[:available])
    67  	if err != nil && err != io.EOF {
    68  		return err
    69  	}
    70  
    71  	_, err = io.CopyN(file, output.contents, int64(int(output.fileLimit)-fileWritten))
    72  	if err != nil && err != io.EOF {
    73  		return err
    74  	}
    75  
    76  	return nil
    77  }
    78  
    79  // WriteJobResults produces files and a model.RunCommandResult in the standard
    80  // format, including truncating the contents of both where necessary to fit
    81  // within system-defined limits.
    82  //
    83  // It will consume only the bytes from the passed io.Readers that it needs to
    84  // correctly form job outputs. Once the command returns, the readers can close.
    85  func WriteJobResults(resultsDir string, stdout, stderr io.Reader, exitcode int, err error) (*model.RunCommandResult, error) {
    86  	result := model.NewRunCommandResult()
    87  
    88  	outputs := []outputResult{
    89  		// Standard output
    90  		{
    91  			stdout,
    92  			model.DownloadFilenameStdout,
    93  			system.MaxStdoutFileLength,
    94  			&result.STDOUT,
    95  			system.MaxStdoutReturnLength,
    96  			&result.StdoutTruncated,
    97  		},
    98  		// Standard error
    99  		{
   100  			stderr,
   101  			model.DownloadFilenameStderr,
   102  			system.MaxStderrFileLength,
   103  			&result.STDERR,
   104  			system.MaxStderrReturnLength,
   105  			&result.StderrTruncated,
   106  		},
   107  		// Exit code
   108  		{
   109  			strings.NewReader(fmt.Sprint(exitcode)),
   110  			model.DownloadFilenameExitCode,
   111  			4,
   112  			nil,
   113  			4,
   114  			nil,
   115  		},
   116  	}
   117  
   118  	wg := multierrgroup.Group{}
   119  	for _, output := range outputs {
   120  		output := output
   121  		wg.Go(func() error {
   122  			return writeOutputResult(resultsDir, output)
   123  		})
   124  	}
   125  
   126  	err = multierr.Append(err, wg.Wait())
   127  	if err != nil {
   128  		result.ErrorMsg = err.Error()
   129  	}
   130  
   131  	result.ExitCode = exitcode
   132  	return result, err
   133  }
   134  
   135  func FailResult(err error) (*model.RunCommandResult, error) {
   136  	return &model.RunCommandResult{ErrorMsg: err.Error()}, err
   137  }