github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/executor/results.go (about) 1 package executor 2 3 import ( 4 "bytes" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "strings" 10 11 "github.com/c2h5oh/datasize" 12 "github.com/filecoin-project/bacalhau/pkg/model" 13 "github.com/filecoin-project/bacalhau/pkg/system" 14 "github.com/filecoin-project/bacalhau/pkg/util/closer" 15 "go.ptx.dk/multierrgroup" 16 "go.uber.org/multierr" 17 ) 18 19 type outputResult struct { 20 contents io.Reader 21 filename string 22 fileLimit datasize.ByteSize 23 summary *string 24 summaryLimit datasize.ByteSize 25 truncated *bool 26 } 27 28 func writeOutputResult(resultsDir string, output outputResult) error { 29 if output.contents == nil { 30 // contents may be nil if something went wrong while trying to get the logs 31 output.contents = bytes.NewReader(nil) 32 } 33 34 var err error 35 36 // Consume the passed buffers up to the limit of the maximum bytes. The 37 // buffers will then contain whatever is left that overflows, and we can 38 // write that directly to disk rather than needing to hold it all in memory. 39 summary := make([]byte, output.summaryLimit+1) 40 summaryRead, err := output.contents.Read(summary) 41 if err != nil && err != io.EOF { 42 return err 43 } 44 45 available := system.Min(summaryRead, int(output.summaryLimit)) 46 47 if output.summary != nil { 48 *(output.summary) = string(summary[:available]) 49 } 50 if output.truncated != nil { 51 *(output.truncated) = summaryRead > int(output.summaryLimit) 52 } 53 if err != nil && err != io.EOF { 54 return err 55 } 56 57 file, err := os.Create(filepath.Join(resultsDir, output.filename)) 58 if err != nil { 59 return err 60 } 61 defer closer.CloseWithLogOnError("file", file) 62 63 // First write the bytes we have already read, and then write whatever 64 // is left in the buffer, but only up to the maximum file limit. 65 available = system.Min(summaryRead, int(output.fileLimit)) 66 fileWritten, err := file.Write(summary[:available]) 67 if err != nil && err != io.EOF { 68 return err 69 } 70 71 _, err = io.CopyN(file, output.contents, int64(int(output.fileLimit)-fileWritten)) 72 if err != nil && err != io.EOF { 73 return err 74 } 75 76 return nil 77 } 78 79 // WriteJobResults produces files and a model.RunCommandResult in the standard 80 // format, including truncating the contents of both where necessary to fit 81 // within system-defined limits. 82 // 83 // It will consume only the bytes from the passed io.Readers that it needs to 84 // correctly form job outputs. Once the command returns, the readers can close. 85 func WriteJobResults(resultsDir string, stdout, stderr io.Reader, exitcode int, err error) (*model.RunCommandResult, error) { 86 result := model.NewRunCommandResult() 87 88 outputs := []outputResult{ 89 // Standard output 90 { 91 stdout, 92 model.DownloadFilenameStdout, 93 system.MaxStdoutFileLength, 94 &result.STDOUT, 95 system.MaxStdoutReturnLength, 96 &result.StdoutTruncated, 97 }, 98 // Standard error 99 { 100 stderr, 101 model.DownloadFilenameStderr, 102 system.MaxStderrFileLength, 103 &result.STDERR, 104 system.MaxStderrReturnLength, 105 &result.StderrTruncated, 106 }, 107 // Exit code 108 { 109 strings.NewReader(fmt.Sprint(exitcode)), 110 model.DownloadFilenameExitCode, 111 4, 112 nil, 113 4, 114 nil, 115 }, 116 } 117 118 wg := multierrgroup.Group{} 119 for _, output := range outputs { 120 output := output 121 wg.Go(func() error { 122 return writeOutputResult(resultsDir, output) 123 }) 124 } 125 126 err = multierr.Append(err, wg.Wait()) 127 if err != nil { 128 result.ErrorMsg = err.Error() 129 } 130 131 result.ExitCode = exitcode 132 return result, err 133 } 134 135 func FailResult(err error) (*model.RunCommandResult, error) { 136 return &model.RunCommandResult{ErrorMsg: err.Error()}, err 137 }