github.com/Jeffail/benthos/v3@v3.65.0/lib/output/writer/files.go (about)

     1  package writer
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"path/filepath"
     8  	"time"
     9  
    10  	"github.com/Jeffail/benthos/v3/internal/bloblang/field"
    11  	"github.com/Jeffail/benthos/v3/internal/interop"
    12  	"github.com/Jeffail/benthos/v3/lib/log"
    13  	"github.com/Jeffail/benthos/v3/lib/metrics"
    14  	"github.com/Jeffail/benthos/v3/lib/types"
    15  )
    16  
    17  //------------------------------------------------------------------------------
    18  
    19  // FilesConfig contains configuration fields for the files output type.
    20  type FilesConfig struct {
    21  	Path string `json:"path" yaml:"path"`
    22  }
    23  
    24  // NewFilesConfig creates a new Config with default values.
    25  func NewFilesConfig() FilesConfig {
    26  	return FilesConfig{
    27  		Path: `${!count("files")}-${!timestamp_unix_nano()}.txt`,
    28  	}
    29  }
    30  
    31  //------------------------------------------------------------------------------
    32  
    33  // Files is a benthos writer.Type implementation that writes message parts each
    34  // to their own file.
    35  type Files struct {
    36  	conf FilesConfig
    37  
    38  	path *field.Expression
    39  
    40  	log   log.Modular
    41  	stats metrics.Type
    42  }
    43  
    44  // NewFiles creates a new file based writer.Type.
    45  //
    46  // Deprecated: use the V2 API instead.
    47  func NewFiles(
    48  	conf FilesConfig,
    49  	log log.Modular,
    50  	stats metrics.Type,
    51  ) (*Files, error) {
    52  	return NewFilesV2(conf, types.NoopMgr(), log, stats)
    53  }
    54  
    55  // NewFilesV2 creates a new file based writer.Type.
    56  func NewFilesV2(
    57  	conf FilesConfig,
    58  	mgr types.Manager,
    59  	log log.Modular,
    60  	stats metrics.Type,
    61  ) (*Files, error) {
    62  	path, err := interop.NewBloblangField(mgr, conf.Path)
    63  	if err != nil {
    64  		return nil, fmt.Errorf("failed to parse path expression: %v", err)
    65  	}
    66  	return &Files{
    67  		conf:  conf,
    68  		path:  path,
    69  		log:   log,
    70  		stats: stats,
    71  	}, nil
    72  }
    73  
    74  // ConnectWithContext is a noop.
    75  func (f *Files) ConnectWithContext(ctx context.Context) error {
    76  	return f.Connect()
    77  }
    78  
    79  // Connect is a noop.
    80  func (f *Files) Connect() error {
    81  	f.log.Infoln("Writing message parts as files.")
    82  	return nil
    83  }
    84  
    85  // WriteWithContext attempts to write message contents to a directory as files.
    86  func (f *Files) WriteWithContext(ctx context.Context, msg types.Message) error {
    87  	return f.Write(msg)
    88  }
    89  
    90  // Write attempts to write message contents to a directory as files.
    91  func (f *Files) Write(msg types.Message) error {
    92  	return IterateBatchedSend(msg, func(i int, p types.Part) error {
    93  		path := f.path.String(i, msg)
    94  
    95  		err := os.MkdirAll(filepath.Dir(path), os.FileMode(0o777))
    96  		if err != nil {
    97  			return err
    98  		}
    99  
   100  		return os.WriteFile(path, p.Get(), os.FileMode(0o666))
   101  	})
   102  }
   103  
   104  // CloseAsync begins cleaning up resources used by this reader asynchronously.
   105  func (f *Files) CloseAsync() {
   106  }
   107  
   108  // WaitForClose will block until either the reader is closed or a specified
   109  // timeout occurs.
   110  func (f *Files) WaitForClose(time.Duration) error {
   111  	return nil
   112  }
   113  
   114  //------------------------------------------------------------------------------