github.com/Jeffail/benthos/v3@v3.65.0/lib/output/hdfs.go (about)

     1  package output
     2  
     3  import (
     4  	"github.com/Jeffail/benthos/v3/internal/docs"
     5  	"github.com/Jeffail/benthos/v3/lib/log"
     6  	"github.com/Jeffail/benthos/v3/lib/message/batch"
     7  	"github.com/Jeffail/benthos/v3/lib/metrics"
     8  	"github.com/Jeffail/benthos/v3/lib/output/writer"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  )
    11  
    12  //------------------------------------------------------------------------------
    13  
    14  func init() {
    15  	Constructors[TypeHDFS] = TypeSpec{
    16  		constructor: fromSimpleConstructor(NewHDFS),
    17  		Summary: `
    18  Sends message parts as files to a HDFS directory.`,
    19  		Description: `
    20  Each file is written with the path specified with the 'path' field, in order to
    21  have a different path for each object you should use function interpolations
    22  described [here](/docs/configuration/interpolation#bloblang-queries).`,
    23  		Async: true,
    24  		FieldSpecs: docs.FieldSpecs{
    25  			docs.FieldCommon("hosts", "A list of hosts to connect to.", "localhost:9000").Array(),
    26  			docs.FieldCommon("user", "A user identifier."),
    27  			docs.FieldCommon("directory", "A directory to store message files within. If the directory does not exist it will be created."),
    28  			docs.FieldCommon(
    29  				"path", "The path to upload messages as, interpolation functions should be used in order to generate unique file paths.",
    30  				`${!count("files")}-${!timestamp_unix_nano()}.txt`,
    31  			).IsInterpolated(),
    32  			docs.FieldCommon("max_in_flight", "The maximum number of messages to have in flight at a given time. Increase this to improve throughput."),
    33  			batch.FieldSpec(),
    34  		},
    35  		Categories: []Category{
    36  			CategoryServices,
    37  		},
    38  	}
    39  }
    40  
    41  //------------------------------------------------------------------------------
    42  
    43  // NewHDFS creates a new HDFS output type.
    44  func NewHDFS(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    45  	h, err := writer.NewHDFSV2(conf.HDFS, mgr, log, stats)
    46  	if err != nil {
    47  		return nil, err
    48  	}
    49  	w, err := NewAsyncWriter(
    50  		TypeHDFS, conf.HDFS.MaxInFlight, h, log, stats,
    51  	)
    52  	if err != nil {
    53  		return nil, err
    54  	}
    55  	return NewBatcherFromConfig(conf.HDFS.Batching, OnlySinglePayloads(w), mgr, log, stats)
    56  }
    57  
    58  //------------------------------------------------------------------------------