github.com/Jeffail/benthos/v3@v3.65.0/lib/input/reader/files.go (about)

     1  package reader
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"time"
    10  
    11  	"github.com/Jeffail/benthos/v3/lib/message"
    12  	"github.com/Jeffail/benthos/v3/lib/types"
    13  )
    14  
    15  //------------------------------------------------------------------------------
    16  
    17  // FilesConfig contains configuration for the Files input type.
    18  type FilesConfig struct {
    19  	Path        string `json:"path" yaml:"path"`
    20  	DeleteFiles bool   `json:"delete_files" yaml:"delete_files"`
    21  }
    22  
    23  // NewFilesConfig creates a new FilesConfig with default values.
    24  func NewFilesConfig() FilesConfig {
    25  	return FilesConfig{
    26  		Path:        "",
    27  		DeleteFiles: false,
    28  	}
    29  }
    30  
    31  //------------------------------------------------------------------------------
    32  
    33  // Files is an input type that reads file contents at a path as messages.
    34  type Files struct {
    35  	targets []string
    36  	delete  bool
    37  }
    38  
    39  // NewFiles creates a new Files input type.
    40  func NewFiles(conf FilesConfig) (*Files, error) {
    41  	f := Files{
    42  		delete: conf.DeleteFiles,
    43  	}
    44  
    45  	if info, err := os.Stat(conf.Path); err != nil {
    46  		return nil, err
    47  	} else if !info.IsDir() {
    48  		f.targets = append(f.targets, conf.Path)
    49  		return &f, nil
    50  	}
    51  
    52  	err := filepath.Walk(conf.Path, func(path string, info os.FileInfo, werr error) error {
    53  		if werr != nil {
    54  			return werr
    55  		}
    56  		if info.IsDir() {
    57  			return nil
    58  		}
    59  		f.targets = append(f.targets, path)
    60  		return nil
    61  	})
    62  
    63  	return &f, err
    64  }
    65  
    66  //------------------------------------------------------------------------------
    67  
    68  // Connect establishes a connection.
    69  func (f *Files) Connect() (err error) {
    70  	return nil
    71  }
    72  
    73  // ConnectWithContext establishes a connection.
    74  func (f *Files) ConnectWithContext(ctx context.Context) (err error) {
    75  	return nil
    76  }
    77  
    78  //------------------------------------------------------------------------------
    79  
    80  // ReadWithContext a new Files message.
    81  func (f *Files) ReadWithContext(ctx context.Context) (types.Message, AsyncAckFn, error) {
    82  	if len(f.targets) == 0 {
    83  		return nil, nil, types.ErrTypeClosed
    84  	}
    85  
    86  	path := f.targets[0]
    87  	f.targets = f.targets[1:]
    88  
    89  	file, openerr := os.Open(path)
    90  	if openerr != nil {
    91  		return nil, nil, fmt.Errorf("failed to read file '%v': %v", path, openerr)
    92  	}
    93  	defer file.Close()
    94  
    95  	msgBytes, readerr := io.ReadAll(file)
    96  	if readerr != nil {
    97  		return nil, nil, readerr
    98  	}
    99  
   100  	msg := message.New([][]byte{msgBytes})
   101  	msg.Get(0).Metadata().Set("path", path)
   102  	return msg, func(ctx context.Context, res types.Response) error {
   103  		if f.delete {
   104  			if res.Error() == nil {
   105  				return os.Remove(path)
   106  			}
   107  		}
   108  		return nil
   109  	}, nil
   110  }
   111  
   112  // Read a new Files message.
   113  func (f *Files) Read() (types.Message, error) {
   114  	msg, _, err := f.ReadWithContext(context.Background())
   115  	return msg, err
   116  }
   117  
   118  // Acknowledge instructs whether unacknowledged messages have been successfully
   119  // propagated.
   120  func (f *Files) Acknowledge(err error) error {
   121  	return nil
   122  }
   123  
   124  // CloseAsync shuts down the Files input and stops processing requests.
   125  func (f *Files) CloseAsync() {
   126  }
   127  
   128  // WaitForClose blocks until the Files input has closed down.
   129  func (f *Files) WaitForClose(timeout time.Duration) error {
   130  	return nil
   131  }
   132  
   133  //------------------------------------------------------------------------------