github.com/observiq/carbon@v0.9.11-0.20200820160507-1b872e368a5e/operator/helper/parser_test.go (about)

     1  package helper
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/observiq/carbon/entry"
    10  	"github.com/observiq/carbon/operator"
    11  	"github.com/observiq/carbon/testutil"
    12  	"github.com/stretchr/testify/mock"
    13  	"github.com/stretchr/testify/require"
    14  	"go.uber.org/zap/zaptest"
    15  )
    16  
    17  func TestParserConfigMissingBase(t *testing.T) {
    18  	config := ParserConfig{}
    19  	context := testutil.NewBuildContext(t)
    20  	_, err := config.Build(context)
    21  	require.Error(t, err)
    22  	require.Contains(t, err.Error(), "missing required `type` field.")
    23  }
    24  
    25  func TestParserConfigInvalidTimeParser(t *testing.T) {
    26  	cfg := NewParserConfig("test-id", "test-type")
    27  	f := entry.NewRecordField("timestamp")
    28  	cfg.TimeParser = &TimeParser{
    29  		ParseFrom:  &f,
    30  		Layout:     "",
    31  		LayoutType: "strptime",
    32  	}
    33  
    34  	_, err := cfg.Build(testutil.NewBuildContext(t))
    35  	require.Error(t, err)
    36  	require.Contains(t, err.Error(), "missing required configuration parameter `layout`")
    37  }
    38  
    39  func TestParserConfigBuildValid(t *testing.T) {
    40  	cfg := NewParserConfig("test-id", "test-type")
    41  	f := entry.NewRecordField("timestamp")
    42  	cfg.TimeParser = &TimeParser{
    43  		ParseFrom:  &f,
    44  		Layout:     "",
    45  		LayoutType: "native",
    46  	}
    47  	_, err := cfg.Build(testutil.NewBuildContext(t))
    48  	require.NoError(t, err)
    49  }
    50  
    51  func TestParserMissingField(t *testing.T) {
    52  	parser := ParserOperator{
    53  		TransformerOperator: TransformerOperator{
    54  			WriterOperator: WriterOperator{
    55  				BasicOperator: BasicOperator{
    56  					OperatorID:    "test-id",
    57  					OperatorType:  "test-type",
    58  					SugaredLogger: zaptest.NewLogger(t).Sugar(),
    59  				},
    60  			},
    61  			OnError: DropOnError,
    62  		},
    63  		ParseFrom: entry.NewRecordField("test"),
    64  	}
    65  	parse := func(i interface{}) (interface{}, error) {
    66  		return i, nil
    67  	}
    68  	ctx := context.Background()
    69  	testEntry := entry.New()
    70  	err := parser.ProcessWith(ctx, testEntry, parse)
    71  	require.Error(t, err)
    72  	require.Contains(t, err.Error(), "Entry is missing the expected parse_from field.")
    73  }
    74  
    75  func TestParserInvalidParse(t *testing.T) {
    76  	buildContext := testutil.NewBuildContext(t)
    77  	parser := ParserOperator{
    78  		TransformerOperator: TransformerOperator{
    79  			WriterOperator: WriterOperator{
    80  				BasicOperator: BasicOperator{
    81  					OperatorID:    "test-id",
    82  					OperatorType:  "test-type",
    83  					SugaredLogger: buildContext.Logger,
    84  				},
    85  			},
    86  			OnError: DropOnError,
    87  		},
    88  		ParseFrom: entry.NewRecordField(),
    89  	}
    90  	parse := func(i interface{}) (interface{}, error) {
    91  		return i, fmt.Errorf("parse failure")
    92  	}
    93  	ctx := context.Background()
    94  	testEntry := entry.New()
    95  	err := parser.ProcessWith(ctx, testEntry, parse)
    96  	require.Error(t, err)
    97  	require.Contains(t, err.Error(), "parse failure")
    98  }
    99  
   100  func TestParserInvalidTimeParse(t *testing.T) {
   101  	buildContext := testutil.NewBuildContext(t)
   102  	parser := ParserOperator{
   103  		TransformerOperator: TransformerOperator{
   104  			WriterOperator: WriterOperator{
   105  				BasicOperator: BasicOperator{
   106  					OperatorID:    "test-id",
   107  					OperatorType:  "test-type",
   108  					SugaredLogger: buildContext.Logger,
   109  				},
   110  			},
   111  			OnError: DropOnError,
   112  		},
   113  		ParseFrom: entry.NewRecordField(),
   114  		ParseTo:   entry.NewRecordField(),
   115  		TimeParser: &TimeParser{
   116  			ParseFrom: func() *entry.Field {
   117  				f := entry.NewRecordField("missing-key")
   118  				return &f
   119  			}(),
   120  		},
   121  	}
   122  	parse := func(i interface{}) (interface{}, error) {
   123  		return i, nil
   124  	}
   125  	ctx := context.Background()
   126  	testEntry := entry.New()
   127  	err := parser.ProcessWith(ctx, testEntry, parse)
   128  	require.Error(t, err)
   129  	require.Contains(t, err.Error(), "time parser: log entry does not have the expected parse_from field")
   130  }
   131  
   132  func TestParserInvalidSeverityParse(t *testing.T) {
   133  	buildContext := testutil.NewBuildContext(t)
   134  	parser := ParserOperator{
   135  		TransformerOperator: TransformerOperator{
   136  			WriterOperator: WriterOperator{
   137  				BasicOperator: BasicOperator{
   138  					OperatorID:    "test-id",
   139  					OperatorType:  "test-type",
   140  					SugaredLogger: buildContext.Logger,
   141  				},
   142  			},
   143  			OnError: DropOnError,
   144  		},
   145  		SeverityParser: &SeverityParser{
   146  			ParseFrom: entry.NewRecordField("missing-key"),
   147  		},
   148  		ParseFrom: entry.NewRecordField(),
   149  		ParseTo:   entry.NewRecordField(),
   150  	}
   151  	parse := func(i interface{}) (interface{}, error) {
   152  		return i, nil
   153  	}
   154  	ctx := context.Background()
   155  	testEntry := entry.New()
   156  	err := parser.ProcessWith(ctx, testEntry, parse)
   157  	require.Error(t, err)
   158  	require.Contains(t, err.Error(), "severity parser: log entry does not have the expected parse_from field")
   159  }
   160  
   161  func TestParserInvalidTimeValidSeverityParse(t *testing.T) {
   162  	buildContext := testutil.NewBuildContext(t)
   163  	parser := ParserOperator{
   164  		TransformerOperator: TransformerOperator{
   165  			WriterOperator: WriterOperator{
   166  				BasicOperator: BasicOperator{
   167  					OperatorID:    "test-id",
   168  					OperatorType:  "test-type",
   169  					SugaredLogger: buildContext.Logger,
   170  				},
   171  			},
   172  			OnError: DropOnError,
   173  		},
   174  		TimeParser: &TimeParser{
   175  			ParseFrom: func() *entry.Field {
   176  				f := entry.NewRecordField("missing-key")
   177  				return &f
   178  			}(),
   179  		},
   180  		SeverityParser: &SeverityParser{
   181  			ParseFrom: entry.NewRecordField("severity"),
   182  			Mapping: map[string]entry.Severity{
   183  				"info": entry.Info,
   184  			},
   185  		},
   186  		ParseFrom: entry.NewRecordField(),
   187  		ParseTo:   entry.NewRecordField(),
   188  	}
   189  	parse := func(i interface{}) (interface{}, error) {
   190  		return i, nil
   191  	}
   192  	ctx := context.Background()
   193  	testEntry := entry.New()
   194  	testEntry.Set(entry.NewRecordField("severity"), "info")
   195  
   196  	err := parser.ProcessWith(ctx, testEntry, parse)
   197  	require.Error(t, err)
   198  	require.Contains(t, err.Error(), "time parser: log entry does not have the expected parse_from field")
   199  
   200  	// But, this should have been set anyways
   201  	require.Equal(t, entry.Info, testEntry.Severity)
   202  }
   203  
   204  func TestParserValidTimeInvalidSeverityParse(t *testing.T) {
   205  	buildContext := testutil.NewBuildContext(t)
   206  	parser := ParserOperator{
   207  		TransformerOperator: TransformerOperator{
   208  			WriterOperator: WriterOperator{
   209  				BasicOperator: BasicOperator{
   210  					OperatorID:    "test-id",
   211  					OperatorType:  "test-type",
   212  					SugaredLogger: buildContext.Logger,
   213  				},
   214  			},
   215  			OnError: DropOnError,
   216  		},
   217  		TimeParser: &TimeParser{
   218  			ParseFrom: func() *entry.Field {
   219  				f := entry.NewRecordField("timestamp")
   220  				return &f
   221  			}(),
   222  			LayoutType: "gotime",
   223  			Layout:     time.Kitchen,
   224  		},
   225  		SeverityParser: &SeverityParser{
   226  			ParseFrom: entry.NewRecordField("missing-key"),
   227  		},
   228  		ParseFrom: entry.NewRecordField(),
   229  		ParseTo:   entry.NewRecordField(),
   230  	}
   231  	parse := func(i interface{}) (interface{}, error) {
   232  		return i, nil
   233  	}
   234  	ctx := context.Background()
   235  	testEntry := entry.New()
   236  	testEntry.Set(entry.NewRecordField("timestamp"), "12:34PM")
   237  
   238  	err := parser.ProcessWith(ctx, testEntry, parse)
   239  	require.Error(t, err)
   240  	require.Contains(t, err.Error(), "severity parser: log entry does not have the expected parse_from field")
   241  
   242  	expected, _ := time.ParseInLocation(time.Kitchen, "12:34PM", time.Local)
   243  	expected = setTimestampYear(expected)
   244  	// But, this should have been set anyways
   245  	require.Equal(t, expected, testEntry.Timestamp)
   246  }
   247  
   248  func TestParserOutput(t *testing.T) {
   249  	output := &testutil.Operator{}
   250  	output.On("ID").Return("test-output")
   251  	output.On("Process", mock.Anything, mock.Anything).Return(nil)
   252  	buildContext := testutil.NewBuildContext(t)
   253  	parser := ParserOperator{
   254  		TransformerOperator: TransformerOperator{
   255  			OnError: DropOnError,
   256  			WriterOperator: WriterOperator{
   257  				BasicOperator: BasicOperator{
   258  					OperatorID:    "test-id",
   259  					OperatorType:  "test-type",
   260  					SugaredLogger: buildContext.Logger,
   261  				},
   262  				OutputOperators: []operator.Operator{output},
   263  			},
   264  		},
   265  		ParseFrom: entry.NewRecordField(),
   266  		ParseTo:   entry.NewRecordField(),
   267  	}
   268  	parse := func(i interface{}) (interface{}, error) {
   269  		return i, nil
   270  	}
   271  	ctx := context.Background()
   272  	testEntry := entry.New()
   273  	err := parser.ProcessWith(ctx, testEntry, parse)
   274  	require.NoError(t, err)
   275  	output.AssertCalled(t, "Process", mock.Anything, mock.Anything)
   276  }
   277  
   278  func TestParserWithPreserve(t *testing.T) {
   279  	output := &testutil.Operator{}
   280  	output.On("ID").Return("test-output")
   281  	output.On("Process", mock.Anything, mock.Anything).Return(nil)
   282  	buildContext := testutil.NewBuildContext(t)
   283  	parser := ParserOperator{
   284  		TransformerOperator: TransformerOperator{
   285  			OnError: DropOnError,
   286  			WriterOperator: WriterOperator{
   287  				BasicOperator: BasicOperator{
   288  					OperatorID:    "test-id",
   289  					OperatorType:  "test-type",
   290  					SugaredLogger: buildContext.Logger,
   291  				},
   292  				OutputOperators: []operator.Operator{output},
   293  			},
   294  		},
   295  		ParseFrom: entry.NewRecordField("parse_from"),
   296  		ParseTo:   entry.NewRecordField("parse_to"),
   297  		Preserve:  true,
   298  	}
   299  	parse := func(i interface{}) (interface{}, error) {
   300  		return i, nil
   301  	}
   302  	ctx := context.Background()
   303  	testEntry := entry.New()
   304  	testEntry.Set(parser.ParseFrom, "test-value")
   305  	err := parser.ProcessWith(ctx, testEntry, parse)
   306  	require.NoError(t, err)
   307  
   308  	actualValue, ok := testEntry.Get(parser.ParseFrom)
   309  	require.True(t, ok)
   310  	require.Equal(t, "test-value", actualValue)
   311  
   312  	actualValue, ok = testEntry.Get(parser.ParseTo)
   313  	require.True(t, ok)
   314  	require.Equal(t, "test-value", actualValue)
   315  }
   316  
   317  func TestParserWithoutPreserve(t *testing.T) {
   318  	output := &testutil.Operator{}
   319  	output.On("ID").Return("test-output")
   320  	output.On("Process", mock.Anything, mock.Anything).Return(nil)
   321  	buildContext := testutil.NewBuildContext(t)
   322  	parser := ParserOperator{
   323  		TransformerOperator: TransformerOperator{
   324  			OnError: DropOnError,
   325  			WriterOperator: WriterOperator{
   326  				BasicOperator: BasicOperator{
   327  					OperatorID:    "test-id",
   328  					OperatorType:  "test-type",
   329  					SugaredLogger: buildContext.Logger,
   330  				},
   331  				OutputOperators: []operator.Operator{output},
   332  			},
   333  		},
   334  		ParseFrom: entry.NewRecordField("parse_from"),
   335  		ParseTo:   entry.NewRecordField("parse_to"),
   336  		Preserve:  false,
   337  	}
   338  	parse := func(i interface{}) (interface{}, error) {
   339  		return i, nil
   340  	}
   341  	ctx := context.Background()
   342  	testEntry := entry.New()
   343  	testEntry.Set(parser.ParseFrom, "test-value")
   344  	err := parser.ProcessWith(ctx, testEntry, parse)
   345  	require.NoError(t, err)
   346  
   347  	_, ok := testEntry.Get(parser.ParseFrom)
   348  	require.False(t, ok)
   349  
   350  	actualValue, ok := testEntry.Get(parser.ParseTo)
   351  	require.True(t, ok)
   352  	require.Equal(t, "test-value", actualValue)
   353  }