github.com/Jeffail/benthos/v3@v3.65.0/internal/docs/yaml_path_test.go (about)

     1  package docs_test
     2  
     3  import (
     4  	"testing"
     5  
     6  	"github.com/Jeffail/benthos/v3/internal/docs"
     7  	"github.com/Jeffail/benthos/v3/lib/config"
     8  	"github.com/Jeffail/gabs/v2"
     9  	"github.com/stretchr/testify/assert"
    10  	"github.com/stretchr/testify/require"
    11  	"gopkg.in/yaml.v3"
    12  )
    13  
    14  func TestSetYAMLPath(t *testing.T) {
    15  	mockProv := docs.NewMappedDocsProvider()
    16  	mockProv.RegisterDocs(docs.ComponentSpec{
    17  		Name: "kafka",
    18  		Type: docs.TypeInput,
    19  		Config: docs.FieldComponent().WithChildren(
    20  			docs.FieldString("addresses", "").Array(),
    21  			docs.FieldString("topics", "").Array(),
    22  		),
    23  	})
    24  	mockProv.RegisterDocs(docs.ComponentSpec{
    25  		Name: "generate",
    26  		Type: docs.TypeInput,
    27  		Config: docs.FieldComponent().WithChildren(
    28  			docs.FieldString("mapping", ""),
    29  		),
    30  	})
    31  	mockProv.RegisterDocs(docs.ComponentSpec{
    32  		Name: "dynamic",
    33  		Type: docs.TypeInput,
    34  		Config: docs.FieldComponent().WithChildren(
    35  			docs.FieldCommon("inputs", "").HasType(docs.FieldTypeInput).Map(),
    36  		),
    37  	})
    38  	mockProv.RegisterDocs(docs.ComponentSpec{
    39  		Name: "nats",
    40  		Type: docs.TypeOutput,
    41  		Config: docs.FieldComponent().WithChildren(
    42  			docs.FieldString("urls", "").Array(),
    43  			docs.FieldString("subject", ""),
    44  			docs.FieldInt("max_in_flight", ""),
    45  		),
    46  	})
    47  	mockProv.RegisterDocs(docs.ComponentSpec{
    48  		Name: "compress",
    49  		Type: docs.TypeProcessor,
    50  		Config: docs.FieldComponent().WithChildren(
    51  			docs.FieldString("algorithm", ""),
    52  		),
    53  	})
    54  	mockProv.RegisterDocs(docs.ComponentSpec{
    55  		Name: "workflow",
    56  		Type: docs.TypeProcessor,
    57  		Config: docs.FieldComponent().WithChildren(
    58  			docs.FieldString("order", "").ArrayOfArrays(),
    59  		),
    60  	})
    61  
    62  	tests := []struct {
    63  		name        string
    64  		input       string
    65  		path        string
    66  		value       string
    67  		output      string
    68  		errContains string
    69  	}{
    70  		{
    71  			name: "set input",
    72  			input: `
    73  input:
    74    kafka:
    75      addresses: [ "foo", "bar" ]
    76      topics: [ "baz" ]
    77  
    78  output:
    79    nats:
    80      urls: [ nats://127.0.0.1:4222 ]
    81      subject: benthos_messages
    82      max_in_flight: 1
    83  `,
    84  			path: "/input",
    85  			value: `
    86  generate:
    87    mapping: 'root = {"foo":"bar"}'`,
    88  			output: `
    89  input:
    90    generate:
    91      mapping: 'root = {"foo":"bar"}'
    92  output:
    93    nats:
    94      urls: [ nats://127.0.0.1:4222 ]
    95      subject: benthos_messages
    96      max_in_flight: 1
    97  `,
    98  		},
    99  		{
   100  			name: "set input addresses total",
   101  			input: `
   102  input:
   103    kafka:
   104      addresses: [ "foo", "bar" ]
   105      topics: [ "baz" ]
   106  `,
   107  			path:  "/input/kafka/addresses",
   108  			value: `"foobar"`,
   109  			output: `
   110  input:
   111    kafka:
   112      addresses: [ "foobar" ]
   113      topics: [ "baz" ]
   114  `,
   115  		},
   116  		{
   117  			name: "set mapping value",
   118  			input: `
   119  input:
   120    kafka:
   121      addresses: [ "foo", "bar" ]
   122      topics: [ "baz" ]
   123  `,
   124  			path:  "/input/dynamic/inputs/foo/type",
   125  			value: `"foobar"`,
   126  			output: `
   127  input:
   128    dynamic:
   129      inputs:
   130        foo:
   131          type: "foobar"
   132    kafka:
   133      addresses: [ "foo", "bar" ]
   134      topics: [ "baz" ]
   135  `,
   136  		},
   137  		{
   138  			name:  "set value to object",
   139  			input: `input: "hello world"`,
   140  			path:  "/input/kafka/addresses",
   141  			value: `"foobar"`,
   142  			output: `
   143  input:
   144    kafka:
   145      addresses: ["foobar"]
   146  `,
   147  		},
   148  		{
   149  			name: "set array index",
   150  			input: `
   151  input:
   152    kafka:
   153      addresses: [ "foo", "bar" ]
   154      topics: [ "baz" ]
   155  `,
   156  			path:  "/input/kafka/addresses/0",
   157  			value: `"baz"`,
   158  			output: `
   159  input:
   160    kafka:
   161      addresses: [ "baz", "bar" ]
   162      topics: [ "baz" ]
   163  `,
   164  		},
   165  		{
   166  			name: "set array index child",
   167  			input: `
   168  input:
   169    kafka:
   170      addresses: [ "foo", "bar" ]
   171      topics: [ "baz" ]
   172    processors:
   173      - compress:
   174          algorithm: gzip
   175  `,
   176  			path:  "/input/processors/0/compress/algorithm",
   177  			value: `"baz"`,
   178  			output: `
   179  input:
   180    kafka:
   181      addresses: [ "foo", "bar" ]
   182      topics: [ "baz" ]
   183    processors:
   184      - compress:
   185          algorithm: baz
   186  `,
   187  		},
   188  		{
   189  			name: "set array append",
   190  			input: `
   191  input:
   192    kafka:
   193      addresses: [ "foo", "bar" ]
   194      topics: [ "baz" ]
   195  `,
   196  			path:  "/input/kafka/addresses/-",
   197  			value: `"baz"`,
   198  			output: `
   199  input:
   200    kafka:
   201      addresses: [ "foo", "bar", "baz" ]
   202      topics: [ "baz" ]
   203  `,
   204  		},
   205  		{
   206  			name: "set array NaN",
   207  			input: `
   208  input:
   209    kafka:
   210      addresses: [ "foo", "bar" ]
   211  `,
   212  			path:        "/input/kafka/addresses/nope",
   213  			value:       `"baz"`,
   214  			errContains: "input.kafka.addresses.nope: failed to parse path segment as array index",
   215  		},
   216  		{
   217  			name: "set array big index",
   218  			input: `
   219  input:
   220    kafka:
   221      addresses: [ "foo", "bar" ]
   222  `,
   223  			path:        "/input/kafka/addresses/2",
   224  			value:       `"baz"`,
   225  			errContains: "input.kafka.addresses.2: target index greater than",
   226  		},
   227  		{
   228  			name: "set nested array big index",
   229  			input: `
   230  input:
   231    kafka:
   232      addresses: [ [ "foo", "bar" ] ]
   233  `,
   234  			path:        "/input/kafka/addresses/0/2",
   235  			value:       `"baz"`,
   236  			errContains: "input.kafka.addresses.0.2: field not recognised",
   237  		},
   238  		{
   239  			name: "set 2D array value abs",
   240  			input: `
   241  pipeline:
   242    processors:
   243      - workflow:
   244          order: []
   245  `,
   246  			path:  "/pipeline/processors/0/workflow/order",
   247  			value: `"baz"`,
   248  			output: `
   249  pipeline:
   250    processors:
   251      - workflow:
   252          order: [["baz"]]
   253  `,
   254  		},
   255  		{
   256  			name: "set 2D array value outter index",
   257  			input: `
   258  pipeline:
   259    processors:
   260      - workflow:
   261          order: []
   262  `,
   263  			path:  "/pipeline/processors/0/workflow/order/-",
   264  			value: `"baz"`,
   265  			output: `
   266  pipeline:
   267    processors:
   268      - workflow:
   269          order: [["baz"]]
   270  `,
   271  		},
   272  		{
   273  			name: "set 2D array value inner index",
   274  			input: `
   275  pipeline:
   276    processors:
   277      - workflow:
   278          order: []
   279  `,
   280  			path:  "/pipeline/processors/0/workflow/order/-/-",
   281  			value: `"baz"`,
   282  			output: `
   283  pipeline:
   284    processors:
   285      - workflow:
   286          order: [["baz"]]
   287  `,
   288  		},
   289  	}
   290  
   291  	for _, test := range tests {
   292  		t.Run(test.name, func(t *testing.T) {
   293  			var input, value yaml.Node
   294  
   295  			require.NoError(t, yaml.Unmarshal([]byte(test.input), &input))
   296  			require.NoError(t, yaml.Unmarshal([]byte(test.value), &value))
   297  
   298  			path, err := gabs.JSONPointerToSlice(test.path)
   299  			require.NoError(t, err)
   300  
   301  			err = config.Spec().SetYAMLPath(mockProv, &input, &value, path...)
   302  			if len(test.errContains) > 0 {
   303  				require.Error(t, err)
   304  				assert.Contains(t, err.Error(), test.errContains)
   305  			} else {
   306  				require.NoError(t, err)
   307  
   308  				var iinput, ioutput interface{}
   309  				require.NoError(t, input.Decode(&iinput))
   310  				require.NoError(t, yaml.Unmarshal([]byte(test.output), &ioutput))
   311  				assert.Equal(t, ioutput, iinput)
   312  			}
   313  		})
   314  	}
   315  }
   316  
   317  func TestGetYAMLPath(t *testing.T) {
   318  	tests := []struct {
   319  		name        string
   320  		input       string
   321  		path        string
   322  		output      string
   323  		errContains string
   324  	}{
   325  		{
   326  			name: "all of input",
   327  			input: `
   328  input:
   329    kafka:
   330      addresses: [ "foo" ]
   331  `,
   332  			path: "/input",
   333  			output: `
   334  kafka:
   335    addresses: [ "foo" ]
   336  `,
   337  		},
   338  		{
   339  			name: "first address of input",
   340  			input: `
   341  input:
   342    kafka:
   343      addresses: [ "foo" ]
   344  `,
   345  			path:   "/input/kafka/addresses/0",
   346  			output: `"foo"`,
   347  		},
   348  		{
   349  			name: "unknown field",
   350  			input: `
   351  input:
   352    kafka:
   353      addresses: [ "foo" ]
   354  `,
   355  			path:        "/input/meow",
   356  			errContains: "input.meow: key not found in mapping",
   357  		},
   358  		{
   359  			name: "bad index",
   360  			input: `
   361  input:
   362    kafka:
   363      addresses: [ "foo" ]
   364  `,
   365  			path:        "/input/kafka/addresses/10",
   366  			errContains: "input.kafka.addresses.10: target index greater",
   367  		},
   368  	}
   369  
   370  	for _, test := range tests {
   371  		t.Run(test.name, func(t *testing.T) {
   372  			var input yaml.Node
   373  			require.NoError(t, yaml.Unmarshal([]byte(test.input), &input))
   374  
   375  			path, err := gabs.JSONPointerToSlice(test.path)
   376  			require.NoError(t, err)
   377  
   378  			output, err := docs.GetYAMLPath(&input, path...)
   379  			if len(test.errContains) > 0 {
   380  				require.Error(t, err)
   381  				assert.Contains(t, err.Error(), test.errContains)
   382  			} else {
   383  				require.NoError(t, err)
   384  
   385  				var expected, actual interface{}
   386  				require.NoError(t, output.Decode(&actual))
   387  				require.NoError(t, yaml.Unmarshal([]byte(test.output), &expected))
   388  				assert.Equal(t, expected, actual)
   389  			}
   390  		})
   391  	}
   392  }
   393  
   394  func TestYAMLLabelsToPath(t *testing.T) {
   395  	mockProv := docs.NewMappedDocsProvider()
   396  	mockProv.RegisterDocs(docs.ComponentSpec{
   397  		Name: "kafka",
   398  		Type: docs.TypeInput,
   399  		Config: docs.FieldComponent().WithChildren(
   400  			docs.FieldString("addresses", "").Array(),
   401  			docs.FieldString("topics", "").Array(),
   402  		),
   403  	})
   404  	mockProv.RegisterDocs(docs.ComponentSpec{
   405  		Name: "dynamic",
   406  		Type: docs.TypeInput,
   407  		Config: docs.FieldComponent().WithChildren(
   408  			docs.FieldCommon("inputs", "").HasType(docs.FieldTypeInput).Map(),
   409  		),
   410  	})
   411  	mockProv.RegisterDocs(docs.ComponentSpec{
   412  		Name: "nats",
   413  		Type: docs.TypeOutput,
   414  		Config: docs.FieldComponent().WithChildren(
   415  			docs.FieldString("urls", "").Array(),
   416  			docs.FieldString("subject", ""),
   417  			docs.FieldInt("max_in_flight", ""),
   418  		),
   419  	})
   420  	mockProv.RegisterDocs(docs.ComponentSpec{
   421  		Name: "compress",
   422  		Type: docs.TypeProcessor,
   423  		Config: docs.FieldComponent().WithChildren(
   424  			docs.FieldString("algorithm", ""),
   425  		),
   426  	})
   427  	mockProv.RegisterDocs(docs.ComponentSpec{
   428  		Name: "for_each",
   429  		Type: docs.TypeProcessor,
   430  		Config: docs.FieldComponent().WithChildren(
   431  			docs.FieldCommon("things", "").HasType(docs.FieldTypeProcessor).Array(),
   432  		),
   433  	})
   434  	mockProv.RegisterDocs(docs.ComponentSpec{
   435  		Name: "mega_for_each",
   436  		Type: docs.TypeProcessor,
   437  		Config: docs.FieldComponent().WithChildren(
   438  			docs.FieldCommon("things", "").HasType(docs.FieldTypeProcessor).ArrayOfArrays(),
   439  		),
   440  	})
   441  	mockProv.RegisterDocs(docs.ComponentSpec{
   442  		Name: "workflow",
   443  		Type: docs.TypeProcessor,
   444  		Config: docs.FieldComponent().WithChildren(
   445  			docs.FieldCommon("things", "").HasType(docs.FieldTypeProcessor).Map(),
   446  		),
   447  	})
   448  
   449  	tests := []struct {
   450  		name   string
   451  		input  string
   452  		output map[string][]string
   453  	}{
   454  		{
   455  			name: "no labels",
   456  			input: `
   457  input:
   458    kafka:
   459      addresses: [ "foo", "bar" ]
   460      topics: [ "baz" ]
   461  
   462  output:
   463    nats:
   464      urls: [ nats://127.0.0.1:4222 ]
   465      subject: benthos_messages
   466      max_in_flight: 1
   467  `,
   468  			output: map[string][]string{},
   469  		},
   470  		{
   471  			name: "basic components all with labels",
   472  			input: `
   473  input:
   474    label: fooinput
   475    kafka:
   476      addresses: [ "foo", "bar" ]
   477      topics: [ "baz" ]
   478  
   479  pipeline:
   480    processors:
   481      - label: fooproc1
   482        compress:
   483          algorithm: nahm8
   484  
   485  output:
   486    label: foooutput
   487    nats:
   488      urls: [ nats://127.0.0.1:4222 ]
   489      subject: benthos_messages
   490      max_in_flight: 1
   491  `,
   492  			output: map[string][]string{
   493  				"fooinput":  {"input"},
   494  				"fooproc1":  {"pipeline", "processors", "0"},
   495  				"foooutput": {"output"},
   496  			},
   497  		},
   498  		{
   499  			name: "Array of procs",
   500  			input: `
   501  pipeline:
   502    processors:
   503      - label: fooproc1
   504        for_each:
   505          things:
   506          - label: fooproc2
   507            compress:
   508              algorithm: nahm8
   509          - label: fooproc3
   510            compress:
   511              algorithm: nahm8
   512  `,
   513  			output: map[string][]string{
   514  				"fooproc1": {"pipeline", "processors", "0"},
   515  				"fooproc2": {"pipeline", "processors", "0", "for_each", "things", "0"},
   516  				"fooproc3": {"pipeline", "processors", "0", "for_each", "things", "1"},
   517  			},
   518  		},
   519  		{
   520  			name: "array of array of procs",
   521  			input: `
   522  pipeline:
   523    processors:
   524      - label: fooproc1
   525        mega_for_each:
   526          things:
   527          -
   528            - label: fooproc2
   529              compress:
   530                algorithm: nahm8
   531            - label: fooproc3
   532              compress:
   533                algorithm: nahm8
   534          -
   535            - label: fooproc4
   536              compress:
   537                algorithm: nahm8
   538  `,
   539  			output: map[string][]string{
   540  				"fooproc1": {"pipeline", "processors", "0"},
   541  				"fooproc2": {"pipeline", "processors", "0", "mega_for_each", "things", "0", "0"},
   542  				"fooproc3": {"pipeline", "processors", "0", "mega_for_each", "things", "0", "1"},
   543  				"fooproc4": {"pipeline", "processors", "0", "mega_for_each", "things", "1", "0"},
   544  			},
   545  		},
   546  		{
   547  			name: "map of procs",
   548  			input: `
   549  pipeline:
   550    processors:
   551      - label: fooproc1
   552        workflow:
   553          things:
   554            first:
   555              label: fooproc2
   556              compress:
   557                algorithm: nahm8
   558            second:
   559              label: fooproc3
   560              compress:
   561                algorithm: nahm8
   562            third:
   563              label: fooproc4
   564              compress:
   565                algorithm: nahm8
   566  `,
   567  			output: map[string][]string{
   568  				"fooproc1": {"pipeline", "processors", "0"},
   569  				"fooproc2": {"pipeline", "processors", "0", "workflow", "things", "first"},
   570  				"fooproc3": {"pipeline", "processors", "0", "workflow", "things", "second"},
   571  				"fooproc4": {"pipeline", "processors", "0", "workflow", "things", "third"},
   572  			},
   573  		},
   574  	}
   575  
   576  	for _, test := range tests {
   577  		t.Run(test.name, func(t *testing.T) {
   578  			var input yaml.Node
   579  			require.NoError(t, yaml.Unmarshal([]byte(test.input), &input))
   580  
   581  			paths := map[string][]string{}
   582  
   583  			config.Spec().YAMLLabelsToPaths(mockProv, &input, paths, nil)
   584  			assert.Equal(t, test.output, paths)
   585  		})
   586  	}
   587  }