github.com/alibaba/ilogtail/pkg@v0.0.0-20250526110833-c53b480d046c/protocol/converter/jsonline_test.go (about)

     1  package protocol
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"testing"
     7  
     8  	. "github.com/smartystreets/goconvey/convey"
     9  
    10  	"github.com/alibaba/ilogtail/pkg/config"
    11  	"github.com/alibaba/ilogtail/pkg/flags"
    12  	"github.com/alibaba/ilogtail/pkg/protocol"
    13  )
    14  
    15  func TestNewConvertToJsonlineLogs(t *testing.T) {
    16  	Convey("When constructing converter with unsupported encoding", t, func() {
    17  		_, err := NewConverter(ProtocolJsonline, EncodingNone, nil, nil, &config.GlobalConfig{})
    18  		So(err, ShouldNotBeNil)
    19  	})
    20  
    21  	Convey("Given a converter with protocol: single, encoding: json, with tag rename and protocol key rename", t, func() {
    22  		keyRenameMap := map[string]string{
    23  			"k8s.node.ip": "ip",
    24  			"host.name":   "hostname",
    25  			"label":       "tag",
    26  			"env":         "env_tag",
    27  		}
    28  		protocolKeyRenameMap := map[string]string{
    29  			"time": "@timestamp",
    30  		}
    31  		c, err := NewConverter(ProtocolJsonline, EncodingJSON, keyRenameMap, protocolKeyRenameMap, &config.GlobalConfig{})
    32  		So(err, ShouldBeNil)
    33  
    34  		Convey("When the logGroup is generated from files and from k8s daemonset environment", func() {
    35  			*flags.K8sFlag = true
    36  			time := []uint32{1662434209, 1662434487}
    37  			method := []string{"PUT", "GET"}
    38  			status := []string{"200", "404"}
    39  			logs := make([]*protocol.Log, 2)
    40  			for i := 0; i < 2; i++ {
    41  				logs[i] = &protocol.Log{
    42  					Time: time[i],
    43  					Contents: []*protocol.Log_Content{
    44  						{Key: "method", Value: method[i]},
    45  						{Key: "status", Value: status[i]},
    46  						{Key: "__tag__:__user_defined_id__", Value: "machine"},
    47  						{Key: "__tag__:__path__", Value: "/root/test/origin/example.log"},
    48  						{Key: "__tag__:_node_name_", Value: "node"},
    49  						{Key: "__tag__:_node_ip_", Value: "172.10.1.19"},
    50  						{Key: "__tag__:_namespace_", Value: "default"},
    51  						{Key: "__tag__:_pod_name_", Value: "container"},
    52  						{Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"},
    53  						{Key: "__tag__:_container_name_", Value: "container"},
    54  						{Key: "__tag__:_container_ip_", Value: "172.10.0.45"},
    55  						{Key: "__tag__:_image_name_", Value: "image"},
    56  						{Key: "__tag__:label", Value: "tag"},
    57  						{Key: "__log_topic__", Value: "file"},
    58  					},
    59  				}
    60  			}
    61  			tags := []*protocol.LogTag{
    62  				{Key: "__hostname__", Value: "alje834hgf"},
    63  				{Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"},
    64  				{Key: "env", Value: "K8S"},
    65  			}
    66  			logGroup := &protocol.LogGroup{
    67  				Logs:     logs,
    68  				Category: "test",
    69  				Topic:    "file",
    70  				Source:   "172.10.0.56",
    71  				LogTags:  tags,
    72  			}
    73  
    74  			Convey("Then the converted log should be valid", func() {
    75  				b, err := c.ToByteStream(logGroup)
    76  				So(err, ShouldBeNil)
    77  
    78  				for _, s := range bytes.Split(b.([]byte), []byte("\n")) {
    79  					unmarshaledLog := make(map[string]interface{})
    80  					err = json.Unmarshal(s, &unmarshaledLog)
    81  					So(err, ShouldBeNil)
    82  					So(unmarshaledLog, ShouldContainKey, "method")
    83  					So(unmarshaledLog, ShouldContainKey, "@timestamp")
    84  					So(unmarshaledLog, ShouldContainKey, "log.file.path")
    85  					So(unmarshaledLog, ShouldContainKey, "hostname")
    86  					So(unmarshaledLog, ShouldContainKey, "host.ip")
    87  					So(unmarshaledLog, ShouldContainKey, "log.topic")
    88  					So(unmarshaledLog, ShouldContainKey, "ip")
    89  					So(unmarshaledLog, ShouldContainKey, "k8s.node.name")
    90  					So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name")
    91  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.name")
    92  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid")
    93  					So(unmarshaledLog, ShouldContainKey, "k8s.container.name")
    94  					So(unmarshaledLog, ShouldContainKey, "k8s.container.ip")
    95  					So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name")
    96  					So(unmarshaledLog, ShouldContainKey, "tag")
    97  					So(unmarshaledLog, ShouldContainKey, "env_tag")
    98  				}
    99  			})
   100  
   101  			Convey("Then the corresponding value of the required fields are returned correctly", func() {
   102  				_, values, err := c.ToByteStreamWithSelectedFields(logGroup, []string{"content.method", "tag.host.name", "tag.ip"})
   103  				So(err, ShouldBeNil)
   104  				So(values, ShouldHaveLength, 0)
   105  			})
   106  		})
   107  	})
   108  
   109  	Convey("Given a converter with protocol: single, encoding: json, with null tag rename", t, func() {
   110  		keyRenameMap := map[string]string{
   111  			"k8s.node.ip": "",
   112  			"host.name":   "",
   113  			"label":       "",
   114  			"env":         "",
   115  		}
   116  		c, err := NewConverter(ProtocolJsonline, EncodingJSON, keyRenameMap, nil, &config.GlobalConfig{})
   117  		So(err, ShouldBeNil)
   118  
   119  		Convey("When the logGroup is generated from files and from k8s daemonset environment", func() {
   120  			*flags.K8sFlag = true
   121  			time := []uint32{1662434209, 1662434487}
   122  			method := []string{"PUT", "GET"}
   123  			status := []string{"200", "404"}
   124  			logs := make([]*protocol.Log, 2)
   125  			for i := 0; i < 2; i++ {
   126  				logs[i] = &protocol.Log{
   127  					Time: time[i],
   128  					Contents: []*protocol.Log_Content{
   129  						{Key: "method", Value: method[i]},
   130  						{Key: "status", Value: status[i]},
   131  						{Key: "__tag__:__user_defined_id__", Value: "machine"},
   132  						{Key: "__tag__:__path__", Value: "/root/test/origin/example.log"},
   133  						{Key: "__tag__:_node_name_", Value: "node"},
   134  						{Key: "__tag__:_node_ip_", Value: "172.10.1.19"},
   135  						{Key: "__tag__:_namespace_", Value: "default"},
   136  						{Key: "__tag__:_pod_name_", Value: "container"},
   137  						{Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"},
   138  						{Key: "__tag__:_container_name_", Value: "container"},
   139  						{Key: "__tag__:_container_ip_", Value: "172.10.0.45"},
   140  						{Key: "__tag__:_image_name_", Value: "image"},
   141  						{Key: "__tag__:label", Value: "tag"},
   142  						{Key: "__log_topic__", Value: "file"},
   143  					},
   144  				}
   145  			}
   146  			tags := []*protocol.LogTag{
   147  				{Key: "__hostname__", Value: "alje834hgf"},
   148  				{Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"},
   149  				{Key: "env", Value: "K8S"},
   150  			}
   151  			logGroup := &protocol.LogGroup{
   152  				Logs:     logs,
   153  				Category: "test",
   154  				Topic:    "file",
   155  				Source:   "172.10.0.56",
   156  				LogTags:  tags,
   157  			}
   158  
   159  			Convey("Then the converted log should be valid", func() {
   160  				b, err := c.ToByteStream(logGroup)
   161  				So(err, ShouldBeNil)
   162  
   163  				for _, s := range bytes.Split(b.([]byte), []byte("\n")) {
   164  					unmarshaledLog := make(map[string]interface{})
   165  					err = json.Unmarshal(s, &unmarshaledLog)
   166  					So(err, ShouldBeNil)
   167  					So(unmarshaledLog, ShouldContainKey, "time")
   168  					So(unmarshaledLog, ShouldContainKey, "method")
   169  					So(unmarshaledLog, ShouldContainKey, "status")
   170  					So(unmarshaledLog, ShouldContainKey, "log.file.path")
   171  					So(unmarshaledLog, ShouldContainKey, "host.ip")
   172  					So(unmarshaledLog, ShouldContainKey, "log.topic")
   173  					So(unmarshaledLog, ShouldContainKey, "k8s.node.name")
   174  					So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name")
   175  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.name")
   176  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid")
   177  					So(unmarshaledLog, ShouldContainKey, "k8s.container.name")
   178  					So(unmarshaledLog, ShouldContainKey, "k8s.container.ip")
   179  					So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name")
   180  				}
   181  			})
   182  		})
   183  
   184  		Convey("When the log is standardized", func() {
   185  			*flags.K8sFlag = true
   186  			time := []uint32{1662434209, 1662434487}
   187  			method := []string{"PUT", "GET"}
   188  			status := []string{"200", "404"}
   189  			logs := make([]*protocol.Log, 2)
   190  			for i := 0; i < 2; i++ {
   191  				logs[i] = &protocol.Log{
   192  					Time: time[i],
   193  					Contents: []*protocol.Log_Content{
   194  						{Key: "method", Value: method[i]},
   195  						{Key: "status", Value: status[i]},
   196  					},
   197  				}
   198  			}
   199  			tags := []*protocol.LogTag{
   200  				{Key: "__user_defined_id__", Value: "machine"},
   201  				{Key: "__hostname__", Value: "alje834hgf"},
   202  				{Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"},
   203  				{Key: "__path__", Value: "/root/test/origin/example.log"},
   204  				{Key: "_node_name_", Value: "node"},
   205  				{Key: "_node_ip_", Value: "172.10.1.19"},
   206  				{Key: "_namespace_", Value: "default"},
   207  				{Key: "_pod_name_", Value: "container"},
   208  				{Key: "_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"},
   209  				{Key: "_container_name_", Value: "container"},
   210  				{Key: "_container_ip_", Value: "172.10.0.45"},
   211  				{Key: "_image_name_", Value: "image"},
   212  				{Key: "label", Value: "tag"},
   213  			}
   214  			logGroup := &protocol.LogGroup{
   215  				Logs:     logs,
   216  				Category: "test",
   217  				Topic:    "topic",
   218  				Source:   "172.10.0.56",
   219  				LogTags:  tags,
   220  			}
   221  
   222  			Convey("Then the converted log should be valid", func() {
   223  				b, err := c.ToByteStream(logGroup)
   224  				So(err, ShouldBeNil)
   225  
   226  				for _, s := range bytes.Split(b.([]byte), []byte("\n")) {
   227  					unmarshaledLog := make(map[string]interface{})
   228  					err = json.Unmarshal(s, &unmarshaledLog)
   229  					So(err, ShouldBeNil)
   230  					So(unmarshaledLog, ShouldContainKey, "time")
   231  					So(unmarshaledLog, ShouldContainKey, "method")
   232  					So(unmarshaledLog, ShouldContainKey, "status")
   233  					So(unmarshaledLog, ShouldContainKey, "log.file.path")
   234  					So(unmarshaledLog, ShouldContainKey, "host.ip")
   235  					So(unmarshaledLog, ShouldContainKey, "log.topic")
   236  					So(unmarshaledLog, ShouldContainKey, "k8s.node.name")
   237  					So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name")
   238  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.name")
   239  					So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid")
   240  					So(unmarshaledLog, ShouldContainKey, "k8s.container.name")
   241  					So(unmarshaledLog, ShouldContainKey, "k8s.container.ip")
   242  					So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name")
   243  				}
   244  			})
   245  		})
   246  	})
   247  }