github.com/alibaba/ilogtail/pkg@v0.0.0-20250526110833-c53b480d046c/protocol/converter/converter_single_log_flatten_test.go (about) 1 // Copyright 2023 iLogtail Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package protocol 16 17 import ( 18 "encoding/json" 19 "testing" 20 21 . "github.com/smartystreets/goconvey/convey" 22 23 "github.com/alibaba/ilogtail/pkg/config" 24 "github.com/alibaba/ilogtail/pkg/flags" 25 "github.com/alibaba/ilogtail/pkg/protocol" 26 ) 27 28 func TestConvertToSimpleFlat(t *testing.T) { 29 30 Convey("Given a converter with protocol: single, encoding: json, with tag rename and protocol key rename", t, func() { 31 keyRenameMap := map[string]string{ 32 "k8s.node.ip": "ip", 33 "host.name": "hostname", 34 "label": "tag", 35 "env": "env_tag", 36 } 37 protocolKeyRenameMap := map[string]string{ 38 "time": "@timestamp", 39 } 40 c, err := NewConverter("custom_single_flatten", "json", keyRenameMap, protocolKeyRenameMap, &config.GlobalConfig{}) 41 So(err, ShouldBeNil) 42 43 Convey("When the logGroup is generated from files and from k8s daemonset environment", func() { 44 *flags.K8sFlag = true 45 time := []uint32{1662434209, 1662434487} 46 method := []string{"PUT", "GET"} 47 status := []string{"200", "404"} 48 logs := make([]*protocol.Log, 2) 49 for i := 0; i < 2; i++ { 50 logs[i] = &protocol.Log{ 51 Time: time[i], 52 Contents: []*protocol.Log_Content{ 53 {Key: "method", Value: method[i]}, 54 {Key: "status", Value: status[i]}, 55 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 56 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 57 {Key: "__tag__:_node_name_", Value: "node"}, 58 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 59 {Key: "__tag__:_namespace_", Value: "default"}, 60 {Key: "__tag__:_pod_name_", Value: "container"}, 61 {Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 62 {Key: "__tag__:_container_name_", Value: "container"}, 63 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 64 {Key: "__tag__:_image_name_", Value: "image"}, 65 {Key: "__tag__:label", Value: "tag"}, 66 {Key: "__log_topic__", Value: "file"}, 67 }, 68 } 69 } 70 tags := []*protocol.LogTag{ 71 {Key: "__hostname__", Value: "alje834hgf"}, 72 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 73 {Key: "env", Value: "K8S"}, 74 } 75 logGroup := &protocol.LogGroup{ 76 Logs: logs, 77 Category: "test", 78 Topic: "file", 79 Source: "172.10.0.56", 80 LogTags: tags, 81 } 82 83 Convey("Then the converted log should be valid", func() { 84 b, err := c.ToByteStream(logGroup) 85 So(err, ShouldBeNil) 86 87 for _, s := range b.([][]byte) { 88 unmarshaledLog := make(map[string]interface{}) 89 err = json.Unmarshal(s, &unmarshaledLog) 90 So(err, ShouldBeNil) 91 So(unmarshaledLog, ShouldContainKey, "method") 92 So(unmarshaledLog, ShouldContainKey, "@timestamp") 93 So(unmarshaledLog, ShouldContainKey, "log.file.path") 94 So(unmarshaledLog, ShouldContainKey, "hostname") 95 So(unmarshaledLog, ShouldContainKey, "host.ip") 96 So(unmarshaledLog, ShouldContainKey, "log.topic") 97 So(unmarshaledLog, ShouldContainKey, "ip") 98 So(unmarshaledLog, ShouldContainKey, "k8s.node.name") 99 So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name") 100 So(unmarshaledLog, ShouldContainKey, "k8s.pod.name") 101 So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid") 102 So(unmarshaledLog, ShouldContainKey, "k8s.container.name") 103 So(unmarshaledLog, ShouldContainKey, "k8s.container.ip") 104 So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name") 105 So(unmarshaledLog, ShouldContainKey, "tag") 106 So(unmarshaledLog, ShouldContainKey, "env_tag") 107 } 108 }) 109 110 Convey("Then the corresponding value of the required fields are returned correctly", func() { 111 _, values, err := c.ToByteStreamWithSelectedFields(logGroup, []string{"content.method", "tag.host.name", "tag.ip"}) 112 So(err, ShouldBeNil) 113 So(values, ShouldHaveLength, 2) 114 So(values[0], ShouldHaveLength, 3) 115 So(values[0]["content.method"], ShouldEqual, "PUT") 116 So(values[0]["tag.host.name"], ShouldEqual, "alje834hgf") 117 So(values[0]["tag.ip"], ShouldEqual, "172.10.1.19") 118 So(values[1], ShouldHaveLength, 3) 119 So(values[1]["content.method"], ShouldEqual, "GET") 120 So(values[1]["tag.host.name"], ShouldEqual, "alje834hgf") 121 So(values[1]["tag.ip"], ShouldEqual, "172.10.1.19") 122 }) 123 124 Convey("Then error should be returned given invalid target field", func() { 125 _, _, err := c.ToByteStreamWithSelectedFields(logGroup, []string{"values.method"}) 126 So(err, ShouldNotBeNil) 127 }) 128 }) 129 }) 130 131 Convey("Given a converter with protocol: single, encoding: json, with null tag rename", t, func() { 132 keyRenameMap := map[string]string{ 133 "k8s.node.ip": "", 134 "host.name": "", 135 "label": "", 136 "env": "", 137 } 138 c, err := NewConverter("custom_single_flatten", "json", keyRenameMap, nil, &config.GlobalConfig{}) 139 So(err, ShouldBeNil) 140 141 Convey("When the logGroup is generated from files and from k8s daemonset environment", func() { 142 *flags.K8sFlag = true 143 time := []uint32{1662434209, 1662434487} 144 method := []string{"PUT", "GET"} 145 status := []string{"200", "404"} 146 logs := make([]*protocol.Log, 2) 147 for i := 0; i < 2; i++ { 148 logs[i] = &protocol.Log{ 149 Time: time[i], 150 Contents: []*protocol.Log_Content{ 151 {Key: "method", Value: method[i]}, 152 {Key: "status", Value: status[i]}, 153 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 154 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 155 {Key: "__tag__:_node_name_", Value: "node"}, 156 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 157 {Key: "__tag__:_namespace_", Value: "default"}, 158 {Key: "__tag__:_pod_name_", Value: "container"}, 159 {Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 160 {Key: "__tag__:_container_name_", Value: "container"}, 161 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 162 {Key: "__tag__:_image_name_", Value: "image"}, 163 {Key: "__tag__:label", Value: "tag"}, 164 {Key: "__log_topic__", Value: "file"}, 165 }, 166 } 167 } 168 tags := []*protocol.LogTag{ 169 {Key: "__hostname__", Value: "alje834hgf"}, 170 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 171 {Key: "env", Value: "K8S"}, 172 } 173 logGroup := &protocol.LogGroup{ 174 Logs: logs, 175 Category: "test", 176 Topic: "file", 177 Source: "172.10.0.56", 178 LogTags: tags, 179 } 180 181 Convey("Then the converted log should be valid", func() { 182 b, err := c.ToByteStream(logGroup) 183 So(err, ShouldBeNil) 184 185 for _, s := range b.([][]byte) { 186 unmarshaledLog := make(map[string]interface{}) 187 err = json.Unmarshal(s, &unmarshaledLog) 188 So(err, ShouldBeNil) 189 So(unmarshaledLog, ShouldContainKey, "time") 190 So(unmarshaledLog, ShouldContainKey, "method") 191 So(unmarshaledLog, ShouldContainKey, "status") 192 So(unmarshaledLog, ShouldContainKey, "log.file.path") 193 So(unmarshaledLog, ShouldContainKey, "host.ip") 194 So(unmarshaledLog, ShouldContainKey, "log.topic") 195 So(unmarshaledLog, ShouldContainKey, "k8s.node.name") 196 So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name") 197 So(unmarshaledLog, ShouldContainKey, "k8s.pod.name") 198 So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid") 199 So(unmarshaledLog, ShouldContainKey, "k8s.container.name") 200 So(unmarshaledLog, ShouldContainKey, "k8s.container.ip") 201 So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name") 202 } 203 }) 204 }) 205 206 Convey("When the log is standardized", func() { 207 *flags.K8sFlag = true 208 time := []uint32{1662434209, 1662434487} 209 method := []string{"PUT", "GET"} 210 status := []string{"200", "404"} 211 logs := make([]*protocol.Log, 2) 212 for i := 0; i < 2; i++ { 213 logs[i] = &protocol.Log{ 214 Time: time[i], 215 Contents: []*protocol.Log_Content{ 216 {Key: "method", Value: method[i]}, 217 {Key: "status", Value: status[i]}, 218 }, 219 } 220 } 221 tags := []*protocol.LogTag{ 222 {Key: "__user_defined_id__", Value: "machine"}, 223 {Key: "__hostname__", Value: "alje834hgf"}, 224 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 225 {Key: "__path__", Value: "/root/test/origin/example.log"}, 226 {Key: "_node_name_", Value: "node"}, 227 {Key: "_node_ip_", Value: "172.10.1.19"}, 228 {Key: "_namespace_", Value: "default"}, 229 {Key: "_pod_name_", Value: "container"}, 230 {Key: "_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 231 {Key: "_container_name_", Value: "container"}, 232 {Key: "_container_ip_", Value: "172.10.0.45"}, 233 {Key: "_image_name_", Value: "image"}, 234 {Key: "label", Value: "tag"}, 235 } 236 logGroup := &protocol.LogGroup{ 237 Logs: logs, 238 Category: "test", 239 Topic: "topic", 240 Source: "172.10.0.56", 241 LogTags: tags, 242 } 243 244 Convey("Then the converted log should be valid", func() { 245 b, err := c.ToByteStream(logGroup) 246 So(err, ShouldBeNil) 247 248 for _, s := range b.([][]byte) { 249 unmarshaledLog := make(map[string]interface{}) 250 err = json.Unmarshal(s, &unmarshaledLog) 251 So(err, ShouldBeNil) 252 So(unmarshaledLog, ShouldContainKey, "time") 253 So(unmarshaledLog, ShouldContainKey, "method") 254 So(unmarshaledLog, ShouldContainKey, "status") 255 So(unmarshaledLog, ShouldContainKey, "log.file.path") 256 So(unmarshaledLog, ShouldContainKey, "host.ip") 257 So(unmarshaledLog, ShouldContainKey, "log.topic") 258 So(unmarshaledLog, ShouldContainKey, "k8s.node.name") 259 So(unmarshaledLog, ShouldContainKey, "k8s.namespace.name") 260 So(unmarshaledLog, ShouldContainKey, "k8s.pod.name") 261 So(unmarshaledLog, ShouldContainKey, "k8s.pod.uid") 262 So(unmarshaledLog, ShouldContainKey, "k8s.container.name") 263 So(unmarshaledLog, ShouldContainKey, "k8s.container.ip") 264 So(unmarshaledLog, ShouldContainKey, "k8s.container.image.name") 265 } 266 }) 267 }) 268 }) 269 }