github.com/alibaba/ilogtail/pkg@v0.0.0-20250526110833-c53b480d046c/protocol/converter/custom_single_log_test.go (about) 1 // Copyright 2022 iLogtail Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package protocol 16 17 import ( 18 "encoding/json" 19 "fmt" 20 "testing" 21 22 . "github.com/smartystreets/goconvey/convey" 23 24 "github.com/alibaba/ilogtail/pkg/config" 25 "github.com/alibaba/ilogtail/pkg/flags" 26 "github.com/alibaba/ilogtail/pkg/protocol" 27 ) 28 29 func TestConvertToSimple(t *testing.T) { 30 Convey("Given a converter with protocol: single, encoding: json, with no tag rename or protocol key rename", t, func() { 31 c, err := NewConverter("custom_single", "json", nil, nil, &config.GlobalConfig{}) 32 So(err, ShouldBeNil) 33 34 Convey("When the logGroup is generated from files and from host environment", func() { 35 *flags.K8sFlag = false 36 time := []uint32{1662434209, 1662434487} 37 method := []string{"PUT", "GET"} 38 status := []string{"200", "404"} 39 logs := make([]*protocol.Log, 2) 40 for i := 0; i < 2; i++ { 41 logs[i] = &protocol.Log{ 42 Time: time[i], 43 Contents: []*protocol.Log_Content{ 44 {Key: "method", Value: method[i]}, 45 {Key: "status", Value: status[i]}, 46 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 47 {Key: "__log_topic__", Value: "file"}, 48 }, 49 } 50 } 51 tags := []*protocol.LogTag{ 52 {Key: "__hostname__", Value: "alje834hgf"}, 53 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 54 } 55 logGroup := &protocol.LogGroup{ 56 Logs: logs, 57 Category: "test", 58 Topic: "file", 59 Source: "172.10.0.56", 60 LogTags: tags, 61 } 62 63 Convey("Then the converted log should be valid", func() { 64 b, err := c.ToByteStream(logGroup) 65 So(err, ShouldBeNil) 66 67 for _, s := range b.([][]byte) { 68 unmarshaledLog := make(map[string]interface{}) 69 err = json.Unmarshal(s, &unmarshaledLog) 70 So(err, ShouldBeNil) 71 So(unmarshaledLog, ShouldHaveLength, 3) 72 So(unmarshaledLog, ShouldContainKey, "time") 73 So(unmarshaledLog, ShouldContainKey, "contents") 74 So(unmarshaledLog, ShouldContainKey, "tags") 75 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 76 So(unmarshaledLog["contents"], ShouldContainKey, "method") 77 So(unmarshaledLog["contents"], ShouldContainKey, "status") 78 So(unmarshaledLog["tags"], ShouldHaveLength, 4) 79 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 80 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 81 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 82 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 83 } 84 }) 85 }) 86 87 Convey("When the logGroup is generated from files and from docker environment", func() { 88 *flags.K8sFlag = false 89 time := []uint32{1662434209, 1662434487} 90 method := []string{"PUT", "GET"} 91 status := []string{"200", "404"} 92 logs := make([]*protocol.Log, 2) 93 for i := 0; i < 2; i++ { 94 logs[i] = &protocol.Log{ 95 Time: time[i], 96 Contents: []*protocol.Log_Content{ 97 {Key: "method", Value: method[i]}, 98 {Key: "status", Value: status[i]}, 99 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 100 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 101 {Key: "__tag__:_container_name_", Value: "container"}, 102 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 103 {Key: "__tag__:_image_name_", Value: "image"}, 104 {Key: "__log_topic__", Value: "file"}, 105 }, 106 } 107 } 108 tags := []*protocol.LogTag{ 109 {Key: "__hostname__", Value: "alje834hgf"}, 110 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 111 } 112 logGroup := &protocol.LogGroup{ 113 Logs: logs, 114 Category: "test", 115 Topic: "file", 116 Source: "172.10.0.56", 117 LogTags: tags, 118 } 119 120 Convey("Then the converted log should be valid", func() { 121 b, err := c.ToByteStream(logGroup) 122 So(err, ShouldBeNil) 123 124 for _, s := range b.([][]byte) { 125 unmarshaledLog := make(map[string]interface{}) 126 err = json.Unmarshal(s, &unmarshaledLog) 127 So(err, ShouldBeNil) 128 So(unmarshaledLog, ShouldHaveLength, 3) 129 So(unmarshaledLog, ShouldContainKey, "time") 130 So(unmarshaledLog, ShouldContainKey, "contents") 131 So(unmarshaledLog, ShouldContainKey, "tags") 132 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 133 So(unmarshaledLog["contents"], ShouldContainKey, "method") 134 So(unmarshaledLog["contents"], ShouldContainKey, "status") 135 So(unmarshaledLog["tags"], ShouldHaveLength, 7) 136 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 137 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 138 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 139 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 140 So(unmarshaledLog["tags"], ShouldContainKey, "container.name") 141 So(unmarshaledLog["tags"], ShouldContainKey, "container.ip") 142 So(unmarshaledLog["tags"], ShouldContainKey, "container.image.name") 143 } 144 }) 145 }) 146 147 Convey("When the logGroup is generated from files and from k8s daemonset environment", func() { 148 *flags.K8sFlag = true 149 time := []uint32{1662434209, 1662434487} 150 method := []string{"PUT", "GET"} 151 status := []string{"200", "404"} 152 logs := make([]*protocol.Log, 2) 153 for i := 0; i < 2; i++ { 154 logs[i] = &protocol.Log{ 155 Time: time[i], 156 Contents: []*protocol.Log_Content{ 157 {Key: "method", Value: method[i]}, 158 {Key: "status", Value: status[i]}, 159 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 160 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 161 {Key: "__tag__:_node_name_", Value: "node"}, 162 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 163 {Key: "__tag__:_namespace_", Value: "default"}, 164 {Key: "__tag__:_pod_name_", Value: "container"}, 165 {Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 166 {Key: "__tag__:_container_name_", Value: "container"}, 167 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 168 {Key: "__tag__:_image_name_", Value: "image"}, 169 {Key: "__tag__:label", Value: "tag"}, 170 {Key: "__log_topic__", Value: "file"}, 171 }, 172 } 173 } 174 tags := []*protocol.LogTag{ 175 {Key: "__hostname__", Value: "alje834hgf"}, 176 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 177 } 178 logGroup := &protocol.LogGroup{ 179 Logs: logs, 180 Category: "test", 181 Topic: "file", 182 Source: "172.10.0.56", 183 LogTags: tags, 184 } 185 186 Convey("Then the converted log should be valid", func() { 187 b, err := c.ToByteStream(logGroup) 188 So(err, ShouldBeNil) 189 190 for _, s := range b.([][]byte) { 191 unmarshaledLog := make(map[string]interface{}) 192 err = json.Unmarshal(s, &unmarshaledLog) 193 So(err, ShouldBeNil) 194 So(unmarshaledLog, ShouldHaveLength, 3) 195 So(unmarshaledLog, ShouldContainKey, "time") 196 So(unmarshaledLog, ShouldContainKey, "contents") 197 So(unmarshaledLog, ShouldContainKey, "tags") 198 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 199 So(unmarshaledLog["contents"], ShouldContainKey, "method") 200 So(unmarshaledLog["contents"], ShouldContainKey, "status") 201 So(unmarshaledLog["tags"], ShouldHaveLength, 13) 202 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 203 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 204 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 205 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 206 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.ip") 207 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 208 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 209 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 210 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.uid") 211 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.name") 212 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.ip") 213 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.image.name") 214 So(unmarshaledLog["tags"], ShouldContainKey, "label") 215 } 216 }) 217 }) 218 219 Convey("When the logGroup is generated from files and from k8s sidecar environment", func() { 220 *flags.K8sFlag = false 221 time := []uint32{1662434209, 1662434487} 222 method := []string{"PUT", "GET"} 223 status := []string{"200", "404"} 224 logs := make([]*protocol.Log, 2) 225 for i := 0; i < 2; i++ { 226 logs[i] = &protocol.Log{ 227 Time: time[i], 228 Contents: []*protocol.Log_Content{ 229 {Key: "method", Value: method[i]}, 230 {Key: "status", Value: status[i]}, 231 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 232 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 233 {Key: "__tag__:_node_name_", Value: "node"}, 234 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 235 {Key: "__tag__:_namespace_", Value: "default"}, 236 {Key: "__tag__:_pod_name_", Value: "container"}, 237 {Key: "__tag__:_pod_ip_", Value: "172.10.0.45"}, 238 {Key: "__log_topic__", Value: "file"}, 239 }, 240 } 241 } 242 tags := []*protocol.LogTag{ 243 {Key: "__hostname__", Value: "alje834hgf"}, 244 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 245 } 246 logGroup := &protocol.LogGroup{ 247 Logs: logs, 248 Category: "test", 249 Topic: "file", 250 Source: "172.10.0.56", 251 LogTags: tags, 252 } 253 254 Convey("Then the converted log should be valid", func() { 255 b, err := c.ToByteStream(logGroup) 256 So(err, ShouldBeNil) 257 258 for _, s := range b.([][]byte) { 259 unmarshaledLog := make(map[string]interface{}) 260 err = json.Unmarshal(s, &unmarshaledLog) 261 So(err, ShouldBeNil) 262 So(unmarshaledLog, ShouldHaveLength, 3) 263 So(unmarshaledLog, ShouldContainKey, "time") 264 So(unmarshaledLog, ShouldContainKey, "contents") 265 So(unmarshaledLog, ShouldContainKey, "tags") 266 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 267 So(unmarshaledLog["contents"], ShouldContainKey, "method") 268 So(unmarshaledLog["contents"], ShouldContainKey, "status") 269 So(unmarshaledLog["tags"], ShouldHaveLength, 9) 270 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 271 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 272 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 273 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 274 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.ip") 275 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 276 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 277 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 278 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.ip") 279 } 280 }) 281 }) 282 283 Convey("When the logGroup is generated from stdout and from docker environment", func() { 284 *flags.K8sFlag = false 285 time := []uint32{1662434209, 1662434487} 286 method := []string{"PUT", "GET"} 287 status := []string{"200", "404"} 288 logs := make([]*protocol.Log, 2) 289 for i := 0; i < 2; i++ { 290 logs[i] = &protocol.Log{ 291 Time: time[i], 292 Contents: []*protocol.Log_Content{ 293 {Key: "method", Value: method[i]}, 294 {Key: "status", Value: status[i]}, 295 {Key: "_container_name_", Value: "container"}, 296 {Key: "_container_ip_", Value: "172.10.0.45"}, 297 {Key: "_image_name_", Value: "image"}, 298 }, 299 } 300 } 301 tags := []*protocol.LogTag{ 302 {Key: "__hostname__", Value: "alje834hgf"}, 303 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 304 } 305 logGroup := &protocol.LogGroup{ 306 Logs: logs, 307 Category: "test", 308 Topic: "", 309 Source: "172.10.0.56", 310 LogTags: tags, 311 } 312 313 Convey("Then the converted log should be valid", func() { 314 b, err := c.ToByteStream(logGroup) 315 So(err, ShouldBeNil) 316 317 for _, s := range b.([][]byte) { 318 unmarshaledLog := make(map[string]interface{}) 319 err = json.Unmarshal(s, &unmarshaledLog) 320 So(err, ShouldBeNil) 321 So(unmarshaledLog, ShouldHaveLength, 3) 322 So(unmarshaledLog, ShouldContainKey, "time") 323 So(unmarshaledLog, ShouldContainKey, "contents") 324 So(unmarshaledLog, ShouldContainKey, "tags") 325 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 326 So(unmarshaledLog["contents"], ShouldContainKey, "method") 327 So(unmarshaledLog["contents"], ShouldContainKey, "status") 328 So(unmarshaledLog["tags"], ShouldHaveLength, 5) 329 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 330 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 331 So(unmarshaledLog["tags"], ShouldContainKey, "container.name") 332 So(unmarshaledLog["tags"], ShouldContainKey, "container.ip") 333 So(unmarshaledLog["tags"], ShouldContainKey, "container.image.name") 334 } 335 }) 336 }) 337 338 Convey("When the logGroup is generated from stdout and from k8s daemonset environment", func() { 339 *flags.K8sFlag = true 340 time := []uint32{1662434209, 1662434487} 341 method := []string{"PUT", "GET"} 342 status := []string{"200", "404"} 343 logs := make([]*protocol.Log, 2) 344 for i := 0; i < 2; i++ { 345 logs[i] = &protocol.Log{ 346 Time: time[i], 347 Contents: []*protocol.Log_Content{ 348 {Key: "method", Value: method[i]}, 349 {Key: "status", Value: status[i]}, 350 {Key: "__tag__:_node_name_", Value: "node"}, 351 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 352 {Key: "_namespace_", Value: "default"}, 353 {Key: "_pod_name_", Value: "container"}, 354 {Key: "_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 355 {Key: "_container_name_", Value: "container"}, 356 {Key: "_container_ip_", Value: "172.10.0.45"}, 357 {Key: "_image_name_", Value: "image"}, 358 {Key: "label", Value: "tag"}, 359 }, 360 } 361 } 362 tags := []*protocol.LogTag{ 363 {Key: "__hostname__", Value: "alje834hgf"}, 364 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 365 } 366 logGroup := &protocol.LogGroup{ 367 Logs: logs, 368 Category: "test", 369 Topic: "", 370 Source: "172.10.0.56", 371 LogTags: tags, 372 } 373 374 Convey("Then the converted log should be valid", func() { 375 b, err := c.ToByteStream(logGroup) 376 So(err, ShouldBeNil) 377 378 for _, s := range b.([][]byte) { 379 unmarshaledLog := make(map[string]interface{}) 380 err = json.Unmarshal(s, &unmarshaledLog) 381 So(err, ShouldBeNil) 382 So(unmarshaledLog, ShouldHaveLength, 3) 383 So(unmarshaledLog, ShouldContainKey, "time") 384 So(unmarshaledLog, ShouldContainKey, "contents") 385 So(unmarshaledLog, ShouldContainKey, "tags") 386 So(unmarshaledLog["contents"], ShouldHaveLength, 3) 387 So(unmarshaledLog["contents"], ShouldContainKey, "method") 388 So(unmarshaledLog["contents"], ShouldContainKey, "status") 389 So(unmarshaledLog["contents"], ShouldContainKey, "label") 390 So(unmarshaledLog["tags"], ShouldHaveLength, 10) 391 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 392 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 393 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.ip") 394 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 395 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 396 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 397 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.uid") 398 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.name") 399 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.ip") 400 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.image.name") 401 } 402 }) 403 }) 404 405 Convey("When the topic is null but __log_topic__ is not", func() { 406 *flags.K8sFlag = false 407 time := []uint32{1662434209, 1662434487} 408 method := []string{"PUT", "GET"} 409 status := []string{"200", "404"} 410 logs := make([]*protocol.Log, 2) 411 for i := 0; i < 2; i++ { 412 logs[i] = &protocol.Log{ 413 Time: time[i], 414 Contents: []*protocol.Log_Content{ 415 {Key: "method", Value: method[i]}, 416 {Key: "status", Value: status[i]}, 417 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 418 {Key: "__log_topic__", Value: "file"}, 419 }, 420 } 421 } 422 tags := []*protocol.LogTag{ 423 {Key: "__hostname__", Value: "alje834hgf"}, 424 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 425 } 426 logGroup := &protocol.LogGroup{ 427 Logs: logs, 428 Category: "test", 429 Topic: "", 430 Source: "172.10.0.56", 431 LogTags: tags, 432 } 433 434 Convey("Then the converted log should be valid", func() { 435 b, err := c.ToByteStream(logGroup) 436 So(err, ShouldBeNil) 437 438 for _, s := range b.([][]byte) { 439 unmarshaledLog := make(map[string]interface{}) 440 err = json.Unmarshal(s, &unmarshaledLog) 441 So(err, ShouldBeNil) 442 So(unmarshaledLog, ShouldHaveLength, 3) 443 So(unmarshaledLog, ShouldContainKey, "time") 444 So(unmarshaledLog, ShouldContainKey, "contents") 445 So(unmarshaledLog, ShouldContainKey, "tags") 446 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 447 So(unmarshaledLog["contents"], ShouldContainKey, "method") 448 So(unmarshaledLog["contents"], ShouldContainKey, "status") 449 So(unmarshaledLog["tags"], ShouldHaveLength, 4) 450 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 451 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 452 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 453 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 454 } 455 }) 456 }) 457 458 Convey("When the topic and __log_topic__ are null", func() { 459 *flags.K8sFlag = false 460 time := []uint32{1662434209, 1662434487} 461 method := []string{"PUT", "GET"} 462 status := []string{"200", "404"} 463 logs := make([]*protocol.Log, 2) 464 for i := 0; i < 2; i++ { 465 logs[i] = &protocol.Log{ 466 Time: time[i], 467 Contents: []*protocol.Log_Content{ 468 {Key: "method", Value: method[i]}, 469 {Key: "status", Value: status[i]}, 470 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 471 }, 472 } 473 } 474 tags := []*protocol.LogTag{ 475 {Key: "__hostname__", Value: "alje834hgf"}, 476 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 477 } 478 logGroup := &protocol.LogGroup{ 479 Logs: logs, 480 Category: "test", 481 Topic: "", 482 Source: "172.10.0.56", 483 LogTags: tags, 484 } 485 486 Convey("Then the converted log should be valid", func() { 487 b, err := c.ToByteStream(logGroup) 488 So(err, ShouldBeNil) 489 490 for _, s := range b.([][]byte) { 491 unmarshaledLog := make(map[string]interface{}) 492 err = json.Unmarshal(s, &unmarshaledLog) 493 So(err, ShouldBeNil) 494 So(unmarshaledLog, ShouldHaveLength, 3) 495 So(unmarshaledLog, ShouldContainKey, "time") 496 So(unmarshaledLog, ShouldContainKey, "contents") 497 So(unmarshaledLog, ShouldContainKey, "tags") 498 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 499 So(unmarshaledLog["contents"], ShouldContainKey, "method") 500 So(unmarshaledLog["contents"], ShouldContainKey, "status") 501 So(unmarshaledLog["tags"], ShouldHaveLength, 3) 502 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 503 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 504 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 505 So(unmarshaledLog["tags"], ShouldNotContainKey, "log.topic") 506 } 507 }) 508 }) 509 510 Convey("When the log is standardized", func() { 511 *flags.K8sFlag = true 512 time := []uint32{1662434209, 1662434487} 513 method := []string{"PUT", "GET"} 514 status := []string{"200", "404"} 515 logs := make([]*protocol.Log, 2) 516 for i := 0; i < 2; i++ { 517 logs[i] = &protocol.Log{ 518 Time: time[i], 519 Contents: []*protocol.Log_Content{ 520 {Key: "method", Value: method[i]}, 521 {Key: "status", Value: status[i]}, 522 }, 523 } 524 } 525 tags := []*protocol.LogTag{ 526 {Key: "__user_defined_id__", Value: "machine"}, 527 {Key: "__hostname__", Value: "alje834hgf"}, 528 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 529 {Key: "__path__", Value: "/root/test/origin/example.log"}, 530 {Key: "_node_name_", Value: "node"}, 531 {Key: "_node_ip_", Value: "172.10.1.19"}, 532 {Key: "_namespace_", Value: "default"}, 533 {Key: "_pod_name_", Value: "container"}, 534 {Key: "_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 535 {Key: "_container_name_", Value: "container"}, 536 {Key: "_container_ip_", Value: "172.10.0.45"}, 537 {Key: "_image_name_", Value: "image"}, 538 {Key: "label", Value: "tag"}, 539 } 540 logGroup := &protocol.LogGroup{ 541 Logs: logs, 542 Category: "test", 543 Topic: "topic", 544 Source: "172.10.0.56", 545 LogTags: tags, 546 } 547 548 Convey("Then the converted log should be valid", func() { 549 b, err := c.ToByteStream(logGroup) 550 So(err, ShouldBeNil) 551 552 for _, s := range b.([][]byte) { 553 unmarshaledLog := make(map[string]interface{}) 554 err = json.Unmarshal(s, &unmarshaledLog) 555 So(err, ShouldBeNil) 556 So(unmarshaledLog, ShouldHaveLength, 3) 557 So(unmarshaledLog, ShouldContainKey, "time") 558 So(unmarshaledLog, ShouldContainKey, "contents") 559 So(unmarshaledLog, ShouldContainKey, "tags") 560 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 561 So(unmarshaledLog["contents"], ShouldContainKey, "method") 562 So(unmarshaledLog["contents"], ShouldContainKey, "status") 563 So(unmarshaledLog["tags"], ShouldHaveLength, 13) 564 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 565 So(unmarshaledLog["tags"], ShouldContainKey, "host.name") 566 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 567 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 568 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.ip") 569 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 570 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 571 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 572 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.uid") 573 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.name") 574 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.ip") 575 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.image.name") 576 So(unmarshaledLog["tags"], ShouldContainKey, "label") 577 } 578 }) 579 }) 580 }) 581 582 Convey("Given a converter with protocol: single, encoding: json, with tag rename and protocol key rename", t, func() { 583 keyRenameMap := map[string]string{ 584 "k8s.node.ip": "ip", 585 "host.name": "hostname", 586 "label": "tag", 587 "env": "env_tag", 588 } 589 protocolKeyRenameMap := map[string]string{ 590 "time": "@timestamp", 591 "contents": "values", 592 "tags": "annos", 593 } 594 c, err := NewConverter("custom_single", "json", keyRenameMap, protocolKeyRenameMap, &config.GlobalConfig{}) 595 So(err, ShouldBeNil) 596 597 Convey("When the logGroup is generated from files and from k8s daemonset environment", func() { 598 *flags.K8sFlag = true 599 time := []uint32{1662434209, 1662434487} 600 method := []string{"PUT", "GET"} 601 status := []string{"200", "404"} 602 logs := make([]*protocol.Log, 2) 603 for i := 0; i < 2; i++ { 604 logs[i] = &protocol.Log{ 605 Time: time[i], 606 Contents: []*protocol.Log_Content{ 607 {Key: "method", Value: method[i]}, 608 {Key: "status", Value: status[i]}, 609 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 610 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 611 {Key: "__tag__:_node_name_", Value: "node"}, 612 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 613 {Key: "__tag__:_namespace_", Value: "default"}, 614 {Key: "__tag__:_pod_name_", Value: "container"}, 615 {Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 616 {Key: "__tag__:_container_name_", Value: "container"}, 617 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 618 {Key: "__tag__:_image_name_", Value: "image"}, 619 {Key: "__tag__:label", Value: "tag"}, 620 {Key: "__log_topic__", Value: "file"}, 621 }, 622 } 623 } 624 tags := []*protocol.LogTag{ 625 {Key: "__hostname__", Value: "alje834hgf"}, 626 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 627 {Key: "env", Value: "K8S"}, 628 } 629 logGroup := &protocol.LogGroup{ 630 Logs: logs, 631 Category: "test", 632 Topic: "file", 633 Source: "172.10.0.56", 634 LogTags: tags, 635 } 636 637 Convey("Then the converted log should be valid", func() { 638 b, err := c.ToByteStream(logGroup) 639 So(err, ShouldBeNil) 640 641 for _, s := range b.([][]byte) { 642 unmarshaledLog := make(map[string]interface{}) 643 err = json.Unmarshal(s, &unmarshaledLog) 644 So(err, ShouldBeNil) 645 So(unmarshaledLog, ShouldHaveLength, 3) 646 So(unmarshaledLog, ShouldContainKey, "@timestamp") 647 So(unmarshaledLog, ShouldContainKey, "values") 648 So(unmarshaledLog, ShouldContainKey, "annos") 649 So(unmarshaledLog["values"], ShouldHaveLength, 2) 650 So(unmarshaledLog["values"], ShouldContainKey, "method") 651 So(unmarshaledLog["values"], ShouldContainKey, "status") 652 So(unmarshaledLog["annos"], ShouldHaveLength, 14) 653 So(unmarshaledLog["annos"], ShouldContainKey, "log.file.path") 654 So(unmarshaledLog["annos"], ShouldContainKey, "hostname") 655 So(unmarshaledLog["annos"], ShouldContainKey, "host.ip") 656 So(unmarshaledLog["annos"], ShouldContainKey, "log.topic") 657 So(unmarshaledLog["annos"], ShouldContainKey, "ip") 658 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.node.name") 659 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.namespace.name") 660 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.pod.name") 661 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.pod.uid") 662 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.container.name") 663 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.container.ip") 664 So(unmarshaledLog["annos"], ShouldContainKey, "k8s.container.image.name") 665 So(unmarshaledLog["annos"], ShouldContainKey, "tag") 666 So(unmarshaledLog["annos"], ShouldContainKey, "env_tag") 667 } 668 }) 669 670 Convey("Then the corresponding value of the required fields are returned correctly", func() { 671 _, values, err := c.ToByteStreamWithSelectedFields(logGroup, []string{"attribute.method", "tag.host.name", "tag.ip", "attribute.unknown"}) 672 So(err, ShouldBeNil) 673 So(values, ShouldHaveLength, 2) 674 So(values[0], ShouldHaveLength, 3) 675 So(values[0]["attribute.method"], ShouldEqual, "PUT") 676 So(values[0]["tag.host.name"], ShouldEqual, "alje834hgf") 677 So(values[0]["tag.ip"], ShouldEqual, "172.10.1.19") 678 So(values[1], ShouldHaveLength, 3) 679 So(values[1]["attribute.method"], ShouldEqual, "GET") 680 So(values[1]["tag.host.name"], ShouldEqual, "alje834hgf") 681 So(values[1]["tag.ip"], ShouldEqual, "172.10.1.19") 682 }) 683 684 Convey("Then error should be returned given invalid target field", func() { 685 _, _, err := c.ToByteStreamWithSelectedFields(logGroup, []string{"values.method"}) 686 So(err, ShouldNotBeNil) 687 }) 688 }) 689 }) 690 691 Convey("Given a converter with protocol: single, encoding: json, with null tag rename", t, func() { 692 keyRenameMap := map[string]string{ 693 "k8s.node.ip": "", 694 "host.name": "", 695 "label": "", 696 "env": "", 697 } 698 c, err := NewConverter("custom_single", "json", keyRenameMap, nil, &config.GlobalConfig{}) 699 So(err, ShouldBeNil) 700 701 Convey("When the logGroup is generated from files and from k8s daemonset environment", func() { 702 *flags.K8sFlag = true 703 time := []uint32{1662434209, 1662434487} 704 method := []string{"PUT", "GET"} 705 status := []string{"200", "404"} 706 logs := make([]*protocol.Log, 2) 707 for i := 0; i < 2; i++ { 708 logs[i] = &protocol.Log{ 709 Time: time[i], 710 Contents: []*protocol.Log_Content{ 711 {Key: "method", Value: method[i]}, 712 {Key: "status", Value: status[i]}, 713 {Key: "__tag__:__user_defined_id__", Value: "machine"}, 714 {Key: "__tag__:__path__", Value: "/root/test/origin/example.log"}, 715 {Key: "__tag__:_node_name_", Value: "node"}, 716 {Key: "__tag__:_node_ip_", Value: "172.10.1.19"}, 717 {Key: "__tag__:_namespace_", Value: "default"}, 718 {Key: "__tag__:_pod_name_", Value: "container"}, 719 {Key: "__tag__:_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 720 {Key: "__tag__:_container_name_", Value: "container"}, 721 {Key: "__tag__:_container_ip_", Value: "172.10.0.45"}, 722 {Key: "__tag__:_image_name_", Value: "image"}, 723 {Key: "__tag__:label", Value: "tag"}, 724 {Key: "__log_topic__", Value: "file"}, 725 }, 726 } 727 } 728 tags := []*protocol.LogTag{ 729 {Key: "__hostname__", Value: "alje834hgf"}, 730 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 731 {Key: "env", Value: "K8S"}, 732 } 733 logGroup := &protocol.LogGroup{ 734 Logs: logs, 735 Category: "test", 736 Topic: "file", 737 Source: "172.10.0.56", 738 LogTags: tags, 739 } 740 741 Convey("Then the converted log should be valid", func() { 742 b, err := c.ToByteStream(logGroup) 743 So(err, ShouldBeNil) 744 745 for _, s := range b.([][]byte) { 746 unmarshaledLog := make(map[string]interface{}) 747 err = json.Unmarshal(s, &unmarshaledLog) 748 So(err, ShouldBeNil) 749 So(unmarshaledLog, ShouldHaveLength, 3) 750 So(unmarshaledLog, ShouldContainKey, "time") 751 So(unmarshaledLog, ShouldContainKey, "contents") 752 So(unmarshaledLog, ShouldContainKey, "tags") 753 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 754 So(unmarshaledLog["contents"], ShouldContainKey, "method") 755 So(unmarshaledLog["contents"], ShouldContainKey, "status") 756 So(unmarshaledLog["tags"], ShouldHaveLength, 10) 757 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 758 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 759 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 760 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 761 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 762 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 763 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.uid") 764 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.name") 765 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.ip") 766 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.image.name") 767 } 768 }) 769 }) 770 771 Convey("When the log is standardized", func() { 772 *flags.K8sFlag = true 773 time := []uint32{1662434209, 1662434487} 774 method := []string{"PUT", "GET"} 775 status := []string{"200", "404"} 776 logs := make([]*protocol.Log, 2) 777 for i := 0; i < 2; i++ { 778 logs[i] = &protocol.Log{ 779 Time: time[i], 780 Contents: []*protocol.Log_Content{ 781 {Key: "method", Value: method[i]}, 782 {Key: "status", Value: status[i]}, 783 }, 784 } 785 } 786 tags := []*protocol.LogTag{ 787 {Key: "__user_defined_id__", Value: "machine"}, 788 {Key: "__hostname__", Value: "alje834hgf"}, 789 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 790 {Key: "__path__", Value: "/root/test/origin/example.log"}, 791 {Key: "_node_name_", Value: "node"}, 792 {Key: "_node_ip_", Value: "172.10.1.19"}, 793 {Key: "_namespace_", Value: "default"}, 794 {Key: "_pod_name_", Value: "container"}, 795 {Key: "_pod_uid_", Value: "12AFERR234SG-SBH6D67HJ9-AAD-VF34"}, 796 {Key: "_container_name_", Value: "container"}, 797 {Key: "_container_ip_", Value: "172.10.0.45"}, 798 {Key: "_image_name_", Value: "image"}, 799 {Key: "label", Value: "tag"}, 800 } 801 logGroup := &protocol.LogGroup{ 802 Logs: logs, 803 Category: "test", 804 Topic: "topic", 805 Source: "172.10.0.56", 806 LogTags: tags, 807 } 808 809 Convey("Then the converted log should be valid", func() { 810 b, err := c.ToByteStream(logGroup) 811 So(err, ShouldBeNil) 812 813 for _, s := range b.([][]byte) { 814 unmarshaledLog := make(map[string]interface{}) 815 err = json.Unmarshal(s, &unmarshaledLog) 816 So(err, ShouldBeNil) 817 So(unmarshaledLog, ShouldHaveLength, 3) 818 So(unmarshaledLog, ShouldContainKey, "time") 819 So(unmarshaledLog, ShouldContainKey, "contents") 820 So(unmarshaledLog, ShouldContainKey, "tags") 821 So(unmarshaledLog["contents"], ShouldHaveLength, 2) 822 So(unmarshaledLog["contents"], ShouldContainKey, "method") 823 So(unmarshaledLog["contents"], ShouldContainKey, "status") 824 So(unmarshaledLog["tags"], ShouldHaveLength, 10) 825 So(unmarshaledLog["tags"], ShouldContainKey, "log.file.path") 826 So(unmarshaledLog["tags"], ShouldContainKey, "host.ip") 827 So(unmarshaledLog["tags"], ShouldContainKey, "log.topic") 828 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.node.name") 829 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.namespace.name") 830 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.name") 831 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.pod.uid") 832 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.name") 833 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.ip") 834 So(unmarshaledLog["tags"], ShouldContainKey, "k8s.container.image.name") 835 } 836 }) 837 }) 838 }) 839 840 Convey("When constructing converter with unsupported encoding", t, func() { 841 _, err := NewConverter("custom_single", "pb", nil, nil, &config.GlobalConfig{}) 842 843 Convey("Then error should be returned", func() { 844 So(err, ShouldNotBeNil) 845 }) 846 }) 847 848 Convey("Given a converter with unsupported encoding", t, func() { 849 *flags.K8sFlag = false 850 c := &Converter{ 851 Protocol: "custom_single", 852 Encoding: "pb", 853 } 854 855 Convey("When performing conversion", func() { 856 logs := []*protocol.Log{ 857 { 858 Time: 1662434209, 859 Contents: []*protocol.Log_Content{ 860 {Key: "method", Value: "PUT"}, 861 }, 862 }, 863 } 864 tags := []*protocol.LogTag{ 865 {Key: "__hostname__", Value: "alje834hgf"}, 866 {Key: "__pack_id__", Value: "AEDCFGHNJUIOPLMN-1E"}, 867 {Key: "__path__", Value: "/root/test/origin/example.log"}, 868 {Key: "__log_topic__", Value: "file"}, 869 } 870 logGroup := &protocol.LogGroup{ 871 Logs: logs, 872 Category: "test", 873 Topic: "topic", 874 Source: "172.10.0.56", 875 LogTags: tags, 876 } 877 878 _, err := c.ToByteStream(logGroup) 879 880 Convey("Then error should be returned", func() { 881 So(err, ShouldNotBeNil) 882 }) 883 }) 884 }) 885 } 886 887 func TestJsonMarshalAndMarshalWithoutHTMLEscaped(t *testing.T) { 888 type TestData struct { 889 ID int 890 Msg string 891 Data interface{} 892 } 893 894 Convey("test json marshal and marchal without html escaped", t, func() { 895 data := TestData{ 896 ID: 0, 897 Msg: ">>>><<<)(*&^%$#@!$@#hello+1447138058167839254", 898 Data: nil, 899 } 900 901 Convey("test marshalWithoutHTMLEscaped", func() { 902 v, _ := marshalWithoutHTMLEscaped(data) 903 904 str := string(v) 905 fmt.Println(str) 906 907 So(str, ShouldEqual, `{"ID":0,"Msg":">>>><<<)(*&^%$#@!$@#hello+1447138058167839254","Data":null}`) 908 }) 909 910 Convey("test json marshal", func() { 911 v, _ := json.Marshal(data) 912 913 str := string(v) 914 fmt.Println(str) 915 916 So(str, ShouldNotEqual, `{"ID":0,"Msg":">>>><<<)(*&^%$#@!$@#hello+1447138058167839254","Data":null}`) 917 }) 918 }) 919 }