github.com/alibaba/ilogtail/pkg@v0.0.0-20250526110833-c53b480d046c/protocol/converter/custom_single_log.go (about)

     1  // Copyright 2022 iLogtail Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package protocol
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/json"
    20  	"fmt"
    21  
    22  	"github.com/alibaba/ilogtail/pkg/protocol"
    23  )
    24  
    25  const (
    26  	protocolKeyTime    = "time"
    27  	protocolKeyContent = "contents"
    28  	protocolKeyTag     = "tags"
    29  	numProtocolKeys    = 3
    30  )
    31  
    32  func (c *Converter) ConvertToSingleProtocolLogs(logGroup *protocol.LogGroup, targetFields []string) ([]map[string]interface{}, []map[string]string, error) {
    33  	convertedLogs, desiredValues := make([]map[string]interface{}, len(logGroup.Logs)), make([]map[string]string, len(logGroup.Logs))
    34  	for i, log := range logGroup.Logs {
    35  		contents, tags := convertLogToMap(log, logGroup.LogTags, logGroup.Source, logGroup.Topic, c.TagKeyRenameMap)
    36  
    37  		desiredValue, err := findTargetValues(targetFields, contents, tags, c.TagKeyRenameMap)
    38  		if err != nil {
    39  			return nil, nil, err
    40  		}
    41  		desiredValues[i] = desiredValue
    42  
    43  		customSingleLog := make(map[string]interface{}, numProtocolKeys)
    44  		if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
    45  			customSingleLog[newKey] = log.Time
    46  		} else {
    47  			customSingleLog[protocolKeyTime] = log.Time
    48  		}
    49  		if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyContent]; ok {
    50  			customSingleLog[newKey] = contents
    51  		} else {
    52  			customSingleLog[protocolKeyContent] = contents
    53  		}
    54  		if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTag]; ok {
    55  			customSingleLog[newKey] = tags
    56  		} else {
    57  			customSingleLog[protocolKeyTag] = tags
    58  		}
    59  		convertedLogs[i] = customSingleLog
    60  	}
    61  	return convertedLogs, desiredValues, nil
    62  }
    63  
    64  func (c *Converter) ConvertToSingleProtocolStream(logGroup *protocol.LogGroup, targetFields []string) ([][]byte, []map[string]string, error) {
    65  	convertedLogs, desiredValues, err := c.ConvertToSingleProtocolLogs(logGroup, targetFields)
    66  	if err != nil {
    67  		return nil, nil, err
    68  	}
    69  
    70  	marshaledLogs := make([][]byte, len(logGroup.Logs))
    71  	for i, log := range convertedLogs {
    72  		switch c.Encoding {
    73  		case EncodingJSON:
    74  			b, err := marshalWithoutHTMLEscaped(log)
    75  			if err != nil {
    76  				return nil, nil, fmt.Errorf("unable to marshal log: %v", log)
    77  			}
    78  			marshaledLogs[i] = b
    79  		default:
    80  			return nil, nil, fmt.Errorf("unsupported encoding format: %s", c.Encoding)
    81  		}
    82  	}
    83  	return marshaledLogs, desiredValues, nil
    84  }
    85  
    86  func marshalWithoutHTMLEscaped(data interface{}) ([]byte, error) {
    87  	bf := bytes.NewBuffer([]byte{})
    88  	jsonEncoder := json.NewEncoder(bf)
    89  	jsonEncoder.SetEscapeHTML(false)
    90  	if err := jsonEncoder.Encode(data); err != nil {
    91  		return nil, err
    92  	}
    93  	b := bytes.TrimRight(bf.Bytes(), "\n")
    94  	return b, nil
    95  }