github.com/alibaba/ilogtail/pkg@v0.0.0-20250526110833-c53b480d046c/protocol/converter/converter_single_log_flatten.go (about)

     1  // Copyright 2023 iLogtail Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package protocol
    16  
    17  import (
    18  	"fmt"
    19  
    20  	"github.com/alibaba/ilogtail/pkg/protocol"
    21  )
    22  
    23  func (c *Converter) ConvertToSingleProtocolLogsFlatten(logGroup *protocol.LogGroup, targetFields []string) ([]map[string]interface{}, []map[string]string, error) {
    24  	convertedLogs, desiredValues := make([]map[string]interface{}, len(logGroup.Logs)), make([]map[string]string, len(logGroup.Logs))
    25  	for i, log := range logGroup.Logs {
    26  		contents, tags := convertLogToMap(log, logGroup.LogTags, logGroup.Source, logGroup.Topic, c.TagKeyRenameMap)
    27  
    28  		desiredValue, err := findTargetValues(targetFields, contents, tags, c.TagKeyRenameMap)
    29  		if err != nil {
    30  			return nil, nil, err
    31  		}
    32  		desiredValues[i] = desiredValue
    33  
    34  		logLength := 1 + len(contents)
    35  		if !c.OnlyContents {
    36  			logLength += len(tags)
    37  		}
    38  
    39  		customSingleLog := make(map[string]interface{}, logLength)
    40  		// merge contents to final logs
    41  		for k, v := range contents {
    42  			customSingleLog[k] = v
    43  		}
    44  		if !c.OnlyContents {
    45  			// merge tags to final logs
    46  			for k, v := range tags {
    47  				customSingleLog[k] = v
    48  			}
    49  		}
    50  
    51  		if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
    52  			customSingleLog[newKey] = log.Time
    53  		} else {
    54  			customSingleLog[protocolKeyTime] = log.Time
    55  		}
    56  
    57  		convertedLogs[i] = customSingleLog
    58  	}
    59  	return convertedLogs, desiredValues, nil
    60  }
    61  
    62  func (c *Converter) ConvertToSingleProtocolStreamFlatten(logGroup *protocol.LogGroup, targetFields []string) ([][]byte, []map[string]string, error) {
    63  	convertedLogs, desiredValues, err := c.ConvertToSingleProtocolLogsFlatten(logGroup, targetFields)
    64  	if err != nil {
    65  		return nil, nil, err
    66  	}
    67  	marshaledLogs := make([][]byte, len(logGroup.Logs))
    68  	for i, log := range convertedLogs {
    69  		switch c.Encoding {
    70  		case EncodingJSON:
    71  			b, err := marshalWithoutHTMLEscaped(log)
    72  			if err != nil {
    73  				return nil, nil, fmt.Errorf("unable to marshal log: %v", log)
    74  			}
    75  			marshaledLogs[i] = b
    76  		default:
    77  			return nil, nil, fmt.Errorf("unsupported encoding format: %s", c.Encoding)
    78  		}
    79  	}
    80  	return marshaledLogs, desiredValues, nil
    81  }