github.com/ystia/yorc/v4@v4.3.0/storage/internal/elastic/utils.go (about)

     1  // Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package elastic
    16  
    17  import (
    18  	"encoding/json"
    19  	"github.com/pkg/errors"
    20  	"github.com/ystia/yorc/v4/log"
    21  	"github.com/ystia/yorc/v4/storage/store"
    22  	"github.com/ystia/yorc/v4/storage/utils"
    23  	"regexp"
    24  	"strconv"
    25  	"strings"
    26  	"time"
    27  )
    28  
    29  // Precompiled regex to extract storeType and timestamp from a key of the form: "_yorc/logs/MyApp/2020-06-07T21:03:17.812178429Z".
    30  var storeTypeAndTimestampRegex = regexp.MustCompile(`(?m)\_yorc\/(\w+)\/.+\/(.*)`)
    31  
    32  // Parse a key of form  "_yorc/logs/MyApp/2020-06-07T21:03:17.812178429Z" to get the store type (logs|events) and the timestamp.
    33  func extractStoreTypeAndTimestamp(k string) (storeType string, timestamp string) {
    34  	res := storeTypeAndTimestampRegex.FindAllStringSubmatch(k, -1)
    35  	for i := range res {
    36  		storeType = res[i][1]
    37  		timestamp = res[i][2]
    38  	}
    39  	return storeType, timestamp
    40  }
    41  
    42  // Precompiled regex to extract storeType and deploymentId from a key of the form: "_yorc/events/" or "_yorc/logs/MyApp" or "_yorc/logs/MyApp/".
    43  var storeTypeAndDeploymentIDRegex = regexp.MustCompile(`(?m)\_yorc\/(\w+)\/?(.+)?\/?`)
    44  
    45  // Parse a key of form "_yorc/events/" or "_yorc/logs/MyApp" or "_yorc/logs/MyApp/" to get the store type (logs|events) and eventually the deploymentId.
    46  func extractStoreTypeAndDeploymentID(k string) (storeType string, deploymentID string) {
    47  	res := storeTypeAndDeploymentIDRegex.FindAllStringSubmatch(k, -1)
    48  	for i := range res {
    49  		storeType = res[i][1]
    50  		if len(res[i]) == 3 {
    51  			deploymentID = res[i][2]
    52  			if strings.HasSuffix(deploymentID, "/") {
    53  				deploymentID = deploymentID[:len(deploymentID)-1]
    54  			}
    55  		}
    56  	}
    57  	return storeType, deploymentID
    58  }
    59  
    60  // We need to append JSON directly into []byte to avoid useless and costly marshaling / unmarshaling.
    61  func appendJSONInBytes(a []byte, v []byte) []byte {
    62  	last := len(a) - 1
    63  	lastByte := a[last]
    64  	// just append v at the end
    65  	a = append(a, v...)
    66  	// then slice
    67  	for i, s := range v {
    68  		a[last+i] = s
    69  	}
    70  	a[last+len(v)] = lastByte
    71  	return a
    72  }
    73  
    74  func _parseInt64StringToInt64(value string) int64 {
    75  	valueInt, err := strconv.ParseInt(value, 10, 64)
    76  	if err != nil {
    77  		log.Printf("Fail parsing _parseInt64StringToInt64 %s", value)
    78  	}
    79  	return valueInt
    80  }
    81  
    82  func parseInt64StringToUint64(value string) (uint64, error) {
    83  	valueInt, err := strconv.ParseInt(value, 10, 64)
    84  	if err != nil {
    85  		return 0, errors.Wrapf(err, "Not able to parse string: %s to int64, error was: %+v", value, err)
    86  	}
    87  	result := uint64(valueInt)
    88  	return result, nil
    89  }
    90  
    91  func _getTimestampFromUint64(nanoTimestamp uint64) time.Time {
    92  	nanoTimestampStr := strconv.FormatUint(nanoTimestamp, 10)
    93  	ts := _parseInt64StringToInt64(nanoTimestampStr)
    94  	return time.Unix(0, ts)
    95  }
    96  
    97  // The max uint is 9223372036854775807 (19 cars), this is the maximum nano for a time (2262-04-12 00:47:16.854775807 +0100 CET).
    98  // Since we use nanotimestamp as ID for events and logs, and since we store this ID as string in ES index, we must ensure that
    99  // the string will be comparable.
   100  func getSortableStringFromUint64(nanoTimestamp uint64) string {
   101  	nanoTimestampStr := strconv.FormatUint(nanoTimestamp, 10)
   102  	if len(nanoTimestampStr) < 19 {
   103  		nanoTimestampStr = strings.Repeat("0", 19-len(nanoTimestampStr)) + nanoTimestampStr
   104  	}
   105  	return nanoTimestampStr
   106  }
   107  
   108  // The document is enriched by adding 'clusterId' and 'iid' properties.
   109  // This addition is done by directly manipulating the []byte in order to avoid costly successive marshal / unmarshal operations.
   110  func buildElasticDocument(k string, rawMessage interface{}) (string, []byte, error) {
   111  	// Extract indice name and timestamp by parsing the key
   112  	storeType, timestamp := extractStoreTypeAndTimestamp(k)
   113  	log.Debugf("storeType is: %s, timestamp: %s", storeType, timestamp)
   114  
   115  	// Convert timestamp to an int64
   116  	eventDate, err := time.Parse(time.RFC3339Nano, timestamp)
   117  	if err != nil {
   118  		return storeType, nil, errors.Wrapf(err, "failed to parse timestamp %+v as time, error was: %+v", timestamp, err)
   119  	}
   120  	// Convert to UnixNano int64
   121  	iid := eventDate.UnixNano()
   122  
   123  	// This is the piece of 'JSON' we want to append
   124  	a := `,"iid":"` + strconv.FormatInt(iid, 10) + `","iidStr":"` + strconv.FormatInt(iid, 10) + `"`
   125  	// v is a json.RawMessage
   126  	raw := rawMessage.(json.RawMessage)
   127  	raw = appendJSONInBytes(raw, []byte(a))
   128  
   129  	return storeType, raw, nil
   130  }
   131  
   132  // An error is returned if :
   133  // - it's not valid (key or value nil)
   134  // - the size of the resulting bulk operation exceed the maximum authorized for a bulk request
   135  // The value is not added if it's size + the current body size exceed the maximum authorized for a bulk request.
   136  // Return a bool indicating if the value has been added to the bulk request body.
   137  func eventuallyAppendValueToBulkRequest(c elasticStoreConf, body *[]byte, kv store.KeyValueIn, maxBulkSizeInBytes int) (bool, error) {
   138  	if err := utils.CheckKeyAndValue(kv.Key, kv.Value); err != nil {
   139  		return false, err
   140  	}
   141  
   142  	storeType, document, err := buildElasticDocument(kv.Key, kv.Value)
   143  	if err != nil {
   144  		return false, err
   145  	}
   146  	log.Debugf("About to add a document of size %d bytes to bulk request", len(document))
   147  
   148  	// The bulk action
   149  	index := `{"index":{"_index":"` + getIndexName(c, storeType) + `","_type":"_doc"}}`
   150  	bulkOperation := make([]byte, 0)
   151  	bulkOperation = append(bulkOperation, index...)
   152  	bulkOperation = append(bulkOperation, "\n"...)
   153  	bulkOperation = append(bulkOperation, document...)
   154  	bulkOperation = append(bulkOperation, "\n"...)
   155  	log.Debugf("About to add a bulk operation of size %d bytes to bulk request, current size of bulk request body is %d bytes", len(bulkOperation), len(*body))
   156  
   157  	// 1 = len("\n") the last newline that will be appended to terminate the bulk request
   158  	estimatedBodySize := len(*body) + len(bulkOperation) + 1
   159  	if len(bulkOperation)+1 > maxBulkSizeInBytes {
   160  		return false, errors.Errorf(
   161  			"A bulk operation size (order + document %s) is greater than the maximum bulk size authorized (%dkB) : %d > %d, this document can't be sent to ES, please adapt your configuration !",
   162  			kv.Key, c.maxBulkSize, len(bulkOperation)+1, maxBulkSizeInBytes,
   163  		)
   164  	}
   165  	if estimatedBodySize > maxBulkSizeInBytes {
   166  		log.Printf(
   167  			"The limit of bulk size (%d kB) will be reached (%d > %d), the current document will be sent in the next bulk request",
   168  			c.maxBulkSize, estimatedBodySize, maxBulkSizeInBytes,
   169  		)
   170  		return false, nil
   171  	}
   172  	log.Debugf("Append document built from key %s to bulk request body, storeType was %s", kv.Key, storeType)
   173  	// Append the bulk operation
   174  	*body = append(*body, bulkOperation...)
   175  	return true, nil
   176  }
   177  
   178  // The index name are prefixed to avoid index name collisions.
   179  func getIndexName(c elasticStoreConf, storeType string) string {
   180  	return c.indicePrefix + strings.ToLower(c.clusterID) + "_" + storeType
   181  }