github.com/ystia/yorc/v4@v4.3.0/storage/internal/elastic/utils.go (about) 1 // Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package elastic 16 17 import ( 18 "encoding/json" 19 "github.com/pkg/errors" 20 "github.com/ystia/yorc/v4/log" 21 "github.com/ystia/yorc/v4/storage/store" 22 "github.com/ystia/yorc/v4/storage/utils" 23 "regexp" 24 "strconv" 25 "strings" 26 "time" 27 ) 28 29 // Precompiled regex to extract storeType and timestamp from a key of the form: "_yorc/logs/MyApp/2020-06-07T21:03:17.812178429Z". 30 var storeTypeAndTimestampRegex = regexp.MustCompile(`(?m)\_yorc\/(\w+)\/.+\/(.*)`) 31 32 // Parse a key of form "_yorc/logs/MyApp/2020-06-07T21:03:17.812178429Z" to get the store type (logs|events) and the timestamp. 33 func extractStoreTypeAndTimestamp(k string) (storeType string, timestamp string) { 34 res := storeTypeAndTimestampRegex.FindAllStringSubmatch(k, -1) 35 for i := range res { 36 storeType = res[i][1] 37 timestamp = res[i][2] 38 } 39 return storeType, timestamp 40 } 41 42 // Precompiled regex to extract storeType and deploymentId from a key of the form: "_yorc/events/" or "_yorc/logs/MyApp" or "_yorc/logs/MyApp/". 43 var storeTypeAndDeploymentIDRegex = regexp.MustCompile(`(?m)\_yorc\/(\w+)\/?(.+)?\/?`) 44 45 // Parse a key of form "_yorc/events/" or "_yorc/logs/MyApp" or "_yorc/logs/MyApp/" to get the store type (logs|events) and eventually the deploymentId. 46 func extractStoreTypeAndDeploymentID(k string) (storeType string, deploymentID string) { 47 res := storeTypeAndDeploymentIDRegex.FindAllStringSubmatch(k, -1) 48 for i := range res { 49 storeType = res[i][1] 50 if len(res[i]) == 3 { 51 deploymentID = res[i][2] 52 if strings.HasSuffix(deploymentID, "/") { 53 deploymentID = deploymentID[:len(deploymentID)-1] 54 } 55 } 56 } 57 return storeType, deploymentID 58 } 59 60 // We need to append JSON directly into []byte to avoid useless and costly marshaling / unmarshaling. 61 func appendJSONInBytes(a []byte, v []byte) []byte { 62 last := len(a) - 1 63 lastByte := a[last] 64 // just append v at the end 65 a = append(a, v...) 66 // then slice 67 for i, s := range v { 68 a[last+i] = s 69 } 70 a[last+len(v)] = lastByte 71 return a 72 } 73 74 func _parseInt64StringToInt64(value string) int64 { 75 valueInt, err := strconv.ParseInt(value, 10, 64) 76 if err != nil { 77 log.Printf("Fail parsing _parseInt64StringToInt64 %s", value) 78 } 79 return valueInt 80 } 81 82 func parseInt64StringToUint64(value string) (uint64, error) { 83 valueInt, err := strconv.ParseInt(value, 10, 64) 84 if err != nil { 85 return 0, errors.Wrapf(err, "Not able to parse string: %s to int64, error was: %+v", value, err) 86 } 87 result := uint64(valueInt) 88 return result, nil 89 } 90 91 func _getTimestampFromUint64(nanoTimestamp uint64) time.Time { 92 nanoTimestampStr := strconv.FormatUint(nanoTimestamp, 10) 93 ts := _parseInt64StringToInt64(nanoTimestampStr) 94 return time.Unix(0, ts) 95 } 96 97 // The max uint is 9223372036854775807 (19 cars), this is the maximum nano for a time (2262-04-12 00:47:16.854775807 +0100 CET). 98 // Since we use nanotimestamp as ID for events and logs, and since we store this ID as string in ES index, we must ensure that 99 // the string will be comparable. 100 func getSortableStringFromUint64(nanoTimestamp uint64) string { 101 nanoTimestampStr := strconv.FormatUint(nanoTimestamp, 10) 102 if len(nanoTimestampStr) < 19 { 103 nanoTimestampStr = strings.Repeat("0", 19-len(nanoTimestampStr)) + nanoTimestampStr 104 } 105 return nanoTimestampStr 106 } 107 108 // The document is enriched by adding 'clusterId' and 'iid' properties. 109 // This addition is done by directly manipulating the []byte in order to avoid costly successive marshal / unmarshal operations. 110 func buildElasticDocument(k string, rawMessage interface{}) (string, []byte, error) { 111 // Extract indice name and timestamp by parsing the key 112 storeType, timestamp := extractStoreTypeAndTimestamp(k) 113 log.Debugf("storeType is: %s, timestamp: %s", storeType, timestamp) 114 115 // Convert timestamp to an int64 116 eventDate, err := time.Parse(time.RFC3339Nano, timestamp) 117 if err != nil { 118 return storeType, nil, errors.Wrapf(err, "failed to parse timestamp %+v as time, error was: %+v", timestamp, err) 119 } 120 // Convert to UnixNano int64 121 iid := eventDate.UnixNano() 122 123 // This is the piece of 'JSON' we want to append 124 a := `,"iid":"` + strconv.FormatInt(iid, 10) + `","iidStr":"` + strconv.FormatInt(iid, 10) + `"` 125 // v is a json.RawMessage 126 raw := rawMessage.(json.RawMessage) 127 raw = appendJSONInBytes(raw, []byte(a)) 128 129 return storeType, raw, nil 130 } 131 132 // An error is returned if : 133 // - it's not valid (key or value nil) 134 // - the size of the resulting bulk operation exceed the maximum authorized for a bulk request 135 // The value is not added if it's size + the current body size exceed the maximum authorized for a bulk request. 136 // Return a bool indicating if the value has been added to the bulk request body. 137 func eventuallyAppendValueToBulkRequest(c elasticStoreConf, body *[]byte, kv store.KeyValueIn, maxBulkSizeInBytes int) (bool, error) { 138 if err := utils.CheckKeyAndValue(kv.Key, kv.Value); err != nil { 139 return false, err 140 } 141 142 storeType, document, err := buildElasticDocument(kv.Key, kv.Value) 143 if err != nil { 144 return false, err 145 } 146 log.Debugf("About to add a document of size %d bytes to bulk request", len(document)) 147 148 // The bulk action 149 index := `{"index":{"_index":"` + getIndexName(c, storeType) + `","_type":"_doc"}}` 150 bulkOperation := make([]byte, 0) 151 bulkOperation = append(bulkOperation, index...) 152 bulkOperation = append(bulkOperation, "\n"...) 153 bulkOperation = append(bulkOperation, document...) 154 bulkOperation = append(bulkOperation, "\n"...) 155 log.Debugf("About to add a bulk operation of size %d bytes to bulk request, current size of bulk request body is %d bytes", len(bulkOperation), len(*body)) 156 157 // 1 = len("\n") the last newline that will be appended to terminate the bulk request 158 estimatedBodySize := len(*body) + len(bulkOperation) + 1 159 if len(bulkOperation)+1 > maxBulkSizeInBytes { 160 return false, errors.Errorf( 161 "A bulk operation size (order + document %s) is greater than the maximum bulk size authorized (%dkB) : %d > %d, this document can't be sent to ES, please adapt your configuration !", 162 kv.Key, c.maxBulkSize, len(bulkOperation)+1, maxBulkSizeInBytes, 163 ) 164 } 165 if estimatedBodySize > maxBulkSizeInBytes { 166 log.Printf( 167 "The limit of bulk size (%d kB) will be reached (%d > %d), the current document will be sent in the next bulk request", 168 c.maxBulkSize, estimatedBodySize, maxBulkSizeInBytes, 169 ) 170 return false, nil 171 } 172 log.Debugf("Append document built from key %s to bulk request body, storeType was %s", kv.Key, storeType) 173 // Append the bulk operation 174 *body = append(*body, bulkOperation...) 175 return true, nil 176 } 177 178 // The index name are prefixed to avoid index name collisions. 179 func getIndexName(c elasticStoreConf, storeType string) string { 180 return c.indicePrefix + strings.ToLower(c.clusterID) + "_" + storeType 181 }