github.com/ystia/yorc/v4@v4.3.0/storage/internal/elastic/store.go (about) 1 // Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package elastic provides an implementation of a storage that index/get documents to/from Elasticsearch 6.x. 16 // This store can only manage logs and events for the moment. It will fail if you try to use it for other store types. 17 package elastic 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "fmt" 24 elasticsearch6 "github.com/elastic/go-elasticsearch/v6" 25 "github.com/elastic/go-elasticsearch/v6/esapi" 26 "github.com/pkg/errors" 27 "github.com/ystia/yorc/v4/config" 28 "github.com/ystia/yorc/v4/log" 29 "github.com/ystia/yorc/v4/storage/encoding" 30 "github.com/ystia/yorc/v4/storage/store" 31 "github.com/ystia/yorc/v4/storage/utils" 32 "math" 33 "strings" 34 "time" 35 ) 36 37 type elasticStore struct { 38 codec encoding.Codec 39 esClient *elasticsearch6.Client 40 cfg elasticStoreConf 41 } 42 43 // NewStore returns a new Elastic store. 44 // Since the elastic store can only manage logs or events, it will panic is it's configured for anything else. 45 // At init stage, we display ES cluster info and initialise indexes if they are not found. 46 func NewStore(cfg config.Configuration, storeConfig config.Store) (store.Store, error) { 47 48 // Just fail if this storage is used for anything different from logs or events 49 for _, t := range storeConfig.Types { 50 if t != "Log" && t != "Event" { 51 return nil, errors.Errorf("Elastic store is not able to manage <%s>, just Log or Event, please change your config", t) 52 } 53 } 54 55 // Get specific config from storage properties 56 elasticStoreConfig, err := getElasticStoreConfig(cfg, storeConfig) 57 if err != nil { 58 return nil, err 59 } 60 61 esClient, err := prepareEsClient(elasticStoreConfig) 62 if err != nil { 63 return nil, err 64 } 65 66 err = initStorageIndex(esClient, elasticStoreConfig, "logs") 67 if err != nil { 68 return nil, errors.Wrapf(err, "Not able to init index for eventType <%s>", "logs") 69 } 70 err = initStorageIndex(esClient, elasticStoreConfig, "events") 71 if err != nil { 72 return nil, errors.Wrapf(err, "Not able to init index for eventType <%s>", "events") 73 } 74 75 return &elasticStore{encoding.JSON, esClient, elasticStoreConfig}, nil 76 } 77 78 // Set index a document (log or event) into ES. 79 func (s *elasticStore) Set(ctx context.Context, k string, v interface{}) error { 80 log.Debugf("Set called will key %s", k) 81 82 if err := utils.CheckKeyAndValue(k, v); err != nil { 83 return err 84 } 85 86 storeType, body, err := buildElasticDocument(k, v) 87 if err != nil { 88 return err 89 } 90 91 indexName := getIndexName(s.cfg, storeType) 92 if log.IsDebug() { 93 log.Debugf("About to index this document into ES index <%s> : %+v", indexName, string(body)) 94 } 95 96 // Prepare ES request 97 req := esapi.IndexRequest{ 98 Index: indexName, 99 DocumentType: "_doc", 100 Body: bytes.NewReader(body), 101 } 102 res, err := req.Do(context.Background(), s.esClient) 103 defer closeResponseBody("IndexRequest:"+indexName, res) 104 if err != nil || res.IsError() { 105 err = handleESResponseError(res, "Index:"+indexName, string(body), err) 106 return err 107 } 108 return nil 109 } 110 111 // SetCollection index collections using ES bulk requests. 112 // We consider both 'max_bulk_size' and 'max_bulk_count' to define bulk requests size. 113 func (s *elasticStore) SetCollection(ctx context.Context, keyValues []store.KeyValueIn) error { 114 totalDocumentCount := len(keyValues) 115 log.Printf("SetCollection called with an array of size %d", totalDocumentCount) 116 start := time.Now() 117 118 if keyValues == nil || totalDocumentCount == 0 { 119 return nil 120 } 121 122 // Just estimate the iteration count 123 iterationCount := int(math.Ceil(float64(totalDocumentCount) / float64(s.cfg.maxBulkCount))) 124 log.Printf( 125 "max_bulk_count is %d, so a minimum of %d iterations will be necessary to bulk index the %d documents", 126 s.cfg.maxBulkCount, iterationCount, totalDocumentCount, 127 ) 128 129 // The current index in []keyValues (also the number of documents indexed) 130 var kvi = 0 131 // The number of iterations 132 var i = 0 133 // Iterate over the []keyValues 134 for { 135 if kvi == totalDocumentCount { 136 // We have reached the end of []keyValues 137 break 138 } 139 fmt.Printf("Bulk iteration %d", i) 140 141 maxBulkSizeInBytes := s.cfg.maxBulkSize * 1024 142 // Prepare a slice of max capacity 143 var body = make([]byte, 0, maxBulkSizeInBytes) 144 // Number of operation in the current bulk request 145 opeCount := 0 146 // Each iteration is a single bulk request 147 for { 148 if kvi == totalDocumentCount || opeCount == s.cfg.maxBulkCount { 149 // We have reached the end of []keyValues OR the max items allowed in a single bulk request (max_bulk_count) 150 break 151 } 152 added, err := eventuallyAppendValueToBulkRequest(s.cfg, &body, keyValues[kvi], maxBulkSizeInBytes) 153 if err != nil { 154 return err 155 } else if !added { 156 // The document hasn't been added (too big), let's include it in next bulk 157 break 158 } else { 159 kvi++ 160 opeCount++ 161 } 162 } 163 // The bulk request must be terminated by a newline 164 body = append(body, "\n"...) 165 // Send the request 166 err := sendBulkRequest(s.esClient, opeCount, &body) 167 if err != nil { 168 return err 169 } 170 // Increment the number of iterations 171 i++ 172 } 173 elapsed := time.Since(start) 174 log.Printf("A total of %d documents have been successfully indexed using %d bulk requests, took %v", kvi, i, elapsed) 175 return nil 176 } 177 178 // Delete removes ES documents using a deleteByRequest query. 179 func (s *elasticStore) Delete(ctx context.Context, k string, recursive bool) error { 180 log.Debugf("Delete called k: %s, recursive: %t", k, recursive) 181 182 // Extract index name and deploymentID by parsing the key 183 storeType, deploymentID := extractStoreTypeAndDeploymentID(k) 184 indexName := getIndexName(s.cfg, storeType) 185 log.Debugf("storeType is: %s, indexName is %s, deploymentID is: %s", storeType, indexName, deploymentID) 186 187 query := `{"query" : { "term": { "deploymentId" : "` + deploymentID + `" }}}` 188 log.Debugf("query is : %s", query) 189 190 var MaxInt = 1024000 191 192 req := esapi.DeleteByQueryRequest{ 193 Index: []string{indexName}, 194 Size: &MaxInt, 195 Body: strings.NewReader(query), 196 Conflicts: "proceed", 197 } 198 res, err := req.Do(context.Background(), s.esClient) 199 defer closeResponseBody("DeleteByQueryRequest:"+indexName, res) 200 err = handleESResponseError(res, "DeleteByQueryRequest:"+indexName, query, err) 201 return err 202 } 203 204 // GetLastModifyIndex return the last index which is found by querying ES using aggregation and a 0 size request. 205 func (s *elasticStore) GetLastModifyIndex(k string) (lastIndex uint64, e error) { 206 log.Debugf("GetLastModifyIndex called k: %s", k) 207 208 // Extract index name and deploymentID by parsing the key 209 storeType, deploymentID := extractStoreTypeAndDeploymentID(k) 210 indexName := getIndexName(s.cfg, storeType) 211 log.Debugf("storeType is: %s, indexName is: %s, deploymentID is: %s", storeType, indexName, deploymentID) 212 213 // The lastIndex is query by using ES aggregation query ~= MAX(iid) HAVING deploymentId 214 query := buildLastModifiedIndexQuery(deploymentID) 215 log.Debugf("buildLastModifiedIndexQuery is : %s", query) 216 217 resSearch, err := s.esClient.Search( 218 s.esClient.Search.WithContext(context.Background()), 219 s.esClient.Search.WithIndex(indexName), 220 s.esClient.Search.WithSize(0), 221 s.esClient.Search.WithBody(strings.NewReader(query)), 222 ) 223 defer closeResponseBody("LastModifiedIndexQuery for "+k, resSearch) 224 e = handleESResponseError(resSearch, "LastModifiedIndexQuery for "+k, query, err) 225 if e != nil { 226 return 227 } 228 229 var r map[string]interface{} 230 if err := json.NewDecoder(resSearch.Body).Decode(&r); err != nil { 231 e = errors.Wrapf( 232 err, 233 "Not able to parse response body after LastModifiedIndexQuery was sent for key %s, status was %s, query was: %s", 234 k, resSearch.Status(), query, 235 ) 236 return 237 } 238 239 total := r["hits"].(map[string]interface{})["total"].(float64) 240 if total > 0 { 241 // ES returns aggregations as float, we have a precision loss of few ns 242 lastIndexR := r["aggregations"].(map[string]interface{})["max_iid"].(map[string]interface{})["last_index"].(map[string]interface{})["value"].(float64) 243 log.Debugf("Received lastIndexReceived: %v, lastIndex: %v", lastIndexR, lastIndex) 244 lastIndex = uint64(lastIndexR) 245 // The ES max result was a float, there is a risk that this is not really the lastIndex 246 // We need to verify 247 lastIndex = s.verifyLastIndex(indexName, deploymentID, lastIndex) 248 } 249 return lastIndex, nil 250 } 251 252 // We need to ensure the lastIndex returned by the aggregation query is really the last 253 // Actually, when elasticsearch aggregates, it returns a float so we loss precession (few ns). 254 // We request the docs with iid > waitIndex to ensure the returned lastIndex is REALLY the last. 255 func (s *elasticStore) verifyLastIndex(indexName string, deploymentID string, estimatedLastIndex uint64) uint64 { 256 query := getListQuery(deploymentID, estimatedLastIndex, 0) 257 // size = 1 no need for the documents 258 hits, _, lastIndex, err := doQueryEs(context.Background(), s.esClient, s.cfg, indexName, query, estimatedLastIndex, 1, "desc") 259 if err != nil { 260 log.Printf("An error occurred while verifying lastIndex, returning the initial value %d, error was : %+v", 261 estimatedLastIndex, err) 262 } 263 log.Printf("%d hits while searching %s (%s) using the estimated lastIndex %d, lastIndex is now %d", 264 hits, indexName, deploymentID, estimatedLastIndex, lastIndex) 265 return lastIndex 266 } 267 268 // List simulates long polling request by : 269 // - periodically querying ES for documents (Aggregation to get the max iid and 0 size result). 270 // - if a some result is found, wait some time (es_refresh_wait_timeout) in order to: 271 // - let ES index recently added documents AND to let 272 // - let Yorc eventually Set a document that has a less iid than the older known document in ES (concurrence issues) 273 // - if no result if found after the the given 'timeout', return empty slice 274 func (s *elasticStore) List(ctx context.Context, k string, waitIndex uint64, timeout time.Duration) ([]store.KeyValueOut, uint64, error) { 275 log.Debugf("List called k: %s, waitIndex: %d, timeout: %v", k, waitIndex, timeout) 276 if err := utils.CheckKey(k); err != nil { 277 return nil, 0, err 278 } 279 280 // Extract indice name by parsing the key 281 storeType, deploymentID := extractStoreTypeAndDeploymentID(k) 282 indexName := getIndexName(s.cfg, storeType) 283 log.Debugf("storeType is: %s, indexName is: %s, deploymentID is: %s", storeType, indexName, deploymentID) 284 285 query := getListQuery(deploymentID, waitIndex, 0) 286 287 now := time.Now() 288 end := now.Add(timeout - s.cfg.esRefreshWaitTimeout) 289 log.Debugf("Now is : %v, date after timeout will be %v (ES timeout duration will be %v)", now, end, timeout-s.cfg.esRefreshWaitTimeout) 290 var values = make([]store.KeyValueOut, 0) 291 var lastIndex = waitIndex 292 var hits = 0 293 var err error 294 for { 295 // first just query to know if they is something to fetch, we just want the max iid (so order desc, size 1) 296 hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 1, "desc") 297 if err != nil { 298 return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events, error was: %+v", err) 299 } 300 now := time.Now() 301 if hits > 0 || now.After(end) { 302 break 303 } 304 305 log.Debugf("hits is %d and timeout not reached, sleeping %v ...", hits, s.cfg.esQueryPeriod) 306 select { 307 case <-time.After(s.cfg.esQueryPeriod): 308 continue 309 case <-ctx.Done(): 310 return values, lastIndex, nil 311 } 312 } 313 if hits > 0 { 314 // we do have something to retrieve, we will just wait esRefreshWaitTimeout to let any document that has just been stored to be indexed 315 // then we just retrieve this 'time window' (between waitIndex and lastIndex) 316 query := getListQuery(deploymentID, waitIndex, lastIndex) 317 if s.cfg.esForceRefresh { 318 // force refresh for this index 319 refreshIndex(s.esClient, indexName) 320 } 321 time.Sleep(s.cfg.esRefreshWaitTimeout) 322 oldHits := hits 323 hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 10000, "asc") 324 if err != nil { 325 return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events (after waiting for refresh)") 326 } 327 if log.IsDebug() && hits > oldHits { 328 log.Debugf("%d > %d so sleeping %v to wait for ES refresh was useful (index %s), %d documents has been fetched", 329 hits, oldHits, s.cfg.esRefreshWaitTimeout, indexName, len(values), 330 ) 331 } 332 } 333 log.Debugf("List called result k: %s, waitIndex: %d, timeout: %v, LastIndex: %d, len(values): %d", 334 k, waitIndex, timeout, lastIndex, len(values)) 335 return values, lastIndex, err 336 } 337 338 // Get is not used for logs nor events: fails in FATAL. 339 func (s *elasticStore) Get(k string, v interface{}) (bool, error) { 340 if err := utils.CheckKeyAndValue(k, v); err != nil { 341 return false, err 342 } 343 return false, errors.Errorf("Function Get(string, interface{}) not yet implemented for Elastic store !") 344 } 345 346 // Exist is not used for logs nor events: fails in FATAL. 347 func (s *elasticStore) Exist(k string) (bool, error) { 348 if err := utils.CheckKey(k); err != nil { 349 return false, err 350 } 351 return false, errors.Errorf("Function Exist(string) not yet implemented for Elastic store !") 352 } 353 354 // Keys is not used for logs nor events: fails in FATAL. 355 func (s *elasticStore) Keys(k string) ([]string, error) { 356 return nil, errors.Errorf("Function Keys(string) not yet implemented for Elastic store !") 357 }