github.com/ystia/yorc/v4@v4.3.0/storage/internal/elastic/store.go (about)

     1  // Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package elastic provides an implementation of a storage that index/get documents to/from Elasticsearch 6.x.
    16  // This store can only manage logs and events for the moment. It will fail if you try to use it for other store types.
    17  package elastic
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"fmt"
    24  	elasticsearch6 "github.com/elastic/go-elasticsearch/v6"
    25  	"github.com/elastic/go-elasticsearch/v6/esapi"
    26  	"github.com/pkg/errors"
    27  	"github.com/ystia/yorc/v4/config"
    28  	"github.com/ystia/yorc/v4/log"
    29  	"github.com/ystia/yorc/v4/storage/encoding"
    30  	"github.com/ystia/yorc/v4/storage/store"
    31  	"github.com/ystia/yorc/v4/storage/utils"
    32  	"math"
    33  	"strings"
    34  	"time"
    35  )
    36  
    37  type elasticStore struct {
    38  	codec    encoding.Codec
    39  	esClient *elasticsearch6.Client
    40  	cfg      elasticStoreConf
    41  }
    42  
    43  // NewStore returns a new Elastic store.
    44  // Since the elastic store can only manage logs or events, it will panic is it's configured for anything else.
    45  // At init stage, we display ES cluster info and initialise indexes if they are not found.
    46  func NewStore(cfg config.Configuration, storeConfig config.Store) (store.Store, error) {
    47  
    48  	// Just fail if this storage is used for anything different from logs or events
    49  	for _, t := range storeConfig.Types {
    50  		if t != "Log" && t != "Event" {
    51  			return nil, errors.Errorf("Elastic store is not able to manage <%s>, just Log or Event, please change your config", t)
    52  		}
    53  	}
    54  
    55  	// Get specific config from storage properties
    56  	elasticStoreConfig, err := getElasticStoreConfig(cfg, storeConfig)
    57  	if err != nil {
    58  		return nil, err
    59  	}
    60  
    61  	esClient, err := prepareEsClient(elasticStoreConfig)
    62  	if err != nil {
    63  		return nil, err
    64  	}
    65  
    66  	err = initStorageIndex(esClient, elasticStoreConfig, "logs")
    67  	if err != nil {
    68  		return nil, errors.Wrapf(err, "Not able to init index for eventType <%s>", "logs")
    69  	}
    70  	err = initStorageIndex(esClient, elasticStoreConfig, "events")
    71  	if err != nil {
    72  		return nil, errors.Wrapf(err, "Not able to init index for eventType <%s>", "events")
    73  	}
    74  
    75  	return &elasticStore{encoding.JSON, esClient, elasticStoreConfig}, nil
    76  }
    77  
    78  // Set index a document (log or event) into ES.
    79  func (s *elasticStore) Set(ctx context.Context, k string, v interface{}) error {
    80  	log.Debugf("Set called will key %s", k)
    81  
    82  	if err := utils.CheckKeyAndValue(k, v); err != nil {
    83  		return err
    84  	}
    85  
    86  	storeType, body, err := buildElasticDocument(k, v)
    87  	if err != nil {
    88  		return err
    89  	}
    90  
    91  	indexName := getIndexName(s.cfg, storeType)
    92  	if log.IsDebug() {
    93  		log.Debugf("About to index this document into ES index <%s> : %+v", indexName, string(body))
    94  	}
    95  
    96  	// Prepare ES request
    97  	req := esapi.IndexRequest{
    98  		Index:        indexName,
    99  		DocumentType: "_doc",
   100  		Body:         bytes.NewReader(body),
   101  	}
   102  	res, err := req.Do(context.Background(), s.esClient)
   103  	defer closeResponseBody("IndexRequest:"+indexName, res)
   104  	if err != nil || res.IsError() {
   105  		err = handleESResponseError(res, "Index:"+indexName, string(body), err)
   106  		return err
   107  	}
   108  	return nil
   109  }
   110  
   111  // SetCollection index collections using ES bulk requests.
   112  // We consider both 'max_bulk_size' and 'max_bulk_count' to define bulk requests size.
   113  func (s *elasticStore) SetCollection(ctx context.Context, keyValues []store.KeyValueIn) error {
   114  	totalDocumentCount := len(keyValues)
   115  	log.Printf("SetCollection called with an array of size %d", totalDocumentCount)
   116  	start := time.Now()
   117  
   118  	if keyValues == nil || totalDocumentCount == 0 {
   119  		return nil
   120  	}
   121  
   122  	// Just estimate the iteration count
   123  	iterationCount := int(math.Ceil(float64(totalDocumentCount) / float64(s.cfg.maxBulkCount)))
   124  	log.Printf(
   125  		"max_bulk_count is %d, so a minimum of %d iterations will be necessary to bulk index the %d documents",
   126  		s.cfg.maxBulkCount, iterationCount, totalDocumentCount,
   127  	)
   128  
   129  	// The current index in []keyValues (also the number of documents indexed)
   130  	var kvi = 0
   131  	// The number of iterations
   132  	var i = 0
   133  	// Iterate over the []keyValues
   134  	for {
   135  		if kvi == totalDocumentCount {
   136  			// We have reached the end of []keyValues
   137  			break
   138  		}
   139  		fmt.Printf("Bulk iteration %d", i)
   140  
   141  		maxBulkSizeInBytes := s.cfg.maxBulkSize * 1024
   142  		// Prepare a slice of max capacity
   143  		var body = make([]byte, 0, maxBulkSizeInBytes)
   144  		// Number of operation in the current bulk request
   145  		opeCount := 0
   146  		// Each iteration is a single bulk request
   147  		for {
   148  			if kvi == totalDocumentCount || opeCount == s.cfg.maxBulkCount {
   149  				// We have reached the end of []keyValues OR the max items allowed in a single bulk request (max_bulk_count)
   150  				break
   151  			}
   152  			added, err := eventuallyAppendValueToBulkRequest(s.cfg, &body, keyValues[kvi], maxBulkSizeInBytes)
   153  			if err != nil {
   154  				return err
   155  			} else if !added {
   156  				// The document hasn't been added (too big), let's include it in next bulk
   157  				break
   158  			} else {
   159  				kvi++
   160  				opeCount++
   161  			}
   162  		}
   163  		// The bulk request must be terminated by a newline
   164  		body = append(body, "\n"...)
   165  		// Send the request
   166  		err := sendBulkRequest(s.esClient, opeCount, &body)
   167  		if err != nil {
   168  			return err
   169  		}
   170  		// Increment the number of iterations
   171  		i++
   172  	}
   173  	elapsed := time.Since(start)
   174  	log.Printf("A total of %d documents have been successfully indexed using %d bulk requests, took %v", kvi, i, elapsed)
   175  	return nil
   176  }
   177  
   178  // Delete removes ES documents using a deleteByRequest query.
   179  func (s *elasticStore) Delete(ctx context.Context, k string, recursive bool) error {
   180  	log.Debugf("Delete called k: %s, recursive: %t", k, recursive)
   181  
   182  	// Extract index name and deploymentID by parsing the key
   183  	storeType, deploymentID := extractStoreTypeAndDeploymentID(k)
   184  	indexName := getIndexName(s.cfg, storeType)
   185  	log.Debugf("storeType is: %s, indexName is %s, deploymentID is: %s", storeType, indexName, deploymentID)
   186  
   187  	query := `{"query" : { "term": { "deploymentId" : "` + deploymentID + `" }}}`
   188  	log.Debugf("query is : %s", query)
   189  
   190  	var MaxInt = 1024000
   191  
   192  	req := esapi.DeleteByQueryRequest{
   193  		Index:     []string{indexName},
   194  		Size:      &MaxInt,
   195  		Body:      strings.NewReader(query),
   196  		Conflicts: "proceed",
   197  	}
   198  	res, err := req.Do(context.Background(), s.esClient)
   199  	defer closeResponseBody("DeleteByQueryRequest:"+indexName, res)
   200  	err = handleESResponseError(res, "DeleteByQueryRequest:"+indexName, query, err)
   201  	return err
   202  }
   203  
   204  // GetLastModifyIndex return the last index which is found by querying ES using aggregation and a 0 size request.
   205  func (s *elasticStore) GetLastModifyIndex(k string) (lastIndex uint64, e error) {
   206  	log.Debugf("GetLastModifyIndex called k: %s", k)
   207  
   208  	// Extract index name and deploymentID by parsing the key
   209  	storeType, deploymentID := extractStoreTypeAndDeploymentID(k)
   210  	indexName := getIndexName(s.cfg, storeType)
   211  	log.Debugf("storeType is: %s, indexName is: %s, deploymentID is: %s", storeType, indexName, deploymentID)
   212  
   213  	// The lastIndex is query by using ES aggregation query ~= MAX(iid) HAVING deploymentId
   214  	query := buildLastModifiedIndexQuery(deploymentID)
   215  	log.Debugf("buildLastModifiedIndexQuery is : %s", query)
   216  
   217  	resSearch, err := s.esClient.Search(
   218  		s.esClient.Search.WithContext(context.Background()),
   219  		s.esClient.Search.WithIndex(indexName),
   220  		s.esClient.Search.WithSize(0),
   221  		s.esClient.Search.WithBody(strings.NewReader(query)),
   222  	)
   223  	defer closeResponseBody("LastModifiedIndexQuery for "+k, resSearch)
   224  	e = handleESResponseError(resSearch, "LastModifiedIndexQuery for "+k, query, err)
   225  	if e != nil {
   226  		return
   227  	}
   228  
   229  	var r map[string]interface{}
   230  	if err := json.NewDecoder(resSearch.Body).Decode(&r); err != nil {
   231  		e = errors.Wrapf(
   232  			err,
   233  			"Not able to parse response body after LastModifiedIndexQuery was sent for key %s, status was %s, query was: %s",
   234  			k, resSearch.Status(), query,
   235  		)
   236  		return
   237  	}
   238  
   239  	total := r["hits"].(map[string]interface{})["total"].(float64)
   240  	if total > 0 {
   241  		// ES returns aggregations as float, we have a precision loss of few ns
   242  		lastIndexR := r["aggregations"].(map[string]interface{})["max_iid"].(map[string]interface{})["last_index"].(map[string]interface{})["value"].(float64)
   243  		log.Debugf("Received lastIndexReceived: %v, lastIndex: %v", lastIndexR, lastIndex)
   244  		lastIndex = uint64(lastIndexR)
   245  		// The ES max result was a float, there is a risk that this is not really the lastIndex
   246  		// We need to verify
   247  		lastIndex = s.verifyLastIndex(indexName, deploymentID, lastIndex)
   248  	}
   249  	return lastIndex, nil
   250  }
   251  
   252  // We need to ensure the lastIndex returned by the aggregation query is really the last
   253  // Actually, when elasticsearch aggregates, it returns a float so we loss precession (few ns).
   254  // We request the docs with iid > waitIndex to ensure the returned lastIndex is REALLY the last.
   255  func (s *elasticStore) verifyLastIndex(indexName string, deploymentID string, estimatedLastIndex uint64) uint64 {
   256  	query := getListQuery(deploymentID, estimatedLastIndex, 0)
   257  	// size = 1 no need for the documents
   258  	hits, _, lastIndex, err := doQueryEs(context.Background(), s.esClient, s.cfg, indexName, query, estimatedLastIndex, 1, "desc")
   259  	if err != nil {
   260  		log.Printf("An error occurred while verifying lastIndex, returning the initial value %d, error was : %+v",
   261  			estimatedLastIndex, err)
   262  	}
   263  	log.Printf("%d hits while searching %s (%s) using the estimated lastIndex %d, lastIndex is now %d",
   264  		hits, indexName, deploymentID, estimatedLastIndex, lastIndex)
   265  	return lastIndex
   266  }
   267  
   268  // List simulates long polling request by :
   269  // - periodically querying ES for documents (Aggregation to get the max iid and 0 size result).
   270  // - if a some result is found, wait some time (es_refresh_wait_timeout) in order to:
   271  //   	- let ES index recently added documents AND to let
   272  // 		- let Yorc eventually Set a document that has a less iid than the older known document in ES (concurrence issues)
   273  // - if no result if found after the the given 'timeout', return empty slice
   274  func (s *elasticStore) List(ctx context.Context, k string, waitIndex uint64, timeout time.Duration) ([]store.KeyValueOut, uint64, error) {
   275  	log.Debugf("List called k: %s, waitIndex: %d, timeout: %v", k, waitIndex, timeout)
   276  	if err := utils.CheckKey(k); err != nil {
   277  		return nil, 0, err
   278  	}
   279  
   280  	// Extract indice name by parsing the key
   281  	storeType, deploymentID := extractStoreTypeAndDeploymentID(k)
   282  	indexName := getIndexName(s.cfg, storeType)
   283  	log.Debugf("storeType is: %s, indexName is: %s, deploymentID is: %s", storeType, indexName, deploymentID)
   284  
   285  	query := getListQuery(deploymentID, waitIndex, 0)
   286  
   287  	now := time.Now()
   288  	end := now.Add(timeout - s.cfg.esRefreshWaitTimeout)
   289  	log.Debugf("Now is : %v, date after timeout will be %v (ES timeout duration will be %v)", now, end, timeout-s.cfg.esRefreshWaitTimeout)
   290  	var values = make([]store.KeyValueOut, 0)
   291  	var lastIndex = waitIndex
   292  	var hits = 0
   293  	var err error
   294  	for {
   295  		// first just query to know if they is something to fetch, we just want the max iid (so order desc, size 1)
   296  		hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 1, "desc")
   297  		if err != nil {
   298  			return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events, error was: %+v", err)
   299  		}
   300  		now := time.Now()
   301  		if hits > 0 || now.After(end) {
   302  			break
   303  		}
   304  
   305  		log.Debugf("hits is %d and timeout not reached, sleeping %v ...", hits, s.cfg.esQueryPeriod)
   306  		select {
   307  		case <-time.After(s.cfg.esQueryPeriod):
   308  			continue
   309  		case <-ctx.Done():
   310  			return values, lastIndex, nil
   311  		}
   312  	}
   313  	if hits > 0 {
   314  		// we do have something to retrieve, we will just wait esRefreshWaitTimeout to let any document that has just been stored to be indexed
   315  		// then we just retrieve this 'time window' (between waitIndex and lastIndex)
   316  		query := getListQuery(deploymentID, waitIndex, lastIndex)
   317  		if s.cfg.esForceRefresh {
   318  			// force refresh for this index
   319  			refreshIndex(s.esClient, indexName)
   320  		}
   321  		time.Sleep(s.cfg.esRefreshWaitTimeout)
   322  		oldHits := hits
   323  		hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 10000, "asc")
   324  		if err != nil {
   325  			return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events (after waiting for refresh)")
   326  		}
   327  		if log.IsDebug() && hits > oldHits {
   328  			log.Debugf("%d > %d so sleeping %v to wait for ES refresh was useful (index %s), %d documents has been fetched",
   329  				hits, oldHits, s.cfg.esRefreshWaitTimeout, indexName, len(values),
   330  			)
   331  		}
   332  	}
   333  	log.Debugf("List called result k: %s, waitIndex: %d, timeout: %v, LastIndex: %d, len(values): %d",
   334  		k, waitIndex, timeout, lastIndex, len(values))
   335  	return values, lastIndex, err
   336  }
   337  
   338  // Get is not used for logs nor events: fails in FATAL.
   339  func (s *elasticStore) Get(k string, v interface{}) (bool, error) {
   340  	if err := utils.CheckKeyAndValue(k, v); err != nil {
   341  		return false, err
   342  	}
   343  	return false, errors.Errorf("Function Get(string, interface{}) not yet implemented for Elastic store !")
   344  }
   345  
   346  // Exist is not used for logs nor events: fails in FATAL.
   347  func (s *elasticStore) Exist(k string) (bool, error) {
   348  	if err := utils.CheckKey(k); err != nil {
   349  		return false, err
   350  	}
   351  	return false, errors.Errorf("Function Exist(string) not yet implemented for Elastic store !")
   352  }
   353  
   354  // Keys is not used for logs nor events: fails in FATAL.
   355  func (s *elasticStore) Keys(k string) ([]string, error) {
   356  	return nil, errors.Errorf("Function Keys(string) not yet implemented for Elastic store !")
   357  }