github.com/ystia/yorc/v4@v4.3.0/storage/internal/elastic/queries.go (about)

     1  // Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package elastic
    16  
    17  import (
    18  	"bytes"
    19  	"strconv"
    20  	"text/template"
    21  )
    22  
    23  // Index creation request
    24  const initStorageTemplateText = `
    25  {
    26       "settings": {
    27          {{ if ne .InitialReplicas -1}}"number_of_replicas": {{ .InitialReplicas}},{{end}}            
    28          {{ if ne .InitialShards -1 }}"number_of_shards": {{ .InitialShards}},{{end}}
    29          "refresh_interval": "1s"
    30       },
    31       "mappings": {
    32           "_doc": {
    33               "_all": {"enabled": false},
    34               "dynamic": "false",
    35               "properties": {
    36                   "deploymentId": { "type": "keyword", "index": true },
    37                   "iid": { "type": "long", "index": true },
    38                   "iidStr": { "type": "keyword","index": false }
    39               }
    40           }
    41       }
    42  }`
    43  
    44  // Get last Modified index
    45  const lastModifiedIndexTemplateText = `
    46  {
    47      "aggs" : {
    48          "max_iid" : {
    49              "filter" : {
    50  {{if .}}
    51                  "bool": {
    52                      "must": [
    53                          { "term": { "deploymentId": "{{ . }}" } }
    54                       ]
    55                  }
    56  {{else}}
    57                  "match_all": {}
    58  {{end}}
    59              },
    60              "aggs" : {
    61                  "last_index" : { "max" : { "field" : "iid" } }
    62              }
    63          }
    64      }
    65  }`
    66  
    67  // Range Query
    68  const rangeQueryTemplateText = `{ "range":{ "iid":{ "gt": "{{ conv .WaitIndex }}"{{if gt .MaxIndex 0}},"lte": "{{ conv .MaxIndex }}"{{end}}}}}`
    69  
    70  const listQueryTemplateText = `
    71  {
    72    "query":{{if .DeploymentID}}{
    73      "bool":{
    74          "must": [
    75            {
    76              "term":{
    77                 "deploymentId": "{{ .DeploymentID }}"
    78              }
    79            },
    80            {{template "rangeQuery" .}}
    81          ]
    82      }
    83    }{{else}}{{template "rangeQuery" .}}{{end}}
    84  }
    85  `
    86  
    87  var templates *template.Template
    88  
    89  func init() {
    90  	funcMap := template.FuncMap{"conv": func(value uint64) string { return strconv.FormatUint(value, 10) }}
    91  
    92  	templates = template.Must(template.New("initStorage").Parse(initStorageTemplateText))
    93  	templates = template.Must(templates.New("lastModifiedIndex").Parse(lastModifiedIndexTemplateText))
    94  
    95  	templates = template.Must(templates.New("rangeQuery").Funcs(funcMap).Parse(rangeQueryTemplateText))
    96  	templates = template.Must(templates.New("listQuery").Parse(listQueryTemplateText))
    97  }
    98  
    99  // Return the query that is used to create indexes for event and log storage.
   100  // We only index the needed fields to optimize ES indexing performance (no dynamic mapping).
   101  func buildInitStorageIndexQuery(elasticStoreConfig elasticStoreConf) string {
   102  	var buffer bytes.Buffer
   103  	templates.ExecuteTemplate(&buffer, "initStorage", elasticStoreConfig)
   104  	return buffer.String()
   105  }
   106  
   107  // This ES aggregation query is built using clusterId and eventually deploymentId.
   108  func buildLastModifiedIndexQuery(deploymentID string) (query string) {
   109  	var buffer bytes.Buffer
   110  	templates.ExecuteTemplate(&buffer, "lastModifiedIndex", deploymentID)
   111  	return buffer.String()
   112  }
   113  
   114  // This ES range query is built using 'waitIndex' and eventually 'maxIndex' and filtered using 'clusterId' and eventually 'deploymentId'.
   115  func getListQuery(deploymentID string, waitIndex uint64, maxIndex uint64) (query string) {
   116  	var buffer bytes.Buffer
   117  
   118  	data := struct {
   119  		WaitIndex    uint64
   120  		MaxIndex     uint64
   121  		DeploymentID string
   122  	}{
   123  		WaitIndex:    waitIndex,
   124  		MaxIndex:     maxIndex,
   125  		DeploymentID: deploymentID,
   126  	}
   127  
   128  	templates.ExecuteTemplate(&buffer, "listQuery", data)
   129  	return buffer.String()
   130  }