github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/query/metadata/tsmeta.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package metadata
    18  
    19  import (
    20  	"sync"
    21  	"time"
    22  
    23  	dtu "github.com/siglens/siglens/pkg/common/dtypeutils"
    24  	"github.com/siglens/siglens/pkg/config"
    25  	"github.com/siglens/siglens/pkg/segment/query/summary"
    26  	"github.com/siglens/siglens/pkg/segment/structs"
    27  )
    28  
    29  /*
    30  Returns all tagTrees that we need to search and what MetricsSegments & MetricsBlocks pass time filtering.
    31  
    32  Returns map[string][]*structs.MetricSearchRequest, mapping a tagsTree to all MetricSearchRequest that pass time filtering
    33  */
    34  func GetMetricsSegmentRequests(mName string, tRange *dtu.MetricsTimeRange, querySummary *summary.QuerySummary, orgid uint64) (map[string][]*structs.MetricsSearchRequest, error) {
    35  	sTime := time.Now()
    36  
    37  	retUpdate := &sync.Mutex{}
    38  	wg := &sync.WaitGroup{}
    39  	parallelism := int(config.GetParallelism())
    40  	retVal := make(map[string][]*structs.MetricsSearchRequest)
    41  	var gErr error
    42  
    43  	globalMetricsMetadata.updateLock.Lock()
    44  	defer globalMetricsMetadata.updateLock.Unlock()
    45  
    46  	for i, mSegMeta := range globalMetricsMetadata.sortedMetricsSegmentMeta {
    47  		if !tRange.CheckRangeOverLap(mSegMeta.EarliestEpochSec, mSegMeta.LatestEpochSec) || mSegMeta.OrgId != orgid {
    48  			continue
    49  		}
    50  		wg.Add(1)
    51  		go func(msm *MetricsSegmentMetadata) {
    52  			defer wg.Done()
    53  			var forceLoaded bool
    54  			if !msm.loadedSearchMetadata {
    55  				err := msm.LoadSearchMetadata()
    56  				if err != nil {
    57  					gErr = err
    58  					return
    59  				}
    60  				forceLoaded = true
    61  			}
    62  
    63  			retBlocks := make(map[uint16]bool)
    64  			for _, mbsu := range msm.mBlockSummary {
    65  				if tRange.CheckRangeOverLap(mbsu.LowTs, mbsu.HighTs) {
    66  					retBlocks[mbsu.Blknum] = true
    67  				}
    68  			}
    69  
    70  			// copy of tag keys map
    71  			allTagKeys := make(map[string]bool)
    72  			for tk := range msm.TagKeys {
    73  				allTagKeys[tk] = true
    74  			}
    75  			if len(retBlocks) == 0 {
    76  				return
    77  			}
    78  			finalReq := &structs.MetricsSearchRequest{
    79  				MetricsKeyBaseDir: msm.MSegmentDir,
    80  				BlocksToSearch:    retBlocks,
    81  				Parallelism:       uint(config.GetParallelism()),
    82  				QueryType:         structs.METRICS_SEARCH,
    83  				AllTagKeys:        allTagKeys,
    84  			}
    85  
    86  			retUpdate.Lock()
    87  			_, ok := retVal[msm.TTreeDir]
    88  			if !ok {
    89  				retVal[msm.TTreeDir] = make([]*structs.MetricsSearchRequest, 0)
    90  			}
    91  			retVal[msm.TTreeDir] = append(retVal[msm.TTreeDir], finalReq)
    92  			retUpdate.Unlock()
    93  
    94  			if forceLoaded {
    95  				msm.clearSearchMetadata()
    96  			}
    97  		}(mSegMeta)
    98  		if i%parallelism == 0 {
    99  			wg.Wait()
   100  		}
   101  	}
   102  	wg.Wait()
   103  	timeElapsed := time.Since(sTime)
   104  	querySummary.UpdateTimeGettingRotatedSearchRequests(timeElapsed)
   105  	return retVal, gErr
   106  }