github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/structs/segmeta.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package structs
    18  
    19  import (
    20  	"fmt"
    21  )
    22  
    23  const MAX_SEGMETA_FSIZE = 10_000_000 // 10 MB
    24  
    25  type ColSizeInfo struct {
    26  	CmiSize uint64 `json:"cmiSize"`
    27  	CsgSize uint64 `json:"csgSize"`
    28  }
    29  
    30  type VtableCounts struct {
    31  	BytesCount       uint64
    32  	RecordCount      uint64
    33  	OnDiskBytesCount uint64
    34  }
    35  
    36  type SegMeta struct {
    37  	SegmentKey         string                  `json:"segmentKey"`
    38  	EarliestEpochMS    uint64                  `json:"earliestEpochMs,omitempty"`
    39  	LatestEpochMS      uint64                  `json:"latestEpochMs,omitempty"`
    40  	SegbaseDir         string                  `json:"segbaseDir,omitempty"`
    41  	VirtualTableName   string                  `json:"virtualTableName"`
    42  	RecordCount        int                     `json:"recordCount,omitempty"`
    43  	BytesReceivedCount uint64                  `json:"bytesReceivedCount,omitempty"`
    44  	OnDiskBytes        uint64                  `json:"onDiskBytes,omitempty"`
    45  	ColumnNames        map[string]*ColSizeInfo `json:"columnNames,omitempty"`
    46  	AllPQIDs           map[string]bool         `json:"pqids,omitempty"`
    47  	NumBlocks          uint16                  `json:"numBlocks,omitempty"`
    48  	OrgId              uint64                  `json:"orgid,omitempty"`
    49  }
    50  
    51  type MetricsMeta struct {
    52  	MSegmentDir        string          `json:"mSegmentDir"`
    53  	NumBlocks          uint16          `json:"numBlocks"`
    54  	BytesReceivedCount uint64          `json:"bytesReceivedCount"`
    55  	OnDiskBytes        uint64          `json:"onDiskBytes"`
    56  	TagKeys            map[string]bool `json:"tagKeys"`
    57  	EarliestEpochSec   uint32          `json:"earliestEpochSec"`
    58  	LatestEpochSec     uint32          `json:"latestEpochSec"`
    59  	TTreeDir           string          `json:"TTreeDir"`
    60  	DatapointCount     uint64          `json:"approximateDatapointCount"`
    61  	OrgId              uint64          `json:"orgid"`
    62  }
    63  
    64  type FileType int
    65  
    66  const (
    67  	Cmi    FileType = iota // columnar micro index
    68  	Csg                    // columnar seg file
    69  	Bsu                    // block summary
    70  	Sid                    // segment info
    71  	Pqmr                   // segment persistent query matched results
    72  	Rollup                 // rollup files
    73  	Sst                    // Segment column stats files
    74  )
    75  
    76  type SegSetFile struct {
    77  	SegKey string `json:"SegKey"`
    78  	// identifier for file. If csg, xxhash of column name, if for block micro files empty string, if spqmr, then qid
    79  	Identifier string   `json:"Identifier"`
    80  	FileType   FileType `json:"FileType"`
    81  }
    82  
    83  type SegSetData struct {
    84  	InUse          bool   `json:"-"` // is this seg file being used? This will tell cleaner if it can delete it or not
    85  	AccessTime     int64  `json:"AccessTime"`
    86  	Size           uint64 `json:"Size"` // size in bytes of file
    87  	SegSetFileName string `json:"SegSetFileName"`
    88  }
    89  
    90  type SegSetDataWithStrFname struct {
    91  	Size       uint64 `json:"Size"`
    92  	IsPresent  bool   `json:"IsPresent"`
    93  	AccessTime int64  `json:"AccessTime"`
    94  	LatestTime uint64 `json:"LatestTime"`
    95  	SegSetFile string `json:"SegSetFile"`
    96  }
    97  
    98  /*
    99     ************ Algo for timestamp encoding *****************
   100     1. As records keep coming in, we add the ts (uint64) to the inProgTss array
   101  
   102     2. Once WIP is full, then instead of writing the full resultion Ts to files,
   103  	  we get the max diff (highTs-lowTs) for this wip block, and then we only store
   104  	  the difference of each Ts from the lowTs in the array.
   105  	  This way we end of using either 2 or 4 bytes per TS instead of 8.
   106  
   107     3. Based on what max diff value is we store the type of TS for this block in "CompTsType"
   108  
   109     4. We then populate the corresponding "CompRecTsXX" array storing only the diffs
   110  
   111     5. During metaloading and query checking, when incoming queryRangeTs is passed,
   112  	  we recompute the actualTs for each record by adding the blockLowTs to the diff
   113  	  of each Ts from the corresponding "CompRecTsXX" array and use it to compare.
   114  	  So basically 2 additional checks traded off for saving 4 bytes for Ts.
   115  
   116     ***********************************************************
   117  */
   118  
   119  const SIZE_OF_BSUM = 18
   120  
   121  // If new member is added to BlockSum, make sure to reset it's value in resetwipblock()
   122  type BlockSummary struct {
   123  	HighTs   uint64
   124  	LowTs    uint64
   125  	RecCount uint16
   126  }
   127  
   128  const SIZE_OF_BlockInfo = 14 // 2 + 4 + 8 bytes
   129  type BlockInfo struct {
   130  	BlkNum    uint16
   131  	BlkLen    uint32
   132  	BlkOffset int64
   133  	// if you add a element here make sure to update the SIZE_OF_BlockInfo
   134  }
   135  
   136  type HostMetaData struct {
   137  	MetaFileName string `json:"metaFileName"`
   138  }
   139  
   140  func (bs *BlockSummary) GetSize() uint64 {
   141  
   142  	totalSize := uint64(16) // highTS/lowTS
   143  	totalSize += 2          // reccount
   144  
   145  	return totalSize
   146  }
   147  
   148  // returns a deep copy of the block summary
   149  func (bs *BlockSummary) Copy() *BlockSummary {
   150  
   151  	return &BlockSummary{
   152  		HighTs:   bs.HighTs,
   153  		LowTs:    bs.LowTs,
   154  		RecCount: bs.RecCount,
   155  	}
   156  }
   157  
   158  type TS_TYPE uint8
   159  
   160  const (
   161  	TS_Type8 = iota + 1
   162  	TS_Type16
   163  	TS_Type32
   164  	TS_Type64
   165  )
   166  
   167  func GetBsuFnameFromSegKey(segkey string) string {
   168  	return fmt.Sprintf("%s.bsu", segkey)
   169  }