github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/reader/record/rrcreader_test.go (about) 1 /* 2 Copyright 2023. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package record 18 19 import ( 20 "fmt" 21 "os" 22 "testing" 23 24 localstorage "github.com/siglens/siglens/pkg/blob/local" 25 "github.com/siglens/siglens/pkg/config" 26 "github.com/siglens/siglens/pkg/segment/query/metadata" 27 "github.com/siglens/siglens/pkg/segment/structs" 28 "github.com/siglens/siglens/pkg/segment/utils" 29 "github.com/stretchr/testify/assert" 30 ) 31 32 func Test_GetJsonFromAllRrc(t *testing.T) { 33 dir := "test/" 34 config.InitializeDefaultConfig() 35 _ = localstorage.InitLocalStorage() 36 numSegKeys := 1 37 numBlocks := 1 38 numRecords := 2 39 metadata.InitMockColumnarMetadataStore(dir, numSegKeys, numBlocks, numRecords) 40 41 segkey := dir + "query_test_" + fmt.Sprint(0) 42 43 segencmap := make(map[uint16]string) 44 segencmap[uint16(0)] = segkey 45 46 allrrc := []*utils.RecordResultContainer{ 47 { 48 SegKeyInfo: utils.SegKeyInfo{ 49 SegKeyEnc: 0, 50 IsRemote: false, 51 }, 52 BlockNum: 0, 53 RecordNum: 0, 54 SortColumnValue: 0, 55 TimeStamp: 0, 56 VirtualTableName: "evts", 57 }, 58 { 59 SegKeyInfo: utils.SegKeyInfo{ 60 SegKeyEnc: 0, 61 IsRemote: false, 62 }, 63 BlockNum: 0, 64 RecordNum: 1, 65 SortColumnValue: 0, 66 TimeStamp: 1, 67 VirtualTableName: "evts", 68 }, 69 } 70 qid := uint64(0) 71 allRecords, _, err := GetJsonFromAllRrc(allrrc, false, qid, segencmap, &structs.QueryAggregators{}) 72 assert.NoError(t, err) 73 assert.Equal(t, 2, len(allRecords)) 74 75 // 11 columns + timestamp + index 76 assert.Equal(t, 14, len(allRecords[0])) 77 78 // checking decoding non random column values 79 80 indexName := "evts" 81 key0Val := "match words 123 abc" 82 key1Val := "value1" 83 key2Vals := []int64{0, 1} 84 key3Vals := []bool{true, false} 85 key6Vals := []int64{0, 2} 86 key7Val := "batch-" + fmt.Sprint(0) 87 key8Val := int64(0) 88 key10Val := segkey 89 for i := 0; i < numRecords; i++ { 90 assert.Equal(t, indexName, allRecords[i]["_index"]) 91 assert.Equal(t, key0Val, allRecords[i]["key0"]) 92 assert.Equal(t, key1Val, allRecords[i]["key1"]) 93 assert.Equal(t, key2Vals[i], allRecords[i]["key2"]) 94 assert.Equal(t, key3Vals[i], allRecords[i]["key3"]) 95 assert.Equal(t, key6Vals[i], allRecords[i]["key6"]) 96 assert.Equal(t, key7Val, allRecords[i]["key7"]) 97 assert.Equal(t, key8Val, allRecords[i]["key8"]) 98 assert.Equal(t, key10Val, allRecords[i]["key10"]) 99 assert.Contains(t, allRecords[i], "key11") 100 assert.Equal(t, uint64(i), allRecords[i][config.GetTimeStampKey()]) 101 } 102 103 os.RemoveAll(dir) 104 } 105 106 func Test_GetJsonFromAllRrc_withAggs_IncludeCols(t *testing.T) { 107 dir := "test/" 108 config.InitializeDefaultConfig() 109 _ = localstorage.InitLocalStorage() 110 numSegKeys := 1 111 numBlocks := 1 112 numRecords := 2 113 metadata.InitMockColumnarMetadataStore(dir, numSegKeys, numBlocks, numRecords) 114 115 segkey := dir + "query_test_" + fmt.Sprint(0) 116 117 segencmap := make(map[uint16]string) 118 segencmap[uint16(0)] = segkey 119 120 allrrc := []*utils.RecordResultContainer{ 121 { 122 SegKeyInfo: utils.SegKeyInfo{ 123 SegKeyEnc: 0, 124 IsRemote: false, 125 }, 126 BlockNum: 0, 127 RecordNum: 0, 128 SortColumnValue: 0, 129 TimeStamp: 0, 130 VirtualTableName: "evts", 131 }, 132 { 133 SegKeyInfo: utils.SegKeyInfo{ 134 SegKeyEnc: 0, 135 IsRemote: false, 136 }, 137 BlockNum: 0, 138 RecordNum: 1, 139 SortColumnValue: 0, 140 TimeStamp: 1, 141 VirtualTableName: "evts", 142 }, 143 } 144 qid := uint64(0) 145 aggNode := &structs.QueryAggregators{} 146 aggNode.PipeCommandType = structs.OutputTransformType 147 aggNode.OutputTransforms = &structs.OutputTransforms{} 148 aggNode.OutputTransforms.OutputColumns = &structs.ColumnsRequest{} 149 aggNode.OutputTransforms.OutputColumns.IncludeColumns = append(aggNode.OutputTransforms.OutputColumns.IncludeColumns, "key0") 150 allRecords, _, err := GetJsonFromAllRrc(allrrc, false, qid, segencmap, aggNode) 151 assert.NoError(t, err) 152 assert.Equal(t, 2, len(allRecords)) 153 154 // 11 columns + timestamp + index 155 assert.Equal(t, 3, len(allRecords[0])) 156 157 // checking decoding non random column values 158 159 indexName := "evts" 160 key0Val := "match words 123 abc" 161 162 for i := 0; i < numRecords; i++ { 163 assert.Equal(t, indexName, allRecords[i]["_index"]) 164 assert.Equal(t, key0Val, allRecords[i]["key0"]) 165 assert.Equal(t, uint64(i), allRecords[i][config.GetTimeStampKey()]) 166 } 167 os.RemoveAll(dir) 168 } 169 170 func Test_GetJsonFromAllRrc_withAggs_ExcludeCols(t *testing.T) { 171 dir := "test/" 172 config.InitializeDefaultConfig() 173 _ = localstorage.InitLocalStorage() 174 numSegKeys := 1 175 numBlocks := 1 176 numRecords := 2 177 metadata.InitMockColumnarMetadataStore(dir, numSegKeys, numBlocks, numRecords) 178 179 segkey := dir + "query_test_" + fmt.Sprint(0) 180 181 segencmap := make(map[uint16]string) 182 segencmap[uint16(0)] = segkey 183 184 allrrc := []*utils.RecordResultContainer{ 185 { 186 SegKeyInfo: utils.SegKeyInfo{ 187 SegKeyEnc: 0, 188 IsRemote: false, 189 }, 190 BlockNum: 0, 191 RecordNum: 0, 192 SortColumnValue: 0, 193 TimeStamp: 0, 194 VirtualTableName: "evts", 195 }, 196 { 197 SegKeyInfo: utils.SegKeyInfo{ 198 SegKeyEnc: 0, 199 IsRemote: false, 200 }, 201 BlockNum: 0, 202 RecordNum: 1, 203 SortColumnValue: 0, 204 TimeStamp: 1, 205 VirtualTableName: "evts", 206 }, 207 } 208 qid := uint64(0) 209 aggNode := &structs.QueryAggregators{} 210 aggNode.PipeCommandType = structs.OutputTransformType 211 aggNode.OutputTransforms = &structs.OutputTransforms{} 212 aggNode.OutputTransforms.OutputColumns = &structs.ColumnsRequest{} 213 aggNode.OutputTransforms.OutputColumns.ExcludeColumns = append(aggNode.OutputTransforms.OutputColumns.ExcludeColumns, "key0") 214 allRecords, _, err := GetJsonFromAllRrc(allrrc, false, qid, segencmap, aggNode) 215 assert.NoError(t, err) 216 assert.Equal(t, 2, len(allRecords)) 217 218 // 11 columns + timestamp + index 219 assert.Equal(t, 13, len(allRecords[0])) 220 221 // checking decoding non random column values 222 223 indexName := "evts" 224 // key0Val := "match words 123 abc" 225 key1Val := "value1" 226 key2Vals := []int64{0, 1} 227 key3Vals := []bool{true, false} 228 key6Vals := []int64{0, 2} 229 key7Val := "batch-" + fmt.Sprint(0) 230 key8Val := int64(0) 231 key10Val := segkey 232 for i := 0; i < numRecords; i++ { 233 assert.Equal(t, indexName, allRecords[i]["_index"]) 234 // assert.Equal(t, key0Val, allRecords[i]["key0"]) 235 assert.Equal(t, key1Val, allRecords[i]["key1"]) 236 assert.Equal(t, key2Vals[i], allRecords[i]["key2"]) // we only encode floats 237 assert.Equal(t, key3Vals[i], allRecords[i]["key3"]) 238 assert.Equal(t, key6Vals[i], allRecords[i]["key6"]) // we only encode floats 239 assert.Equal(t, key7Val, allRecords[i]["key7"]) 240 assert.Equal(t, key8Val, allRecords[i]["key8"]) 241 assert.Equal(t, key10Val, allRecords[i]["key10"]) 242 assert.Contains(t, allRecords[i], "key11") 243 assert.Equal(t, uint64(i), allRecords[i][config.GetTimeStampKey()]) 244 } 245 246 os.RemoveAll(dir) 247 }