github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/util_test.go (about) 1 package storage 2 3 import ( 4 "context" 5 "sort" 6 "testing" 7 "time" 8 9 "github.com/davecgh/go-spew/spew" 10 "github.com/prometheus/common/model" 11 "github.com/prometheus/prometheus/model/labels" 12 "github.com/stretchr/testify/assert" 13 14 "github.com/grafana/loki/pkg/chunkenc" 15 "github.com/grafana/loki/pkg/ingester/client" 16 "github.com/grafana/loki/pkg/logproto" 17 "github.com/grafana/loki/pkg/logql/syntax" 18 "github.com/grafana/loki/pkg/logqlmodel/stats" 19 "github.com/grafana/loki/pkg/querier/astmapper" 20 "github.com/grafana/loki/pkg/storage/chunk" 21 "github.com/grafana/loki/pkg/storage/chunk/cache" 22 chunkclient "github.com/grafana/loki/pkg/storage/chunk/client" 23 "github.com/grafana/loki/pkg/storage/chunk/fetcher" 24 "github.com/grafana/loki/pkg/storage/config" 25 "github.com/grafana/loki/pkg/storage/stores" 26 index_stats "github.com/grafana/loki/pkg/storage/stores/index/stats" 27 loki_util "github.com/grafana/loki/pkg/util" 28 util_log "github.com/grafana/loki/pkg/util/log" 29 ) 30 31 var ( 32 fooLabelsWithName = labels.Labels{{Name: "foo", Value: "bar"}, {Name: "__name__", Value: "logs"}} 33 fooLabels = labels.Labels{{Name: "foo", Value: "bar"}} 34 ) 35 36 var from = time.Unix(0, time.Millisecond.Nanoseconds()) 37 38 func assertStream(t *testing.T, expected, actual []logproto.Stream) { 39 if len(expected) != len(actual) { 40 t.Fatalf("error stream length are different expected %d actual %d\n%s", len(expected), len(actual), spew.Sdump(expected, actual)) 41 return 42 } 43 sort.Slice(expected, func(i int, j int) bool { return expected[i].Labels < expected[j].Labels }) 44 sort.Slice(actual, func(i int, j int) bool { return actual[i].Labels < actual[j].Labels }) 45 for i := range expected { 46 assert.Equal(t, expected[i].Labels, actual[i].Labels) 47 if len(expected[i].Entries) != len(actual[i].Entries) { 48 t.Fatalf("error entries length are different expected %d actual %d\n%s", len(expected[i].Entries), len(actual[i].Entries), spew.Sdump(expected[i].Entries, actual[i].Entries)) 49 50 return 51 } 52 for j := range expected[i].Entries { 53 assert.Equal(t, expected[i].Entries[j].Timestamp.UnixNano(), actual[i].Entries[j].Timestamp.UnixNano()) 54 assert.Equal(t, expected[i].Entries[j].Line, actual[i].Entries[j].Line) 55 } 56 } 57 } 58 59 func assertSeries(t *testing.T, expected, actual []logproto.Series) { 60 if len(expected) != len(actual) { 61 t.Fatalf("error stream length are different expected %d actual %d\n%s", len(expected), len(actual), spew.Sdump(expected, actual)) 62 return 63 } 64 sort.Slice(expected, func(i int, j int) bool { return expected[i].Labels < expected[j].Labels }) 65 sort.Slice(actual, func(i int, j int) bool { return actual[i].Labels < actual[j].Labels }) 66 for i := range expected { 67 assert.Equal(t, expected[i].Labels, actual[i].Labels) 68 if len(expected[i].Samples) != len(actual[i].Samples) { 69 t.Fatalf("error entries length are different expected %d actual%d\n%s", len(expected[i].Samples), len(actual[i].Samples), spew.Sdump(expected[i].Samples, actual[i].Samples)) 70 71 return 72 } 73 for j := range expected[i].Samples { 74 assert.Equal(t, expected[i].Samples[j].Timestamp, actual[i].Samples[j].Timestamp) 75 assert.Equal(t, expected[i].Samples[j].Value, actual[i].Samples[j].Value) 76 assert.Equal(t, expected[i].Samples[j].Hash, actual[i].Samples[j].Hash) 77 } 78 } 79 } 80 81 func newLazyChunk(stream logproto.Stream) *LazyChunk { 82 return &LazyChunk{ 83 Fetcher: nil, 84 IsValid: true, 85 Chunk: newChunk(stream), 86 } 87 } 88 89 func newLazyInvalidChunk(stream logproto.Stream) *LazyChunk { 90 return &LazyChunk{ 91 Fetcher: nil, 92 IsValid: false, 93 Chunk: newChunk(stream), 94 } 95 } 96 97 func newChunk(stream logproto.Stream) chunk.Chunk { 98 lbs, err := syntax.ParseLabels(stream.Labels) 99 if err != nil { 100 panic(err) 101 } 102 if !lbs.Has(labels.MetricName) { 103 builder := labels.NewBuilder(lbs) 104 builder.Set(labels.MetricName, "logs") 105 lbs = builder.Labels() 106 } 107 from, through := loki_util.RoundToMilliseconds(stream.Entries[0].Timestamp, stream.Entries[len(stream.Entries)-1].Timestamp) 108 chk := chunkenc.NewMemChunk(chunkenc.EncGZIP, chunkenc.UnorderedHeadBlockFmt, 256*1024, 0) 109 for _, e := range stream.Entries { 110 _ = chk.Append(&e) 111 } 112 chk.Close() 113 c := chunk.NewChunk("fake", client.Fingerprint(lbs), lbs, chunkenc.NewFacade(chk, 0, 0), from, through) 114 // force the checksum creation 115 if err := c.Encode(); err != nil { 116 panic(err) 117 } 118 return c 119 } 120 121 func newMatchers(matchers string) []*labels.Matcher { 122 res, err := syntax.ParseMatchers(matchers) 123 if err != nil { 124 panic(err) 125 } 126 return res 127 } 128 129 func newQuery(query string, start, end time.Time, shards []astmapper.ShardAnnotation, deletes []*logproto.Delete) *logproto.QueryRequest { 130 req := &logproto.QueryRequest{ 131 Selector: query, 132 Start: start, 133 Limit: 1000, 134 End: end, 135 Direction: logproto.FORWARD, 136 Deletes: deletes, 137 } 138 for _, shard := range shards { 139 req.Shards = append(req.Shards, shard.String()) 140 } 141 return req 142 } 143 144 func newSampleQuery(query string, start, end time.Time, deletes []*logproto.Delete) *logproto.SampleQueryRequest { 145 req := &logproto.SampleQueryRequest{ 146 Selector: query, 147 Start: start, 148 End: end, 149 Deletes: deletes, 150 } 151 return req 152 } 153 154 type mockChunkStore struct { 155 schemas config.SchemaConfig 156 chunks []chunk.Chunk 157 client *mockChunkStoreClient 158 f chunk.RequestChunkFilterer 159 } 160 161 // mockChunkStore cannot implement both chunk.Store and chunk.Client, 162 // since there is a conflict in signature for DeleteChunk method. 163 var ( 164 _ stores.Store = &mockChunkStore{} 165 _ chunkclient.Client = &mockChunkStoreClient{} 166 ) 167 168 func newMockChunkStore(streams []*logproto.Stream) *mockChunkStore { 169 chunks := make([]chunk.Chunk, 0, len(streams)) 170 for _, s := range streams { 171 chunks = append(chunks, newChunk(*s)) 172 } 173 return &mockChunkStore{schemas: config.SchemaConfig{}, chunks: chunks, client: &mockChunkStoreClient{chunks: chunks, scfg: config.SchemaConfig{}}} 174 } 175 176 func (m *mockChunkStore) Put(ctx context.Context, chunks []chunk.Chunk) error { return nil } 177 func (m *mockChunkStore) PutOne(ctx context.Context, from, through model.Time, chunk chunk.Chunk) error { 178 return nil 179 } 180 181 func (m *mockChunkStore) GetSeries(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([]labels.Labels, error) { 182 result := make([]labels.Labels, 0, len(m.chunks)) 183 unique := map[uint64]struct{}{} 184 Outer: 185 for _, c := range m.chunks { 186 if _, ok := unique[c.Fingerprint]; !ok { 187 for _, m := range matchers { 188 if !m.Matches(c.Metric.Get(m.Name)) { 189 continue Outer 190 } 191 } 192 l := labels.NewBuilder(c.Metric).Del(labels.MetricName).Labels() 193 if m.f != nil { 194 if m.f.ForRequest(ctx).ShouldFilter(l) { 195 continue 196 } 197 } 198 199 result = append(result, l) 200 unique[c.Fingerprint] = struct{}{} 201 } 202 } 203 sort.Slice(result, func(i, j int) bool { return labels.Compare(result[i], result[j]) < 0 }) 204 return result, nil 205 } 206 207 func (m *mockChunkStore) LabelValuesForMetricName(ctx context.Context, userID string, from, through model.Time, metricName string, labelName string, matchers ...*labels.Matcher) ([]string, error) { 208 return nil, nil 209 } 210 211 func (m *mockChunkStore) LabelNamesForMetricName(ctx context.Context, userID string, from, through model.Time, metricName string) ([]string, error) { 212 return nil, nil 213 } 214 215 func (m *mockChunkStore) SetChunkFilterer(f chunk.RequestChunkFilterer) { 216 m.f = f 217 } 218 219 func (m *mockChunkStore) DeleteChunk(ctx context.Context, from, through model.Time, userID, chunkID string, metric labels.Labels, partiallyDeletedInterval *model.Interval) error { 220 return nil 221 } 222 223 func (m *mockChunkStore) DeleteSeriesIDs(ctx context.Context, from, through model.Time, userID string, metric labels.Labels) error { 224 return nil 225 } 226 func (m *mockChunkStore) Stop() {} 227 func (m *mockChunkStore) Get(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([]chunk.Chunk, error) { 228 return nil, nil 229 } 230 231 func (m *mockChunkStore) GetChunkFetcher(_ model.Time) *fetcher.Fetcher { 232 return nil 233 } 234 235 func (m *mockChunkStore) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([][]chunk.Chunk, []*fetcher.Fetcher, error) { 236 refs := make([]chunk.Chunk, 0, len(m.chunks)) 237 // transform real chunks into ref chunks. 238 for _, c := range m.chunks { 239 r, err := chunk.ParseExternalKey("fake", m.schemas.ExternalKey(c.ChunkRef)) 240 if err != nil { 241 panic(err) 242 } 243 refs = append(refs, r) 244 } 245 246 cache, err := cache.New(cache.Config{Prefix: "chunks"}, nil, util_log.Logger, stats.ChunkCache) 247 if err != nil { 248 panic(err) 249 } 250 251 f, err := fetcher.New(cache, false, m.schemas, m.client, 10, 100) 252 if err != nil { 253 panic(err) 254 } 255 return [][]chunk.Chunk{refs}, []*fetcher.Fetcher{f}, nil 256 } 257 258 func (m *mockChunkStore) Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*index_stats.Stats, error) { 259 return nil, nil 260 } 261 262 type mockChunkStoreClient struct { 263 chunks []chunk.Chunk 264 scfg config.SchemaConfig 265 } 266 267 func (m mockChunkStoreClient) Stop() { 268 panic("implement me") 269 } 270 271 func (m mockChunkStoreClient) PutChunks(ctx context.Context, chunks []chunk.Chunk) error { 272 return nil 273 } 274 275 func (m mockChunkStoreClient) GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) { 276 var res []chunk.Chunk 277 for _, c := range chunks { 278 for _, sc := range m.chunks { 279 // only returns chunks requested using the external key 280 if m.scfg.ExternalKey(c.ChunkRef) == m.scfg.ExternalKey(sc.ChunkRef) { 281 res = append(res, sc) 282 } 283 } 284 } 285 return res, nil 286 } 287 288 func (m mockChunkStoreClient) DeleteChunk(ctx context.Context, userID, chunkID string) error { 289 return nil 290 } 291 292 func (m mockChunkStoreClient) IsChunkNotFoundErr(_ error) bool { 293 return false 294 } 295 296 var streamsFixture = []*logproto.Stream{ 297 { 298 Labels: "{foo=\"bar\"}", 299 Entries: []logproto.Entry{ 300 { 301 Timestamp: from, 302 Line: "1", 303 }, 304 305 { 306 Timestamp: from.Add(time.Millisecond), 307 Line: "2", 308 }, 309 { 310 Timestamp: from.Add(2 * time.Millisecond), 311 Line: "3", 312 }, 313 }, 314 }, 315 { 316 Labels: "{foo=\"bar\"}", 317 Entries: []logproto.Entry{ 318 { 319 Timestamp: from.Add(2 * time.Millisecond), 320 Line: "3", 321 }, 322 { 323 Timestamp: from.Add(3 * time.Millisecond), 324 Line: "4", 325 }, 326 327 { 328 Timestamp: from.Add(4 * time.Millisecond), 329 Line: "5", 330 }, 331 { 332 Timestamp: from.Add(5 * time.Millisecond), 333 Line: "6", 334 }, 335 }, 336 }, 337 { 338 Labels: "{foo=\"bazz\"}", 339 Entries: []logproto.Entry{ 340 { 341 Timestamp: from, 342 Line: "1", 343 }, 344 345 { 346 Timestamp: from.Add(time.Millisecond), 347 Line: "2", 348 }, 349 { 350 Timestamp: from.Add(2 * time.Millisecond), 351 Line: "3", 352 }, 353 }, 354 }, 355 { 356 Labels: "{foo=\"bazz\"}", 357 Entries: []logproto.Entry{ 358 { 359 Timestamp: from.Add(2 * time.Millisecond), 360 Line: "3", 361 }, 362 { 363 Timestamp: from.Add(3 * time.Millisecond), 364 Line: "4", 365 }, 366 367 { 368 Timestamp: from.Add(4 * time.Millisecond), 369 Line: "5", 370 }, 371 { 372 Timestamp: from.Add(5 * time.Millisecond), 373 Line: "6", 374 }, 375 }, 376 }, 377 } 378 var storeFixture = newMockChunkStore(streamsFixture)