github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/batch_test.go (about)

     1  package storage
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/cespare/xxhash/v2"
    10  	"github.com/pkg/errors"
    11  	"github.com/prometheus/prometheus/model/labels"
    12  	"github.com/prometheus/prometheus/promql"
    13  	"github.com/stretchr/testify/require"
    14  	"github.com/weaveworks/common/user"
    15  
    16  	"github.com/grafana/loki/pkg/chunkenc"
    17  	"github.com/grafana/loki/pkg/iter"
    18  	"github.com/grafana/loki/pkg/logproto"
    19  	"github.com/grafana/loki/pkg/logql"
    20  	"github.com/grafana/loki/pkg/logql/log"
    21  	"github.com/grafana/loki/pkg/logqlmodel/stats"
    22  	"github.com/grafana/loki/pkg/storage/config"
    23  )
    24  
    25  var NilMetrics = NewChunkMetrics(nil, 0)
    26  
    27  func Test_batchIterSafeStart(t *testing.T) {
    28  	stream := logproto.Stream{
    29  		Labels: fooLabelsWithName.String(),
    30  		Entries: []logproto.Entry{
    31  			{
    32  				Timestamp: from,
    33  				Line:      "1",
    34  			},
    35  			{
    36  				Timestamp: from.Add(time.Millisecond),
    37  				Line:      "2",
    38  			},
    39  		},
    40  	}
    41  	chks := []*LazyChunk{
    42  		newLazyChunk(stream),
    43  	}
    44  
    45  	s := config.SchemaConfig{
    46  		Configs: []config.PeriodConfig{
    47  			{
    48  				From:      config.DayTime{Time: 0},
    49  				Schema:    "v11",
    50  				RowShards: 16,
    51  			},
    52  		},
    53  	}
    54  
    55  	batch := newBatchChunkIterator(context.Background(), s, chks, 1, logproto.FORWARD, from, from.Add(4*time.Millisecond), NilMetrics, []*labels.Matcher{}, nil)
    56  
    57  	// if it was started already, we should see a panic before this
    58  	time.Sleep(time.Millisecond)
    59  
    60  	// ensure idempotency
    61  	batch.Start()
    62  	batch.Start()
    63  
    64  	require.NotNil(t, batch.Next())
    65  }
    66  
    67  func Test_newLogBatchChunkIterator(t *testing.T) {
    68  	tests := map[string]struct {
    69  		chunks     []*LazyChunk
    70  		expected   []logproto.Stream
    71  		matchers   string
    72  		start, end time.Time
    73  		direction  logproto.Direction
    74  		batchSize  int
    75  	}{
    76  		"forward with overlap": {
    77  			[]*LazyChunk{
    78  				newLazyChunk(logproto.Stream{
    79  					Labels: fooLabelsWithName.String(),
    80  					Entries: []logproto.Entry{
    81  						{
    82  							Timestamp: from,
    83  							Line:      "1",
    84  						},
    85  						{
    86  							Timestamp: from.Add(time.Millisecond),
    87  							Line:      "2",
    88  						},
    89  					},
    90  				}),
    91  				newLazyChunk(logproto.Stream{
    92  					Labels: fooLabelsWithName.String(),
    93  					Entries: []logproto.Entry{
    94  						{
    95  							Timestamp: from.Add(time.Millisecond),
    96  							Line:      "2",
    97  						},
    98  						{
    99  							Timestamp: from.Add(2 * time.Millisecond),
   100  							Line:      "3",
   101  						},
   102  					},
   103  				}),
   104  				newLazyChunk(logproto.Stream{
   105  					Labels: fooLabelsWithName.String(),
   106  					Entries: []logproto.Entry{
   107  						{
   108  							Timestamp: from.Add(time.Millisecond),
   109  							Line:      "2",
   110  						},
   111  						{
   112  							Timestamp: from.Add(2 * time.Millisecond),
   113  							Line:      "3",
   114  						},
   115  					},
   116  				}),
   117  				newLazyChunk(logproto.Stream{
   118  					Labels: fooLabelsWithName.String(),
   119  					Entries: []logproto.Entry{
   120  						{
   121  							Timestamp: from.Add(2 * time.Millisecond),
   122  							Line:      "3",
   123  						},
   124  						{
   125  							Timestamp: from.Add(3 * time.Millisecond),
   126  							Line:      "4",
   127  						},
   128  					},
   129  				}),
   130  				newLazyChunk(logproto.Stream{
   131  					Labels: fooLabelsWithName.String(),
   132  					Entries: []logproto.Entry{
   133  						{
   134  							Timestamp: from.Add(2 * time.Millisecond),
   135  							Line:      "3",
   136  						},
   137  						{
   138  							Timestamp: from.Add(3 * time.Millisecond),
   139  							Line:      "4",
   140  						},
   141  					},
   142  				}),
   143  				newLazyChunk(logproto.Stream{
   144  					Labels: fooLabelsWithName.String(),
   145  					Entries: []logproto.Entry{
   146  						{
   147  							Timestamp: from.Add(3 * time.Millisecond),
   148  							Line:      "4",
   149  						},
   150  						{
   151  							Timestamp: from.Add(4 * time.Millisecond),
   152  							Line:      "5",
   153  						},
   154  					},
   155  				}),
   156  			},
   157  			[]logproto.Stream{
   158  				{
   159  					Labels: fooLabels.String(),
   160  					Entries: []logproto.Entry{
   161  						{
   162  							Timestamp: from,
   163  							Line:      "1",
   164  						},
   165  						{
   166  							Timestamp: from.Add(time.Millisecond),
   167  							Line:      "2",
   168  						},
   169  						{
   170  							Timestamp: from.Add(2 * time.Millisecond),
   171  							Line:      "3",
   172  						},
   173  						{
   174  							Timestamp: from.Add(3 * time.Millisecond),
   175  							Line:      "4",
   176  						},
   177  					},
   178  				},
   179  			},
   180  			fooLabelsWithName.String(),
   181  			from, from.Add(4 * time.Millisecond),
   182  			logproto.FORWARD,
   183  			2,
   184  		},
   185  		"forward all overlap and all chunks have a from time less than query from time": {
   186  			[]*LazyChunk{
   187  				newLazyChunk(logproto.Stream{
   188  					Labels: fooLabelsWithName.String(),
   189  					Entries: []logproto.Entry{
   190  						{
   191  							Timestamp: from,
   192  							Line:      "1",
   193  						},
   194  						{
   195  							Timestamp: from.Add(time.Millisecond),
   196  							Line:      "2",
   197  						},
   198  					},
   199  				}),
   200  				newLazyChunk(logproto.Stream{
   201  					Labels: fooLabelsWithName.String(),
   202  					Entries: []logproto.Entry{
   203  						{
   204  							Timestamp: from,
   205  							Line:      "1",
   206  						},
   207  						{
   208  							Timestamp: from.Add(time.Millisecond),
   209  							Line:      "2",
   210  						},
   211  						{
   212  							Timestamp: from.Add(2 * time.Millisecond),
   213  							Line:      "3",
   214  						},
   215  					},
   216  				}),
   217  				newLazyChunk(logproto.Stream{
   218  					Labels: fooLabelsWithName.String(),
   219  					Entries: []logproto.Entry{
   220  						{
   221  							Timestamp: from,
   222  							Line:      "1",
   223  						},
   224  						{
   225  							Timestamp: from.Add(time.Millisecond),
   226  							Line:      "2",
   227  						},
   228  						{
   229  							Timestamp: from.Add(2 * time.Millisecond),
   230  							Line:      "3",
   231  						},
   232  					},
   233  				}),
   234  				newLazyChunk(logproto.Stream{
   235  					Labels: fooLabelsWithName.String(),
   236  					Entries: []logproto.Entry{
   237  						{
   238  							Timestamp: from,
   239  							Line:      "1",
   240  						},
   241  						{
   242  							Timestamp: from.Add(2 * time.Millisecond),
   243  							Line:      "3",
   244  						},
   245  						{
   246  							Timestamp: from.Add(3 * time.Millisecond),
   247  							Line:      "4",
   248  						},
   249  					},
   250  				}),
   251  				newLazyChunk(logproto.Stream{
   252  					Labels: fooLabelsWithName.String(),
   253  					Entries: []logproto.Entry{
   254  						{
   255  							Timestamp: from,
   256  							Line:      "1",
   257  						},
   258  						{
   259  							Timestamp: from.Add(2 * time.Millisecond),
   260  							Line:      "3",
   261  						},
   262  						{
   263  							Timestamp: from.Add(3 * time.Millisecond),
   264  							Line:      "4",
   265  						},
   266  					},
   267  				}),
   268  				newLazyChunk(logproto.Stream{
   269  					Labels: fooLabelsWithName.String(),
   270  					Entries: []logproto.Entry{
   271  						{
   272  							Timestamp: from,
   273  							Line:      "1",
   274  						},
   275  						{
   276  							Timestamp: from.Add(3 * time.Millisecond),
   277  							Line:      "4",
   278  						},
   279  						{
   280  							Timestamp: from.Add(4 * time.Millisecond),
   281  							Line:      "5",
   282  						},
   283  					},
   284  				}),
   285  			},
   286  			[]logproto.Stream{
   287  				{
   288  					Labels: fooLabels.String(),
   289  					Entries: []logproto.Entry{
   290  						{
   291  							Timestamp: from.Add(time.Millisecond),
   292  							Line:      "2",
   293  						},
   294  						{
   295  							Timestamp: from.Add(2 * time.Millisecond),
   296  							Line:      "3",
   297  						},
   298  						{
   299  							Timestamp: from.Add(3 * time.Millisecond),
   300  							Line:      "4",
   301  						},
   302  						{
   303  							Timestamp: from.Add(4 * time.Millisecond),
   304  							Line:      "5",
   305  						},
   306  					},
   307  				},
   308  			},
   309  			fooLabelsWithName.String(),
   310  			from.Add(1 * time.Millisecond), from.Add(5 * time.Millisecond),
   311  			logproto.FORWARD,
   312  			2,
   313  		},
   314  		"forward with overlapping non-continuous entries": {
   315  			[]*LazyChunk{
   316  				newLazyChunk(logproto.Stream{
   317  					Labels: fooLabelsWithName.String(),
   318  					Entries: []logproto.Entry{
   319  						{
   320  							Timestamp: from,
   321  							Line:      "1",
   322  						},
   323  						{
   324  							Timestamp: from.Add(time.Millisecond),
   325  							Line:      "2",
   326  						},
   327  						{
   328  							Timestamp: from.Add(3 * time.Millisecond),
   329  							Line:      "4",
   330  						},
   331  					},
   332  				}),
   333  				newLazyChunk(logproto.Stream{
   334  					Labels: fooLabelsWithName.String(),
   335  					Entries: []logproto.Entry{
   336  						{
   337  							Timestamp: from.Add(time.Millisecond),
   338  							Line:      "2",
   339  						},
   340  						{
   341  							Timestamp: from.Add(2 * time.Millisecond),
   342  							Line:      "3",
   343  						},
   344  					},
   345  				}),
   346  				newLazyChunk(logproto.Stream{
   347  					Labels: fooLabelsWithName.String(),
   348  					Entries: []logproto.Entry{
   349  						{
   350  							Timestamp: from.Add(time.Millisecond),
   351  							Line:      "2",
   352  						},
   353  						{
   354  							Timestamp: from.Add(3 * time.Millisecond),
   355  							Line:      "4",
   356  						},
   357  					},
   358  				}),
   359  				newLazyChunk(logproto.Stream{
   360  					Labels: fooLabelsWithName.String(),
   361  					Entries: []logproto.Entry{
   362  						{
   363  							Timestamp: from.Add(2 * time.Millisecond),
   364  							Line:      "3",
   365  						},
   366  						{
   367  							Timestamp: from.Add(3 * time.Millisecond),
   368  							Line:      "4",
   369  						},
   370  					},
   371  				}),
   372  			},
   373  			[]logproto.Stream{
   374  				{
   375  					Labels: fooLabels.String(),
   376  					Entries: []logproto.Entry{
   377  						{
   378  							Timestamp: from,
   379  							Line:      "1",
   380  						},
   381  						{
   382  							Timestamp: from.Add(time.Millisecond),
   383  							Line:      "2",
   384  						},
   385  						{
   386  							Timestamp: from.Add(2 * time.Millisecond),
   387  							Line:      "3",
   388  						},
   389  					},
   390  				},
   391  			},
   392  			fooLabelsWithName.String(),
   393  			from, from.Add(3 * time.Millisecond),
   394  			logproto.FORWARD,
   395  			2,
   396  		},
   397  		"backward with overlap": {
   398  			[]*LazyChunk{
   399  				newLazyChunk(logproto.Stream{
   400  					Labels: fooLabelsWithName.String(),
   401  					Entries: []logproto.Entry{
   402  						{
   403  							Timestamp: from,
   404  							Line:      "1",
   405  						},
   406  						{
   407  							Timestamp: from.Add(time.Millisecond),
   408  							Line:      "2",
   409  						},
   410  					},
   411  				}),
   412  				newLazyChunk(logproto.Stream{
   413  					Labels: fooLabelsWithName.String(),
   414  					Entries: []logproto.Entry{
   415  						{
   416  							Timestamp: from.Add(time.Millisecond),
   417  							Line:      "2",
   418  						},
   419  						{
   420  							Timestamp: from.Add(2 * time.Millisecond),
   421  							Line:      "3",
   422  						},
   423  					},
   424  				}),
   425  				newLazyChunk(logproto.Stream{
   426  					Labels: fooLabelsWithName.String(),
   427  					Entries: []logproto.Entry{
   428  						{
   429  							Timestamp: from.Add(time.Millisecond),
   430  							Line:      "2",
   431  						},
   432  						{
   433  							Timestamp: from.Add(2 * time.Millisecond),
   434  							Line:      "3",
   435  						},
   436  					},
   437  				}),
   438  				newLazyChunk(logproto.Stream{
   439  					Labels: fooLabelsWithName.String(),
   440  					Entries: []logproto.Entry{
   441  						{
   442  							Timestamp: from.Add(2 * time.Millisecond),
   443  							Line:      "3",
   444  						},
   445  						{
   446  							Timestamp: from.Add(3 * time.Millisecond),
   447  							Line:      "4",
   448  						},
   449  					},
   450  				}),
   451  				newLazyChunk(logproto.Stream{
   452  					Labels: fooLabelsWithName.String(),
   453  					Entries: []logproto.Entry{
   454  						{
   455  							Timestamp: from.Add(2 * time.Millisecond),
   456  							Line:      "3",
   457  						},
   458  						{
   459  							Timestamp: from.Add(3 * time.Millisecond),
   460  							Line:      "4",
   461  						},
   462  					},
   463  				}),
   464  				newLazyChunk(logproto.Stream{
   465  					Labels: fooLabelsWithName.String(),
   466  					Entries: []logproto.Entry{
   467  						{
   468  							Timestamp: from.Add(3 * time.Millisecond),
   469  							Line:      "4",
   470  						},
   471  						{
   472  							Timestamp: from.Add(4 * time.Millisecond),
   473  							Line:      "5",
   474  						},
   475  					},
   476  				}),
   477  			},
   478  			[]logproto.Stream{
   479  				{
   480  					Labels: fooLabels.String(),
   481  					Entries: []logproto.Entry{
   482  						{
   483  							Timestamp: from.Add(3 * time.Millisecond),
   484  							Line:      "4",
   485  						},
   486  						{
   487  							Timestamp: from.Add(2 * time.Millisecond),
   488  							Line:      "3",
   489  						},
   490  						{
   491  							Timestamp: from.Add(time.Millisecond),
   492  							Line:      "2",
   493  						},
   494  						{
   495  							Timestamp: from,
   496  							Line:      "1",
   497  						},
   498  					},
   499  				},
   500  			},
   501  			fooLabelsWithName.String(),
   502  			from, from.Add(4 * time.Millisecond),
   503  			logproto.BACKWARD,
   504  			2,
   505  		},
   506  		"backward all overlap and all chunks have a through time greater than query through time": {
   507  			[]*LazyChunk{
   508  				newLazyChunk(logproto.Stream{
   509  					Labels: fooLabelsWithName.String(),
   510  					Entries: []logproto.Entry{
   511  						{
   512  							Timestamp: from,
   513  							Line:      "1",
   514  						},
   515  						{
   516  							Timestamp: from.Add(time.Millisecond),
   517  							Line:      "2",
   518  						},
   519  						{
   520  							Timestamp: from.Add(4 * time.Millisecond),
   521  							Line:      "5",
   522  						},
   523  					},
   524  				}),
   525  				newLazyChunk(logproto.Stream{
   526  					Labels: fooLabelsWithName.String(),
   527  					Entries: []logproto.Entry{
   528  						{
   529  							Timestamp: from.Add(time.Millisecond),
   530  							Line:      "2",
   531  						},
   532  						{
   533  							Timestamp: from.Add(2 * time.Millisecond),
   534  							Line:      "3",
   535  						},
   536  						{
   537  							Timestamp: from.Add(4 * time.Millisecond),
   538  							Line:      "5",
   539  						},
   540  					},
   541  				}),
   542  				newLazyChunk(logproto.Stream{
   543  					Labels: fooLabelsWithName.String(),
   544  					Entries: []logproto.Entry{
   545  						{
   546  							Timestamp: from.Add(time.Millisecond),
   547  							Line:      "2",
   548  						},
   549  						{
   550  							Timestamp: from.Add(2 * time.Millisecond),
   551  							Line:      "3",
   552  						},
   553  						{
   554  							Timestamp: from.Add(4 * time.Millisecond),
   555  							Line:      "5",
   556  						},
   557  					},
   558  				}),
   559  				newLazyChunk(logproto.Stream{
   560  					Labels: fooLabelsWithName.String(),
   561  					Entries: []logproto.Entry{
   562  						{
   563  							Timestamp: from.Add(2 * time.Millisecond),
   564  							Line:      "3",
   565  						},
   566  						{
   567  							Timestamp: from.Add(3 * time.Millisecond),
   568  							Line:      "4",
   569  						},
   570  						{
   571  							Timestamp: from.Add(4 * time.Millisecond),
   572  							Line:      "5",
   573  						},
   574  					},
   575  				}),
   576  				newLazyChunk(logproto.Stream{
   577  					Labels: fooLabelsWithName.String(),
   578  					Entries: []logproto.Entry{
   579  						{
   580  							Timestamp: from.Add(2 * time.Millisecond),
   581  							Line:      "3",
   582  						},
   583  						{
   584  							Timestamp: from.Add(3 * time.Millisecond),
   585  							Line:      "4",
   586  						},
   587  						{
   588  							Timestamp: from.Add(4 * time.Millisecond),
   589  							Line:      "5",
   590  						},
   591  					},
   592  				}),
   593  				newLazyChunk(logproto.Stream{
   594  					Labels: fooLabelsWithName.String(),
   595  					Entries: []logproto.Entry{
   596  						{
   597  							Timestamp: from.Add(3 * time.Millisecond),
   598  							Line:      "4",
   599  						},
   600  						{
   601  							Timestamp: from.Add(4 * time.Millisecond),
   602  							Line:      "5",
   603  						},
   604  					},
   605  				}),
   606  			},
   607  			[]logproto.Stream{
   608  				{
   609  					Labels: fooLabels.String(),
   610  					Entries: []logproto.Entry{
   611  						{
   612  							Timestamp: from.Add(3 * time.Millisecond),
   613  							Line:      "4",
   614  						},
   615  						{
   616  							Timestamp: from.Add(2 * time.Millisecond),
   617  							Line:      "3",
   618  						},
   619  						{
   620  							Timestamp: from.Add(time.Millisecond),
   621  							Line:      "2",
   622  						},
   623  						{
   624  							Timestamp: from,
   625  							Line:      "1",
   626  						},
   627  					},
   628  				},
   629  			},
   630  			fooLabelsWithName.String(),
   631  			from, from.Add(4 * time.Millisecond),
   632  			logproto.BACKWARD,
   633  			2,
   634  		},
   635  		"backward with overlapping non-continuous entries": {
   636  			[]*LazyChunk{
   637  				newLazyChunk(logproto.Stream{
   638  					Labels: fooLabelsWithName.String(),
   639  					Entries: []logproto.Entry{
   640  						{
   641  							Timestamp: from.Add(0 * time.Millisecond),
   642  							Line:      "0",
   643  						},
   644  						{
   645  							Timestamp: from.Add(3 * time.Millisecond),
   646  							Line:      "3",
   647  						},
   648  					},
   649  				}),
   650  				newLazyChunk(logproto.Stream{
   651  					Labels: fooLabelsWithName.String(),
   652  					Entries: []logproto.Entry{
   653  						{
   654  							Timestamp: from.Add(1 * time.Millisecond),
   655  							Line:      "1",
   656  						},
   657  						{
   658  							Timestamp: from.Add(6 * time.Millisecond),
   659  							Line:      "6",
   660  						},
   661  					},
   662  				}),
   663  				newLazyChunk(logproto.Stream{
   664  					Labels: fooLabelsWithName.String(),
   665  					Entries: []logproto.Entry{
   666  						{
   667  							Timestamp: from.Add(2 * time.Millisecond),
   668  							Line:      "2",
   669  						},
   670  						{
   671  							Timestamp: from.Add(5 * time.Millisecond),
   672  							Line:      "5",
   673  						},
   674  					},
   675  				}),
   676  				newLazyChunk(logproto.Stream{
   677  					Labels: fooLabelsWithName.String(),
   678  					Entries: []logproto.Entry{
   679  						{
   680  							Timestamp: from.Add(4 * time.Millisecond),
   681  							Line:      "4",
   682  						},
   683  						{
   684  							Timestamp: from.Add(7 * time.Millisecond),
   685  							Line:      "7",
   686  						},
   687  					},
   688  				}),
   689  			},
   690  			[]logproto.Stream{
   691  				{
   692  					Labels: fooLabels.String(),
   693  					Entries: []logproto.Entry{
   694  						{
   695  							Timestamp: from.Add(7 * time.Millisecond),
   696  							Line:      "7",
   697  						},
   698  						{
   699  							Timestamp: from.Add(6 * time.Millisecond),
   700  							Line:      "6",
   701  						},
   702  						{
   703  							Timestamp: from.Add(5 * time.Millisecond),
   704  							Line:      "5",
   705  						},
   706  						{
   707  							Timestamp: from.Add(4 * time.Millisecond),
   708  							Line:      "4",
   709  						},
   710  						{
   711  							Timestamp: from.Add(3 * time.Millisecond),
   712  							Line:      "3",
   713  						},
   714  						{
   715  							Timestamp: from.Add(2 * time.Millisecond),
   716  							Line:      "2",
   717  						},
   718  						{
   719  							Timestamp: from.Add(1 * time.Millisecond),
   720  							Line:      "1",
   721  						},
   722  						{
   723  							Timestamp: from.Add(0 * time.Millisecond),
   724  							Line:      "0",
   725  						},
   726  					},
   727  				},
   728  			},
   729  			fooLabelsWithName.String(),
   730  			from, from.Add(8 * time.Millisecond),
   731  			logproto.BACKWARD,
   732  			2,
   733  		},
   734  		"forward without overlap": {
   735  			[]*LazyChunk{
   736  				newLazyChunk(logproto.Stream{
   737  					Labels: fooLabelsWithName.String(),
   738  					Entries: []logproto.Entry{
   739  						{
   740  							Timestamp: from,
   741  							Line:      "1",
   742  						},
   743  						{
   744  							Timestamp: from.Add(time.Millisecond),
   745  							Line:      "2",
   746  						},
   747  					},
   748  				}),
   749  				newLazyChunk(logproto.Stream{
   750  					Labels: fooLabelsWithName.String(),
   751  					Entries: []logproto.Entry{
   752  						{
   753  							Timestamp: from.Add(2 * time.Millisecond),
   754  							Line:      "3",
   755  						},
   756  					},
   757  				}),
   758  				newLazyChunk(logproto.Stream{
   759  					Labels: fooLabelsWithName.String(),
   760  					Entries: []logproto.Entry{
   761  						{
   762  							Timestamp: from.Add(3 * time.Millisecond),
   763  							Line:      "4",
   764  						},
   765  					},
   766  				}),
   767  			},
   768  			[]logproto.Stream{
   769  				{
   770  					Labels: fooLabels.String(),
   771  					Entries: []logproto.Entry{
   772  						{
   773  							Timestamp: from,
   774  							Line:      "1",
   775  						},
   776  						{
   777  							Timestamp: from.Add(time.Millisecond),
   778  							Line:      "2",
   779  						},
   780  						{
   781  							Timestamp: from.Add(2 * time.Millisecond),
   782  							Line:      "3",
   783  						},
   784  					},
   785  				},
   786  			},
   787  			fooLabelsWithName.String(),
   788  			from, from.Add(3 * time.Millisecond),
   789  			logproto.FORWARD,
   790  			2,
   791  		},
   792  		"backward without overlap": {
   793  			[]*LazyChunk{
   794  				newLazyChunk(logproto.Stream{
   795  					Labels: fooLabelsWithName.String(),
   796  					Entries: []logproto.Entry{
   797  						{
   798  							Timestamp: from,
   799  							Line:      "1",
   800  						},
   801  						{
   802  							Timestamp: from.Add(time.Millisecond),
   803  							Line:      "2",
   804  						},
   805  					},
   806  				}),
   807  				newLazyChunk(logproto.Stream{
   808  					Labels: fooLabelsWithName.String(),
   809  					Entries: []logproto.Entry{
   810  						{
   811  							Timestamp: from.Add(2 * time.Millisecond),
   812  							Line:      "3",
   813  						},
   814  					},
   815  				}),
   816  				newLazyChunk(logproto.Stream{
   817  					Labels: fooLabelsWithName.String(),
   818  					Entries: []logproto.Entry{
   819  						{
   820  							Timestamp: from.Add(3 * time.Millisecond),
   821  							Line:      "4",
   822  						},
   823  					},
   824  				}),
   825  			},
   826  			[]logproto.Stream{
   827  				{
   828  					Labels: fooLabels.String(),
   829  					Entries: []logproto.Entry{
   830  						{
   831  							Timestamp: from.Add(2 * time.Millisecond),
   832  							Line:      "3",
   833  						},
   834  						{
   835  							Timestamp: from.Add(time.Millisecond),
   836  							Line:      "2",
   837  						},
   838  						{
   839  							Timestamp: from,
   840  							Line:      "1",
   841  						},
   842  					},
   843  				},
   844  			},
   845  			fooLabelsWithName.String(),
   846  			from, from.Add(3 * time.Millisecond),
   847  			logproto.BACKWARD,
   848  			2,
   849  		},
   850  		// This test is rather complex under the hood.
   851  		// It should cause three sub batches in the iterator.
   852  		// The first batch has no overlap -- it cannot as the first. It has bounds [1,2)
   853  		// The second batch has one chunk overlap, but it includes no entries in the overlap.
   854  		// It has bounds [2,4).
   855  		// The third batch finally consumes the overlap, with bounds [4,max).
   856  		// Notably it also ends up testing the code paths for increasing batch sizes past
   857  		// the default due to nextChunks with the same start timestamp.
   858  		"forward identicals": {
   859  			[]*LazyChunk{
   860  				newLazyChunk(logproto.Stream{
   861  					Labels: fooLabelsWithName.String(),
   862  					Entries: []logproto.Entry{
   863  						{
   864  							Timestamp: from,
   865  							Line:      "1",
   866  						},
   867  					},
   868  				}),
   869  				newLazyChunk(logproto.Stream{
   870  					Labels: fooLabelsWithName.String(),
   871  					Entries: []logproto.Entry{
   872  						{
   873  							Timestamp: from,
   874  							Line:      "1",
   875  						},
   876  					},
   877  				}),
   878  				newLazyChunk(logproto.Stream{
   879  					Labels: fooLabelsWithName.String(),
   880  					Entries: []logproto.Entry{
   881  						{
   882  							Timestamp: from,
   883  							Line:      "1",
   884  						},
   885  						{
   886  							Timestamp: from.Add(3 * time.Millisecond),
   887  							Line:      "4",
   888  						},
   889  					},
   890  				}),
   891  				newLazyChunk(logproto.Stream{
   892  					Labels: fooLabelsWithName.String(),
   893  					Entries: []logproto.Entry{
   894  						{
   895  							Timestamp: from.Add(time.Millisecond),
   896  							Line:      "2",
   897  						},
   898  					},
   899  				}),
   900  				newLazyChunk(logproto.Stream{
   901  					Labels: fooLabelsWithName.String(),
   902  					Entries: []logproto.Entry{
   903  						{
   904  							Timestamp: from.Add(time.Millisecond),
   905  							Line:      "2",
   906  						},
   907  					},
   908  				}),
   909  				newLazyChunk(logproto.Stream{
   910  					Labels: fooLabelsWithName.String(),
   911  					Entries: []logproto.Entry{
   912  						{
   913  							Timestamp: from.Add(time.Millisecond),
   914  							Line:      "2",
   915  						},
   916  					},
   917  				}),
   918  				newLazyChunk(logproto.Stream{
   919  					Labels: fooLabelsWithName.String(),
   920  					Entries: []logproto.Entry{
   921  						{
   922  							Timestamp: from.Add(3 * time.Millisecond),
   923  							Line:      "4",
   924  						},
   925  					},
   926  				}),
   927  			},
   928  			[]logproto.Stream{
   929  				{
   930  					Labels: fooLabels.String(),
   931  					Entries: []logproto.Entry{
   932  						{
   933  							Timestamp: from,
   934  							Line:      "1",
   935  						},
   936  						{
   937  							Timestamp: from.Add(time.Millisecond),
   938  							Line:      "2",
   939  						},
   940  						{
   941  							Timestamp: from.Add(3 * time.Millisecond),
   942  							Line:      "4",
   943  						},
   944  					},
   945  				},
   946  			},
   947  			fooLabelsWithName.String(),
   948  			from, from.Add(4 * time.Millisecond),
   949  			logproto.FORWARD,
   950  			1,
   951  		},
   952  	}
   953  
   954  	s := config.SchemaConfig{
   955  		Configs: []config.PeriodConfig{
   956  			{
   957  				From:      config.DayTime{Time: 0},
   958  				Schema:    "v11",
   959  				RowShards: 16,
   960  			},
   961  		},
   962  	}
   963  
   964  	for name, tt := range tests {
   965  		tt := tt
   966  		t.Run(name, func(t *testing.T) {
   967  			it, err := newLogBatchIterator(context.Background(), s, NilMetrics, tt.chunks, tt.batchSize, newMatchers(tt.matchers), log.NewNoopPipeline(), tt.direction, tt.start, tt.end, nil)
   968  			require.NoError(t, err)
   969  			streams, _, err := iter.ReadBatch(it, 1000)
   970  			_ = it.Close()
   971  			if err != nil {
   972  				t.Fatalf("error reading batch %s", err)
   973  			}
   974  
   975  			assertStream(t, tt.expected, streams.Streams)
   976  		})
   977  	}
   978  }
   979  
   980  func Test_newSampleBatchChunkIterator(t *testing.T) {
   981  	tests := map[string]struct {
   982  		chunks     []*LazyChunk
   983  		expected   []logproto.Series
   984  		matchers   string
   985  		start, end time.Time
   986  		batchSize  int
   987  	}{
   988  		"forward with overlap": {
   989  			[]*LazyChunk{
   990  				newLazyChunk(logproto.Stream{
   991  					Labels: fooLabelsWithName.String(),
   992  					Entries: []logproto.Entry{
   993  						{
   994  							Timestamp: from,
   995  							Line:      "1",
   996  						},
   997  						{
   998  							Timestamp: from.Add(time.Millisecond),
   999  							Line:      "2",
  1000  						},
  1001  					},
  1002  				}),
  1003  				newLazyChunk(logproto.Stream{
  1004  					Labels: fooLabelsWithName.String(),
  1005  					Entries: []logproto.Entry{
  1006  						{
  1007  							Timestamp: from.Add(time.Millisecond),
  1008  							Line:      "2",
  1009  						},
  1010  						{
  1011  							Timestamp: from.Add(2 * time.Millisecond),
  1012  							Line:      "3",
  1013  						},
  1014  					},
  1015  				}),
  1016  				newLazyChunk(logproto.Stream{
  1017  					Labels: fooLabelsWithName.String(),
  1018  					Entries: []logproto.Entry{
  1019  						{
  1020  							Timestamp: from.Add(time.Millisecond),
  1021  							Line:      "2",
  1022  						},
  1023  						{
  1024  							Timestamp: from.Add(2 * time.Millisecond),
  1025  							Line:      "3",
  1026  						},
  1027  					},
  1028  				}),
  1029  				newLazyChunk(logproto.Stream{
  1030  					Labels: fooLabelsWithName.String(),
  1031  					Entries: []logproto.Entry{
  1032  						{
  1033  							Timestamp: from.Add(2 * time.Millisecond),
  1034  							Line:      "3",
  1035  						},
  1036  						{
  1037  							Timestamp: from.Add(3 * time.Millisecond),
  1038  							Line:      "4",
  1039  						},
  1040  					},
  1041  				}),
  1042  				newLazyChunk(logproto.Stream{
  1043  					Labels: fooLabelsWithName.String(),
  1044  					Entries: []logproto.Entry{
  1045  						{
  1046  							Timestamp: from.Add(2 * time.Millisecond),
  1047  							Line:      "3",
  1048  						},
  1049  						{
  1050  							Timestamp: from.Add(3 * time.Millisecond),
  1051  							Line:      "4",
  1052  						},
  1053  					},
  1054  				}),
  1055  				newLazyChunk(logproto.Stream{
  1056  					Labels: fooLabelsWithName.String(),
  1057  					Entries: []logproto.Entry{
  1058  						{
  1059  							Timestamp: from.Add(3 * time.Millisecond),
  1060  							Line:      "4",
  1061  						},
  1062  						{
  1063  							Timestamp: from.Add(4 * time.Millisecond),
  1064  							Line:      "5",
  1065  						},
  1066  					},
  1067  				}),
  1068  			},
  1069  			[]logproto.Series{
  1070  				{
  1071  					Labels: fooLabels.String(),
  1072  					Samples: []logproto.Sample{
  1073  						{
  1074  							Timestamp: from.UnixNano(),
  1075  							Hash:      xxhash.Sum64String("1"),
  1076  							Value:     1.,
  1077  						},
  1078  						{
  1079  							Timestamp: from.Add(time.Millisecond).UnixNano(),
  1080  							Hash:      xxhash.Sum64String("2"),
  1081  							Value:     1.,
  1082  						},
  1083  						{
  1084  							Timestamp: from.Add(2 * time.Millisecond).UnixNano(),
  1085  							Hash:      xxhash.Sum64String("3"),
  1086  							Value:     1.,
  1087  						},
  1088  						{
  1089  							Timestamp: from.Add(3 * time.Millisecond).UnixNano(),
  1090  							Hash:      xxhash.Sum64String("4"),
  1091  							Value:     1.,
  1092  						},
  1093  					},
  1094  				},
  1095  			},
  1096  			fooLabelsWithName.String(),
  1097  			from, from.Add(4 * time.Millisecond),
  1098  			2,
  1099  		},
  1100  		"forward with overlapping non-continuous entries": {
  1101  			[]*LazyChunk{
  1102  				newLazyChunk(logproto.Stream{
  1103  					Labels: fooLabelsWithName.String(),
  1104  					Entries: []logproto.Entry{
  1105  						{
  1106  							Timestamp: from,
  1107  							Line:      "1",
  1108  						},
  1109  						{
  1110  							Timestamp: from.Add(time.Millisecond),
  1111  							Line:      "2",
  1112  						},
  1113  						{
  1114  							Timestamp: from.Add(3 * time.Millisecond),
  1115  							Line:      "4",
  1116  						},
  1117  					},
  1118  				}),
  1119  				newLazyChunk(logproto.Stream{
  1120  					Labels: fooLabelsWithName.String(),
  1121  					Entries: []logproto.Entry{
  1122  						{
  1123  							Timestamp: from.Add(time.Millisecond),
  1124  							Line:      "2",
  1125  						},
  1126  						{
  1127  							Timestamp: from.Add(2 * time.Millisecond),
  1128  							Line:      "3",
  1129  						},
  1130  					},
  1131  				}),
  1132  				newLazyChunk(logproto.Stream{
  1133  					Labels: fooLabelsWithName.String(),
  1134  					Entries: []logproto.Entry{
  1135  						{
  1136  							Timestamp: from.Add(time.Millisecond),
  1137  							Line:      "2",
  1138  						},
  1139  						{
  1140  							Timestamp: from.Add(3 * time.Millisecond),
  1141  							Line:      "4",
  1142  						},
  1143  					},
  1144  				}),
  1145  				newLazyChunk(logproto.Stream{
  1146  					Labels: fooLabelsWithName.String(),
  1147  					Entries: []logproto.Entry{
  1148  						{
  1149  							Timestamp: from.Add(2 * time.Millisecond),
  1150  							Line:      "3",
  1151  						},
  1152  						{
  1153  							Timestamp: from.Add(3 * time.Millisecond),
  1154  							Line:      "4",
  1155  						},
  1156  					},
  1157  				}),
  1158  			},
  1159  			[]logproto.Series{
  1160  				{
  1161  					Labels: fooLabels.String(),
  1162  					Samples: []logproto.Sample{
  1163  						{
  1164  							Timestamp: from.UnixNano(),
  1165  							Hash:      xxhash.Sum64String("1"),
  1166  							Value:     1.,
  1167  						},
  1168  						{
  1169  							Timestamp: from.Add(time.Millisecond).UnixNano(),
  1170  							Hash:      xxhash.Sum64String("2"),
  1171  							Value:     1.,
  1172  						},
  1173  						{
  1174  							Timestamp: from.Add(2 * time.Millisecond).UnixNano(),
  1175  							Hash:      xxhash.Sum64String("3"),
  1176  							Value:     1.,
  1177  						},
  1178  					},
  1179  				},
  1180  			},
  1181  			fooLabelsWithName.String(),
  1182  			from, from.Add(3 * time.Millisecond),
  1183  			2,
  1184  		},
  1185  		"forward last chunk boundaries equal to end": {
  1186  			[]*LazyChunk{
  1187  				newLazyChunk(logproto.Stream{
  1188  					Labels: fooLabelsWithName.String(),
  1189  					Entries: []logproto.Entry{
  1190  						{
  1191  							Timestamp: time.Unix(1, 0),
  1192  							Line:      "1",
  1193  						},
  1194  						{
  1195  							Timestamp: time.Unix(2, 0),
  1196  							Line:      "2",
  1197  						},
  1198  					},
  1199  				}),
  1200  				newLazyChunk(logproto.Stream{
  1201  					Labels: fooLabelsWithName.String(),
  1202  					Entries: []logproto.Entry{
  1203  						{
  1204  							Timestamp: time.Unix(2, 0),
  1205  							Line:      "2",
  1206  						},
  1207  						{
  1208  							Timestamp: time.Unix(3, 0),
  1209  							Line:      "3",
  1210  						},
  1211  					},
  1212  				}),
  1213  				newLazyChunk(logproto.Stream{
  1214  					Labels: fooLabelsWithName.String(),
  1215  					Entries: []logproto.Entry{
  1216  						{
  1217  							Timestamp: time.Unix(3, 0),
  1218  							Line:      "3",
  1219  						},
  1220  						{
  1221  							Timestamp: time.Unix(4, 0),
  1222  							Line:      "4",
  1223  						},
  1224  					},
  1225  				}),
  1226  			},
  1227  			[]logproto.Series{
  1228  				{
  1229  					Labels: fooLabels.String(),
  1230  					Samples: []logproto.Sample{
  1231  						{
  1232  							Timestamp: time.Unix(1, 0).UnixNano(),
  1233  							Hash:      xxhash.Sum64String("1"),
  1234  							Value:     1.,
  1235  						},
  1236  						{
  1237  							Timestamp: time.Unix(2, 0).UnixNano(),
  1238  							Hash:      xxhash.Sum64String("2"),
  1239  							Value:     1.,
  1240  						},
  1241  					},
  1242  				},
  1243  			},
  1244  			fooLabelsWithName.String(),
  1245  			time.Unix(1, 0), time.Unix(3, 0),
  1246  			2,
  1247  		},
  1248  		"forward last chunk boundaries equal to end and start": {
  1249  			[]*LazyChunk{
  1250  				newLazyChunk(logproto.Stream{
  1251  					Labels: fooLabelsWithName.String(),
  1252  					Entries: []logproto.Entry{
  1253  						{
  1254  							Timestamp: time.Unix(1, 0),
  1255  							Line:      "1",
  1256  						},
  1257  						{
  1258  							Timestamp: time.Unix(1, 0),
  1259  							Line:      "2",
  1260  						},
  1261  					},
  1262  				}),
  1263  				newLazyChunk(logproto.Stream{
  1264  					Labels: fooLabelsWithName.String(),
  1265  					Entries: []logproto.Entry{
  1266  						{
  1267  							Timestamp: time.Unix(1, 0),
  1268  							Line:      "2",
  1269  						},
  1270  						{
  1271  							Timestamp: time.Unix(2, 0),
  1272  							Line:      "3",
  1273  						},
  1274  					},
  1275  				}),
  1276  			},
  1277  			[]logproto.Series{
  1278  				{
  1279  					Labels: fooLabels.String(),
  1280  					Samples: []logproto.Sample{
  1281  						{
  1282  							Timestamp: time.Unix(1, 0).UnixNano(),
  1283  							Hash:      xxhash.Sum64String("1"),
  1284  							Value:     1.,
  1285  						},
  1286  						{
  1287  							Timestamp: time.Unix(1, 0).UnixNano(),
  1288  							Hash:      xxhash.Sum64String("2"),
  1289  							Value:     1.,
  1290  						},
  1291  					},
  1292  				},
  1293  			},
  1294  			fooLabelsWithName.String(),
  1295  			time.Unix(1, 0), time.Unix(1, 0),
  1296  			2,
  1297  		},
  1298  		"forward without overlap": {
  1299  			[]*LazyChunk{
  1300  				newLazyChunk(logproto.Stream{
  1301  					Labels: fooLabelsWithName.String(),
  1302  					Entries: []logproto.Entry{
  1303  						{
  1304  							Timestamp: from,
  1305  							Line:      "1",
  1306  						},
  1307  						{
  1308  							Timestamp: from.Add(time.Millisecond),
  1309  							Line:      "2",
  1310  						},
  1311  					},
  1312  				}),
  1313  				newLazyChunk(logproto.Stream{
  1314  					Labels: fooLabelsWithName.String(),
  1315  					Entries: []logproto.Entry{
  1316  						{
  1317  							Timestamp: from.Add(2 * time.Millisecond),
  1318  							Line:      "3",
  1319  						},
  1320  					},
  1321  				}),
  1322  				newLazyChunk(logproto.Stream{
  1323  					Labels: fooLabelsWithName.String(),
  1324  					Entries: []logproto.Entry{
  1325  						{
  1326  							Timestamp: from.Add(3 * time.Millisecond),
  1327  							Line:      "4",
  1328  						},
  1329  					},
  1330  				}),
  1331  			},
  1332  			[]logproto.Series{
  1333  				{
  1334  					Labels: fooLabels.String(),
  1335  					Samples: []logproto.Sample{
  1336  						{
  1337  							Timestamp: from.UnixNano(),
  1338  							Hash:      xxhash.Sum64String("1"),
  1339  							Value:     1.,
  1340  						},
  1341  						{
  1342  							Timestamp: from.Add(time.Millisecond).UnixNano(),
  1343  							Hash:      xxhash.Sum64String("2"),
  1344  							Value:     1.,
  1345  						},
  1346  						{
  1347  							Timestamp: from.Add(2 * time.Millisecond).UnixNano(),
  1348  							Hash:      xxhash.Sum64String("3"),
  1349  							Value:     1.,
  1350  						},
  1351  					},
  1352  				},
  1353  			},
  1354  			fooLabelsWithName.String(),
  1355  			from, from.Add(3 * time.Millisecond),
  1356  			2,
  1357  		},
  1358  	}
  1359  
  1360  	s := config.SchemaConfig{
  1361  		Configs: []config.PeriodConfig{
  1362  			{
  1363  				From:      config.DayTime{Time: 0},
  1364  				Schema:    "v11",
  1365  				RowShards: 16,
  1366  			},
  1367  		},
  1368  	}
  1369  
  1370  	for name, tt := range tests {
  1371  		tt := tt
  1372  		t.Run(name, func(t *testing.T) {
  1373  			ex, err := log.NewLineSampleExtractor(log.CountExtractor, nil, nil, false, false)
  1374  			require.NoError(t, err)
  1375  
  1376  			it, err := newSampleBatchIterator(context.Background(), s, NilMetrics, tt.chunks, tt.batchSize, newMatchers(tt.matchers), ex, tt.start, tt.end, nil)
  1377  			require.NoError(t, err)
  1378  			series, _, err := iter.ReadSampleBatch(it, 1000)
  1379  			_ = it.Close()
  1380  			if err != nil {
  1381  				t.Fatalf("error reading batch %s", err)
  1382  			}
  1383  
  1384  			assertSeries(t, tt.expected, series.Series)
  1385  		})
  1386  	}
  1387  }
  1388  
  1389  func TestPartitionOverlappingchunks(t *testing.T) {
  1390  	var (
  1391  		oneThroughFour = newLazyChunk(logproto.Stream{
  1392  			Labels: fooLabelsWithName.String(),
  1393  			Entries: []logproto.Entry{
  1394  				{
  1395  					Timestamp: from,
  1396  					Line:      "1",
  1397  				},
  1398  				{
  1399  					Timestamp: from.Add(3 * time.Millisecond),
  1400  					Line:      "4",
  1401  				},
  1402  			},
  1403  		})
  1404  		two = newLazyChunk(logproto.Stream{
  1405  			Labels: fooLabelsWithName.String(),
  1406  			Entries: []logproto.Entry{
  1407  				{
  1408  					Timestamp: from.Add(1 * time.Millisecond),
  1409  					Line:      "2",
  1410  				},
  1411  			},
  1412  		})
  1413  		three = newLazyChunk(logproto.Stream{
  1414  			Labels: fooLabelsWithName.String(),
  1415  			Entries: []logproto.Entry{
  1416  				{
  1417  					Timestamp: from.Add(2 * time.Millisecond),
  1418  					Line:      "3",
  1419  				},
  1420  			},
  1421  		})
  1422  	)
  1423  
  1424  	for i, tc := range []struct {
  1425  		input    []*LazyChunk
  1426  		expected [][]*LazyChunk
  1427  	}{
  1428  		{
  1429  			input: []*LazyChunk{
  1430  				oneThroughFour,
  1431  				two,
  1432  				three,
  1433  			},
  1434  			expected: [][]*LazyChunk{
  1435  				{oneThroughFour},
  1436  				{two, three},
  1437  			},
  1438  		},
  1439  		{
  1440  			input: []*LazyChunk{
  1441  				two,
  1442  				oneThroughFour,
  1443  				three,
  1444  			},
  1445  			expected: [][]*LazyChunk{
  1446  				{oneThroughFour},
  1447  				{two, three},
  1448  			},
  1449  		},
  1450  		{
  1451  			input: []*LazyChunk{
  1452  				two,
  1453  				two,
  1454  				three,
  1455  				three,
  1456  			},
  1457  			expected: [][]*LazyChunk{
  1458  				{two, three},
  1459  				{two, three},
  1460  			},
  1461  		},
  1462  	} {
  1463  		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
  1464  			out := partitionOverlappingChunks(tc.input)
  1465  			require.Equal(t, tc.expected, out)
  1466  		})
  1467  	}
  1468  }
  1469  
  1470  func TestBuildHeapIterator(t *testing.T) {
  1471  	var (
  1472  		firstChunk = newLazyChunk(logproto.Stream{
  1473  			Labels: "{foo=\"bar\"}",
  1474  			Entries: []logproto.Entry{
  1475  				{
  1476  					Timestamp: from,
  1477  					Line:      "1",
  1478  				},
  1479  				{
  1480  					Timestamp: from.Add(time.Millisecond),
  1481  					Line:      "2",
  1482  				},
  1483  				{
  1484  					Timestamp: from.Add(2 * time.Millisecond),
  1485  					Line:      "3",
  1486  				},
  1487  			},
  1488  		})
  1489  		secondChunk = newLazyInvalidChunk(logproto.Stream{
  1490  			Labels: "{foo=\"bar\"}",
  1491  			Entries: []logproto.Entry{
  1492  				{
  1493  					Timestamp: from.Add(3 * time.Millisecond),
  1494  					Line:      "4",
  1495  				},
  1496  				{
  1497  					Timestamp: from.Add(4 * time.Millisecond),
  1498  					Line:      "5",
  1499  				},
  1500  			},
  1501  		})
  1502  		thirdChunk = newLazyChunk(logproto.Stream{
  1503  			Labels: "{foo=\"bar\"}",
  1504  			Entries: []logproto.Entry{
  1505  				{
  1506  					Timestamp: from.Add(5 * time.Millisecond),
  1507  					Line:      "6",
  1508  				},
  1509  			},
  1510  		})
  1511  	)
  1512  
  1513  	for i, tc := range []struct {
  1514  		input    [][]*LazyChunk
  1515  		expected []logproto.Stream
  1516  	}{
  1517  		{
  1518  			[][]*LazyChunk{
  1519  				{firstChunk},
  1520  				{thirdChunk},
  1521  			},
  1522  			[]logproto.Stream{
  1523  				{
  1524  					Labels: "{foo=\"bar\"}",
  1525  					Entries: []logproto.Entry{
  1526  						{
  1527  							Timestamp: from,
  1528  							Line:      "1",
  1529  						},
  1530  						{
  1531  							Timestamp: from.Add(time.Millisecond),
  1532  							Line:      "2",
  1533  						},
  1534  						{
  1535  							Timestamp: from.Add(2 * time.Millisecond),
  1536  							Line:      "3",
  1537  						},
  1538  						{
  1539  							Timestamp: from.Add(5 * time.Millisecond),
  1540  							Line:      "6",
  1541  						},
  1542  					},
  1543  				},
  1544  			},
  1545  		},
  1546  		{
  1547  			[][]*LazyChunk{
  1548  				{secondChunk},
  1549  				{firstChunk, thirdChunk},
  1550  			},
  1551  			[]logproto.Stream{
  1552  				{
  1553  					Labels: "{foo=\"bar\"}",
  1554  					Entries: []logproto.Entry{
  1555  						{
  1556  							Timestamp: from,
  1557  							Line:      "1",
  1558  						},
  1559  						{
  1560  							Timestamp: from.Add(time.Millisecond),
  1561  							Line:      "2",
  1562  						},
  1563  						{
  1564  							Timestamp: from.Add(2 * time.Millisecond),
  1565  							Line:      "3",
  1566  						},
  1567  						{
  1568  							Timestamp: from.Add(5 * time.Millisecond),
  1569  							Line:      "6",
  1570  						},
  1571  					},
  1572  				},
  1573  			},
  1574  		},
  1575  	} {
  1576  		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
  1577  			ctx = user.InjectOrgID(context.Background(), "test-user")
  1578  			b := &logBatchIterator{
  1579  				batchChunkIterator: &batchChunkIterator{
  1580  					direction: logproto.FORWARD,
  1581  				},
  1582  				ctx:      ctx,
  1583  				pipeline: log.NewNoopPipeline(),
  1584  			}
  1585  			it, err := b.buildHeapIterator(tc.input, from, from.Add(6*time.Millisecond), b.pipeline.ForStream(labels.Labels{labels.Label{Name: "foo", Value: "bar"}}), nil)
  1586  			if err != nil {
  1587  				t.Errorf("buildHeapIterator error = %v", err)
  1588  				return
  1589  			}
  1590  			req := newQuery("{foo=\"bar\"}", from, from.Add(6*time.Millisecond), nil, nil)
  1591  			streams, _, err := iter.ReadBatch(it, req.Limit)
  1592  			_ = it.Close()
  1593  			if err != nil {
  1594  				t.Fatalf("error reading batch %s", err)
  1595  			}
  1596  			assertStream(t, tc.expected, streams.Streams)
  1597  		})
  1598  	}
  1599  }
  1600  
  1601  func Test_IsInvalidChunkError(t *testing.T) {
  1602  	tests := []struct {
  1603  		name           string
  1604  		err            error
  1605  		expectedResult bool
  1606  	}{
  1607  		{
  1608  			"invalid chunk cheksum error from cortex",
  1609  			promql.ErrStorage{Err: chunkenc.ErrInvalidChecksum},
  1610  			true,
  1611  		},
  1612  		{
  1613  			"invalid chunk cheksum error from loki",
  1614  			promql.ErrStorage{Err: chunkenc.ErrInvalidChecksum},
  1615  			true,
  1616  		},
  1617  		{
  1618  			"cache error",
  1619  			promql.ErrStorage{Err: errors.New("error fetching from cache")},
  1620  			false,
  1621  		},
  1622  		{
  1623  			"no error from cortex or loki",
  1624  			nil,
  1625  			false,
  1626  		},
  1627  	}
  1628  	for _, tc := range tests {
  1629  		result := isInvalidChunkError(tc.err)
  1630  		require.Equal(t, tc.expectedResult, result)
  1631  	}
  1632  }
  1633  
  1634  func TestBatchCancel(t *testing.T) {
  1635  	createChunk := func(from time.Time) *LazyChunk {
  1636  		return newLazyChunk(logproto.Stream{
  1637  			Labels: fooLabelsWithName.String(),
  1638  			Entries: []logproto.Entry{
  1639  				{
  1640  					Timestamp: from,
  1641  					Line:      "1",
  1642  				},
  1643  				{
  1644  					Timestamp: from.Add(time.Millisecond),
  1645  					Line:      "2",
  1646  				},
  1647  			},
  1648  		})
  1649  	}
  1650  	chunks := []*LazyChunk{
  1651  		createChunk(from), createChunk(from.Add(10 * time.Millisecond)), createChunk(from.Add(30 * time.Millisecond)),
  1652  	}
  1653  	ctx, cancel := context.WithCancel(context.Background())
  1654  	cancel()
  1655  
  1656  	s := config.SchemaConfig{
  1657  		Configs: []config.PeriodConfig{
  1658  			{
  1659  				From:      config.DayTime{Time: 0},
  1660  				Schema:    "v11",
  1661  				RowShards: 16,
  1662  			},
  1663  		},
  1664  	}
  1665  
  1666  	it, err := newLogBatchIterator(ctx, s, NilMetrics, chunks, 1, newMatchers(fooLabels.String()), log.NewNoopPipeline(), logproto.FORWARD, from, time.Now(), nil)
  1667  	require.NoError(t, err)
  1668  	defer require.NoError(t, it.Close())
  1669  	for it.Next() {
  1670  	}
  1671  	require.Equal(t, context.Canceled, it.Error())
  1672  }
  1673  
  1674  var entry logproto.Entry
  1675  
  1676  func Benchmark_store_OverlappingChunks(b *testing.B) {
  1677  	b.ReportAllocs()
  1678  	st := &store{
  1679  		chunkMetrics: NilMetrics,
  1680  		cfg: Config{
  1681  			MaxChunkBatchSize: 50,
  1682  		},
  1683  		Store: newMockChunkStore(newOverlappingStreams(200, 200)),
  1684  	}
  1685  	b.ResetTimer()
  1686  	statsCtx, ctx := stats.NewContext(user.InjectOrgID(context.Background(), "fake"))
  1687  	start := time.Now()
  1688  	for i := 0; i < b.N; i++ {
  1689  		it, err := st.SelectLogs(ctx, logql.SelectLogParams{QueryRequest: &logproto.QueryRequest{
  1690  			Selector:  `{foo="bar"}`,
  1691  			Direction: logproto.BACKWARD,
  1692  			Limit:     0,
  1693  			Shards:    nil,
  1694  			Start:     time.Unix(0, 1),
  1695  			End:       time.Unix(0, time.Now().UnixNano()),
  1696  		}})
  1697  		if err != nil {
  1698  			b.Fatal(err)
  1699  		}
  1700  		for it.Next() {
  1701  			entry = it.Entry()
  1702  		}
  1703  		if err := it.Close(); err != nil {
  1704  			b.Fatal(err)
  1705  		}
  1706  	}
  1707  	r := statsCtx.Result(time.Since(start), 0, 0)
  1708  	b.Log("Total chunks:" + fmt.Sprintf("%d", r.TotalChunksRef()))
  1709  	b.Log("Total bytes decompressed:" + fmt.Sprintf("%d", r.TotalDecompressedBytes()))
  1710  }
  1711  
  1712  func newOverlappingStreams(streamCount int, entryCount int) []*logproto.Stream {
  1713  	streams := make([]*logproto.Stream, streamCount)
  1714  	for i := range streams {
  1715  		streams[i] = &logproto.Stream{
  1716  			Labels:  fmt.Sprintf(`{foo="bar",id="%d"}`, i),
  1717  			Entries: make([]logproto.Entry, entryCount),
  1718  		}
  1719  		for j := range streams[i].Entries {
  1720  			streams[i].Entries[j] = logproto.Entry{
  1721  				Timestamp: time.Unix(0, int64(1+j)),
  1722  				Line:      "a very compressible log line duh",
  1723  			}
  1724  		}
  1725  	}
  1726  	return streams
  1727  }