github.com/mithrandie/csvq@v1.18.1/lib/query/analytic_function.go (about)

     1  package query
     2  
     3  import (
     4  	"context"
     5  	"math"
     6  	"sort"
     7  	"strings"
     8  
     9  	"github.com/mithrandie/csvq/lib/parser"
    10  	"github.com/mithrandie/csvq/lib/value"
    11  )
    12  
    13  var AnalyticFunctions = map[string]AnalyticFunction{
    14  	"ROW_NUMBER":   RowNumber{},
    15  	"RANK":         Rank{},
    16  	"DENSE_RANK":   DenseRank{},
    17  	"CUME_DIST":    CumeDist{},
    18  	"PERCENT_RANK": PercentRank{},
    19  	"NTILE":        NTile{},
    20  	"FIRST_VALUE":  FirstValue{},
    21  	"LAST_VALUE":   LastValue{},
    22  	"NTH_VALUE":    NthValue{},
    23  	"LAG":          Lag{},
    24  	"LEAD":         Lead{},
    25  	"LISTAGG":      AnalyticListAgg{},
    26  	"JSON_AGG":     AnalyticJsonAgg{},
    27  }
    28  
    29  type AnalyticFunction interface {
    30  	CheckArgsLen(expr parser.AnalyticFunction) error
    31  	Execute(context.Context, *ReferenceScope, Partition, parser.AnalyticFunction) (map[int]value.Primary, error)
    32  }
    33  
    34  type Partition []int
    35  
    36  func (p Partition) Reverse() {
    37  	sort.Sort(sort.Reverse(sort.IntSlice(p)))
    38  }
    39  
    40  type Partitions map[string]Partition
    41  
    42  func Analyze(ctx context.Context, scope *ReferenceScope, view *View, fn parser.AnalyticFunction, partitionIndices []int) error {
    43  	var anfn AnalyticFunction
    44  	var aggfn AggregateFunction
    45  	var udfn *UserDefinedFunction
    46  	var err error
    47  
    48  	fieldIdentifier := FormatFieldIdentifier(fn)
    49  	fieldLabel := FormatFieldLabel(fn)
    50  
    51  	uname := strings.ToUpper(fn.Name)
    52  	if f, ok := AnalyticFunctions[uname]; ok {
    53  		anfn = f
    54  	} else if f, ok := AggregateFunctions[uname]; ok {
    55  		aggfn = f
    56  	} else {
    57  		if udfn, err = scope.GetFunction(fn, uname); err != nil || !udfn.IsAggregate {
    58  			return NewFunctionNotExistError(fn, fn.Name)
    59  		}
    60  	}
    61  
    62  	if anfn != nil {
    63  		if err := anfn.CheckArgsLen(fn); err != nil {
    64  			return err
    65  		}
    66  	} else if aggfn != nil {
    67  		if len(fn.Args) != 1 {
    68  			return NewFunctionArgumentLengthError(fn, fn.Name, []int{1})
    69  		}
    70  
    71  		if _, ok := fn.Args[0].(parser.AllColumns); ok {
    72  			fn.Args[0] = parser.NewIntegerValue(1)
    73  		}
    74  	} else {
    75  		if err := udfn.CheckArgsLen(fn, fn.Name, len(fn.Args)-1); err != nil {
    76  			return err
    77  		}
    78  	}
    79  
    80  	if view.sortValuesInEachCell == nil {
    81  		view.sortValuesInEachCell = make([][]*SortValue, view.RecordLen())
    82  	}
    83  
    84  	partitionKeys := make([]string, view.RecordLen())
    85  	if err = NewGoroutineTaskManager(view.RecordLen(), -1, scope.Tx.Flags.CPU).Run(ctx, func(index int) error {
    86  		keyBuf := GetComparisonKeysBuf()
    87  
    88  		if view.sortValuesInEachCell[index] == nil {
    89  			view.sortValuesInEachCell[index] = make([]*SortValue, cap(view.RecordSet[index]))
    90  		}
    91  		if partitionIndices != nil {
    92  			sortValues := make(SortValues, len(partitionIndices))
    93  			for j, idx := range partitionIndices {
    94  				if idx < len(view.sortValuesInEachCell[index]) && view.sortValuesInEachCell[index][idx] != nil {
    95  					sortValues[j] = view.sortValuesInEachCell[index][idx]
    96  				} else {
    97  					sortValues[j] = NewSortValue(view.RecordSet[index][idx][0], scope.Tx.Flags)
    98  					if idx < len(view.sortValuesInEachCell[index]) {
    99  						view.sortValuesInEachCell[index][idx] = sortValues[j]
   100  					}
   101  				}
   102  			}
   103  			sortValues.Serialize(keyBuf)
   104  		}
   105  
   106  		partitionKeys[index] = keyBuf.String()
   107  		PutComparisonkeysBuf(keyBuf)
   108  		return nil
   109  	}); err != nil {
   110  		return err
   111  	}
   112  
   113  	partitions := make(Partitions, 20)
   114  	partitionMapKeys := make([]string, 0, 20)
   115  	for i, key := range partitionKeys {
   116  		if _, ok := partitions[key]; ok {
   117  			partitions[key] = append(partitions[key], i)
   118  		} else {
   119  			partitions[key] = make(Partition, 1, 40)
   120  			partitions[key][0] = i
   121  			partitionMapKeys = append(partitionMapKeys, key)
   122  		}
   123  	}
   124  
   125  	calcCnt := view.RecordLen() * len(partitionMapKeys)
   126  	minReq := -1
   127  	if MinimumRequiredPerCPUCore < calcCnt {
   128  		minReq = int(math.Ceil(float64(len(partitionMapKeys)) / (math.Floor(float64(calcCnt) / MinimumRequiredPerCPUCore))))
   129  	}
   130  	gm := NewGoroutineTaskManager(len(partitionMapKeys), minReq, scope.Tx.Flags.CPU)
   131  
   132  	var analyzeFn = func(thIdx int) {
   133  		defer func() {
   134  			if !gm.HasError() {
   135  				if panicReport := recover(); panicReport != nil {
   136  					gm.SetError(NewFatalError(panicReport))
   137  				}
   138  			}
   139  
   140  			if 1 < gm.Number {
   141  				gm.Done()
   142  			}
   143  		}()
   144  
   145  		start, end := gm.RecordRange(thIdx)
   146  		seqScope := scope.CreateScopeForSequentialEvaluation(view)
   147  
   148  	AnalyzeLoop:
   149  		for i := start; i < end; i++ {
   150  			if gm.HasError() {
   151  				break AnalyzeLoop
   152  			}
   153  			if i&15 == 0 && ctx.Err() != nil {
   154  				break AnalyzeLoop
   155  			}
   156  
   157  			if anfn != nil {
   158  				list, e := anfn.Execute(ctx, seqScope, partitions[partitionMapKeys[i]], fn)
   159  				if e != nil {
   160  					gm.SetError(e)
   161  					break AnalyzeLoop
   162  				}
   163  				for idx, val := range list {
   164  					view.RecordSet[idx] = append(view.RecordSet[idx], NewCell(val))
   165  				}
   166  			} else {
   167  				partition := partitions[partitionMapKeys[i]]
   168  				frameSet := WindowFrameSet(partition, fn.AnalyticClause)
   169  				valueCache := make(map[int]value.Primary, len(partition))
   170  
   171  				udfnArgsExprs := fn.Args[1:]
   172  				udfnArgs := make([]value.Primary, len(udfnArgsExprs))
   173  
   174  				for _, frame := range frameSet {
   175  					values, e := windowValues(ctx, seqScope, frame, partition, fn, valueCache)
   176  					if e != nil {
   177  						gm.SetError(e)
   178  						break AnalyzeLoop
   179  					}
   180  
   181  					if aggfn != nil {
   182  						val := aggfn(values, scope.Tx.Flags)
   183  
   184  						for _, idx := range frame.Records {
   185  							view.RecordSet[idx] = append(view.RecordSet[idx], NewCell(val))
   186  						}
   187  					} else { //User Defined Function
   188  						for _, idx := range frame.Records {
   189  							seqScope.Records[0].recordIndex = idx
   190  
   191  							for i, v := range udfnArgsExprs {
   192  								arg, e := Evaluate(ctx, seqScope, v)
   193  								if e != nil {
   194  									gm.SetError(e)
   195  									break AnalyzeLoop
   196  								}
   197  								udfnArgs[i] = arg
   198  							}
   199  
   200  							val, e := udfn.ExecuteAggregate(ctx, seqScope, values, udfnArgs)
   201  							if e != nil {
   202  								gm.SetError(e)
   203  								break AnalyzeLoop
   204  							}
   205  
   206  							view.RecordSet[idx] = append(view.RecordSet[idx], NewCell(val))
   207  						}
   208  					}
   209  				}
   210  			}
   211  		}
   212  	}
   213  
   214  	if 1 < gm.Number {
   215  		for i := 0; i < gm.Number; i++ {
   216  			gm.Add()
   217  			go analyzeFn(i)
   218  		}
   219  		gm.Wait()
   220  	} else {
   221  		analyzeFn(0)
   222  	}
   223  
   224  	if gm.HasError() {
   225  		return gm.Err()
   226  	}
   227  	if ctx.Err() != nil {
   228  		return ConvertContextError(ctx.Err())
   229  	}
   230  
   231  	view.Header, _ = AddHeaderField(view.Header, fieldIdentifier, fieldLabel, "")
   232  
   233  	return nil
   234  }
   235  
   236  type WindowFrame struct {
   237  	Low     int
   238  	High    int
   239  	Records []int
   240  }
   241  
   242  func WindowFrameSet(partition Partition, expr parser.AnalyticClause) []WindowFrame {
   243  	var singleFrameSet = func(partition Partition) []WindowFrame {
   244  		indices := make([]int, len(partition))
   245  		for i, idx := range partition {
   246  			indices[i] = idx
   247  		}
   248  		return []WindowFrame{{Low: 0, High: len(partition) - 1, Records: indices}}
   249  	}
   250  
   251  	var frameIndex = func(current int, length int, framePosition parser.WindowFramePosition) int {
   252  		var idx int
   253  
   254  		switch framePosition.Direction.Token {
   255  		case parser.CURRENT:
   256  			idx = current
   257  		case parser.PRECEDING:
   258  			if !framePosition.Unbounded.IsEmpty() {
   259  				idx = 0
   260  			} else {
   261  				idx = current - framePosition.Offset
   262  			}
   263  		case parser.FOLLOWING:
   264  			if !framePosition.Unbounded.IsEmpty() {
   265  				idx = length - 1
   266  			} else {
   267  				idx = current + framePosition.Offset
   268  			}
   269  		}
   270  
   271  		return idx
   272  	}
   273  
   274  	length := len(partition)
   275  
   276  	if expr.OrderByClause == nil {
   277  		return singleFrameSet(partition)
   278  	}
   279  
   280  	frameSet := make([]WindowFrame, 0, length)
   281  
   282  	var windowClause parser.WindowingClause
   283  	if expr.WindowingClause == nil {
   284  		windowClause = parser.WindowingClause{
   285  			FrameLow: parser.WindowFramePosition{
   286  				Direction: parser.Token{Token: parser.PRECEDING},
   287  				Unbounded: parser.Token{Token: parser.UNBOUNDED},
   288  			},
   289  		}
   290  	} else {
   291  		windowClause = expr.WindowingClause.(parser.WindowingClause)
   292  	}
   293  	frameLow := windowClause.FrameLow.(parser.WindowFramePosition)
   294  
   295  	if windowClause.FrameHigh == nil {
   296  		for current := 0; current < length; current++ {
   297  			frameSet = append(frameSet, WindowFrame{
   298  				Low:     frameIndex(current, length, frameLow),
   299  				High:    current,
   300  				Records: []int{partition[current]},
   301  			})
   302  		}
   303  	} else {
   304  		frameHigh := windowClause.FrameHigh.(parser.WindowFramePosition)
   305  		if frameLow.Direction.Token == parser.PRECEDING && !frameLow.Unbounded.IsEmpty() && frameHigh.Direction.Token == parser.FOLLOWING && !frameHigh.Unbounded.IsEmpty() {
   306  			return singleFrameSet(partition)
   307  		}
   308  
   309  		for current := 0; current < length; current++ {
   310  			frameSet = append(frameSet, WindowFrame{
   311  				Low:     frameIndex(current, length, frameLow),
   312  				High:    frameIndex(current, length, frameHigh),
   313  				Records: []int{partition[current]},
   314  			})
   315  		}
   316  	}
   317  
   318  	return frameSet
   319  }
   320  
   321  func windowValues(ctx context.Context, scope *ReferenceScope, frame WindowFrame, partition Partition, expr parser.AnalyticFunction, valueCache map[int]value.Primary) ([]value.Primary, error) {
   322  	values := make([]value.Primary, 0, frame.High-frame.Low+1)
   323  
   324  	anScope := scope.CreateScopeForAnalytics()
   325  	for i := frame.Low; i <= frame.High; i++ {
   326  		if i < 0 || len(partition) <= i {
   327  			continue
   328  		}
   329  
   330  		recordIdx := partition[i]
   331  		if v, ok := valueCache[recordIdx]; ok {
   332  			values = append(values, v)
   333  		} else {
   334  			anScope.Records[0].recordIndex = recordIdx
   335  			p, e := Evaluate(ctx, anScope, expr.Args[0])
   336  			if e != nil {
   337  				return nil, e
   338  			}
   339  			valueCache[recordIdx] = p
   340  			values = append(values, p)
   341  		}
   342  	}
   343  
   344  	if expr.IsDistinct() {
   345  		values = Distinguish(values, scope.Tx.Flags)
   346  	}
   347  	return values, nil
   348  }
   349  
   350  func CheckArgsLen(expr parser.AnalyticFunction, length []int) error {
   351  	if len(length) == 1 {
   352  		if len(expr.Args) != length[0] {
   353  			return NewFunctionArgumentLengthError(expr, expr.Name, length)
   354  		}
   355  	} else {
   356  		if len(expr.Args) < length[0] {
   357  			return NewFunctionArgumentLengthErrorWithCustomArgs(expr, expr.Name, "at least "+FormatCount(length[0], "argument"))
   358  		}
   359  		if length[1] < len(expr.Args) {
   360  			return NewFunctionArgumentLengthErrorWithCustomArgs(expr, expr.Name, "at most "+FormatCount(length[1], "argument"))
   361  		}
   362  	}
   363  	return nil
   364  }
   365  
   366  type RowNumber struct{}
   367  
   368  func (fn RowNumber) CheckArgsLen(expr parser.AnalyticFunction) error {
   369  	return CheckArgsLen(expr, []int{0})
   370  }
   371  
   372  func (fn RowNumber) Execute(_ context.Context, _ *ReferenceScope, partition Partition, _ parser.AnalyticFunction) (map[int]value.Primary, error) {
   373  	list := make(map[int]value.Primary, len(partition))
   374  	var number int64 = 0
   375  	for _, idx := range partition {
   376  		number++
   377  		list[idx] = value.NewInteger(number)
   378  	}
   379  
   380  	return list, nil
   381  }
   382  
   383  type Rank struct{}
   384  
   385  func (fn Rank) CheckArgsLen(expr parser.AnalyticFunction) error {
   386  	return CheckArgsLen(expr, []int{0})
   387  }
   388  
   389  func (fn Rank) Execute(_ context.Context, scope *ReferenceScope, partition Partition, _ parser.AnalyticFunction) (map[int]value.Primary, error) {
   390  	list := make(map[int]value.Primary, len(partition))
   391  	var number int64 = 0
   392  	var rank int64 = 0
   393  	var currentRank SortValues
   394  	for _, idx := range partition {
   395  		number++
   396  		if scope.Records[0].view.sortValuesInEachRecord == nil || !scope.Records[0].view.sortValuesInEachRecord[idx].EquivalentTo(currentRank) {
   397  			rank = number
   398  			if scope.Records[0].view.sortValuesInEachRecord != nil {
   399  				currentRank = scope.Records[0].view.sortValuesInEachRecord[idx]
   400  			}
   401  		}
   402  		list[idx] = value.NewInteger(rank)
   403  	}
   404  
   405  	return list, nil
   406  }
   407  
   408  type DenseRank struct{}
   409  
   410  func (fn DenseRank) CheckArgsLen(expr parser.AnalyticFunction) error {
   411  	return CheckArgsLen(expr, []int{0})
   412  }
   413  
   414  func (fn DenseRank) Execute(_ context.Context, scope *ReferenceScope, partition Partition, _ parser.AnalyticFunction) (map[int]value.Primary, error) {
   415  	list := make(map[int]value.Primary, len(partition))
   416  	var rank int64 = 0
   417  	var currentRank SortValues
   418  	for _, idx := range partition {
   419  		if scope.Records[0].view.sortValuesInEachRecord == nil || !scope.Records[0].view.sortValuesInEachRecord[idx].EquivalentTo(currentRank) {
   420  			rank++
   421  			if scope.Records[0].view.sortValuesInEachRecord != nil {
   422  				currentRank = scope.Records[0].view.sortValuesInEachRecord[idx]
   423  			}
   424  		}
   425  		list[idx] = value.NewInteger(rank)
   426  	}
   427  
   428  	return list, nil
   429  }
   430  
   431  type CumeDist struct{}
   432  
   433  func (fn CumeDist) CheckArgsLen(expr parser.AnalyticFunction) error {
   434  	return CheckArgsLen(expr, []int{0})
   435  }
   436  
   437  func (fn CumeDist) Execute(_ context.Context, scope *ReferenceScope, partition Partition, _ parser.AnalyticFunction) (map[int]value.Primary, error) {
   438  	list := make(map[int]value.Primary, len(partition))
   439  
   440  	groups := perseCumulativeGroups(partition, scope.Records[0].view)
   441  	total := float64(len(partition))
   442  	cumulative := float64(0)
   443  	for _, group := range groups {
   444  		cumulative += float64(len(group))
   445  		dist := cumulative / total
   446  
   447  		for _, idx := range group {
   448  			list[idx] = value.NewFloat(dist)
   449  		}
   450  	}
   451  
   452  	return list, nil
   453  }
   454  
   455  type PercentRank struct{}
   456  
   457  func (fn PercentRank) CheckArgsLen(expr parser.AnalyticFunction) error {
   458  	return CheckArgsLen(expr, []int{0})
   459  }
   460  
   461  func (fn PercentRank) Execute(_ context.Context, scope *ReferenceScope, partition Partition, _ parser.AnalyticFunction) (map[int]value.Primary, error) {
   462  	list := make(map[int]value.Primary, len(partition))
   463  
   464  	groups := perseCumulativeGroups(partition, scope.Records[0].view)
   465  	denom := float64(len(partition) - 1)
   466  	cumulative := float64(0)
   467  	for _, group := range groups {
   468  		var dist float64 = 1
   469  		if 0 < denom {
   470  			dist = cumulative / denom
   471  		}
   472  
   473  		for _, idx := range group {
   474  			list[idx] = value.NewFloat(dist)
   475  		}
   476  
   477  		cumulative += float64(len(group))
   478  	}
   479  
   480  	return list, nil
   481  }
   482  
   483  func perseCumulativeGroups(partition Partition, view *View) [][]int {
   484  	groups := make([][]int, 0)
   485  	var currentRank SortValues
   486  	for _, idx := range partition {
   487  		if view.sortValuesInEachRecord == nil || !view.sortValuesInEachRecord[idx].EquivalentTo(currentRank) {
   488  			groups = append(groups, []int{idx})
   489  			if view.sortValuesInEachRecord != nil {
   490  				currentRank = view.sortValuesInEachRecord[idx]
   491  			}
   492  		} else {
   493  			groups[len(groups)-1] = append(groups[len(groups)-1], idx)
   494  		}
   495  	}
   496  	return groups
   497  }
   498  
   499  type NTile struct{}
   500  
   501  func (fn NTile) CheckArgsLen(expr parser.AnalyticFunction) error {
   502  	return CheckArgsLen(expr, []int{1})
   503  }
   504  
   505  func (fn NTile) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   506  	tileNumber := 0
   507  	p, err := Evaluate(ctx, scope, expr.Args[0])
   508  	if err != nil {
   509  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the first argument must be an integer")
   510  	}
   511  	i := value.ToInteger(p)
   512  	if value.IsNull(i) {
   513  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the first argument must be an integer")
   514  	}
   515  	tileNumber = int(i.(*value.Integer).Raw())
   516  	value.Discard(i)
   517  	if tileNumber < 1 {
   518  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the first argument must be greater than 0")
   519  	}
   520  
   521  	total := len(partition)
   522  	perTile := total / tileNumber
   523  	mod := total % tileNumber
   524  
   525  	if perTile < 1 {
   526  		perTile = 1
   527  		mod = 0
   528  	}
   529  
   530  	list := make(map[int]value.Primary, len(partition))
   531  	var tile int64 = 1
   532  	var count = 0
   533  	for _, idx := range partition {
   534  		count++
   535  
   536  		switch {
   537  		case perTile+1 < count:
   538  			tile++
   539  			count = 1
   540  		case perTile+1 == count:
   541  			if 0 < mod {
   542  				mod--
   543  			} else {
   544  				tile++
   545  				count = 1
   546  			}
   547  		}
   548  		list[idx] = value.NewInteger(tile)
   549  	}
   550  
   551  	return list, nil
   552  }
   553  
   554  type FirstValue struct{}
   555  
   556  func (fn FirstValue) CheckArgsLen(expr parser.AnalyticFunction) error {
   557  	return CheckArgsLen(expr, []int{1})
   558  }
   559  
   560  func (fn FirstValue) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   561  	return setNthValue(ctx, scope, partition, expr, 1)
   562  }
   563  
   564  type LastValue struct{}
   565  
   566  func (fn LastValue) CheckArgsLen(expr parser.AnalyticFunction) error {
   567  	return CheckArgsLen(expr, []int{1})
   568  }
   569  
   570  func (fn LastValue) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   571  	partition.Reverse()
   572  	return setNthValue(ctx, scope, partition, expr, 1)
   573  }
   574  
   575  type NthValue struct{}
   576  
   577  func (fn NthValue) CheckArgsLen(expr parser.AnalyticFunction) error {
   578  	return CheckArgsLen(expr, []int{2})
   579  }
   580  
   581  func (fn NthValue) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   582  	n := 0
   583  	p, err := Evaluate(ctx, scope, expr.Args[1])
   584  	if err != nil {
   585  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be an integer")
   586  	}
   587  	pi := value.ToInteger(p)
   588  	if value.IsNull(pi) {
   589  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be an integer")
   590  	}
   591  	n = int(pi.(*value.Integer).Raw())
   592  	value.Discard(pi)
   593  	if n < 1 {
   594  		return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be greater than 0")
   595  	}
   596  
   597  	return setNthValue(ctx, scope, partition, expr, n)
   598  }
   599  
   600  func setNthValue(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction, n int) (map[int]value.Primary, error) {
   601  	frameSet := WindowFrameSet(partition, expr.AnalyticClause)
   602  	list := make(map[int]value.Primary, len(partition))
   603  
   604  	valueCache := make(map[int]value.Primary, len(partition))
   605  
   606  	anScope := scope.CreateScopeForAnalytics()
   607  	for _, frame := range frameSet {
   608  		var val value.Primary = value.NewNull()
   609  		count := 0
   610  
   611  		for i := frame.Low; i <= frame.High; i++ {
   612  			if i < 0 || len(partition) <= i {
   613  				continue
   614  			}
   615  
   616  			recordIdx := partition[i]
   617  			if v, ok := valueCache[recordIdx]; ok {
   618  				val = v
   619  			} else {
   620  				anScope.Records[0].recordIndex = recordIdx
   621  				p, err := Evaluate(ctx, anScope, expr.Args[0])
   622  				if err != nil {
   623  					return nil, err
   624  				}
   625  				valueCache[recordIdx] = p
   626  				val = p
   627  			}
   628  			if expr.IgnoreNulls() && value.IsNull(val) {
   629  				continue
   630  			}
   631  
   632  			count++
   633  			if count == n {
   634  				break
   635  			}
   636  		}
   637  
   638  		for _, idx := range frame.Records {
   639  			list[idx] = val
   640  		}
   641  	}
   642  
   643  	return list, nil
   644  }
   645  
   646  type Lag struct{}
   647  
   648  func (fn Lag) CheckArgsLen(expr parser.AnalyticFunction) error {
   649  	return CheckArgsLen(expr, []int{1, 3})
   650  }
   651  
   652  func (fn Lag) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   653  	return setLag(ctx, scope, partition, expr)
   654  }
   655  
   656  type Lead struct{}
   657  
   658  func (fn Lead) CheckArgsLen(expr parser.AnalyticFunction) error {
   659  	return CheckArgsLen(expr, []int{1, 3})
   660  }
   661  
   662  func (fn Lead) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   663  	partition.Reverse()
   664  	return setLag(ctx, scope, partition, expr)
   665  }
   666  
   667  func setLag(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   668  	offset := 1
   669  	if 1 < len(expr.Args) {
   670  		p, err := Evaluate(ctx, scope, expr.Args[1])
   671  		if err != nil {
   672  			return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be an integer")
   673  		}
   674  		i := value.ToInteger(p)
   675  		if value.IsNull(i) {
   676  			return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be an integer")
   677  		}
   678  		offset = int(i.(*value.Integer).Raw())
   679  		value.Discard(i)
   680  	}
   681  
   682  	var defaultValue value.Primary = value.NewNull()
   683  	if 2 < len(expr.Args) {
   684  		p, err := Evaluate(ctx, scope, expr.Args[2])
   685  		if err != nil {
   686  			return nil, err
   687  		}
   688  		defaultValue = p
   689  	}
   690  
   691  	anScope := scope.CreateScopeForAnalytics()
   692  	list := make(map[int]value.Primary, len(partition))
   693  	values := make([]value.Primary, 0)
   694  	for _, idx := range partition {
   695  		anScope.Records[0].recordIndex = idx
   696  		p, err := Evaluate(ctx, anScope, expr.Args[0])
   697  		if err != nil {
   698  			return nil, err
   699  		}
   700  
   701  		values = append(values, p)
   702  
   703  		lagIdx := len(values) - 1 - offset
   704  		val := defaultValue
   705  		if 0 <= lagIdx && lagIdx < len(values) {
   706  			for i := lagIdx; i >= 0; i-- {
   707  				if expr.IgnoreNulls() && value.IsNull(values[i]) {
   708  					continue
   709  				}
   710  				val = values[i]
   711  				break
   712  			}
   713  		}
   714  		list[idx] = val
   715  	}
   716  
   717  	return list, nil
   718  }
   719  
   720  type AnalyticListAgg struct{}
   721  
   722  func (fn AnalyticListAgg) CheckArgsLen(expr parser.AnalyticFunction) error {
   723  	return CheckArgsLen(expr, []int{1, 2})
   724  }
   725  
   726  func (fn AnalyticListAgg) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   727  	separator := ""
   728  	if len(expr.Args) == 2 {
   729  		p, err := Evaluate(ctx, scope, expr.Args[1])
   730  		if err != nil {
   731  			return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be a string")
   732  		}
   733  		s := value.ToString(p)
   734  		if value.IsNull(s) {
   735  			return nil, NewFunctionInvalidArgumentError(expr, expr.Name, "the second argument must be a string")
   736  		}
   737  		separator = s.(*value.String).Raw()
   738  		value.Discard(s)
   739  	}
   740  
   741  	anScope := scope.CreateScopeForAnalytics()
   742  	values := make([]value.Primary, len(partition))
   743  	for i, idx := range partition {
   744  		anScope.Records[0].recordIndex = idx
   745  		val, e := Evaluate(ctx, anScope, expr.Args[0])
   746  		if e != nil {
   747  			return nil, e
   748  		}
   749  		values[i] = val
   750  	}
   751  	if expr.IsDistinct() {
   752  		values = Distinguish(values, scope.Tx.Flags)
   753  	}
   754  
   755  	val := ListAgg(values, separator)
   756  
   757  	list := make(map[int]value.Primary, len(partition))
   758  	for _, idx := range partition {
   759  		list[idx] = val
   760  	}
   761  
   762  	return list, nil
   763  }
   764  
   765  type AnalyticJsonAgg struct{}
   766  
   767  func (fn AnalyticJsonAgg) CheckArgsLen(expr parser.AnalyticFunction) error {
   768  	return CheckArgsLen(expr, []int{1})
   769  }
   770  
   771  func (fn AnalyticJsonAgg) Execute(ctx context.Context, scope *ReferenceScope, partition Partition, expr parser.AnalyticFunction) (map[int]value.Primary, error) {
   772  	anScope := scope.CreateScopeForAnalytics()
   773  	values := make([]value.Primary, len(partition))
   774  	for i, idx := range partition {
   775  		anScope.Records[0].recordIndex = idx
   776  		val, e := Evaluate(ctx, anScope, expr.Args[0])
   777  		if e != nil {
   778  			return nil, e
   779  		}
   780  		values[i] = val
   781  	}
   782  	if expr.IsDistinct() {
   783  		values = Distinguish(values, scope.Tx.Flags)
   784  	}
   785  
   786  	val := JsonAgg(values)
   787  
   788  	list := make(map[int]value.Primary, len(partition))
   789  	for _, idx := range partition {
   790  		list[idx] = val
   791  	}
   792  
   793  	return list, nil
   794  }