github.com/tobgu/qframe@v0.4.0/benchmark_test.go (about)

     1  package qframe_test
     2  
     3  import (
     4  	"bytes"
     5  	stdcsv "encoding/csv"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  	"math/rand"
    10  	"testing"
    11  
    12  	qf "github.com/tobgu/qframe"
    13  	"github.com/tobgu/qframe/config/csv"
    14  	"github.com/tobgu/qframe/config/groupby"
    15  	"github.com/tobgu/qframe/filter"
    16  	"github.com/tobgu/qframe/types"
    17  )
    18  
    19  func genInts(seed int64, size int) []int {
    20  	result := make([]int, size)
    21  	r := rand.New(rand.NewSource(seed))
    22  	if seed == noSeed {
    23  		// Sorted slice
    24  		for ix := range result {
    25  			result[ix] = ix
    26  		}
    27  	} else {
    28  		// Random slice
    29  		for ix := range result {
    30  			result[ix] = r.Intn(size)
    31  		}
    32  	}
    33  
    34  	return result
    35  }
    36  
    37  func genIntsWithCardinality(seed int64, size, cardinality int) []int {
    38  	result := genInts(seed, size)
    39  	for i, x := range result {
    40  		result[i] = x % cardinality
    41  	}
    42  
    43  	return result
    44  }
    45  
    46  func genStringsWithCardinality(seed int64, size, cardinality, strLen int) []string {
    47  	baseStr := "abcdefghijklmnopqrstuvxyz"[:strLen]
    48  	result := make([]string, size)
    49  	for i, x := range genIntsWithCardinality(seed, size, cardinality) {
    50  		result[i] = fmt.Sprintf("%s%d", baseStr, x)
    51  	}
    52  	return result
    53  }
    54  
    55  const noSeed int64 = 0
    56  const seed1 int64 = 1
    57  const seed2 int64 = 2
    58  const seed3 int64 = 3
    59  const seed4 int64 = 4
    60  const frameSize = 100000
    61  
    62  func exampleIntFrame(size int) qf.QFrame {
    63  	return qf.New(map[string]interface{}{
    64  		"S1": genInts(seed1, size),
    65  		"S2": genInts(seed2, size),
    66  		"S3": genInts(seed3, size),
    67  		"S4": genInts(seed4, size)})
    68  }
    69  
    70  func BenchmarkQFrame_FilterIntBuiltIn(b *testing.B) {
    71  	data := exampleIntFrame(frameSize)
    72  
    73  	b.ReportAllocs()
    74  	b.ResetTimer()
    75  	for i := 0; i < b.N; i++ {
    76  		newData := data.Filter(qf.Or(
    77  			qf.Filter{Column: "S1", Comparator: "<", Arg: frameSize / 10},
    78  			qf.Filter{Column: "S2", Comparator: "<", Arg: frameSize / 10},
    79  			qf.Filter{Column: "S3", Comparator: ">", Arg: int(0.9 * frameSize)}))
    80  
    81  		if newData.Len() != 27142 {
    82  			b.Errorf("Length was %d, Err: %s", newData.Len(), newData.Err)
    83  		}
    84  	}
    85  }
    86  
    87  func lessThan(limit int) func(int) bool {
    88  	return func(x int) bool { return x < limit }
    89  }
    90  
    91  func greaterThan(limit int) func(int) bool {
    92  	return func(x int) bool { return x > limit }
    93  }
    94  
    95  func BenchmarkQFrame_FilterIntGeneral(b *testing.B) {
    96  	data := exampleIntFrame(frameSize)
    97  
    98  	b.ReportAllocs()
    99  	b.ResetTimer()
   100  	for i := 0; i < b.N; i++ {
   101  		newData := data.Filter(qf.Or(
   102  			qf.Filter{Column: "S1", Comparator: lessThan(frameSize / 10)},
   103  			qf.Filter{Column: "S2", Comparator: lessThan(frameSize / 10)},
   104  			qf.Filter{Column: "S3", Comparator: greaterThan(int(0.9 * frameSize))}))
   105  
   106  		if newData.Len() != 27142 {
   107  			b.Errorf("Length was %d, Err: %s", newData.Len(), newData.Err)
   108  		}
   109  	}
   110  }
   111  
   112  func rangeSlice(size int) []int {
   113  	result := make([]int, size)
   114  	for i := 0; i < size; i++ {
   115  		result[i] = i
   116  	}
   117  	return result
   118  }
   119  
   120  func BenchmarkQFrame_FilterIntBuiltinIn(b *testing.B) {
   121  	data := exampleIntFrame(frameSize)
   122  	slice := rangeSlice(frameSize / 100)
   123  	b.ReportAllocs()
   124  	b.ResetTimer()
   125  	for i := 0; i < b.N; i++ {
   126  		newData := data.Filter(qf.Filter{Column: "S1", Comparator: "in", Arg: slice})
   127  		if newData.Err != nil {
   128  			b.Errorf("Length was Err: %s", newData.Err)
   129  		}
   130  	}
   131  }
   132  
   133  func intInFilter(input []int) func(int) bool {
   134  	set := make(map[int]struct{}, len(input))
   135  	for _, x := range input {
   136  		set[x] = struct{}{}
   137  	}
   138  
   139  	return func(x int) bool {
   140  		_, ok := set[x]
   141  		return ok
   142  	}
   143  }
   144  
   145  func BenchmarkQFrame_FilterIntGeneralIn(b *testing.B) {
   146  	data := exampleIntFrame(frameSize)
   147  	slice := rangeSlice(frameSize / 100)
   148  	b.ReportAllocs()
   149  	b.ResetTimer()
   150  	for i := 0; i < b.N; i++ {
   151  		newData := data.Filter(qf.Filter{Column: "S1", Comparator: intInFilter(slice)})
   152  		if newData.Err != nil {
   153  			b.Errorf("Length was Err: %s", newData.Err)
   154  		}
   155  	}
   156  }
   157  
   158  func BenchmarkQFrame_FilterNot(b *testing.B) {
   159  	data := qf.New(map[string]interface{}{
   160  		"S1": genInts(seed1, frameSize)})
   161  	f := qf.Filter{Column: "S1", Comparator: "<", Arg: frameSize - frameSize/10, Inverse: true}
   162  
   163  	b.Run("qframe", func(b *testing.B) {
   164  		b.ReportAllocs()
   165  		b.ResetTimer()
   166  		for i := 0; i < b.N; i++ {
   167  			newData := data.Filter(f)
   168  			if newData.Len() != 9882 {
   169  				b.Errorf("Length was %d", newData.Len())
   170  			}
   171  		}
   172  	})
   173  
   174  	b.Run("filter", func(b *testing.B) {
   175  		b.ReportAllocs()
   176  		b.ResetTimer()
   177  		for i := 0; i < b.N; i++ {
   178  			clause := qf.Not(qf.Filter(filter.Filter{Column: "S1", Comparator: "<", Arg: frameSize - frameSize/10}))
   179  			newData := data.Filter(clause)
   180  			if newData.Len() != 9882 {
   181  				b.Errorf("Length was %d", newData.Len())
   182  			}
   183  		}
   184  	})
   185  }
   186  
   187  func BenchmarkQFrame_Sort(b *testing.B) {
   188  	data := qf.New(map[string]interface{}{
   189  		"S1": genInts(seed1, frameSize),
   190  		"S2": genInts(seed2, frameSize),
   191  		"S3": genInts(seed3, frameSize),
   192  		"S4": genInts(seed4, frameSize)})
   193  
   194  	b.ReportAllocs()
   195  	b.ResetTimer()
   196  
   197  	for i := 0; i < b.N; i++ {
   198  		newData := data.Sort(qf.Order{Column: "S1"}, qf.Order{Column: "S2", Reverse: true})
   199  		if newData.Err != nil {
   200  			b.Errorf("Unexpected sort error: %s", newData.Err)
   201  		}
   202  	}
   203  }
   204  
   205  func BenchmarkQFrame_Sort1Col(b *testing.B) {
   206  	data := qf.New(map[string]interface{}{
   207  		"S1": genInts(seed1, frameSize),
   208  		"S2": genInts(seed2, frameSize),
   209  		"S3": genInts(seed3, frameSize),
   210  		"S4": genInts(seed4, frameSize)})
   211  
   212  	b.ReportAllocs()
   213  	b.ResetTimer()
   214  
   215  	for i := 0; i < b.N; i++ {
   216  		newData := data.Sort(qf.Order{Column: "S1"})
   217  		if newData.Err != nil {
   218  			b.Errorf("Unexpected sort error: %s", newData.Err)
   219  		}
   220  	}
   221  }
   222  
   223  func BenchmarkQFrame_SortSorted(b *testing.B) {
   224  	data := qf.New(map[string]interface{}{
   225  		"S1": genInts(noSeed, frameSize),
   226  		"S2": genInts(noSeed, frameSize),
   227  		"S3": genInts(noSeed, frameSize),
   228  		"S4": genInts(noSeed, frameSize)})
   229  
   230  	b.ReportAllocs()
   231  	b.ResetTimer()
   232  
   233  	for i := 0; i < b.N; i++ {
   234  		newData := data.Sort(qf.Order{Column: "S1"}, qf.Order{Column: "S2", Reverse: true})
   235  		if newData.Err != nil {
   236  			b.Errorf("Unexpected sort error: %s", newData.Err)
   237  		}
   238  	}
   239  }
   240  
   241  func csvBytes(rowCount int) []byte {
   242  	buf := new(bytes.Buffer)
   243  	writer := stdcsv.NewWriter(buf)
   244  	_ = writer.Write([]string{"INT1", "INT2", "FLOAT1", "FLOAT2", "BOOL1", "STRING1", "STRING2"})
   245  	for i := 0; i < rowCount; i++ {
   246  		_ = writer.Write([]string{"123", "1234567", "5.2534", "9834543.25", "true", fmt.Sprintf("Foo bar baz %d", i%10000), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"})
   247  	}
   248  	writer.Flush()
   249  
   250  	csvBytes, _ := io.ReadAll(buf)
   251  	return csvBytes
   252  }
   253  
   254  func csvEnumBytes(rowCount, cardinality int) []byte {
   255  	buf := new(bytes.Buffer)
   256  	writer := stdcsv.NewWriter(buf)
   257  	_ = writer.Write([]string{"COL1", "COL2"})
   258  	for i := 0; i < rowCount; i++ {
   259  		_ = writer.Write([]string{
   260  			fmt.Sprintf("Foo bar baz %d", i%cardinality),
   261  			fmt.Sprintf("AB%d", i%cardinality)})
   262  	}
   263  	writer.Flush()
   264  
   265  	csvBytes, _ := io.ReadAll(buf)
   266  	return csvBytes
   267  }
   268  
   269  func BenchmarkQFrame_ReadCSV(b *testing.B) {
   270  	rowCount := 100000
   271  	input := csvBytes(rowCount)
   272  
   273  	b.ReportAllocs()
   274  	b.ResetTimer()
   275  
   276  	for i := 0; i < b.N; i++ {
   277  		r := bytes.NewReader(input)
   278  		df := qf.ReadCSV(r, csv.RowCountHint(rowCount))
   279  		if df.Err != nil {
   280  			b.Errorf("Unexpected CSV error: %s", df.Err)
   281  		}
   282  
   283  		if df.Len() != rowCount {
   284  			b.Errorf("Unexpected size: %d", df.Len())
   285  		}
   286  	}
   287  }
   288  
   289  func BenchmarkQFrame_ReadCSVEnum(b *testing.B) {
   290  	rowCount := 100000
   291  	cardinality := 20
   292  	input := csvEnumBytes(rowCount, cardinality)
   293  
   294  	for _, t := range []string{"enum"} {
   295  		b.Run(fmt.Sprintf("Type %s", t), func(b *testing.B) {
   296  			b.ReportAllocs()
   297  			b.ResetTimer()
   298  			for i := 0; i < b.N; i++ {
   299  				r := bytes.NewReader(input)
   300  				df := qf.ReadCSV(r, csv.Types(map[string]string{"COL1": t, "COL2": t}))
   301  				if df.Err != nil {
   302  					b.Errorf("Unexpected CSV error: %s", df.Err)
   303  				}
   304  
   305  				if df.Len() != rowCount {
   306  					b.Errorf("Unexpected size: %d", df.Len())
   307  				}
   308  			}
   309  		})
   310  	}
   311  }
   312  
   313  func jsonRecords(rowCount int) []byte {
   314  	record := map[string]interface{}{
   315  		"INT1":    123,
   316  		"INT2":    1234567,
   317  		"FLOAT1":  5.2534,
   318  		"FLOAT2":  9834543.25,
   319  		"BOOL1":   true,
   320  		"STRING1": "Foo bar baz",
   321  		"STRING2": "ABCDEFGHIJKLMNOPQRSTUVWXYZ"}
   322  	records := make([]map[string]interface{}, rowCount)
   323  	for i := range records {
   324  		records[i] = record
   325  	}
   326  
   327  	result, err := json.Marshal(records)
   328  	if err != nil {
   329  		panic(err)
   330  	}
   331  	return result
   332  }
   333  
   334  func intSlice(value, size int) []int {
   335  	result := make([]int, size)
   336  	for i := range result {
   337  		result[i] = value
   338  	}
   339  
   340  	return result
   341  }
   342  
   343  func floatSlice(value float64, size int) []float64 {
   344  	result := make([]float64, size)
   345  	for i := range result {
   346  		result[i] = value
   347  	}
   348  
   349  	return result
   350  }
   351  
   352  func boolSlice(value bool, size int) []bool {
   353  	result := make([]bool, size)
   354  	for i := range result {
   355  		result[i] = value
   356  	}
   357  
   358  	return result
   359  }
   360  
   361  func stringSlice(value string, size int) []string {
   362  	result := make([]string, size)
   363  	for i := range result {
   364  		result[i] = value
   365  	}
   366  
   367  	return result
   368  }
   369  
   370  func exampleData(rowCount int) map[string]interface{} {
   371  	return map[string]interface{}{
   372  		"INT1":    intSlice(123, rowCount),
   373  		"INT2":    intSlice(1234567, rowCount),
   374  		"FLOAT1":  floatSlice(5.2534, rowCount),
   375  		"FLOAT2":  floatSlice(9834543.25, rowCount),
   376  		"BOOL1":   boolSlice(false, rowCount),
   377  		"STRING1": stringSlice("Foo bar baz", rowCount),
   378  		"STRING2": stringSlice("ABCDEFGHIJKLMNOPQRSTUVWXYZ", rowCount)}
   379  }
   380  
   381  func BenchmarkQFrame_FromJSONRecords(b *testing.B) {
   382  	rowCount := 10000
   383  	input := jsonRecords(rowCount)
   384  	b.ReportAllocs()
   385  	b.ResetTimer()
   386  
   387  	for i := 0; i < b.N; i++ {
   388  		r := bytes.NewReader(input)
   389  		df := qf.ReadJSON(r)
   390  		if df.Err != nil {
   391  			b.Errorf("Unexpected JSON error: %s", df.Err)
   392  		}
   393  
   394  		if df.Len() != rowCount {
   395  			b.Errorf("Unexpected size: %d", df.Len())
   396  		}
   397  	}
   398  }
   399  
   400  func BenchmarkQFrame_ToCSV(b *testing.B) {
   401  	rowCount := 100000
   402  	input := exampleData(rowCount)
   403  	df := qf.New(input)
   404  	if df.Err != nil {
   405  		b.Errorf("Unexpected New error: %s", df.Err)
   406  	}
   407  
   408  	b.ReportAllocs()
   409  	b.ResetTimer()
   410  
   411  	for i := 0; i < b.N; i++ {
   412  		buf := new(bytes.Buffer)
   413  		err := df.ToCSV(buf)
   414  		if err != nil {
   415  			b.Errorf("Unexpected ToCSV error: %s", err)
   416  		}
   417  	}
   418  }
   419  
   420  // NOP writer just to make sure we don't contaminate the benchmarks
   421  // the performance characteristics of the writer implementation.
   422  type dummyWriter struct{}
   423  
   424  func (_ dummyWriter) Write(b []byte) (int, error) {
   425  	return len(b), nil
   426  }
   427  
   428  func BenchmarkQFrame_ToJSONRecords(b *testing.B) {
   429  	rowCount := 100000
   430  	input := exampleData(rowCount)
   431  	df := qf.New(input)
   432  	if df.Err != nil {
   433  		b.Errorf("Unexpected New error: %s", df.Err)
   434  	}
   435  
   436  	b.ReportAllocs()
   437  	b.ResetTimer()
   438  
   439  	for i := 0; i < b.N; i++ {
   440  		err := df.ToJSON(dummyWriter{})
   441  		if err != nil {
   442  			b.Errorf("Unexpected ToJSON error: %s", err)
   443  		}
   444  	}
   445  }
   446  
   447  func BenchmarkQFrame_FilterEnumVsString(b *testing.B) {
   448  	rowCount := 100000
   449  	cardinality := 9
   450  	input := csvEnumBytes(rowCount, cardinality)
   451  
   452  	table := []struct {
   453  		types         map[string]string
   454  		column        string
   455  		filter        string
   456  		expectedCount int
   457  		comparator    string
   458  	}{
   459  		{
   460  			types:         map[string]string{"COL1": "enum", "COL2": "enum"},
   461  			column:        "COL1",
   462  			filter:        "Foo bar baz 5",
   463  			expectedCount: 55556,
   464  		},
   465  		{
   466  			types:         map[string]string{},
   467  			column:        "COL1",
   468  			filter:        "Foo bar baz 5",
   469  			expectedCount: 55556,
   470  		},
   471  		{
   472  			types:         map[string]string{},
   473  			column:        "COL2",
   474  			filter:        "AB5",
   475  			expectedCount: 55556,
   476  		},
   477  		{
   478  			types:         map[string]string{},
   479  			column:        "COL1",
   480  			filter:        "%bar baz 5%",
   481  			expectedCount: 11111,
   482  			comparator:    "like",
   483  		},
   484  		{
   485  			types:         map[string]string{},
   486  			column:        "COL1",
   487  			filter:        "%bar baz 5%",
   488  			expectedCount: 11111,
   489  			comparator:    "ilike",
   490  		},
   491  		{
   492  			types:         map[string]string{"COL1": "enum", "COL2": "enum"},
   493  			column:        "COL1",
   494  			filter:        "%bar baz 5%",
   495  			expectedCount: 11111,
   496  			comparator:    "ilike",
   497  		},
   498  	}
   499  	for _, tc := range table {
   500  		r := bytes.NewReader(input)
   501  		df := qf.ReadCSV(r, csv.Types(tc.types))
   502  		if tc.comparator == "" {
   503  			tc.comparator = "<"
   504  		}
   505  
   506  		b.Run(fmt.Sprintf("Filter %s %s, enum: %t", tc.filter, tc.comparator, len(tc.types) > 0), func(b *testing.B) {
   507  			b.ReportAllocs()
   508  			b.ResetTimer()
   509  			for i := 0; i < b.N; i++ {
   510  				newDf := df.Filter(qf.Filter{Comparator: tc.comparator, Column: tc.column, Arg: tc.filter})
   511  				if newDf.Len() != tc.expectedCount {
   512  					b.Errorf("Unexpected count: %d, expected: %d", newDf.Len(), tc.expectedCount)
   513  				}
   514  			}
   515  		})
   516  	}
   517  }
   518  
   519  func benchApply(b *testing.B, name string, input qf.QFrame, fn interface{}) {
   520  	b.Helper()
   521  	b.Run(name, func(b *testing.B) {
   522  		b.ReportAllocs()
   523  		b.ResetTimer()
   524  		for i := 0; i < b.N; i++ {
   525  			result := input.Apply(qf.Instruction{Fn: fn, DstCol: "COL1", SrcCol1: "COL1"})
   526  			if result.Err != nil {
   527  				b.Errorf("Err: %d, %s", result.Len(), result.Err)
   528  			}
   529  		}
   530  	})
   531  
   532  }
   533  
   534  func BenchmarkQFrame_ApplyStringToString(b *testing.B) {
   535  	rowCount := 100000
   536  	cardinality := 9
   537  	input := csvEnumBytes(rowCount, cardinality)
   538  	r := bytes.NewReader(input)
   539  	df := qf.ReadCSV(r)
   540  
   541  	benchApply(b, "Instruction with custom function", df, toUpper)
   542  	benchApply(b, "Instruction with builtin function", df, "ToUpper")
   543  }
   544  
   545  func BenchmarkQFrame_ApplyEnum(b *testing.B) {
   546  	rowCount := 100000
   547  	cardinality := 9
   548  	input := csvEnumBytes(rowCount, cardinality)
   549  	r := bytes.NewReader(input)
   550  	df := qf.ReadCSV(r, csv.Types(map[string]string{"COL1": "enum"}))
   551  
   552  	benchApply(b, "Instruction with custom function", df, toUpper)
   553  	benchApply(b, "Instruction with built in function", df, "ToUpper")
   554  	benchApply(b, "Instruction int function (for reference)", df, func(x *string) int { return len(*x) })
   555  }
   556  
   557  func BenchmarkQFrame_IntView(b *testing.B) {
   558  	f := qf.New(map[string]interface{}{"S1": genInts(seed1, frameSize)}).Sort(qf.Order{Column: "S1"})
   559  	v, err := f.IntView("S1")
   560  	if err != nil {
   561  		b.Error(err)
   562  	}
   563  
   564  	b.Run("For loop", func(b *testing.B) {
   565  		b.ReportAllocs()
   566  		b.ResetTimer()
   567  		for i := 0; i < b.N; i++ {
   568  			result := 0
   569  			for j := 0; j < v.Len(); j++ {
   570  				result += v.ItemAt(j)
   571  			}
   572  
   573  			// Don't allow the result to be optimized away
   574  			if result == 0 {
   575  				b.Fail()
   576  			}
   577  		}
   578  	})
   579  
   580  	b.Run("Slice", func(b *testing.B) {
   581  		b.ReportAllocs()
   582  		b.ResetTimer()
   583  		for i := 0; i < b.N; i++ {
   584  			result := 0
   585  			for _, j := range v.Slice() {
   586  				result += j
   587  			}
   588  
   589  			// Don't allow the result to be optimized away
   590  			if result == 0 {
   591  				b.Fail()
   592  			}
   593  		}
   594  	})
   595  
   596  }
   597  
   598  func BenchmarkQFrame_StringView(b *testing.B) {
   599  	rowCount := 100000
   600  	cardinality := 9
   601  	input := csvEnumBytes(rowCount, cardinality)
   602  	r := bytes.NewReader(input)
   603  	f := qf.ReadCSV(r).Sort(qf.Order{Column: "COL1"})
   604  	v, err := f.StringView("COL1")
   605  	if err != nil {
   606  		b.Error(err)
   607  	}
   608  
   609  	b.Run("For loop", func(b *testing.B) {
   610  		b.ReportAllocs()
   611  		b.ResetTimer()
   612  		for i := 0; i < b.N; i++ {
   613  			var last *string
   614  			for j := 0; j < v.Len(); j++ {
   615  				last = v.ItemAt(j)
   616  			}
   617  
   618  			// Don't allow the result to be optimized away
   619  			if len(*last) == 0 {
   620  				b.Fail()
   621  			}
   622  		}
   623  	})
   624  
   625  	b.Run("Slice", func(b *testing.B) {
   626  		b.ReportAllocs()
   627  		b.ResetTimer()
   628  		for i := 0; i < b.N; i++ {
   629  			var last *string
   630  			for _, j := range v.Slice() {
   631  				last = j
   632  			}
   633  
   634  			// Don't allow the result to be optimized away
   635  			if len(*last) == 0 {
   636  				b.Fail()
   637  			}
   638  		}
   639  	})
   640  }
   641  
   642  func BenchmarkQFrame_EvalInt(b *testing.B) {
   643  	df := exampleIntFrame(100000)
   644  	b.ReportAllocs()
   645  	b.ResetTimer()
   646  	for i := 0; i < b.N; i++ {
   647  		result := df.Eval("RESULT", qf.Expr("+", qf.Expr("+", types.ColumnName("S1"), types.ColumnName("S2")), qf.Val(2)))
   648  		if result.Err != nil {
   649  			b.Errorf("Err: %d, %s", result.Len(), result.Err)
   650  		}
   651  	}
   652  }
   653  
   654  func BenchmarkGroupBy(b *testing.B) {
   655  	table := []struct {
   656  		name         string
   657  		size         int
   658  		cardinality1 int
   659  		cardinality2 int
   660  		cardinality3 int
   661  		cols         []string
   662  	}{
   663  		{name: "single col", size: 100000, cardinality1: 1000, cardinality2: 10, cardinality3: 2, cols: []string{"COL1"}},
   664  		{name: "triple col", size: 100000, cardinality1: 1000, cardinality2: 10, cardinality3: 2, cols: []string{"COL1", "COL2", "COL3"}},
   665  		{name: "high cardinality", size: 100000, cardinality1: 50000, cardinality2: 1, cardinality3: 1, cols: []string{"COL1"}},
   666  		{name: "low cardinality", size: 100000, cardinality1: 5, cardinality2: 1, cardinality3: 1, cols: []string{"COL1"}},
   667  		{name: "small frame", size: 100, cardinality1: 20, cardinality2: 1, cardinality3: 1, cols: []string{"COL1"}},
   668  	}
   669  
   670  	for _, tc := range table {
   671  		for _, dataType := range []string{"string", "integer"} {
   672  			b.Run(fmt.Sprintf("%s dataType=%s", tc.name, dataType), func(b *testing.B) {
   673  				var input map[string]interface{}
   674  				if dataType == "integer" {
   675  					input = map[string]interface{}{
   676  						"COL1": genIntsWithCardinality(seed1, tc.size, tc.cardinality1),
   677  						"COL2": genIntsWithCardinality(seed2, tc.size, tc.cardinality2),
   678  						"COL3": genIntsWithCardinality(seed3, tc.size, tc.cardinality3),
   679  					}
   680  				} else {
   681  					input = map[string]interface{}{
   682  						"COL1": genStringsWithCardinality(seed1, tc.size, tc.cardinality1, 10),
   683  						"COL2": genStringsWithCardinality(seed2, tc.size, tc.cardinality2, 10),
   684  						"COL3": genStringsWithCardinality(seed3, tc.size, tc.cardinality3, 10),
   685  					}
   686  				}
   687  				df := qf.New(input)
   688  				b.ReportAllocs()
   689  				b.ResetTimer()
   690  				var stats qf.GroupStats
   691  				for i := 0; i < b.N; i++ {
   692  					grouper := df.GroupBy(groupby.Columns(tc.cols...))
   693  					if grouper.Err != nil {
   694  						b.Errorf(grouper.Err.Error())
   695  					}
   696  					stats = grouper.Stats
   697  				}
   698  
   699  				_ = stats
   700  				// b.Logf("Stats: %#v", stats)
   701  
   702  				/*
   703  					// Remember to put -alloc_space there otherwise it will be empty since no space is used anymore
   704  					go tool pprof -alloc_space qframe.test mem_singlegroup.prof/
   705  
   706  					(pprof) web
   707  					(pprof) list insertEntry
   708  
   709  				*/
   710  			})
   711  		}
   712  	}
   713  }
   714  
   715  func BenchmarkDistinctNull(b *testing.B) {
   716  	inputLen := 100000
   717  	input := make([]*string, inputLen)
   718  	foo := "foo"
   719  	input[0] = &foo
   720  	df := qf.New(map[string]interface{}{"COL1": input})
   721  
   722  	table := []struct {
   723  		groupByNull bool
   724  		expectedLen int
   725  	}{
   726  		{groupByNull: false, expectedLen: inputLen},
   727  		{groupByNull: true, expectedLen: 2},
   728  	}
   729  
   730  	for _, tc := range table {
   731  		b.Run(fmt.Sprintf("groupByNull=%v", tc.groupByNull), func(b *testing.B) {
   732  			b.ReportAllocs()
   733  			for i := 0; i < b.N; i++ {
   734  				out := df.Distinct(groupby.Columns("COL1"), groupby.Null(tc.groupByNull))
   735  				if out.Err != nil {
   736  					b.Errorf(out.Err.Error())
   737  
   738  				}
   739  				if tc.expectedLen != out.Len() {
   740  					b.Errorf("%d != %d", tc.expectedLen, out.Len())
   741  				}
   742  			}
   743  		})
   744  	}
   745  }
   746  
   747  /*
   748  Go 1.7
   749  
   750  go test -bench=.
   751  tpp
   752  go tool pprof dataframe.test filter_cpu.out
   753  
   754  Initial results:
   755  BenchmarkDataFrame_Filter-2     	      30	  40542568 ns/op	 7750730 B/op	  300134 allocs/op
   756  BenchmarkQCacheFrame_Filter-2   	     300	   3997702 ns/op	  991720 B/op	      14 allocs/op
   757  
   758  After converting bool index to int index before subsetting:
   759  BenchmarkDataFrame_Filter-2     	      30	  40330898 ns/op	 7750731 B/op	  300134 allocs/op
   760  BenchmarkQCacheFrame_Filter-2   	     500	   2631666 ns/op	 2098409 B/op	      38 allocs/op
   761  
   762  Only evolve indexes, don't realize the dataframe (note that the tests tests are running slower in general,
   763  the BenchmarkDataFrame_Filter is the exact same as above):
   764  BenchmarkDataFrame_Filter-2     	      30	  46309948 ns/op	 7750730 B/op	  300134 allocs/op
   765  BenchmarkQCacheFrame_Filter-2   	    1000	   2083198 ns/op	  606505 B/op	      29 allocs/op
   766  
   767  Initial sorting implementation using built in interface-based sort.Sort. Not sure if this is actually
   768  OK going forward since the Sort is not guaranteed to be stable.
   769  BenchmarkDataFrame_Sort-2     	       5	 245155627 ns/op	50547024 B/op	     148 allocs/op
   770  BenchmarkQFrame_Sort-2        	      20	  78297649 ns/op	  401504 B/op	       3 allocs/op
   771  
   772  Sorting using a copy of the stdlib Sort but with the Interface switched to a concrete type. A fair
   773  bit quicker but not as quick as expected.
   774  BenchmarkDataFrame_Filter-2   	      30	  46760882 ns/op	 7750731 B/op	  300134 allocs/op
   775  BenchmarkQFrame_Filter-2      	    1000	   2062230 ns/op	  606504 B/op	      29 allocs/op
   776  BenchmarkDataFrame_Sort-2     	       5	 242068573 ns/op	50547024 B/op	     148 allocs/op
   777  BenchmarkQFrame_Sort-2        	      30	  50057905 ns/op	  401408 B/op	       1 allocs/op
   778  
   779  Sorting done using above copy but using stable sort for all but the last order by column.
   780  BenchmarkDataFrame_Filter-2   	      30	  44818293 ns/op	 7750731 B/op	  300134 allocs/op
   781  BenchmarkQFrame_Filter-2      	    1000	   2126636 ns/op	  606505 B/op	      29 allocs/op
   782  BenchmarkDataFrame_Sort-2     	       5	 239796901 ns/op	50547024 B/op	     148 allocs/op
   783  BenchmarkQFrame_Sort-2        	      10	 119140365 ns/op	  401408 B/op	       1 allocs/op
   784  
   785  Test using timsort instead of built in sort, gives stability by default. Better, but slightly disappointing.
   786  BenchmarkDataFrame_Filter-2   	      30	  44576205 ns/op	 7750731 B/op	  300134 allocs/op
   787  BenchmarkQFrame_Filter-2      	    1000	   2121513 ns/op	  606504 B/op	      29 allocs/op
   788  BenchmarkDataFrame_Sort-2     	       5	 245788389 ns/op	50547024 B/op	     148 allocs/op
   789  BenchmarkQFrame_Sort-2        	      20	  94122521 ns/op	 3854980 B/op	      25 allocs/op
   790  
   791  // timsort
   792  BenchmarkDataFrame_Filter-2    	      30	  47960157 ns/op	 7750731 B/op	  300134 allocs/op
   793  BenchmarkQFrame_Filter-2       	    1000	   2174167 ns/op	  606504 B/op	      29 allocs/op
   794  BenchmarkDataFrame_Sort-2      	       5	 281561310 ns/op	50547024 B/op	     148 allocs/op
   795  BenchmarkQFrame_Sort-2         	      20	  98123611 ns/op	 3854984 B/op	      25 allocs/op
   796  BenchmarkQFrame_Sort1Col-2     	      30	  45322479 ns/op	 2128192 B/op	      13 allocs/op
   797  BenchmarkQFrame_SortSorted-2   	     300	   4428537 ns/op	 2011788 B/op	       9 allocs/op
   798  
   799  // stdlib specific
   800  BenchmarkDataFrame_Filter-2    	      20	  50015836 ns/op	 7750730 B/op	  300134 allocs/op
   801  BenchmarkQFrame_Filter-2       	     500	   2205289 ns/op	  606504 B/op	      29 allocs/op
   802  BenchmarkDataFrame_Sort-2      	       5	 270738781 ns/op	50547024 B/op	     148 allocs/op
   803  BenchmarkQFrame_Sort-2         	      10	 137043496 ns/op	  401408 B/op	       1 allocs/op
   804  BenchmarkQFrame_Sort1Col-2     	      50	  30669308 ns/op	  401408 B/op	       1 allocs/op
   805  BenchmarkQFrame_SortSorted-2   	      50	  28217092 ns/op	  401408 B/op	       1 allocs/op
   806  
   807  // stdlib
   808  BenchmarkDataFrame_Filter-2    	      30	  50137069 ns/op	 7750731 B/op	  300134 allocs/op
   809  BenchmarkQFrame_Filter-2       	    1000	   2308053 ns/op	  606504 B/op	      29 allocs/op
   810  BenchmarkDataFrame_Sort-2      	       5	 288688150 ns/op	50547024 B/op	     148 allocs/op
   811  BenchmarkQFrame_Sort-2         	      10	 206407019 ns/op	  401536 B/op	       3 allocs/op
   812  BenchmarkQFrame_Sort1Col-2     	      30	  46005496 ns/op	  401472 B/op	       2 allocs/op
   813  BenchmarkQFrame_SortSorted-2   	      20	  54300644 ns/op	  401536 B/op	       3 allocs/op
   814  
   815  // stdlib specific + co-locate data to sort on, ~2x speedup compared to separate index
   816  BenchmarkDataFrame_Filter-2    	      30	  46678558 ns/op	 7750731 B/op	  300134 allocs/op
   817  BenchmarkQFrame_Filter-2       	    1000	   2218767 ns/op	  606504 B/op	      29 allocs/op
   818  BenchmarkDataFrame_Sort-2      	       5	 254261311 ns/op	50547024 B/op	     148 allocs/op
   819  BenchmarkQFrame_Sort-2         	      20	  68903882 ns/op	 3612672 B/op	       3 allocs/op
   820  BenchmarkQFrame_Sort1Col-2     	     100	  15970577 ns/op	 2007040 B/op	       2 allocs/op
   821  BenchmarkQFrame_SortSorted-2   	     100	  14389450 ns/op	 3612672 B/op	       3 allocs/op
   822  
   823  // Different sort implementation that likely performs better for multi column sort but
   824  // slightly worse for singe column sort.
   825  BenchmarkQFrame_Sort-2         	      30	  47600788 ns/op	  401626 B/op	       4 allocs/op
   826  BenchmarkQFrame_Sort1Col-2     	      30	  43807643 ns/op	  401472 B/op	       3 allocs/op
   827  BenchmarkQFrame_SortSorted-2   	      50	  24775838 ns/op	  401536 B/op	       4 allocs/op
   828  
   829  // Initial CSV implementation for int, 4 x 100000.
   830  BenchmarkQFrame_IntFromCSV-2      	      20	  55921060 ns/op	30167012 B/op	     261 allocs/op
   831  BenchmarkDataFrame_IntFromCSV-2   	       5	 243541282 ns/op	41848809 B/op	  900067 allocs/op
   832  
   833  // Type detecting CSV implementation, 100000 x "123", "1234567", "5.2534", "9834543.25", "true", "Foo bar baz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
   834  BenchmarkQFrame_IntFromCSV-2   	      10	 101362864 ns/op	87707785 B/op	  200491 allocs/op
   835  
   836  // JSON, 10000 rows
   837  BenchmarkDataFrame_ReadJSON-2          	      10	 176107262 ns/op	24503045 B/op	  670112 allocs/op
   838  BenchmarkQFrame_FromJSONRecords-2   	      10	 117408651 ns/op	15132420 B/op	  430089 allocs/op
   839  BenchmarkQFrame_FromJSONColumns-2   	      10	 104641079 ns/op	15342302 B/op	  220842 allocs/op
   840  
   841  // JSON with easyjson generated unmarshal
   842  BenchmarkQFrame_FromJSONColumns-2   	      50	  24764232 ns/op	 6730738 B/op	   20282 allocs/op
   843  
   844  // ToCSV, vanilla implementation based on stdlib csv, 100000 records
   845  BenchmarkQFrame_ToCSV-2   	       5	 312478023 ns/op	26365360 B/op	  600017 allocs/op
   846  
   847  // ToJSON, performance is not super impressive... 100000 records
   848  BenchmarkQFrame_ToJSONRecords-2   	       2	 849280921 ns/op	181573400 B/op	 3400028 allocs/op
   849  BenchmarkQFrame_ToJSONColumns-2   	       5	 224702680 ns/op	33782697 B/op	     513 allocs/op
   850  
   851  // Testing jsoniter with some success
   852  BenchmarkQFrame_ToJSONRecords-2   	       2	 646738504 ns/op	137916264 B/op	 3600006 allocs/op
   853  BenchmarkQFrame_ToJSONColumns-2   	      20	  99932317 ns/op	34144682 B/op	     490 allocs/op
   854  
   855  // Python, as a comparison, with corresponding list of dictionaries:
   856  >>> import json
   857  >>> import time
   858  >>> t0 = time.time(); j = json.dumps(x); time.time() - t0
   859  0.33017611503601074
   860  >>> import ujson
   861  >>> t0 = time.time(); j = ujson.dumps(x); time.time() - t0
   862  0.17484211921691895
   863  
   864  // Custom encoder for JSON records, now we're talking
   865  BenchmarkQFrame_ToJSONRecords-2   	      20	  87437635 ns/op	53638858 B/op	      35 allocs/op
   866  BenchmarkQFrame_ToJSONColumns-2   	      10	 102566155 ns/op	37746546 B/op	     547 allocs/op
   867  
   868  // Reuse string pointers when reading CSV
   869  Before:
   870  BenchmarkQFrame_ReadCSV-2   	      10	 119385221 ns/op	92728576 B/op	  400500 allocs/op
   871  
   872  After:
   873  BenchmarkQFrame_ReadCSV-2   	      10	 108917111 ns/op	86024686 B/op	   20790 allocs/op
   874  
   875  // Initial CSV read Enum, 2 x 100000 cells with cardinality 20
   876  BenchmarkQFrame_ReadCSVEnum/Type_enum-2         	      50	  28081769 ns/op	19135232 B/op	     213 allocs/op
   877  BenchmarkQFrame_ReadCSVEnum/Type_string-2       	      50	  28563580 ns/op	20526743 B/op	     238 allocs/op
   878  
   879  Total saving 1,4 Mb in line with what was expected given that one byte is used per entry instead of eight
   880  
   881  // Enum vs string filtering
   882  BenchmarkQFrame_FilterEnumVsString/Test_0-2         	    2000	    714369 ns/op	  335888 B/op	       3 allocs/op
   883  BenchmarkQFrame_FilterEnumVsString/Test_1-2         	    1000	   1757913 ns/op	  335888 B/op	       3 allocs/op
   884  BenchmarkQFrame_FilterEnumVsString/Test_2-2         	    1000	   1792186 ns/op	  335888 B/op	       3 allocs/op
   885  
   886  // Initial "(i)like" matching of strings using regexes
   887  
   888  Case sensitive:
   889  BenchmarkQFrame_FilterEnumVsString/Test_3-2         	     100	  11765579 ns/op	  162600 B/op	      74 allocs/op
   890  
   891  Case insensitive:
   892  BenchmarkQFrame_FilterEnumVsString/Test_4-2         	      30	  41680939 ns/op	  163120 B/op	      91 allocs/op
   893  
   894  // Remove the need for regexp in many cases:
   895  BenchmarkQFrame_FilterEnumVsString/Filter_Foo_bar_baz_5_<-2         	    2000	    692662 ns/op	  335888 B/op	       3 allocs/op
   896  BenchmarkQFrame_FilterEnumVsString/Filter_Foo_bar_baz_5_<#01-2      	    1000	   1620056 ns/op	  335893 B/op	       3 allocs/op
   897  BenchmarkQFrame_FilterEnumVsString/Filter_AB5_<-2                   	    1000	   1631806 ns/op	  335888 B/op	       3 allocs/op
   898  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_like-2        	     500	   3245751 ns/op	  155716 B/op	       4 allocs/op
   899  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_ilike-2       	     100	  11418693 ns/op	  155873 B/op	       8 allocs/op
   900  
   901  // Enum string matching, speedy:
   902  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_ilike,_enum:_false-2      	     100	  11583233 ns/op	  155792 B/op	       8 allocs/op
   903  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_ilike,_enum:_true-2       	    2000	    729671 ns/op	  155989 B/op	      13 allocs/op
   904  
   905  // Inverse (not) filtering:
   906  BenchmarkQFrame_FilterNot-2   	    2000	    810831 ns/op	  147459 B/op	       2 allocs/op
   907  
   908  // Performance tweak for single, simple, clause statements to put them on par with calling the
   909  // Qframe Filter function directly
   910  
   911  // Before
   912  BenchmarkQFrame_FilterNot/qframe-2         	    2000	    716280 ns/op	  147465 B/op	       2 allocs/op
   913  BenchmarkQFrame_FilterNot/filter-2         	    2000	   1158211 ns/op	  516161 B/op	       4 allocs/op
   914  
   915  // After
   916  BenchmarkQFrame_FilterNot/qframe-2         	    2000	    713147 ns/op	  147465 B/op	       2 allocs/op
   917  BenchmarkQFrame_FilterNot/filter-2         	    2000	    726766 ns/op	  147521 B/op	       3 allocs/op
   918  
   919  // Restructure string column to use a byte blob with offsets and lengths
   920  BenchmarkQFrame_ReadCSV-2       	      20	  85906027 ns/op	84728656 B/op	     500 allocs/op
   921  
   922  // Fix string clause to make better use of the new string blob structure:
   923  BenchmarkQFrame_FilterEnumVsString/Filter_Foo_bar_baz_5_<,_enum:_true-2         	    2000	    691081 ns/op	  335888 B/op	       3 allocs/op
   924  BenchmarkQFrame_FilterEnumVsString/Filter_Foo_bar_baz_5_<,_enum:_false-2        	    1000	   1902665 ns/op	  335889 B/op	       3 allocs/op
   925  BenchmarkQFrame_FilterEnumVsString/Filter_AB5_<,_enum:_false-2                  	    1000	   1935237 ns/op	  335888 B/op	       3 allocs/op
   926  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_like,_enum:_false-2       	     500	   3855434 ns/op	  155680 B/op	       4 allocs/op
   927  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_ilike,_enum:_false-2      	     100	  11881963 ns/op	  155792 B/op	       8 allocs/op
   928  BenchmarkQFrame_FilterEnumVsString/Filter_%bar_baz_5%_ilike,_enum:_true-2       	    2000	    691971 ns/op	  155824 B/op	       9 allocs/op
   929  
   930  // Compare string to upper, first as general custom function, second as specialized built in function.
   931  BenchmarkQFrame_ApplyStringToString/Apply_with_custom_function-2         	      30	  42895890 ns/op	17061043 B/op	  400020 allocs/op
   932  BenchmarkQFrame_ApplyStringToString/Apply_with_built_in_function-2       	     100	  12163217 ns/op	 2107024 B/op	       7 allocs/op
   933  
   934  // Compare apply for enums
   935  BenchmarkQFrame_ApplyEnum/Apply_with_custom_function-2         	      50	  38505068 ns/op	15461041 B/op	  300020 allocs/op
   936  BenchmarkQFrame_ApplyEnum/Apply_with_built_in_function-2       	  300000	      3566 ns/op	    1232 B/op	      23 allocs/op
   937  BenchmarkQFrame_ApplyEnum/Apply_int_function_(for_reference)-2 	    1000	   1550604 ns/op	  803491 B/op	       6 allocs/op
   938  
   939  // The difference in using built in filter vs general filter func passed as argument. Basically the overhead of a function
   940  // call for each row. Smaller than I would have thought actually.
   941  BenchmarkQFrame_FilterIntBuiltIn-2   	    1000	   1685483 ns/op	  221184 B/op	       2 allocs/op
   942  BenchmarkQFrame_FilterIntGeneral-2   	     500	   2631678 ns/op	  221239 B/op	       5 allocs/op
   943  
   944  // Only minor difference in performance between built in and general filtering here. Map access dominates
   945  // the execution time.
   946  BenchmarkQFrame_FilterIntGeneralIn-2   	     500	   3321307 ns/op	  132571 B/op	      10 allocs/op
   947  BenchmarkQFrame_FilterIntBuiltinIn-2   	     500	   3055410 ns/op	  132591 B/op	      10 allocs/op
   948  
   949  // Without the sort the slice version is actually a bit faster even though it allocates a new slice and iterates
   950  // over the data twice.
   951  BenchmarkQFrame_IntView/For_loop-2         	    2000	    763169 ns/op	       0 B/op	       0 allocs/op
   952  BenchmarkQFrame_IntView/Slice-2            	    2000	    806672 ns/op	  802816 B/op	       1 allocs/op
   953  
   954  BenchmarkQFrame_StringView/For_loop-2         	     200	   6242471 ns/op	 1600000 B/op	  100000 allocs/op
   955  BenchmarkQFrame_StringView/Slice-2            	     100	  14006634 ns/op	 4002816 B/op	  200001 allocs/op
   956  
   957  // Same as above but modified to work with enums in COL1
   958  BenchmarkQFrame_StringView/For_loop-2         	    1000	   1651190 ns/op	       0 B/op	       0 allocs/op
   959  BenchmarkQFrame_StringView/Slice-2            	     500	   2697675 ns/op	  802816 B/op	       1 allocs/op
   960  
   961  // Most of the time is spent in icolumn.Apply2
   962  BenchmarkQFrame_EvalInt-2   	     500	   2461435 ns/op	 2416968 B/op	      69 allocs/op
   963  
   964  // Hash based group by and distinct
   965  BenchmarkGroupBy/single_col_dataType=string-2         	     100	  15649028 ns/op	 2354704 B/op	    7012 allocs/op
   966  BenchmarkGroupBy/single_col_dataType=integer-2        	     200	   9231345 ns/op	 2354672 B/op	    7012 allocs/op
   967  BenchmarkGroupBy/triple_col_dataType=string-2         	      20	  61141105 ns/op	 5300345 B/op	   49990 allocs/op
   968  BenchmarkGroupBy/triple_col_dataType=integer-2        	      50	  28986440 ns/op	 5300250 B/op	   49990 allocs/op
   969  BenchmarkGroupBy/high_cardinality_dataType=string-2   	      30	  36929671 ns/op	10851690 B/op	   62115 allocs/op
   970  BenchmarkGroupBy/high_cardinality_dataType=integer-2  	      50	  28362647 ns/op	10851660 B/op	   62115 allocs/op
   971  BenchmarkGroupBy/low_cardinality_dataType=string-2    	     100	  12705659 ns/op	 3194024 B/op	     114 allocs/op
   972  BenchmarkGroupBy/low_cardinality_dataType=integer-2   	     200	   7764495 ns/op	 3193995 B/op	     114 allocs/op
   973  BenchmarkGroupBy/small_frame_dataType=string-2        	  100000	     18085 ns/op	    5736 B/op	      62 allocs/op
   974  BenchmarkGroupBy/small_frame_dataType=integer-2       	  100000	     12313 ns/op	    5704 B/op	      62 allocs/op
   975  P
   976  BenchmarkDistinctNull/groupByNull=false-2         	      30	  38197889 ns/op	15425856 B/op	      13 allocs/op
   977  BenchmarkDistinctNull/groupByNull=true-2          	     100	  10925589 ns/op	 1007945 B/op	      10 allocs/op
   978  */