github.com/viant/toolbox@v0.34.5/data/compacted_test.go (about)

     1  package data
     2  
     3  import (
     4  	"encoding/json"
     5  	"github.com/stretchr/testify/assert"
     6  	"github.com/viant/toolbox"
     7  	"testing"
     8  	"time"
     9  )
    10  
    11  func TestNewCollection(t *testing.T) {
    12  	if ! canRun64BitArch() {
    13  		t.Skip()
    14  	}
    15  
    16  	collection := NewCompactedSlice(true, true)
    17  	collection.Add(map[string]interface{}{
    18  		"f1":  1,
    19  		"f12": 1,
    20  		"f15": 1,
    21  		"f20": 1,
    22  		"f11": nil,
    23  		"f13": "",
    24  	})
    25  
    26  	collection.Add(map[string]interface{}{
    27  		"f1":  1,
    28  		"f32": 1,
    29  		"f35": 1,
    30  		"f30": 1,
    31  		"f31": nil,
    32  		"f33": "",
    33  		"f11": 0,
    34  		"f36": 0.0,
    35  	})
    36  
    37  	var actual = []map[string]interface{}{}
    38  	err := collection.Range(func(data interface{}) (bool, error) {
    39  		actual = append(actual, toolbox.AsMap(data))
    40  		return true, nil
    41  	})
    42  	assert.Nil(t, err)
    43  	assert.Equal(t, 2, len(actual))
    44  	assert.Equal(t, map[string]interface{}{
    45  		"f1":  1,
    46  		"f12": 1,
    47  		"f15": 1,
    48  		"f20": 1,
    49  	}, actual[0])
    50  
    51  	assert.Equal(t, map[string]interface{}{
    52  		"f1":  1,
    53  		"f32": 1,
    54  		"f35": 1,
    55  		"f30": 1,
    56  	}, actual[1])
    57  }
    58  
    59  func Test_optimizedStorage(t *testing.T) {
    60  	if ! canRun64BitArch() {
    61  		t.Skip()
    62  	}
    63  	collection := NewCompactedSlice(true, true)
    64  	var data = []interface{}{nil, nil, nil, "123", nil, nil, "abc", 12, nil, nil, nil, "a"}
    65  	var compressed = []interface{}{nilGroup(3), "123", nilGroup(2), "abc", 12, nilGroup(3), "a"}
    66  	var optimized = collection.compress(data)
    67  	assert.EqualValues(t, compressed, optimized)
    68  	collection.fields = make([]*Field, 12)
    69  	var uncompressed = make([]interface{}, len(collection.fields))
    70  	collection.uncompress(compressed, uncompressed)
    71  	assert.EqualValues(t, data, uncompressed)
    72  }
    73  
    74  func TestCompactedSlice_SortedRange(t *testing.T) {
    75  	if ! canRun64BitArch() {
    76  		t.Skip()
    77  	}
    78  	var useCases = []struct {
    79  		description string
    80  		data        []map[string]interface{}
    81  		expected    []interface{}
    82  		indexBy     []string
    83  		hasError    bool
    84  	}{
    85  		{
    86  			description: "int sorting",
    87  			indexBy:     []string{"id"},
    88  			data: []map[string]interface{}{
    89  				{
    90  					"id":   10,
    91  					"name": "name 10",
    92  				},
    93  				{
    94  					"id":   3,
    95  					"name": "name 3",
    96  				},
    97  				{
    98  					"id":   1,
    99  					"name": "name 1",
   100  				},
   101  				{
   102  					"id":   2,
   103  					"name": "name 2",
   104  				},
   105  			},
   106  			expected: []interface{}{
   107  				1, 2, 3, 10,
   108  			},
   109  		},
   110  		{
   111  			description: "float sorting",
   112  			indexBy:     []string{"id"},
   113  			data: []map[string]interface{}{
   114  				{
   115  					"id":   10.0,
   116  					"name": "name 10",
   117  				},
   118  				{
   119  					"id":   3.1,
   120  					"name": "name 3",
   121  				},
   122  				{
   123  					"id":   1.2,
   124  					"name": "name 1",
   125  				},
   126  				{
   127  					"id":   2.2,
   128  					"name": "name 2",
   129  				},
   130  			},
   131  			expected: []interface{}{
   132  				1.2, 2.2, 3.1, 10.0,
   133  			},
   134  		},
   135  		{
   136  			description: "string sorting",
   137  			indexBy:     []string{"id"},
   138  			data: []map[string]interface{}{
   139  				{
   140  					"id":   "010",
   141  					"name": "name 10",
   142  				},
   143  				{
   144  					"id":   "003",
   145  					"name": "name 3",
   146  				},
   147  				{
   148  					"id":   "001",
   149  					"name": "name 1",
   150  				},
   151  				{
   152  					"id":   "022",
   153  					"name": "name 2",
   154  				},
   155  			},
   156  			expected: []interface{}{
   157  				"001", "003", "010", "022",
   158  			},
   159  		},
   160  		{
   161  			description: "combined index sorting",
   162  			indexBy:     []string{"id"},
   163  			data: []map[string]interface{}{
   164  				{
   165  					"id":   1,
   166  					"u":    1,
   167  					"name": "name 10",
   168  				},
   169  				{
   170  					"id":   3,
   171  					"u":    2,
   172  					"name": "name 3",
   173  				},
   174  				{
   175  					"id":   2,
   176  					"u":    2,
   177  					"name": "name 1",
   178  				},
   179  				{
   180  					"id":   4,
   181  					"u":    6,
   182  					"name": "name 2",
   183  				},
   184  			},
   185  			expected: []interface{}{
   186  				1, 2, 3, 4,
   187  			},
   188  		},
   189  		{
   190  			description: "missing Field",
   191  			indexBy:     []string{"field1"},
   192  			data: []map[string]interface{}{
   193  				{
   194  					"id":   1,
   195  					"u":    1,
   196  					"name": "name 10",
   197  				},
   198  			},
   199  			hasError: true,
   200  		},
   201  		{
   202  			description: "unsupported index type Field",
   203  			indexBy:     []string{"id"},
   204  			data: []map[string]interface{}{
   205  				{
   206  					"id":   time.Now(),
   207  					"u":    1,
   208  					"name": "name 10",
   209  				},
   210  			},
   211  			hasError: true,
   212  		},
   213  	}
   214  
   215  	for _, useCase := range useCases {
   216  		collection := NewCompactedSlice(true, true)
   217  		var actual = make([]interface{}, 0)
   218  		for _, item := range useCase.data {
   219  			collection.Add(item)
   220  		}
   221  		err := collection.SortedRange(useCase.indexBy, func(item interface{}) (b bool, e error) {
   222  			record := toolbox.AsMap(item)
   223  			actual = append(actual, record[useCase.indexBy[0]])
   224  			return true, nil
   225  		})
   226  		if useCase.hasError {
   227  			assert.NotNil(t, err, useCase.description)
   228  			continue
   229  		}
   230  		if !assert.Nil(t, err, useCase.description) {
   231  			continue
   232  		}
   233  		assert.EqualValues(t, useCase.expected, actual, useCase.description)
   234  	}
   235  
   236  }
   237  
   238  func TestCompactedSlice_SortedIterator(t *testing.T) {
   239  	if ! canRun64BitArch() {
   240  		t.Skip()
   241  	}
   242  	var useCases = []struct {
   243  		description string
   244  		data        []map[string]interface{}
   245  		expected    []interface{}
   246  		indexBy     []string
   247  		hasError    bool
   248  	}{
   249  		{
   250  			description: "int sorting",
   251  			indexBy:     []string{"id"},
   252  			data: []map[string]interface{}{
   253  				{
   254  					"id":   10,
   255  					"name": "name 10",
   256  				},
   257  				{
   258  					"id":   3,
   259  					"name": "name 3",
   260  				},
   261  				{
   262  					"id":   1,
   263  					"name": "name 1",
   264  				},
   265  				{
   266  					"id":   2,
   267  					"name": "name 2",
   268  				},
   269  			},
   270  			expected: []interface{}{
   271  				1, 2, 3, 10,
   272  			},
   273  		},
   274  		{
   275  			description: "float sorting",
   276  			indexBy:     []string{"id"},
   277  			data: []map[string]interface{}{
   278  				{
   279  					"id":   10.0,
   280  					"name": "name 10",
   281  				},
   282  				{
   283  					"id":   3.1,
   284  					"name": "name 3",
   285  				},
   286  				{
   287  					"id":   1.2,
   288  					"name": "name 1",
   289  				},
   290  				{
   291  					"id":   2.2,
   292  					"name": "name 2",
   293  				},
   294  			},
   295  			expected: []interface{}{
   296  				1.2, 2.2, 3.1, 10.0,
   297  			},
   298  		},
   299  		{
   300  			description: "string sorting",
   301  			indexBy:     []string{"id"},
   302  			data: []map[string]interface{}{
   303  				{
   304  					"id":   "010",
   305  					"name": "name 10",
   306  				},
   307  				{
   308  					"id":   "003",
   309  					"name": "name 3",
   310  				},
   311  				{
   312  					"id":   "001",
   313  					"name": "name 1",
   314  				},
   315  				{
   316  					"id":   "022",
   317  					"name": "name 2",
   318  				},
   319  			},
   320  			expected: []interface{}{
   321  				"001", "003", "010", "022",
   322  			},
   323  		},
   324  		{
   325  			description: "combined index sorting",
   326  			indexBy:     []string{"id"},
   327  			data: []map[string]interface{}{
   328  				{
   329  					"id":   1,
   330  					"u":    1,
   331  					"name": "name 10",
   332  				},
   333  				{
   334  					"id":   3,
   335  					"u":    2,
   336  					"name": "name 3",
   337  				},
   338  				{
   339  					"id":   2,
   340  					"u":    2,
   341  					"name": "name 1",
   342  				},
   343  				{
   344  					"id":   4,
   345  					"u":    6,
   346  					"name": "name 2",
   347  				},
   348  			},
   349  			expected: []interface{}{
   350  				1, 2, 3, 4,
   351  			},
   352  		},
   353  		{
   354  			description: "missing Field",
   355  			indexBy:     []string{"field1"},
   356  			data: []map[string]interface{}{
   357  				{
   358  					"id":   1,
   359  					"u":    1,
   360  					"name": "name 10",
   361  				},
   362  			},
   363  			hasError: true,
   364  		},
   365  		{
   366  			description: "unsupported index type Field",
   367  			indexBy:     []string{"id"},
   368  			data: []map[string]interface{}{
   369  				{
   370  					"id":   time.Now(),
   371  					"u":    1,
   372  					"name": "name 10",
   373  				},
   374  			},
   375  			hasError: true,
   376  		},
   377  	}
   378  
   379  	for _, useCase := range useCases {
   380  		collection := NewCompactedSlice(true, true)
   381  		var actual = make([]interface{}, 0)
   382  		for _, item := range useCase.data {
   383  			collection.Add(item)
   384  		}
   385  		iterator, err := collection.SortedIterator(useCase.indexBy)
   386  		if useCase.hasError {
   387  			assert.NotNil(t, err, useCase.description)
   388  			continue
   389  		}
   390  		if !assert.Nil(t, err, useCase.description) {
   391  			continue
   392  		}
   393  
   394  		var record map[string]interface{}
   395  		for iterator.HasNext() {
   396  			err = iterator.Next(&record)
   397  			assert.Nil(t, err)
   398  			actual = append(actual, record[useCase.indexBy[0]])
   399  		}
   400  		assert.EqualValues(t, useCase.expected, actual, useCase.description)
   401  	}
   402  
   403  }
   404  
   405  func TestCompactedSlice_Iterator(t *testing.T) {
   406  	if ! canRun64BitArch() {
   407  		t.Skip()
   408  	}
   409  	var useCases = []struct {
   410  		description string
   411  		data        []map[string]interface{}
   412  		expected    []interface{}
   413  		indexBy     []string
   414  		hasError    bool
   415  	}{
   416  		{
   417  			description: "int sorting",
   418  			indexBy:     []string{"id"},
   419  			data: []map[string]interface{}{
   420  				{
   421  					"id":   10,
   422  					"name": "name 10",
   423  				},
   424  				{
   425  					"id":   3,
   426  					"name": "name 3",
   427  				},
   428  				{
   429  					"id":   1,
   430  					"name": "name 1",
   431  				},
   432  			},
   433  			expected: []interface{}{
   434  				10, 3, 1,
   435  			},
   436  		},
   437  		{
   438  			description: "float sorting",
   439  			indexBy:     []string{"id"},
   440  			data: []map[string]interface{}{
   441  				{
   442  					"id":   10.0,
   443  					"name": "name 10",
   444  				},
   445  				{
   446  					"id":   3.1,
   447  					"name": "name 3",
   448  				},
   449  				{
   450  					"id":   2.2,
   451  					"name": "name 2",
   452  				},
   453  			},
   454  			expected: []interface{}{
   455  				10.0, 3.1, 2.2,
   456  			},
   457  		},
   458  	}
   459  
   460  	for _, useCase := range useCases {
   461  		collection := NewCompactedSlice(true, true)
   462  		var actual = make([]interface{}, 0)
   463  		for _, item := range useCase.data {
   464  			collection.Add(item)
   465  		}
   466  		iterator := collection.Iterator()
   467  
   468  		var record map[string]interface{}
   469  		for iterator.HasNext() {
   470  			err := iterator.Next(&record)
   471  			assert.Nil(t, err)
   472  			actual = append(actual, record[useCase.indexBy[0]])
   473  		}
   474  		assert.EqualValues(t, useCase.expected, actual, useCase.description)
   475  	}
   476  
   477  }
   478  
   479  
   480  func TestCompactedSlice_MarshalJSON(t *testing.T) {
   481  	if ! canRun64BitArch() {
   482  		t.Skip()
   483  	}
   484  	var useCases = []struct {
   485  		description string
   486  		data        []map[string]interface{}
   487  		hasError    bool
   488  	}{
   489  		{
   490  			description: "array marshaling",
   491  			data: []map[string]interface{}{
   492  				{
   493  					"id":   float64(10),
   494  					"name": "name 10",
   495  				},
   496  				{
   497  					"id":   float64(3),
   498  					"name": "name 3",
   499  				},
   500  				{
   501  					"id":   float64(1),
   502  					"name": "name 1",
   503  				},
   504  			},
   505  
   506  		},
   507  
   508  	}
   509  
   510  	for _, useCase := range useCases {
   511  		collection := NewCompactedSlice(true, true)
   512  
   513  		for _, item := range useCase.data {
   514  			collection.Add(item)
   515  		}
   516  		rawJSON, err := json.Marshal(collection)
   517  		if ! assert.Nil(t, err, useCase.description) {
   518  			continue
   519  		}
   520  		actual := []map[string]interface{}{}
   521  		json.Unmarshal(rawJSON, &actual)
   522  		assert.EqualValues(t, useCase.data, actual)
   523  
   524  
   525  
   526  
   527  	}
   528  
   529  }
   530  
   531  
   532  func canRun64BitArch() bool {
   533  	isNot64BitArch := 32 << uintptr(^uintptr(0)>>63) < 64
   534  	return ! isNot64BitArch
   535  }