github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/inverted/delta_analyzer_test.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package inverted
    13  
    14  import (
    15  	"testing"
    16  
    17  	"github.com/stretchr/testify/assert"
    18  )
    19  
    20  func TestDeltaAnalyzer(t *testing.T) {
    21  	t.Run("without previous indexing", func(t *testing.T) {
    22  		previous := []Property(nil)
    23  		next := []Property{
    24  			{
    25  				Name: "prop1",
    26  				Items: []Countable{
    27  					{
    28  						Data:          []byte("value1"),
    29  						TermFrequency: 7,
    30  					},
    31  					{
    32  						Data:          []byte("value2"),
    33  						TermFrequency: 3,
    34  					},
    35  				},
    36  			},
    37  			{
    38  				Name: "prop2",
    39  				Items: []Countable{
    40  					{
    41  						Data:          []byte("value3"),
    42  						TermFrequency: 7,
    43  					},
    44  					{
    45  						Data:          []byte("value4"),
    46  						TermFrequency: 3,
    47  					},
    48  				},
    49  			},
    50  		}
    51  
    52  		res := Delta(previous, next)
    53  		assert.Equal(t, next, res.ToAdd)
    54  		assert.Len(t, res.ToDelete, 0)
    55  	})
    56  
    57  	t.Run("with previous indexing and no changes", func(t *testing.T) {
    58  		previous := []Property{
    59  			{
    60  				Name: "prop1",
    61  				Items: []Countable{
    62  					{
    63  						Data:          []byte("value1"),
    64  						TermFrequency: 7,
    65  					},
    66  					{
    67  						Data:          []byte("value2"),
    68  						TermFrequency: 3,
    69  					},
    70  				},
    71  			},
    72  			{
    73  				Name: "prop2",
    74  				Items: []Countable{
    75  					{
    76  						Data:          []byte("value3"),
    77  						TermFrequency: 7,
    78  					},
    79  					{
    80  						Data:          []byte("value4"),
    81  						TermFrequency: 3,
    82  					},
    83  				},
    84  			},
    85  		}
    86  		next := []Property{
    87  			{
    88  				Name: "prop1",
    89  				Items: []Countable{
    90  					{
    91  						Data:          []byte("value1"),
    92  						TermFrequency: 7,
    93  					},
    94  					{
    95  						Data:          []byte("value2"),
    96  						TermFrequency: 3,
    97  					},
    98  				},
    99  			},
   100  			{
   101  				Name: "prop2",
   102  				Items: []Countable{
   103  					{
   104  						Data:          []byte("value3"),
   105  						TermFrequency: 7,
   106  					},
   107  					{
   108  						Data:          []byte("value4"),
   109  						TermFrequency: 3,
   110  					},
   111  				},
   112  			},
   113  		}
   114  
   115  		res := Delta(previous, next)
   116  		assert.Len(t, res.ToDelete, 0)
   117  		assert.Len(t, res.ToAdd, 0)
   118  	})
   119  
   120  	t.Run("with previous indexing - only additions", func(t *testing.T) {
   121  		previous := []Property{
   122  			{
   123  				Name: "prop1",
   124  				Items: []Countable{
   125  					{
   126  						Data:          []byte("value2"),
   127  						TermFrequency: 3,
   128  					},
   129  				},
   130  			},
   131  			{
   132  				Name: "prop2",
   133  				Items: []Countable{
   134  					{
   135  						Data:          []byte("value4"),
   136  						TermFrequency: 3,
   137  					},
   138  				},
   139  			},
   140  		}
   141  		next := []Property{
   142  			{
   143  				Name: "prop1",
   144  				Items: []Countable{
   145  					{
   146  						Data:          []byte("value1"),
   147  						TermFrequency: 7,
   148  					},
   149  					{
   150  						Data:          []byte("value2"),
   151  						TermFrequency: 3,
   152  					},
   153  				},
   154  			},
   155  			{
   156  				Name: "prop2",
   157  				Items: []Countable{
   158  					{
   159  						Data:          []byte("value3"),
   160  						TermFrequency: 7,
   161  					},
   162  					{
   163  						Data:          []byte("value4"),
   164  						TermFrequency: 3,
   165  					},
   166  				},
   167  			},
   168  		}
   169  
   170  		expectedAdd := []Property{
   171  			{
   172  				Name: "prop1",
   173  				Items: []Countable{
   174  					{
   175  						Data:          []byte("value1"),
   176  						TermFrequency: 7,
   177  					},
   178  				},
   179  			},
   180  			{
   181  				Name: "prop2",
   182  				Items: []Countable{
   183  					{
   184  						Data:          []byte("value3"),
   185  						TermFrequency: 7,
   186  					},
   187  				},
   188  			},
   189  		}
   190  
   191  		res := Delta(previous, next)
   192  		assert.Equal(t, expectedAdd, res.ToAdd)
   193  		assert.Len(t, res.ToDelete, 0)
   194  	})
   195  
   196  	t.Run("with previous indexing - both additions and deletions", func(t *testing.T) {
   197  		previous := []Property{
   198  			{
   199  				Name: "prop1",
   200  				Items: []Countable{
   201  					{
   202  						Data:          []byte("value2"),
   203  						TermFrequency: 3,
   204  					},
   205  				},
   206  			},
   207  			{
   208  				Name: "prop2",
   209  				Items: []Countable{
   210  					{
   211  						Data:          []byte("value4"),
   212  						TermFrequency: 3,
   213  					},
   214  				},
   215  			},
   216  		}
   217  		next := []Property{
   218  			{
   219  				Name: "prop1",
   220  				Items: []Countable{
   221  					{
   222  						Data:          []byte("value1"),
   223  						TermFrequency: 7,
   224  					},
   225  				},
   226  			},
   227  			{
   228  				Name: "prop2",
   229  				Items: []Countable{
   230  					{
   231  						Data:          []byte("value3"),
   232  						TermFrequency: 7,
   233  					},
   234  					{
   235  						Data:          []byte("value4"),
   236  						TermFrequency: 3,
   237  					},
   238  				},
   239  			},
   240  		}
   241  
   242  		expectedAdd := []Property{
   243  			{
   244  				Name: "prop1",
   245  				Items: []Countable{
   246  					{
   247  						Data:          []byte("value1"),
   248  						TermFrequency: 7,
   249  					},
   250  				},
   251  			},
   252  			{
   253  				Name: "prop2",
   254  				Items: []Countable{
   255  					{
   256  						Data:          []byte("value3"),
   257  						TermFrequency: 7,
   258  					},
   259  				},
   260  			},
   261  		}
   262  
   263  		expectedDelete := []Property{
   264  			{
   265  				Name: "prop1",
   266  				Items: []Countable{
   267  					{
   268  						Data:          []byte("value2"),
   269  						TermFrequency: 3,
   270  					},
   271  				},
   272  			},
   273  		}
   274  
   275  		res := Delta(previous, next)
   276  		assert.Equal(t, expectedAdd, res.ToAdd)
   277  		assert.Equal(t, expectedDelete, res.ToDelete)
   278  	})
   279  }
   280  
   281  func TestDeltaAnalyzer_Arrays(t *testing.T) {
   282  	lexInt64 := func(val int64) []byte {
   283  		bytes, _ := LexicographicallySortableInt64(val)
   284  		return bytes
   285  	}
   286  	lexBool := func(val bool) []byte {
   287  		if val {
   288  			return []uint8{1}
   289  		}
   290  		return []uint8{0}
   291  	}
   292  
   293  	t.Run("with previous indexing - both additions and deletions", func(t *testing.T) {
   294  		previous := []Property{
   295  			{
   296  				Name: "ints",
   297  				Items: []Countable{
   298  					{Data: lexInt64(101)},
   299  					{Data: lexInt64(101)},
   300  					{Data: lexInt64(101)},
   301  					{Data: lexInt64(101)},
   302  					{Data: lexInt64(101)},
   303  					{Data: lexInt64(101)},
   304  					{Data: lexInt64(102)},
   305  					{Data: lexInt64(103)},
   306  					{Data: lexInt64(104)},
   307  				},
   308  				Length:             9,
   309  				HasFilterableIndex: true,
   310  				HasSearchableIndex: false,
   311  			},
   312  			{
   313  				Name: "booleans",
   314  				Items: []Countable{
   315  					{Data: lexBool(true)},
   316  					{Data: lexBool(true)},
   317  					{Data: lexBool(true)},
   318  					{Data: lexBool(false)},
   319  				},
   320  				Length:             4,
   321  				HasFilterableIndex: true,
   322  				HasSearchableIndex: false,
   323  			},
   324  			{
   325  				Name:               "numbers",
   326  				Items:              []Countable{},
   327  				Length:             0,
   328  				HasFilterableIndex: true,
   329  				HasSearchableIndex: false,
   330  			},
   331  			{
   332  				Name: "texts",
   333  				Items: []Countable{
   334  					{Data: []byte("aaa")},
   335  					{Data: []byte("bbb")},
   336  					{Data: []byte("ccc")},
   337  				},
   338  				Length:             3,
   339  				HasFilterableIndex: true,
   340  				HasSearchableIndex: false,
   341  			},
   342  			{
   343  				Name: "dates",
   344  				Items: []Countable{
   345  					{Data: []byte("2021-06-01T22:18:59.640162Z")},
   346  					{Data: []byte("2022-06-01T22:18:59.640162Z")},
   347  				},
   348  				Length:             2,
   349  				HasFilterableIndex: true,
   350  				HasSearchableIndex: false,
   351  			},
   352  			{
   353  				Name: "_creationTimeUnix",
   354  				Items: []Countable{
   355  					{Data: []byte("1703778000000")},
   356  				},
   357  				Length:             0,
   358  				HasFilterableIndex: true,
   359  				HasSearchableIndex: false,
   360  			},
   361  			{
   362  				Name: "_lastUpdateTimeUnix",
   363  				Items: []Countable{
   364  					{Data: []byte("1703778000000")},
   365  				},
   366  				Length:             0,
   367  				HasFilterableIndex: true,
   368  				HasSearchableIndex: false,
   369  			},
   370  		}
   371  		next := []Property{
   372  			{
   373  				Name: "ints",
   374  				Items: []Countable{
   375  					{Data: lexInt64(101)},
   376  					{Data: lexInt64(101)},
   377  					{Data: lexInt64(101)},
   378  					{Data: lexInt64(101)},
   379  					{Data: lexInt64(103)},
   380  					{Data: lexInt64(104)},
   381  					{Data: lexInt64(105)},
   382  				},
   383  				Length:             7,
   384  				HasFilterableIndex: true,
   385  				HasSearchableIndex: false,
   386  			},
   387  			{
   388  				Name: "booleans",
   389  				Items: []Countable{
   390  					{Data: lexBool(true)},
   391  					{Data: lexBool(true)},
   392  					{Data: lexBool(true)},
   393  					{Data: lexBool(false)},
   394  				},
   395  				Length:             4,
   396  				HasFilterableIndex: true,
   397  				HasSearchableIndex: false,
   398  			},
   399  			{
   400  				Name:               "texts",
   401  				Items:              []Countable{},
   402  				Length:             0,
   403  				HasFilterableIndex: true,
   404  				HasSearchableIndex: false,
   405  			},
   406  			{
   407  				Name: "_creationTimeUnix",
   408  				Items: []Countable{
   409  					{Data: []byte("1703778000000")},
   410  				},
   411  				Length:             0,
   412  				HasFilterableIndex: true,
   413  				HasSearchableIndex: false,
   414  			},
   415  			{
   416  				Name: "_lastUpdateTimeUnix",
   417  				Items: []Countable{
   418  					{Data: []byte("1703778500000")},
   419  				},
   420  				Length:             0,
   421  				HasFilterableIndex: true,
   422  				HasSearchableIndex: false,
   423  			},
   424  		}
   425  
   426  		expectedAdd := []Property{
   427  			{
   428  				Name: "ints",
   429  				Items: []Countable{
   430  					{Data: lexInt64(105)},
   431  				},
   432  				Length:             7,
   433  				HasFilterableIndex: true,
   434  				HasSearchableIndex: false,
   435  			},
   436  			{
   437  				Name:               "texts",
   438  				Items:              []Countable{},
   439  				Length:             0,
   440  				HasFilterableIndex: true,
   441  				HasSearchableIndex: false,
   442  			},
   443  			{
   444  				Name: "_lastUpdateTimeUnix",
   445  				Items: []Countable{
   446  					{Data: []byte("1703778500000")},
   447  				},
   448  				Length:             0,
   449  				HasFilterableIndex: true,
   450  				HasSearchableIndex: false,
   451  			},
   452  		}
   453  		expectedDelete := []Property{
   454  			{
   455  				Name: "ints",
   456  				Items: []Countable{
   457  					{Data: lexInt64(102)},
   458  				},
   459  				Length:             9,
   460  				HasFilterableIndex: true,
   461  				HasSearchableIndex: false,
   462  			},
   463  			{
   464  				Name: "texts",
   465  				Items: []Countable{
   466  					{Data: []byte("aaa")},
   467  					{Data: []byte("bbb")},
   468  					{Data: []byte("ccc")},
   469  				},
   470  				Length:             3,
   471  				HasFilterableIndex: true,
   472  				HasSearchableIndex: false,
   473  			},
   474  			{
   475  				Name: "_lastUpdateTimeUnix",
   476  				Items: []Countable{
   477  					{Data: []byte("1703778000000")},
   478  				},
   479  				Length:             0,
   480  				HasFilterableIndex: true,
   481  				HasSearchableIndex: false,
   482  			},
   483  			{
   484  				Name:               "numbers",
   485  				Items:              []Countable{},
   486  				Length:             0,
   487  				HasFilterableIndex: true,
   488  				HasSearchableIndex: false,
   489  			},
   490  			{
   491  				Name: "dates",
   492  				Items: []Countable{
   493  					{Data: []byte("2021-06-01T22:18:59.640162Z")},
   494  					{Data: []byte("2022-06-01T22:18:59.640162Z")},
   495  				},
   496  				Length:             2,
   497  				HasFilterableIndex: true,
   498  				HasSearchableIndex: false,
   499  			},
   500  		}
   501  
   502  		delta := Delta(previous, next)
   503  		assert.Equal(t, expectedAdd, delta.ToAdd)
   504  		assert.Equal(t, expectedDelete, delta.ToDelete)
   505  	})
   506  }
   507  
   508  func TestDeltaNilAnalyzer(t *testing.T) {
   509  	previous := []NilProperty{
   510  		{
   511  			Name:                "ints",
   512  			AddToPropertyLength: false,
   513  		},
   514  		{
   515  			Name:                "booleans",
   516  			AddToPropertyLength: true,
   517  		},
   518  		{
   519  			Name:                "numbers",
   520  			AddToPropertyLength: true,
   521  		},
   522  	}
   523  	next := []NilProperty{
   524  		{
   525  			Name:                "booleans",
   526  			AddToPropertyLength: true,
   527  		},
   528  		{
   529  			Name:                "texts",
   530  			AddToPropertyLength: true,
   531  		},
   532  		{
   533  			Name:                "dates",
   534  			AddToPropertyLength: false,
   535  		},
   536  	}
   537  
   538  	expectedAdd := []NilProperty{
   539  		{
   540  			Name:                "texts",
   541  			AddToPropertyLength: true,
   542  		},
   543  		{
   544  			Name:                "dates",
   545  			AddToPropertyLength: false,
   546  		},
   547  	}
   548  	expectedDelete := []NilProperty{
   549  		{
   550  			Name:                "ints",
   551  			AddToPropertyLength: false,
   552  		},
   553  		{
   554  			Name:                "numbers",
   555  			AddToPropertyLength: true,
   556  		},
   557  	}
   558  
   559  	deltaNil := DeltaNil(previous, next)
   560  	assert.Equal(t, expectedAdd, deltaNil.ToAdd)
   561  	assert.Equal(t, expectedDelete, deltaNil.ToDelete)
   562  }