github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/graveler/committed/import_test.go (about)

     1  package committed_test
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"testing"
     7  
     8  	"github.com/golang/mock/gomock"
     9  	"github.com/treeverse/lakefs/pkg/graveler"
    10  	"github.com/treeverse/lakefs/pkg/graveler/committed"
    11  	"github.com/treeverse/lakefs/pkg/graveler/committed/mock"
    12  )
    13  
    14  func Test_import(t *testing.T) {
    15  	tests := []struct {
    16  		name           string
    17  		sourceRange    *testMetaRange
    18  		destRange      *testMetaRange
    19  		prefixes       []graveler.Prefix
    20  		expectedResult []testRunResult
    21  	}{
    22  		{
    23  			name: "source range smaller than dest range",
    24  			sourceRange: newTestMetaRange([]testRange{
    25  				{
    26  					rng: committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
    27  					records: []testValueRecord{
    28  						{"a/1", "a1"}, {"a/2", "a2"},
    29  					},
    30  				},
    31  				{
    32  					rng: committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
    33  					records: []testValueRecord{
    34  						{"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
    35  					},
    36  				},
    37  			}),
    38  			destRange: newTestMetaRange([]testRange{
    39  				{
    40  					rng: committed.Range{ID: "a/7-a/9", MinKey: committed.Key("a/7"), MaxKey: committed.Key("a/9"), Count: 3, EstimatedSize: 1536},
    41  					records: []testValueRecord{
    42  						{"a/7", "a7"}, {"a/8", "a8"}, {"a/9", "a9"},
    43  					},
    44  				},
    45  			}),
    46  			prefixes: []graveler.Prefix{
    47  				"a",
    48  			},
    49  			expectedResult: []testRunResult{
    50  				{
    51  					expectedActions: []writeAction{
    52  						{
    53  							action: actionTypeWriteRange,
    54  							rng:    committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
    55  						},
    56  						{
    57  							action: actionTypeWriteRange,
    58  							rng:    committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
    59  						},
    60  					},
    61  				},
    62  			},
    63  		},
    64  		{
    65  			name: "dest range smaller than compared prefix",
    66  			sourceRange: newTestMetaRange([]testRange{
    67  				{
    68  					rng: committed.Range{ID: "b/1-b/2", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/2"), Count: 2, EstimatedSize: 1024},
    69  					records: []testValueRecord{
    70  						{"b/1", "b1"}, {"b/2", "b2"},
    71  					},
    72  				},
    73  			}),
    74  			destRange: newTestMetaRange([]testRange{
    75  				{
    76  					rng: committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
    77  					records: []testValueRecord{
    78  						{"a/1", "a1"}, {"a/2", "a2"},
    79  					},
    80  				},
    81  				{
    82  					rng: committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
    83  					records: []testValueRecord{
    84  						{"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
    85  					},
    86  				},
    87  			}),
    88  			prefixes: []graveler.Prefix{
    89  				"b",
    90  			},
    91  			expectedResult: []testRunResult{{
    92  				expectedActions: []writeAction{
    93  					{
    94  						action: actionTypeWriteRange,
    95  						rng:    committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
    96  					},
    97  					{
    98  						action: actionTypeWriteRange,
    99  						rng:    committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
   100  					},
   101  					{
   102  						action: actionTypeWriteRange,
   103  						rng:    committed.Range{ID: "b/1-b/2", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/2"), Count: 2, EstimatedSize: 1024},
   104  					},
   105  				},
   106  			}},
   107  		},
   108  		{
   109  			name: "same range",
   110  			sourceRange: newTestMetaRange([]testRange{
   111  				{
   112  					rng: committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
   113  					records: []testValueRecord{
   114  						{"a/1", "a1"}, {"a/2", "a2"},
   115  					},
   116  				},
   117  				{
   118  					rng: committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
   119  					records: []testValueRecord{
   120  						{"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
   121  					},
   122  				},
   123  			}),
   124  			destRange: newTestMetaRange([]testRange{
   125  				{
   126  					rng: committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
   127  					records: []testValueRecord{
   128  						{"a/1", "a1"}, {"a/2", "a2"},
   129  					},
   130  				},
   131  				{
   132  					rng: committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
   133  					records: []testValueRecord{
   134  						{"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
   135  					},
   136  				},
   137  				{
   138  					rng: committed.Range{ID: "c/4-c/7", MinKey: committed.Key("c/4"), MaxKey: committed.Key("c/7"), Count: 2, EstimatedSize: 1024},
   139  					records: []testValueRecord{
   140  						{"c/4", "c4"}, {"c/7", "c7"},
   141  					},
   142  				},
   143  			}),
   144  			prefixes: []graveler.Prefix{
   145  				"a",
   146  			},
   147  			expectedResult: []testRunResult{{
   148  				expectedActions: []writeAction{
   149  					{
   150  						action: actionTypeWriteRange,
   151  						rng:    committed.Range{ID: "a/1-a/2", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/2"), Count: 2, EstimatedSize: 1024},
   152  					},
   153  					{
   154  						action: actionTypeWriteRange,
   155  						rng:    committed.Range{ID: "a/3-a/6", MinKey: committed.Key("a/3"), MaxKey: committed.Key("a/6"), Count: 4, EstimatedSize: 2048},
   156  					},
   157  					{
   158  						action: actionTypeWriteRange,
   159  						rng:    committed.Range{ID: "c/4-c/7", MinKey: committed.Key("c/4"), MaxKey: committed.Key("c/7"), Count: 2, EstimatedSize: 1024},
   160  					},
   161  				},
   162  			}},
   163  		},
   164  		{
   165  			name: "dest range is bounded by compared prefix",
   166  			sourceRange: newTestMetaRange([]testRange{
   167  				{
   168  					rng: committed.Range{ID: "a/4-a/8", MinKey: committed.Key("a/4"), MaxKey: committed.Key("a/8"), Count: 5, EstimatedSize: 2560},
   169  					records: []testValueRecord{
   170  						{"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"}, {"a/7", "a7"}, {"a/8", "a8"},
   171  					},
   172  				},
   173  			}),
   174  			destRange: newTestMetaRange([]testRange{
   175  				{
   176  					rng: committed.Range{ID: "a/1-a/6", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/6"), Count: 6, EstimatedSize: 3072},
   177  					records: []testValueRecord{
   178  						{"a/1", "a1"}, {"a/2", "a2"}, {"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
   179  					},
   180  				},
   181  				{
   182  					rng: committed.Range{ID: "b/1-b/6", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/6"), Count: 2, EstimatedSize: 1024},
   183  					records: []testValueRecord{
   184  						{"b/1", "b1"}, {"b/2", "b2"}, {"b/3", "b3"}, {"b/4", "b4"}, {"b/5", "b5"}, {"b/6", "b6"},
   185  					},
   186  				},
   187  			}),
   188  			prefixes: []graveler.Prefix{
   189  				"a",
   190  			},
   191  			expectedResult: []testRunResult{{
   192  				expectedActions: []writeAction{
   193  					{
   194  						action: actionTypeWriteRange,
   195  						rng:    committed.Range{ID: "a/4-a/8", MinKey: committed.Key("a/4"), MaxKey: committed.Key("a/8"), Count: 5, EstimatedSize: 2560},
   196  					},
   197  					{
   198  						action: actionTypeWriteRange,
   199  						rng:    committed.Range{ID: "b/1-b/6", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/6"), Count: 2, EstimatedSize: 1024},
   200  					},
   201  				},
   202  			}},
   203  		},
   204  		{
   205  			name: "multiple ranges in dest are bounded by compared prefix",
   206  			sourceRange: newTestMetaRange([]testRange{
   207  				{
   208  					rng: committed.Range{ID: "a/4-a/8", MinKey: committed.Key("a/4"), MaxKey: committed.Key("a/8"), Count: 5, EstimatedSize: 2560},
   209  					records: []testValueRecord{
   210  						{"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"}, {"a/7", "a7"}, {"a/8", "a8"},
   211  					},
   212  				},
   213  			}),
   214  			destRange: newTestMetaRange([]testRange{
   215  				{
   216  					rng: committed.Range{ID: "a/1-a/6", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/6"), Count: 6, EstimatedSize: 3072},
   217  					records: []testValueRecord{
   218  						{"a/1", "a1"}, {"a/2", "a2"}, {"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"}, {"a/6", "a6"},
   219  					},
   220  				},
   221  				{
   222  					rng: committed.Range{ID: "a/11-a/16", MinKey: committed.Key("a/11"), MaxKey: committed.Key("a/16"), Count: 6, EstimatedSize: 3072},
   223  					records: []testValueRecord{
   224  						{"a/11", "a11"}, {"a/12", "a12"}, {"a/13", "a13"}, {"a/14", "a14"}, {"a/15", "a15"}, {"a/16", "a16"},
   225  					},
   226  				},
   227  				{
   228  					rng: committed.Range{ID: "a/21-a/26", MinKey: committed.Key("a/21"), MaxKey: committed.Key("a/26"), Count: 6, EstimatedSize: 3072},
   229  					records: []testValueRecord{
   230  						{"a/21", "a21"}, {"a/22", "a22"}, {"a/23", "a23"}, {"a/24", "a24"}, {"a/25", "a25"}, {"a/26", "a26"},
   231  					},
   232  				},
   233  				{
   234  					rng: committed.Range{ID: "b/1-b/6", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/6"), Count: 2, EstimatedSize: 1024},
   235  					records: []testValueRecord{
   236  						{"b/1", "b1"}, {"b/2", "b2"}, {"b/3", "b3"}, {"b/4", "b4"}, {"b/5", "b5"}, {"b/6", "b6"},
   237  					},
   238  				},
   239  			}),
   240  			prefixes: []graveler.Prefix{
   241  				"a",
   242  			},
   243  			expectedResult: []testRunResult{{
   244  				expectedActions: []writeAction{
   245  					{
   246  						action: actionTypeWriteRange,
   247  						rng:    committed.Range{ID: "a/4-a/8", MinKey: committed.Key("a/4"), MaxKey: committed.Key("a/8"), Count: 5, EstimatedSize: 2560},
   248  					},
   249  					{
   250  						action: actionTypeWriteRange,
   251  						rng:    committed.Range{ID: "b/1-b/6", MinKey: committed.Key("b/1"), MaxKey: committed.Key("b/6"), Count: 2, EstimatedSize: 1024},
   252  					},
   253  				},
   254  			}},
   255  		},
   256  		{
   257  			name: "intersected ranges - multiple prefixes",
   258  			sourceRange: newTestMetaRange([]testRange{
   259  				{
   260  					rng: committed.Range{ID: "a/1-a/4", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/4"), Count: 4, EstimatedSize: 2048},
   261  					records: []testValueRecord{
   262  						{"a/1", "a1"}, {"a/2", "a2"}, {"a/3", "a3"}, {"a/4", "a4"},
   263  					},
   264  				},
   265  				{
   266  					rng: committed.Range{ID: "a/8-c/3", MinKey: committed.Key("a/8"), MaxKey: committed.Key("c/3"), Count: 2, EstimatedSize: 1024},
   267  					records: []testValueRecord{
   268  						{"a/8", "a8"}, {"c/3", "c3"},
   269  					},
   270  				},
   271  				{
   272  					rng: committed.Range{ID: "c/4-c/7", MinKey: committed.Key("c/4"), MaxKey: committed.Key("c/7"), Count: 2, EstimatedSize: 1024},
   273  					records: []testValueRecord{
   274  						{"c/4", "c4"}, {"c/7", "c7"},
   275  					},
   276  				},
   277  			}),
   278  			destRange: newTestMetaRange([]testRange{
   279  				{
   280  					rng: committed.Range{ID: "a/2-a/5", MinKey: committed.Key("a/2"), MaxKey: committed.Key("a/5"), Count: 4, EstimatedSize: 2048},
   281  					records: []testValueRecord{
   282  						{"a/2", "a2"}, {"a/3", "a3"}, {"a/4", "a4"}, {"a/5", "a5"},
   283  					},
   284  				},
   285  				{
   286  					rng: committed.Range{ID: "a/12-a/15", MinKey: committed.Key("a/12"), MaxKey: committed.Key("a/15"), Count: 4, EstimatedSize: 2048},
   287  					records: []testValueRecord{
   288  						{"a/12", "a12"}, {"a/13", "a13"}, {"a/14", "a14"}, {"a/15", "a15"},
   289  					},
   290  				},
   291  				{
   292  					rng: committed.Range{ID: "a/8-b/4", MinKey: committed.Key("a/8"), MaxKey: committed.Key("b/4"), Count: 4, EstimatedSize: 2048},
   293  					records: []testValueRecord{
   294  						{"a/8", "a8"}, {"b/1", "b1"}, {"b/2", "b2"}, {"b/4", "b4"},
   295  					},
   296  				},
   297  				{
   298  					rng: committed.Range{ID: "b/6-b/7", MinKey: committed.Key("b/6"), MaxKey: committed.Key("b/7"), Count: 2, EstimatedSize: 1024},
   299  					records: []testValueRecord{
   300  						{"b/6", "b6"}, {"b/7", "b7"},
   301  					},
   302  				},
   303  				{
   304  					rng: committed.Range{ID: "b/8-c/2", MinKey: committed.Key("b/8"), MaxKey: committed.Key("c/2"), Count: 4, EstimatedSize: 2048},
   305  					records: []testValueRecord{
   306  						{"b/8", "b8"}, {"b/9", "b9"}, {"c/1", "c1"}, {"c/2", "c2"},
   307  					},
   308  				},
   309  				{
   310  					rng: committed.Range{ID: "c/5-d/2", MinKey: committed.Key("c/5"), MaxKey: committed.Key("d/2"), Count: 4, EstimatedSize: 2048},
   311  					records: []testValueRecord{
   312  						{"c/5", "c5"}, {"c/6", "c6"}, {"d/1", "d1"}, {"d/2", "d2"},
   313  					},
   314  				},
   315  			}),
   316  			prefixes: []graveler.Prefix{
   317  				"a",
   318  				"c",
   319  			},
   320  			expectedResult: []testRunResult{{
   321  				expectedActions: []writeAction{
   322  					{
   323  						action: actionTypeWriteRange,
   324  						rng:    committed.Range{ID: "a/1-a/4", MinKey: committed.Key("a/1"), MaxKey: committed.Key("a/4"), Count: 4, EstimatedSize: 2048},
   325  					},
   326  					{action: actionTypeWriteRecord, key: "a/8", identity: "a8"},
   327  					{action: actionTypeWriteRecord, key: "b/1", identity: "b1"},
   328  					{action: actionTypeWriteRecord, key: "b/2", identity: "b2"},
   329  					{action: actionTypeWriteRecord, key: "b/4", identity: "b4"},
   330  					{
   331  						action: actionTypeWriteRange,
   332  						rng:    committed.Range{ID: "b/6-b/7", MinKey: committed.Key("b/6"), MaxKey: committed.Key("b/7"), Count: 2, EstimatedSize: 1024},
   333  					},
   334  					{action: actionTypeWriteRecord, key: "b/8", identity: "b8"},
   335  					{action: actionTypeWriteRecord, key: "b/9", identity: "b9"},
   336  					{action: actionTypeWriteRecord, key: "c/3", identity: "c3"},
   337  					{action: actionTypeWriteRecord, key: "c/4", identity: "c4"},
   338  					{action: actionTypeWriteRecord, key: "c/7", identity: "c7"},
   339  					{action: actionTypeWriteRecord, key: "d/1", identity: "d1"},
   340  					{action: actionTypeWriteRecord, key: "d/2", identity: "d2"},
   341  				},
   342  			}},
   343  		},
   344  	}
   345  
   346  	for _, tst := range tests {
   347  		for _, expectedResult := range tst.expectedResult {
   348  			t.Run(tst.name, func(t *testing.T) {
   349  				ctrl := gomock.NewController(t)
   350  				defer ctrl.Finish()
   351  				ctx := context.Background()
   352  				writer := mock.NewMockMetaRangeWriter(ctrl)
   353  				for _, action := range expectedResult.expectedActions {
   354  					switch action.action {
   355  					case actionTypeWriteRecord:
   356  						writer.EXPECT().WriteRecord(newRecordMatcher(action.key, action.identity))
   357  					case actionTypeWriteRange:
   358  						writer.EXPECT().WriteRange(gomock.Eq(action.rng))
   359  					}
   360  				}
   361  				metaRangeManager := mock.NewMockMetaRangeManager(ctrl)
   362  				metaRangeManager.EXPECT().NewWriter(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(writer)
   363  				sourceMetaRangeID := tst.sourceRange.GetMetaRangeID()
   364  				destMetaRangeID := tst.destRange.GetMetaRangeID()
   365  				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), graveler.MetaRangeID("")).AnyTimes().Return(committed.NewEmptyIterator(), nil) // empty base
   366  				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil)
   367  				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil)
   368  
   369  				rangeManager := mock.NewMockRangeManager(ctrl)
   370  
   371  				writer.EXPECT().Abort().AnyTimes()
   372  				metaRangeId := graveler.MetaRangeID("import")
   373  				writer.EXPECT().Close(gomock.Any()).Return(&metaRangeId, nil).AnyTimes()
   374  				committedManager := committed.NewCommittedManager(metaRangeManager, rangeManager, params)
   375  				_, err := committedManager.Import(ctx, "ns", destMetaRangeID, sourceMetaRangeID, tst.prefixes)
   376  				if !errors.Is(err, expectedResult.expectedErr) {
   377  					t.Fatalf("Import error = '%v', expected '%v'", err, expectedResult.expectedErr)
   378  				}
   379  			})
   380  		}
   381  	}
   382  }