github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/graveler/committed/commit_test.go (about) 1 package committed_test 2 3 import ( 4 "context" 5 "errors" 6 "testing" 7 8 "github.com/golang/mock/gomock" 9 "github.com/stretchr/testify/assert" 10 "github.com/treeverse/lakefs/pkg/graveler" 11 "github.com/treeverse/lakefs/pkg/graveler/committed" 12 "github.com/treeverse/lakefs/pkg/graveler/committed/mock" 13 "github.com/treeverse/lakefs/pkg/graveler/testutil" 14 ) 15 16 func makeV(k, id string) *graveler.ValueRecord { 17 return &graveler.ValueRecord{Key: graveler.Key(k), Value: &graveler.Value{Identity: []byte(id)}} 18 } 19 20 func makeTombstoneV(k string) *graveler.ValueRecord { 21 return &graveler.ValueRecord{Key: graveler.Key(k)} 22 } 23 24 func TestCommitAdd(t *testing.T) { 25 ctrl := gomock.NewController(t) 26 27 base := testutil.NewFakeIterator(). 28 AddRange(&committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 2}). 29 AddValueRecords(makeV("a", "base:a"), makeV("c", "base:c")). 30 AddRange(&committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("d"), Count: 1}). 31 AddValueRecords(makeV("d", "base:d")) 32 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 33 *makeV("b", "dest:b"), 34 *makeV("e", "dest:e"), 35 *makeV("f", "dest:f"), 36 }) 37 38 writer := mock.NewMockMetaRangeWriter(ctrl) 39 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "base:a"))) 40 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("b", "dest:b"))) 41 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 42 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("d"), Count: 1})) 43 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("e", "dest:e"))) 44 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("f", "dest:f"))) 45 46 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 47 assert.NoError(t, err) 48 assert.Equal(t, graveler.DiffSummary{ 49 Count: map[graveler.DiffType]int{ 50 graveler.DiffTypeAdded: 3, 51 }, 52 }, summary) 53 } 54 55 func TestCommitChangeWithinBaseRange(t *testing.T) { 56 ctrl := gomock.NewController(t) 57 58 base := testutil.NewFakeIterator(). 59 AddRange(&committed.Range{ID: "outer-range", MinKey: committed.Key("k1"), MaxKey: committed.Key("k6"), Count: 2}). 60 AddValueRecords(makeV("k1", "base:k1"), makeV("k6", "base:k6")). 61 AddRange(&committed.Range{ID: "last", MinKey: committed.Key("k10"), MaxKey: committed.Key("k13"), Count: 2}). 62 AddValueRecords(makeV("k10", "base:k10"), makeV("k13", "base:k13")) 63 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 64 *makeV("k2", "dest:k2"), 65 *makeV("k3", "dest:k3"), 66 *makeV("k4", "dest:k4"), 67 *makeV("k5", "dest:k5"), 68 }) 69 70 writer := mock.NewMockMetaRangeWriter(ctrl) 71 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k1", "base:k1"))) 72 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k2", "dest:k2"))) 73 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k3", "dest:k3"))) 74 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k4", "dest:k4"))) 75 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k5", "dest:k5"))) 76 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k6", "base:k6"))) 77 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "last", MinKey: committed.Key("k10"), MaxKey: committed.Key("k13"), Count: 2})) 78 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 79 assert.NoError(t, err) 80 assert.Equal(t, graveler.DiffSummary{ 81 Count: map[graveler.DiffType]int{ 82 graveler.DiffTypeAdded: 4, 83 }, 84 }, summary) 85 } 86 87 func TestCommitBaseRangesWithinChanges(t *testing.T) { 88 ctrl := gomock.NewController(t) 89 90 base := testutil.NewFakeIterator(). 91 AddRange(&committed.Range{ID: "inner-range-1", MinKey: committed.Key("k2"), MaxKey: committed.Key("k3"), Count: 2}). 92 AddValueRecords(makeV("k2", "base:k2"), makeV("k3", "base:k3")). 93 AddRange(&committed.Range{ID: "inner-range-2", MinKey: committed.Key("k4"), MaxKey: committed.Key("k5"), Count: 2}). 94 AddValueRecords(makeV("k4", "base:k4"), makeV("k5", "base:k5")) 95 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 96 *makeV("k1", "changes:k1"), 97 *makeV("k6", "changes:k6"), 98 }) 99 100 writer := mock.NewMockMetaRangeWriter(ctrl) 101 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k1", "changes:k1"))) 102 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k2", "base:k2"))) 103 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k3", "base:k3"))) 104 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "inner-range-2", MinKey: committed.Key("k4"), MaxKey: committed.Key("k5"), Count: 2})) 105 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("k6", "changes:k6"))) 106 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 107 assert.NoError(t, err) 108 assert.Equal(t, graveler.DiffSummary{ 109 Count: map[graveler.DiffType]int{ 110 graveler.DiffTypeAdded: 2, 111 }, 112 }, summary) 113 } 114 115 func TestCommitReplace(t *testing.T) { 116 ctrl := gomock.NewController(t) 117 118 base := testutil.NewFakeIterator(). 119 AddRange(&committed.Range{ID: "base:one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3}). 120 AddValueRecords(makeV("a", "base:a"), makeV("b", "base:b"), makeV("c", "base:c")). 121 AddRange(&committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1}). 122 AddValueRecords(makeV("d", "base:d")). 123 AddRange(&committed.Range{ID: "three", MinKey: committed.Key("e"), MaxKey: committed.Key("ez"), Count: 1}). 124 AddValueRecords(makeV("e", "base:e")) 125 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 126 *makeV("b", "changes:b"), 127 *makeV("e", "changes:e"), 128 }) 129 130 writer := mock.NewMockMetaRangeWriter(ctrl) 131 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "base:a"))) 132 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("b", "changes:b"))) 133 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 134 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1})) 135 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("e", "changes:e"))) 136 137 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 138 assert.NoError(t, err) 139 assert.Equal(t, graveler.DiffSummary{ 140 Count: map[graveler.DiffType]int{ 141 graveler.DiffTypeChanged: 2, 142 }, 143 }, summary) 144 } 145 146 // TestCommitOverrideNoChange verify that if we have changes include the same entry as in base (based on identity) we will take the one in base 147 // and will not consider it as a change. 148 func TestCommitOverrideNoChange(t *testing.T) { 149 ctrl := gomock.NewController(t) 150 151 base := testutil.NewFakeIterator(). 152 AddRange(&committed.Range{ID: "base:one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3}). 153 AddValueRecords( 154 makeV("a", "base:a"), 155 &graveler.ValueRecord{Key: graveler.Key("b"), Value: &graveler.Value{Identity: []byte("base:b"), Data: []byte("base")}}, 156 makeV("c", "base:c"), 157 ). 158 AddRange(&committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1}). 159 AddValueRecords(makeV("d", "base:d")). 160 AddRange(&committed.Range{ID: "three", MinKey: committed.Key("e"), MaxKey: committed.Key("ez"), Count: 1}). 161 AddValueRecords( 162 &graveler.ValueRecord{Key: graveler.Key("e"), Value: &graveler.Value{Identity: []byte("base:e"), Data: []byte("base")}}, 163 ) 164 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 165 {Key: graveler.Key("b"), Value: &graveler.Value{Identity: []byte("base:b"), Data: []byte("new")}}, 166 {Key: graveler.Key("e"), Value: &graveler.Value{Identity: []byte("base:e"), Data: []byte("new")}}, 167 }) 168 169 writer := mock.NewMockMetaRangeWriter(ctrl) 170 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "base:a"))) 171 writer.EXPECT().WriteRecord(gomock.Eq(graveler.ValueRecord{Key: graveler.Key("b"), Value: &graveler.Value{Identity: []byte("base:b"), Data: []byte("base")}})) 172 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 173 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1})) 174 writer.EXPECT().WriteRecord(gomock.Eq(graveler.ValueRecord{Key: graveler.Key("e"), Value: &graveler.Value{Identity: []byte("base:e"), Data: []byte("base")}})) 175 176 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 177 assert.Error(t, err, graveler.ErrNoChanges) 178 assert.Equal(t, graveler.DiffSummary{ 179 Count: map[graveler.DiffType]int{}, 180 }, summary) 181 } 182 183 func TestCommitDelete(t *testing.T) { 184 ctrl := gomock.NewController(t) 185 186 base := testutil.NewFakeIterator(). 187 AddRange(&committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3}). 188 AddValueRecords(makeV("a", "base:a"), makeV("b", "base:b"), makeV("c", "base:c")). 189 AddRange(&committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1}). 190 AddValueRecords(makeV("d", "base:d")). 191 AddRange(&committed.Range{ID: "three", MinKey: committed.Key("ez"), MaxKey: committed.Key("ez"), Count: 1}). 192 AddValueRecords(makeV("e", "base:e")) 193 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 194 *makeTombstoneV("b"), 195 *makeTombstoneV("e"), 196 }) 197 198 writer := mock.NewMockMetaRangeWriter(ctrl) 199 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "base:a"))) 200 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 201 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1})) 202 203 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 204 assert.NoError(t, err) 205 assert.Equal(t, graveler.DiffSummary{ 206 Count: map[graveler.DiffType]int{ 207 graveler.DiffTypeRemoved: 2, 208 }, 209 }, summary) 210 } 211 212 func TestCommitCopiesLeftoverChanges(t *testing.T) { 213 ctrl := gomock.NewController(t) 214 215 base := testutil.NewFakeIterator(). 216 AddRange(&committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3}). 217 AddValueRecords(makeV("a", "base:a"), makeV("b", "base:b"), makeV("c", "base:c")). 218 AddRange(&committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1}). 219 AddValueRecords(makeV("d", "base:d")) 220 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 221 *makeV("b", "changes:b"), 222 *makeV("e", "changes:e"), 223 *makeV("f", "changes:f"), 224 }) 225 226 writer := mock.NewMockMetaRangeWriter(ctrl) 227 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "base:a"))) 228 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("b", "changes:b"))) 229 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 230 writer.EXPECT().WriteRange(gomock.Eq(committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1})) 231 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("e", "changes:e"))) 232 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("f", "changes:f"))) 233 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 234 assert.NoError(t, err) 235 assert.Equal(t, graveler.DiffSummary{ 236 Count: map[graveler.DiffType]int{ 237 graveler.DiffTypeChanged: 1, 238 graveler.DiffTypeAdded: 2, 239 }, 240 }, summary) 241 } 242 243 func TestCommitTombstoneNoBase(t *testing.T) { 244 ctrl := gomock.NewController(t) 245 246 base := testutil.NewFakeIterator() 247 248 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 249 *makeTombstoneV("b"), 250 *makeTombstoneV("e"), 251 *makeTombstoneV("f"), 252 }) 253 254 writer := mock.NewMockMetaRangeWriter(ctrl) 255 256 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 257 assert.Error(t, err, graveler.ErrNoChanges) 258 assert.Equal(t, graveler.DiffSummary{ 259 Count: map[graveler.DiffType]int{}, 260 }, summary) 261 } 262 263 func TestCommitDeleteNonExistingRecord(t *testing.T) { 264 ctrl := gomock.NewController(t) 265 266 base := testutil.NewFakeIterator(). 267 AddRange(&committed.Range{ID: "base:one", MinKey: committed.Key("c"), MaxKey: committed.Key("d"), Count: 2}). 268 AddValueRecords(makeV("c", "base:c"), makeV("d", "base:d")) 269 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 270 *makeV("a", "changes:a"), 271 *makeTombstoneV("e"), 272 *makeTombstoneV("f"), 273 }) 274 275 writer := mock.NewMockMetaRangeWriter(ctrl) 276 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("a", "changes:a"))) 277 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 278 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("d", "base:d"))) 279 280 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 281 282 assert.NoError(t, err) 283 assert.Equal(t, graveler.DiffSummary{ 284 Count: map[graveler.DiffType]int{ 285 graveler.DiffTypeAdded: 1, 286 }, 287 }, summary) 288 } 289 290 func TestCommitTombstonesBeforeRange(t *testing.T) { 291 ctrl := gomock.NewController(t) 292 293 base := testutil.NewFakeIterator(). 294 AddRange(&committed.Range{ID: "base:range", MinKey: committed.Key("b"), MaxKey: committed.Key("c"), Count: 2}). 295 AddValueRecords(makeV("b", "base:b"), makeV("c", "base:c")) 296 297 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 298 *makeTombstoneV("a"), 299 *makeV("d", "changes:d"), 300 }) 301 302 writer := mock.NewMockMetaRangeWriter(ctrl) 303 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("b", "base:b"))) 304 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("c", "base:c"))) 305 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("d", "changes:d"))) 306 307 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 308 assert.NoError(t, err) 309 assert.Equal(t, graveler.DiffSummary{ 310 Count: map[graveler.DiffType]int{ 311 graveler.DiffTypeAdded: 1, 312 }, 313 }, summary) 314 } 315 316 func TestCommitCopiesLeftoverBase(t *testing.T) { 317 ctrl := gomock.NewController(t) 318 319 range1 := &committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3} 320 range2 := &committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1} 321 range4 := &committed.Range{ID: "four", MinKey: committed.Key("g"), MaxKey: committed.Key("hz"), Count: 2} 322 base := testutil.NewFakeIterator(). 323 AddRange(range1). 324 AddValueRecords(makeV("a", "base:a"), makeV("b", "base:b"), makeV("c", "base:c")). 325 AddRange(range2). 326 AddValueRecords(makeV("d", "base:d")). 327 AddRange(&committed.Range{ID: "three", MinKey: committed.Key("e"), MaxKey: committed.Key("f"), Count: 1}). 328 AddValueRecords(makeV("e", "base:e"), makeV("f", "base:f")). 329 AddRange(range4). 330 AddValueRecords(makeV("g", "base:g"), makeV("h", "base:h")) 331 332 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{ 333 *makeTombstoneV("e"), 334 }) 335 336 writer := mock.NewMockMetaRangeWriter(ctrl) 337 writer.EXPECT().WriteRange(gomock.Eq(*range1)) 338 writer.EXPECT().WriteRange(gomock.Eq(*range2)) 339 writer.EXPECT().WriteRecord(gomock.Eq(*makeV("f", "base:f"))) 340 writer.EXPECT().WriteRange(gomock.Eq(*range4)) 341 342 summary, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 343 assert.NoError(t, err) 344 assert.Equal(t, graveler.DiffSummary{ 345 Count: map[graveler.DiffType]int{ 346 graveler.DiffTypeRemoved: 1, 347 }, 348 }, summary) 349 } 350 351 func TestCommitNoChangesFails(t *testing.T) { 352 ctrl := gomock.NewController(t) 353 354 range1 := &committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3} 355 range2 := &committed.Range{ID: "two", MinKey: committed.Key("d"), MaxKey: committed.Key("dz"), Count: 1} 356 base := testutil.NewFakeIterator(). 357 AddRange(range1). 358 AddValueRecords(makeV("a", "base:a"), makeV("b", "base:b"), makeV("c", "base:c")). 359 AddRange(range2). 360 AddValueRecords(makeV("d", "base:d")) 361 362 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{}) 363 364 writer := mock.NewMockMetaRangeWriter(ctrl) 365 writer.EXPECT().WriteRange(gomock.Any()).AnyTimes() 366 367 _, err := committed.Commit(context.Background(), writer, base, changes, &committed.CommitOptions{}) 368 assert.Error(t, err, graveler.ErrNoChanges) 369 } 370 371 func TestCommitCancelContext(t *testing.T) { 372 ctrl := gomock.NewController(t) 373 374 t.Run("base", func(t *testing.T) { 375 base := testutil.NewFakeIterator(). 376 AddRange(&committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 2}). 377 AddValueRecords(makeV("a", "base:a"), makeV("c", "base:c")) 378 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{}) 379 writer := mock.NewMockMetaRangeWriter(ctrl) 380 ctx, cancel := context.WithCancel(context.Background()) 381 cancel() 382 _, err := committed.Commit(ctx, writer, base, changes, &committed.CommitOptions{}) 383 assert.True(t, errors.Is(err, context.Canceled), "context canceled error") 384 }) 385 386 t.Run("change", func(t *testing.T) { 387 base := testutil.NewFakeIterator() 388 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{*makeV("b", "dest:b")}) 389 writer := mock.NewMockMetaRangeWriter(ctrl) 390 ctx, cancel := context.WithCancel(context.Background()) 391 cancel() 392 _, err := committed.Commit(ctx, writer, base, changes, &committed.CommitOptions{}) 393 assert.True(t, errors.Is(err, context.Canceled), "context canceled error") 394 }) 395 396 t.Run("base_and_change", func(t *testing.T) { 397 base := testutil.NewFakeIterator(). 398 AddRange(&committed.Range{ID: "one", MinKey: committed.Key("a"), MaxKey: committed.Key("cz"), Count: 3}). 399 AddValueRecords(makeV("a", "base:a"), makeV("c", "base:c")) 400 changes := testutil.NewValueIteratorFake([]graveler.ValueRecord{*makeV("b", "dest:b")}) 401 writer := mock.NewMockMetaRangeWriter(ctrl) 402 ctx, cancel := context.WithCancel(context.Background()) 403 cancel() 404 _, err := committed.Commit(ctx, writer, base, changes, &committed.CommitOptions{}) 405 assert.True(t, errors.Is(err, context.Canceled), "context canceled error") 406 }) 407 }