github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/graveler/ref/commit_ordered_iterator_test.go (about) 1 package ref_test 2 3 import ( 4 "context" 5 "encoding/hex" 6 "fmt" 7 "sort" 8 "testing" 9 "time" 10 11 "github.com/go-openapi/swag" 12 "github.com/go-test/deep" 13 "github.com/golang/mock/gomock" 14 "github.com/treeverse/lakefs/pkg/graveler" 15 "github.com/treeverse/lakefs/pkg/graveler/ref" 16 "github.com/treeverse/lakefs/pkg/kv/mock" 17 "github.com/treeverse/lakefs/pkg/testutil" 18 ) 19 20 func newFakeAddressProvider(commits []*graveler.Commit) *fakeAddressProvider { 21 identityToFakeIdentity := make(map[string]string) 22 for _, c := range commits { 23 identityToFakeIdentity[hex.EncodeToString(c.Identity())] = c.Message 24 } 25 return &fakeAddressProvider{identityToFakeIdentity: identityToFakeIdentity} 26 } 27 28 type testCommit struct { 29 id string 30 parents []string 31 version *int 32 } 33 34 func TestOrderedCommitIterator(t *testing.T) { 35 ctx := context.Background() 36 cases := []struct { 37 Name string 38 Commits []*testCommit 39 expectedCommits []string 40 expectedAncestryLeaves []string 41 }{ 42 { 43 Name: "two_ancestries", 44 Commits: []*testCommit{ 45 {id: "e0", parents: []string{}}, 46 {id: "c1", parents: []string{"e0"}}, 47 {id: "b2", parents: []string{"e0"}}, 48 {id: "d3", parents: []string{"c1"}}, 49 {id: "e4", parents: []string{"b2"}}, 50 {id: "f5", parents: []string{"d3"}}, 51 {id: "a6", parents: []string{"e4"}}, 52 {id: "c7", parents: []string{"f5"}}, 53 }, 54 expectedCommits: []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"}, 55 expectedAncestryLeaves: []string{"a6", "c7"}, 56 }, 57 { 58 Name: "with_merge", 59 Commits: []*testCommit{ 60 {id: "e0", parents: []string{}}, 61 {id: "c1", parents: []string{"e0"}}, 62 {id: "b2", parents: []string{"e0"}}, 63 {id: "d3", parents: []string{"c1"}}, 64 {id: "e4", parents: []string{"b2"}}, 65 {id: "f5", parents: []string{"d3"}}, 66 {id: "a6", parents: []string{"e4"}}, 67 {id: "c7", parents: []string{"f5", "a6"}}, 68 }, 69 expectedCommits: []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"}, 70 expectedAncestryLeaves: []string{"a6", "c7"}, 71 }, 72 { 73 Name: "with_merge_old_version", 74 Commits: []*testCommit{ 75 {id: "e0", parents: []string{}}, 76 {id: "c1", parents: []string{"e0"}}, 77 {id: "b2", parents: []string{"e0"}}, 78 {id: "d3", parents: []string{"c1"}}, 79 {id: "e4", parents: []string{"b2"}}, 80 {id: "f5", parents: []string{"d3"}}, 81 {id: "a6", parents: []string{"e4"}}, 82 {id: "c7", parents: []string{"f5", "a6"}, version: swag.Int(int(graveler.CommitVersionInitial))}, 83 }, 84 expectedCommits: []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"}, 85 expectedAncestryLeaves: []string{"c7", "f5"}, 86 }, 87 { 88 Name: "criss_cross_merge", 89 Commits: []*testCommit{ 90 {id: "e0", parents: []string{}}, 91 {id: "c1", parents: []string{"e0"}}, 92 {id: "b2", parents: []string{"e0"}}, 93 {id: "d3", parents: []string{"c1", "b2"}}, 94 {id: "a4", parents: []string{"c1", "b2"}}, 95 {id: "c5", parents: []string{"d3"}}, 96 {id: "f6", parents: []string{"a4"}}, 97 }, 98 expectedCommits: []string{"a4", "b2", "c1", "c5", "d3", "e0", "f6"}, 99 expectedAncestryLeaves: []string{"b2", "c5", "f6"}, 100 }, 101 { 102 Name: "merges_in_history", 103 // from git core tests: 104 // E---D---C---B---A 105 // \"-_ \ \ 106 // \ `---------G \ 107 // \ \ 108 // F----------------H 109 Commits: []*testCommit{ 110 {id: "e", parents: []string{}}, 111 {id: "d", parents: []string{"e"}}, 112 {id: "f", parents: []string{"e"}}, 113 {id: "c", parents: []string{"d"}}, 114 {id: "b", parents: []string{"c"}}, 115 {id: "a", parents: []string{"b"}}, 116 {id: "g", parents: []string{"b", "e"}}, 117 {id: "h", parents: []string{"a", "f"}}, 118 }, 119 expectedCommits: []string{"a", "b", "c", "d", "e", "f", "g", "h"}, 120 expectedAncestryLeaves: []string{"f", "g", "h"}, 121 }, 122 { 123 Name: "merge_to_and_from_main", 124 // E---D----C---B------A 125 // \ / \ / 126 // F----G---H---I--J 127 Commits: []*testCommit{ 128 {id: "e", parents: []string{}}, 129 {id: "d", parents: []string{"e"}}, 130 {id: "f", parents: []string{"e"}}, 131 {id: "g", parents: []string{"f"}}, 132 {id: "c", parents: []string{"d", "g"}}, 133 {id: "b", parents: []string{"c"}}, 134 {id: "h", parents: []string{"g"}}, 135 {id: "i", parents: []string{"h", "b"}}, 136 {id: "j", parents: []string{"i"}}, 137 {id: "a", parents: []string{"b", "j"}}, 138 }, 139 expectedCommits: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, 140 expectedAncestryLeaves: []string{"a", "j"}, 141 }, 142 } 143 for _, tst := range cases { 144 t.Run(tst.Name, func(t *testing.T) { 145 var commits []*graveler.Commit 146 for _, tstCommit := range tst.Commits { 147 parents := make([]graveler.CommitID, 0, len(tstCommit.parents)) 148 for _, parent := range tstCommit.parents { 149 parents = append(parents, graveler.CommitID(parent)) 150 } 151 version := graveler.CurrentCommitVersion 152 if tstCommit.version != nil { 153 version = graveler.CommitVersion(*tstCommit.version) 154 } 155 commits = append(commits, &graveler.Commit{ 156 Message: tstCommit.id, 157 Parents: parents, 158 Version: version, 159 }) 160 } 161 r, store := testRefManagerWithAddressProvider(t, newFakeAddressProvider(commits)) 162 repository, err := r.CreateBareRepository(ctx, graveler.RepositoryID(tst.Name), graveler.Repository{ 163 StorageNamespace: "s3://foo", 164 CreationDate: time.Now(), 165 DefaultBranchID: "main", 166 }) 167 testutil.Must(t, err) 168 for _, c := range commits { 169 _, err := r.AddCommit(ctx, repository, *c) 170 testutil.Must(t, err) 171 } 172 173 iterator, err := ref.NewOrderedCommitIterator(ctx, store, repository, false) 174 if err != nil { 175 t.Fatal("create ordered commit iterator", err) 176 } 177 var actualOrderedCommits []string 178 for iterator.Next() { 179 c := iterator.Value() 180 actualOrderedCommits = append( 181 actualOrderedCommits, 182 string(c.CommitID), 183 ) 184 } 185 testutil.Must(t, iterator.Err()) 186 if diff := deep.Equal(tst.expectedCommits, actualOrderedCommits); diff != nil { 187 t.Errorf("Unexpected ordered commits from iterator. diff=%s", diff) 188 } 189 iterator.Close() 190 iterator, err = ref.NewOrderedCommitIterator(ctx, store, repository, true) 191 if err != nil { 192 t.Fatal("create ordered commit iterator", err) 193 } 194 var actualAncestryLeaves []string 195 for iterator.Next() { 196 c := iterator.Value() 197 actualAncestryLeaves = append( 198 actualAncestryLeaves, 199 string(c.CommitID), 200 ) 201 } 202 testutil.Must(t, iterator.Err()) 203 if diff := deep.Equal(tst.expectedAncestryLeaves, actualAncestryLeaves); diff != nil { 204 t.Errorf("Unexpected ancestry leaves from iterator. diff=%s", diff) 205 } 206 iterator.Close() 207 }) 208 } 209 } 210 211 func TestOrderedCommitIteratorGrid(t *testing.T) { 212 // Construct the following grid, taken from https://github.com/git/git/blob/master/t/t6600-test-reach.sh 213 // (10,10) 214 // / \ 215 // (10,9) (9,10) 216 // / \ / \ 217 // (10,8) (9,9) (8,10) 218 // / \ / \ / \ 219 // ( continued...) 220 // \ / \ / \ / 221 // (3,1) (2,2) (1,3) 222 // \ / \ / 223 // (2,1) (1, 2) 224 // \ / 225 // (1,1) 226 grid := make([][]*graveler.Commit, 10) 227 commits := make([]*graveler.Commit, 0, 100) 228 expectedCommitIDS := make([]string, 0, 100) 229 expectedAncestryLeaves := make([]string, 0, 10) 230 for i := 0; i < 10; i++ { 231 grid[i] = make([]*graveler.Commit, 10) 232 for j := 0; j < 10; j++ { 233 parents := make([]graveler.CommitID, 0, 2) 234 if i > 0 { 235 parents = append(parents, graveler.CommitID(fmt.Sprintf("%d-%d", i-1, j))) 236 } 237 if j > 0 { 238 parents = append(parents, graveler.CommitID(fmt.Sprintf("%d-%d", i, j-1))) 239 } 240 c := &graveler.Commit{Message: fmt.Sprintf("%d-%d", i, j), Parents: parents, Version: graveler.CurrentCommitVersion} 241 grid[i][j] = c 242 expectedCommitIDS = append(expectedCommitIDS, fmt.Sprintf("%d-%d", i, j)) 243 if i == 9 { 244 // nodes with i == 9 are never the first parent of any node, since for all nodes 245 expectedAncestryLeaves = append(expectedAncestryLeaves, fmt.Sprintf("%d-%d", i, j)) 246 } 247 commits = append(commits, c) 248 } 249 } 250 sort.Strings(expectedCommitIDS) 251 r, store := testRefManagerWithAddressProvider(t, newFakeAddressProvider(commits)) 252 ctx := context.Background() 253 repo := graveler.RepositoryRecord{ 254 RepositoryID: graveler.RepositoryID("repo"), 255 Repository: &graveler.Repository{ 256 StorageNamespace: "s3://foo", 257 CreationDate: time.Now(), 258 DefaultBranchID: "main", 259 }, 260 } 261 repository, err := r.CreateBareRepository(ctx, repo.RepositoryID, *repo.Repository) 262 testutil.Must(t, err) 263 for _, c := range commits { 264 _, err := r.AddCommit(ctx, repository, *c) 265 testutil.Must(t, err) 266 } 267 iterator, err := ref.NewOrderedCommitIterator(ctx, store, &repo, false) 268 if err != nil { 269 t.Fatal("create ordered commit iterator", err) 270 } 271 var actualOrderedCommits []string 272 for iterator.Next() { 273 c := iterator.Value() 274 actualOrderedCommits = append( 275 actualOrderedCommits, 276 string(c.CommitID), 277 ) 278 } 279 testutil.Must(t, iterator.Err()) 280 if diff := deep.Equal(expectedCommitIDS, actualOrderedCommits); diff != nil { 281 t.Errorf("Unexpected ordered commits from iterator. diff=%s", diff) 282 } 283 iterator.Close() 284 iterator, err = ref.NewOrderedCommitIterator(ctx, store, &repo, true) 285 if err != nil { 286 t.Fatal("create ordered commit iterator", err) 287 } 288 var actualAncestryLeaves []string 289 for iterator.Next() { 290 c := iterator.Value() 291 actualAncestryLeaves = append( 292 actualAncestryLeaves, 293 string(c.CommitID), 294 ) 295 } 296 testutil.Must(t, iterator.Err()) 297 if diff := deep.Equal(expectedAncestryLeaves, actualAncestryLeaves); diff != nil { 298 t.Errorf("Unexpected ancestry leaves from iterator. diff=%s", diff) 299 } 300 iterator.Close() 301 } 302 303 func TestOrderedCommitIterator_CloseTwice(t *testing.T) { 304 ctrl := gomock.NewController(t) 305 ctx := context.Background() 306 entIt := mock.NewMockEntriesIterator(ctrl) 307 entIt.EXPECT().Close().Times(1) 308 store := mock.NewMockStore(ctrl) 309 store.EXPECT().Scan(ctx, gomock.Any(), gomock.Any()).Return(entIt, nil).Times(1) 310 repository := &graveler.RepositoryRecord{ 311 RepositoryID: "CommitIterClose", 312 Repository: &graveler.Repository{ 313 StorageNamespace: "", 314 CreationDate: time.Time{}, 315 DefaultBranchID: "", 316 State: 0, 317 InstanceUID: "rid", 318 }, 319 } 320 it, err := ref.NewOrderedCommitIterator(ctx, store, repository, false) 321 if err != nil { 322 t.Fatal("failed to create ordered commit iterator", err) 323 } 324 it.Close() 325 // calling 'Close()` again to verify the iterator doesn't call the internal iterator close method 326 it.Close() 327 } 328 329 func TestOrderedCommitIterator_NextAfterClose(t *testing.T) { 330 ctrl := gomock.NewController(t) 331 ctx := context.Background() 332 entIt := mock.NewMockEntriesIterator(ctrl) 333 entIt.EXPECT().Close().Times(1) 334 store := mock.NewMockStore(ctrl) 335 store.EXPECT().Scan(ctx, gomock.Any(), gomock.Any()).Return(entIt, nil).Times(1) 336 repository := &graveler.RepositoryRecord{ 337 RepositoryID: "CommitIterClose", 338 Repository: &graveler.Repository{ 339 InstanceUID: "rid", 340 }, 341 } 342 it, err := ref.NewOrderedCommitIterator(ctx, store, repository, false) 343 if err != nil { 344 t.Fatal("failed to create kv ordered commit iterator", err) 345 } 346 it.Close() 347 // calling Next after Close should not crash 348 if it.Next() { 349 t.Fatal("Next() should return false after Close()") 350 } 351 }