github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/graveler/ref/commit_ordered_iterator_test.go (about)

     1  package ref_test
     2  
     3  import (
     4  	"context"
     5  	"encoding/hex"
     6  	"fmt"
     7  	"sort"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/go-openapi/swag"
    12  	"github.com/go-test/deep"
    13  	"github.com/golang/mock/gomock"
    14  	"github.com/treeverse/lakefs/pkg/graveler"
    15  	"github.com/treeverse/lakefs/pkg/graveler/ref"
    16  	"github.com/treeverse/lakefs/pkg/kv/mock"
    17  	"github.com/treeverse/lakefs/pkg/testutil"
    18  )
    19  
    20  func newFakeAddressProvider(commits []*graveler.Commit) *fakeAddressProvider {
    21  	identityToFakeIdentity := make(map[string]string)
    22  	for _, c := range commits {
    23  		identityToFakeIdentity[hex.EncodeToString(c.Identity())] = c.Message
    24  	}
    25  	return &fakeAddressProvider{identityToFakeIdentity: identityToFakeIdentity}
    26  }
    27  
    28  type testCommit struct {
    29  	id      string
    30  	parents []string
    31  	version *int
    32  }
    33  
    34  func TestOrderedCommitIterator(t *testing.T) {
    35  	ctx := context.Background()
    36  	cases := []struct {
    37  		Name                   string
    38  		Commits                []*testCommit
    39  		expectedCommits        []string
    40  		expectedAncestryLeaves []string
    41  	}{
    42  		{
    43  			Name: "two_ancestries",
    44  			Commits: []*testCommit{
    45  				{id: "e0", parents: []string{}},
    46  				{id: "c1", parents: []string{"e0"}},
    47  				{id: "b2", parents: []string{"e0"}},
    48  				{id: "d3", parents: []string{"c1"}},
    49  				{id: "e4", parents: []string{"b2"}},
    50  				{id: "f5", parents: []string{"d3"}},
    51  				{id: "a6", parents: []string{"e4"}},
    52  				{id: "c7", parents: []string{"f5"}},
    53  			},
    54  			expectedCommits:        []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"},
    55  			expectedAncestryLeaves: []string{"a6", "c7"},
    56  		},
    57  		{
    58  			Name: "with_merge",
    59  			Commits: []*testCommit{
    60  				{id: "e0", parents: []string{}},
    61  				{id: "c1", parents: []string{"e0"}},
    62  				{id: "b2", parents: []string{"e0"}},
    63  				{id: "d3", parents: []string{"c1"}},
    64  				{id: "e4", parents: []string{"b2"}},
    65  				{id: "f5", parents: []string{"d3"}},
    66  				{id: "a6", parents: []string{"e4"}},
    67  				{id: "c7", parents: []string{"f5", "a6"}},
    68  			},
    69  			expectedCommits:        []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"},
    70  			expectedAncestryLeaves: []string{"a6", "c7"},
    71  		},
    72  		{
    73  			Name: "with_merge_old_version",
    74  			Commits: []*testCommit{
    75  				{id: "e0", parents: []string{}},
    76  				{id: "c1", parents: []string{"e0"}},
    77  				{id: "b2", parents: []string{"e0"}},
    78  				{id: "d3", parents: []string{"c1"}},
    79  				{id: "e4", parents: []string{"b2"}},
    80  				{id: "f5", parents: []string{"d3"}},
    81  				{id: "a6", parents: []string{"e4"}},
    82  				{id: "c7", parents: []string{"f5", "a6"}, version: swag.Int(int(graveler.CommitVersionInitial))},
    83  			},
    84  			expectedCommits:        []string{"a6", "b2", "c1", "c7", "d3", "e0", "e4", "f5"},
    85  			expectedAncestryLeaves: []string{"c7", "f5"},
    86  		},
    87  		{
    88  			Name: "criss_cross_merge",
    89  			Commits: []*testCommit{
    90  				{id: "e0", parents: []string{}},
    91  				{id: "c1", parents: []string{"e0"}},
    92  				{id: "b2", parents: []string{"e0"}},
    93  				{id: "d3", parents: []string{"c1", "b2"}},
    94  				{id: "a4", parents: []string{"c1", "b2"}},
    95  				{id: "c5", parents: []string{"d3"}},
    96  				{id: "f6", parents: []string{"a4"}},
    97  			},
    98  			expectedCommits:        []string{"a4", "b2", "c1", "c5", "d3", "e0", "f6"},
    99  			expectedAncestryLeaves: []string{"b2", "c5", "f6"},
   100  		},
   101  		{
   102  			Name: "merges_in_history",
   103  			// from git core tests:
   104  			// E---D---C---B---A
   105  			// \"-_         \   \
   106  			//  \  `---------G   \
   107  			//   \                \
   108  			//    F----------------H
   109  			Commits: []*testCommit{
   110  				{id: "e", parents: []string{}},
   111  				{id: "d", parents: []string{"e"}},
   112  				{id: "f", parents: []string{"e"}},
   113  				{id: "c", parents: []string{"d"}},
   114  				{id: "b", parents: []string{"c"}},
   115  				{id: "a", parents: []string{"b"}},
   116  				{id: "g", parents: []string{"b", "e"}},
   117  				{id: "h", parents: []string{"a", "f"}},
   118  			},
   119  			expectedCommits:        []string{"a", "b", "c", "d", "e", "f", "g", "h"},
   120  			expectedAncestryLeaves: []string{"f", "g", "h"},
   121  		},
   122  		{
   123  			Name: "merge_to_and_from_main",
   124  			// E---D----C---B------A
   125  			//  \      /     \    /
   126  			//   F----G---H---I--J
   127  			Commits: []*testCommit{
   128  				{id: "e", parents: []string{}},
   129  				{id: "d", parents: []string{"e"}},
   130  				{id: "f", parents: []string{"e"}},
   131  				{id: "g", parents: []string{"f"}},
   132  				{id: "c", parents: []string{"d", "g"}},
   133  				{id: "b", parents: []string{"c"}},
   134  				{id: "h", parents: []string{"g"}},
   135  				{id: "i", parents: []string{"h", "b"}},
   136  				{id: "j", parents: []string{"i"}},
   137  				{id: "a", parents: []string{"b", "j"}},
   138  			},
   139  			expectedCommits:        []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"},
   140  			expectedAncestryLeaves: []string{"a", "j"},
   141  		},
   142  	}
   143  	for _, tst := range cases {
   144  		t.Run(tst.Name, func(t *testing.T) {
   145  			var commits []*graveler.Commit
   146  			for _, tstCommit := range tst.Commits {
   147  				parents := make([]graveler.CommitID, 0, len(tstCommit.parents))
   148  				for _, parent := range tstCommit.parents {
   149  					parents = append(parents, graveler.CommitID(parent))
   150  				}
   151  				version := graveler.CurrentCommitVersion
   152  				if tstCommit.version != nil {
   153  					version = graveler.CommitVersion(*tstCommit.version)
   154  				}
   155  				commits = append(commits, &graveler.Commit{
   156  					Message: tstCommit.id,
   157  					Parents: parents,
   158  					Version: version,
   159  				})
   160  			}
   161  			r, store := testRefManagerWithAddressProvider(t, newFakeAddressProvider(commits))
   162  			repository, err := r.CreateBareRepository(ctx, graveler.RepositoryID(tst.Name), graveler.Repository{
   163  				StorageNamespace: "s3://foo",
   164  				CreationDate:     time.Now(),
   165  				DefaultBranchID:  "main",
   166  			})
   167  			testutil.Must(t, err)
   168  			for _, c := range commits {
   169  				_, err := r.AddCommit(ctx, repository, *c)
   170  				testutil.Must(t, err)
   171  			}
   172  
   173  			iterator, err := ref.NewOrderedCommitIterator(ctx, store, repository, false)
   174  			if err != nil {
   175  				t.Fatal("create ordered commit iterator", err)
   176  			}
   177  			var actualOrderedCommits []string
   178  			for iterator.Next() {
   179  				c := iterator.Value()
   180  				actualOrderedCommits = append(
   181  					actualOrderedCommits,
   182  					string(c.CommitID),
   183  				)
   184  			}
   185  			testutil.Must(t, iterator.Err())
   186  			if diff := deep.Equal(tst.expectedCommits, actualOrderedCommits); diff != nil {
   187  				t.Errorf("Unexpected ordered commits from iterator. diff=%s", diff)
   188  			}
   189  			iterator.Close()
   190  			iterator, err = ref.NewOrderedCommitIterator(ctx, store, repository, true)
   191  			if err != nil {
   192  				t.Fatal("create ordered commit iterator", err)
   193  			}
   194  			var actualAncestryLeaves []string
   195  			for iterator.Next() {
   196  				c := iterator.Value()
   197  				actualAncestryLeaves = append(
   198  					actualAncestryLeaves,
   199  					string(c.CommitID),
   200  				)
   201  			}
   202  			testutil.Must(t, iterator.Err())
   203  			if diff := deep.Equal(tst.expectedAncestryLeaves, actualAncestryLeaves); diff != nil {
   204  				t.Errorf("Unexpected ancestry leaves from iterator. diff=%s", diff)
   205  			}
   206  			iterator.Close()
   207  		})
   208  	}
   209  }
   210  
   211  func TestOrderedCommitIteratorGrid(t *testing.T) {
   212  	// Construct the following grid, taken from https://github.com/git/git/blob/master/t/t6600-test-reach.sh
   213  	//             (10,10)
   214  	//            /       \
   215  	//         (10,9)    (9,10)
   216  	//        /     \   /      \
   217  	//    (10,8)    (9,9)      (8,10)
   218  	//   /     \    /   \      /    \
   219  	//         ( continued...)
   220  	//   \     /    \   /      \    /
   221  	//    (3,1)     (2,2)      (1,3)
   222  	//        \     /    \     /
   223  	//         (2,1)      (1, 2)
   224  	//              \    /
   225  	//              (1,1)
   226  	grid := make([][]*graveler.Commit, 10)
   227  	commits := make([]*graveler.Commit, 0, 100)
   228  	expectedCommitIDS := make([]string, 0, 100)
   229  	expectedAncestryLeaves := make([]string, 0, 10)
   230  	for i := 0; i < 10; i++ {
   231  		grid[i] = make([]*graveler.Commit, 10)
   232  		for j := 0; j < 10; j++ {
   233  			parents := make([]graveler.CommitID, 0, 2)
   234  			if i > 0 {
   235  				parents = append(parents, graveler.CommitID(fmt.Sprintf("%d-%d", i-1, j)))
   236  			}
   237  			if j > 0 {
   238  				parents = append(parents, graveler.CommitID(fmt.Sprintf("%d-%d", i, j-1)))
   239  			}
   240  			c := &graveler.Commit{Message: fmt.Sprintf("%d-%d", i, j), Parents: parents, Version: graveler.CurrentCommitVersion}
   241  			grid[i][j] = c
   242  			expectedCommitIDS = append(expectedCommitIDS, fmt.Sprintf("%d-%d", i, j))
   243  			if i == 9 {
   244  				// nodes with i == 9 are never the first parent of any node, since for all nodes
   245  				expectedAncestryLeaves = append(expectedAncestryLeaves, fmt.Sprintf("%d-%d", i, j))
   246  			}
   247  			commits = append(commits, c)
   248  		}
   249  	}
   250  	sort.Strings(expectedCommitIDS)
   251  	r, store := testRefManagerWithAddressProvider(t, newFakeAddressProvider(commits))
   252  	ctx := context.Background()
   253  	repo := graveler.RepositoryRecord{
   254  		RepositoryID: graveler.RepositoryID("repo"),
   255  		Repository: &graveler.Repository{
   256  			StorageNamespace: "s3://foo",
   257  			CreationDate:     time.Now(),
   258  			DefaultBranchID:  "main",
   259  		},
   260  	}
   261  	repository, err := r.CreateBareRepository(ctx, repo.RepositoryID, *repo.Repository)
   262  	testutil.Must(t, err)
   263  	for _, c := range commits {
   264  		_, err := r.AddCommit(ctx, repository, *c)
   265  		testutil.Must(t, err)
   266  	}
   267  	iterator, err := ref.NewOrderedCommitIterator(ctx, store, &repo, false)
   268  	if err != nil {
   269  		t.Fatal("create ordered commit iterator", err)
   270  	}
   271  	var actualOrderedCommits []string
   272  	for iterator.Next() {
   273  		c := iterator.Value()
   274  		actualOrderedCommits = append(
   275  			actualOrderedCommits,
   276  			string(c.CommitID),
   277  		)
   278  	}
   279  	testutil.Must(t, iterator.Err())
   280  	if diff := deep.Equal(expectedCommitIDS, actualOrderedCommits); diff != nil {
   281  		t.Errorf("Unexpected ordered commits from iterator. diff=%s", diff)
   282  	}
   283  	iterator.Close()
   284  	iterator, err = ref.NewOrderedCommitIterator(ctx, store, &repo, true)
   285  	if err != nil {
   286  		t.Fatal("create ordered commit iterator", err)
   287  	}
   288  	var actualAncestryLeaves []string
   289  	for iterator.Next() {
   290  		c := iterator.Value()
   291  		actualAncestryLeaves = append(
   292  			actualAncestryLeaves,
   293  			string(c.CommitID),
   294  		)
   295  	}
   296  	testutil.Must(t, iterator.Err())
   297  	if diff := deep.Equal(expectedAncestryLeaves, actualAncestryLeaves); diff != nil {
   298  		t.Errorf("Unexpected ancestry leaves from iterator. diff=%s", diff)
   299  	}
   300  	iterator.Close()
   301  }
   302  
   303  func TestOrderedCommitIterator_CloseTwice(t *testing.T) {
   304  	ctrl := gomock.NewController(t)
   305  	ctx := context.Background()
   306  	entIt := mock.NewMockEntriesIterator(ctrl)
   307  	entIt.EXPECT().Close().Times(1)
   308  	store := mock.NewMockStore(ctrl)
   309  	store.EXPECT().Scan(ctx, gomock.Any(), gomock.Any()).Return(entIt, nil).Times(1)
   310  	repository := &graveler.RepositoryRecord{
   311  		RepositoryID: "CommitIterClose",
   312  		Repository: &graveler.Repository{
   313  			StorageNamespace: "",
   314  			CreationDate:     time.Time{},
   315  			DefaultBranchID:  "",
   316  			State:            0,
   317  			InstanceUID:      "rid",
   318  		},
   319  	}
   320  	it, err := ref.NewOrderedCommitIterator(ctx, store, repository, false)
   321  	if err != nil {
   322  		t.Fatal("failed to create ordered commit iterator", err)
   323  	}
   324  	it.Close()
   325  	// calling 'Close()` again to verify the iterator doesn't call the internal iterator close method
   326  	it.Close()
   327  }
   328  
   329  func TestOrderedCommitIterator_NextAfterClose(t *testing.T) {
   330  	ctrl := gomock.NewController(t)
   331  	ctx := context.Background()
   332  	entIt := mock.NewMockEntriesIterator(ctrl)
   333  	entIt.EXPECT().Close().Times(1)
   334  	store := mock.NewMockStore(ctrl)
   335  	store.EXPECT().Scan(ctx, gomock.Any(), gomock.Any()).Return(entIt, nil).Times(1)
   336  	repository := &graveler.RepositoryRecord{
   337  		RepositoryID: "CommitIterClose",
   338  		Repository: &graveler.Repository{
   339  			InstanceUID: "rid",
   340  		},
   341  	}
   342  	it, err := ref.NewOrderedCommitIterator(ctx, store, repository, false)
   343  	if err != nil {
   344  		t.Fatal("failed to create kv ordered commit iterator", err)
   345  	}
   346  	it.Close()
   347  	// calling Next after Close should not crash
   348  	if it.Next() {
   349  		t.Fatal("Next() should return false after Close()")
   350  	}
   351  }