github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/libraries/doltcore/doltdb/commit_hooks_test.go (about)

     1  // Copyright 2021 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package doltdb
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"errors"
    21  	"io"
    22  	"path/filepath"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/dolthub/go-mysql-server/sql"
    27  	"github.com/stretchr/testify/assert"
    28  	"github.com/stretchr/testify/require"
    29  	"go.uber.org/zap/buffer"
    30  
    31  	"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
    32  	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
    33  	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
    34  	"github.com/dolthub/dolt/go/libraries/utils/filesys"
    35  	"github.com/dolthub/dolt/go/libraries/utils/test"
    36  	"github.com/dolthub/dolt/go/store/datas"
    37  	"github.com/dolthub/dolt/go/store/types"
    38  )
    39  
    40  const defaultBranch = "main"
    41  
    42  func TestPushOnWriteHook(t *testing.T) {
    43  	ctx := context.Background()
    44  
    45  	// destination repo
    46  	testDir, err := test.ChangeToTestDir("TestReplicationDest")
    47  
    48  	if err != nil {
    49  		panic("Couldn't change the working directory to the test directory.")
    50  	}
    51  
    52  	committerName := "Bill Billerson"
    53  	committerEmail := "bigbillieb@fake.horse"
    54  
    55  	tmpDir := filepath.Join(testDir, dbfactory.DoltDataDir)
    56  	err = filesys.LocalFS.MkDirs(tmpDir)
    57  
    58  	if err != nil {
    59  		t.Fatal("Failed to create noms directory")
    60  	}
    61  
    62  	destDB, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS)
    63  
    64  	// source repo
    65  	testDir, err = test.ChangeToTestDir("TestReplicationSource")
    66  
    67  	if err != nil {
    68  		panic("Couldn't change the working directory to the test directory.")
    69  	}
    70  
    71  	tmpDir = filepath.Join(testDir, dbfactory.DoltDataDir)
    72  	err = filesys.LocalFS.MkDirs(tmpDir)
    73  
    74  	if err != nil {
    75  		t.Fatal("Failed to create noms directory")
    76  	}
    77  
    78  	ddb, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS)
    79  	err = ddb.WriteEmptyRepo(context.Background(), "main", committerName, committerEmail)
    80  
    81  	if err != nil {
    82  		t.Fatal("Unexpected error creating empty repo", err)
    83  	}
    84  
    85  	// prepare a commit in the source repo
    86  	cs, _ := NewCommitSpec("main")
    87  	optCmt, err := ddb.Resolve(context.Background(), cs, nil)
    88  	if err != nil {
    89  		t.Fatal("Couldn't find commit")
    90  	}
    91  	commit, ok := optCmt.ToCommit()
    92  	assert.True(t, ok)
    93  
    94  	meta, err := commit.GetCommitMeta(context.Background())
    95  	assert.NoError(t, err)
    96  
    97  	if meta.Name != committerName || meta.Email != committerEmail {
    98  		t.Error("Unexpected metadata")
    99  	}
   100  
   101  	root, err := commit.GetRootValue(context.Background())
   102  
   103  	assert.NoError(t, err)
   104  
   105  	names, err := root.GetTableNames(context.Background(), DefaultSchemaName)
   106  	assert.NoError(t, err)
   107  	if len(names) != 0 {
   108  		t.Fatal("There should be no tables in empty db")
   109  	}
   110  
   111  	tSchema := createTestSchema(t)
   112  	rowData := createTestRowData(t, ddb.vrw, ddb.ns, tSchema)
   113  	tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
   114  
   115  	if err != nil {
   116  		t.Fatal("Failed to create test table with data")
   117  	}
   118  
   119  	root, err = root.PutTable(context.Background(), TableName{Name: "test"}, tbl)
   120  	assert.NoError(t, err)
   121  
   122  	r, valHash, err := ddb.WriteRootValue(context.Background(), root)
   123  	assert.NoError(t, err)
   124  	root = r
   125  
   126  	meta, err = datas.NewCommitMeta(committerName, committerEmail, "Sample data")
   127  	if err != nil {
   128  		t.Error("Failed to commit")
   129  	}
   130  
   131  	// setup hook
   132  	hook := NewPushOnWriteHook(destDB, tmpDir)
   133  	ddb.SetCommitHooks(ctx, []CommitHook{hook})
   134  
   135  	t.Run("replicate to remote", func(t *testing.T) {
   136  		srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta)
   137  		require.NoError(t, err)
   138  
   139  		ds, err := ddb.db.GetDataset(ctx, "refs/heads/main")
   140  		require.NoError(t, err)
   141  
   142  		_, err = hook.Execute(ctx, ds, ddb.db)
   143  		require.NoError(t, err)
   144  
   145  		cs, _ = NewCommitSpec(defaultBranch)
   146  		optCmt, err := destDB.Resolve(context.Background(), cs, nil)
   147  		require.NoError(t, err)
   148  		destCommit, ok := optCmt.ToCommit()
   149  		require.True(t, ok)
   150  
   151  		srcHash, _ := srcCommit.HashOf()
   152  		destHash, _ := destCommit.HashOf()
   153  		assert.Equal(t, srcHash, destHash)
   154  	})
   155  
   156  	t.Run("replicate handle error logs to writer", func(t *testing.T) {
   157  		var buffer = &bytes.Buffer{}
   158  		err = hook.SetLogger(ctx, buffer)
   159  		assert.NoError(t, err)
   160  
   161  		msg := "prince charles is a vampire"
   162  		hook.HandleError(ctx, errors.New(msg))
   163  
   164  		assert.Contains(t, buffer.String(), msg)
   165  	})
   166  }
   167  
   168  func TestLogHook(t *testing.T) {
   169  	msg := []byte("hello")
   170  	var err error
   171  	t.Run("new log hook", func(t *testing.T) {
   172  		ctx := context.Background()
   173  		hook := NewLogHook(msg)
   174  		var buffer = &bytes.Buffer{}
   175  		err = hook.SetLogger(ctx, buffer)
   176  		assert.NoError(t, err)
   177  		hook.Execute(ctx, datas.Dataset{}, nil)
   178  		assert.Equal(t, buffer.Bytes(), msg)
   179  	})
   180  }
   181  
   182  func TestAsyncPushOnWrite(t *testing.T) {
   183  	ctx := context.Background()
   184  
   185  	// destination repo
   186  	testDir, err := test.ChangeToTestDir("TestReplicationDest")
   187  
   188  	if err != nil {
   189  		panic("Couldn't change the working directory to the test directory.")
   190  	}
   191  
   192  	committerName := "Bill Billerson"
   193  	committerEmail := "bigbillieb@fake.horse"
   194  
   195  	tmpDir := filepath.Join(testDir, dbfactory.DoltDataDir)
   196  	err = filesys.LocalFS.MkDirs(tmpDir)
   197  
   198  	if err != nil {
   199  		t.Fatal("Failed to create noms directory")
   200  	}
   201  
   202  	destDB, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS)
   203  
   204  	// source repo
   205  	testDir, err = test.ChangeToTestDir("TestReplicationSource")
   206  
   207  	if err != nil {
   208  		panic("Couldn't change the working directory to the test directory.")
   209  	}
   210  
   211  	tmpDir = filepath.Join(testDir, dbfactory.DoltDataDir)
   212  	err = filesys.LocalFS.MkDirs(tmpDir)
   213  
   214  	if err != nil {
   215  		t.Fatal("Failed to create noms directory")
   216  	}
   217  
   218  	ddb, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS)
   219  	err = ddb.WriteEmptyRepo(context.Background(), "main", committerName, committerEmail)
   220  
   221  	if err != nil {
   222  		t.Fatal("Unexpected error creating empty repo", err)
   223  	}
   224  
   225  	t.Run("replicate to remote", func(t *testing.T) {
   226  		bThreads := sql.NewBackgroundThreads()
   227  		defer bThreads.Shutdown()
   228  		hook, err := NewAsyncPushOnWriteHook(bThreads, destDB, tmpDir, &buffer.Buffer{})
   229  		if err != nil {
   230  			t.Fatal("Unexpected error creating push hook", err)
   231  		}
   232  
   233  		for i := 0; i < 200; i++ {
   234  			cs, _ := NewCommitSpec("main")
   235  			optCmt, err := ddb.Resolve(context.Background(), cs, nil)
   236  			if err != nil {
   237  				t.Fatal("Couldn't find commit")
   238  			}
   239  			commit, ok := optCmt.ToCommit()
   240  			assert.True(t, ok)
   241  
   242  			meta, err := commit.GetCommitMeta(context.Background())
   243  			assert.NoError(t, err)
   244  
   245  			if meta.Name != committerName || meta.Email != committerEmail {
   246  				t.Error("Unexpected metadata")
   247  			}
   248  
   249  			root, err := commit.GetRootValue(context.Background())
   250  
   251  			assert.NoError(t, err)
   252  
   253  			tSchema := createTestSchema(t)
   254  			rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema)
   255  			require.NoError(t, err)
   256  			tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
   257  			require.NoError(t, err)
   258  
   259  			if err != nil {
   260  				t.Fatal("Failed to create test table with data")
   261  			}
   262  
   263  			root, err = root.PutTable(context.Background(), TableName{Name: "test"}, tbl)
   264  			assert.NoError(t, err)
   265  
   266  			r, valHash, err := ddb.WriteRootValue(context.Background(), root)
   267  			assert.NoError(t, err)
   268  			root = r
   269  
   270  			meta, err = datas.NewCommitMeta(committerName, committerEmail, "Sample data")
   271  			if err != nil {
   272  				t.Error("Failed to create CommitMeta")
   273  			}
   274  
   275  			_, err = ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta)
   276  			require.NoError(t, err)
   277  			ds, err := ddb.db.GetDataset(ctx, "refs/heads/main")
   278  			require.NoError(t, err)
   279  			_, err = hook.Execute(ctx, ds, ddb.db)
   280  			require.NoError(t, err)
   281  		}
   282  	})
   283  
   284  	t.Run("does not over replicate branch delete", func(t *testing.T) {
   285  		// We used to have a bug where a branch delete would be
   286  		// replicated over and over again endlessly.
   287  
   288  		// The test construction here is that we put a counting commit
   289  		// hook on *destDB*.  Then we call the async push hook as if we
   290  		// need to replicate certain head updates.  We call once for a
   291  		// branch that does exist and once for a branch which does not
   292  		// exist. Calling with a branch which does not exist looks the
   293  		// same as the call which is made after a branch delete.
   294  
   295  		counts := &countingCommitHook{make(map[string]int)}
   296  		destDB.SetCommitHooks(context.Background(), []CommitHook{counts})
   297  
   298  		bThreads := sql.NewBackgroundThreads()
   299  		hook, err := NewAsyncPushOnWriteHook(bThreads, destDB, tmpDir, &buffer.Buffer{})
   300  		require.NoError(t, err, "create push on write hook without an error")
   301  
   302  		// Pretend we replicate a HEAD which does exist.
   303  		ds, err := ddb.db.GetDataset(ctx, "refs/heads/main")
   304  		require.NoError(t, err)
   305  		_, err = hook.Execute(ctx, ds, ddb.db)
   306  		require.NoError(t, err)
   307  
   308  		// Pretend we replicate a HEAD which does not exist, i.e., a branch delete.
   309  		ds, err = ddb.db.GetDataset(ctx, "refs/heads/does_not_exist")
   310  		require.NoError(t, err)
   311  		_, err = hook.Execute(ctx, ds, ddb.db)
   312  		require.NoError(t, err)
   313  
   314  		// Wait a bit for background thread to fire, in case it is
   315  		// going to betray us. TODO: Structure AsyncPushOnWriteHook to
   316  		// be more testable, so we do not have to rely on
   317  		// non-determinstic goroutine scheduling and best-effort sleeps
   318  		// to observe the potential failure here.
   319  		time.Sleep(10 * time.Second)
   320  
   321  		// Shutdown thread to get final replication if necessary.
   322  		bThreads.Shutdown()
   323  
   324  		// If all went well, the branch delete was executed exactly once.
   325  		require.Equal(t, 1, counts.counts["refs/heads/does_not_exist"])
   326  	})
   327  }
   328  
   329  var _ CommitHook = (*countingCommitHook)(nil)
   330  
   331  type countingCommitHook struct {
   332  	// The number of times Execute() got called for given dataset.
   333  	counts map[string]int
   334  }
   335  
   336  func (c *countingCommitHook) Execute(ctx context.Context, ds datas.Dataset, db datas.Database) (func(context.Context) error, error) {
   337  	c.counts[ds.ID()] += 1
   338  	return nil, nil
   339  }
   340  
   341  func (c *countingCommitHook) HandleError(ctx context.Context, err error) error {
   342  	return nil
   343  }
   344  
   345  func (c *countingCommitHook) SetLogger(ctx context.Context, wr io.Writer) error {
   346  	return nil
   347  }
   348  
   349  func (c *countingCommitHook) ExecuteForWorkingSets() bool {
   350  	return false
   351  }