github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/libraries/doltcore/doltdb/commit_hooks_test.go (about) 1 // Copyright 2021 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package doltdb 16 17 import ( 18 "bytes" 19 "context" 20 "errors" 21 "io" 22 "path/filepath" 23 "testing" 24 "time" 25 26 "github.com/dolthub/go-mysql-server/sql" 27 "github.com/stretchr/testify/assert" 28 "github.com/stretchr/testify/require" 29 "go.uber.org/zap/buffer" 30 31 "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" 32 "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" 33 "github.com/dolthub/dolt/go/libraries/doltcore/ref" 34 "github.com/dolthub/dolt/go/libraries/utils/filesys" 35 "github.com/dolthub/dolt/go/libraries/utils/test" 36 "github.com/dolthub/dolt/go/store/datas" 37 "github.com/dolthub/dolt/go/store/types" 38 ) 39 40 const defaultBranch = "main" 41 42 func TestPushOnWriteHook(t *testing.T) { 43 ctx := context.Background() 44 45 // destination repo 46 testDir, err := test.ChangeToTestDir("TestReplicationDest") 47 48 if err != nil { 49 panic("Couldn't change the working directory to the test directory.") 50 } 51 52 committerName := "Bill Billerson" 53 committerEmail := "bigbillieb@fake.horse" 54 55 tmpDir := filepath.Join(testDir, dbfactory.DoltDataDir) 56 err = filesys.LocalFS.MkDirs(tmpDir) 57 58 if err != nil { 59 t.Fatal("Failed to create noms directory") 60 } 61 62 destDB, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS) 63 64 // source repo 65 testDir, err = test.ChangeToTestDir("TestReplicationSource") 66 67 if err != nil { 68 panic("Couldn't change the working directory to the test directory.") 69 } 70 71 tmpDir = filepath.Join(testDir, dbfactory.DoltDataDir) 72 err = filesys.LocalFS.MkDirs(tmpDir) 73 74 if err != nil { 75 t.Fatal("Failed to create noms directory") 76 } 77 78 ddb, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS) 79 err = ddb.WriteEmptyRepo(context.Background(), "main", committerName, committerEmail) 80 81 if err != nil { 82 t.Fatal("Unexpected error creating empty repo", err) 83 } 84 85 // prepare a commit in the source repo 86 cs, _ := NewCommitSpec("main") 87 optCmt, err := ddb.Resolve(context.Background(), cs, nil) 88 if err != nil { 89 t.Fatal("Couldn't find commit") 90 } 91 commit, ok := optCmt.ToCommit() 92 assert.True(t, ok) 93 94 meta, err := commit.GetCommitMeta(context.Background()) 95 assert.NoError(t, err) 96 97 if meta.Name != committerName || meta.Email != committerEmail { 98 t.Error("Unexpected metadata") 99 } 100 101 root, err := commit.GetRootValue(context.Background()) 102 103 assert.NoError(t, err) 104 105 names, err := root.GetTableNames(context.Background(), DefaultSchemaName) 106 assert.NoError(t, err) 107 if len(names) != 0 { 108 t.Fatal("There should be no tables in empty db") 109 } 110 111 tSchema := createTestSchema(t) 112 rowData := createTestRowData(t, ddb.vrw, ddb.ns, tSchema) 113 tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData) 114 115 if err != nil { 116 t.Fatal("Failed to create test table with data") 117 } 118 119 root, err = root.PutTable(context.Background(), TableName{Name: "test"}, tbl) 120 assert.NoError(t, err) 121 122 r, valHash, err := ddb.WriteRootValue(context.Background(), root) 123 assert.NoError(t, err) 124 root = r 125 126 meta, err = datas.NewCommitMeta(committerName, committerEmail, "Sample data") 127 if err != nil { 128 t.Error("Failed to commit") 129 } 130 131 // setup hook 132 hook := NewPushOnWriteHook(destDB, tmpDir) 133 ddb.SetCommitHooks(ctx, []CommitHook{hook}) 134 135 t.Run("replicate to remote", func(t *testing.T) { 136 srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta) 137 require.NoError(t, err) 138 139 ds, err := ddb.db.GetDataset(ctx, "refs/heads/main") 140 require.NoError(t, err) 141 142 _, err = hook.Execute(ctx, ds, ddb.db) 143 require.NoError(t, err) 144 145 cs, _ = NewCommitSpec(defaultBranch) 146 optCmt, err := destDB.Resolve(context.Background(), cs, nil) 147 require.NoError(t, err) 148 destCommit, ok := optCmt.ToCommit() 149 require.True(t, ok) 150 151 srcHash, _ := srcCommit.HashOf() 152 destHash, _ := destCommit.HashOf() 153 assert.Equal(t, srcHash, destHash) 154 }) 155 156 t.Run("replicate handle error logs to writer", func(t *testing.T) { 157 var buffer = &bytes.Buffer{} 158 err = hook.SetLogger(ctx, buffer) 159 assert.NoError(t, err) 160 161 msg := "prince charles is a vampire" 162 hook.HandleError(ctx, errors.New(msg)) 163 164 assert.Contains(t, buffer.String(), msg) 165 }) 166 } 167 168 func TestLogHook(t *testing.T) { 169 msg := []byte("hello") 170 var err error 171 t.Run("new log hook", func(t *testing.T) { 172 ctx := context.Background() 173 hook := NewLogHook(msg) 174 var buffer = &bytes.Buffer{} 175 err = hook.SetLogger(ctx, buffer) 176 assert.NoError(t, err) 177 hook.Execute(ctx, datas.Dataset{}, nil) 178 assert.Equal(t, buffer.Bytes(), msg) 179 }) 180 } 181 182 func TestAsyncPushOnWrite(t *testing.T) { 183 ctx := context.Background() 184 185 // destination repo 186 testDir, err := test.ChangeToTestDir("TestReplicationDest") 187 188 if err != nil { 189 panic("Couldn't change the working directory to the test directory.") 190 } 191 192 committerName := "Bill Billerson" 193 committerEmail := "bigbillieb@fake.horse" 194 195 tmpDir := filepath.Join(testDir, dbfactory.DoltDataDir) 196 err = filesys.LocalFS.MkDirs(tmpDir) 197 198 if err != nil { 199 t.Fatal("Failed to create noms directory") 200 } 201 202 destDB, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS) 203 204 // source repo 205 testDir, err = test.ChangeToTestDir("TestReplicationSource") 206 207 if err != nil { 208 panic("Couldn't change the working directory to the test directory.") 209 } 210 211 tmpDir = filepath.Join(testDir, dbfactory.DoltDataDir) 212 err = filesys.LocalFS.MkDirs(tmpDir) 213 214 if err != nil { 215 t.Fatal("Failed to create noms directory") 216 } 217 218 ddb, _ := LoadDoltDB(context.Background(), types.Format_Default, LocalDirDoltDB, filesys.LocalFS) 219 err = ddb.WriteEmptyRepo(context.Background(), "main", committerName, committerEmail) 220 221 if err != nil { 222 t.Fatal("Unexpected error creating empty repo", err) 223 } 224 225 t.Run("replicate to remote", func(t *testing.T) { 226 bThreads := sql.NewBackgroundThreads() 227 defer bThreads.Shutdown() 228 hook, err := NewAsyncPushOnWriteHook(bThreads, destDB, tmpDir, &buffer.Buffer{}) 229 if err != nil { 230 t.Fatal("Unexpected error creating push hook", err) 231 } 232 233 for i := 0; i < 200; i++ { 234 cs, _ := NewCommitSpec("main") 235 optCmt, err := ddb.Resolve(context.Background(), cs, nil) 236 if err != nil { 237 t.Fatal("Couldn't find commit") 238 } 239 commit, ok := optCmt.ToCommit() 240 assert.True(t, ok) 241 242 meta, err := commit.GetCommitMeta(context.Background()) 243 assert.NoError(t, err) 244 245 if meta.Name != committerName || meta.Email != committerEmail { 246 t.Error("Unexpected metadata") 247 } 248 249 root, err := commit.GetRootValue(context.Background()) 250 251 assert.NoError(t, err) 252 253 tSchema := createTestSchema(t) 254 rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema) 255 require.NoError(t, err) 256 tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData) 257 require.NoError(t, err) 258 259 if err != nil { 260 t.Fatal("Failed to create test table with data") 261 } 262 263 root, err = root.PutTable(context.Background(), TableName{Name: "test"}, tbl) 264 assert.NoError(t, err) 265 266 r, valHash, err := ddb.WriteRootValue(context.Background(), root) 267 assert.NoError(t, err) 268 root = r 269 270 meta, err = datas.NewCommitMeta(committerName, committerEmail, "Sample data") 271 if err != nil { 272 t.Error("Failed to create CommitMeta") 273 } 274 275 _, err = ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta) 276 require.NoError(t, err) 277 ds, err := ddb.db.GetDataset(ctx, "refs/heads/main") 278 require.NoError(t, err) 279 _, err = hook.Execute(ctx, ds, ddb.db) 280 require.NoError(t, err) 281 } 282 }) 283 284 t.Run("does not over replicate branch delete", func(t *testing.T) { 285 // We used to have a bug where a branch delete would be 286 // replicated over and over again endlessly. 287 288 // The test construction here is that we put a counting commit 289 // hook on *destDB*. Then we call the async push hook as if we 290 // need to replicate certain head updates. We call once for a 291 // branch that does exist and once for a branch which does not 292 // exist. Calling with a branch which does not exist looks the 293 // same as the call which is made after a branch delete. 294 295 counts := &countingCommitHook{make(map[string]int)} 296 destDB.SetCommitHooks(context.Background(), []CommitHook{counts}) 297 298 bThreads := sql.NewBackgroundThreads() 299 hook, err := NewAsyncPushOnWriteHook(bThreads, destDB, tmpDir, &buffer.Buffer{}) 300 require.NoError(t, err, "create push on write hook without an error") 301 302 // Pretend we replicate a HEAD which does exist. 303 ds, err := ddb.db.GetDataset(ctx, "refs/heads/main") 304 require.NoError(t, err) 305 _, err = hook.Execute(ctx, ds, ddb.db) 306 require.NoError(t, err) 307 308 // Pretend we replicate a HEAD which does not exist, i.e., a branch delete. 309 ds, err = ddb.db.GetDataset(ctx, "refs/heads/does_not_exist") 310 require.NoError(t, err) 311 _, err = hook.Execute(ctx, ds, ddb.db) 312 require.NoError(t, err) 313 314 // Wait a bit for background thread to fire, in case it is 315 // going to betray us. TODO: Structure AsyncPushOnWriteHook to 316 // be more testable, so we do not have to rely on 317 // non-determinstic goroutine scheduling and best-effort sleeps 318 // to observe the potential failure here. 319 time.Sleep(10 * time.Second) 320 321 // Shutdown thread to get final replication if necessary. 322 bThreads.Shutdown() 323 324 // If all went well, the branch delete was executed exactly once. 325 require.Equal(t, 1, counts.counts["refs/heads/does_not_exist"]) 326 }) 327 } 328 329 var _ CommitHook = (*countingCommitHook)(nil) 330 331 type countingCommitHook struct { 332 // The number of times Execute() got called for given dataset. 333 counts map[string]int 334 } 335 336 func (c *countingCommitHook) Execute(ctx context.Context, ds datas.Dataset, db datas.Database) (func(context.Context) error, error) { 337 c.counts[ds.ID()] += 1 338 return nil, nil 339 } 340 341 func (c *countingCommitHook) HandleError(ctx context.Context, err error) error { 342 return nil 343 } 344 345 func (c *countingCommitHook) SetLogger(ctx context.Context, wr io.Writer) error { 346 return nil 347 } 348 349 func (c *countingCommitHook) ExecuteForWorkingSets() bool { 350 return false 351 }