github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/cmd/gitannex/e2e_test.go (about) 1 package gitannex 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "runtime" 12 "strings" 13 "testing" 14 15 "github.com/stretchr/testify/require" 16 17 "github.com/rclone/rclone/fs" 18 "github.com/rclone/rclone/lib/buildinfo" 19 ) 20 21 // checkRcloneBinaryVersion runs whichever rclone is on the PATH and checks 22 // whether it reports a version that matches the test's expectations. Returns 23 // nil when the version is the expected version, otherwise returns an error. 24 func checkRcloneBinaryVersion(t *testing.T) error { 25 // versionInfo is a subset of information produced by "core/version". 26 type versionInfo struct { 27 Version string 28 IsGit bool 29 GoTags string 30 } 31 32 cmd := exec.Command("rclone", "rc", "--loopback", "core/version") 33 stdout, err := cmd.Output() 34 if err != nil { 35 return fmt.Errorf("failed to get rclone version: %w", err) 36 } 37 38 var parsed versionInfo 39 if err := json.Unmarshal(stdout, &parsed); err != nil { 40 return fmt.Errorf("failed to parse rclone version: %w", err) 41 } 42 if parsed.Version != fs.Version { 43 return fmt.Errorf("expected version %q, but got %q", fs.Version, parsed.Version) 44 } 45 if parsed.IsGit != strings.HasSuffix(fs.Version, "-DEV") { 46 return errors.New("expected rclone to be a dev build") 47 } 48 _, tagString := buildinfo.GetLinkingAndTags() 49 if parsed.GoTags != tagString { 50 // TODO: Skip the test when tags do not match. 51 t.Logf("expected tag string %q, but got %q. Not skipping!", tagString, parsed.GoTags) 52 } 53 return nil 54 } 55 56 // countFilesRecursively returns the number of files nested underneath `dir`. It 57 // counts files only and excludes directories. 58 func countFilesRecursively(t *testing.T, dir string) int { 59 remoteFiles, err := os.ReadDir(dir) 60 require.NoError(t, err) 61 62 var count int 63 for _, f := range remoteFiles { 64 if f.IsDir() { 65 subdir := filepath.Join(dir, f.Name()) 66 count += countFilesRecursively(t, subdir) 67 } else { 68 count++ 69 } 70 } 71 return count 72 } 73 74 func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool { 75 remoteFiles, err := os.ReadDir(dir) 76 require.NoError(t, err) 77 78 for _, f := range remoteFiles { 79 fPath := filepath.Join(dir, f.Name()) 80 if f.IsDir() { 81 if findFileWithContents(t, fPath, wantContents) { 82 return true 83 } 84 } else { 85 contents, err := os.ReadFile(fPath) 86 require.NoError(t, err) 87 if bytes.Equal(contents, wantContents) { 88 return true 89 } 90 } 91 } 92 return false 93 } 94 95 type e2eTestingContext struct { 96 tempDir string 97 binDir string 98 homeDir string 99 configDir string 100 rcloneConfigDir string 101 ephemeralRepoDir string 102 } 103 104 // makeE2eTestingContext sets up a new e2eTestingContext rooted under 105 // `t.TempDir()`. It creates the skeleton directory structure shown below in the 106 // temp directory without creating any files. 107 // 108 // . 109 // |-- bin 110 // | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY} 111 // |-- ephemeralRepo 112 // `-- user 113 // `-- .config 114 // `-- rclone 115 // `-- rclone.conf 116 func makeE2eTestingContext(t *testing.T) e2eTestingContext { 117 tempDir := t.TempDir() 118 119 binDir := filepath.Join(tempDir, "bin") 120 homeDir := filepath.Join(tempDir, "user") 121 configDir := filepath.Join(homeDir, ".config") 122 rcloneConfigDir := filepath.Join(configDir, "rclone") 123 ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo") 124 125 for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} { 126 require.NoError(t, os.Mkdir(dir, 0700)) 127 } 128 129 return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} 130 } 131 132 // Install the symlink that enables git-annex to invoke "rclone gitannex" 133 // without explicitly specifying the subcommand. 134 func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) { 135 rcloneBinaryPath, err := exec.LookPath("rclone") 136 require.NoError(t, err) 137 require.NoError(t, os.Symlink( 138 rcloneBinaryPath, 139 filepath.Join(e.binDir, "git-annex-remote-rclone-builtin"))) 140 } 141 142 // Install a rclone.conf file in an appropriate location in the fake home 143 // directory. The config defines an rclone remote named "MyRcloneRemote" using 144 // the local backend. 145 func (e *e2eTestingContext) installRcloneConfig(t *testing.T) { 146 // Install the rclone.conf file that defines the remote. 147 rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf") 148 rcloneConfigContents := "[MyRcloneRemote]\ntype = local" 149 require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600)) 150 } 151 152 // runInRepo runs the given command from within the ephemeral repo directory. To 153 // prevent accidental changes in the real home directory, it sets the HOME 154 // variable to a subdirectory of the temp directory. It also ensures that the 155 // git-annex-remote-rclone-builtin symlink will be found by extending the PATH. 156 func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) { 157 fmt.Printf("+ %s %v\n", command, args) 158 cmd := exec.Command(command, args...) 159 cmd.Dir = e.ephemeralRepoDir 160 cmd.Env = []string{ 161 "HOME=" + e.homeDir, 162 "PATH=" + os.Getenv("PATH") + ":" + e.binDir, 163 } 164 cmd.Stdout = os.Stdout 165 cmd.Stderr = os.Stderr 166 require.NoError(t, cmd.Run()) 167 } 168 169 // createGitRepo creates an empty git repository in the ephemeral repo 170 // directory. It makes "global" config changes that are ultimately scoped to the 171 // calling test thanks to runInRepo() overriding the HOME environment variable. 172 func (e *e2eTestingContext) createGitRepo(t *testing.T) { 173 e.runInRepo(t, "git", "annex", "version") 174 e.runInRepo(t, "git", "config", "--global", "user.name", "User Name") 175 e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com") 176 e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main") 177 e.runInRepo(t, "git", "init") 178 e.runInRepo(t, "git", "annex", "init") 179 } 180 181 func skipE2eTestIfNecessary(t *testing.T) { 182 if testing.Short() { 183 t.Skip("Skipping due to short mode.") 184 } 185 186 // TODO: Support e2e tests on Windows. Need to evaluate the semantics of the 187 // HOME and PATH environment variables. 188 switch runtime.GOOS { 189 case "darwin", 190 "freebsd", 191 "linux", 192 "netbsd", 193 "openbsd", 194 "plan9", 195 "solaris": 196 default: 197 t.Skipf("GOOS %q is not supported.", runtime.GOOS) 198 } 199 200 if err := checkRcloneBinaryVersion(t); err != nil { 201 t.Skipf("Skipping due to rclone version: %s", err) 202 } 203 204 if _, err := exec.LookPath("git-annex"); err != nil { 205 t.Skipf("Skipping because git-annex was not found: %s", err) 206 } 207 } 208 209 // This end-to-end test runs `git annex testremote` in a temporary git repo. 210 // This test will be skipped unless the `rclone` binary on PATH reports the 211 // expected version. 212 // 213 // When run on CI, an rclone binary built from HEAD will be on the PATH. When 214 // running locally, you will likely need to ensure the current binary is on the 215 // PATH like so: 216 // 217 // go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/... 218 // 219 // In the future, this test will probably be extended to test a number of 220 // parameters like repo layouts, and runtime may suffer from a combinatorial 221 // explosion. 222 func TestEndToEnd(t *testing.T) { 223 skipE2eTestIfNecessary(t) 224 225 for _, mode := range allLayoutModes() { 226 mode := mode 227 t.Run(string(mode), func(t *testing.T) { 228 t.Parallel() 229 230 testingContext := makeE2eTestingContext(t) 231 testingContext.installRcloneGitannexSymlink(t) 232 testingContext.installRcloneConfig(t) 233 testingContext.createGitRepo(t) 234 235 testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote", 236 "type=external", "externaltype=rclone-builtin", "encryption=none", 237 "rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir, 238 "rclonelayout="+string(mode)) 239 240 testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote") 241 }) 242 } 243 } 244 245 // For each layout mode, migrate a single remote from git-annex-remote-rclone to 246 // git-annex-remote-rclone-builtin and run `git annex testremote`. 247 func TestEndToEndMigration(t *testing.T) { 248 skipE2eTestIfNecessary(t) 249 250 if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil { 251 t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err) 252 } 253 254 for _, mode := range allLayoutModes() { 255 mode := mode 256 t.Run(string(mode), func(t *testing.T) { 257 t.Parallel() 258 259 tc := makeE2eTestingContext(t) 260 tc.installRcloneGitannexSymlink(t) 261 tc.installRcloneConfig(t) 262 tc.createGitRepo(t) 263 264 remoteStorage := filepath.Join(tc.tempDir, "remotePrefix") 265 require.NoError(t, os.Mkdir(remoteStorage, 0777)) 266 267 tc.runInRepo(t, 268 "git", "annex", "initremote", "MigratedRemote", 269 "type=external", "externaltype=rclone", "encryption=none", 270 "target=MyRcloneRemote", 271 "rclone_layout="+string(mode), 272 "prefix="+remoteStorage, 273 ) 274 275 fooFileContents := []byte{1, 2, 3, 4} 276 fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo") 277 require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700)) 278 tc.runInRepo(t, "git", "annex", "add", "foo") 279 tc.runInRepo(t, "git", "commit", "-m", "Add foo file") 280 // Git-annex objects are not writable, which prevents `testing` from 281 // cleaning up the temp directory. We can work around this by 282 // explicitly dropping any files we add to the annex. 283 t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") }) 284 285 tc.runInRepo(t, "git", "annex", "copy", "--to=MigratedRemote", "foo") 286 tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo") 287 288 tc.runInRepo(t, 289 "git", "annex", "enableremote", "MigratedRemote", 290 "externaltype=rclone-builtin", 291 "rcloneremotename=MyRcloneRemote", 292 "rclonelayout="+string(mode), 293 "rcloneprefix="+remoteStorage, 294 ) 295 296 tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo") 297 298 tc.runInRepo(t, "git", "annex", "testremote", "MigratedRemote") 299 }) 300 } 301 } 302 303 // For each layout mode, create two git-annex remotes with externaltype=rclone 304 // and externaltype=rclone-builtin respectively. Test that files copied to one 305 // remote are present on the other. Similarly, test that files deleted from one 306 // are removed on the other. 307 func TestEndToEndRepoLayoutCompat(t *testing.T) { 308 skipE2eTestIfNecessary(t) 309 310 if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil { 311 t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err) 312 } 313 314 for _, mode := range allLayoutModes() { 315 mode := mode 316 t.Run(string(mode), func(t *testing.T) { 317 t.Parallel() 318 319 tc := makeE2eTestingContext(t) 320 tc.installRcloneGitannexSymlink(t) 321 tc.installRcloneConfig(t) 322 tc.createGitRepo(t) 323 324 remoteStorage := filepath.Join(tc.tempDir, "remotePrefix") 325 require.NoError(t, os.Mkdir(remoteStorage, 0777)) 326 327 tc.runInRepo(t, 328 "git", "annex", "initremote", "Control", 329 "type=external", "externaltype=rclone", "encryption=none", 330 "target=MyRcloneRemote", 331 "rclone_layout="+string(mode), 332 "prefix="+remoteStorage) 333 334 tc.runInRepo(t, 335 "git", "annex", "initremote", "Experiment", 336 "type=external", "externaltype=rclone-builtin", "encryption=none", 337 "rcloneremotename=MyRcloneRemote", 338 "rclonelayout="+string(mode), 339 "rcloneprefix="+remoteStorage) 340 341 fooFileContents := []byte{1, 2, 3, 4} 342 fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo") 343 require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700)) 344 tc.runInRepo(t, "git", "annex", "add", "foo") 345 tc.runInRepo(t, "git", "commit", "-m", "Add foo file") 346 // Git-annex objects are not writable, which prevents `testing` from 347 // cleaning up the temp directory. We can work around this by 348 // explicitly dropping any files we add to the annex. 349 t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") }) 350 351 require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) 352 require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) 353 354 // Copy the file to Control and verify it's present on Experiment. 355 356 tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo") 357 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 358 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 359 360 tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo") 361 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 362 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 363 364 // Drop the file locally and verify we can copy it back from Experiment. 365 366 tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") 367 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 368 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 369 370 tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo") 371 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 372 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 373 374 // Drop the file from Experiment, copy it back to Experiment, and 375 // verify it's still present on Control. 376 377 tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo") 378 require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) 379 require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) 380 381 tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo") 382 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 383 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 384 385 tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo") 386 require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) 387 require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) 388 389 // Drop the file from Control. 390 391 tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo") 392 require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) 393 require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) 394 }) 395 } 396 }