github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/cmd/gitannex/gitannex.go (about) 1 // Package gitannex provides the "gitannex" command, which enables [git-annex] 2 // to communicate with rclone by implementing the [external special remote 3 // protocol]. The protocol is line delimited and spoken over stdin and stdout. 4 // 5 // # Milestones 6 // 7 // (Tracked in [issue #7625].) 8 // 9 // 1. ✅ Minimal support for the [external special remote protocol]. Tested on 10 // "local" and "drive" backends. 11 // 2. Add support for the ASYNC protocol extension. This may improve performance. 12 // 3. Support the [simple export interface]. This will enable `git-annex 13 // export` functionality. 14 // 4. Once the draft is finalized, support import/export interface. 15 // 16 // [git-annex]: https://git-annex.branchable.com/ 17 // [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/ 18 // [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/ 19 // [issue #7625]: https://github.com/rclone/rclone/issues/7625 20 package gitannex 21 22 import ( 23 "bufio" 24 "context" 25 _ "embed" 26 "errors" 27 "fmt" 28 "io" 29 "os" 30 "path/filepath" 31 "strings" 32 33 "github.com/rclone/rclone/cmd" 34 "github.com/rclone/rclone/fs" 35 "github.com/rclone/rclone/fs/cache" 36 "github.com/rclone/rclone/fs/operations" 37 "github.com/spf13/cobra" 38 ) 39 40 const subcommandName string = "gitannex" 41 const uniqueCommandName string = "git-annex-remote-rclone-builtin" 42 43 //go:embed gitannex.md 44 var gitannexHelp string 45 46 func init() { 47 os.Args = maybeTransformArgs(os.Args) 48 cmd.Root.AddCommand(command) 49 } 50 51 // maybeTransformArgs returns a modified version of `args` with the "gitannex" 52 // subcommand inserted when `args` indicates that the program was executed as 53 // "git-annex-remote-rclone-builtin". One way this can happen is when rclone is 54 // invoked via symlink. Otherwise, returns `args`. 55 func maybeTransformArgs(args []string) []string { 56 if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName { 57 return args 58 } 59 newArgs := make([]string, 0, len(args)+1) 60 newArgs = append(newArgs, args[0]) 61 newArgs = append(newArgs, subcommandName) 62 newArgs = append(newArgs, args[1:]...) 63 return newArgs 64 } 65 66 // messageParser helps parse messages we receive from git-annex into a sequence 67 // of parameters. Messages are not quite trivial to parse because they are 68 // separated by spaces, but the final parameter may itself contain spaces. 69 // 70 // This abstraction is necessary because simply splitting on space doesn't cut 71 // it. Also, we cannot know how many parameters to parse until we've parsed the 72 // first parameter. 73 type messageParser struct { 74 line string 75 } 76 77 // nextSpaceDelimitedParameter consumes the next space-delimited parameter. 78 func (m *messageParser) nextSpaceDelimitedParameter() (string, error) { 79 m.line = strings.TrimRight(m.line, "\r\n") 80 if len(m.line) == 0 { 81 return "", errors.New("nothing remains to parse") 82 } 83 84 before, after, found := strings.Cut(m.line, " ") 85 if found { 86 if len(before) == 0 { 87 return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line) 88 } 89 m.line = after 90 return before, nil 91 } 92 93 remaining := m.line 94 m.line = "" 95 return remaining, nil 96 } 97 98 // finalParameter consumes the final parameter, which may contain spaces. 99 func (m *messageParser) finalParameter() string { 100 m.line = strings.TrimRight(m.line, "\r\n") 101 if len(m.line) == 0 { 102 return "" 103 } 104 105 param := m.line 106 m.line = "" 107 return param 108 } 109 110 // configDefinition describes a configuration value required by this command. We 111 // use "GETCONFIG" messages to query git-annex for these values at runtime. 112 type configDefinition struct { 113 name string 114 description string 115 destination *string 116 defaultValue *string 117 } 118 119 // server contains this command's current state. 120 type server struct { 121 reader *bufio.Reader 122 writer io.Writer 123 124 // When true, the server prints a transcript of messages sent and received 125 // to stderr. 126 verbose bool 127 128 extensionInfo bool 129 extensionAsync bool 130 extensionGetGitRemoteName bool 131 extensionUnavailableResponse bool 132 133 configsDone bool 134 configPrefix string 135 configRcloneRemoteName string 136 configRcloneLayout string 137 } 138 139 func (s *server) sendMsg(msg string) { 140 msg = msg + "\n" 141 if _, err := io.WriteString(s.writer, msg); err != nil { 142 panic(err) 143 } 144 if s.verbose { 145 _, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg)) 146 if err != nil { 147 panic(fmt.Errorf("failed to write verbose message to stderr: %w", err)) 148 } 149 } 150 } 151 152 func (s *server) getMsg() (*messageParser, error) { 153 msg, err := s.reader.ReadString('\n') 154 if err != nil { 155 if len(msg) == 0 { 156 // Git-annex closes stdin when it is done with us, so failing to 157 // read a new line is not an error. 158 return nil, nil 159 } 160 return nil, fmt.Errorf("expected message to end with newline: %q", msg) 161 } 162 if s.verbose { 163 _, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg)) 164 if err != nil { 165 return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err) 166 } 167 } 168 return &messageParser{msg}, nil 169 } 170 171 func (s *server) run() error { 172 // The remote sends the first message. 173 s.sendMsg("VERSION 1") 174 175 for { 176 message, err := s.getMsg() 177 if err != nil { 178 return fmt.Errorf("error receiving message: %w", err) 179 } 180 181 if message == nil { 182 break 183 } 184 185 command, err := message.nextSpaceDelimitedParameter() 186 if err != nil { 187 return fmt.Errorf("failed to parse command") 188 } 189 190 switch command { 191 // 192 // Git-annex requires that these requests are supported. 193 // 194 case "INITREMOTE": 195 err = s.handleInitRemote() 196 case "PREPARE": 197 err = s.handlePrepare() 198 case "EXPORTSUPPORTED": 199 // Indicate that we do not support exports. 200 s.sendMsg("EXPORTSUPPORTED-FAILURE") 201 case "TRANSFER": 202 err = s.handleTransfer(message) 203 case "CHECKPRESENT": 204 err = s.handleCheckPresent(message) 205 case "REMOVE": 206 err = s.handleRemove(message) 207 case "ERROR": 208 errorMessage := message.finalParameter() 209 err = fmt.Errorf("received error message from git-annex: %s", errorMessage) 210 211 // 212 // These requests are optional. 213 // 214 case "EXTENSIONS": 215 // Git-annex just told us which protocol extensions it supports. 216 // Respond with the list of extensions that we want to use (none). 217 err = s.handleExtensions(message) 218 case "LISTCONFIGS": 219 s.handleListConfigs() 220 case "GETCOST": 221 // Git-annex wants to know the "cost" of using this remote. It 222 // probably depends on the backend we will be using, but let's just 223 // consider this an "expensive remote" per git-annex's 224 // Config/Cost.hs. 225 s.sendMsg("COST 200") 226 case "GETAVAILABILITY": 227 // Indicate that this is a cloud service. 228 s.sendMsg("AVAILABILITY GLOBAL") 229 case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO": 230 s.sendMsg("UNSUPPORTED-REQUEST") 231 default: 232 err = fmt.Errorf("received unexpected message from git-annex: %s", message.line) 233 } 234 if err != nil { 235 return err 236 } 237 } 238 239 return nil 240 } 241 242 // Idempotently handle an incoming INITREMOTE message. This should perform 243 // one-time setup operations, but we may receive the command again, e.g. when 244 // this git-annex remote is initialized in a different repository. 245 func (s *server) handleInitRemote() error { 246 if err := s.queryConfigs(); err != nil { 247 return fmt.Errorf("failed to get configs: %w", err) 248 } 249 250 remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName)) 251 if err != nil { 252 s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote") 253 return fmt.Errorf("failed to open root directory of rclone remote: %w", err) 254 } 255 256 if !remoteRootFs.Features().CanHaveEmptyDirectories { 257 s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories") 258 return fmt.Errorf("rclone remote does not support empty directories") 259 } 260 261 if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil { 262 s.sendMsg("INITREMOTE-FAILURE failed to mkdir") 263 return fmt.Errorf("failed to mkdir: %w", err) 264 } 265 266 s.sendMsg("INITREMOTE-SUCCESS") 267 return nil 268 } 269 270 // Get a list of configs with pointers to fields of `s`. 271 func (s *server) getRequiredConfigs() []configDefinition { 272 defaultRclonePrefix := "git-annex-rclone" 273 defaultRcloneLayout := "nodir" 274 275 return []configDefinition{ 276 { 277 "rcloneremotename", 278 "Name of the rclone remote to use. " + 279 "Must match a remote known to rclone. " + 280 "(Note that rclone remotes are a distinct concept from git-annex remotes.)", 281 &s.configRcloneRemoteName, 282 nil, 283 }, 284 { 285 "rcloneprefix", 286 "Directory where rclone will write git-annex content. " + 287 fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) + 288 "This directory will be created on init if it does not exist.", 289 &s.configPrefix, 290 &defaultRclonePrefix, 291 }, 292 { 293 "rclonelayout", 294 "Defines where, within the rcloneprefix directory, rclone will write git-annex content. " + 295 fmt.Sprintf("Must be one of %v. ", allLayoutModes()) + 296 fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout), 297 &s.configRcloneLayout, 298 &defaultRcloneLayout, 299 }, 300 } 301 } 302 303 // Query git-annex for config values. 304 func (s *server) queryConfigs() error { 305 if s.configsDone { 306 return nil 307 } 308 309 // Send a "GETCONFIG" message for each required config and parse git-annex's 310 // "VALUE" response. 311 for _, config := range s.getRequiredConfigs() { 312 s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name)) 313 314 message, err := s.getMsg() 315 if err != nil { 316 return err 317 } 318 319 valueKeyword, err := message.nextSpaceDelimitedParameter() 320 if err != nil || valueKeyword != "VALUE" { 321 return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line) 322 } 323 324 value := message.finalParameter() 325 if value == "" && config.defaultValue == nil { 326 return fmt.Errorf("config value of %q must not be empty", config.name) 327 } 328 if value == "" { 329 *config.destination = *config.defaultValue 330 continue 331 } 332 *config.destination = value 333 } 334 335 s.configsDone = true 336 return nil 337 } 338 339 func (s *server) handlePrepare() error { 340 if err := s.queryConfigs(); err != nil { 341 s.sendMsg("PREPARE-FAILURE Error getting configs") 342 return fmt.Errorf("error getting configs: %w", err) 343 } 344 s.sendMsg("PREPARE-SUCCESS") 345 return nil 346 } 347 348 // Git-annex is asking us to return the list of settings that we use. Keep this 349 // in sync with `handlePrepare()`. 350 func (s *server) handleListConfigs() { 351 for _, config := range s.getRequiredConfigs() { 352 s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description)) 353 } 354 s.sendMsg("CONFIGEND") 355 } 356 357 func (s *server) handleTransfer(message *messageParser) error { 358 argMode, err := message.nextSpaceDelimitedParameter() 359 if err != nil { 360 s.sendMsg("TRANSFER-FAILURE failed to parse direction") 361 return fmt.Errorf("malformed arguments for TRANSFER: %w", err) 362 } 363 argKey, err := message.nextSpaceDelimitedParameter() 364 if err != nil { 365 s.sendMsg("TRANSFER-FAILURE failed to parse key") 366 return fmt.Errorf("malformed arguments for TRANSFER: %w", err) 367 } 368 argFile := message.finalParameter() 369 if argFile == "" { 370 s.sendMsg("TRANSFER-FAILURE failed to parse file path") 371 return errors.New("failed to parse file path") 372 } 373 374 if err := s.queryConfigs(); err != nil { 375 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey)) 376 return fmt.Errorf("error getting configs: %w", err) 377 } 378 379 layout := parseLayoutMode(s.configRcloneLayout) 380 if layout == layoutModeUnknown { 381 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey)) 382 return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) 383 } 384 385 remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) 386 if err != nil { 387 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey)) 388 return fmt.Errorf("error building fs string: %w", err) 389 } 390 391 remoteFs, err := cache.Get(context.TODO(), remoteFsString) 392 if err != nil { 393 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey)) 394 return err 395 } 396 397 localDir := filepath.Dir(argFile) 398 localFs, err := cache.Get(context.TODO(), localDir) 399 if err != nil { 400 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey)) 401 return fmt.Errorf("failed to get local fs: %w", err) 402 } 403 404 remoteFileName := argKey 405 localFileName := filepath.Base(argFile) 406 407 switch argMode { 408 case "STORE": 409 err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName) 410 if err != nil { 411 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) 412 return err 413 } 414 415 case "RETRIEVE": 416 err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName) 417 // It is non-fatal when retrieval fails because the file is missing on 418 // the remote. 419 if err == fs.ErrorObjectNotFound { 420 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey)) 421 return nil 422 } 423 if err != nil { 424 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) 425 return err 426 } 427 428 default: 429 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey)) 430 return fmt.Errorf("received malformed TRANSFER mode: %v", argMode) 431 } 432 433 s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey)) 434 return nil 435 } 436 437 func (s *server) handleCheckPresent(message *messageParser) error { 438 argKey := message.finalParameter() 439 if argKey == "" { 440 return errors.New("failed to parse response for CHECKPRESENT") 441 } 442 443 if err := s.queryConfigs(); err != nil { 444 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey)) 445 return fmt.Errorf("error getting configs: %s", err) 446 } 447 448 layout := parseLayoutMode(s.configRcloneLayout) 449 if layout == layoutModeUnknown { 450 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) 451 return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) 452 } 453 454 remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) 455 if err != nil { 456 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) 457 return fmt.Errorf("error building fs string: %w", err) 458 } 459 460 remoteFs, err := cache.Get(context.TODO(), remoteFsString) 461 if err != nil { 462 s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey)) 463 return err 464 } 465 466 _, err = remoteFs.NewObject(context.TODO(), argKey) 467 if err == fs.ErrorObjectNotFound { 468 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) 469 return nil 470 } 471 if err != nil { 472 s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey)) 473 return err 474 } 475 476 s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey)) 477 return nil 478 } 479 480 func (s *server) queryDirhash(msg string) (string, error) { 481 s.sendMsg(msg) 482 parser, err := s.getMsg() 483 if err != nil { 484 return "", err 485 } 486 keyword, err := parser.nextSpaceDelimitedParameter() 487 if err != nil { 488 return "", err 489 } 490 if keyword != "VALUE" { 491 return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword) 492 } 493 dirhash, err := parser.nextSpaceDelimitedParameter() 494 if err != nil { 495 return "", fmt.Errorf("failed to parse dirhash: %w", err) 496 } 497 return dirhash, nil 498 } 499 500 func (s *server) handleRemove(message *messageParser) error { 501 argKey := message.finalParameter() 502 if argKey == "" { 503 return errors.New("failed to parse key for REMOVE") 504 } 505 506 layout := parseLayoutMode(s.configRcloneLayout) 507 if layout == layoutModeUnknown { 508 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) 509 return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) 510 } 511 512 remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) 513 if err != nil { 514 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) 515 return fmt.Errorf("error building fs string: %w", err) 516 } 517 518 remoteFs, err := cache.Get(context.TODO(), remoteFsString) 519 if err != nil { 520 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) 521 return fmt.Errorf("error getting remote fs: %w", err) 522 } 523 524 fileObj, err := remoteFs.NewObject(context.TODO(), argKey) 525 // It is non-fatal when removal fails because the file is missing on the 526 // remote. 527 if errors.Is(err, fs.ErrorObjectNotFound) { 528 s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) 529 return nil 530 } 531 if err != nil { 532 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err)) 533 return fmt.Errorf("error getting new fs object: %w", err) 534 } 535 if err := operations.DeleteFile(context.TODO(), fileObj); err != nil { 536 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey)) 537 return fmt.Errorf("error deleting file: %q", argKey) 538 } 539 s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) 540 return nil 541 } 542 543 func (s *server) handleExtensions(message *messageParser) error { 544 for { 545 extension, err := message.nextSpaceDelimitedParameter() 546 if err != nil { 547 break 548 } 549 switch extension { 550 case "INFO": 551 s.extensionInfo = true 552 case "ASYNC": 553 s.extensionAsync = true 554 case "GETGITREMOTENAME": 555 s.extensionGetGitRemoteName = true 556 case "UNAVAILABLERESPONSE": 557 s.extensionUnavailableResponse = true 558 } 559 } 560 s.sendMsg("EXTENSIONS") 561 return nil 562 } 563 564 var command = &cobra.Command{ 565 Aliases: []string{uniqueCommandName}, 566 Use: subcommandName, 567 Short: "Speaks with git-annex over stdin/stdout.", 568 Long: gitannexHelp, 569 Annotations: map[string]string{ 570 "versionIntroduced": "v1.67.0", 571 }, 572 Run: func(command *cobra.Command, args []string) { 573 cmd.CheckArgs(0, 0, command, args) 574 575 s := server{ 576 reader: bufio.NewReader(os.Stdin), 577 writer: os.Stdout, 578 } 579 err := s.run() 580 if err != nil { 581 s.sendMsg(fmt.Sprintf("ERROR %s", err.Error())) 582 panic(err) 583 } 584 }, 585 }