github.com/artpar/rclone@v1.67.3/cmd/gitannex/gitannex.go (about) 1 // Package gitannex provides the "gitannex" command, which enables [git-annex] 2 // to communicate with rclone by implementing the [external special remote 3 // protocol]. The protocol is line delimited and spoken over stdin and stdout. 4 // 5 // # Milestones 6 // 7 // (Tracked in [issue #7625].) 8 // 9 // 1. ✅ Minimal support for the [external special remote protocol]. Tested on 10 // "local" and "drive" backends. 11 // 2. Add support for the ASYNC protocol extension. This may improve performance. 12 // 3. Support the [simple export interface]. This will enable `git-annex 13 // export` functionality. 14 // 4. Once the draft is finalized, support import/export interface. 15 // 16 // [git-annex]: https://git-annex.branchable.com/ 17 // [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/ 18 // [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/ 19 // [issue #7625]: https://github.com/artpar/rclone/issues/7625 20 package gitannex 21 22 import ( 23 "bufio" 24 "context" 25 _ "embed" 26 "errors" 27 "fmt" 28 "io" 29 "os" 30 "path/filepath" 31 "strings" 32 33 "github.com/artpar/rclone/cmd" 34 "github.com/artpar/rclone/fs" 35 "github.com/artpar/rclone/fs/cache" 36 "github.com/artpar/rclone/fs/operations" 37 "github.com/spf13/cobra" 38 ) 39 40 const subcommandName string = "gitannex" 41 const uniqueCommandName string = "git-annex-remote-rclone-builtin" 42 43 //go:embed gitannex.md 44 var gitannexHelp string 45 46 func init() { 47 os.Args = maybeTransformArgs(os.Args) 48 cmd.Root.AddCommand(command) 49 } 50 51 // maybeTransformArgs returns a modified version of `args` with the "gitannex" 52 // subcommand inserted when `args` indicates that the program was executed as 53 // "git-annex-remote-rclone-builtin". One way this can happen is when rclone is 54 // invoked via symlink. Otherwise, returns `args`. 55 func maybeTransformArgs(args []string) []string { 56 if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName { 57 return args 58 } 59 newArgs := make([]string, 0, len(args)+1) 60 newArgs = append(newArgs, args[0]) 61 newArgs = append(newArgs, subcommandName) 62 newArgs = append(newArgs, args[1:]...) 63 return newArgs 64 } 65 66 // messageParser helps parse messages we receive from git-annex into a sequence 67 // of parameters. Messages are not quite trivial to parse because they are 68 // separated by spaces, but the final parameter may itself contain spaces. 69 // 70 // This abstraction is necessary because simply splitting on space doesn't cut 71 // it. Also, we cannot know how many parameters to parse until we've parsed the 72 // first parameter. 73 type messageParser struct { 74 line string 75 } 76 77 // nextSpaceDelimitedParameter consumes the next space-delimited parameter. 78 func (m *messageParser) nextSpaceDelimitedParameter() (string, error) { 79 m.line = strings.TrimRight(m.line, "\r\n") 80 if len(m.line) == 0 { 81 return "", errors.New("nothing remains to parse") 82 } 83 84 before, after, found := strings.Cut(m.line, " ") 85 if found { 86 if len(before) == 0 { 87 return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line) 88 } 89 m.line = after 90 return before, nil 91 } 92 93 remaining := m.line 94 m.line = "" 95 return remaining, nil 96 } 97 98 // finalParameter consumes the final parameter, which may contain spaces. 99 func (m *messageParser) finalParameter() (string, error) { 100 m.line = strings.TrimRight(m.line, "\r\n") 101 if len(m.line) == 0 { 102 return "", errors.New("nothing remains to parse") 103 } 104 105 param := m.line 106 m.line = "" 107 return param, nil 108 } 109 110 // configDefinition describes a configuration value required by this command. We 111 // use "GETCONFIG" messages to query git-annex for these values at runtime. 112 type configDefinition struct { 113 name string 114 description string 115 destination *string 116 } 117 118 // server contains this command's current state. 119 type server struct { 120 reader *bufio.Reader 121 writer io.Writer 122 123 // When true, the server prints a transcript of messages sent and received 124 // to stderr. 125 verbose bool 126 127 extensionInfo bool 128 extensionAsync bool 129 extensionGetGitRemoteName bool 130 extensionUnavailableResponse bool 131 132 configsDone bool 133 configPrefix string 134 configRcloneRemoteName string 135 } 136 137 func (s *server) sendMsg(msg string) { 138 msg = msg + "\n" 139 if _, err := io.WriteString(s.writer, msg); err != nil { 140 panic(err) 141 } 142 if s.verbose { 143 _, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg)) 144 if err != nil { 145 panic(fmt.Errorf("failed to write verbose message to stderr: %w", err)) 146 } 147 } 148 } 149 150 func (s *server) getMsg() (*messageParser, error) { 151 msg, err := s.reader.ReadString('\n') 152 if err != nil { 153 if len(msg) == 0 { 154 // Git-annex closes stdin when it is done with us, so failing to 155 // read a new line is not an error. 156 return nil, nil 157 } 158 return nil, fmt.Errorf("expected message to end with newline: %q", msg) 159 } 160 if s.verbose { 161 _, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg)) 162 if err != nil { 163 return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err) 164 } 165 } 166 return &messageParser{msg}, nil 167 } 168 169 func (s *server) run() error { 170 // The remote sends the first message. 171 s.sendMsg("VERSION 1") 172 173 for { 174 message, err := s.getMsg() 175 if err != nil { 176 return fmt.Errorf("error receiving message: %w", err) 177 } 178 179 if message == nil { 180 break 181 } 182 183 command, err := message.nextSpaceDelimitedParameter() 184 if err != nil { 185 return fmt.Errorf("failed to parse command") 186 } 187 188 switch command { 189 // 190 // Git-annex requires that these requests are supported. 191 // 192 case "INITREMOTE": 193 err = s.handleInitRemote() 194 case "PREPARE": 195 err = s.handlePrepare() 196 case "EXPORTSUPPORTED": 197 // Indicate that we do not support exports. 198 s.sendMsg("EXPORTSUPPORTED-FAILURE") 199 case "TRANSFER": 200 err = s.handleTransfer(message) 201 case "CHECKPRESENT": 202 err = s.handleCheckPresent(message) 203 case "REMOVE": 204 err = s.handleRemove(message) 205 case "ERROR": 206 errorMessage, parseErr := message.finalParameter() 207 if parseErr != nil { 208 err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr) 209 break 210 } 211 err = fmt.Errorf("received error message from git-annex: %s", errorMessage) 212 213 // 214 // These requests are optional. 215 // 216 case "EXTENSIONS": 217 // Git-annex just told us which protocol extensions it supports. 218 // Respond with the list of extensions that we want to use (none). 219 err = s.handleExtensions(message) 220 case "LISTCONFIGS": 221 s.handleListConfigs() 222 case "GETCOST": 223 // Git-annex wants to know the "cost" of using this remote. It 224 // probably depends on the backend we will be using, but let's just 225 // consider this an "expensive remote" per git-annex's 226 // Config/Cost.hs. 227 s.sendMsg("COST 200") 228 case "GETAVAILABILITY": 229 // Indicate that this is a cloud service. 230 s.sendMsg("AVAILABILITY GLOBAL") 231 case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO": 232 s.sendMsg("UNSUPPORTED-REQUEST") 233 default: 234 err = fmt.Errorf("received unexpected message from git-annex: %s", message.line) 235 } 236 if err != nil { 237 return err 238 } 239 } 240 241 return nil 242 } 243 244 // Idempotently handle an incoming INITREMOTE message. This should perform 245 // one-time setup operations, but we may receive the command again, e.g. when 246 // this git-annex remote is initialized in a different repository. 247 func (s *server) handleInitRemote() error { 248 if err := s.queryConfigs(); err != nil { 249 return fmt.Errorf("failed to get configs: %w", err) 250 } 251 252 remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName)) 253 if err != nil { 254 s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote") 255 return fmt.Errorf("failed to open root directory of rclone remote: %w", err) 256 } 257 258 if !remoteRootFs.Features().CanHaveEmptyDirectories { 259 s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories") 260 return fmt.Errorf("rclone remote does not support empty directories") 261 } 262 263 if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil { 264 s.sendMsg("INITREMOTE-FAILURE failed to mkdir") 265 return fmt.Errorf("failed to mkdir: %w", err) 266 } 267 268 s.sendMsg("INITREMOTE-SUCCESS") 269 return nil 270 } 271 272 // Get a list of configs with pointers to fields of `s`. 273 func (s *server) getRequiredConfigs() []configDefinition { 274 return []configDefinition{ 275 { 276 "rcloneremotename", 277 "Name of the rclone remote to use. " + 278 "Must match a remote known to rclone. " + 279 "(Note that rclone remotes are a distinct concept from git-annex remotes.)", 280 &s.configRcloneRemoteName, 281 }, 282 { 283 "rcloneprefix", 284 "Directory where rclone will write git-annex content. " + 285 "If not specified, defaults to \"git-annex-rclone\". " + 286 "This directory be created on init if it does not exist.", 287 &s.configPrefix, 288 }, 289 } 290 } 291 292 // Query git-annex for config values. 293 func (s *server) queryConfigs() error { 294 if s.configsDone { 295 return nil 296 } 297 298 // Send a "GETCONFIG" message for each required config and parse git-annex's 299 // "VALUE" response. 300 for _, config := range s.getRequiredConfigs() { 301 s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name)) 302 303 message, err := s.getMsg() 304 if err != nil { 305 return err 306 } 307 308 valueKeyword, err := message.nextSpaceDelimitedParameter() 309 if err != nil || valueKeyword != "VALUE" { 310 return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line) 311 } 312 313 value, err := message.finalParameter() 314 if err != nil || value == "" { 315 return fmt.Errorf("config value of %q must not be empty", config.name) 316 } 317 318 *config.destination = value 319 } 320 321 s.configsDone = true 322 return nil 323 } 324 325 func (s *server) handlePrepare() error { 326 if err := s.queryConfigs(); err != nil { 327 s.sendMsg("PREPARE-FAILURE Error getting configs") 328 return fmt.Errorf("error getting configs: %w", err) 329 } 330 s.sendMsg("PREPARE-SUCCESS") 331 return nil 332 } 333 334 // Git-annex is asking us to return the list of settings that we use. Keep this 335 // in sync with `handlePrepare()`. 336 func (s *server) handleListConfigs() { 337 for _, config := range s.getRequiredConfigs() { 338 s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description)) 339 } 340 s.sendMsg("CONFIGEND") 341 } 342 343 func (s *server) handleTransfer(message *messageParser) error { 344 argMode, err := message.nextSpaceDelimitedParameter() 345 if err != nil { 346 s.sendMsg("TRANSFER-FAILURE failed to parse direction") 347 return fmt.Errorf("malformed arguments for TRANSFER: %w", err) 348 } 349 argKey, err := message.nextSpaceDelimitedParameter() 350 if err != nil { 351 s.sendMsg("TRANSFER-FAILURE failed to parse key") 352 return fmt.Errorf("malformed arguments for TRANSFER: %w", err) 353 } 354 argFile, err := message.finalParameter() 355 if err != nil { 356 s.sendMsg("TRANSFER-FAILURE failed to parse file") 357 return fmt.Errorf("malformed arguments for TRANSFER: %w", err) 358 } 359 360 if err := s.queryConfigs(); err != nil { 361 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey)) 362 return fmt.Errorf("error getting configs: %w", err) 363 } 364 365 remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) 366 if err != nil { 367 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey)) 368 return err 369 } 370 371 localDir := filepath.Dir(argFile) 372 localFs, err := cache.Get(context.TODO(), localDir) 373 if err != nil { 374 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey)) 375 return fmt.Errorf("failed to get local fs: %w", err) 376 } 377 378 remoteFileName := argKey 379 localFileName := filepath.Base(argFile) 380 381 switch argMode { 382 case "STORE": 383 err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName) 384 if err != nil { 385 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) 386 return err 387 } 388 389 case "RETRIEVE": 390 err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName) 391 // It is non-fatal when retrieval fails because the file is missing on 392 // the remote. 393 if err == fs.ErrorObjectNotFound { 394 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey)) 395 return nil 396 } 397 if err != nil { 398 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) 399 return err 400 } 401 402 default: 403 s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey)) 404 return fmt.Errorf("received malformed TRANSFER mode: %v", argMode) 405 } 406 407 s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey)) 408 return nil 409 } 410 411 func (s *server) handleCheckPresent(message *messageParser) error { 412 argKey, err := message.finalParameter() 413 if err != nil { 414 return err 415 } 416 417 if err := s.queryConfigs(); err != nil { 418 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey)) 419 return fmt.Errorf("error getting configs: %s", err) 420 } 421 422 remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) 423 if err != nil { 424 s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey)) 425 return err 426 } 427 428 _, err = remoteFs.NewObject(context.TODO(), argKey) 429 if err == fs.ErrorObjectNotFound { 430 s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) 431 return nil 432 } 433 if err != nil { 434 s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey)) 435 return err 436 } 437 438 s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey)) 439 return nil 440 } 441 442 func (s *server) handleRemove(message *messageParser) error { 443 argKey, err := message.finalParameter() 444 if err != nil { 445 return err 446 } 447 448 remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) 449 if err != nil { 450 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) 451 return fmt.Errorf("error getting remote fs: %w", err) 452 } 453 454 fileObj, err := remoteFs.NewObject(context.TODO(), argKey) 455 // It is non-fatal when removal fails because the file is missing on the 456 // remote. 457 if errors.Is(err, fs.ErrorObjectNotFound) { 458 s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) 459 return nil 460 } 461 if err != nil { 462 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err)) 463 return fmt.Errorf("error getting new fs object: %w", err) 464 } 465 if err := operations.DeleteFile(context.TODO(), fileObj); err != nil { 466 s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey)) 467 return fmt.Errorf("error deleting file: %q", argKey) 468 } 469 s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) 470 return nil 471 } 472 473 func (s *server) handleExtensions(message *messageParser) error { 474 for { 475 extension, err := message.nextSpaceDelimitedParameter() 476 if err != nil { 477 break 478 } 479 switch extension { 480 case "INFO": 481 s.extensionInfo = true 482 case "ASYNC": 483 s.extensionAsync = true 484 case "GETGITREMOTENAME": 485 s.extensionGetGitRemoteName = true 486 case "UNAVAILABLERESPONSE": 487 s.extensionUnavailableResponse = true 488 } 489 } 490 s.sendMsg("EXTENSIONS") 491 return nil 492 } 493 494 var command = &cobra.Command{ 495 Aliases: []string{uniqueCommandName}, 496 Use: subcommandName, 497 Short: "Speaks with git-annex over stdin/stdout.", 498 Long: gitannexHelp, 499 Annotations: map[string]string{ 500 "versionIntroduced": "v1.67.0", 501 }, 502 Run: func(command *cobra.Command, args []string) { 503 cmd.CheckArgs(0, 0, command, args) 504 505 s := server{ 506 reader: bufio.NewReader(os.Stdin), 507 writer: os.Stdout, 508 } 509 err := s.run() 510 if err != nil { 511 s.sendMsg(fmt.Sprintf("ERROR %s", err.Error())) 512 panic(err) 513 } 514 }, 515 }