github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/hdfs/hdfs.go (about)

     1  //go:build !plan9
     2  
     3  // Package hdfs provides an interface to the HDFS storage system.
     4  package hdfs
     5  
     6  import (
     7  	"path"
     8  	"strings"
     9  
    10  	"github.com/rclone/rclone/fs"
    11  	"github.com/rclone/rclone/fs/config"
    12  	"github.com/rclone/rclone/lib/encoder"
    13  )
    14  
    15  func init() {
    16  	fsi := &fs.RegInfo{
    17  		Name:        "hdfs",
    18  		Description: "Hadoop distributed file system",
    19  		NewFs:       NewFs,
    20  		Options: []fs.Option{{
    21  			Name:      "namenode",
    22  			Help:      "Hadoop name nodes and ports.\n\nE.g. \"namenode-1:8020,namenode-2:8020,...\" to connect to host namenodes at port 8020.",
    23  			Required:  true,
    24  			Sensitive: true,
    25  			Default:   fs.CommaSepList{},
    26  		}, {
    27  			Name: "username",
    28  			Help: "Hadoop user name.",
    29  			Examples: []fs.OptionExample{{
    30  				Value: "root",
    31  				Help:  "Connect to hdfs as root.",
    32  			}},
    33  			Sensitive: true,
    34  		}, {
    35  			Name: "service_principal_name",
    36  			Help: `Kerberos service principal name for the namenode.
    37  
    38  Enables KERBEROS authentication. Specifies the Service Principal Name
    39  (SERVICE/FQDN) for the namenode. E.g. \"hdfs/namenode.hadoop.docker\"
    40  for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
    41  			Advanced:  true,
    42  			Sensitive: true,
    43  		}, {
    44  			Name: "data_transfer_protection",
    45  			Help: `Kerberos data transfer protection: authentication|integrity|privacy.
    46  
    47  Specifies whether or not authentication, data signature integrity
    48  checks, and wire encryption are required when communicating with
    49  the datanodes. Possible values are 'authentication', 'integrity'
    50  and 'privacy'. Used only with KERBEROS enabled.`,
    51  			Examples: []fs.OptionExample{{
    52  				Value: "privacy",
    53  				Help:  "Ensure authentication, integrity and encryption enabled.",
    54  			}},
    55  			Advanced: true,
    56  		}, {
    57  			Name:     config.ConfigEncoding,
    58  			Help:     config.ConfigEncodingHelp,
    59  			Advanced: true,
    60  			Default:  (encoder.Display | encoder.EncodeInvalidUtf8 | encoder.EncodeColon),
    61  		}},
    62  	}
    63  	fs.Register(fsi)
    64  }
    65  
    66  // Options for this backend
    67  type Options struct {
    68  	Namenode               fs.CommaSepList      `config:"namenode"`
    69  	Username               string               `config:"username"`
    70  	ServicePrincipalName   string               `config:"service_principal_name"`
    71  	DataTransferProtection string               `config:"data_transfer_protection"`
    72  	Enc                    encoder.MultiEncoder `config:"encoding"`
    73  }
    74  
    75  // xPath make correct file path with leading '/'
    76  func xPath(root string, tail string) string {
    77  	if !strings.HasPrefix(root, "/") {
    78  		root = "/" + root
    79  	}
    80  	return path.Join(root, tail)
    81  }