github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/crypt/cipher.go (about)

     1  package crypt
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"crypto/aes"
     7  	gocipher "crypto/cipher"
     8  	"crypto/rand"
     9  	"encoding/base32"
    10  	"fmt"
    11  	"io"
    12  	"strconv"
    13  	"strings"
    14  	"sync"
    15  	"unicode/utf8"
    16  
    17  	"github.com/pkg/errors"
    18  	"github.com/rclone/rclone/backend/crypt/pkcs7"
    19  	"github.com/rclone/rclone/fs"
    20  	"github.com/rclone/rclone/fs/accounting"
    21  	"github.com/rfjakob/eme"
    22  	"golang.org/x/crypto/nacl/secretbox"
    23  	"golang.org/x/crypto/scrypt"
    24  )
    25  
    26  // Constants
    27  const (
    28  	nameCipherBlockSize = aes.BlockSize
    29  	fileMagic           = "RCLONE\x00\x00"
    30  	fileMagicSize       = len(fileMagic)
    31  	fileNonceSize       = 24
    32  	fileHeaderSize      = fileMagicSize + fileNonceSize
    33  	blockHeaderSize     = secretbox.Overhead
    34  	blockDataSize       = 64 * 1024
    35  	blockSize           = blockHeaderSize + blockDataSize
    36  	encryptedSuffix     = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
    37  )
    38  
    39  // Errors returned by cipher
    40  var (
    41  	ErrorBadDecryptUTF8          = errors.New("bad decryption - utf-8 invalid")
    42  	ErrorBadDecryptControlChar   = errors.New("bad decryption - contains control chars")
    43  	ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize")
    44  	ErrorTooShortAfterDecode     = errors.New("too short after base32 decode")
    45  	ErrorTooLongAfterDecode      = errors.New("too long after base32 decode")
    46  	ErrorEncryptedFileTooShort   = errors.New("file is too short to be encrypted")
    47  	ErrorEncryptedFileBadHeader  = errors.New("file has truncated block header")
    48  	ErrorEncryptedBadMagic       = errors.New("not an encrypted file - bad magic string")
    49  	ErrorEncryptedBadBlock       = errors.New("failed to authenticate decrypted block - bad password?")
    50  	ErrorBadBase32Encoding       = errors.New("bad base32 filename encoding")
    51  	ErrorFileClosed              = errors.New("file already closed")
    52  	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
    53  	ErrorBadSeek                 = errors.New("Seek beyond end of file")
    54  	defaultSalt                  = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
    55  	obfuscQuoteRune              = '!'
    56  )
    57  
    58  // Global variables
    59  var (
    60  	fileMagicBytes = []byte(fileMagic)
    61  )
    62  
    63  // ReadSeekCloser is the interface of the read handles
    64  type ReadSeekCloser interface {
    65  	io.Reader
    66  	io.Seeker
    67  	io.Closer
    68  	fs.RangeSeeker
    69  }
    70  
    71  // OpenRangeSeek opens the file handle at the offset with the limit given
    72  type OpenRangeSeek func(ctx context.Context, offset, limit int64) (io.ReadCloser, error)
    73  
    74  // NameEncryptionMode is the type of file name encryption in use
    75  type NameEncryptionMode int
    76  
    77  // NameEncryptionMode levels
    78  const (
    79  	NameEncryptionOff NameEncryptionMode = iota
    80  	NameEncryptionStandard
    81  	NameEncryptionObfuscated
    82  )
    83  
    84  // NewNameEncryptionMode turns a string into a NameEncryptionMode
    85  func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
    86  	s = strings.ToLower(s)
    87  	switch s {
    88  	case "off":
    89  		mode = NameEncryptionOff
    90  	case "standard":
    91  		mode = NameEncryptionStandard
    92  	case "obfuscate":
    93  		mode = NameEncryptionObfuscated
    94  	default:
    95  		err = errors.Errorf("Unknown file name encryption mode %q", s)
    96  	}
    97  	return mode, err
    98  }
    99  
   100  // String turns mode into a human readable string
   101  func (mode NameEncryptionMode) String() (out string) {
   102  	switch mode {
   103  	case NameEncryptionOff:
   104  		out = "off"
   105  	case NameEncryptionStandard:
   106  		out = "standard"
   107  	case NameEncryptionObfuscated:
   108  		out = "obfuscate"
   109  	default:
   110  		out = fmt.Sprintf("Unknown mode #%d", mode)
   111  	}
   112  	return out
   113  }
   114  
   115  // Cipher defines an encoding and decoding cipher for the crypt backend
   116  type Cipher struct {
   117  	dataKey        [32]byte                  // Key for secretbox
   118  	nameKey        [32]byte                  // 16,24 or 32 bytes
   119  	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
   120  	block          gocipher.Block
   121  	mode           NameEncryptionMode
   122  	buffers        sync.Pool // encrypt/decrypt buffers
   123  	cryptoRand     io.Reader // read crypto random numbers from here
   124  	dirNameEncrypt bool
   125  }
   126  
   127  // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
   128  func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
   129  	c := &Cipher{
   130  		mode:           mode,
   131  		cryptoRand:     rand.Reader,
   132  		dirNameEncrypt: dirNameEncrypt,
   133  	}
   134  	c.buffers.New = func() interface{} {
   135  		return make([]byte, blockSize)
   136  	}
   137  	err := c.Key(password, salt)
   138  	if err != nil {
   139  		return nil, err
   140  	}
   141  	return c, nil
   142  }
   143  
   144  // Key creates all the internal keys from the password passed in using
   145  // scrypt.
   146  //
   147  // If salt is "" we use a fixed salt just to make attackers lives
   148  // slighty harder than using no salt.
   149  //
   150  // Note that empty passsword makes all 0x00 keys which is used in the
   151  // tests.
   152  func (c *Cipher) Key(password, salt string) (err error) {
   153  	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
   154  	var saltBytes = defaultSalt
   155  	if salt != "" {
   156  		saltBytes = []byte(salt)
   157  	}
   158  	var key []byte
   159  	if password == "" {
   160  		key = make([]byte, keySize)
   161  	} else {
   162  		key, err = scrypt.Key([]byte(password), saltBytes, 16384, 8, 1, keySize)
   163  		if err != nil {
   164  			return err
   165  		}
   166  	}
   167  	copy(c.dataKey[:], key)
   168  	copy(c.nameKey[:], key[len(c.dataKey):])
   169  	copy(c.nameTweak[:], key[len(c.dataKey)+len(c.nameKey):])
   170  	// Key the name cipher
   171  	c.block, err = aes.NewCipher(c.nameKey[:])
   172  	return err
   173  }
   174  
   175  // getBlock gets a block from the pool of size blockSize
   176  func (c *Cipher) getBlock() []byte {
   177  	return c.buffers.Get().([]byte)
   178  }
   179  
   180  // putBlock returns a block to the pool of size blockSize
   181  func (c *Cipher) putBlock(buf []byte) {
   182  	if len(buf) != blockSize {
   183  		panic("bad blocksize returned to pool")
   184  	}
   185  	c.buffers.Put(buf)
   186  }
   187  
   188  // encodeFileName encodes a filename using a modified version of
   189  // standard base32 as described in RFC4648
   190  //
   191  // The standard encoding is modified in two ways
   192  //  * it becomes lower case (no-one likes upper case filenames!)
   193  //  * we strip the padding character `=`
   194  func encodeFileName(in []byte) string {
   195  	encoded := base32.HexEncoding.EncodeToString(in)
   196  	encoded = strings.TrimRight(encoded, "=")
   197  	return strings.ToLower(encoded)
   198  }
   199  
   200  // decodeFileName decodes a filename as encoded by encodeFileName
   201  func decodeFileName(in string) ([]byte, error) {
   202  	if strings.HasSuffix(in, "=") {
   203  		return nil, ErrorBadBase32Encoding
   204  	}
   205  	// First figure out how many padding characters to add
   206  	roundUpToMultipleOf8 := (len(in) + 7) &^ 7
   207  	equals := roundUpToMultipleOf8 - len(in)
   208  	in = strings.ToUpper(in) + "========"[:equals]
   209  	return base32.HexEncoding.DecodeString(in)
   210  }
   211  
   212  // encryptSegment encrypts a path segment
   213  //
   214  // This uses EME with AES
   215  //
   216  // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
   217  // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
   218  // Rogaway.
   219  //
   220  // This makes for deterministic encryption which is what we want - the
   221  // same filename must encrypt to the same thing.
   222  //
   223  // This means that
   224  //  * filenames with the same name will encrypt the same
   225  //  * filenames which start the same won't have a common prefix
   226  func (c *Cipher) encryptSegment(plaintext string) string {
   227  	if plaintext == "" {
   228  		return ""
   229  	}
   230  	paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext))
   231  	ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt)
   232  	return encodeFileName(ciphertext)
   233  }
   234  
   235  // decryptSegment decrypts a path segment
   236  func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
   237  	if ciphertext == "" {
   238  		return "", nil
   239  	}
   240  	rawCiphertext, err := decodeFileName(ciphertext)
   241  	if err != nil {
   242  		return "", err
   243  	}
   244  	if len(rawCiphertext)%nameCipherBlockSize != 0 {
   245  		return "", ErrorNotAMultipleOfBlocksize
   246  	}
   247  	if len(rawCiphertext) == 0 {
   248  		// not possible if decodeFilename() working correctly
   249  		return "", ErrorTooShortAfterDecode
   250  	}
   251  	if len(rawCiphertext) > 2048 {
   252  		return "", ErrorTooLongAfterDecode
   253  	}
   254  	paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt)
   255  	plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext)
   256  	if err != nil {
   257  		return "", err
   258  	}
   259  	return string(plaintext), err
   260  }
   261  
   262  // Simple obfuscation routines
   263  func (c *Cipher) obfuscateSegment(plaintext string) string {
   264  	if plaintext == "" {
   265  		return ""
   266  	}
   267  
   268  	// If the string isn't valid UTF8 then don't rotate; just
   269  	// prepend a !.
   270  	if !utf8.ValidString(plaintext) {
   271  		return "!." + plaintext
   272  	}
   273  
   274  	// Calculate a simple rotation based on the filename and
   275  	// the nameKey
   276  	var dir int
   277  	for _, runeValue := range plaintext {
   278  		dir += int(runeValue)
   279  	}
   280  	dir = dir % 256
   281  
   282  	// We'll use this number to store in the result filename...
   283  	var result bytes.Buffer
   284  	_, _ = result.WriteString(strconv.Itoa(dir) + ".")
   285  
   286  	// but we'll augment it with the nameKey for real calculation
   287  	for i := 0; i < len(c.nameKey); i++ {
   288  		dir += int(c.nameKey[i])
   289  	}
   290  
   291  	// Now for each character, depending on the range it is in
   292  	// we will actually rotate a different amount
   293  	for _, runeValue := range plaintext {
   294  		switch {
   295  		case runeValue == obfuscQuoteRune:
   296  			// Quote the Quote character
   297  			_, _ = result.WriteRune(obfuscQuoteRune)
   298  			_, _ = result.WriteRune(obfuscQuoteRune)
   299  
   300  		case runeValue >= '0' && runeValue <= '9':
   301  			// Number
   302  			thisdir := (dir % 9) + 1
   303  			newRune := '0' + (int(runeValue)-'0'+thisdir)%10
   304  			_, _ = result.WriteRune(rune(newRune))
   305  
   306  		case (runeValue >= 'A' && runeValue <= 'Z') ||
   307  			(runeValue >= 'a' && runeValue <= 'z'):
   308  			// ASCII letter.  Try to avoid trivial A->a mappings
   309  			thisdir := dir%25 + 1
   310  			// Calculate the offset of this character in A-Za-z
   311  			pos := int(runeValue - 'A')
   312  			if pos >= 26 {
   313  				pos -= 6 // It's lower case
   314  			}
   315  			// Rotate the character to the new location
   316  			pos = (pos + thisdir) % 52
   317  			if pos >= 26 {
   318  				pos += 6 // and handle lower case offset again
   319  			}
   320  			_, _ = result.WriteRune(rune('A' + pos))
   321  
   322  		case runeValue >= 0xA0 && runeValue <= 0xFF:
   323  			// Latin 1 supplement
   324  			thisdir := (dir % 95) + 1
   325  			newRune := 0xA0 + (int(runeValue)-0xA0+thisdir)%96
   326  			_, _ = result.WriteRune(rune(newRune))
   327  
   328  		case runeValue >= 0x100:
   329  			// Some random Unicode range; we have no good rules here
   330  			thisdir := (dir % 127) + 1
   331  			base := int(runeValue - runeValue%256)
   332  			newRune := rune(base + (int(runeValue)-base+thisdir)%256)
   333  			// If the new character isn't a valid UTF8 char
   334  			// then don't rotate it.  Quote it instead
   335  			if !utf8.ValidRune(newRune) {
   336  				_, _ = result.WriteRune(obfuscQuoteRune)
   337  				_, _ = result.WriteRune(runeValue)
   338  			} else {
   339  				_, _ = result.WriteRune(newRune)
   340  			}
   341  
   342  		default:
   343  			// Leave character untouched
   344  			_, _ = result.WriteRune(runeValue)
   345  		}
   346  	}
   347  	return result.String()
   348  }
   349  
   350  func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
   351  	if ciphertext == "" {
   352  		return "", nil
   353  	}
   354  	pos := strings.Index(ciphertext, ".")
   355  	if pos == -1 {
   356  		return "", ErrorNotAnEncryptedFile
   357  	} // No .
   358  	num := ciphertext[:pos]
   359  	if num == "!" {
   360  		// No rotation; probably original was not valid unicode
   361  		return ciphertext[pos+1:], nil
   362  	}
   363  	dir, err := strconv.Atoi(num)
   364  	if err != nil {
   365  		return "", ErrorNotAnEncryptedFile // Not a number
   366  	}
   367  
   368  	// add the nameKey to get the real rotate distance
   369  	for i := 0; i < len(c.nameKey); i++ {
   370  		dir += int(c.nameKey[i])
   371  	}
   372  
   373  	var result bytes.Buffer
   374  
   375  	inQuote := false
   376  	for _, runeValue := range ciphertext[pos+1:] {
   377  		switch {
   378  		case inQuote:
   379  			_, _ = result.WriteRune(runeValue)
   380  			inQuote = false
   381  
   382  		case runeValue == obfuscQuoteRune:
   383  			inQuote = true
   384  
   385  		case runeValue >= '0' && runeValue <= '9':
   386  			// Number
   387  			thisdir := (dir % 9) + 1
   388  			newRune := '0' + int(runeValue) - '0' - thisdir
   389  			if newRune < '0' {
   390  				newRune += 10
   391  			}
   392  			_, _ = result.WriteRune(rune(newRune))
   393  
   394  		case (runeValue >= 'A' && runeValue <= 'Z') ||
   395  			(runeValue >= 'a' && runeValue <= 'z'):
   396  			thisdir := dir%25 + 1
   397  			pos := int(runeValue - 'A')
   398  			if pos >= 26 {
   399  				pos -= 6
   400  			}
   401  			pos = pos - thisdir
   402  			if pos < 0 {
   403  				pos += 52
   404  			}
   405  			if pos >= 26 {
   406  				pos += 6
   407  			}
   408  			_, _ = result.WriteRune(rune('A' + pos))
   409  
   410  		case runeValue >= 0xA0 && runeValue <= 0xFF:
   411  			thisdir := (dir % 95) + 1
   412  			newRune := 0xA0 + int(runeValue) - 0xA0 - thisdir
   413  			if newRune < 0xA0 {
   414  				newRune += 96
   415  			}
   416  			_, _ = result.WriteRune(rune(newRune))
   417  
   418  		case runeValue >= 0x100:
   419  			thisdir := (dir % 127) + 1
   420  			base := int(runeValue - runeValue%256)
   421  			newRune := rune(base + (int(runeValue) - base - thisdir))
   422  			if int(newRune) < base {
   423  				newRune += 256
   424  			}
   425  			_, _ = result.WriteRune(newRune)
   426  
   427  		default:
   428  			_, _ = result.WriteRune(runeValue)
   429  
   430  		}
   431  	}
   432  
   433  	return result.String(), nil
   434  }
   435  
   436  // encryptFileName encrypts a file path
   437  func (c *Cipher) encryptFileName(in string) string {
   438  	segments := strings.Split(in, "/")
   439  	for i := range segments {
   440  		// Skip directory name encryption if the user chose to
   441  		// leave them intact
   442  		if !c.dirNameEncrypt && i != (len(segments)-1) {
   443  			continue
   444  		}
   445  		if c.mode == NameEncryptionStandard {
   446  			segments[i] = c.encryptSegment(segments[i])
   447  		} else {
   448  			segments[i] = c.obfuscateSegment(segments[i])
   449  		}
   450  	}
   451  	return strings.Join(segments, "/")
   452  }
   453  
   454  // EncryptFileName encrypts a file path
   455  func (c *Cipher) EncryptFileName(in string) string {
   456  	if c.mode == NameEncryptionOff {
   457  		return in + encryptedSuffix
   458  	}
   459  	return c.encryptFileName(in)
   460  }
   461  
   462  // EncryptDirName encrypts a directory path
   463  func (c *Cipher) EncryptDirName(in string) string {
   464  	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
   465  		return in
   466  	}
   467  	return c.encryptFileName(in)
   468  }
   469  
   470  // decryptFileName decrypts a file path
   471  func (c *Cipher) decryptFileName(in string) (string, error) {
   472  	segments := strings.Split(in, "/")
   473  	for i := range segments {
   474  		var err error
   475  		// Skip directory name decryption if the user chose to
   476  		// leave them intact
   477  		if !c.dirNameEncrypt && i != (len(segments)-1) {
   478  			continue
   479  		}
   480  		if c.mode == NameEncryptionStandard {
   481  			segments[i], err = c.decryptSegment(segments[i])
   482  		} else {
   483  			segments[i], err = c.deobfuscateSegment(segments[i])
   484  		}
   485  
   486  		if err != nil {
   487  			return "", err
   488  		}
   489  	}
   490  	return strings.Join(segments, "/"), nil
   491  }
   492  
   493  // DecryptFileName decrypts a file path
   494  func (c *Cipher) DecryptFileName(in string) (string, error) {
   495  	if c.mode == NameEncryptionOff {
   496  		remainingLength := len(in) - len(encryptedSuffix)
   497  		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
   498  			return in[:remainingLength], nil
   499  		}
   500  		return "", ErrorNotAnEncryptedFile
   501  	}
   502  	return c.decryptFileName(in)
   503  }
   504  
   505  // DecryptDirName decrypts a directory path
   506  func (c *Cipher) DecryptDirName(in string) (string, error) {
   507  	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
   508  		return in, nil
   509  	}
   510  	return c.decryptFileName(in)
   511  }
   512  
   513  // NameEncryptionMode returns the encryption mode in use for names
   514  func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
   515  	return c.mode
   516  }
   517  
   518  // nonce is an NACL secretbox nonce
   519  type nonce [fileNonceSize]byte
   520  
   521  // pointer returns the nonce as a *[24]byte for secretbox
   522  func (n *nonce) pointer() *[fileNonceSize]byte {
   523  	return (*[fileNonceSize]byte)(n)
   524  }
   525  
   526  // fromReader fills the nonce from an io.Reader - normally the OSes
   527  // crypto random number generator
   528  func (n *nonce) fromReader(in io.Reader) error {
   529  	read, err := io.ReadFull(in, (*n)[:])
   530  	if read != fileNonceSize {
   531  		return errors.Wrap(err, "short read of nonce")
   532  	}
   533  	return nil
   534  }
   535  
   536  // fromBuf fills the nonce from the buffer passed in
   537  func (n *nonce) fromBuf(buf []byte) {
   538  	read := copy((*n)[:], buf)
   539  	if read != fileNonceSize {
   540  		panic("buffer to short to read nonce")
   541  	}
   542  }
   543  
   544  // carry 1 up the nonce from position i
   545  func (n *nonce) carry(i int) {
   546  	for ; i < len(*n); i++ {
   547  		digit := (*n)[i]
   548  		newDigit := digit + 1
   549  		(*n)[i] = newDigit
   550  		if newDigit >= digit {
   551  			// exit if no carry
   552  			break
   553  		}
   554  	}
   555  }
   556  
   557  // increment to add 1 to the nonce
   558  func (n *nonce) increment() {
   559  	n.carry(0)
   560  }
   561  
   562  // add a uint64 to the nonce
   563  func (n *nonce) add(x uint64) {
   564  	carry := uint16(0)
   565  	for i := 0; i < 8; i++ {
   566  		digit := (*n)[i]
   567  		xDigit := byte(x)
   568  		x >>= 8
   569  		carry += uint16(digit) + uint16(xDigit)
   570  		(*n)[i] = byte(carry)
   571  		carry >>= 8
   572  	}
   573  	if carry != 0 {
   574  		n.carry(8)
   575  	}
   576  }
   577  
   578  // encrypter encrypts an io.Reader on the fly
   579  type encrypter struct {
   580  	mu       sync.Mutex
   581  	in       io.Reader
   582  	c        *Cipher
   583  	nonce    nonce
   584  	buf      []byte
   585  	readBuf  []byte
   586  	bufIndex int
   587  	bufSize  int
   588  	err      error
   589  }
   590  
   591  // newEncrypter creates a new file handle encrypting on the fly
   592  func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
   593  	fh := &encrypter{
   594  		in:      in,
   595  		c:       c,
   596  		buf:     c.getBlock(),
   597  		readBuf: c.getBlock(),
   598  		bufSize: fileHeaderSize,
   599  	}
   600  	// Initialise nonce
   601  	if nonce != nil {
   602  		fh.nonce = *nonce
   603  	} else {
   604  		err := fh.nonce.fromReader(c.cryptoRand)
   605  		if err != nil {
   606  			return nil, err
   607  		}
   608  	}
   609  	// Copy magic into buffer
   610  	copy(fh.buf, fileMagicBytes)
   611  	// Copy nonce into buffer
   612  	copy(fh.buf[fileMagicSize:], fh.nonce[:])
   613  	return fh, nil
   614  }
   615  
   616  // Read as per io.Reader
   617  func (fh *encrypter) Read(p []byte) (n int, err error) {
   618  	fh.mu.Lock()
   619  	defer fh.mu.Unlock()
   620  
   621  	if fh.err != nil {
   622  		return 0, fh.err
   623  	}
   624  	if fh.bufIndex >= fh.bufSize {
   625  		// Read data
   626  		// FIXME should overlap the reads with a go-routine and 2 buffers?
   627  		readBuf := fh.readBuf[:blockDataSize]
   628  		n, err = io.ReadFull(fh.in, readBuf)
   629  		if n == 0 {
   630  			// err can't be nil since:
   631  			// n == len(buf) if and only if err == nil.
   632  			return fh.finish(err)
   633  		}
   634  		// possibly err != nil here, but we will process the
   635  		// data and the next call to ReadFull will return 0, err
   636  		// Write nonce to start of block
   637  		copy(fh.buf, fh.nonce[:])
   638  		// Encrypt the block using the nonce
   639  		block := fh.buf
   640  		secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
   641  		fh.bufIndex = 0
   642  		fh.bufSize = blockHeaderSize + n
   643  		fh.nonce.increment()
   644  	}
   645  	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
   646  	fh.bufIndex += n
   647  	return n, nil
   648  }
   649  
   650  // finish sets the final error and tidies up
   651  func (fh *encrypter) finish(err error) (int, error) {
   652  	if fh.err != nil {
   653  		return 0, fh.err
   654  	}
   655  	fh.err = err
   656  	fh.c.putBlock(fh.buf)
   657  	fh.buf = nil
   658  	fh.c.putBlock(fh.readBuf)
   659  	fh.readBuf = nil
   660  	return 0, err
   661  }
   662  
   663  // Encrypt data encrypts the data stream
   664  func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
   665  	in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
   666  	out, err := c.newEncrypter(in, nil)
   667  	if err != nil {
   668  		return nil, nil, err
   669  	}
   670  	return wrap(out), out, nil // and wrap the accounting back on
   671  }
   672  
   673  // EncryptData encrypts the data stream
   674  func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
   675  	out, _, err := c.encryptData(in)
   676  	return out, err
   677  }
   678  
   679  // decrypter decrypts an io.ReaderCloser on the fly
   680  type decrypter struct {
   681  	mu           sync.Mutex
   682  	rc           io.ReadCloser
   683  	nonce        nonce
   684  	initialNonce nonce
   685  	c            *Cipher
   686  	buf          []byte
   687  	readBuf      []byte
   688  	bufIndex     int
   689  	bufSize      int
   690  	err          error
   691  	limit        int64 // limit of bytes to read, -1 for unlimited
   692  	open         OpenRangeSeek
   693  }
   694  
   695  // newDecrypter creates a new file handle decrypting on the fly
   696  func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
   697  	fh := &decrypter{
   698  		rc:      rc,
   699  		c:       c,
   700  		buf:     c.getBlock(),
   701  		readBuf: c.getBlock(),
   702  		limit:   -1,
   703  	}
   704  	// Read file header (magic + nonce)
   705  	readBuf := fh.readBuf[:fileHeaderSize]
   706  	_, err := io.ReadFull(fh.rc, readBuf)
   707  	if err == io.EOF || err == io.ErrUnexpectedEOF {
   708  		// This read from 0..fileHeaderSize-1 bytes
   709  		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
   710  	} else if err != nil {
   711  		return nil, fh.finishAndClose(err)
   712  	}
   713  	// check the magic
   714  	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
   715  		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
   716  	}
   717  	// retrieve the nonce
   718  	fh.nonce.fromBuf(readBuf[fileMagicSize:])
   719  	fh.initialNonce = fh.nonce
   720  	return fh, nil
   721  }
   722  
   723  // newDecrypterSeek creates a new file handle decrypting on the fly
   724  func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
   725  	var rc io.ReadCloser
   726  	doRangeSeek := false
   727  	setLimit := false
   728  	// Open initially with no seek
   729  	if offset == 0 && limit < 0 {
   730  		// If no offset or limit then open whole file
   731  		rc, err = open(ctx, 0, -1)
   732  	} else if offset == 0 {
   733  		// If no offset open the header + limit worth of the file
   734  		_, underlyingLimit, _, _ := calculateUnderlying(offset, limit)
   735  		rc, err = open(ctx, 0, int64(fileHeaderSize)+underlyingLimit)
   736  		setLimit = true
   737  	} else {
   738  		// Otherwise just read the header to start with
   739  		rc, err = open(ctx, 0, int64(fileHeaderSize))
   740  		doRangeSeek = true
   741  	}
   742  	if err != nil {
   743  		return nil, err
   744  	}
   745  	// Open the stream which fills in the nonce
   746  	fh, err = c.newDecrypter(rc)
   747  	if err != nil {
   748  		return nil, err
   749  	}
   750  	fh.open = open // will be called by fh.RangeSeek
   751  	if doRangeSeek {
   752  		_, err = fh.RangeSeek(ctx, offset, io.SeekStart, limit)
   753  		if err != nil {
   754  			_ = fh.Close()
   755  			return nil, err
   756  		}
   757  	}
   758  	if setLimit {
   759  		fh.limit = limit
   760  	}
   761  	return fh, nil
   762  }
   763  
   764  // read data into internal buffer - call with fh.mu held
   765  func (fh *decrypter) fillBuffer() (err error) {
   766  	// FIXME should overlap the reads with a go-routine and 2 buffers?
   767  	readBuf := fh.readBuf
   768  	n, err := io.ReadFull(fh.rc, readBuf)
   769  	if n == 0 {
   770  		// err can't be nil since:
   771  		// n == len(buf) if and only if err == nil.
   772  		return err
   773  	}
   774  	// possibly err != nil here, but we will process the data and
   775  	// the next call to ReadFull will return 0, err
   776  
   777  	// Check header + 1 byte exists
   778  	if n <= blockHeaderSize {
   779  		if err != nil {
   780  			return err // return pending error as it is likely more accurate
   781  		}
   782  		return ErrorEncryptedFileBadHeader
   783  	}
   784  	// Decrypt the block using the nonce
   785  	block := fh.buf
   786  	_, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
   787  	if !ok {
   788  		if err != nil {
   789  			return err // return pending error as it is likely more accurate
   790  		}
   791  		return ErrorEncryptedBadBlock
   792  	}
   793  	fh.bufIndex = 0
   794  	fh.bufSize = n - blockHeaderSize
   795  	fh.nonce.increment()
   796  	return nil
   797  }
   798  
   799  // Read as per io.Reader
   800  func (fh *decrypter) Read(p []byte) (n int, err error) {
   801  	fh.mu.Lock()
   802  	defer fh.mu.Unlock()
   803  
   804  	if fh.err != nil {
   805  		return 0, fh.err
   806  	}
   807  	if fh.bufIndex >= fh.bufSize {
   808  		err = fh.fillBuffer()
   809  		if err != nil {
   810  			return 0, fh.finish(err)
   811  		}
   812  	}
   813  	toCopy := fh.bufSize - fh.bufIndex
   814  	if fh.limit >= 0 && fh.limit < int64(toCopy) {
   815  		toCopy = int(fh.limit)
   816  	}
   817  	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
   818  	fh.bufIndex += n
   819  	if fh.limit >= 0 {
   820  		fh.limit -= int64(n)
   821  		if fh.limit == 0 {
   822  			return n, fh.finish(io.EOF)
   823  		}
   824  	}
   825  	return n, nil
   826  }
   827  
   828  // calculateUnderlying converts an (offset, limit) in a crypted file
   829  // into an (underlyingOffset, underlyingLimit) for the underlying
   830  // file.
   831  //
   832  // It also returns number of bytes to discard after reading the first
   833  // block and number of blocks this is from the start so the nonce can
   834  // be incremented.
   835  func calculateUnderlying(offset, limit int64) (underlyingOffset, underlyingLimit, discard, blocks int64) {
   836  	// blocks we need to seek, plus bytes we need to discard
   837  	blocks, discard = offset/blockDataSize, offset%blockDataSize
   838  
   839  	// Offset in underlying stream we need to seek
   840  	underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
   841  
   842  	// work out how many blocks we need to read
   843  	underlyingLimit = int64(-1)
   844  	if limit >= 0 {
   845  		// bytes to read beyond the first block
   846  		bytesToRead := limit - (blockDataSize - discard)
   847  
   848  		// Read the first block
   849  		blocksToRead := int64(1)
   850  
   851  		if bytesToRead > 0 {
   852  			// Blocks that need to be read plus left over blocks
   853  			extraBlocksToRead, endBytes := bytesToRead/blockDataSize, bytesToRead%blockDataSize
   854  			if endBytes != 0 {
   855  				// If left over bytes must read another block
   856  				extraBlocksToRead++
   857  			}
   858  			blocksToRead += extraBlocksToRead
   859  		}
   860  
   861  		// Must read a whole number of blocks
   862  		underlyingLimit = blocksToRead * (blockHeaderSize + blockDataSize)
   863  	}
   864  	return
   865  }
   866  
   867  // RangeSeek behaves like a call to Seek(offset int64, whence
   868  // int) with the output wrapped in an io.LimitedReader
   869  // limiting the total length to limit.
   870  //
   871  // RangeSeek with a limit of < 0 is equivalent to a regular Seek.
   872  func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, limit int64) (int64, error) {
   873  	fh.mu.Lock()
   874  	defer fh.mu.Unlock()
   875  
   876  	if fh.open == nil {
   877  		return 0, fh.finish(errors.New("can't seek - not initialised with newDecrypterSeek"))
   878  	}
   879  	if whence != io.SeekStart {
   880  		return 0, fh.finish(errors.New("can only seek from the start"))
   881  	}
   882  
   883  	// Reset error or return it if not EOF
   884  	if fh.err == io.EOF {
   885  		fh.unFinish()
   886  	} else if fh.err != nil {
   887  		return 0, fh.err
   888  	}
   889  
   890  	underlyingOffset, underlyingLimit, discard, blocks := calculateUnderlying(offset, limit)
   891  
   892  	// Move the nonce on the correct number of blocks from the start
   893  	fh.nonce = fh.initialNonce
   894  	fh.nonce.add(uint64(blocks))
   895  
   896  	// Can we seek underlying stream directly?
   897  	if do, ok := fh.rc.(fs.RangeSeeker); ok {
   898  		// Seek underlying stream directly
   899  		_, err := do.RangeSeek(ctx, underlyingOffset, 0, underlyingLimit)
   900  		if err != nil {
   901  			return 0, fh.finish(err)
   902  		}
   903  	} else {
   904  		// if not reopen with seek
   905  		_ = fh.rc.Close() // close underlying file
   906  		fh.rc = nil
   907  
   908  		// Re-open the underlying object with the offset given
   909  		rc, err := fh.open(ctx, underlyingOffset, underlyingLimit)
   910  		if err != nil {
   911  			return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset and limit"))
   912  		}
   913  
   914  		// Set the file handle
   915  		fh.rc = rc
   916  	}
   917  
   918  	// Fill the buffer
   919  	err := fh.fillBuffer()
   920  	if err != nil {
   921  		return 0, fh.finish(err)
   922  	}
   923  
   924  	// Discard bytes from the buffer
   925  	if int(discard) > fh.bufSize {
   926  		return 0, fh.finish(ErrorBadSeek)
   927  	}
   928  	fh.bufIndex = int(discard)
   929  
   930  	// Set the limit
   931  	fh.limit = limit
   932  
   933  	return offset, nil
   934  }
   935  
   936  // Seek implements the io.Seeker interface
   937  func (fh *decrypter) Seek(offset int64, whence int) (int64, error) {
   938  	return fh.RangeSeek(context.TODO(), offset, whence, -1)
   939  }
   940  
   941  // finish sets the final error and tidies up
   942  func (fh *decrypter) finish(err error) error {
   943  	if fh.err != nil {
   944  		return fh.err
   945  	}
   946  	fh.err = err
   947  	fh.c.putBlock(fh.buf)
   948  	fh.buf = nil
   949  	fh.c.putBlock(fh.readBuf)
   950  	fh.readBuf = nil
   951  	return err
   952  }
   953  
   954  // unFinish undoes the effects of finish
   955  func (fh *decrypter) unFinish() {
   956  	// Clear error
   957  	fh.err = nil
   958  
   959  	// reinstate the buffers
   960  	fh.buf = fh.c.getBlock()
   961  	fh.readBuf = fh.c.getBlock()
   962  
   963  	// Empty the buffer
   964  	fh.bufIndex = 0
   965  	fh.bufSize = 0
   966  }
   967  
   968  // Close
   969  func (fh *decrypter) Close() error {
   970  	fh.mu.Lock()
   971  	defer fh.mu.Unlock()
   972  
   973  	// Check already closed
   974  	if fh.err == ErrorFileClosed {
   975  		return fh.err
   976  	}
   977  	// Closed before reading EOF so not finish()ed yet
   978  	if fh.err == nil {
   979  		_ = fh.finish(io.EOF)
   980  	}
   981  	// Show file now closed
   982  	fh.err = ErrorFileClosed
   983  	if fh.rc == nil {
   984  		return nil
   985  	}
   986  	return fh.rc.Close()
   987  }
   988  
   989  // finishAndClose does finish then Close()
   990  //
   991  // Used when we are returning a nil fh from new
   992  func (fh *decrypter) finishAndClose(err error) error {
   993  	_ = fh.finish(err)
   994  	_ = fh.Close()
   995  	return err
   996  }
   997  
   998  // DecryptData decrypts the data stream
   999  func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
  1000  	out, err := c.newDecrypter(rc)
  1001  	if err != nil {
  1002  		return nil, err
  1003  	}
  1004  	return out, nil
  1005  }
  1006  
  1007  // DecryptDataSeek decrypts the data stream from offset
  1008  //
  1009  // The open function must return a ReadCloser opened to the offset supplied
  1010  //
  1011  // You must use this form of DecryptData if you might want to Seek the file handle
  1012  func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
  1013  	out, err := c.newDecrypterSeek(ctx, open, offset, limit)
  1014  	if err != nil {
  1015  		return nil, err
  1016  	}
  1017  	return out, nil
  1018  }
  1019  
  1020  // EncryptedSize calculates the size of the data when encrypted
  1021  func (c *Cipher) EncryptedSize(size int64) int64 {
  1022  	blocks, residue := size/blockDataSize, size%blockDataSize
  1023  	encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
  1024  	if residue != 0 {
  1025  		encryptedSize += blockHeaderSize + residue
  1026  	}
  1027  	return encryptedSize
  1028  }
  1029  
  1030  // DecryptedSize calculates the size of the data when decrypted
  1031  func (c *Cipher) DecryptedSize(size int64) (int64, error) {
  1032  	size -= int64(fileHeaderSize)
  1033  	if size < 0 {
  1034  		return 0, ErrorEncryptedFileTooShort
  1035  	}
  1036  	blocks, residue := size/blockSize, size%blockSize
  1037  	decryptedSize := blocks * blockDataSize
  1038  	if residue != 0 {
  1039  		residue -= blockHeaderSize
  1040  		if residue <= 0 {
  1041  			return 0, ErrorEncryptedFileBadHeader
  1042  		}
  1043  	}
  1044  	decryptedSize += residue
  1045  	return decryptedSize, nil
  1046  }
  1047  
  1048  // check interfaces
  1049  var (
  1050  	_ io.ReadCloser  = (*decrypter)(nil)
  1051  	_ io.Seeker      = (*decrypter)(nil)
  1052  	_ fs.RangeSeeker = (*decrypter)(nil)
  1053  	_ io.Reader      = (*encrypter)(nil)
  1054  )