github.com/Cloud-Foundations/Dominator@v0.3.4/lib/objectserver/filesystem/add.go (about)

     1  package filesystem
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"errors"
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"path"
    11  	"time"
    12  
    13  	"github.com/Cloud-Foundations/Dominator/lib/fsutil"
    14  	"github.com/Cloud-Foundations/Dominator/lib/hash"
    15  	"github.com/Cloud-Foundations/Dominator/lib/objectcache"
    16  )
    17  
    18  const (
    19  	buflen = 65536
    20  )
    21  
    22  // This must be called with the lock held. The object must not already exist.
    23  func (objSrv *ObjectServer) add(object *objectType) {
    24  	objSrv.objects[object.hash] = object
    25  	objSrv.addUnreferenced(object)
    26  	objSrv.lastMutationTime = time.Now()
    27  	objSrv.totalBytes += object.size
    28  }
    29  
    30  func (objSrv *ObjectServer) addObject(reader io.Reader, length uint64,
    31  	expectedHash *hash.Hash) (hash.Hash, bool, error) {
    32  	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
    33  	if err != nil {
    34  		return hashVal, false, err
    35  	}
    36  	filename := path.Join(objSrv.BaseDirectory,
    37  		objectcache.HashToFilename(hashVal))
    38  	// Check for existing object and collision.
    39  	if isNew, err := objSrv.addOrCompare(hashVal, data, filename); err != nil {
    40  		return hashVal, false, err
    41  	} else {
    42  		object := &objectType{hash: hashVal, size: uint64(len(data))}
    43  		objSrv.rwLock.Lock()
    44  		if _, ok := objSrv.objects[object.hash]; !ok {
    45  			objSrv.add(object)
    46  		}
    47  		objSrv.rwLock.Unlock()
    48  		if objSrv.addCallback != nil {
    49  			objSrv.addCallback(hashVal, uint64(len(data)), isNew)
    50  		}
    51  		return hashVal, isNew, nil
    52  	}
    53  }
    54  
    55  func (objSrv *ObjectServer) addOrCompare(hashVal hash.Hash, data []byte,
    56  	filename string) (bool, error) {
    57  	fi, err := os.Lstat(filename)
    58  	if err == nil {
    59  		if !fi.Mode().IsRegular() {
    60  			return false, errors.New("existing non-file: " + filename)
    61  		}
    62  		if err := collisionCheck(data, filename, fi.Size()); err != nil {
    63  			return false, errors.New("collision detected: " + err.Error())
    64  		}
    65  		// No collision and no error: it's the same object. Go home early.
    66  		return false, nil
    67  	}
    68  	if objSrv.gc != nil { // Have external garbage collector: trigger it inline.
    69  		objSrv.garbageCollector()
    70  	}
    71  	err = os.MkdirAll(path.Dir(filename), fsutil.PrivateDirPerms)
    72  	if err != nil {
    73  		return false, err
    74  	}
    75  	err = fsutil.CopyToFile(filename, fsutil.PrivateFilePerms,
    76  		bytes.NewReader(data), uint64(len(data)))
    77  	if err != nil {
    78  		return false, err
    79  	}
    80  	return true, nil
    81  }
    82  
    83  func collisionCheck(data []byte, filename string, size int64) error {
    84  	file, err := os.Open(filename)
    85  	if err != nil {
    86  		return err
    87  	}
    88  	defer file.Close()
    89  	if int64(len(data)) != size {
    90  		return fmt.Errorf("length mismatch. Data=%d, existing object=%d",
    91  			len(data), size)
    92  	}
    93  	reader := bufio.NewReader(file)
    94  	buffer := make([]byte, 0, buflen)
    95  	for len(data) > 0 {
    96  		numToRead := len(data)
    97  		if numToRead > cap(buffer) {
    98  			numToRead = cap(buffer)
    99  		}
   100  		buf := buffer[:numToRead]
   101  		nread, err := reader.Read(buf)
   102  		if err != nil {
   103  			return err
   104  		}
   105  		if bytes.Compare(data[:nread], buf[:nread]) != 0 {
   106  			return errors.New("content mismatch")
   107  		}
   108  		data = data[nread:]
   109  	}
   110  	return nil
   111  }