github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/lib/objectserver/filesystem/add.go (about)

     1  package filesystem
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"errors"
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"path"
    11  	"syscall"
    12  	"time"
    13  
    14  	"github.com/Cloud-Foundations/Dominator/lib/fsutil"
    15  	"github.com/Cloud-Foundations/Dominator/lib/hash"
    16  	"github.com/Cloud-Foundations/Dominator/lib/objectcache"
    17  )
    18  
    19  const (
    20  	filePerms = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP
    21  	buflen    = 65536
    22  )
    23  
    24  func (objSrv *ObjectServer) addObject(reader io.Reader, length uint64,
    25  	expectedHash *hash.Hash) (hash.Hash, bool, error) {
    26  	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
    27  	if err != nil {
    28  		return hashVal, false, err
    29  	}
    30  	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hashVal))
    31  	// Check for existing object and collision.
    32  	if isNew, err := objSrv.addOrCompare(hashVal, data, filename); err != nil {
    33  		return hashVal, false, err
    34  	} else {
    35  		objSrv.rwLock.Lock()
    36  		objSrv.sizesMap[hashVal] = uint64(len(data))
    37  		objSrv.lastMutationTime = time.Now()
    38  		objSrv.rwLock.Unlock()
    39  		if objSrv.addCallback != nil {
    40  			objSrv.addCallback(hashVal, uint64(len(data)), isNew)
    41  		}
    42  		return hashVal, isNew, nil
    43  	}
    44  }
    45  
    46  func (objSrv *ObjectServer) addOrCompare(hashVal hash.Hash, data []byte,
    47  	filename string) (bool, error) {
    48  	fi, err := os.Lstat(filename)
    49  	if err == nil {
    50  		if !fi.Mode().IsRegular() {
    51  			return false, errors.New("existing non-file: " + filename)
    52  		}
    53  		if err := collisionCheck(data, filename, fi.Size()); err != nil {
    54  			return false, errors.New("collision detected: " + err.Error())
    55  		}
    56  		// No collision and no error: it's the same object. Go home early.
    57  		return false, nil
    58  	}
    59  	objSrv.garbageCollector()
    60  	if err = os.MkdirAll(path.Dir(filename), syscall.S_IRWXU); err != nil {
    61  		return false, err
    62  	}
    63  	if err := fsutil.CopyToFile(filename, filePerms, bytes.NewReader(data),
    64  		uint64(len(data))); err != nil {
    65  		return false, err
    66  	}
    67  	return true, nil
    68  }
    69  
    70  func collisionCheck(data []byte, filename string, size int64) error {
    71  	file, err := os.Open(filename)
    72  	if err != nil {
    73  		return err
    74  	}
    75  	defer file.Close()
    76  	if int64(len(data)) != size {
    77  		return errors.New(fmt.Sprintf(
    78  			"length mismatch. Data=%d, existing object=%d",
    79  			len(data), size))
    80  	}
    81  	reader := bufio.NewReader(file)
    82  	buffer := make([]byte, 0, buflen)
    83  	for len(data) > 0 {
    84  		numToRead := len(data)
    85  		if numToRead > cap(buffer) {
    86  			numToRead = cap(buffer)
    87  		}
    88  		buf := buffer[:numToRead]
    89  		nread, err := reader.Read(buf)
    90  		if err != nil {
    91  			return err
    92  		}
    93  		if bytes.Compare(data[:nread], buf[:nread]) != 0 {
    94  			return errors.New("content mismatch")
    95  		}
    96  		data = data[nread:]
    97  	}
    98  	return nil
    99  }