github.com/informationsea/shellflow@v0.1.3/filecache.go (about)

     1  package main
     2  
     3  import (
     4  	"bytes"
     5  	"compress/gzip"
     6  	"crypto/sha256"
     7  	"database/sql"
     8  	"fmt"
     9  	"io"
    10  	"os"
    11  	"path"
    12  
    13  	_ "github.com/mattn/go-sqlite3"
    14  )
    15  
    16  var fileStatCache = make(map[string]os.FileInfo)
    17  
    18  func ClearCache() {
    19  	if Sha256CacheConnection != nil {
    20  		Sha256CacheConnection.Connection.Exec("DROP TABLE IF EXISTS Sha256cache")
    21  		Sha256CacheConnection.Connection.Close()
    22  		Sha256CacheConnection = nil
    23  	}
    24  	fileStatCache = make(map[string]os.FileInfo)
    25  }
    26  
    27  func Stat(name string) (os.FileInfo, error) {
    28  	if stat, ok := fileStatCache[name]; ok {
    29  		return stat, nil
    30  	}
    31  	stat, err := os.Stat(name)
    32  	if err != nil {
    33  		return nil, err
    34  	}
    35  	fileStatCache[name] = stat
    36  	return stat, nil
    37  }
    38  
    39  type Sha256Cache struct {
    40  	CacheFilePath string
    41  	Connection    *sql.DB
    42  }
    43  
    44  func NewSha256Cache() *Sha256Cache {
    45  	err := os.MkdirAll(WorkflowLogDir, 0755)
    46  	if err != nil && !os.IsExist(err) {
    47  		fmt.Fprintf(os.Stderr, "(Ignored) Cannot create workflow log directory: %s\n", err)
    48  		return nil
    49  		// panic(err.Error())
    50  	}
    51  
    52  	filename := path.Join(WorkflowLogDir, "files.sqlite3")
    53  	conn, err := sql.Open("sqlite3", filename)
    54  	if err != nil {
    55  		fmt.Fprintf(os.Stderr, "(Ignored) Cannot open sqlite3 file: %s\n", err)
    56  		return nil
    57  		// panic(err.Error())
    58  	}
    59  
    60  	_, err = conn.Exec("CREATE TABLE IF NOT EXISTS Sha256Cache(path TEXT, modified TEXT, size INTEGER, sha256 TEXT, PRIMARY KEY(path, modified, size))")
    61  	if err != nil {
    62  		fmt.Fprintf(os.Stderr, "(Ignored) Cannot create cache table: %s\n", err)
    63  		return nil
    64  		// panic(err.Error())
    65  	}
    66  
    67  	return &Sha256Cache{
    68  		CacheFilePath: filename,
    69  		Connection:    conn,
    70  	}
    71  }
    72  
    73  var Sha256CacheConnection *Sha256Cache
    74  
    75  func CalcSha256ForFile(filepath string, maximumContentLogSize int64) (HashSum, error) {
    76  	if Sha256CacheConnection == nil {
    77  		Sha256CacheConnection = NewSha256Cache()
    78  	}
    79  
    80  	stat, err := Stat(filepath)
    81  	if err != nil {
    82  		return nil, err
    83  	}
    84  
    85  	if Sha256CacheConnection != nil {
    86  		result := Sha256CacheConnection.Connection.QueryRow("SELECT sha256 FROM Sha256Cache WHERE path = ? AND modified = ? AND size = ?", filepath, stat.ModTime().String(), stat.Size())
    87  		var hashString string
    88  		err = result.Scan(&hashString)
    89  		if err == nil {
    90  			var hashSum HashSum
    91  			_, err = fmt.Sscanf(hashString, "%x", &hashSum)
    92  			return hashSum, nil
    93  		}
    94  		if err != sql.ErrNoRows {
    95  			fmt.Fprintf(os.Stderr, "(Ignored) Cannot scan sqlite database: %s\n", err)
    96  			//return nil, err
    97  		}
    98  	}
    99  
   100  	backupContent := stat.Size() <= maximumContentLogSize
   101  	var content = bytes.NewBuffer(nil)
   102  	var reader io.Reader
   103  
   104  	file, err := os.Open(filepath)
   105  	if err != nil {
   106  		return nil, err
   107  	}
   108  	defer file.Close()
   109  
   110  	fmt.Printf("calculating SHA256 for %s\n", filepath)
   111  
   112  	if backupContent {
   113  		reader = io.TeeReader(file, content)
   114  	} else {
   115  		reader = file
   116  	}
   117  
   118  	hash := sha256.New()
   119  	_, err = io.Copy(hash, reader)
   120  	if err != nil {
   121  		return nil, err
   122  	}
   123  
   124  	hashResult := hash.Sum(nil)
   125  	hashString := fmt.Sprintf("%x", hashResult)
   126  
   127  	// backup copy
   128  	if backupContent {
   129  		backupPath := path.Join(WorkflowLogDir, "__backup", hashString[:1], hashString[:2], hashString+".gz")
   130  		backupDir := path.Dir(backupPath)
   131  		err = os.MkdirAll(backupDir, 0755)
   132  		if err != nil && !os.IsExist(err) {
   133  			return nil, err
   134  		}
   135  		backupFile, err := os.OpenFile(backupPath, os.O_CREATE|os.O_WRONLY, 0644)
   136  		if err != nil {
   137  			return nil, err
   138  		}
   139  		defer backupFile.Close()
   140  		gzipEncoder := gzip.NewWriter(backupFile)
   141  		defer gzipEncoder.Close()
   142  		gzipEncoder.Write(content.Bytes())
   143  	}
   144  
   145  	// register to file log
   146  	if Sha256CacheConnection != nil {
   147  		_, err = Sha256CacheConnection.Connection.Exec("INSERT INTO Sha256Cache(path, modified, size, sha256) VALUES(?, ?, ?, ?)", filepath, stat.ModTime().String(), stat.Size(), hashString)
   148  		if err != nil {
   149  			fmt.Fprintf(os.Stderr, "(Ignored) Cannot log SHA256 into database: %s\n", err.Error())
   150  			return hashResult, nil
   151  		}
   152  	}
   153  
   154  	return hashResult, nil
   155  }