github.com/GuanceCloud/cliutils@v1.1.21/diskcache/diskcache.go (about)

     1  // Unless explicitly stated otherwise all files in this repository are licensed
     2  // under the MIT License.
     3  // This product includes software developed at Guance Cloud (https://www.guance.com/).
     4  // Copyright 2021-present Guance, Inc.
     5  
     6  // Package diskcache is a simple local-disk cache implements.
     7  //
     8  // The diskcache package is a local-disk cache, it implements following functions:
     9  //
    10  //  1. Concurrent Put()/Get().
    11  //  2. Recoverable last-read-position on restart.
    12  //  3. Exclusive Open() on same path.
    13  //  4. Errors during Get() are retriable.
    14  //  5. Auto-rotate on batch size.
    15  //  6. Drop in FIFO policy when max capacity reached.
    16  //  7. We can configure various specifics in environments without to modify options source code.
    17  package diskcache
    18  
    19  import (
    20  	"errors"
    21  	"fmt"
    22  	"os"
    23  	"sync"
    24  	"time"
    25  )
    26  
    27  const (
    28  	dataHeaderLen = 4
    29  
    30  	// EOFHint labels a file's end.
    31  	EOFHint = uint32(0xdeadbeef)
    32  )
    33  
    34  // Generic diskcache errors.
    35  var (
    36  	// Invalid read size.
    37  	ErrUnexpectedReadSize = errors.New("unexpected read size")
    38  
    39  	// Data send to Put() exceed the maxDataSize.
    40  	ErrTooLargeData = errors.New("too large data")
    41  
    42  	// Get on no data cache.
    43  	ErrEOF = errors.New("EOF")
    44  
    45  	// Invalid cache filename.
    46  	ErrInvalidDataFileName       = errors.New("invalid datafile name")
    47  	ErrInvalidDataFileNameSuffix = errors.New("invalid datafile name suffix")
    48  
    49  	// Invalid file header.
    50  	ErrBadHeader = errors.New("bad header")
    51  )
    52  
    53  // DiskCache is the representation of a disk cache.
    54  // A DiskCache is safe for concurrent use by multiple goroutines.
    55  // Do not Open the same-path diskcache among goroutines.
    56  type DiskCache struct {
    57  	path string
    58  
    59  	dataFiles []string
    60  
    61  	// current writing/reading file.
    62  	curWriteFile,
    63  	curReadfile string
    64  
    65  	// current write/read fd
    66  	wfd, rfd *os.File
    67  
    68  	// If current write file go nothing put for a
    69  	// long time(wakeup), we rotate it manually.
    70  	wfdLastWrite time.Time
    71  
    72  	// how long to wakeup a sleeping write-file
    73  	wakeup time.Duration
    74  
    75  	wlock, // used to exclude concurrent Put.
    76  	rlock *sync.Mutex // used to exclude concurrent Get.
    77  	rwlock *sync.Mutex // used to exclude switch/rotate/drop/Close
    78  
    79  	flock *flock // disabled multi-Open on same path
    80  	pos   *pos   // current read fd position info
    81  
    82  	// specs of current diskcache
    83  	size, // current byte size
    84  	curBatchSize, // current writing file's size
    85  	batchSize, // current batch size(static)
    86  	capacity int64 // capacity of the diskcache
    87  	maxDataSize int32 // max data size of single Put()
    88  
    89  	// File permission, default 0750/0640
    90  	dirPerms,
    91  	filePerms os.FileMode
    92  
    93  	// various flags
    94  	noSync, // NoSync if enabled, may cause data missing, default false
    95  	noFallbackOnError, // ignore Fn() error
    96  	noPos, // no position
    97  	noLock bool // no file lock
    98  
    99  	// labels used to export prometheus flags
   100  	labels []string
   101  }
   102  
   103  func (c *DiskCache) String() string {
   104  	c.rwlock.Lock()
   105  	defer c.rwlock.Unlock()
   106  
   107  	// nolint: lll
   108  	// if there too many files(>10), only print file count
   109  	if n := len(c.dataFiles); n > 10 {
   110  		return fmt.Sprintf("%s/[size: %d][fallback: %v][nosync: %v][nopos: %v][nolock: %v][files: %d][maxDataSize: %d][batchSize: %d][capacity: %d][dataFiles: %d]",
   111  			c.path, c.size, c.noFallbackOnError, c.noSync, c.noPos, c.noLock, len(c.dataFiles), c.maxDataSize, c.batchSize, c.capacity, n,
   112  		)
   113  	} else {
   114  		// nolint: lll
   115  		return fmt.Sprintf("%s/[size: %d][fallback: %v][nosync: %v][nopos: %v][nolock: %v][files: %d][maxDataSize: %d][batchSize: %d][capacity: %d][dataFiles: %v]",
   116  			c.path, c.size, c.noFallbackOnError, c.noSync, c.noLock, c.noPos, len(c.dataFiles), c.maxDataSize, c.batchSize, c.capacity, c.dataFiles,
   117  		)
   118  	}
   119  }