github.com/keakon/golog@v0.0.0-20230330091222-cac71197c18d/writer.go (about)

     1  package golog
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"errors"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"os"
    11  	"path/filepath"
    12  	"runtime"
    13  	"sort"
    14  	"sync"
    15  	"time"
    16  )
    17  
    18  const (
    19  	defaultBufferSize = 1024 * 1024 * 4
    20  
    21  	fileFlag      = os.O_WRONLY | os.O_CREATE | os.O_APPEND
    22  	fileMode      = 0644
    23  	flushDuration = time.Millisecond * 100
    24  
    25  	rotateByDateFormat = "-20060102.log"   // -YYYYmmdd.log
    26  	rotateByHourFormat = "-2006010215.log" // -YYYYmmddHH.log
    27  )
    28  
    29  // RotateDuration specifies rotate duration type, should be either RotateByDate or RotateByHour.
    30  type RotateDuration uint8
    31  
    32  const (
    33  	// RotateByDate set the log file to be rotated each day.
    34  	RotateByDate RotateDuration = iota
    35  	// RotateByHour set the log file to be rotated each hour.
    36  	RotateByHour
    37  )
    38  
    39  // DiscardWriter is a WriteCloser which write everything to devNull
    40  type DiscardWriter struct {
    41  	io.Writer
    42  }
    43  
    44  // NewDiscardWriter creates a new ConsoleWriter.
    45  func NewDiscardWriter() *DiscardWriter {
    46  	return &DiscardWriter{Writer: ioutil.Discard}
    47  }
    48  
    49  // Close sets its Writer to nil.
    50  func (w *DiscardWriter) Close() error {
    51  	w.Writer = nil
    52  	return nil
    53  }
    54  
    55  // A ConsoleWriter is a writer which should not be actually closed.
    56  type ConsoleWriter struct {
    57  	*os.File // faster than io.Writer
    58  }
    59  
    60  // NewConsoleWriter creates a new ConsoleWriter.
    61  func NewConsoleWriter(f *os.File) *ConsoleWriter {
    62  	return &ConsoleWriter{File: f}
    63  }
    64  
    65  // NewStdoutWriter creates a new stdout writer.
    66  func NewStdoutWriter() *ConsoleWriter {
    67  	return NewConsoleWriter(os.Stdout)
    68  }
    69  
    70  // NewStderrWriter creates a new stderr writer.
    71  func NewStderrWriter() *ConsoleWriter {
    72  	return NewConsoleWriter(os.Stderr)
    73  }
    74  
    75  // Close sets its File to nil.
    76  func (w *ConsoleWriter) Close() error {
    77  	w.File = nil
    78  	return nil
    79  }
    80  
    81  // NewFileWriter creates a FileWriter by its path.
    82  func NewFileWriter(path string) (*os.File, error) {
    83  	return os.OpenFile(path, fileFlag, fileMode)
    84  }
    85  
    86  type bufferedFileWriter struct {
    87  	file       *os.File
    88  	buffer     *bufio.Writer
    89  	bufferSize uint32
    90  }
    91  
    92  type BufferedFileWriterOption func(*bufferedFileWriter)
    93  
    94  // BufferSize sets the buffer size.
    95  func BufferSize(size uint32) BufferedFileWriterOption {
    96  	return func(w *bufferedFileWriter) {
    97  		if size >= 1024 {
    98  			w.bufferSize = size
    99  		}
   100  	}
   101  }
   102  
   103  // A BufferedFileWriter is a buffered file writer.
   104  // The written bytes will be flushed to the log file every 0.1 second,
   105  // or when reaching the buffer capacity (4 MB).
   106  type BufferedFileWriter struct {
   107  	bufferedFileWriter
   108  	lock       sync.Mutex
   109  	stopChan   chan struct{}
   110  	updateChan chan struct{}
   111  	updated    bool
   112  }
   113  
   114  // NewBufferedFileWriter creates a new BufferedFileWriter.
   115  func NewBufferedFileWriter(path string, options ...BufferedFileWriterOption) (*BufferedFileWriter, error) {
   116  	f, err := os.OpenFile(path, fileFlag, fileMode)
   117  	if err != nil {
   118  		return nil, err
   119  	}
   120  	w := &BufferedFileWriter{
   121  		bufferedFileWriter: bufferedFileWriter{
   122  			file:       f,
   123  			bufferSize: defaultBufferSize,
   124  		},
   125  		updateChan: make(chan struct{}, 1),
   126  		stopChan:   make(chan struct{}),
   127  	}
   128  
   129  	for _, option := range options {
   130  		option(&w.bufferedFileWriter)
   131  	}
   132  	w.buffer = bufio.NewWriterSize(f, int(w.bufferSize))
   133  
   134  	go w.schedule()
   135  	return w, nil
   136  }
   137  
   138  func (w *BufferedFileWriter) schedule() {
   139  	timer := time.NewTimer(0)
   140  	for {
   141  		select {
   142  		case <-w.updateChan:
   143  			// something has been written to the buffer, it can be flushed to the file later
   144  			stopTimer(timer)
   145  			timer.Reset(flushDuration)
   146  		case <-w.stopChan:
   147  			stopTimer(timer)
   148  			return
   149  		}
   150  
   151  		select {
   152  		case <-timer.C:
   153  			var err error
   154  			w.lock.Lock()
   155  			if w.file != nil { // not closed
   156  				w.updated = false
   157  				err = w.buffer.Flush()
   158  			}
   159  			w.lock.Unlock()
   160  			if err != nil {
   161  				logError(err)
   162  			}
   163  		case <-w.stopChan:
   164  			stopTimer(timer)
   165  			return
   166  		}
   167  	}
   168  }
   169  
   170  // Write writes a byte slice to the buffer.
   171  func (w *BufferedFileWriter) Write(p []byte) (n int, err error) {
   172  	w.lock.Lock()
   173  	n, err = w.buffer.Write(p)
   174  	if !w.updated && n > 0 && w.buffer.Buffered() > 0 { // checks w.updated to prevent notifying w.updateChan twice
   175  		w.updated = true
   176  		w.lock.Unlock()
   177  
   178  		select { // ignores if blocked
   179  		case w.updateChan <- struct{}{}:
   180  		default:
   181  		}
   182  	} else {
   183  		w.lock.Unlock()
   184  	}
   185  	return
   186  }
   187  
   188  // Close flushes the buffer, then closes the file writer.
   189  func (w *BufferedFileWriter) Close() error {
   190  	close(w.stopChan)
   191  	w.lock.Lock()
   192  	err := w.buffer.Flush()
   193  	w.buffer = nil
   194  	if err == nil {
   195  		err = w.file.Close()
   196  	} else {
   197  		e := w.file.Close()
   198  		if e != nil {
   199  			logError(e)
   200  		}
   201  	}
   202  	w.file = nil
   203  	w.lock.Unlock()
   204  	return err
   205  }
   206  
   207  // A RotatingFileWriter is a buffered file writer which will rotate before reaching its maxSize.
   208  // An exception is when a record is larger than maxSize, it won't be separated into 2 files.
   209  // It keeps at most backupCount backups.
   210  type RotatingFileWriter struct {
   211  	BufferedFileWriter
   212  	path        string
   213  	pos         uint64
   214  	maxSize     uint64
   215  	backupCount uint8
   216  }
   217  
   218  // NewRotatingFileWriter creates a new RotatingFileWriter.
   219  func NewRotatingFileWriter(path string, maxSize uint64, backupCount uint8, options ...BufferedFileWriterOption) (*RotatingFileWriter, error) {
   220  	if maxSize == 0 {
   221  		return nil, errors.New("maxSize cannot be 0")
   222  	}
   223  
   224  	if backupCount == 0 {
   225  		return nil, errors.New("backupCount cannot be 0")
   226  	}
   227  
   228  	f, err := os.OpenFile(path, fileFlag, fileMode)
   229  	if err != nil {
   230  		return nil, err
   231  	}
   232  
   233  	stat, err := f.Stat()
   234  	if err != nil {
   235  		e := f.Close()
   236  		if e != nil {
   237  			logError(e)
   238  		}
   239  		return nil, err
   240  	}
   241  
   242  	w := RotatingFileWriter{
   243  		BufferedFileWriter: BufferedFileWriter{
   244  			bufferedFileWriter: bufferedFileWriter{
   245  				file:       f,
   246  				bufferSize: defaultBufferSize,
   247  			},
   248  			updateChan: make(chan struct{}, 1),
   249  			stopChan:   make(chan struct{}),
   250  		},
   251  		path:        path,
   252  		pos:         uint64(stat.Size()),
   253  		maxSize:     maxSize,
   254  		backupCount: backupCount,
   255  	}
   256  
   257  	for _, option := range options {
   258  		option(&w.bufferedFileWriter)
   259  	}
   260  	w.buffer = bufio.NewWriterSize(f, int(w.bufferSize))
   261  
   262  	go w.schedule()
   263  	return &w, nil
   264  }
   265  
   266  // Write writes a byte slice to the buffer and rotates if reaching its maxSize.
   267  func (w *RotatingFileWriter) Write(p []byte) (n int, err error) {
   268  	w.lock.Lock()
   269  	defer w.lock.Unlock()
   270  
   271  	n, err = w.buffer.Write(p)
   272  	if n > 0 {
   273  		w.pos += uint64(n)
   274  
   275  		if w.pos >= w.maxSize {
   276  			e := w.rotate()
   277  			if e != nil {
   278  				logError(e)
   279  				if err == nil { // don't shadow Write() error
   280  					err = e
   281  				}
   282  			}
   283  			return // w.rotate() also calls w.buffer.Flush(), no need to notify w.updateChan
   284  		}
   285  
   286  		if !w.updated && w.buffer.Buffered() > 0 {
   287  			w.updated = true
   288  
   289  			select { // ignores if blocked
   290  			case w.updateChan <- struct{}{}:
   291  			default:
   292  			}
   293  		}
   294  	}
   295  
   296  	return
   297  }
   298  
   299  // rotate rotates the log file. It should be called within a lock block.
   300  func (w *RotatingFileWriter) rotate() error {
   301  	if w.file == nil { // was closed
   302  		return os.ErrClosed
   303  	}
   304  
   305  	err := w.buffer.Flush()
   306  	if err != nil {
   307  		return err
   308  	}
   309  
   310  	err = w.file.Close()
   311  	w.pos = 0
   312  	if err != nil {
   313  		w.file = nil
   314  		w.buffer = nil
   315  		return err
   316  	}
   317  
   318  	for i := w.backupCount; i > 1; i-- {
   319  		oldPath := fmt.Sprintf("%s.%d", w.path, i-1)
   320  		newPath := fmt.Sprintf("%s.%d", w.path, i)
   321  		e := os.Rename(oldPath, newPath)
   322  		if e != nil {
   323  			logError(e)
   324  		}
   325  	}
   326  
   327  	err = os.Rename(w.path, w.path+".1")
   328  	if err != nil {
   329  		w.file = nil
   330  		w.buffer = nil
   331  		return err
   332  	}
   333  
   334  	f, err := os.OpenFile(w.path, fileFlag, fileMode)
   335  	if err != nil {
   336  		w.file = nil
   337  		w.buffer = nil
   338  		return err
   339  	}
   340  
   341  	w.file = f
   342  	w.buffer.Reset(f)
   343  	return nil
   344  }
   345  
   346  // A TimedRotatingFileWriter is a buffered file writer which will rotate by time.
   347  // Its rotateDuration can be either RotateByDate or RotateByHour.
   348  // It keeps at most backupCount backups.
   349  type TimedRotatingFileWriter struct {
   350  	BufferedFileWriter
   351  	pathPrefix     string
   352  	rotateDuration RotateDuration
   353  	backupCount    uint8
   354  }
   355  
   356  // NewTimedRotatingFileWriter creates a new TimedRotatingFileWriter.
   357  func NewTimedRotatingFileWriter(pathPrefix string, rotateDuration RotateDuration, backupCount uint8, options ...BufferedFileWriterOption) (*TimedRotatingFileWriter, error) {
   358  	if backupCount == 0 {
   359  		return nil, errors.New("backupCount cannot be 0")
   360  	}
   361  
   362  	f, err := openTimedRotatingFile(pathPrefix, rotateDuration)
   363  	if err != nil {
   364  		return nil, err
   365  	}
   366  
   367  	w := TimedRotatingFileWriter{
   368  		BufferedFileWriter: BufferedFileWriter{
   369  			bufferedFileWriter: bufferedFileWriter{
   370  				file:       f,
   371  				bufferSize: defaultBufferSize,
   372  			},
   373  			updateChan: make(chan struct{}, 1),
   374  			stopChan:   make(chan struct{}),
   375  		},
   376  		pathPrefix:     pathPrefix,
   377  		rotateDuration: rotateDuration,
   378  		backupCount:    backupCount,
   379  	}
   380  
   381  	for _, option := range options {
   382  		option(&w.bufferedFileWriter)
   383  	}
   384  	w.buffer = bufio.NewWriterSize(f, int(w.bufferSize))
   385  
   386  	go w.schedule()
   387  	return &w, nil
   388  }
   389  
   390  func (w *TimedRotatingFileWriter) schedule() {
   391  	lock := &w.lock
   392  	flushTimer := time.NewTimer(0)
   393  	duration := nextRotateDuration(w.rotateDuration)
   394  	rotateTimer := time.NewTimer(duration)
   395  
   396  	for {
   397  	updateLoop:
   398  		for {
   399  			select {
   400  			case <-w.updateChan:
   401  				stopTimer(flushTimer)
   402  				flushTimer.Reset(flushDuration)
   403  				break updateLoop
   404  			case <-rotateTimer.C:
   405  				err := w.rotate(rotateTimer)
   406  				if err != nil {
   407  					logError(err)
   408  				}
   409  			case <-w.stopChan:
   410  				stopTimer(flushTimer)
   411  				stopTimer(rotateTimer)
   412  				return
   413  			}
   414  		}
   415  
   416  	flushLoop:
   417  		for {
   418  			select {
   419  			case <-flushTimer.C:
   420  				lock.Lock()
   421  				var err error
   422  				if w.file != nil { // not closed
   423  					w.updated = false
   424  					err = w.buffer.Flush()
   425  				}
   426  				lock.Unlock()
   427  				if err != nil {
   428  					logError(err)
   429  				}
   430  				break flushLoop
   431  			case <-rotateTimer.C:
   432  				err := w.rotate(rotateTimer)
   433  				if err != nil {
   434  					logError(err)
   435  				}
   436  			case <-w.stopChan:
   437  				stopTimer(flushTimer)
   438  				stopTimer(rotateTimer)
   439  				return
   440  			}
   441  		}
   442  	}
   443  }
   444  
   445  // rotate rotates the log file.
   446  func (w *TimedRotatingFileWriter) rotate(timer *time.Timer) error {
   447  	w.lock.Lock()
   448  	if w.file == nil { // was closed
   449  		w.lock.Unlock()
   450  		return nil // usually happens when program exits, should be ignored
   451  	}
   452  
   453  	err := w.buffer.Flush()
   454  	if err != nil {
   455  		w.lock.Unlock()
   456  		return err
   457  	}
   458  
   459  	err = w.file.Close()
   460  	if err != nil {
   461  		w.lock.Unlock()
   462  		return err
   463  	}
   464  
   465  	f, err := openTimedRotatingFile(w.pathPrefix, w.rotateDuration)
   466  	if err != nil {
   467  		w.buffer = nil
   468  		w.file = nil
   469  		w.lock.Unlock()
   470  		return err
   471  	}
   472  
   473  	w.file = f
   474  	w.buffer.Reset(f)
   475  
   476  	duration := nextRotateDuration(w.rotateDuration)
   477  	timer.Reset(duration)
   478  	w.lock.Unlock()
   479  
   480  	go w.purge()
   481  	return nil
   482  }
   483  
   484  // purge removes the outdated backups.
   485  func (w *TimedRotatingFileWriter) purge() {
   486  	pathes, err := filepath.Glob(w.pathPrefix + "*")
   487  	if err != nil {
   488  		logError(err)
   489  		return
   490  	}
   491  
   492  	count := len(pathes) - int(w.backupCount) - 1
   493  	if count > 0 {
   494  		var name string
   495  		w.lock.Lock()
   496  		if w.file != nil { // not closed
   497  			name = w.file.Name()
   498  		}
   499  		w.lock.Unlock()
   500  		sort.Strings(pathes)
   501  		for i := 0; i < count; i++ {
   502  			path := pathes[i]
   503  			if path != name {
   504  				err = os.Remove(path)
   505  				if err != nil {
   506  					logError(err)
   507  				}
   508  			}
   509  		}
   510  	}
   511  }
   512  
   513  // openTimedRotatingFile opens a log file for TimedRotatingFileWriter
   514  func openTimedRotatingFile(path string, rotateDuration RotateDuration) (*os.File, error) {
   515  	var pathSuffix string
   516  	t := now()
   517  	switch rotateDuration {
   518  	case RotateByDate:
   519  		pathSuffix = t.Format(rotateByDateFormat)
   520  	case RotateByHour:
   521  		pathSuffix = t.Format(rotateByHourFormat)
   522  	default:
   523  		return nil, errors.New("invalid rotateDuration")
   524  	}
   525  
   526  	return os.OpenFile(path+pathSuffix, fileFlag, fileMode)
   527  }
   528  
   529  // nextRotateDuration returns the next rotate duration for the rotateTimer.
   530  // It is defined as a variable in order to mock it in the unit testing.
   531  var nextRotateDuration = func(rotateDuration RotateDuration) time.Duration {
   532  	now := now()
   533  	var nextTime time.Time
   534  	if rotateDuration == RotateByDate {
   535  		nextTime = time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())
   536  	} else {
   537  		nextTime = time.Date(now.Year(), now.Month(), now.Day(), now.Hour()+1, 0, 0, 0, now.Location())
   538  	}
   539  	return nextTime.Sub(now)
   540  }
   541  
   542  type ConcurrentFileWriter struct {
   543  	bufferedFileWriter
   544  	cpuCount    int
   545  	locks       []sync.Mutex
   546  	buffers     []*bytes.Buffer
   547  	stopChan    chan struct{}
   548  	stoppedChan chan struct{}
   549  }
   550  
   551  // NewBufferedFileWriter creates a new BufferedFileWriter.
   552  func NewConcurrentFileWriter(path string, options ...BufferedFileWriterOption) (*ConcurrentFileWriter, error) {
   553  	f, err := os.OpenFile(path, fileFlag, fileMode)
   554  	if err != nil {
   555  		return nil, err
   556  	}
   557  
   558  	cpuCount := runtime.GOMAXPROCS(0)
   559  
   560  	w := &ConcurrentFileWriter{
   561  		bufferedFileWriter: bufferedFileWriter{
   562  			file:       f,
   563  			bufferSize: defaultBufferSize,
   564  		},
   565  		cpuCount:    cpuCount,
   566  		locks:       make([]sync.Mutex, cpuCount),
   567  		buffers:     make([]*bytes.Buffer, cpuCount),
   568  		stopChan:    make(chan struct{}),
   569  		stoppedChan: make(chan struct{}, 1),
   570  	}
   571  
   572  	for _, option := range options {
   573  		option(&w.bufferedFileWriter)
   574  	}
   575  
   576  	w.buffer = bufio.NewWriterSize(f, int(w.bufferSize))
   577  	for i := 0; i < cpuCount; i++ {
   578  		w.buffers[i] = bytes.NewBuffer(make([]byte, 0, w.bufferSize))
   579  	}
   580  
   581  	go w.schedule()
   582  	return w, nil
   583  }
   584  
   585  func (w *ConcurrentFileWriter) schedule() {
   586  	timer := time.NewTimer(flushDuration)
   587  	for {
   588  		select {
   589  		case <-timer.C:
   590  			for shard := 0; shard < w.cpuCount; shard++ {
   591  				w.locks[shard].Lock()
   592  				buffer := w.buffers[shard]
   593  				if buffer.Len() > 0 {
   594  					w.buffer.Write(buffer.Bytes())
   595  					buffer.Reset()
   596  				}
   597  				w.locks[shard].Unlock()
   598  			}
   599  
   600  			if w.buffer.Buffered() > 0 {
   601  				err := w.buffer.Flush()
   602  				if err != nil {
   603  					logError(err)
   604  				}
   605  			}
   606  
   607  			timer.Reset(flushDuration)
   608  		case <-w.stopChan:
   609  			stopTimer(timer)
   610  			w.stoppedChan <- struct{}{}
   611  			return
   612  		}
   613  	}
   614  }
   615  
   616  // Write writes a byte slice to the buffer.
   617  func (w *ConcurrentFileWriter) Write(p []byte) (n int, err error) {
   618  	shard := runtime_procPin()
   619  	runtime_procUnpin() // can't hold the lock for long
   620  
   621  	w.locks[shard].Lock()
   622  	n, err = w.buffers[shard].Write(p)
   623  	w.locks[shard].Unlock()
   624  	return
   625  }
   626  
   627  // Close flushes the buffer, then closes the file writer.
   628  func (w *ConcurrentFileWriter) Close() (err error) {
   629  	close(w.stopChan) // stops schedule()
   630  	<-w.stoppedChan   // waits for schedule() to finish, so the rest code can run without locks
   631  
   632  	for shard := 0; shard < w.cpuCount; shard++ {
   633  		buffer := w.buffers[shard]
   634  		if buffer.Len() > 0 {
   635  			w.buffer.Write(buffer.Bytes())
   636  			buffer.Reset()
   637  		}
   638  	}
   639  
   640  	if w.buffer.Buffered() > 0 {
   641  		err = w.buffer.Flush()
   642  	}
   643  	if err == nil {
   644  		err = w.file.Close()
   645  	} else {
   646  		e := w.file.Close()
   647  		if e != nil {
   648  			logError(e)
   649  		}
   650  	}
   651  	w.file = nil
   652  	return err
   653  }