github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/fsutil/file_allocator.go (about)

     1  //  Copyright 2022 PingCAP, Inc.
     2  //
     3  //  Licensed under the Apache License, Version 2.0 (the "License");
     4  //  you may not use this file except in compliance with the License.
     5  //  You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  //  Unless required by applicable law or agreed to in writing, software
    10  //  distributed under the License is distributed on an "AS IS" BASIS,
    11  //  See the License for the specific language governing permissions and
    12  //  limitations under the License.
    13  
    14  package fsutil
    15  
    16  import (
    17  	"fmt"
    18  	"os"
    19  	"path/filepath"
    20  	"sync"
    21  
    22  	"github.com/pingcap/tiflow/pkg/redo"
    23  )
    24  
    25  // FileAllocator has two functionalities:
    26  //  1. create new file or reuse the existing file (the existing tmp file will be cleared)
    27  //     beforehand for file write.
    28  //  2. pre-allocate disk space to mitigate the overhead of file metadata updating.
    29  //
    30  // ref: https://github.com/etcd-io/etcd/pull/4785
    31  type FileAllocator struct {
    32  	dir    string
    33  	prefix string
    34  	size   int64
    35  	count  int
    36  	wg     sync.WaitGroup
    37  	// fileCh is unbuffered because we want only one file to be written next,
    38  	// and another file to be standby.
    39  	fileCh chan *os.File
    40  	doneCh chan struct{}
    41  	errCh  chan error
    42  }
    43  
    44  // NewFileAllocator creates a file allocator and starts a background file allocation goroutine.
    45  func NewFileAllocator(dir string, prefix string, size int64) *FileAllocator {
    46  	allocator := &FileAllocator{
    47  		dir:    dir,
    48  		prefix: prefix,
    49  		size:   size,
    50  		fileCh: make(chan *os.File),
    51  		errCh:  make(chan error, 1),
    52  		doneCh: make(chan struct{}),
    53  	}
    54  
    55  	allocator.wg.Add(1)
    56  	go func() {
    57  		defer allocator.wg.Done()
    58  		allocator.run()
    59  	}()
    60  
    61  	return allocator
    62  }
    63  
    64  // Open returns a file for writing, this tmp file needs to be renamed after calling Open().
    65  func (fl *FileAllocator) Open() (f *os.File, err error) {
    66  	select {
    67  	case f = <-fl.fileCh:
    68  	case err = <-fl.errCh:
    69  	}
    70  
    71  	return f, err
    72  }
    73  
    74  // Close closes the doneCh to notify the background goroutine to exit.
    75  func (fl *FileAllocator) Close() error {
    76  	close(fl.doneCh)
    77  	fl.wg.Wait()
    78  	return <-fl.errCh
    79  }
    80  
    81  func (fl *FileAllocator) alloc() (f *os.File, err error) {
    82  	if _, err := os.Stat(fl.dir); err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	filePath := filepath.Join(fl.dir, fmt.Sprintf("%s_%d.tmp", fl.prefix, fl.count%2))
    87  	f, err = os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, redo.DefaultFileMode)
    88  	if err != nil {
    89  		return nil, err
    90  	}
    91  	err = PreAllocate(f, fl.size)
    92  	if err != nil {
    93  		return nil, err
    94  	}
    95  	fl.count++
    96  	return f, nil
    97  }
    98  
    99  func (fl *FileAllocator) run() {
   100  	defer close(fl.errCh)
   101  	for {
   102  		f, err := fl.alloc()
   103  		if err != nil {
   104  			fl.errCh <- err
   105  			return
   106  		}
   107  
   108  		select {
   109  		case fl.fileCh <- f:
   110  		case <-fl.doneCh:
   111  			os.Remove(f.Name())
   112  			f.Close()
   113  			return
   114  		}
   115  	}
   116  }