github.com/scottcagno/storage@v1.8.0/pkg/lsmtree/sstable-manager.go (about)

     1  package lsmtree
     2  
     3  import (
     4  	"os"
     5  	"path/filepath"
     6  	"strings"
     7  )
     8  
     9  type indexKey struct {
    10  	level int
    11  	key   string
    12  }
    13  
    14  type indexVal struct {
    15  	path   string
    16  	offset int64
    17  }
    18  
    19  type ssTableManager struct {
    20  	baseDir  string
    21  	index    map[indexKey]indexVal
    22  	level    map[int]int
    23  	sstcount int
    24  }
    25  
    26  func openSSTableManager(base string) (*ssTableManager, error) {
    27  	// sanitize base path
    28  	path, err := initBasePath(base)
    29  	if err != nil {
    30  		return nil, err
    31  	}
    32  	// create new ss-table-manager to return
    33  	sstm := &ssTableManager{
    34  		baseDir: path,
    35  		index:   make(map[indexKey]indexVal),
    36  		level:   make(map[int]int),
    37  	}
    38  	// initialize
    39  	err = sstm.load()
    40  	if err != nil {
    41  		return nil, err
    42  	}
    43  	return sstm, nil
    44  }
    45  
    46  func (sstm *ssTableManager) load() error {
    47  	// read the base dir for this level
    48  	dirs, err := os.ReadDir(sstm.baseDir)
    49  	if err != nil {
    50  		return err
    51  	}
    52  	// iterate dirs
    53  	for _, dir := range dirs {
    54  		// skip anything that is not a directory
    55  		if !dir.IsDir() {
    56  			continue
    57  		}
    58  		// get level
    59  		level, err := dirToLevel(dir.Name())
    60  		if err != nil {
    61  			return err
    62  		}
    63  		// add level to levels
    64  		if _, ok := sstm.level[level]; !ok {
    65  			sstm.level[level] = 0
    66  		}
    67  		// now let us add the file count within those levels
    68  		files, err := os.ReadDir(dir.Name())
    69  		if err != nil {
    70  			return err
    71  		}
    72  		// count the files
    73  		for _, file := range files {
    74  			// if the file is a sst-table data file, increment
    75  			if !file.IsDir() && strings.HasSuffix(file.Name(), dataFileSuffix) {
    76  				sstm.level[level]++
    77  				sstm.sstcount++
    78  			}
    79  		}
    80  	}
    81  	return nil
    82  }
    83  
    84  // createSSAndIndexTables creates a new ss-table and ss-table-index using
    85  // the provided entry batch, and returns nil on success.
    86  func (sstm *ssTableManager) createSSAndIndexTables(memt *rbTree) error {
    87  	// create level-0 path for newly flushed ss-tables
    88  	path := filepath.Join(sstm.baseDir, levelToDir(0))
    89  	// read the base dir for this level
    90  	files, err := os.ReadDir(path)
    91  	if err != nil {
    92  		return err
    93  	}
    94  	// init seq
    95  	var seq int64
    96  	// count the files to get the sequence number
    97  	for _, file := range files {
    98  		// if the file is a sst-table data file, increment
    99  		if !file.IsDir() && strings.HasSuffix(file.Name(), dataFileSuffix) {
   100  			seq++
   101  		}
   102  	}
   103  	// get data file name
   104  	dataFileName := filepath.Join(path, toDataFileName(seq))
   105  	// open data file
   106  	dataFile, err := os.OpenFile(dataFileName, os.O_CREATE|os.O_RDWR, 0666)
   107  	if err != nil {
   108  		return err
   109  	}
   110  	// remember to close
   111  	defer func(dataFile *os.File) {
   112  		err := dataFile.Close()
   113  		if err != nil {
   114  			panic("closing dataFile: " + err.Error())
   115  		}
   116  	}(dataFile)
   117  	// get index file name
   118  	indexFileName := filepath.Join(path, toIndexFileName(seq))
   119  	// open index file
   120  	indexFile, err := os.OpenFile(indexFileName, os.O_CREATE|os.O_RDWR, 0666)
   121  	if err != nil {
   122  		return err
   123  	}
   124  	// remember to close
   125  	defer func(indexFile *os.File) {
   126  		err := indexFile.Close()
   127  		if err != nil {
   128  			panic("closing indexFile: " + err.Error())
   129  		}
   130  	}(indexFile)
   131  	// range mem-table and write entries and indexes
   132  	memt.rangeFront(func(e *Entry) bool {
   133  		// write entry to data file
   134  		offset, err := writeEntry(dataFile, e)
   135  		if err != nil {
   136  			// for now, just panic
   137  			panic(err)
   138  		}
   139  		// write index to index file
   140  		_, err = writeIndex(indexFile, &Index{
   141  			Key:    e.Key,
   142  			Offset: offset,
   143  		})
   144  		if err != nil {
   145  			// for now, just panic
   146  			panic(err)
   147  		}
   148  		return true
   149  	})
   150  	// sync data file
   151  	err = dataFile.Sync()
   152  	if err != nil {
   153  		return err
   154  	}
   155  	// sync index file
   156  	err = indexFile.Sync()
   157  	if err != nil {
   158  		return err
   159  	}
   160  	return nil
   161  }
   162  
   163  func (sstm *ssTableManager) get(e *Entry) (*Entry, error) {
   164  	return nil, nil
   165  }