github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/internal/pacertoy/pebble/main.go

// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package main

import (
	"context"
	"fmt"
	"math"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/petermattis/pebble/internal/rate"

	"golang.org/x/exp/rand"
)

const (
	// If measureLatency is true, the simulator outputs p50, p95, and p99
	// latencies after all writes are completed. In this mode, all writes
	// are immediately queued. If this is disabled, writes come in continually
	// at different rates.
	measureLatency        = false

	writeAmount           = 2000 << 20 // 2 GB

	// Max rate for all compactions. This is intentionally set low enough that
	// user writes will have to be delayed.
	maxCompactionRate     = 100 << 20 // 100 MB/s
	minCompactionRate     = 20 << 20 // 20 MB/s

	memtableSize          = 64 << 20 // 64 MB
	maxMemtableCount      = 5
	drainDelayThreshold   = 1.05 * memtableSize
	maxFlushRate          = 30 << 20 // 30 MB/s
	minFlushRate          = 4 << 20 // 4 MB/s

	l0CompactionThreshold = 1

	levelRatio            = 10
	numLevels             = 7

	compactionDebtSlowdownThreshold = 2 * memtableSize
)

type compactionPacer struct {
	level      int64
	maxDrainer *rate.Limiter
	minDrainer *rate.Limiter
}

func newCompactionPacer() *compactionPacer {
	p := &compactionPacer{
		maxDrainer: rate.NewLimiter(maxCompactionRate, maxCompactionRate),
		minDrainer: rate.NewLimiter(minCompactionRate, minCompactionRate),
	}
	return p
}

func (p *compactionPacer) fill(n int64) {
	atomic.AddInt64(&p.level, n)
}

func (p *compactionPacer) drain(n int64, delay bool) bool {
	p.maxDrainer.WaitN(context.Background(), int(n))

	if delay {
		p.minDrainer.WaitN(context.Background(), int(n))
	}
	level := atomic.AddInt64(&p.level, -n)
	return level <= compactionDebtSlowdownThreshold
}

type flushPacer struct {
	level             int64
	drainDelayLevel   float64
	fillCond          sync.Cond
	// minDrainer is the drainer which sets the minimum speed of draining.
	minDrainer        *rate.Limiter
	// maxDrainer is the drainer which sets the maximum speed of draining.
	maxDrainer        *rate.Limiter
}

func newFlushPacer(mu *sync.Mutex) *flushPacer {
	p := &flushPacer{
		drainDelayLevel: drainDelayThreshold,
		minDrainer:      rate.NewLimiter(minFlushRate, minFlushRate),
		maxDrainer:      rate.NewLimiter(maxFlushRate, maxFlushRate),
	}
	p.fillCond.L = mu
	return p
}

func (p *flushPacer) fill(n int64) {
	atomic.AddInt64(&p.level, n)
	p.fillCond.Signal()
}

func (p *flushPacer) drain(n int64, delay bool) bool {
	p.maxDrainer.WaitN(context.Background(), int(n))

	if delay {
		p.minDrainer.WaitN(context.Background(), int(n))
	}
	level := atomic.AddInt64(&p.level, -n)
	p.fillCond.Signal()
	return float64(level) <= p.drainDelayLevel
}

type DB struct {
	mu         sync.Mutex
	flushPacer *flushPacer
	flushCond  sync.Cond
	memtables  []*int64
	fill       int64
	drain      int64

	compactionMu        sync.Mutex
	compactionPacer     *compactionPacer
	// L0 is represented as an array of integers whereas every other level
	// is represented as a single integer.
	L0                  []*int64
	// Non-L0 sstables. sstables[0] == L1.
	sstables            []int64
	maxSSTableSizes     []int64
	compactionFlushCond sync.Cond
	prevCompactionDebt  float64
}

func newDB() *DB {
	db := &DB{}
	db.flushPacer = newFlushPacer(&db.mu)
	db.flushCond.L = &db.mu
	db.memtables = append(db.memtables, new(int64))

	db.compactionFlushCond.L = &db.compactionMu
	db.L0 = append(db.L0, new(int64))
	db.compactionPacer = newCompactionPacer()

	db.maxSSTableSizes = make([]int64, numLevels-1)
	base := int64(levelRatio)
	for i := uint64(0); i < numLevels-2; i++ {
		// Each level is 10 times larger than the one above it.
		db.maxSSTableSizes[i] = memtableSize * l0CompactionThreshold * base
		base *= levelRatio

		// Begin with each level full.
		newLevel := db.maxSSTableSizes[i]

		db.sstables = append(db.sstables, newLevel)
	}
	db.sstables = append(db.sstables, 0)
	db.maxSSTableSizes[numLevels-2] = math.MaxInt64

	go db.drainMemtable()
	go db.drainCompaction()

	return db
}

// drainCompaction simulates background compactions.
func (db *DB) drainCompaction() {
	rng := rand.New(rand.NewSource(1))

	for {
		db.compactionMu.Lock()

		for len(db.L0) <= l0CompactionThreshold {
			db.compactionFlushCond.Wait()
		}
		l0Table := db.L0[0]
		db.compactionMu.Unlock()

		var delay bool
		for i, size := int64(0), int64(0); i < *l0Table; i += size {
			size = 10000 + rng.Int63n(500)
			if size > (*l0Table - i) {
				size = *l0Table - i
			}
			delay = db.compactionPacer.drain(size, delay)
		}

		db.compactionMu.Lock()
		db.L0 = db.L0[1:]
		db.compactionMu.Unlock()

		singleTableSize := int64(memtableSize)
		tablesToCompact := 0
		for i := range db.sstables {
			newSSTableSize := atomic.AddInt64(&db.sstables[i], singleTableSize)
			if newSSTableSize > db.maxSSTableSizes[i] {
				atomic.AddInt64(&db.sstables[i], -singleTableSize)
				tablesToCompact++
			} else {
				// Lower levels do not need compaction if level above it did not
				// need compaction.
				break
			}
		}

		totalCompactionBytes := int64(tablesToCompact * memtableSize)

		for t := 0; t < tablesToCompact; t++ {
			db.compactionPacer.fill(memtableSize)
			for i, size := int64(0), int64(0); i < memtableSize; i += size {
				size = 10000 + rng.Int63n(500)
				if size > (totalCompactionBytes - i) {
					size = totalCompactionBytes - i
				}
				delay = db.compactionPacer.drain(size, delay)
			}

			db.delayMemtableDrain()
		}
	}
}

// fillCompaction fills L0 sstables.
func (db *DB) fillCompaction(size int64) {
	db.compactionMu.Lock()

	db.compactionPacer.fill(size)

	last := db.L0[len(db.L0)-1]
	if *last+size > memtableSize {
		last = new(int64)
		db.L0 = append(db.L0, last)
		db.compactionFlushCond.Signal()
	}
	*last += size

	db.compactionMu.Unlock()
}

// drainMemtable simulates memtable flushing.
func (db *DB) drainMemtable() {
	rng := rand.New(rand.NewSource(2))

	for {
		db.mu.Lock()
		for len(db.memtables) <= 1 {
			db.flushCond.Wait()
		}
		memtable := db.memtables[0]
		db.mu.Unlock()

		var delay bool
		for i, size := int64(0), int64(0); i < *memtable; i += size {
			size = 1000 + rng.Int63n(50)
			if size > (*memtable - i) {
				size = *memtable - i
			}
			delay = db.flushPacer.drain(size, delay)
			atomic.AddInt64(&db.drain, size)

			db.fillCompaction(size)
		}

		db.delayMemtableDrain()

		db.mu.Lock()
		db.memtables = db.memtables[1:]
		db.mu.Unlock()
	}
}

// delayMemtableDrain applies memtable drain delays depending on compaction debt.
func (db *DB) delayMemtableDrain() {
	totalCompactionBytes := atomic.LoadInt64(&db.compactionPacer.level)
	compactionDebt := math.Max(float64(totalCompactionBytes)-l0CompactionThreshold*memtableSize, 0.0)

	db.mu.Lock()
	if compactionDebt > compactionDebtSlowdownThreshold {
		// Compaction debt is above the threshold and the debt is growing. Throttle memtable flushing.
		drainLimit := maxFlushRate * rate.Limit(compactionDebtSlowdownThreshold/compactionDebt)
		if drainLimit > 0 && drainLimit <= maxFlushRate {
			db.flushPacer.maxDrainer.SetLimit(drainLimit)
		}
	} else {
		// Continuously speed up memtable flushing to make sure that slowdown signal did not
		// decrease the memtable flush rate by too much.
		drainLimit := db.flushPacer.maxDrainer.Limit() * 1.05
		if drainLimit > 0 && drainLimit <= maxFlushRate {
			db.flushPacer.maxDrainer.SetLimit(drainLimit)
		}
	}

	db.prevCompactionDebt = compactionDebt
	db.mu.Unlock()
}

// fillMemtable simulates memtable filling.
func (db *DB) fillMemtable(size int64) {
	db.mu.Lock()

	for len(db.memtables) > maxMemtableCount {
		db.flushPacer.fillCond.Wait()
	}
	db.flushPacer.fill(size)
	atomic.AddInt64(&db.fill, size)

	last := db.memtables[len(db.memtables)-1]
	if *last+size > memtableSize {
		last = new(int64)
		db.memtables = append(db.memtables, last)
		db.flushCond.Signal()
	}
	*last += size

	db.mu.Unlock()
}

// printLevels prints the levels.
func (db *DB) printLevels() {
	db.mu.Lock()
	for i := range db.sstables {
		fmt.Printf("Level %d: %d/%d\n", i+1, db.sstables[i]/(1024*1024), db.maxSSTableSizes[i]/(1024*1024))
	}
	db.mu.Unlock()
}

// simulateWrite simulates user writes.
func simulateWrite(db *DB, measureLatencyMode bool) {
	limiter := rate.NewLimiter(10<<20, 10<<20) // 10 MB/s
	fmt.Printf("filling at 10 MB/sec\n")

	setRate := func(mb int) {
		fmt.Printf("filling at %d MB/sec\n", mb)
		limiter.SetLimit(rate.Limit(mb << 20))
	}

	if !measureLatencyMode {
		go func() {
			rng := rand.New(rand.NewSource(3))
			for {
				secs := 5 + rng.Intn(5)
				time.Sleep(time.Duration(secs) * time.Second)
				mb := 10 + rng.Intn(20)
				setRate(mb)
			}
		}()
	}

	rng := rand.New(rand.NewSource(uint64(4)))

	totalWrites := int64(0)
	percentiles := []int64{50, 95, 99}
	percentileIndex := 0
	percentileTimes := make([]time.Time, 0)

	startTime := time.Now()
	for totalWrites <= writeAmount {
		size := 1000 + rng.Int63n(50)
		if !measureLatencyMode {
			limiter.WaitN(context.Background(), int(size))
		}
		db.fillMemtable(size)

		// Calculate latency percentiles
		totalWrites += size
		if percentileIndex < len(percentiles) && totalWrites > (percentiles[percentileIndex] * writeAmount / 100) {
			percentileTimes = append(percentileTimes, time.Now())
			percentileIndex += 1
		}
	}

	time.Sleep(time.Second * 10)
	// Latency should only be measured when `limiter.WaitN` is removed.
	if measureLatencyMode {
		fmt.Printf("_____p50______p95______p99\n")
		fmt.Printf("%8s %8s %8s\n",
			time.Duration(percentileTimes[0].Sub(startTime).Seconds())*time.Second,
			time.Duration(percentileTimes[1].Sub(startTime).Seconds())*time.Second,
			time.Duration(percentileTimes[2].Sub(startTime).Seconds())*time.Second)
	}

	os.Exit(0)
}

func main() {
	db := newDB()

	go simulateWrite(db, measureLatency)

	tick := time.NewTicker(time.Second)
	start := time.Now()
	lastNow := start
	var lastFill, lastDrain int64

	for i := 0; ; i++ {
		select {
		case <-tick.C:
			if (i % 20) == 0 {
				fmt.Printf("_elapsed___memtbs____dirty_____fill____drain____cdebt__l0count___max-f-rate\n")
			}

			if (i % 7) == 0 {
				//db.printLevels()
			}

			db.mu.Lock()
			memtableCount := len(db.memtables)
			db.mu.Unlock()
			dirty := atomic.LoadInt64(&db.flushPacer.level)
			fill := atomic.LoadInt64(&db.fill)
			drain := atomic.LoadInt64(&db.drain)

			db.compactionMu.Lock()
			compactionL0 := len(db.L0)
			db.compactionMu.Unlock()
			totalCompactionBytes := atomic.LoadInt64(&db.compactionPacer.level)
			compactionDebt := math.Max(float64(totalCompactionBytes)-l0CompactionThreshold*memtableSize, 0.0)
			maxFlushRate := db.flushPacer.maxDrainer.Limit()

			now := time.Now()
			elapsed := now.Sub(lastNow).Seconds()
			fmt.Printf("%8s %8d %8.1f %8.1f %8.1f %8.1f %8d %12.1f\n",
				time.Duration(now.Sub(start).Seconds()+0.5)*time.Second,
				memtableCount,
				float64(dirty)/(1024.0*1024.0),
				float64(fill-lastFill)/(1024.0*1024.0*elapsed),
				float64(drain-lastDrain)/(1024.0*1024.0*elapsed),
				compactionDebt/(1024.0*1024.0),
				compactionL0,
				maxFlushRate/(1024.0*1024.0))

			lastNow = now
			lastFill = fill
			lastDrain = drain
		}
	}
}