github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/syncing_write.go (about)

     1  // Copyright 2017 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package kvserver
    12  
    13  import (
    14  	"context"
    15  	"os"
    16  	"runtime/debug"
    17  	"strings"
    18  	"time"
    19  
    20  	"github.com/cockroachdb/cockroach/pkg/settings"
    21  	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
    22  	"github.com/cockroachdb/cockroach/pkg/storage"
    23  	"github.com/cockroachdb/cockroach/pkg/util/log"
    24  	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
    25  	"golang.org/x/time/rate"
    26  )
    27  
    28  // bulkIOWriteBurst is the burst for the BulkIOWriteLimiter.
    29  const bulkIOWriteBurst = 2 * 1024 * 1024 // 2MB
    30  
    31  const bulkIOWriteLimiterLongWait = 500 * time.Millisecond
    32  
    33  func limitBulkIOWrite(ctx context.Context, limiter *rate.Limiter, cost int) {
    34  	// The limiter disallows anything greater than its burst (set to
    35  	// BulkIOWriteLimiterBurst), so cap the batch size if it would overflow.
    36  	//
    37  	// TODO(dan): This obviously means the limiter is no longer accounting for the
    38  	// full cost. I've tried calling WaitN in a loop to fully cover the cost, but
    39  	// that didn't seem to be as smooth in practice (NB [dt]: that was when this
    40  	// limit was done before writing the whole file, rather than on individual
    41  	// chunks).
    42  	if cost > bulkIOWriteBurst {
    43  		cost = bulkIOWriteBurst
    44  	}
    45  
    46  	begin := timeutil.Now()
    47  	if err := limiter.WaitN(ctx, cost); err != nil {
    48  		log.Errorf(ctx, "error rate limiting bulk io write: %+v", err)
    49  	}
    50  
    51  	if d := timeutil.Since(begin); d > bulkIOWriteLimiterLongWait {
    52  		log.Warningf(ctx, "bulk io write limiter took %s (>%s):\n%s",
    53  			d, bulkIOWriteLimiterLongWait, debug.Stack())
    54  	}
    55  }
    56  
    57  // sstWriteSyncRate wraps "kv.bulk_sst.sync_size". 0 disables syncing.
    58  var sstWriteSyncRate = settings.RegisterByteSizeSetting(
    59  	"kv.bulk_sst.sync_size",
    60  	"threshold after which non-Rocks SST writes must fsync (0 disables)",
    61  	bulkIOWriteBurst,
    62  )
    63  
    64  // writeFileSyncing is essentially ioutil.WriteFile -- writes data to a file
    65  // named by filename -- but with rate limiting and periodic fsyncing controlled
    66  // by settings and the passed limiter (should be the store's limiter). Periodic
    67  // fsync provides smooths out disk IO, as mentioned in #20352 and #20279, and
    68  // provides back-pressure, along with the explicit rate limiting. If the file
    69  // does not exist, WriteFile creates it with permissions perm; otherwise
    70  // WriteFile truncates it before writing.
    71  func writeFileSyncing(
    72  	ctx context.Context,
    73  	filename string,
    74  	data []byte,
    75  	eng storage.Engine,
    76  	perm os.FileMode,
    77  	settings *cluster.Settings,
    78  	limiter *rate.Limiter,
    79  ) error {
    80  	chunkSize := sstWriteSyncRate.Get(&settings.SV)
    81  	sync := true
    82  	if chunkSize == 0 {
    83  		chunkSize = bulkIOWriteBurst
    84  		sync = false
    85  	}
    86  
    87  	f, err := eng.Create(filename)
    88  	if err != nil {
    89  		if strings.Contains(err.Error(), "No such file or directory") {
    90  			return os.ErrNotExist
    91  		}
    92  		return err
    93  	}
    94  
    95  	for i := int64(0); i < int64(len(data)); i += chunkSize {
    96  		end := i + chunkSize
    97  		if l := int64(len(data)); end > l {
    98  			end = l
    99  		}
   100  		chunk := data[i:end]
   101  
   102  		// rate limit
   103  		limitBulkIOWrite(ctx, limiter, len(chunk))
   104  		_, err = f.Write(chunk)
   105  		if err == nil && sync {
   106  			err = f.Sync()
   107  		}
   108  		if err != nil {
   109  			break
   110  		}
   111  	}
   112  
   113  	closeErr := f.Close()
   114  	if err == nil {
   115  		err = closeErr
   116  	}
   117  	return err
   118  }