vitess.io/vitess@v0.16.2/go/vt/vtgate/buffer/buffer.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package buffer provides a buffer for PRIMARY traffic during failovers.
    18  //
    19  // Instead of returning an error to the application (when the vttablet primary
    20  // becomes unavailable), the buffer will automatically retry buffered requests
    21  // after the end of the failover was detected.
    22  //
    23  // Buffering (stalling) requests will increase the number of requests in flight
    24  // within vtgate and at upstream layers. Therefore, it is important to limit
    25  // the size of the buffer and the buffering duration (window) per request.
    26  // See the file flags.go for the available configuration and its defaults.
    27  package buffer
    28  
    29  import (
    30  	"context"
    31  	"fmt"
    32  	"sync"
    33  
    34  	"vitess.io/vitess/go/sync2"
    35  	"vitess.io/vitess/go/vt/discovery"
    36  	"vitess.io/vitess/go/vt/log"
    37  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    38  	"vitess.io/vitess/go/vt/topo/topoproto"
    39  	"vitess.io/vitess/go/vt/vterrors"
    40  
    41  	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
    42  )
    43  
    44  var (
    45  	ShardMissingError    = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "destination shard is missing after a resharding operation")
    46  	bufferFullError      = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "primary buffer is full")
    47  	entryEvictedError    = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "buffer full: request evicted for newer request")
    48  	contextCanceledError = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "context was canceled before failover finished")
    49  )
    50  
    51  // bufferMode specifies how the buffer is configured for a given shard.
    52  type bufferMode int
    53  
    54  const (
    55  	// bufferModeDisabled will let all requests pass through and do nothing.
    56  	bufferModeDisabled bufferMode = iota
    57  	// bufferModeEnabled means all requests should be buffered.
    58  	bufferModeEnabled
    59  	// bufferModeDryRun will track the failover, but not actually buffer requests.
    60  	bufferModeDryRun
    61  )
    62  
    63  // RetryDoneFunc will be returned for each buffered request and must be called
    64  // after the buffered request was retried.
    65  // Without this signal, the buffer would not know how many buffered requests are
    66  // currently retried.
    67  type RetryDoneFunc context.CancelFunc
    68  
    69  // CausedByFailover returns true if "err" was supposedly caused by a failover.
    70  // To simplify things, we've merged the detection for different MySQL flavors
    71  // in one function. Supported flavors: MariaDB, MySQL
    72  func CausedByFailover(err error) bool {
    73  	log.V(2).Infof("Checking error (type: %T) if it is caused by a failover. err: %v", err, err)
    74  	return vterrors.Code(err) == vtrpcpb.Code_CLUSTER_EVENT
    75  }
    76  
    77  // Buffer is used to track ongoing PRIMARY tablet failovers and buffer
    78  // requests while the PRIMARY tablet is unavailable.
    79  // Once the new PRIMARY starts accepting requests, buffering stops and requests
    80  // queued so far will be automatically retried.
    81  //
    82  // There should be exactly one instance of this buffer. For each failover, an
    83  // instance of "ShardBuffer" will be created.
    84  type Buffer struct {
    85  	// Immutable configuration fields.
    86  	config *Config
    87  
    88  	// bufferSizeSema limits how many requests can be buffered
    89  	// ("-buffer_size") and is shared by all shardBuffer instances.
    90  	bufferSizeSema *sync2.Semaphore
    91  
    92  	// mu guards all fields in this group.
    93  	// In particular, it is used to serialize the following Go routines:
    94  	// - 1. Requests which may buffer (RLock, can be run in parallel)
    95  	// - 2. Request which starts buffering (based on the seen error)
    96  	// - 3. HealthCheck subscriber ("StatsUpdate") which stops buffering
    97  	// - 4. Timer which may stop buffering after -buffer_max_failover_duration
    98  	mu sync.RWMutex
    99  	// buffers holds a shardBuffer object per shard, even if no failover is in
   100  	// progress.
   101  	// Key Format: "<keyspace>/<shard>"
   102  	buffers map[string]*shardBuffer
   103  	// stopped is true after Shutdown() was run.
   104  	stopped bool
   105  }
   106  
   107  // New creates a new Buffer object.
   108  func New(cfg *Config) *Buffer {
   109  	return &Buffer{
   110  		config:         cfg,
   111  		bufferSizeSema: sync2.NewSemaphore(cfg.Size, 0),
   112  		buffers:        make(map[string]*shardBuffer),
   113  	}
   114  }
   115  
   116  // WaitForFailoverEnd blocks until a pending buffering due to a failover for
   117  // keyspace/shard is over.
   118  // If there is no ongoing failover, "err" is checked. If it's caused by a
   119  // failover, buffering may be started.
   120  // It returns an error if buffering failed (e.g. buffer full).
   121  // If it does not return an error, it may return a RetryDoneFunc which must be
   122  // called after the request was retried.
   123  func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, err error) (RetryDoneFunc, error) {
   124  	// If an err is given, it must be related to a failover.
   125  	// We never buffer requests with other errors.
   126  	if err != nil && !CausedByFailover(err) {
   127  		return nil, nil
   128  	}
   129  
   130  	sb := b.getOrCreateBuffer(keyspace, shard)
   131  	if sb == nil {
   132  		// Buffer is shut down. Ignore all calls.
   133  		requestsSkipped.Add([]string{keyspace, shard, skippedShutdown}, 1)
   134  		return nil, nil
   135  	}
   136  	if sb.disabled() {
   137  		requestsSkipped.Add([]string{keyspace, shard, skippedDisabled}, 1)
   138  		return nil, nil
   139  	}
   140  
   141  	return sb.waitForFailoverEnd(ctx, keyspace, shard, err)
   142  }
   143  
   144  // ProcessPrimaryHealth notifies the buffer to record a new primary
   145  // and end any failover buffering that may be in progress
   146  func (b *Buffer) ProcessPrimaryHealth(th *discovery.TabletHealth) {
   147  	if th.Target.TabletType != topodatapb.TabletType_PRIMARY {
   148  		panic(fmt.Sprintf("BUG: non-PRIMARY TabletHealth object must not be forwarded: %#v", th))
   149  	}
   150  	timestamp := th.PrimaryTermStartTime
   151  	if timestamp == 0 {
   152  		// Primarys where TabletExternallyReparented was never called will return 0.
   153  		// Ignore them.
   154  		return
   155  	}
   156  
   157  	sb := b.getOrCreateBuffer(th.Target.Keyspace, th.Target.Shard)
   158  	if sb == nil {
   159  		// Buffer is shut down. Ignore all calls.
   160  		return
   161  	}
   162  	sb.recordExternallyReparentedTimestamp(timestamp, th.Tablet.Alias)
   163  }
   164  
   165  func (b *Buffer) HandleKeyspaceEvent(ksevent *discovery.KeyspaceEvent) {
   166  	for _, shard := range ksevent.Shards {
   167  		sb := b.getOrCreateBuffer(shard.Target.Keyspace, shard.Target.Shard)
   168  		if sb != nil {
   169  			sb.recordKeyspaceEvent(shard.Tablet, shard.Serving)
   170  		}
   171  	}
   172  }
   173  
   174  // getOrCreateBuffer returns the ShardBuffer for the given keyspace and shard.
   175  // It returns nil if Buffer is shut down and all calls should be ignored.
   176  func (b *Buffer) getOrCreateBuffer(keyspace, shard string) *shardBuffer {
   177  	key := topoproto.KeyspaceShardString(keyspace, shard)
   178  	b.mu.RLock()
   179  	sb, ok := b.buffers[key]
   180  	stopped := b.stopped
   181  	b.mu.RUnlock()
   182  
   183  	if stopped {
   184  		return nil
   185  	}
   186  	if ok {
   187  		return sb
   188  	}
   189  
   190  	b.mu.Lock()
   191  	defer b.mu.Unlock()
   192  	// Look it up again because it could have been created in the meantime.
   193  	sb, ok = b.buffers[key]
   194  	if !ok {
   195  		sb = newShardBufferHealthCheck(b, b.config.bufferingMode(keyspace, shard), keyspace, shard)
   196  		b.buffers[key] = sb
   197  	}
   198  	return sb
   199  }
   200  
   201  // Shutdown blocks until all pending ShardBuffer objects are shut down.
   202  // In particular, it guarantees that all launched Go routines are stopped after
   203  // it returns.
   204  func (b *Buffer) Shutdown() {
   205  	b.shutdown()
   206  	b.waitForShutdown()
   207  }
   208  
   209  func (b *Buffer) shutdown() {
   210  	b.mu.Lock()
   211  	defer b.mu.Unlock()
   212  
   213  	for _, sb := range b.buffers {
   214  		sb.shutdown()
   215  	}
   216  	b.stopped = true
   217  }
   218  
   219  func (b *Buffer) waitForShutdown() {
   220  	b.mu.RLock()
   221  	defer b.mu.RUnlock()
   222  
   223  	for _, sb := range b.buffers {
   224  		sb.waitForShutdown()
   225  	}
   226  }