vitess.io/vitess@v0.16.2/go/vt/vtgate/buffer/buffer.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package buffer provides a buffer for PRIMARY traffic during failovers. 18 // 19 // Instead of returning an error to the application (when the vttablet primary 20 // becomes unavailable), the buffer will automatically retry buffered requests 21 // after the end of the failover was detected. 22 // 23 // Buffering (stalling) requests will increase the number of requests in flight 24 // within vtgate and at upstream layers. Therefore, it is important to limit 25 // the size of the buffer and the buffering duration (window) per request. 26 // See the file flags.go for the available configuration and its defaults. 27 package buffer 28 29 import ( 30 "context" 31 "fmt" 32 "sync" 33 34 "vitess.io/vitess/go/sync2" 35 "vitess.io/vitess/go/vt/discovery" 36 "vitess.io/vitess/go/vt/log" 37 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 38 "vitess.io/vitess/go/vt/topo/topoproto" 39 "vitess.io/vitess/go/vt/vterrors" 40 41 vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" 42 ) 43 44 var ( 45 ShardMissingError = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "destination shard is missing after a resharding operation") 46 bufferFullError = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "primary buffer is full") 47 entryEvictedError = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "buffer full: request evicted for newer request") 48 contextCanceledError = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "context was canceled before failover finished") 49 ) 50 51 // bufferMode specifies how the buffer is configured for a given shard. 52 type bufferMode int 53 54 const ( 55 // bufferModeDisabled will let all requests pass through and do nothing. 56 bufferModeDisabled bufferMode = iota 57 // bufferModeEnabled means all requests should be buffered. 58 bufferModeEnabled 59 // bufferModeDryRun will track the failover, but not actually buffer requests. 60 bufferModeDryRun 61 ) 62 63 // RetryDoneFunc will be returned for each buffered request and must be called 64 // after the buffered request was retried. 65 // Without this signal, the buffer would not know how many buffered requests are 66 // currently retried. 67 type RetryDoneFunc context.CancelFunc 68 69 // CausedByFailover returns true if "err" was supposedly caused by a failover. 70 // To simplify things, we've merged the detection for different MySQL flavors 71 // in one function. Supported flavors: MariaDB, MySQL 72 func CausedByFailover(err error) bool { 73 log.V(2).Infof("Checking error (type: %T) if it is caused by a failover. err: %v", err, err) 74 return vterrors.Code(err) == vtrpcpb.Code_CLUSTER_EVENT 75 } 76 77 // Buffer is used to track ongoing PRIMARY tablet failovers and buffer 78 // requests while the PRIMARY tablet is unavailable. 79 // Once the new PRIMARY starts accepting requests, buffering stops and requests 80 // queued so far will be automatically retried. 81 // 82 // There should be exactly one instance of this buffer. For each failover, an 83 // instance of "ShardBuffer" will be created. 84 type Buffer struct { 85 // Immutable configuration fields. 86 config *Config 87 88 // bufferSizeSema limits how many requests can be buffered 89 // ("-buffer_size") and is shared by all shardBuffer instances. 90 bufferSizeSema *sync2.Semaphore 91 92 // mu guards all fields in this group. 93 // In particular, it is used to serialize the following Go routines: 94 // - 1. Requests which may buffer (RLock, can be run in parallel) 95 // - 2. Request which starts buffering (based on the seen error) 96 // - 3. HealthCheck subscriber ("StatsUpdate") which stops buffering 97 // - 4. Timer which may stop buffering after -buffer_max_failover_duration 98 mu sync.RWMutex 99 // buffers holds a shardBuffer object per shard, even if no failover is in 100 // progress. 101 // Key Format: "<keyspace>/<shard>" 102 buffers map[string]*shardBuffer 103 // stopped is true after Shutdown() was run. 104 stopped bool 105 } 106 107 // New creates a new Buffer object. 108 func New(cfg *Config) *Buffer { 109 return &Buffer{ 110 config: cfg, 111 bufferSizeSema: sync2.NewSemaphore(cfg.Size, 0), 112 buffers: make(map[string]*shardBuffer), 113 } 114 } 115 116 // WaitForFailoverEnd blocks until a pending buffering due to a failover for 117 // keyspace/shard is over. 118 // If there is no ongoing failover, "err" is checked. If it's caused by a 119 // failover, buffering may be started. 120 // It returns an error if buffering failed (e.g. buffer full). 121 // If it does not return an error, it may return a RetryDoneFunc which must be 122 // called after the request was retried. 123 func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, err error) (RetryDoneFunc, error) { 124 // If an err is given, it must be related to a failover. 125 // We never buffer requests with other errors. 126 if err != nil && !CausedByFailover(err) { 127 return nil, nil 128 } 129 130 sb := b.getOrCreateBuffer(keyspace, shard) 131 if sb == nil { 132 // Buffer is shut down. Ignore all calls. 133 requestsSkipped.Add([]string{keyspace, shard, skippedShutdown}, 1) 134 return nil, nil 135 } 136 if sb.disabled() { 137 requestsSkipped.Add([]string{keyspace, shard, skippedDisabled}, 1) 138 return nil, nil 139 } 140 141 return sb.waitForFailoverEnd(ctx, keyspace, shard, err) 142 } 143 144 // ProcessPrimaryHealth notifies the buffer to record a new primary 145 // and end any failover buffering that may be in progress 146 func (b *Buffer) ProcessPrimaryHealth(th *discovery.TabletHealth) { 147 if th.Target.TabletType != topodatapb.TabletType_PRIMARY { 148 panic(fmt.Sprintf("BUG: non-PRIMARY TabletHealth object must not be forwarded: %#v", th)) 149 } 150 timestamp := th.PrimaryTermStartTime 151 if timestamp == 0 { 152 // Primarys where TabletExternallyReparented was never called will return 0. 153 // Ignore them. 154 return 155 } 156 157 sb := b.getOrCreateBuffer(th.Target.Keyspace, th.Target.Shard) 158 if sb == nil { 159 // Buffer is shut down. Ignore all calls. 160 return 161 } 162 sb.recordExternallyReparentedTimestamp(timestamp, th.Tablet.Alias) 163 } 164 165 func (b *Buffer) HandleKeyspaceEvent(ksevent *discovery.KeyspaceEvent) { 166 for _, shard := range ksevent.Shards { 167 sb := b.getOrCreateBuffer(shard.Target.Keyspace, shard.Target.Shard) 168 if sb != nil { 169 sb.recordKeyspaceEvent(shard.Tablet, shard.Serving) 170 } 171 } 172 } 173 174 // getOrCreateBuffer returns the ShardBuffer for the given keyspace and shard. 175 // It returns nil if Buffer is shut down and all calls should be ignored. 176 func (b *Buffer) getOrCreateBuffer(keyspace, shard string) *shardBuffer { 177 key := topoproto.KeyspaceShardString(keyspace, shard) 178 b.mu.RLock() 179 sb, ok := b.buffers[key] 180 stopped := b.stopped 181 b.mu.RUnlock() 182 183 if stopped { 184 return nil 185 } 186 if ok { 187 return sb 188 } 189 190 b.mu.Lock() 191 defer b.mu.Unlock() 192 // Look it up again because it could have been created in the meantime. 193 sb, ok = b.buffers[key] 194 if !ok { 195 sb = newShardBufferHealthCheck(b, b.config.bufferingMode(keyspace, shard), keyspace, shard) 196 b.buffers[key] = sb 197 } 198 return sb 199 } 200 201 // Shutdown blocks until all pending ShardBuffer objects are shut down. 202 // In particular, it guarantees that all launched Go routines are stopped after 203 // it returns. 204 func (b *Buffer) Shutdown() { 205 b.shutdown() 206 b.waitForShutdown() 207 } 208 209 func (b *Buffer) shutdown() { 210 b.mu.Lock() 211 defer b.mu.Unlock() 212 213 for _, sb := range b.buffers { 214 sb.shutdown() 215 } 216 b.stopped = true 217 } 218 219 func (b *Buffer) waitForShutdown() { 220 b.mu.RLock() 221 defer b.mu.RUnlock() 222 223 for _, sb := range b.buffers { 224 sb.waitForShutdown() 225 } 226 }