github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/sql/rowexec/backfiller.go (about) 1 // Copyright 2016 The Cockroach Authors. 2 // 3 // Use of this software is governed by the Business Source License 4 // included in the file licenses/BSL.txt. 5 // 6 // As of the Change Date specified in that file, in accordance with 7 // the Business Source License, use of this software will be governed 8 // by the Apache License, Version 2.0, included in the file 9 // licenses/APL.txt. 10 11 package rowexec 12 13 import ( 14 "context" 15 "fmt" 16 17 "github.com/cockroachdb/cockroach/pkg/clusterversion" 18 "github.com/cockroachdb/cockroach/pkg/jobs" 19 "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" 20 "github.com/cockroachdb/cockroach/pkg/keys" 21 "github.com/cockroachdb/cockroach/pkg/kv" 22 "github.com/cockroachdb/cockroach/pkg/roachpb" 23 "github.com/cockroachdb/cockroach/pkg/sql/backfill" 24 "github.com/cockroachdb/cockroach/pkg/sql/execinfra" 25 "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" 26 "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" 27 "github.com/cockroachdb/cockroach/pkg/sql/types" 28 "github.com/cockroachdb/cockroach/pkg/util/hlc" 29 "github.com/cockroachdb/cockroach/pkg/util/log" 30 "github.com/cockroachdb/cockroach/pkg/util/timeutil" 31 "github.com/cockroachdb/cockroach/pkg/util/tracing" 32 "github.com/cockroachdb/errors" 33 "github.com/cockroachdb/logtags" 34 ) 35 36 type chunkBackfiller interface { 37 // prepare must be called before runChunk. 38 prepare(ctx context.Context) error 39 40 // close should always be called to close a backfiller if prepare() was called. 41 close(ctx context.Context) 42 43 // runChunk returns the next-key and an error. next-key is nil 44 // once the backfill is complete. 45 runChunk( 46 ctx context.Context, 47 mutations []sqlbase.DescriptorMutation, 48 span roachpb.Span, 49 chunkSize int64, 50 readAsOf hlc.Timestamp, 51 ) (roachpb.Key, error) 52 53 // CurrentBufferFill returns how fractionally full the configured buffer is. 54 CurrentBufferFill() float32 55 56 // flush must be called after the last chunk to finish buffered work. 57 flush(ctx context.Context) error 58 } 59 60 // backfiller is a processor that implements a distributed backfill of 61 // an entity, like indexes or columns, during a schema change. 62 type backfiller struct { 63 chunks chunkBackfiller 64 // name is the name of the kind of entity this backfiller processes. 65 name string 66 // mutationFilter returns true if the mutation should be processed by the 67 // chunkBackfiller. 68 filter backfill.MutationFilter 69 70 spec execinfrapb.BackfillerSpec 71 output execinfra.RowReceiver 72 out execinfra.ProcOutputHelper 73 flowCtx *execinfra.FlowCtx 74 processorID int32 75 } 76 77 // OutputTypes is part of the processor interface. 78 func (*backfiller) OutputTypes() []*types.T { 79 // No output types. 80 return nil 81 } 82 83 func (b backfiller) getMutationsToProcess( 84 ctx context.Context, 85 ) ([]sqlbase.DescriptorMutation, error) { 86 var mutations []sqlbase.DescriptorMutation 87 desc := b.spec.Table 88 if len(desc.Mutations) == 0 { 89 return nil, errors.Errorf("no schema changes for table ID=%d", desc.ID) 90 } 91 const noNewIndex = -1 92 // The first index of a mutation in the mutation list that will be 93 // processed. 94 firstMutationIdx := noNewIndex 95 mutationID := desc.Mutations[0].MutationID 96 for i, m := range desc.Mutations { 97 if m.MutationID != mutationID { 98 break 99 } 100 if b.filter(m) { 101 mutations = append(mutations, m) 102 if firstMutationIdx == noNewIndex { 103 firstMutationIdx = i 104 } 105 } 106 } 107 108 if firstMutationIdx == noNewIndex || 109 len(b.spec.Spans) == 0 { 110 return nil, errors.Errorf("completed processing all spans for %s backfill (%d, %d)", b.name, desc.ID, mutationID) 111 } 112 return mutations, nil 113 } 114 115 // Run is part of the Processor interface. 116 func (b *backfiller) Run(ctx context.Context) { 117 opName := fmt.Sprintf("%sBackfiller", b.name) 118 ctx = logtags.AddTag(ctx, opName, int(b.spec.Table.ID)) 119 ctx, span := execinfra.ProcessorSpan(ctx, opName) 120 defer tracing.FinishSpan(span) 121 meta := b.doRun(ctx) 122 execinfra.SendTraceData(ctx, b.output) 123 if emitHelper(ctx, &b.out, nil /* row */, meta, func(ctx context.Context) {}) { 124 b.output.ProducerDone() 125 } 126 } 127 128 func (b *backfiller) doRun(ctx context.Context) *execinfrapb.ProducerMetadata { 129 if err := b.out.Init(&execinfrapb.PostProcessSpec{}, nil, b.flowCtx.NewEvalCtx(), b.output); err != nil { 130 return &execinfrapb.ProducerMetadata{Err: err} 131 } 132 mutations, err := b.getMutationsToProcess(ctx) 133 if err != nil { 134 return &execinfrapb.ProducerMetadata{Err: err} 135 } 136 finishedSpans, err := b.mainLoop(ctx, mutations) 137 if err != nil { 138 return &execinfrapb.ProducerMetadata{Err: err} 139 } 140 st := b.flowCtx.Cfg.Settings 141 if !st.Version.IsActive(ctx, clusterversion.VersionAtomicChangeReplicasTrigger) { 142 // There is a node of older version which could be the coordinator. 143 // So we communicate the finished work by writing to the jobs row. 144 err = WriteResumeSpan( 145 ctx, 146 b.flowCtx.Cfg.DB, 147 b.flowCtx.Codec(), 148 b.spec.Table.ID, 149 b.spec.Table.Mutations[0].MutationID, 150 b.filter, 151 finishedSpans, 152 b.flowCtx.Cfg.JobRegistry, 153 ) 154 return &execinfrapb.ProducerMetadata{Err: err} 155 } 156 var prog execinfrapb.RemoteProducerMetadata_BulkProcessorProgress 157 prog.CompletedSpans = append(prog.CompletedSpans, finishedSpans...) 158 return &execinfrapb.ProducerMetadata{BulkProcessorProgress: &prog} 159 } 160 161 // mainLoop invokes runChunk on chunks of rows. 162 // It does not close the output. 163 func (b *backfiller) mainLoop( 164 ctx context.Context, mutations []sqlbase.DescriptorMutation, 165 ) (roachpb.Spans, error) { 166 if err := b.chunks.prepare(ctx); err != nil { 167 return nil, err 168 } 169 defer b.chunks.close(ctx) 170 171 // As we approach the end of the configured duration, we may want to actually 172 // opportunistically wrap up a bit early. Specifically, if doing so can avoid 173 // starting a new fresh buffer that would need to then be flushed shortly 174 // thereafter with very little in it, resulting in many small SSTs that are 175 // almost as expensive to for their recipients but don't actually add much 176 // data. Instead, if our buffer is full enough that it is likely to flush soon 177 // and we're near the end of the alloted time, go ahead and stop there, flush 178 // and return. 179 opportunisticCheckpointAfter := (b.spec.Duration * 4) / 5 180 // opportunisticFillThreshold is the buffer fill fraction above which we'll 181 // conclude that running another chunk risks starting *but not really filling* 182 // a new buffer. This can be set pretty high -- if a single chunk is likely to 183 // fill more than this amount and cause a flush, then it likely also fills 184 // a non-trivial part of the next buffer. 185 const opportunisticCheckpointThreshold = 0.8 186 start := timeutil.Now() 187 totalChunks := 0 188 totalSpans := 0 189 var finishedSpans roachpb.Spans 190 191 for i := range b.spec.Spans { 192 log.VEventf(ctx, 2, "%s backfiller starting span %d of %d: %s", 193 b.name, i+1, len(b.spec.Spans), b.spec.Spans[i].Span) 194 chunks := 0 195 todo := b.spec.Spans[i].Span 196 for todo.Key != nil { 197 log.VEventf(ctx, 3, "%s backfiller starting chunk %d: %s", b.name, chunks, todo) 198 var err error 199 todo.Key, err = b.chunks.runChunk(ctx, mutations, todo, b.spec.ChunkSize, b.spec.ReadAsOf) 200 if err != nil { 201 return nil, err 202 } 203 chunks++ 204 running := timeutil.Since(start) 205 if running > opportunisticCheckpointAfter && b.chunks.CurrentBufferFill() > opportunisticCheckpointThreshold { 206 break 207 } 208 if running > b.spec.Duration { 209 break 210 } 211 } 212 totalChunks += chunks 213 214 // If we exited the loop with a non-nil resume key, we ran out of time. 215 if todo.Key != nil { 216 log.VEventf(ctx, 2, 217 "%s backfiller ran out of time on span %d of %d, will resume it at %s next time", 218 b.name, i+1, len(b.spec.Spans), todo) 219 finishedSpans = append(finishedSpans, roachpb.Span{Key: b.spec.Spans[i].Span.Key, EndKey: todo.Key}) 220 break 221 } 222 log.VEventf(ctx, 2, "%s backfiller finished span %d of %d: %s", 223 b.name, i+1, len(b.spec.Spans), b.spec.Spans[i].Span) 224 totalSpans++ 225 finishedSpans = append(finishedSpans, b.spec.Spans[i].Span) 226 } 227 228 log.VEventf(ctx, 3, "%s backfiller flushing...", b.name) 229 if err := b.chunks.flush(ctx); err != nil { 230 return nil, err 231 } 232 log.VEventf(ctx, 2, "%s backfiller finished %d spans in %d chunks in %s", 233 b.name, totalSpans, totalChunks, timeutil.Since(start)) 234 235 return finishedSpans, nil 236 } 237 238 // GetResumeSpans returns a ResumeSpanList from a job. 239 func GetResumeSpans( 240 ctx context.Context, 241 jobsRegistry *jobs.Registry, 242 txn *kv.Txn, 243 codec keys.SQLCodec, 244 tableID sqlbase.ID, 245 mutationID sqlbase.MutationID, 246 filter backfill.MutationFilter, 247 ) ([]roachpb.Span, *jobs.Job, int, error) { 248 tableDesc, err := sqlbase.GetTableDescFromID(ctx, txn, codec, tableID) 249 if err != nil { 250 return nil, nil, 0, err 251 } 252 253 // Find the index of the first mutation that is being worked on. 254 const noIndex = -1 255 mutationIdx := noIndex 256 if len(tableDesc.Mutations) > 0 { 257 for i, m := range tableDesc.Mutations { 258 if m.MutationID != mutationID { 259 break 260 } 261 if mutationIdx == noIndex && filter(m) { 262 mutationIdx = i 263 } 264 } 265 } 266 267 if mutationIdx == noIndex { 268 return nil, nil, 0, errors.AssertionFailedf( 269 "mutation %d has completed", errors.Safe(mutationID)) 270 } 271 272 // Find the job. 273 var jobID int64 274 if len(tableDesc.MutationJobs) > 0 { 275 // TODO (lucy): We need to get rid of MutationJobs. This is the only place 276 // where we need to get the job where it's not completely straightforward to 277 // remove the use of MutationJobs, since the backfiller doesn't otherwise 278 // know which job it's associated with. 279 for _, job := range tableDesc.MutationJobs { 280 if job.MutationID == mutationID { 281 jobID = job.JobID 282 break 283 } 284 } 285 } 286 287 if jobID == 0 { 288 log.Errorf(ctx, "mutation with no job: %d, table desc: %+v", mutationID, tableDesc) 289 return nil, nil, 0, errors.AssertionFailedf( 290 "no job found for mutation %d", errors.Safe(mutationID)) 291 } 292 293 job, err := jobsRegistry.LoadJobWithTxn(ctx, jobID, txn) 294 if err != nil { 295 return nil, nil, 0, errors.Wrapf(err, "can't find job %d", errors.Safe(jobID)) 296 } 297 details, ok := job.Details().(jobspb.SchemaChangeDetails) 298 if !ok { 299 return nil, nil, 0, errors.AssertionFailedf( 300 "expected SchemaChangeDetails job type, got %T", job.Details()) 301 } 302 // Return the resume spans from the job using the mutation idx. 303 return details.ResumeSpanList[mutationIdx].ResumeSpans, job, mutationIdx, nil 304 } 305 306 // SetResumeSpansInJob adds a list of resume spans into a job details field. 307 func SetResumeSpansInJob( 308 ctx context.Context, spans []roachpb.Span, mutationIdx int, txn *kv.Txn, job *jobs.Job, 309 ) error { 310 details, ok := job.Details().(jobspb.SchemaChangeDetails) 311 if !ok { 312 return errors.Errorf("expected SchemaChangeDetails job type, got %T", job.Details()) 313 } 314 details.ResumeSpanList[mutationIdx].ResumeSpans = spans 315 return job.WithTxn(txn).SetDetails(ctx, details) 316 } 317 318 // WriteResumeSpan writes a checkpoint for the backfill work on origSpan. 319 // origSpan is the span of keys that were assigned to be backfilled, 320 // resume is the left over work from origSpan. 321 func WriteResumeSpan( 322 ctx context.Context, 323 db *kv.DB, 324 codec keys.SQLCodec, 325 id sqlbase.ID, 326 mutationID sqlbase.MutationID, 327 filter backfill.MutationFilter, 328 finished roachpb.Spans, 329 jobsRegistry *jobs.Registry, 330 ) error { 331 ctx, traceSpan := tracing.ChildSpan(ctx, "checkpoint") 332 defer tracing.FinishSpan(traceSpan) 333 334 return db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { 335 resumeSpans, job, mutationIdx, error := GetResumeSpans( 336 ctx, jobsRegistry, txn, codec, id, mutationID, filter, 337 ) 338 if error != nil { 339 return error 340 } 341 342 resumeSpans = roachpb.SubtractSpans(resumeSpans, finished) 343 return SetResumeSpansInJob(ctx, resumeSpans, mutationIdx, txn, job) 344 }) 345 }