github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/sql/rowexec/backfiller.go (about)

     1  // Copyright 2016 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package rowexec
    12  
    13  import (
    14  	"context"
    15  	"fmt"
    16  
    17  	"github.com/cockroachdb/cockroach/pkg/clusterversion"
    18  	"github.com/cockroachdb/cockroach/pkg/jobs"
    19  	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
    20  	"github.com/cockroachdb/cockroach/pkg/keys"
    21  	"github.com/cockroachdb/cockroach/pkg/kv"
    22  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    23  	"github.com/cockroachdb/cockroach/pkg/sql/backfill"
    24  	"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
    25  	"github.com/cockroachdb/cockroach/pkg/sql/execinfrapb"
    26  	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
    27  	"github.com/cockroachdb/cockroach/pkg/sql/types"
    28  	"github.com/cockroachdb/cockroach/pkg/util/hlc"
    29  	"github.com/cockroachdb/cockroach/pkg/util/log"
    30  	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
    31  	"github.com/cockroachdb/cockroach/pkg/util/tracing"
    32  	"github.com/cockroachdb/errors"
    33  	"github.com/cockroachdb/logtags"
    34  )
    35  
    36  type chunkBackfiller interface {
    37  	// prepare must be called before runChunk.
    38  	prepare(ctx context.Context) error
    39  
    40  	// close should always be called to close a backfiller if prepare() was called.
    41  	close(ctx context.Context)
    42  
    43  	// runChunk returns the next-key and an error. next-key is nil
    44  	// once the backfill is complete.
    45  	runChunk(
    46  		ctx context.Context,
    47  		mutations []sqlbase.DescriptorMutation,
    48  		span roachpb.Span,
    49  		chunkSize int64,
    50  		readAsOf hlc.Timestamp,
    51  	) (roachpb.Key, error)
    52  
    53  	// CurrentBufferFill returns how fractionally full the configured buffer is.
    54  	CurrentBufferFill() float32
    55  
    56  	// flush must be called after the last chunk to finish buffered work.
    57  	flush(ctx context.Context) error
    58  }
    59  
    60  // backfiller is a processor that implements a distributed backfill of
    61  // an entity, like indexes or columns, during a schema change.
    62  type backfiller struct {
    63  	chunks chunkBackfiller
    64  	// name is the name of the kind of entity this backfiller processes.
    65  	name string
    66  	// mutationFilter returns true if the mutation should be processed by the
    67  	// chunkBackfiller.
    68  	filter backfill.MutationFilter
    69  
    70  	spec        execinfrapb.BackfillerSpec
    71  	output      execinfra.RowReceiver
    72  	out         execinfra.ProcOutputHelper
    73  	flowCtx     *execinfra.FlowCtx
    74  	processorID int32
    75  }
    76  
    77  // OutputTypes is part of the processor interface.
    78  func (*backfiller) OutputTypes() []*types.T {
    79  	// No output types.
    80  	return nil
    81  }
    82  
    83  func (b backfiller) getMutationsToProcess(
    84  	ctx context.Context,
    85  ) ([]sqlbase.DescriptorMutation, error) {
    86  	var mutations []sqlbase.DescriptorMutation
    87  	desc := b.spec.Table
    88  	if len(desc.Mutations) == 0 {
    89  		return nil, errors.Errorf("no schema changes for table ID=%d", desc.ID)
    90  	}
    91  	const noNewIndex = -1
    92  	// The first index of a mutation in the mutation list that will be
    93  	// processed.
    94  	firstMutationIdx := noNewIndex
    95  	mutationID := desc.Mutations[0].MutationID
    96  	for i, m := range desc.Mutations {
    97  		if m.MutationID != mutationID {
    98  			break
    99  		}
   100  		if b.filter(m) {
   101  			mutations = append(mutations, m)
   102  			if firstMutationIdx == noNewIndex {
   103  				firstMutationIdx = i
   104  			}
   105  		}
   106  	}
   107  
   108  	if firstMutationIdx == noNewIndex ||
   109  		len(b.spec.Spans) == 0 {
   110  		return nil, errors.Errorf("completed processing all spans for %s backfill (%d, %d)", b.name, desc.ID, mutationID)
   111  	}
   112  	return mutations, nil
   113  }
   114  
   115  // Run is part of the Processor interface.
   116  func (b *backfiller) Run(ctx context.Context) {
   117  	opName := fmt.Sprintf("%sBackfiller", b.name)
   118  	ctx = logtags.AddTag(ctx, opName, int(b.spec.Table.ID))
   119  	ctx, span := execinfra.ProcessorSpan(ctx, opName)
   120  	defer tracing.FinishSpan(span)
   121  	meta := b.doRun(ctx)
   122  	execinfra.SendTraceData(ctx, b.output)
   123  	if emitHelper(ctx, &b.out, nil /* row */, meta, func(ctx context.Context) {}) {
   124  		b.output.ProducerDone()
   125  	}
   126  }
   127  
   128  func (b *backfiller) doRun(ctx context.Context) *execinfrapb.ProducerMetadata {
   129  	if err := b.out.Init(&execinfrapb.PostProcessSpec{}, nil, b.flowCtx.NewEvalCtx(), b.output); err != nil {
   130  		return &execinfrapb.ProducerMetadata{Err: err}
   131  	}
   132  	mutations, err := b.getMutationsToProcess(ctx)
   133  	if err != nil {
   134  		return &execinfrapb.ProducerMetadata{Err: err}
   135  	}
   136  	finishedSpans, err := b.mainLoop(ctx, mutations)
   137  	if err != nil {
   138  		return &execinfrapb.ProducerMetadata{Err: err}
   139  	}
   140  	st := b.flowCtx.Cfg.Settings
   141  	if !st.Version.IsActive(ctx, clusterversion.VersionAtomicChangeReplicasTrigger) {
   142  		// There is a node of older version which could be the coordinator.
   143  		// So we communicate the finished work by writing to the jobs row.
   144  		err = WriteResumeSpan(
   145  			ctx,
   146  			b.flowCtx.Cfg.DB,
   147  			b.flowCtx.Codec(),
   148  			b.spec.Table.ID,
   149  			b.spec.Table.Mutations[0].MutationID,
   150  			b.filter,
   151  			finishedSpans,
   152  			b.flowCtx.Cfg.JobRegistry,
   153  		)
   154  		return &execinfrapb.ProducerMetadata{Err: err}
   155  	}
   156  	var prog execinfrapb.RemoteProducerMetadata_BulkProcessorProgress
   157  	prog.CompletedSpans = append(prog.CompletedSpans, finishedSpans...)
   158  	return &execinfrapb.ProducerMetadata{BulkProcessorProgress: &prog}
   159  }
   160  
   161  // mainLoop invokes runChunk on chunks of rows.
   162  // It does not close the output.
   163  func (b *backfiller) mainLoop(
   164  	ctx context.Context, mutations []sqlbase.DescriptorMutation,
   165  ) (roachpb.Spans, error) {
   166  	if err := b.chunks.prepare(ctx); err != nil {
   167  		return nil, err
   168  	}
   169  	defer b.chunks.close(ctx)
   170  
   171  	// As we approach the end of the configured duration, we may want to actually
   172  	// opportunistically wrap up a bit early. Specifically, if doing so can avoid
   173  	// starting a new fresh buffer that would need to then be flushed shortly
   174  	// thereafter with very little in it, resulting in many small SSTs that are
   175  	// almost as expensive to for their recipients but don't actually add much
   176  	// data. Instead, if our buffer is full enough that it is likely to flush soon
   177  	// and we're near the end of the alloted time, go ahead and stop there, flush
   178  	// and return.
   179  	opportunisticCheckpointAfter := (b.spec.Duration * 4) / 5
   180  	// opportunisticFillThreshold is the buffer fill fraction above which we'll
   181  	// conclude that running another chunk risks starting *but not really filling*
   182  	// a new buffer. This can be set pretty high -- if a single chunk is likely to
   183  	// fill more than this amount and cause a flush, then it likely also fills
   184  	// a non-trivial part of the next buffer.
   185  	const opportunisticCheckpointThreshold = 0.8
   186  	start := timeutil.Now()
   187  	totalChunks := 0
   188  	totalSpans := 0
   189  	var finishedSpans roachpb.Spans
   190  
   191  	for i := range b.spec.Spans {
   192  		log.VEventf(ctx, 2, "%s backfiller starting span %d of %d: %s",
   193  			b.name, i+1, len(b.spec.Spans), b.spec.Spans[i].Span)
   194  		chunks := 0
   195  		todo := b.spec.Spans[i].Span
   196  		for todo.Key != nil {
   197  			log.VEventf(ctx, 3, "%s backfiller starting chunk %d: %s", b.name, chunks, todo)
   198  			var err error
   199  			todo.Key, err = b.chunks.runChunk(ctx, mutations, todo, b.spec.ChunkSize, b.spec.ReadAsOf)
   200  			if err != nil {
   201  				return nil, err
   202  			}
   203  			chunks++
   204  			running := timeutil.Since(start)
   205  			if running > opportunisticCheckpointAfter && b.chunks.CurrentBufferFill() > opportunisticCheckpointThreshold {
   206  				break
   207  			}
   208  			if running > b.spec.Duration {
   209  				break
   210  			}
   211  		}
   212  		totalChunks += chunks
   213  
   214  		// If we exited the loop with a non-nil resume key, we ran out of time.
   215  		if todo.Key != nil {
   216  			log.VEventf(ctx, 2,
   217  				"%s backfiller ran out of time on span %d of %d, will resume it at %s next time",
   218  				b.name, i+1, len(b.spec.Spans), todo)
   219  			finishedSpans = append(finishedSpans, roachpb.Span{Key: b.spec.Spans[i].Span.Key, EndKey: todo.Key})
   220  			break
   221  		}
   222  		log.VEventf(ctx, 2, "%s backfiller finished span %d of %d: %s",
   223  			b.name, i+1, len(b.spec.Spans), b.spec.Spans[i].Span)
   224  		totalSpans++
   225  		finishedSpans = append(finishedSpans, b.spec.Spans[i].Span)
   226  	}
   227  
   228  	log.VEventf(ctx, 3, "%s backfiller flushing...", b.name)
   229  	if err := b.chunks.flush(ctx); err != nil {
   230  		return nil, err
   231  	}
   232  	log.VEventf(ctx, 2, "%s backfiller finished %d spans in %d chunks in %s",
   233  		b.name, totalSpans, totalChunks, timeutil.Since(start))
   234  
   235  	return finishedSpans, nil
   236  }
   237  
   238  // GetResumeSpans returns a ResumeSpanList from a job.
   239  func GetResumeSpans(
   240  	ctx context.Context,
   241  	jobsRegistry *jobs.Registry,
   242  	txn *kv.Txn,
   243  	codec keys.SQLCodec,
   244  	tableID sqlbase.ID,
   245  	mutationID sqlbase.MutationID,
   246  	filter backfill.MutationFilter,
   247  ) ([]roachpb.Span, *jobs.Job, int, error) {
   248  	tableDesc, err := sqlbase.GetTableDescFromID(ctx, txn, codec, tableID)
   249  	if err != nil {
   250  		return nil, nil, 0, err
   251  	}
   252  
   253  	// Find the index of the first mutation that is being worked on.
   254  	const noIndex = -1
   255  	mutationIdx := noIndex
   256  	if len(tableDesc.Mutations) > 0 {
   257  		for i, m := range tableDesc.Mutations {
   258  			if m.MutationID != mutationID {
   259  				break
   260  			}
   261  			if mutationIdx == noIndex && filter(m) {
   262  				mutationIdx = i
   263  			}
   264  		}
   265  	}
   266  
   267  	if mutationIdx == noIndex {
   268  		return nil, nil, 0, errors.AssertionFailedf(
   269  			"mutation %d has completed", errors.Safe(mutationID))
   270  	}
   271  
   272  	// Find the job.
   273  	var jobID int64
   274  	if len(tableDesc.MutationJobs) > 0 {
   275  		// TODO (lucy): We need to get rid of MutationJobs. This is the only place
   276  		// where we need to get the job where it's not completely straightforward to
   277  		// remove the use of MutationJobs, since the backfiller doesn't otherwise
   278  		// know which job it's associated with.
   279  		for _, job := range tableDesc.MutationJobs {
   280  			if job.MutationID == mutationID {
   281  				jobID = job.JobID
   282  				break
   283  			}
   284  		}
   285  	}
   286  
   287  	if jobID == 0 {
   288  		log.Errorf(ctx, "mutation with no job: %d, table desc: %+v", mutationID, tableDesc)
   289  		return nil, nil, 0, errors.AssertionFailedf(
   290  			"no job found for mutation %d", errors.Safe(mutationID))
   291  	}
   292  
   293  	job, err := jobsRegistry.LoadJobWithTxn(ctx, jobID, txn)
   294  	if err != nil {
   295  		return nil, nil, 0, errors.Wrapf(err, "can't find job %d", errors.Safe(jobID))
   296  	}
   297  	details, ok := job.Details().(jobspb.SchemaChangeDetails)
   298  	if !ok {
   299  		return nil, nil, 0, errors.AssertionFailedf(
   300  			"expected SchemaChangeDetails job type, got %T", job.Details())
   301  	}
   302  	// Return the resume spans from the job using the mutation idx.
   303  	return details.ResumeSpanList[mutationIdx].ResumeSpans, job, mutationIdx, nil
   304  }
   305  
   306  // SetResumeSpansInJob adds a list of resume spans into a job details field.
   307  func SetResumeSpansInJob(
   308  	ctx context.Context, spans []roachpb.Span, mutationIdx int, txn *kv.Txn, job *jobs.Job,
   309  ) error {
   310  	details, ok := job.Details().(jobspb.SchemaChangeDetails)
   311  	if !ok {
   312  		return errors.Errorf("expected SchemaChangeDetails job type, got %T", job.Details())
   313  	}
   314  	details.ResumeSpanList[mutationIdx].ResumeSpans = spans
   315  	return job.WithTxn(txn).SetDetails(ctx, details)
   316  }
   317  
   318  // WriteResumeSpan writes a checkpoint for the backfill work on origSpan.
   319  // origSpan is the span of keys that were assigned to be backfilled,
   320  // resume is the left over work from origSpan.
   321  func WriteResumeSpan(
   322  	ctx context.Context,
   323  	db *kv.DB,
   324  	codec keys.SQLCodec,
   325  	id sqlbase.ID,
   326  	mutationID sqlbase.MutationID,
   327  	filter backfill.MutationFilter,
   328  	finished roachpb.Spans,
   329  	jobsRegistry *jobs.Registry,
   330  ) error {
   331  	ctx, traceSpan := tracing.ChildSpan(ctx, "checkpoint")
   332  	defer tracing.FinishSpan(traceSpan)
   333  
   334  	return db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
   335  		resumeSpans, job, mutationIdx, error := GetResumeSpans(
   336  			ctx, jobsRegistry, txn, codec, id, mutationID, filter,
   337  		)
   338  		if error != nil {
   339  			return error
   340  		}
   341  
   342  		resumeSpans = roachpb.SubtractSpans(resumeSpans, finished)
   343  		return SetResumeSpansInJob(ctx, resumeSpans, mutationIdx, txn, job)
   344  	})
   345  }