github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/db/checkpoint/testutils.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package checkpoint
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    22  	"github.com/matrixorigin/matrixone/pkg/container/types"
    23  	"github.com/matrixorigin/matrixone/pkg/logutil"
    24  	"github.com/matrixorigin/matrixone/pkg/util/fault"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logtail"
    27  )
    28  
    29  type TestRunner interface {
    30  	EnableCheckpoint()
    31  	DisableCheckpoint()
    32  
    33  	CleanPenddingCheckpoint()
    34  	ForceGlobalCheckpoint(end types.TS, versionInterval time.Duration) error
    35  	ForceIncrementalCheckpoint(end types.TS) error
    36  	IsAllChangesFlushed(start, end types.TS, printTree bool) bool
    37  	MaxLSNInRange(end types.TS) uint64
    38  
    39  	ExistPendingEntryToGC() bool
    40  	MaxGlobalCheckpoint() *CheckpointEntry
    41  	ForceFlush(ts types.TS, ctx context.Context, duration time.Duration) (err error)
    42  	GetDirtyCollector() logtail.Collector
    43  }
    44  
    45  // DisableCheckpoint stops generating checkpoint
    46  func (r *runner) DisableCheckpoint() {
    47  	r.disabled.Store(true)
    48  }
    49  
    50  func (r *runner) EnableCheckpoint() {
    51  	r.disabled.Store(false)
    52  }
    53  
    54  func (r *runner) CleanPenddingCheckpoint() {
    55  	prev := r.MaxCheckpoint()
    56  	if prev == nil {
    57  		return
    58  	}
    59  	if !prev.IsFinished() {
    60  		r.storage.Lock()
    61  		r.storage.entries.Delete(prev)
    62  		r.storage.Unlock()
    63  	}
    64  	if prev.IsRunning() {
    65  		logutil.Warnf("Delete a running checkpoint entry")
    66  	}
    67  	prev = r.MaxGlobalCheckpoint()
    68  	if prev == nil {
    69  		return
    70  	}
    71  	if !prev.IsFinished() {
    72  		r.storage.Lock()
    73  		r.storage.entries.Delete(prev)
    74  		r.storage.Unlock()
    75  	}
    76  	if prev.IsRunning() {
    77  		logutil.Warnf("Delete a running checkpoint entry")
    78  	}
    79  }
    80  
    81  func (r *runner) ForceGlobalCheckpoint(end types.TS, versionInterval time.Duration) error {
    82  	if r.GetPenddingIncrementalCount() == 0 {
    83  		err := r.ForceIncrementalCheckpoint(end)
    84  		if err != nil {
    85  			return err
    86  		}
    87  	} else {
    88  		end = r.MaxCheckpoint().GetEnd()
    89  	}
    90  	r.globalCheckpointQueue.Enqueue(&globalCheckpointContext{
    91  		force:    true,
    92  		end:      end,
    93  		interval: versionInterval,
    94  	})
    95  	return nil
    96  }
    97  func (r *runner) ForceFlush(ts types.TS, ctx context.Context, forceDuration time.Duration) (err error) {
    98  	makeCtx := func() *DirtyCtx {
    99  		tree := r.source.ScanInRangePruned(types.TS{}, ts)
   100  		tree.GetTree().Compact()
   101  		if tree.IsEmpty() {
   102  			return nil
   103  		}
   104  		entry := logtail.NewDirtyTreeEntry(types.TS{}, ts, tree.GetTree())
   105  		dirtyCtx := new(DirtyCtx)
   106  		dirtyCtx.tree = entry
   107  		dirtyCtx.force = true
   108  		// logutil.Infof("try flush %v",tree.String())
   109  		return dirtyCtx
   110  	}
   111  	op := func() (ok bool, err error) {
   112  		dirtyCtx := makeCtx()
   113  		if dirtyCtx == nil {
   114  			return true, nil
   115  		}
   116  		if _, err = r.dirtyEntryQueue.Enqueue(dirtyCtx); err != nil {
   117  			return true, nil
   118  		}
   119  		return false, nil
   120  	}
   121  
   122  	if forceDuration == 0 {
   123  		forceDuration = r.options.forceFlushTimeout
   124  	}
   125  	err = common.RetryWithIntervalAndTimeout(
   126  		op,
   127  		forceDuration,
   128  		r.options.forceFlushCheckInterval, false)
   129  	if err != nil {
   130  		return moerr.NewInternalError(ctx, "force flush failed: %v", err)
   131  	}
   132  	_, sarg, _ := fault.TriggerFault("tae: flush timeout")
   133  	if sarg != "" {
   134  		err = moerr.NewInternalError(ctx, sarg)
   135  	}
   136  	return
   137  }
   138  
   139  func (r *runner) ForceIncrementalCheckpoint(end types.TS) error {
   140  	prev := r.MaxCheckpoint()
   141  	if prev != nil && !prev.IsFinished() {
   142  		return moerr.NewInternalError(context.Background(), "prev checkpoint not finished")
   143  	}
   144  	start := types.TS{}
   145  	if prev != nil {
   146  		start = prev.end.Next()
   147  	}
   148  	entry := NewCheckpointEntry(start, end, ET_Incremental)
   149  	r.storage.Lock()
   150  	r.storage.entries.Set(entry)
   151  	now := time.Now()
   152  	r.storage.Unlock()
   153  	r.doIncrementalCheckpoint(entry)
   154  	if err := r.saveCheckpoint(entry.start, entry.end); err != nil {
   155  		return err
   156  	}
   157  	entry.SetState(ST_Finished)
   158  	logutil.Infof("%s is done, takes %s", entry.String(), time.Since(now))
   159  	return nil
   160  }
   161  
   162  func (r *runner) IsAllChangesFlushed(start, end types.TS, printTree bool) bool {
   163  	tree := r.source.ScanInRangePruned(start, end)
   164  	tree.GetTree().Compact()
   165  	if printTree && !tree.IsEmpty() {
   166  		logutil.Infof("%v", tree.String())
   167  	}
   168  	return tree.IsEmpty()
   169  }