github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/db/checkpoint/snapshot.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package checkpoint
    16  
    17  import (
    18  	"context"
    19  	"github.com/matrixorigin/matrixone/pkg/container/types"
    20  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    21  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio"
    22  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    23  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    24  	"sort"
    25  )
    26  
    27  type GetCheckpointRange = func(snapshot types.TS, files []*MetaFile) ([]*MetaFile, int, error)
    28  
    29  func SpecifiedCheckpoint(snapshot types.TS, files []*MetaFile) ([]*MetaFile, int, error) {
    30  	for i, file := range files {
    31  		if snapshot.LessEq(&file.end) {
    32  			return files, i, nil
    33  		}
    34  	}
    35  	return files, len(files) - 1, nil
    36  }
    37  
    38  func AllAfterAndGCheckpoint(snapshot types.TS, files []*MetaFile) ([]*MetaFile, int, error) {
    39  	prev := &MetaFile{}
    40  	for i, file := range files {
    41  		if snapshot.LessEq(&file.end) &&
    42  			snapshot.Less(&prev.end) &&
    43  			file.start.IsEmpty() {
    44  			return files, i - 1, nil
    45  		}
    46  		prev = file
    47  	}
    48  	return files, len(files) - 1, nil
    49  }
    50  
    51  func ListSnapshotCheckpoint(
    52  	ctx context.Context,
    53  	fs fileservice.FileService,
    54  	snapshot types.TS,
    55  	tid uint64,
    56  	listFunc GetCheckpointRange,
    57  ) ([]*CheckpointEntry, error) {
    58  	files, idx, err := ListSnapshotMeta(ctx, fs, snapshot, listFunc)
    59  	if err != nil {
    60  		return nil, err
    61  	}
    62  	if len(files) == 0 {
    63  		return nil, nil
    64  	}
    65  	return ListSnapshotCheckpointWithMeta(ctx, fs, files, idx, types.TS{}, false)
    66  }
    67  
    68  func ListSnapshotMeta(
    69  	ctx context.Context,
    70  	fs fileservice.FileService,
    71  	snapshot types.TS,
    72  	listFunc GetCheckpointRange,
    73  ) ([]*MetaFile, int, error) {
    74  	dirs, err := fs.List(ctx, CheckpointDir)
    75  	if err != nil {
    76  		return nil, 0, err
    77  	}
    78  	if len(dirs) == 0 {
    79  		return nil, 0, nil
    80  	}
    81  	metaFiles := make([]*MetaFile, 0)
    82  	for i, dir := range dirs {
    83  		start, end := blockio.DecodeCheckpointMetadataFileName(dir.Name)
    84  		metaFiles = append(metaFiles, &MetaFile{
    85  			start: start,
    86  			end:   end,
    87  			index: i,
    88  			name:  dir.Name,
    89  		})
    90  	}
    91  	sort.Slice(metaFiles, func(i, j int) bool {
    92  		return metaFiles[i].end.Less(&metaFiles[j].end)
    93  	})
    94  
    95  	if listFunc == nil {
    96  		listFunc = AllAfterAndGCheckpoint
    97  	}
    98  	return listFunc(snapshot, metaFiles)
    99  }
   100  
   101  func ListSnapshotCheckpointWithMeta(
   102  	ctx context.Context,
   103  	fs fileservice.FileService,
   104  	files []*MetaFile,
   105  	idx int,
   106  	gcStage types.TS,
   107  	isAll bool,
   108  ) ([]*CheckpointEntry, error) {
   109  	reader, err := blockio.NewFileReader(fs, CheckpointDir+files[idx].name)
   110  	if err != nil {
   111  		return nil, nil
   112  	}
   113  	bats, closeCB, err := reader.LoadAllColumns(ctx, nil, common.DebugAllocator)
   114  	if err != nil {
   115  		return nil, nil
   116  	}
   117  	defer func() {
   118  		if closeCB != nil {
   119  			closeCB()
   120  		}
   121  	}()
   122  	bat := containers.NewBatch()
   123  	defer bat.Close()
   124  	colNames := CheckpointSchema.Attrs()
   125  	colTypes := CheckpointSchema.Types()
   126  	for i := range bats[0].Vecs {
   127  		if len(bats) == 0 {
   128  			continue
   129  		}
   130  		var vec containers.Vector
   131  		if bats[0].Vecs[i].Length() == 0 {
   132  			vec = containers.MakeVector(colTypes[i], common.DebugAllocator)
   133  		} else {
   134  			vec = containers.ToTNVector(bats[0].Vecs[i], common.DebugAllocator)
   135  		}
   136  		bat.AddVector(colNames[i], vec)
   137  	}
   138  	entries, maxGlobalEnd := replayCheckpointEntries(bat, 3)
   139  	sort.Slice(entries, func(i, j int) bool {
   140  		return entries[i].end.Less(&entries[j].end)
   141  	})
   142  	if isAll && gcStage.IsEmpty() {
   143  		return entries, nil
   144  	}
   145  	for i := range entries {
   146  		if !gcStage.IsEmpty() {
   147  			if entries[i].end.Less(&gcStage) {
   148  				continue
   149  			}
   150  			return entries[i:], nil
   151  		}
   152  
   153  		if entries[i].end.Equal(&maxGlobalEnd) &&
   154  			entries[i].entryType == ET_Global {
   155  			return entries[i:], nil
   156  		}
   157  
   158  	}
   159  	return entries, nil
   160  }