github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/tables/jobs/flushobj.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package jobs
    16  
    17  import (
    18  	"context"
    19  	"math/rand"
    20  	"time"
    21  
    22  	"github.com/matrixorigin/matrixone/pkg/perfcounter"
    23  
    24  	"github.com/matrixorigin/matrixone/pkg/objectio"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
    27  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    28  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    29  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tasks"
    30  )
    31  
    32  type flushObjTask struct {
    33  	*tasks.BaseTask
    34  	data      *containers.Batch
    35  	delta     *containers.Batch
    36  	meta      *catalog.ObjectEntry
    37  	fs        *objectio.ObjectFS
    38  	name      objectio.ObjectName
    39  	blocks    []objectio.BlockObject
    40  	schemaVer uint32
    41  	seqnums   []uint16
    42  	stat      objectio.ObjectStats
    43  	isAObj    bool
    44  
    45  	Stats objectio.ObjectStats
    46  }
    47  
    48  func NewFlushObjTask(
    49  	ctx *tasks.Context,
    50  	schemaVer uint32,
    51  	seqnums []uint16,
    52  	fs *objectio.ObjectFS,
    53  	meta *catalog.ObjectEntry,
    54  	data *containers.Batch,
    55  	delta *containers.Batch,
    56  	isAObj bool,
    57  ) *flushObjTask {
    58  	task := &flushObjTask{
    59  		schemaVer: schemaVer,
    60  		seqnums:   seqnums,
    61  		data:      data,
    62  		meta:      meta,
    63  		fs:        fs,
    64  		delta:     delta,
    65  		isAObj:    isAObj,
    66  	}
    67  	task.BaseTask = tasks.NewBaseTask(task, tasks.IOTask, ctx)
    68  	return task
    69  }
    70  
    71  func (task *flushObjTask) Scope() *common.ID { return task.meta.AsCommonID() }
    72  
    73  func (task *flushObjTask) Execute(ctx context.Context) (err error) {
    74  	if v := ctx.Value(TestFlushBailoutPos1{}); v != nil {
    75  		time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond)
    76  	}
    77  	seg := task.meta.ID.Segment()
    78  	name := objectio.BuildObjectName(seg, 0)
    79  	task.name = name
    80  	writer, err := blockio.NewBlockWriterNew(task.fs.Service, name, task.schemaVer, task.seqnums)
    81  	if err != nil {
    82  		return err
    83  	}
    84  	if task.isAObj {
    85  		writer.SetAppendable()
    86  	}
    87  	if task.meta.GetSchema().HasPK() {
    88  		writer.SetPrimaryKey(uint16(task.meta.GetSchema().GetSingleSortKeyIdx()))
    89  	} else if task.meta.GetSchema().HasSortKey() {
    90  		writer.SetSortKey(uint16(task.meta.GetSchema().GetSingleSortKeyIdx()))
    91  	}
    92  
    93  	cnBatch := containers.ToCNBatch(task.data)
    94  	for _, vec := range cnBatch.Vecs {
    95  		if vec == nil {
    96  			// this task has been canceled
    97  			return nil
    98  		}
    99  	}
   100  	_, err = writer.WriteBatch(cnBatch)
   101  	if err != nil {
   102  		return err
   103  	}
   104  	if task.delta != nil {
   105  		_, err := writer.WriteTombstoneBatch(containers.ToCNBatch(task.delta))
   106  		if err != nil {
   107  			return err
   108  		}
   109  	}
   110  	task.blocks, _, err = writer.Sync(ctx)
   111  	if err != nil {
   112  		return err
   113  	}
   114  	task.Stats = writer.GetObjectStats()[objectio.SchemaData]
   115  
   116  	perfcounter.Update(ctx, func(counter *perfcounter.CounterSet) {
   117  		counter.TAE.Block.Flush.Add(1)
   118  	})
   119  	task.stat = writer.Stats()
   120  	return err
   121  }