github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/logstore/driver/logservicedriver/driver.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package logservicedriver
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	"github.com/matrixorigin/matrixone/pkg/logutil"
    22  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    23  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logstore/driver"
    24  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logstore/sm"
    25  )
    26  
    27  const (
    28  	ReplayReadSize = common.M * 64
    29  	MaxReadSize    = common.M * 64
    30  )
    31  
    32  func RetryWithTimeout(timeoutDuration time.Duration, fn func() (shouldReturn bool)) error {
    33  	ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration)
    34  	defer cancel()
    35  	for {
    36  		select {
    37  		case <-ctx.Done():
    38  			return ErrRetryTimeOut
    39  		default:
    40  			if fn() {
    41  				return nil
    42  			}
    43  		}
    44  	}
    45  }
    46  
    47  type LogServiceDriver struct {
    48  	clientPool *clientpool
    49  	config     *Config
    50  	appendable *driverAppender
    51  	*driverInfo
    52  	*readCache
    53  
    54  	closeCtx        context.Context
    55  	closeCancel     context.CancelFunc
    56  	preAppendLoop   sm.Queue
    57  	appendQueue     chan any
    58  	appendedQueue   chan any
    59  	appendedLoop    *sm.Loop
    60  	postAppendQueue chan any
    61  	postAppendLoop  *sm.Loop
    62  
    63  	truncateQueue sm.Queue
    64  
    65  	flushtimes  int
    66  	appendtimes int
    67  
    68  	readDuration time.Duration
    69  }
    70  
    71  func NewLogServiceDriver(cfg *Config) *LogServiceDriver {
    72  	clientpoolConfig := &clientConfig{
    73  		cancelDuration:        cfg.NewClientDuration,
    74  		recordSize:            cfg.NewRecordSize,
    75  		clientFactory:         cfg.ClientFactory,
    76  		GetClientRetryTimeOut: cfg.GetClientRetryTimeOut,
    77  		retryDuration:         cfg.RetryTimeout,
    78  	}
    79  	d := &LogServiceDriver{
    80  		clientPool:      newClientPool(cfg.ClientPoolMaxSize, cfg.ClientPoolMaxSize, clientpoolConfig),
    81  		config:          cfg,
    82  		appendable:      newDriverAppender(),
    83  		driverInfo:      newDriverInfo(),
    84  		readCache:       newReadCache(),
    85  		appendQueue:     make(chan any, 10000),
    86  		appendedQueue:   make(chan any, 10000),
    87  		postAppendQueue: make(chan any, 10000),
    88  	}
    89  	d.closeCtx, d.closeCancel = context.WithCancel(context.Background())
    90  	d.preAppendLoop = sm.NewSafeQueue(10000, 10000, d.onPreAppend)
    91  	d.preAppendLoop.Start()
    92  	d.appendedLoop = sm.NewLoop(d.appendedQueue, d.postAppendQueue, d.onAppendedQueue, 10000)
    93  	d.appendedLoop.Start()
    94  	d.postAppendLoop = sm.NewLoop(d.postAppendQueue, nil, d.onPostAppendQueue, 10000)
    95  	d.postAppendLoop.Start()
    96  	d.truncateQueue = sm.NewSafeQueue(10000, 10000, d.onTruncate)
    97  	d.truncateQueue.Start()
    98  	return d
    99  }
   100  
   101  func (d *LogServiceDriver) Close() error {
   102  	logutil.Infof("append%d,flush%d", d.appendtimes, d.flushtimes)
   103  	d.clientPool.Close()
   104  	d.closeCancel()
   105  	d.preAppendLoop.Stop()
   106  	d.appendedLoop.Stop()
   107  	d.postAppendLoop.Stop()
   108  	d.truncateQueue.Stop()
   109  	close(d.appendQueue)
   110  	close(d.appendedQueue)
   111  	close(d.postAppendQueue)
   112  	return nil
   113  }
   114  
   115  func (d *LogServiceDriver) Replay(h driver.ApplyHandle) error {
   116  	d.PreReplay()
   117  	r := newReplayer(h, ReplayReadSize, d)
   118  	r.replay()
   119  	d.onReplay(r)
   120  	r.d.resetReadCache()
   121  	d.PostReplay()
   122  	logutil.Info("open-tae", common.OperationField("replay"),
   123  		common.OperandField("wal"),
   124  		common.AnyField("backend", "logservice"),
   125  		common.AnyField("apply cost", r.applyDuration),
   126  		common.AnyField("read cost", d.readDuration))
   127  
   128  	return nil
   129  }