github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/redo/writer/file/file_log_writer.go (about)

     1  //  Copyright 2023 PingCAP, Inc.
     2  //
     3  //  Licensed under the Apache License, Version 2.0 (the "License");
     4  //  you may not use this file except in compliance with the License.
     5  //  You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  //  Unless required by applicable law or agreed to in writing, software
    10  //  distributed under the License is distributed on an "AS IS" BASIS,
    11  //  See the License for the specific language governing permissions and
    12  //  limitations under the License.
    13  
    14  package file
    15  
    16  import (
    17  	"context"
    18  	"path/filepath"
    19  
    20  	"github.com/pingcap/log"
    21  	"github.com/pingcap/tiflow/cdc/model/codec"
    22  	"github.com/pingcap/tiflow/cdc/redo/writer"
    23  	"github.com/pingcap/tiflow/pkg/config"
    24  	"github.com/pingcap/tiflow/pkg/errors"
    25  	"go.uber.org/zap"
    26  )
    27  
    28  var _ writer.RedoLogWriter = &logWriter{}
    29  
    30  // logWriter implement the RedoLogWriter interface
    31  type logWriter struct {
    32  	cfg           *writer.LogWriterConfig
    33  	backendWriter fileWriter
    34  }
    35  
    36  // NewLogWriter create a new logWriter.
    37  func NewLogWriter(
    38  	ctx context.Context, cfg *writer.LogWriterConfig, opts ...writer.Option,
    39  ) (lw *logWriter, err error) {
    40  	if cfg == nil {
    41  		err := errors.New("LogWriterConfig can not be nil")
    42  		return nil, errors.WrapError(errors.ErrRedoConfigInvalid, err)
    43  	}
    44  
    45  	if cfg.UseExternalStorage {
    46  		// When an external storage is used, we use redoDir as a temporary dir to store redo logs
    47  		// before we flush them to S3.
    48  		changeFeedID := cfg.ChangeFeedID
    49  		dataDir := config.GetGlobalServerConfig().DataDir
    50  		cfg.Dir = filepath.Join(dataDir, config.DefaultRedoDir,
    51  			changeFeedID.Namespace, changeFeedID.ID)
    52  	} else {
    53  		// When local storage or NFS is used, we use redoDir as the final storage path.
    54  		cfg.Dir = cfg.URI.Path
    55  	}
    56  
    57  	lw = &logWriter{cfg: cfg}
    58  	if lw.backendWriter, err = NewFileWriter(ctx, cfg, opts...); err != nil {
    59  		return nil, err
    60  	}
    61  	return
    62  }
    63  
    64  func (l *logWriter) WriteEvents(ctx context.Context, events ...writer.RedoEvent) error {
    65  	select {
    66  	case <-ctx.Done():
    67  		return errors.Trace(ctx.Err())
    68  	default:
    69  	}
    70  
    71  	if l.isStopped() {
    72  		return errors.ErrRedoWriterStopped.GenWithStackByArgs()
    73  	}
    74  	if len(events) == 0 {
    75  		return nil
    76  	}
    77  
    78  	for _, event := range events {
    79  		if event == nil {
    80  			log.Warn("writing nil event to redo log, ignore this",
    81  				zap.String("capture", l.cfg.CaptureID))
    82  			continue
    83  		}
    84  
    85  		rl := event.ToRedoLog()
    86  		data, err := codec.MarshalRedoLog(rl, nil)
    87  		if err != nil {
    88  			return errors.WrapError(errors.ErrMarshalFailed, err)
    89  		}
    90  
    91  		l.backendWriter.AdvanceTs(rl.GetCommitTs())
    92  		_, err = l.backendWriter.Write(data)
    93  		if err != nil {
    94  			return err
    95  		}
    96  	}
    97  	return nil
    98  }
    99  
   100  // FlushLog implement FlushLog api
   101  func (l *logWriter) FlushLog(ctx context.Context) (err error) {
   102  	select {
   103  	case <-ctx.Done():
   104  		return errors.Trace(ctx.Err())
   105  	default:
   106  	}
   107  
   108  	if l.isStopped() {
   109  		return errors.ErrRedoWriterStopped.GenWithStackByArgs()
   110  	}
   111  
   112  	return l.backendWriter.Flush()
   113  }
   114  
   115  // Close implements RedoLogWriter.Close.
   116  func (l *logWriter) Close() (err error) {
   117  	return l.backendWriter.Close()
   118  }
   119  
   120  func (l *logWriter) isStopped() bool {
   121  	return !l.backendWriter.IsRunning()
   122  }