github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/cdc/puller/sorter/memory_backend.go (about)

     1  // Copyright 2021 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package sorter
    15  
    16  import (
    17  	"sync/atomic"
    18  
    19  	"github.com/pingcap/failpoint"
    20  	"github.com/pingcap/log"
    21  	"github.com/pingcap/ticdc/cdc/model"
    22  	"go.uber.org/zap"
    23  )
    24  
    25  type memoryBackEnd struct {
    26  	events        []*model.PolymorphicEvent
    27  	estimatedSize int64
    28  	borrowed      int32
    29  }
    30  
    31  func newMemoryBackEnd() *memoryBackEnd {
    32  	return &memoryBackEnd{}
    33  }
    34  
    35  func (m *memoryBackEnd) reader() (backEndReader, error) {
    36  	failpoint.Inject("sorterDebug", func() {
    37  		if atomic.SwapInt32(&m.borrowed, 1) != 0 {
    38  			log.Panic("memoryBackEnd: already borrowed")
    39  		}
    40  	})
    41  
    42  	return &memoryBackEndReader{
    43  		backEnd:   m,
    44  		readIndex: 0,
    45  	}, nil
    46  }
    47  
    48  func (m *memoryBackEnd) writer() (backEndWriter, error) {
    49  	failpoint.Inject("sorterDebug", func() {
    50  		if atomic.SwapInt32(&m.borrowed, 1) != 0 {
    51  			log.Panic("memoryBackEnd: already borrowed")
    52  		}
    53  	})
    54  
    55  	return &memoryBackEndWriter{backEnd: m}, nil
    56  }
    57  
    58  func (m *memoryBackEnd) free() error {
    59  	failpoint.Inject("sorterDebug", func() {
    60  		if atomic.LoadInt32(&m.borrowed) != 0 {
    61  			log.Panic("fileBackEnd: trying to free borrowed file")
    62  		}
    63  	})
    64  
    65  	if pool != nil {
    66  		atomic.AddInt64(&pool.memoryUseEstimate, -m.estimatedSize)
    67  	}
    68  
    69  	return nil
    70  }
    71  
    72  type memoryBackEndReader struct {
    73  	backEnd   *memoryBackEnd
    74  	readIndex int
    75  }
    76  
    77  func (r *memoryBackEndReader) readNext() (*model.PolymorphicEvent, error) {
    78  	// Check for "EOF"
    79  	if r.readIndex >= len(r.backEnd.events) {
    80  		return nil, nil
    81  	}
    82  
    83  	ret := r.backEnd.events[r.readIndex]
    84  	// Sets the slot to nil to prevent delaying GC.
    85  	r.backEnd.events[r.readIndex] = nil
    86  	r.readIndex++
    87  	return ret, nil
    88  }
    89  
    90  func (r *memoryBackEndReader) resetAndClose() error {
    91  	failpoint.Inject("sorterDebug", func() {
    92  		atomic.StoreInt32(&r.backEnd.borrowed, 0)
    93  	})
    94  
    95  	if pool != nil {
    96  		atomic.AddInt64(&pool.memoryUseEstimate, -r.backEnd.estimatedSize)
    97  	}
    98  	r.backEnd.estimatedSize = 0
    99  
   100  	return nil
   101  }
   102  
   103  type memoryBackEndWriter struct {
   104  	backEnd      *memoryBackEnd
   105  	bytesWritten int64
   106  	// for debugging only
   107  	maxTs uint64
   108  }
   109  
   110  func (w *memoryBackEndWriter) writeNext(event *model.PolymorphicEvent) error {
   111  	w.backEnd.events = append(w.backEnd.events, event)
   112  	// 8 * 5 is for the 5 fields in PolymorphicEvent, each of which is thought of as a 64-bit pointer
   113  	w.bytesWritten += 8*5 + event.RawKV.ApproximateSize()
   114  
   115  	failpoint.Inject("sorterDebug", func() {
   116  		if event.CRTs < w.maxTs {
   117  			log.Panic("memoryBackEnd: ts regressed, bug?",
   118  				zap.Uint64("prev-ts", w.maxTs),
   119  				zap.Uint64("cur-ts", event.CRTs))
   120  		}
   121  		w.maxTs = event.CRTs
   122  	})
   123  	return nil
   124  }
   125  
   126  func (w *memoryBackEndWriter) writtenCount() int {
   127  	return len(w.backEnd.events)
   128  }
   129  
   130  // dataSize for the memoryBackEnd returns only an estimation, as there is no serialization taking place.
   131  func (w *memoryBackEndWriter) dataSize() uint64 {
   132  	return uint64(w.bytesWritten)
   133  }
   134  
   135  func (w *memoryBackEndWriter) flushAndClose() error {
   136  	failpoint.Inject("sorterDebug", func() {
   137  		atomic.StoreInt32(&w.backEnd.borrowed, 0)
   138  	})
   139  
   140  	w.backEnd.estimatedSize = w.bytesWritten
   141  	if pool != nil {
   142  		atomic.AddInt64(&pool.memoryUseEstimate, w.bytesWritten)
   143  	}
   144  
   145  	return nil
   146  }