github.com/vipernet-xyz/tm@v0.34.24/test/maverick/consensus/wal.go (about) 1 package consensus 2 3 import ( 4 "encoding/binary" 5 "errors" 6 "fmt" 7 "hash/crc32" 8 "io" 9 "path/filepath" 10 "time" 11 12 "github.com/gogo/protobuf/proto" 13 14 // tmjson "github.com/vipernet-xyz/tm/libs/json" 15 tmcon "github.com/vipernet-xyz/tm/consensus" 16 auto "github.com/vipernet-xyz/tm/libs/autofile" 17 "github.com/vipernet-xyz/tm/libs/log" 18 tmos "github.com/vipernet-xyz/tm/libs/os" 19 "github.com/vipernet-xyz/tm/libs/service" 20 tmcons "github.com/vipernet-xyz/tm/proto/tendermint/consensus" 21 tmtime "github.com/vipernet-xyz/tm/types/time" 22 ) 23 24 const ( 25 // time.Time + max consensus msg size 26 maxMsgSizeBytes = maxMsgSize + 24 27 28 // how often the WAL should be sync'd during period sync'ing 29 walDefaultFlushInterval = 2 * time.Second 30 ) 31 32 //-------------------------------------------------------- 33 // types and functions for savings consensus messages 34 // func init() { 35 // tmjson.RegisterType(msgInfo{}, "tendermint/wal/MsgInfo") 36 // tmjson.RegisterType(timeoutInfo{}, "tendermint/wal/TimeoutInfo") 37 // tmjson.RegisterType(tmcon.EndHeightMessage {}, "tendermint/wal/EndHeightMessage ") 38 // } 39 40 // Write ahead logger writes msgs to disk before they are processed. 41 // Can be used for crash-recovery and deterministic replay. 42 // TODO: currently the wal is overwritten during replay catchup, give it a mode 43 // so it's either reading or appending - must read to end to start appending 44 // again. 45 type BaseWAL struct { 46 service.BaseService 47 48 group *auto.Group 49 50 enc *WALEncoder 51 52 flushTicker *time.Ticker 53 flushInterval time.Duration 54 } 55 56 var _ tmcon.WAL = &BaseWAL{} 57 58 // NewWAL returns a new write-ahead logger based on `baseWAL`, which implements 59 // WAL. It's flushed and synced to disk every 2s and once when stopped. 60 func NewWAL(walFile string, groupOptions ...func(*auto.Group)) (*BaseWAL, error) { 61 err := tmos.EnsureDir(filepath.Dir(walFile), 0700) 62 if err != nil { 63 return nil, fmt.Errorf("failed to ensure WAL directory is in place: %w", err) 64 } 65 66 group, err := auto.OpenGroup(walFile, groupOptions...) 67 if err != nil { 68 return nil, err 69 } 70 wal := &BaseWAL{ 71 group: group, 72 enc: NewWALEncoder(group), 73 flushInterval: walDefaultFlushInterval, 74 } 75 wal.BaseService = *service.NewBaseService(nil, "baseWAL", wal) 76 return wal, nil 77 } 78 79 // SetFlushInterval allows us to override the periodic flush interval for the WAL. 80 func (wal *BaseWAL) SetFlushInterval(i time.Duration) { 81 wal.flushInterval = i 82 } 83 84 func (wal *BaseWAL) Group() *auto.Group { 85 return wal.group 86 } 87 88 func (wal *BaseWAL) SetLogger(l log.Logger) { 89 wal.BaseService.Logger = l 90 wal.group.SetLogger(l) 91 } 92 93 func (wal *BaseWAL) OnStart() error { 94 size, err := wal.group.Head.Size() 95 if err != nil { 96 return err 97 } else if size == 0 { 98 if err := wal.WriteSync(tmcon.EndHeightMessage{Height: 0}); err != nil { 99 return err 100 } 101 } 102 err = wal.group.Start() 103 if err != nil { 104 return err 105 } 106 wal.flushTicker = time.NewTicker(wal.flushInterval) 107 go wal.processFlushTicks() 108 return nil 109 } 110 111 func (wal *BaseWAL) processFlushTicks() { 112 for { 113 select { 114 case <-wal.flushTicker.C: 115 if err := wal.FlushAndSync(); err != nil { 116 wal.Logger.Error("Periodic WAL flush failed", "err", err) 117 } 118 case <-wal.Quit(): 119 return 120 } 121 } 122 } 123 124 // FlushAndSync flushes and fsync's the underlying group's data to disk. 125 // See auto#FlushAndSync 126 func (wal *BaseWAL) FlushAndSync() error { 127 return wal.group.FlushAndSync() 128 } 129 130 // Stop the underlying autofile group. 131 // Use Wait() to ensure it's finished shutting down 132 // before cleaning up files. 133 func (wal *BaseWAL) OnStop() { 134 wal.flushTicker.Stop() 135 if err := wal.FlushAndSync(); err != nil { 136 wal.Logger.Error("error on flush data to disk", "error", err) 137 } 138 if err := wal.group.Stop(); err != nil { 139 wal.Logger.Error("error trying to stop wal", "error", err) 140 } 141 wal.group.Close() 142 } 143 144 // Wait for the underlying autofile group to finish shutting down 145 // so it's safe to cleanup files. 146 func (wal *BaseWAL) Wait() { 147 wal.group.Wait() 148 } 149 150 // Write is called in newStep and for each receive on the 151 // peerMsgQueue and the timeoutTicker. 152 // NOTE: does not call fsync() 153 func (wal *BaseWAL) Write(msg tmcon.WALMessage) error { 154 if wal == nil { 155 return nil 156 } 157 158 if err := wal.enc.Encode(&tmcon.TimedWALMessage{Time: tmtime.Now(), Msg: msg}); err != nil { 159 wal.Logger.Error("Error writing msg to consensus wal. WARNING: recover may not be possible for the current height", 160 "err", err, "msg", msg) 161 return err 162 } 163 164 return nil 165 } 166 167 // WriteSync is called when we receive a msg from ourselves 168 // so that we write to disk before sending signed messages. 169 // NOTE: calls fsync() 170 func (wal *BaseWAL) WriteSync(msg tmcon.WALMessage) error { 171 if wal == nil { 172 return nil 173 } 174 175 if err := wal.Write(msg); err != nil { 176 return err 177 } 178 179 if err := wal.FlushAndSync(); err != nil { 180 wal.Logger.Error(`WriteSync failed to flush consensus wal. 181 WARNING: may result in creating alternative proposals / votes for the current height iff the node restarted`, 182 "err", err) 183 return err 184 } 185 186 return nil 187 } 188 189 // WALSearchOptions are optional arguments to SearchForEndHeight. 190 type WALSearchOptions struct { 191 // IgnoreDataCorruptionErrors set to true will result in skipping data corruption errors. 192 IgnoreDataCorruptionErrors bool 193 } 194 195 // SearchForEndHeight searches for the EndHeightMessage with the given height 196 // and returns an auto.GroupReader, whenever it was found or not and an error. 197 // Group reader will be nil if found equals false. 198 // 199 // CONTRACT: caller must close group reader. 200 func (wal *BaseWAL) SearchForEndHeight( 201 height int64, 202 options *tmcon.WALSearchOptions) (rd io.ReadCloser, found bool, err error) { 203 var ( 204 msg *tmcon.TimedWALMessage 205 gr *auto.GroupReader 206 ) 207 lastHeightFound := int64(-1) 208 209 // NOTE: starting from the last file in the group because we're usually 210 // searching for the last height. See replay.go 211 min, max := wal.group.MinIndex(), wal.group.MaxIndex() 212 wal.Logger.Info("Searching for height", "height", height, "min", min, "max", max) 213 for index := max; index >= min; index-- { 214 gr, err = wal.group.NewReader(index) 215 if err != nil { 216 return nil, false, err 217 } 218 219 dec := NewWALDecoder(gr) 220 for { 221 msg, err = dec.Decode() 222 if err == io.EOF { 223 // OPTIMISATION: no need to look for height in older files if we've seen h < height 224 if lastHeightFound > 0 && lastHeightFound < height { 225 gr.Close() 226 return nil, false, nil 227 } 228 // check next file 229 break 230 } 231 if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) { 232 wal.Logger.Error("Corrupted entry. Skipping...", "err", err) 233 // do nothing 234 continue 235 } else if err != nil { 236 gr.Close() 237 return nil, false, err 238 } 239 240 if m, ok := msg.Msg.(tmcon.EndHeightMessage); ok { 241 lastHeightFound = m.Height 242 if m.Height == height { // found 243 wal.Logger.Info("Found", "height", height, "index", index) 244 return gr, true, nil 245 } 246 } 247 } 248 gr.Close() 249 } 250 251 return nil, false, nil 252 } 253 254 // ///////////////////////////////////////////////////////////////////////////// 255 256 // A WALEncoder writes custom-encoded WAL messages to an output stream. 257 // 258 // Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value 259 type WALEncoder struct { 260 wr io.Writer 261 } 262 263 // NewWALEncoder returns a new encoder that writes to wr. 264 func NewWALEncoder(wr io.Writer) *WALEncoder { 265 return &WALEncoder{wr} 266 } 267 268 // Encode writes the custom encoding of v to the stream. It returns an error if 269 // the encoded size of v is greater than 1MB. Any error encountered 270 // during the write is also returned. 271 func (enc *WALEncoder) Encode(v *tmcon.TimedWALMessage) error { 272 pbMsg, err := WALToProto(v.Msg) 273 if err != nil { 274 return err 275 } 276 pv := tmcons.TimedWALMessage{ 277 Time: v.Time, 278 Msg: pbMsg, 279 } 280 281 data, err := proto.Marshal(&pv) 282 if err != nil { 283 panic(fmt.Errorf("encode timed wall message failure: %w", err)) 284 } 285 286 crc := crc32.Checksum(data, crc32c) 287 length := uint32(len(data)) 288 if length > maxMsgSizeBytes { 289 return fmt.Errorf("msg is too big: %d bytes, max: %d bytes", length, maxMsgSizeBytes) 290 } 291 totalLength := 8 + int(length) 292 293 msg := make([]byte, totalLength) 294 binary.BigEndian.PutUint32(msg[0:4], crc) 295 binary.BigEndian.PutUint32(msg[4:8], length) 296 copy(msg[8:], data) 297 298 _, err = enc.wr.Write(msg) 299 return err 300 } 301 302 // ///////////////////////////////////////////////////////////////////////////// 303 304 // IsDataCorruptionError returns true if data has been corrupted inside WAL. 305 func IsDataCorruptionError(err error) bool { 306 _, ok := err.(DataCorruptionError) 307 return ok 308 } 309 310 // DataCorruptionError is an error that occures if data on disk was corrupted. 311 type DataCorruptionError struct { 312 cause error 313 } 314 315 func (e DataCorruptionError) Error() string { 316 return fmt.Sprintf("DataCorruptionError[%v]", e.cause) 317 } 318 319 func (e DataCorruptionError) Cause() error { 320 return e.cause 321 } 322 323 // A WALDecoder reads and decodes custom-encoded WAL messages from an input 324 // stream. See WALEncoder for the format used. 325 // 326 // It will also compare the checksums and make sure data size is equal to the 327 // length from the header. If that is not the case, error will be returned. 328 type WALDecoder struct { 329 rd io.Reader 330 } 331 332 // NewWALDecoder returns a new decoder that reads from rd. 333 func NewWALDecoder(rd io.Reader) *WALDecoder { 334 return &WALDecoder{rd} 335 } 336 337 // Decode reads the next custom-encoded value from its reader and returns it. 338 func (dec *WALDecoder) Decode() (*tmcon.TimedWALMessage, error) { 339 b := make([]byte, 4) 340 341 _, err := dec.rd.Read(b) 342 if errors.Is(err, io.EOF) { 343 return nil, err 344 } 345 if err != nil { 346 return nil, DataCorruptionError{fmt.Errorf("failed to read checksum: %v", err)} 347 } 348 crc := binary.BigEndian.Uint32(b) 349 350 b = make([]byte, 4) 351 _, err = dec.rd.Read(b) 352 if err != nil { 353 return nil, DataCorruptionError{fmt.Errorf("failed to read length: %v", err)} 354 } 355 length := binary.BigEndian.Uint32(b) 356 357 if length > maxMsgSizeBytes { 358 return nil, DataCorruptionError{fmt.Errorf( 359 "length %d exceeded maximum possible value of %d bytes", 360 length, 361 maxMsgSizeBytes)} 362 } 363 364 data := make([]byte, length) 365 n, err := dec.rd.Read(data) 366 if err != nil { 367 return nil, DataCorruptionError{fmt.Errorf("failed to read data: %v (read: %d, wanted: %d)", err, n, length)} 368 } 369 370 // check checksum before decoding data 371 actualCRC := crc32.Checksum(data, crc32c) 372 if actualCRC != crc { 373 return nil, DataCorruptionError{fmt.Errorf("checksums do not match: read: %v, actual: %v", crc, actualCRC)} 374 } 375 376 var res = new(tmcons.TimedWALMessage) 377 err = proto.Unmarshal(data, res) 378 if err != nil { 379 return nil, DataCorruptionError{fmt.Errorf("failed to decode data: %v", err)} 380 } 381 382 walMsg, err := WALFromProto(res.Msg) 383 if err != nil { 384 return nil, DataCorruptionError{fmt.Errorf("failed to convert from proto: %w", err)} 385 } 386 tMsgWal := &tmcon.TimedWALMessage{ 387 Time: res.Time, 388 Msg: walMsg, 389 } 390 391 return tMsgWal, err 392 } 393 394 type nilWAL struct{} 395 396 var _ tmcon.WAL = nilWAL{} 397 398 func (nilWAL) Write(m tmcon.WALMessage) error { return nil } 399 func (nilWAL) WriteSync(m tmcon.WALMessage) error { return nil } 400 func (nilWAL) FlushAndSync() error { return nil } 401 func (nilWAL) SearchForEndHeight(height int64, 402 options *tmcon.WALSearchOptions) (rd io.ReadCloser, found bool, err error) { 403 return nil, false, nil 404 } 405 func (nilWAL) Start() error { return nil } 406 func (nilWAL) Stop() error { return nil } 407 func (nilWAL) Wait() {}