github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/syndtr/goleveldb/leveldb/journal/journal.go (about) 1 // Copyright 2011 The LevelDB-Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record.go?r=1d5ccbe03246da926391ee12d1c6caae054ff4b0 6 // License, authors and contributors informations can be found at bellow URLs respectively: 7 // https://code.google.com/p/leveldb-go/source/browse/LICENSE 8 // https://code.google.com/p/leveldb-go/source/browse/AUTHORS 9 // https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS 10 11 // Package journal reads and writes sequences of journals. Each journal is a stream 12 // of bytes that completes before the next journal starts. 13 // 14 // When reading, call Next to obtain an io.Reader for the next journal. Next will 15 // return io.EOF when there are no more journals. It is valid to call Next 16 // without reading the current journal to exhaustion. 17 // 18 // When writing, call Next to obtain an io.Writer for the next journal. Calling 19 // Next finishes the current journal. Call Close to finish the final journal. 20 // 21 // Optionally, call Flush to finish the current journal and flush the underlying 22 // writer without starting a new journal. To start a new journal after flushing, 23 // call Next. 24 // 25 // Neither Readers or Writers are safe to use concurrently. 26 // 27 // Example code: 28 // func read(r io.Reader) ([]string, error) { 29 // var ss []string 30 // journals := journal.NewReader(r, nil, true, true) 31 // for { 32 // j, err := journals.Next() 33 // if err == io.EOF { 34 // break 35 // } 36 // if err != nil { 37 // return nil, err 38 // } 39 // s, err := ioutil.ReadAll(j) 40 // if err != nil { 41 // return nil, err 42 // } 43 // ss = append(ss, string(s)) 44 // } 45 // return ss, nil 46 // } 47 // 48 // func write(w io.Writer, ss []string) error { 49 // journals := journal.NewWriter(w) 50 // for _, s := range ss { 51 // j, err := journals.Next() 52 // if err != nil { 53 // return err 54 // } 55 // if _, err := j.Write([]byte(s)), err != nil { 56 // return err 57 // } 58 // } 59 // return journals.Close() 60 // } 61 // 62 // The wire format is that the stream is divided into 32KiB blocks, and each 63 // block contains a number of tightly packed chunks. Chunks cannot cross block 64 // boundaries. The last block may be shorter than 32 KiB. Any unused bytes in a 65 // block must be zero. 66 // 67 // A journal maps to one or more chunks. Each chunk has a 7 byte header (a 4 68 // byte checksum, a 2 byte little-endian uint16 length, and a 1 byte chunk type) 69 // followed by a payload. The checksum is over the chunk type and the payload. 70 // 71 // There are four chunk types: whether the chunk is the full journal, or the 72 // first, middle or last chunk of a multi-chunk journal. A multi-chunk journal 73 // has one first chunk, zero or more middle chunks, and one last chunk. 74 // 75 // The wire format allows for limited recovery in the face of data corruption: 76 // on a format error (such as a checksum mismatch), the reader moves to the 77 // next block and looks for the next full or first chunk. 78 package journal 79 80 import ( 81 "encoding/binary" 82 "fmt" 83 "io" 84 85 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/errors" 86 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/storage" 87 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/util" 88 ) 89 90 // These constants are part of the wire format and should not be changed. 91 const ( 92 fullChunkType = 1 93 firstChunkType = 2 94 middleChunkType = 3 95 lastChunkType = 4 96 ) 97 98 const ( 99 blockSize = 32 * 1024 100 headerSize = 7 101 ) 102 103 type flusher interface { 104 Flush() error 105 } 106 107 // ErrCorrupted is the error type that generated by corrupted block or chunk. 108 type ErrCorrupted struct { 109 Size int 110 Reason string 111 } 112 113 func (e *ErrCorrupted) Error() string { 114 return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size) 115 } 116 117 // Dropper is the interface that wrap simple Drop method. The Drop 118 // method will be called when the journal reader dropping a block or chunk. 119 type Dropper interface { 120 Drop(err error) 121 } 122 123 // Reader reads journals from an underlying io.Reader. 124 type Reader struct { 125 // r is the underlying reader. 126 r io.Reader 127 // the dropper. 128 dropper Dropper 129 // strict flag. 130 strict bool 131 // checksum flag. 132 checksum bool 133 // seq is the sequence number of the current journal. 134 seq int 135 // buf[i:j] is the unread portion of the current chunk's payload. 136 // The low bound, i, excludes the chunk header. 137 i, j int 138 // n is the number of bytes of buf that are valid. Once reading has started, 139 // only the final block can have n < blockSize. 140 n int 141 // last is whether the current chunk is the last chunk of the journal. 142 last bool 143 // err is any accumulated error. 144 err error 145 // buf is the buffer. 146 buf [blockSize]byte 147 } 148 149 // NewReader returns a new reader. The dropper may be nil, and if 150 // strict is true then corrupted or invalid chunk will halt the journal 151 // reader entirely. 152 func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader { 153 return &Reader{ 154 r: r, 155 dropper: dropper, 156 strict: strict, 157 checksum: checksum, 158 last: true, 159 } 160 } 161 162 var errSkip = errors.New("leveldb/journal: skipped") 163 164 func (r *Reader) corrupt(n int, reason string, skip bool) error { 165 if r.dropper != nil { 166 r.dropper.Drop(&ErrCorrupted{n, reason}) 167 } 168 if r.strict && !skip { 169 r.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrCorrupted{n, reason}) 170 return r.err 171 } 172 return errSkip 173 } 174 175 // nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the 176 // next block into the buffer if necessary. 177 func (r *Reader) nextChunk(first bool) error { 178 for { 179 if r.j+headerSize <= r.n { 180 checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4]) 181 length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6]) 182 chunkType := r.buf[r.j+6] 183 184 if checksum == 0 && length == 0 && chunkType == 0 { 185 // Drop entire block. 186 m := r.n - r.j 187 r.i = r.n 188 r.j = r.n 189 return r.corrupt(m, "zero header", false) 190 } else { 191 m := r.n - r.j 192 r.i = r.j + headerSize 193 r.j = r.j + headerSize + int(length) 194 if r.j > r.n { 195 // Drop entire block. 196 r.i = r.n 197 r.j = r.n 198 return r.corrupt(m, "chunk length overflows block", false) 199 } else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() { 200 // Drop entire block. 201 r.i = r.n 202 r.j = r.n 203 return r.corrupt(m, "checksum mismatch", false) 204 } 205 } 206 if first && chunkType != fullChunkType && chunkType != firstChunkType { 207 m := r.j - r.i 208 r.i = r.j 209 // Report the error, but skip it. 210 return r.corrupt(m+headerSize, "orphan chunk", true) 211 } 212 r.last = chunkType == fullChunkType || chunkType == lastChunkType 213 return nil 214 } 215 216 // The last block. 217 if r.n < blockSize && r.n > 0 { 218 if !first { 219 return r.corrupt(0, "missing chunk part", false) 220 } 221 r.err = io.EOF 222 return r.err 223 } 224 225 // Read block. 226 n, err := io.ReadFull(r.r, r.buf[:]) 227 if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 228 return err 229 } 230 if n == 0 { 231 if !first { 232 return r.corrupt(0, "missing chunk part", false) 233 } 234 r.err = io.EOF 235 return r.err 236 } 237 r.i, r.j, r.n = 0, 0, n 238 } 239 } 240 241 // Next returns a reader for the next journal. It returns io.EOF if there are no 242 // more journals. The reader returned becomes stale after the next Next call, 243 // and should no longer be used. If strict is false, the reader will returns 244 // io.ErrUnexpectedEOF error when found corrupted journal. 245 func (r *Reader) Next() (io.Reader, error) { 246 r.seq++ 247 if r.err != nil { 248 return nil, r.err 249 } 250 r.i = r.j 251 for { 252 if err := r.nextChunk(true); err == nil { 253 break 254 } else if err != errSkip { 255 return nil, err 256 } 257 } 258 return &singleReader{r, r.seq, nil}, nil 259 } 260 261 // Reset resets the journal reader, allows reuse of the journal reader. Reset returns 262 // last accumulated error. 263 func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error { 264 r.seq++ 265 err := r.err 266 r.r = reader 267 r.dropper = dropper 268 r.strict = strict 269 r.checksum = checksum 270 r.i = 0 271 r.j = 0 272 r.n = 0 273 r.last = true 274 r.err = nil 275 return err 276 } 277 278 type singleReader struct { 279 r *Reader 280 seq int 281 err error 282 } 283 284 func (x *singleReader) Read(p []byte) (int, error) { 285 r := x.r 286 if r.seq != x.seq { 287 return 0, errors.New("leveldb/journal: stale reader") 288 } 289 if x.err != nil { 290 return 0, x.err 291 } 292 if r.err != nil { 293 return 0, r.err 294 } 295 for r.i == r.j { 296 if r.last { 297 return 0, io.EOF 298 } 299 x.err = r.nextChunk(false) 300 if x.err != nil { 301 if x.err == errSkip { 302 x.err = io.ErrUnexpectedEOF 303 } 304 return 0, x.err 305 } 306 } 307 n := copy(p, r.buf[r.i:r.j]) 308 r.i += n 309 return n, nil 310 } 311 312 func (x *singleReader) ReadByte() (byte, error) { 313 r := x.r 314 if r.seq != x.seq { 315 return 0, errors.New("leveldb/journal: stale reader") 316 } 317 if x.err != nil { 318 return 0, x.err 319 } 320 if r.err != nil { 321 return 0, r.err 322 } 323 for r.i == r.j { 324 if r.last { 325 return 0, io.EOF 326 } 327 x.err = r.nextChunk(false) 328 if x.err != nil { 329 if x.err == errSkip { 330 x.err = io.ErrUnexpectedEOF 331 } 332 return 0, x.err 333 } 334 } 335 c := r.buf[r.i] 336 r.i++ 337 return c, nil 338 } 339 340 // Writer writes journals to an underlying io.Writer. 341 type Writer struct { 342 // w is the underlying writer. 343 w io.Writer 344 // seq is the sequence number of the current journal. 345 seq int 346 // f is w as a flusher. 347 f flusher 348 // buf[i:j] is the bytes that will become the current chunk. 349 // The low bound, i, includes the chunk header. 350 i, j int 351 // buf[:written] has already been written to w. 352 // written is zero unless Flush has been called. 353 written int 354 // first is whether the current chunk is the first chunk of the journal. 355 first bool 356 // pending is whether a chunk is buffered but not yet written. 357 pending bool 358 // err is any accumulated error. 359 err error 360 // buf is the buffer. 361 buf [blockSize]byte 362 } 363 364 // NewWriter returns a new Writer. 365 func NewWriter(w io.Writer) *Writer { 366 f, _ := w.(flusher) 367 return &Writer{ 368 w: w, 369 f: f, 370 } 371 } 372 373 // fillHeader fills in the header for the pending chunk. 374 func (w *Writer) fillHeader(last bool) { 375 if w.i+headerSize > w.j || w.j > blockSize { 376 panic("leveldb/journal: bad writer state") 377 } 378 if last { 379 if w.first { 380 w.buf[w.i+6] = fullChunkType 381 } else { 382 w.buf[w.i+6] = lastChunkType 383 } 384 } else { 385 if w.first { 386 w.buf[w.i+6] = firstChunkType 387 } else { 388 w.buf[w.i+6] = middleChunkType 389 } 390 } 391 binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], util.NewCRC(w.buf[w.i+6:w.j]).Value()) 392 binary.LittleEndian.PutUint16(w.buf[w.i+4:w.i+6], uint16(w.j-w.i-headerSize)) 393 } 394 395 // writeBlock writes the buffered block to the underlying writer, and reserves 396 // space for the next chunk's header. 397 func (w *Writer) writeBlock() { 398 _, w.err = w.w.Write(w.buf[w.written:]) 399 w.i = 0 400 w.j = headerSize 401 w.written = 0 402 } 403 404 // writePending finishes the current journal and writes the buffer to the 405 // underlying writer. 406 func (w *Writer) writePending() { 407 if w.err != nil { 408 return 409 } 410 if w.pending { 411 w.fillHeader(true) 412 w.pending = false 413 } 414 _, w.err = w.w.Write(w.buf[w.written:w.j]) 415 w.written = w.j 416 } 417 418 // Close finishes the current journal and closes the writer. 419 func (w *Writer) Close() error { 420 w.seq++ 421 w.writePending() 422 if w.err != nil { 423 return w.err 424 } 425 w.err = errors.New("leveldb/journal: closed Writer") 426 return nil 427 } 428 429 // Flush finishes the current journal, writes to the underlying writer, and 430 // flushes it if that writer implements interface{ Flush() error }. 431 func (w *Writer) Flush() error { 432 w.seq++ 433 w.writePending() 434 if w.err != nil { 435 return w.err 436 } 437 if w.f != nil { 438 w.err = w.f.Flush() 439 return w.err 440 } 441 return nil 442 } 443 444 // Reset resets the journal writer, allows reuse of the journal writer. Reset 445 // will also closes the journal writer if not already. 446 func (w *Writer) Reset(writer io.Writer) (err error) { 447 w.seq++ 448 if w.err == nil { 449 w.writePending() 450 err = w.err 451 } 452 w.w = writer 453 w.f, _ = writer.(flusher) 454 w.i = 0 455 w.j = 0 456 w.written = 0 457 w.first = false 458 w.pending = false 459 w.err = nil 460 return 461 } 462 463 // Next returns a writer for the next journal. The writer returned becomes stale 464 // after the next Close, Flush or Next call, and should no longer be used. 465 func (w *Writer) Next() (io.Writer, error) { 466 w.seq++ 467 if w.err != nil { 468 return nil, w.err 469 } 470 if w.pending { 471 w.fillHeader(true) 472 } 473 w.i = w.j 474 w.j = w.j + headerSize 475 // Check if there is room in the block for the header. 476 if w.j > blockSize { 477 // Fill in the rest of the block with zeroes. 478 for k := w.i; k < blockSize; k++ { 479 w.buf[k] = 0 480 } 481 w.writeBlock() 482 if w.err != nil { 483 return nil, w.err 484 } 485 } 486 w.first = true 487 w.pending = true 488 return singleWriter{w, w.seq}, nil 489 } 490 491 type singleWriter struct { 492 w *Writer 493 seq int 494 } 495 496 func (x singleWriter) Write(p []byte) (int, error) { 497 w := x.w 498 if w.seq != x.seq { 499 return 0, errors.New("leveldb/journal: stale writer") 500 } 501 if w.err != nil { 502 return 0, w.err 503 } 504 n0 := len(p) 505 for len(p) > 0 { 506 // Write a block, if it is full. 507 if w.j == blockSize { 508 w.fillHeader(false) 509 w.writeBlock() 510 if w.err != nil { 511 return 0, w.err 512 } 513 w.first = false 514 } 515 // Copy bytes into the buffer. 516 n := copy(w.buf[w.j:], p) 517 w.j += n 518 p = p[n:] 519 } 520 return n0, nil 521 }