github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/syndtr/goleveldb/leveldb/storage/file_storage.go (about) 1 // Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> 2 // All rights reservefs. 3 // 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 7 package storage 8 9 import ( 10 "errors" 11 "fmt" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "runtime" 16 "strconv" 17 "strings" 18 "sync" 19 "time" 20 ) 21 22 var ( 23 errFileOpen = errors.New("leveldb/storage: file still open") 24 errReadOnly = errors.New("leveldb/storage: storage is read-only") 25 ) 26 27 type fileLock interface { 28 release() error 29 } 30 31 type fileStorageLock struct { 32 fs *fileStorage 33 } 34 35 func (lock *fileStorageLock) Release() { 36 if lock.fs != nil { 37 lock.fs.mu.Lock() 38 defer lock.fs.mu.Unlock() 39 if lock.fs.slock == lock { 40 lock.fs.slock = nil 41 } 42 } 43 } 44 45 const logSizeThreshold = 1024 * 1024 // 1 MiB 46 47 // fileStorage is a file-system backed storage. 48 type fileStorage struct { 49 path string 50 readOnly bool 51 52 mu sync.Mutex 53 flock fileLock 54 slock *fileStorageLock 55 logw *os.File 56 logSize int64 57 buf []byte 58 // Opened file counter; if open < 0 means closed. 59 open int 60 day int 61 } 62 63 // OpenFile returns a new filesytem-backed storage implementation with the given 64 // path. This also acquire a file lock, so any subsequent attempt to open the 65 // same path will fail. 66 // 67 // The storage must be closed after use, by calling Close method. 68 func OpenFile(path string, readOnly bool) (Storage, error) { 69 if fi, err := os.Stat(path); err == nil { 70 if !fi.IsDir() { 71 return nil, fmt.Errorf("leveldb/storage: open %s: not a directory", path) 72 } 73 } else if os.IsNotExist(err) && !readOnly { 74 if err := os.MkdirAll(path, 0755); err != nil { 75 return nil, err 76 } 77 } else { 78 return nil, err 79 } 80 81 flock, err := newFileLock(filepath.Join(path, "LOCK"), readOnly) 82 if err != nil { 83 return nil, err 84 } 85 86 defer func() { 87 if err != nil { 88 flock.release() 89 } 90 }() 91 92 var ( 93 logw *os.File 94 logSize int64 95 ) 96 if !readOnly { 97 logw, err = os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644) 98 if err != nil { 99 return nil, err 100 } 101 logSize, err = logw.Seek(0, os.SEEK_END) 102 if err != nil { 103 logw.Close() 104 return nil, err 105 } 106 } 107 108 fs := &fileStorage{ 109 path: path, 110 readOnly: readOnly, 111 flock: flock, 112 logw: logw, 113 logSize: logSize, 114 } 115 runtime.SetFinalizer(fs, (*fileStorage).Close) 116 return fs, nil 117 } 118 119 func (fs *fileStorage) Lock() (Lock, error) { 120 fs.mu.Lock() 121 defer fs.mu.Unlock() 122 if fs.open < 0 { 123 return nil, ErrClosed 124 } 125 if fs.readOnly { 126 return &fileStorageLock{}, nil 127 } 128 if fs.slock != nil { 129 return nil, ErrLocked 130 } 131 fs.slock = &fileStorageLock{fs: fs} 132 return fs.slock, nil 133 } 134 135 func itoa(buf []byte, i int, wid int) []byte { 136 u := uint(i) 137 if u == 0 && wid <= 1 { 138 return append(buf, '0') 139 } 140 141 // Assemble decimal in reverse order. 142 var b [32]byte 143 bp := len(b) 144 for ; u > 0 || wid > 0; u /= 10 { 145 bp-- 146 wid-- 147 b[bp] = byte(u%10) + '0' 148 } 149 return append(buf, b[bp:]...) 150 } 151 152 func (fs *fileStorage) printDay(t time.Time) { 153 if fs.day == t.Day() { 154 return 155 } 156 fs.day = t.Day() 157 fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n")) 158 } 159 160 func (fs *fileStorage) doLog(t time.Time, str string) { 161 if fs.logSize > logSizeThreshold { 162 // Rotate log file. 163 fs.logw.Close() 164 fs.logw = nil 165 fs.logSize = 0 166 rename(filepath.Join(fs.path, "LOG"), filepath.Join(fs.path, "LOG.old")) 167 } 168 if fs.logw == nil { 169 var err error 170 fs.logw, err = os.OpenFile(filepath.Join(fs.path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644) 171 if err != nil { 172 return 173 } 174 // Force printDay on new log file. 175 fs.day = 0 176 } 177 fs.printDay(t) 178 hour, min, sec := t.Clock() 179 msec := t.Nanosecond() / 1e3 180 // time 181 fs.buf = itoa(fs.buf[:0], hour, 2) 182 fs.buf = append(fs.buf, ':') 183 fs.buf = itoa(fs.buf, min, 2) 184 fs.buf = append(fs.buf, ':') 185 fs.buf = itoa(fs.buf, sec, 2) 186 fs.buf = append(fs.buf, '.') 187 fs.buf = itoa(fs.buf, msec, 6) 188 fs.buf = append(fs.buf, ' ') 189 // write 190 fs.buf = append(fs.buf, []byte(str)...) 191 fs.buf = append(fs.buf, '\n') 192 fs.logw.Write(fs.buf) 193 } 194 195 func (fs *fileStorage) Log(str string) { 196 if !fs.readOnly { 197 t := time.Now() 198 fs.mu.Lock() 199 defer fs.mu.Unlock() 200 if fs.open < 0 { 201 return 202 } 203 fs.doLog(t, str) 204 } 205 } 206 207 func (fs *fileStorage) log(str string) { 208 if !fs.readOnly { 209 fs.doLog(time.Now(), str) 210 } 211 } 212 213 func (fs *fileStorage) SetMeta(fd FileDesc) (err error) { 214 if !FileDescOk(fd) { 215 return ErrInvalidFile 216 } 217 if fs.readOnly { 218 return errReadOnly 219 } 220 221 fs.mu.Lock() 222 defer fs.mu.Unlock() 223 if fs.open < 0 { 224 return ErrClosed 225 } 226 defer func() { 227 if err != nil { 228 fs.log(fmt.Sprintf("CURRENT: %v", err)) 229 } 230 }() 231 path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num) 232 w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 233 if err != nil { 234 return 235 } 236 _, err = fmt.Fprintln(w, fsGenName(fd)) 237 // Close the file first. 238 if cerr := w.Close(); cerr != nil { 239 fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, cerr)) 240 } 241 if err != nil { 242 return 243 } 244 return rename(path, filepath.Join(fs.path, "CURRENT")) 245 } 246 247 func (fs *fileStorage) GetMeta() (fd FileDesc, err error) { 248 fs.mu.Lock() 249 defer fs.mu.Unlock() 250 if fs.open < 0 { 251 return FileDesc{}, ErrClosed 252 } 253 dir, err := os.Open(fs.path) 254 if err != nil { 255 return 256 } 257 names, err := dir.Readdirnames(0) 258 // Close the dir first before checking for Readdirnames error. 259 if ce := dir.Close(); ce != nil { 260 fs.log(fmt.Sprintf("close dir: %v", ce)) 261 } 262 if err != nil { 263 return 264 } 265 // Find latest CURRENT file. 266 var rem []string 267 var pend bool 268 var cerr error 269 for _, name := range names { 270 if strings.HasPrefix(name, "CURRENT") { 271 pend1 := len(name) > 7 272 var pendNum int64 273 // Make sure it is valid name for a CURRENT file, otherwise skip it. 274 if pend1 { 275 if name[7] != '.' || len(name) < 9 { 276 fs.log(fmt.Sprintf("skipping %s: invalid file name", name)) 277 continue 278 } 279 var e1 error 280 if pendNum, e1 = strconv.ParseInt(name[8:], 10, 0); e1 != nil { 281 fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", name, e1)) 282 continue 283 } 284 } 285 path := filepath.Join(fs.path, name) 286 r, e1 := os.OpenFile(path, os.O_RDONLY, 0) 287 if e1 != nil { 288 return FileDesc{}, e1 289 } 290 b, e1 := ioutil.ReadAll(r) 291 if e1 != nil { 292 r.Close() 293 return FileDesc{}, e1 294 } 295 var fd1 FileDesc 296 if len(b) < 1 || b[len(b)-1] != '\n' || !fsParseNamePtr(string(b[:len(b)-1]), &fd1) { 297 fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", name)) 298 if pend1 { 299 rem = append(rem, name) 300 } 301 if !pend1 || cerr == nil { 302 metaFd, _ := fsParseName(name) 303 cerr = &ErrCorrupted{ 304 Fd: metaFd, 305 Err: errors.New("leveldb/storage: corrupted or incomplete meta file"), 306 } 307 } 308 } else if pend1 && pendNum != fd1.Num { 309 fs.log(fmt.Sprintf("skipping %s: inconsistent pending-file num: %d vs %d", name, pendNum, fd1.Num)) 310 rem = append(rem, name) 311 } else if fd1.Num < fd.Num { 312 fs.log(fmt.Sprintf("skipping %s: obsolete", name)) 313 if pend1 { 314 rem = append(rem, name) 315 } 316 } else { 317 fd = fd1 318 pend = pend1 319 } 320 if err := r.Close(); err != nil { 321 fs.log(fmt.Sprintf("close %s: %v", name, err)) 322 } 323 } 324 } 325 // Don't remove any files if there is no valid CURRENT file. 326 if fd.Nil() { 327 if cerr != nil { 328 err = cerr 329 } else { 330 err = os.ErrNotExist 331 } 332 return 333 } 334 if !fs.readOnly { 335 // Rename pending CURRENT file to an effective CURRENT. 336 if pend { 337 path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num) 338 if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil { 339 fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", fd.Num, err)) 340 } 341 } 342 // Remove obsolete or incomplete pending CURRENT files. 343 for _, name := range rem { 344 path := filepath.Join(fs.path, name) 345 if err := os.Remove(path); err != nil { 346 fs.log(fmt.Sprintf("remove %s: %v", name, err)) 347 } 348 } 349 } 350 return 351 } 352 353 func (fs *fileStorage) List(ft FileType) (fds []FileDesc, err error) { 354 fs.mu.Lock() 355 defer fs.mu.Unlock() 356 if fs.open < 0 { 357 return nil, ErrClosed 358 } 359 dir, err := os.Open(fs.path) 360 if err != nil { 361 return 362 } 363 names, err := dir.Readdirnames(0) 364 // Close the dir first before checking for Readdirnames error. 365 if cerr := dir.Close(); cerr != nil { 366 fs.log(fmt.Sprintf("close dir: %v", cerr)) 367 } 368 if err == nil { 369 for _, name := range names { 370 if fd, ok := fsParseName(name); ok && fd.Type&ft != 0 { 371 fds = append(fds, fd) 372 } 373 } 374 } 375 return 376 } 377 378 func (fs *fileStorage) Open(fd FileDesc) (Reader, error) { 379 if !FileDescOk(fd) { 380 return nil, ErrInvalidFile 381 } 382 383 fs.mu.Lock() 384 defer fs.mu.Unlock() 385 if fs.open < 0 { 386 return nil, ErrClosed 387 } 388 of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_RDONLY, 0) 389 if err != nil { 390 if fsHasOldName(fd) && os.IsNotExist(err) { 391 of, err = os.OpenFile(filepath.Join(fs.path, fsGenOldName(fd)), os.O_RDONLY, 0) 392 if err == nil { 393 goto ok 394 } 395 } 396 return nil, err 397 } 398 ok: 399 fs.open++ 400 return &fileWrap{File: of, fs: fs, fd: fd}, nil 401 } 402 403 func (fs *fileStorage) Create(fd FileDesc) (Writer, error) { 404 if !FileDescOk(fd) { 405 return nil, ErrInvalidFile 406 } 407 if fs.readOnly { 408 return nil, errReadOnly 409 } 410 411 fs.mu.Lock() 412 defer fs.mu.Unlock() 413 if fs.open < 0 { 414 return nil, ErrClosed 415 } 416 of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 417 if err != nil { 418 return nil, err 419 } 420 fs.open++ 421 return &fileWrap{File: of, fs: fs, fd: fd}, nil 422 } 423 424 func (fs *fileStorage) Remove(fd FileDesc) error { 425 if !FileDescOk(fd) { 426 return ErrInvalidFile 427 } 428 if fs.readOnly { 429 return errReadOnly 430 } 431 432 fs.mu.Lock() 433 defer fs.mu.Unlock() 434 if fs.open < 0 { 435 return ErrClosed 436 } 437 err := os.Remove(filepath.Join(fs.path, fsGenName(fd))) 438 if err != nil { 439 if fsHasOldName(fd) && os.IsNotExist(err) { 440 if e1 := os.Remove(filepath.Join(fs.path, fsGenOldName(fd))); !os.IsNotExist(e1) { 441 fs.log(fmt.Sprintf("remove %s: %v (old name)", fd, err)) 442 err = e1 443 } 444 } else { 445 fs.log(fmt.Sprintf("remove %s: %v", fd, err)) 446 } 447 } 448 return err 449 } 450 451 func (fs *fileStorage) Rename(oldfd, newfd FileDesc) error { 452 if !FileDescOk(oldfd) || !FileDescOk(newfd) { 453 return ErrInvalidFile 454 } 455 if oldfd == newfd { 456 return nil 457 } 458 if fs.readOnly { 459 return errReadOnly 460 } 461 462 fs.mu.Lock() 463 defer fs.mu.Unlock() 464 if fs.open < 0 { 465 return ErrClosed 466 } 467 return rename(filepath.Join(fs.path, fsGenName(oldfd)), filepath.Join(fs.path, fsGenName(newfd))) 468 } 469 470 func (fs *fileStorage) Close() error { 471 fs.mu.Lock() 472 defer fs.mu.Unlock() 473 if fs.open < 0 { 474 return ErrClosed 475 } 476 // Clear the finalizer. 477 runtime.SetFinalizer(fs, nil) 478 479 if fs.open > 0 { 480 fs.log(fmt.Sprintf("close: warning, %d files still open", fs.open)) 481 } 482 fs.open = -1 483 if fs.logw != nil { 484 fs.logw.Close() 485 } 486 return fs.flock.release() 487 } 488 489 type fileWrap struct { 490 *os.File 491 fs *fileStorage 492 fd FileDesc 493 closed bool 494 } 495 496 func (fw *fileWrap) Sync() error { 497 if err := fw.File.Sync(); err != nil { 498 return err 499 } 500 if fw.fd.Type == TypeManifest { 501 // Also sync parent directory if file type is manifest. 502 // See: https://code.google.com/p/leveldb/issues/detail?id=190. 503 if err := syncDir(fw.fs.path); err != nil { 504 fw.fs.log(fmt.Sprintf("syncDir: %v", err)) 505 return err 506 } 507 } 508 return nil 509 } 510 511 func (fw *fileWrap) Close() error { 512 fw.fs.mu.Lock() 513 defer fw.fs.mu.Unlock() 514 if fw.closed { 515 return ErrClosed 516 } 517 fw.closed = true 518 fw.fs.open-- 519 err := fw.File.Close() 520 if err != nil { 521 fw.fs.log(fmt.Sprintf("close %s: %v", fw.fd, err)) 522 } 523 return err 524 } 525 526 func fsGenName(fd FileDesc) string { 527 switch fd.Type { 528 case TypeManifest: 529 return fmt.Sprintf("MANIFEST-%06d", fd.Num) 530 case TypeJournal: 531 return fmt.Sprintf("%06d.log", fd.Num) 532 case TypeTable: 533 return fmt.Sprintf("%06d.ldb", fd.Num) 534 case TypeTemp: 535 return fmt.Sprintf("%06d.tmp", fd.Num) 536 default: 537 panic("invalid file type") 538 } 539 } 540 541 func fsHasOldName(fd FileDesc) bool { 542 return fd.Type == TypeTable 543 } 544 545 func fsGenOldName(fd FileDesc) string { 546 switch fd.Type { 547 case TypeTable: 548 return fmt.Sprintf("%06d.sst", fd.Num) 549 } 550 return fsGenName(fd) 551 } 552 553 func fsParseName(name string) (fd FileDesc, ok bool) { 554 var tail string 555 _, err := fmt.Sscanf(name, "%d.%s", &fd.Num, &tail) 556 if err == nil { 557 switch tail { 558 case "log": 559 fd.Type = TypeJournal 560 case "ldb", "sst": 561 fd.Type = TypeTable 562 case "tmp": 563 fd.Type = TypeTemp 564 default: 565 return 566 } 567 return fd, true 568 } 569 n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &fd.Num, &tail) 570 if n == 1 { 571 fd.Type = TypeManifest 572 return fd, true 573 } 574 return 575 } 576 577 func fsParseNamePtr(name string, fd *FileDesc) bool { 578 _fd, ok := fsParseName(name) 579 if fd != nil { 580 *fd = _fd 581 } 582 return ok 583 }