storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/os-readdir_unix.go (about)

     1  //go:build (linux && !appengine) || darwin || freebsd || netbsd || openbsd
     2  // +build linux,!appengine darwin freebsd netbsd openbsd
     3  
     4  /*
     5   * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
     6   *
     7   * Licensed under the Apache License, Version 2.0 (the "License");
     8   * you may not use this file except in compliance with the License.
     9   * You may obtain a copy of the License at
    10   *
    11   *     http://www.apache.org/licenses/LICENSE-2.0
    12   *
    13   * Unless required by applicable law or agreed to in writing, software
    14   * distributed under the License is distributed on an "AS IS" BASIS,
    15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16   * See the License for the specific language governing permissions and
    17   * limitations under the License.
    18   */
    19  
    20  package cmd
    21  
    22  import (
    23  	"bytes"
    24  	"fmt"
    25  	"os"
    26  	"sync"
    27  	"syscall"
    28  	"unsafe"
    29  
    30  	"golang.org/x/sys/unix"
    31  )
    32  
    33  func access(name string) error {
    34  	if err := unix.Access(name, unix.R_OK|unix.W_OK); err != nil {
    35  		return &os.PathError{Op: "lstat", Path: name, Err: err}
    36  	}
    37  	return nil
    38  }
    39  
    40  // The buffer must be at least a block long.
    41  // refer https://github.com/golang/go/issues/24015
    42  const blockSize = 8 << 10 // 8192
    43  
    44  // By default atleast 128 entries in single getdents call (1MiB buffer)
    45  var direntPool = sync.Pool{
    46  	New: func() interface{} {
    47  		buf := make([]byte, blockSize*128)
    48  		return &buf
    49  	},
    50  }
    51  
    52  // unexpectedFileMode is a sentinel (and bogus) os.FileMode
    53  // value used to represent a syscall.DT_UNKNOWN Dirent.Type.
    54  const unexpectedFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice
    55  
    56  func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err error) {
    57  	// golang.org/issue/15653
    58  	dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0]))
    59  	if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
    60  		return consumed, nil, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v)
    61  	}
    62  	if len(buf) < int(dirent.Reclen) {
    63  		return consumed, nil, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen)
    64  	}
    65  	consumed = int(dirent.Reclen)
    66  	if direntInode(dirent) == 0 { // File absent in directory.
    67  		return
    68  	}
    69  	switch dirent.Type {
    70  	case syscall.DT_REG:
    71  		typ = 0
    72  	case syscall.DT_DIR:
    73  		typ = os.ModeDir
    74  	case syscall.DT_LNK:
    75  		typ = os.ModeSymlink
    76  	default:
    77  		// Skip all other file types. Revisit if/when this code needs
    78  		// to handle such files, MinIO is only interested in
    79  		// files and directories.
    80  		typ = unexpectedFileMode
    81  	}
    82  
    83  	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
    84  	nameLen, err := direntNamlen(dirent)
    85  	if err != nil {
    86  		return consumed, nil, typ, err
    87  	}
    88  
    89  	return consumed, nameBuf[:nameLen], typ, nil
    90  }
    91  
    92  // Return all the entries at the directory dirPath.
    93  func readDir(dirPath string) (entries []string, err error) {
    94  	return readDirN(dirPath, -1)
    95  }
    96  
    97  // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
    98  // the directory itself, if the dirPath doesn't exist this function doesn't return
    99  // an error.
   100  func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
   101  	f, err := os.Open(dirPath)
   102  	if err != nil {
   103  		if osErrToFileErr(err) == errFileNotFound {
   104  			return nil
   105  		}
   106  		return osErrToFileErr(err)
   107  	}
   108  	defer f.Close()
   109  
   110  	buf := make([]byte, blockSize)
   111  	boff := 0 // starting read position in buf
   112  	nbuf := 0 // end valid data in buf
   113  
   114  	for {
   115  		if boff >= nbuf {
   116  			boff = 0
   117  			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
   118  			if err != nil {
   119  				if isSysErrNotDir(err) {
   120  					return nil
   121  				}
   122  				err = osErrToFileErr(err)
   123  				if err == errFileNotFound {
   124  					return nil
   125  				}
   126  				return err
   127  			}
   128  			if nbuf <= 0 {
   129  				break // EOF
   130  			}
   131  		}
   132  		consumed, name, typ, err := parseDirEnt(buf[boff:nbuf])
   133  		if err != nil {
   134  			return err
   135  		}
   136  		boff += consumed
   137  		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
   138  			continue
   139  		}
   140  
   141  		// Fallback for filesystems (like old XFS) that don't
   142  		// support Dirent.Type and have DT_UNKNOWN (0) there
   143  		// instead.
   144  		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
   145  			fi, err := os.Stat(pathJoin(dirPath, string(name)))
   146  			if err != nil {
   147  				// It got deleted in the meantime, not found
   148  				// or returns too many symlinks ignore this
   149  				// file/directory.
   150  				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
   151  					isSysErrTooManySymlinks(err) {
   152  					continue
   153  				}
   154  				return err
   155  			}
   156  
   157  			// Ignore symlinked directories.
   158  			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
   159  				continue
   160  			}
   161  
   162  			typ = fi.Mode() & os.ModeType
   163  		}
   164  		if err = fn(string(name), typ); err == errDoneForNow {
   165  			// fn() requested to return by caller.
   166  			return nil
   167  		}
   168  	}
   169  
   170  	return err
   171  }
   172  
   173  // Return count entries at the directory dirPath and all entries
   174  // if count is set to -1
   175  func readDirN(dirPath string, count int) (entries []string, err error) {
   176  	f, err := os.Open(dirPath)
   177  	if err != nil {
   178  		return nil, osErrToFileErr(err)
   179  	}
   180  	defer f.Close()
   181  
   182  	bufp := direntPool.Get().(*[]byte)
   183  	defer direntPool.Put(bufp)
   184  
   185  	nameTmp := direntPool.Get().(*[]byte)
   186  	defer direntPool.Put(nameTmp)
   187  	tmp := *nameTmp
   188  
   189  	boff := 0 // starting read position in buf
   190  	nbuf := 0 // end valid data in buf
   191  
   192  	for count != 0 {
   193  		if boff >= nbuf {
   194  			boff = 0
   195  			nbuf, err = syscall.ReadDirent(int(f.Fd()), *bufp)
   196  			if err != nil {
   197  				if isSysErrNotDir(err) {
   198  					return nil, errFileNotFound
   199  				}
   200  				return nil, osErrToFileErr(err)
   201  			}
   202  			if nbuf <= 0 {
   203  				break
   204  			}
   205  		}
   206  		consumed, name, typ, err := parseDirEnt((*bufp)[boff:nbuf])
   207  		if err != nil {
   208  			return nil, err
   209  		}
   210  		boff += consumed
   211  		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
   212  			continue
   213  		}
   214  
   215  		// Fallback for filesystems (like old XFS) that don't
   216  		// support Dirent.Type and have DT_UNKNOWN (0) there
   217  		// instead.
   218  		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
   219  			fi, err := os.Stat(pathJoin(dirPath, string(name)))
   220  			if err != nil {
   221  				// It got deleted in the meantime, not found
   222  				// or returns too many symlinks ignore this
   223  				// file/directory.
   224  				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
   225  					isSysErrTooManySymlinks(err) {
   226  					continue
   227  				}
   228  				return nil, err
   229  			}
   230  
   231  			// Ignore symlinked directories.
   232  			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
   233  				continue
   234  			}
   235  
   236  			typ = fi.Mode() & os.ModeType
   237  		}
   238  
   239  		var nameStr string
   240  		if typ.IsRegular() {
   241  			nameStr = string(name)
   242  		} else if typ.IsDir() {
   243  			// Use temp buffer to append a slash to avoid string concat.
   244  			tmp = tmp[:len(name)+1]
   245  			copy(tmp, name)
   246  			tmp[len(tmp)-1] = '/' // SlashSeparator
   247  			nameStr = string(tmp)
   248  		}
   249  
   250  		count--
   251  		entries = append(entries, nameStr)
   252  	}
   253  
   254  	return
   255  }
   256  
   257  func globalSync() {
   258  	syscall.Sync()
   259  }