github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/common/fileutils/fileutils.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package fileutils
    18  
    19  import (
    20  	"io"
    21  	"os"
    22  	"path"
    23  	"syscall"
    24  
    25  	"github.com/siglens/siglens/pkg/config"
    26  	"github.com/siglens/siglens/pkg/utils/semaphore"
    27  	log "github.com/sirupsen/logrus"
    28  )
    29  
    30  /*
    31  	Exports utility functions for limiting the amount of open file descriptors across siglens
    32  */
    33  
    34  var GLOBAL_FD_LIMITER *semaphore.WeightedSemaphore
    35  var DEFAULT_MAX_OPEN_FDs uint64 = 8192
    36  
    37  /*
    38  what percent of max open FDs should we actually use?
    39  the remaining percent would be used for the FDs used by fasthttp, and misc file opens
    40  
    41  TODO: how to improve on this limit?
    42  */
    43  var OPEN_FD_THRESHOLD_PERCENT uint64 = 70
    44  
    45  func init() {
    46  	var numFiles uint64
    47  	var rLimit syscall.Rlimit
    48  	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
    49  	if err != nil {
    50  		log.Errorf("Failed to get max possible number of open file descriptors. Defaulting to %+v. Error: %v",
    51  			DEFAULT_MAX_OPEN_FDs, err)
    52  		numFiles = DEFAULT_MAX_OPEN_FDs
    53  	} else {
    54  		numFiles = rLimit.Cur
    55  	}
    56  	var newLimit syscall.Rlimit
    57  	newLimit.Max = numFiles
    58  	newLimit.Cur = numFiles
    59  
    60  	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &newLimit)
    61  	if err != nil {
    62  		log.Errorf("Failed to set max possible number of open file descriptors. Defaulting to %+v. Error: %v",
    63  			rLimit.Cur, err)
    64  	} else {
    65  		numFiles = newLimit.Cur
    66  	}
    67  	maxPossibleOpenFds := int64(numFiles*OPEN_FD_THRESHOLD_PERCENT) / 100
    68  	log.Infof("Initialized FD limiter with %+v as max number of open files", maxPossibleOpenFds)
    69  	GLOBAL_FD_LIMITER = semaphore.NewDefaultWeightedSemaphore(maxPossibleOpenFds, "GlobalFileLimiter")
    70  }
    71  
    72  func IsDirEmpty(name string) bool {
    73  	f, err := os.Open(name)
    74  	if err != nil {
    75  		return false
    76  	}
    77  	defer f.Close()
    78  
    79  	// read in ONLY one file
    80  	_, err = f.Readdir(1)
    81  
    82  	// and if the file is EOF... well, the dir is empty.
    83  	return err == io.EOF
    84  }
    85  
    86  func RecursivelyDeleteEmptyParentDirectories(filePath string) {
    87  	temp := path.Dir(filePath)
    88  	for {
    89  		if temp == config.GetDataPath() {
    90  			break
    91  		}
    92  		if IsDirEmpty(temp) {
    93  			os.RemoveAll(temp)
    94  		} else {
    95  			break
    96  		}
    97  		temp = path.Dir(temp)
    98  	}
    99  }
   100  
   101  func GetAllFilesInDirectory(path string) []string {
   102  	retVal := make([]string, 0)
   103  	files, err := os.ReadDir(path)
   104  
   105  	if err != nil {
   106  		log.Errorf("GetAllFilesInDirectory Error: %v", err)
   107  		return retVal
   108  	}
   109  
   110  	for _, file := range files {
   111  		if file.IsDir() {
   112  			res := GetAllFilesInDirectory(path + file.Name() + "/")
   113  			retVal = append(retVal, res...)
   114  		} else {
   115  			retVal = append(retVal, path+file.Name())
   116  		}
   117  	}
   118  	return retVal
   119  }