github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/client/local/fs_object_client.go (about) 1 package local 2 3 import ( 4 "context" 5 "flag" 6 "fmt" 7 "io" 8 "os" 9 "path/filepath" 10 "time" 11 12 "github.com/go-kit/log/level" 13 "github.com/pkg/errors" 14 "github.com/thanos-io/thanos/pkg/runutil" 15 16 "github.com/grafana/loki/pkg/ruler/rulestore/local" 17 "github.com/grafana/loki/pkg/storage/chunk/client" 18 "github.com/grafana/loki/pkg/storage/chunk/client/util" 19 util_log "github.com/grafana/loki/pkg/util/log" 20 ) 21 22 // FSConfig is the config for a FSObjectClient. 23 type FSConfig struct { 24 Directory string `yaml:"directory"` 25 } 26 27 // RegisterFlags registers flags. 28 func (cfg *FSConfig) RegisterFlags(f *flag.FlagSet) { 29 cfg.RegisterFlagsWithPrefix("", f) 30 } 31 32 // RegisterFlags registers flags with prefix. 33 func (cfg *FSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { 34 f.StringVar(&cfg.Directory, prefix+"local.chunk-directory", "", "Directory to store chunks in.") 35 } 36 37 func (cfg *FSConfig) ToCortexLocalConfig() local.Config { 38 return local.Config{ 39 Directory: cfg.Directory, 40 } 41 } 42 43 // FSObjectClient holds config for filesystem as object store 44 type FSObjectClient struct { 45 cfg FSConfig 46 pathSeparator string 47 } 48 49 // NewFSObjectClient makes a chunk.Client which stores chunks as files in the local filesystem. 50 func NewFSObjectClient(cfg FSConfig) (*FSObjectClient, error) { 51 // filepath.Clean cleans up the path by removing unwanted duplicate slashes, dots etc. 52 // This is needed because DeleteObject works on paths which are already cleaned up and it 53 // checks whether it is about to delete the configured directory when it becomes empty 54 cfg.Directory = filepath.Clean(cfg.Directory) 55 if err := util.EnsureDirectory(cfg.Directory); err != nil { 56 return nil, err 57 } 58 59 return &FSObjectClient{ 60 cfg: cfg, 61 pathSeparator: string(os.PathSeparator), 62 }, nil 63 } 64 65 // Stop implements ObjectClient 66 func (FSObjectClient) Stop() {} 67 68 // GetObject from the store 69 func (f *FSObjectClient) GetObject(_ context.Context, objectKey string) (io.ReadCloser, int64, error) { 70 fl, err := os.Open(filepath.Join(f.cfg.Directory, filepath.FromSlash(objectKey))) 71 if err != nil { 72 return nil, 0, err 73 } 74 stats, err := fl.Stat() 75 if err != nil { 76 return nil, 0, err 77 } 78 return fl, stats.Size(), nil 79 } 80 81 // PutObject into the store 82 func (f *FSObjectClient) PutObject(_ context.Context, objectKey string, object io.ReadSeeker) error { 83 fullPath := filepath.Join(f.cfg.Directory, filepath.FromSlash(objectKey)) 84 err := util.EnsureDirectory(filepath.Dir(fullPath)) 85 if err != nil { 86 return err 87 } 88 89 fl, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) 90 if err != nil { 91 return err 92 } 93 94 defer runutil.CloseWithLogOnErr(util_log.Logger, fl, "fullPath: %s", fullPath) 95 96 _, err = io.Copy(fl, object) 97 if err != nil { 98 return err 99 } 100 101 err = fl.Sync() 102 if err != nil { 103 return err 104 } 105 106 return fl.Close() 107 } 108 109 // List implements chunk.ObjectClient. 110 // FSObjectClient assumes that prefix is a directory, and only supports "" and "/" delimiters. 111 func (f *FSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error) { 112 if delimiter != "" && delimiter != "/" { 113 return nil, nil, fmt.Errorf("unsupported delimiter: %q", delimiter) 114 } 115 116 folderPath := filepath.Join(f.cfg.Directory, filepath.FromSlash(prefix)) 117 118 info, err := os.Stat(folderPath) 119 if err != nil { 120 if os.IsNotExist(err) { 121 return nil, nil, nil 122 } 123 return nil, nil, err 124 } 125 if !info.IsDir() { 126 // When listing single file, return this file only. 127 return []client.StorageObject{{Key: info.Name(), ModifiedAt: info.ModTime()}}, nil, nil 128 } 129 130 var storageObjects []client.StorageObject 131 var commonPrefixes []client.StorageCommonPrefix 132 133 err = filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error { 134 if err != nil { 135 return err 136 } 137 138 // Ignore starting folder itself. 139 if path == folderPath { 140 return nil 141 } 142 143 relPath, err := filepath.Rel(f.cfg.Directory, path) 144 if err != nil { 145 return err 146 } 147 148 relPath = filepath.ToSlash(relPath) 149 150 if info.IsDir() { 151 if delimiter == "" { 152 // Go into directory 153 return nil 154 } 155 156 empty, err := isDirEmpty(path) 157 if err != nil { 158 return err 159 } 160 161 if !empty { 162 commonPrefixes = append(commonPrefixes, client.StorageCommonPrefix(relPath+delimiter)) 163 } 164 return filepath.SkipDir 165 } 166 167 storageObjects = append(storageObjects, client.StorageObject{Key: relPath, ModifiedAt: info.ModTime()}) 168 return nil 169 }) 170 171 return storageObjects, commonPrefixes, err 172 } 173 174 func (f *FSObjectClient) DeleteObject(ctx context.Context, objectKey string) error { 175 // inspired from https://github.com/thanos-io/thanos/blob/55cb8ca38b3539381dc6a781e637df15c694e50a/pkg/objstore/filesystem/filesystem.go#L195 176 file := filepath.Join(f.cfg.Directory, filepath.FromSlash(objectKey)) 177 178 for file != f.cfg.Directory { 179 if err := os.Remove(file); err != nil { 180 return err 181 } 182 183 file = filepath.Dir(file) 184 empty, err := isDirEmpty(file) 185 if err != nil { 186 return err 187 } 188 189 if !empty { 190 break 191 } 192 } 193 194 return nil 195 } 196 197 // DeleteChunksBefore implements BucketClient 198 func (f *FSObjectClient) DeleteChunksBefore(ctx context.Context, ts time.Time) error { 199 return filepath.Walk(f.cfg.Directory, func(path string, info os.FileInfo, err error) error { 200 if !info.IsDir() && info.ModTime().Before(ts) { 201 level.Info(util_log.Logger).Log("msg", "file has exceeded the retention period, removing it", "filepath", info.Name()) 202 if err := os.Remove(path); err != nil { 203 return err 204 } 205 } 206 return nil 207 }) 208 } 209 210 // IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations. 211 func (f *FSObjectClient) IsObjectNotFoundErr(err error) bool { 212 return os.IsNotExist(errors.Cause(err)) 213 } 214 215 // copied from https://github.com/thanos-io/thanos/blob/55cb8ca38b3539381dc6a781e637df15c694e50a/pkg/objstore/filesystem/filesystem.go#L181 216 func isDirEmpty(name string) (ok bool, err error) { 217 f, err := os.Open(name) 218 if err != nil { 219 return false, err 220 } 221 defer runutil.CloseWithErrCapture(&err, f, "dir open") 222 223 if _, err = f.Readdir(1); err == io.EOF { 224 return true, nil 225 } 226 return false, err 227 }