github.com/bpfs/defs@v0.0.15/afero/gcsfs/fs.go (about) 1 // Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>. 2 // 3 // The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp 4 // licensed under Apache License 2.0. 5 // 6 // Licensed under the Apache License, Version 2.0 (the "License"); 7 // you may not use this file except in compliance with the License. 8 // You may obtain a copy of the License at 9 // http://www.apache.org/licenses/LICENSE-2.0 10 // 11 // Unless required by applicable law or agreed to in writing, software 12 // distributed under the License is distributed on an "AS IS" BASIS, 13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 // See the License for the specific language governing permissions and 15 // limitations under the License. 16 17 package gcsfs 18 19 import ( 20 "context" 21 "errors" 22 "os" 23 "path/filepath" 24 "strings" 25 "syscall" 26 "time" 27 28 "github.com/googleapis/google-cloud-go-testing/storage/stiface" 29 ) 30 31 const ( 32 defaultFileMode = 0o755 33 gsPrefix = "gs://" 34 ) 35 36 // Fs is a Fs implementation that uses functions provided by google cloud storage 37 type Fs struct { 38 ctx context.Context 39 client stiface.Client 40 separator string 41 42 buckets map[string]stiface.BucketHandle 43 rawGcsObjects map[string]*GcsFile 44 45 autoRemoveEmptyFolders bool // trigger for creating "virtual folders" (not required by GCSs) 46 } 47 48 func NewGcsFs(ctx context.Context, client stiface.Client) *Fs { 49 return NewGcsFsWithSeparator(ctx, client, "/") 50 } 51 52 func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs { 53 return &Fs{ 54 ctx: ctx, 55 client: client, 56 separator: folderSep, 57 rawGcsObjects: make(map[string]*GcsFile), 58 59 autoRemoveEmptyFolders: true, 60 } 61 } 62 63 // normSeparators will normalize all "\\" and "/" to the provided separator 64 func (fs *Fs) normSeparators(s string) string { 65 return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1) 66 } 67 68 func (fs *Fs) ensureTrailingSeparator(s string) string { 69 if len(s) > 0 && !strings.HasSuffix(s, fs.separator) { 70 return s + fs.separator 71 } 72 return s 73 } 74 75 func (fs *Fs) ensureNoLeadingSeparator(s string) string { 76 if len(s) > 0 && strings.HasPrefix(s, fs.separator) { 77 s = s[len(fs.separator):] 78 } 79 80 return s 81 } 82 83 func ensureNoPrefix(s string) string { 84 if len(s) > 0 && strings.HasPrefix(s, gsPrefix) { 85 return s[len(gsPrefix):] 86 } 87 return s 88 } 89 90 func validateName(s string) error { 91 if len(s) == 0 { 92 return ErrNoBucketInName 93 } 94 return nil 95 } 96 97 // Splits provided name into bucket name and path 98 func (fs *Fs) splitName(name string) (bucketName string, path string) { 99 splitName := strings.Split(name, fs.separator) 100 101 return splitName[0], strings.Join(splitName[1:], fs.separator) 102 } 103 104 func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) { 105 bucket := fs.buckets[name] 106 if bucket == nil { 107 bucket = fs.client.Bucket(name) 108 _, err := bucket.Attrs(fs.ctx) 109 if err != nil { 110 return nil, err 111 } 112 } 113 return bucket, nil 114 } 115 116 func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) { 117 bucketName, path := fs.splitName(name) 118 119 bucket, err := fs.getBucket(bucketName) 120 if err != nil { 121 return nil, err 122 } 123 124 return bucket.Object(path), nil 125 } 126 127 func (fs *Fs) Name() string { return "GcsFs" } 128 129 func (fs *Fs) Create(name string) (*GcsFile, error) { 130 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) 131 if err := validateName(name); err != nil { 132 return nil, err 133 } 134 135 if !fs.autoRemoveEmptyFolders { 136 baseDir := filepath.Base(name) 137 if stat, err := fs.Stat(baseDir); err != nil || !stat.IsDir() { 138 err = fs.MkdirAll(baseDir, 0) 139 if err != nil { 140 return nil, err 141 } 142 } 143 } 144 145 obj, err := fs.getObj(name) 146 if err != nil { 147 return nil, err 148 } 149 w := obj.NewWriter(fs.ctx) 150 err = w.Close() 151 if err != nil { 152 return nil, err 153 } 154 file := NewGcsFile(fs.ctx, fs, obj, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0, name) 155 156 fs.rawGcsObjects[name] = file 157 return file, nil 158 } 159 160 func (fs *Fs) Mkdir(name string, _ os.FileMode) error { 161 name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name)))) 162 if err := validateName(name); err != nil { 163 return err 164 } 165 // folder creation logic has to additionally check for folder name presence 166 bucketName, path := fs.splitName(name) 167 if bucketName == "" { 168 return ErrNoBucketInName 169 } 170 if path == "" { 171 // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent 172 return ErrEmptyObjectName 173 } 174 175 obj, err := fs.getObj(name) 176 if err != nil { 177 return err 178 } 179 w := obj.NewWriter(fs.ctx) 180 return w.Close() 181 } 182 183 func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { 184 path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path)))) 185 if err := validateName(path); err != nil { 186 return err 187 } 188 // folder creation logic has to additionally check for folder name presence 189 bucketName, splitPath := fs.splitName(path) 190 if bucketName == "" { 191 return ErrNoBucketInName 192 } 193 if splitPath == "" { 194 // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent 195 return ErrEmptyObjectName 196 } 197 198 root := "" 199 folders := strings.Split(path, fs.separator) 200 for i, f := range folders { 201 if f == "" && i != 0 { 202 continue // it's the last item - it should be empty 203 } 204 // Don't force a delimiter prefix 205 if root != "" { 206 root = root + fs.separator + f 207 } else { 208 // we have to have at least bucket name + folder name to create successfully 209 root = f 210 continue 211 } 212 213 if err := fs.Mkdir(root, perm); err != nil { 214 return err 215 } 216 } 217 return nil 218 } 219 220 func (fs *Fs) Open(name string) (*GcsFile, error) { 221 return fs.OpenFile(name, os.O_RDONLY, 0) 222 } 223 224 func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) { 225 var file *GcsFile 226 var err error 227 228 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) 229 if err = validateName(name); err != nil { 230 return nil, err 231 } 232 233 f, found := fs.rawGcsObjects[name] 234 if found { 235 file = NewGcsFileFromOldFH(flag, fileMode, f.resource) 236 } else { 237 var obj stiface.ObjectHandle 238 obj, err = fs.getObj(name) 239 if err != nil { 240 return nil, err 241 } 242 file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name) 243 } 244 245 if flag == os.O_RDONLY { 246 _, err = file.Stat() 247 if err != nil { 248 return nil, err 249 } 250 } 251 252 if flag&os.O_TRUNC != 0 { 253 err = file.resource.obj.Delete(fs.ctx) 254 if err != nil { 255 return nil, err 256 } 257 return fs.Create(name) 258 } 259 260 if flag&os.O_APPEND != 0 { 261 _, err = file.Seek(0, 2) 262 if err != nil { 263 return nil, err 264 } 265 } 266 267 if flag&os.O_CREATE != 0 { 268 _, err = file.Stat() 269 if err == nil { // the file actually exists 270 return nil, syscall.EPERM 271 } 272 273 _, err = file.WriteString("") 274 if err != nil { 275 return nil, err 276 } 277 } 278 return file, nil 279 } 280 281 func (fs *Fs) Remove(name string) error { 282 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) 283 if err := validateName(name); err != nil { 284 return err 285 } 286 287 obj, err := fs.getObj(name) 288 if err != nil { 289 return err 290 } 291 info, err := fs.Stat(name) 292 if err != nil { 293 return err 294 } 295 delete(fs.rawGcsObjects, name) 296 297 if info.IsDir() { 298 // it's a folder, we ha to check its contents - it cannot be removed, if not empty 299 var dir *GcsFile 300 dir, err = fs.Open(name) 301 if err != nil { 302 return err 303 } 304 var infos []os.FileInfo 305 infos, err = dir.Readdir(0) 306 if err != nil { 307 return err 308 } 309 if len(infos) > 0 { 310 return syscall.ENOTEMPTY 311 } 312 313 // it's an empty folder, we can continue 314 name = fs.ensureTrailingSeparator(name) 315 obj, err = fs.getObj(name) 316 if err != nil { 317 return err 318 } 319 320 return obj.Delete(fs.ctx) 321 } 322 return obj.Delete(fs.ctx) 323 } 324 325 func (fs *Fs) RemoveAll(path string) error { 326 path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path))) 327 if err := validateName(path); err != nil { 328 return err 329 } 330 331 pathInfo, err := fs.Stat(path) 332 if errors.Is(err, ErrFileNotFound) { 333 // return early if file doesn't exist 334 return nil 335 } 336 if err != nil { 337 return err 338 } 339 340 if !pathInfo.IsDir() { 341 return fs.Remove(path) 342 } 343 344 var dir *GcsFile 345 dir, err = fs.Open(path) 346 if err != nil { 347 return err 348 } 349 350 var infos []os.FileInfo 351 infos, err = dir.Readdir(0) 352 if err != nil { 353 return err 354 } 355 for _, info := range infos { 356 nameToRemove := fs.normSeparators(info.Name()) 357 err = fs.RemoveAll(path + fs.separator + nameToRemove) 358 if err != nil { 359 return err 360 } 361 } 362 363 return fs.Remove(path) 364 } 365 366 func (fs *Fs) Rename(oldName, newName string) error { 367 oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName))) 368 if err := validateName(oldName); err != nil { 369 return err 370 } 371 372 newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName))) 373 if err := validateName(newName); err != nil { 374 return err 375 } 376 377 src, err := fs.getObj(oldName) 378 if err != nil { 379 return err 380 } 381 dst, err := fs.getObj(newName) 382 if err != nil { 383 return err 384 } 385 386 if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil { 387 return err 388 } 389 delete(fs.rawGcsObjects, oldName) 390 return src.Delete(fs.ctx) 391 } 392 393 func (fs *Fs) Stat(name string) (os.FileInfo, error) { 394 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) 395 if err := validateName(name); err != nil { 396 return nil, err 397 } 398 399 return newFileInfo(name, fs, defaultFileMode) 400 } 401 402 func (fs *Fs) Chmod(_ string, _ os.FileMode) error { 403 return errors.New("method Chmod is not implemented in GCS") 404 } 405 406 func (fs *Fs) Chtimes(_ string, _, _ time.Time) error { 407 return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly") 408 } 409 410 func (fs *Fs) Chown(_ string, _, _ int) error { 411 return errors.New("method Chown is not implemented for GCS") 412 }