github.com/t2y/goofys@v0.19.1-0.20190123053037-27053313e616/internal/dir.go (about) 1 // Copyright 2015 - 2017 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 "fmt" 19 "sort" 20 "strings" 21 "sync" 22 "time" 23 24 "github.com/aws/aws-sdk-go/aws" 25 "github.com/aws/aws-sdk-go/service/s3" 26 27 "github.com/jacobsa/fuse" 28 "github.com/jacobsa/fuse/fuseops" 29 "github.com/jacobsa/fuse/fuseutil" 30 ) 31 32 type DirInodeData struct { 33 // these 2 refer to readdir of the Children 34 lastOpenDir *string 35 lastOpenDirIdx int 36 seqOpenDirScore uint8 37 DirTime time.Time 38 39 Children []*Inode 40 } 41 42 type DirHandleEntry struct { 43 Name *string 44 Inode fuseops.InodeID 45 Type fuseutil.DirentType 46 Offset fuseops.DirOffset 47 48 Attributes *InodeAttributes 49 ETag *string 50 StorageClass *string 51 } 52 53 type DirHandle struct { 54 inode *Inode 55 56 mu sync.Mutex // everything below is protected by mu 57 Entries []*DirHandleEntry 58 Marker *string 59 BaseOffset int 60 } 61 62 func NewDirHandle(inode *Inode) (dh *DirHandle) { 63 dh = &DirHandle{inode: inode} 64 return 65 } 66 67 func (inode *Inode) OpenDir() (dh *DirHandle) { 68 inode.logFuse("OpenDir") 69 70 parent := inode.Parent 71 if parent != nil && inode.fs.flags.TypeCacheTTL != 0 { 72 parent.mu.Lock() 73 defer parent.mu.Unlock() 74 75 num := len(parent.dir.Children) 76 77 if parent.dir.lastOpenDir == nil && num > 0 && *parent.dir.Children[0].Name == *inode.Name { 78 if parent.dir.seqOpenDirScore < 255 { 79 parent.dir.seqOpenDirScore += 1 80 } 81 // 2.1) if I open a/a, a/'s score is now 2 82 // ie: handle the depth first search case 83 if parent.dir.seqOpenDirScore >= 2 { 84 fuseLog.Debugf("%v in readdir mode", *parent.FullName()) 85 } 86 } else if parent.dir.lastOpenDir != nil && parent.dir.lastOpenDirIdx+1 < num && 87 // we are reading the next one as expected 88 *parent.dir.Children[parent.dir.lastOpenDirIdx+1].Name == *inode.Name && 89 // check that inode positions haven't moved 90 *parent.dir.Children[parent.dir.lastOpenDirIdx].Name == *parent.dir.lastOpenDir { 91 // 2.2) if I open b/, root's score is now 2 92 // ie: handle the breath first search case 93 if parent.dir.seqOpenDirScore < 255 { 94 parent.dir.seqOpenDirScore++ 95 } 96 parent.dir.lastOpenDirIdx += 1 97 if parent.dir.seqOpenDirScore == 2 { 98 fuseLog.Debugf("%v in readdir mode", *parent.FullName()) 99 } 100 } else { 101 parent.dir.seqOpenDirScore = 0 102 parent.dir.lastOpenDirIdx = parent.findChildIdxUnlocked(*inode.Name) 103 if parent.dir.lastOpenDirIdx == -1 { 104 panic(fmt.Sprintf("%v is not under %v", *inode.Name, *parent.FullName())) 105 } 106 } 107 108 parent.dir.lastOpenDir = inode.Name 109 inode.mu.Lock() 110 defer inode.mu.Unlock() 111 112 if inode.dir.lastOpenDir == nil { 113 // 1) if I open a/, root's score = 1 (a is the first dir), 114 // so make a/'s count at 1 too 115 inode.dir.seqOpenDirScore = parent.dir.seqOpenDirScore 116 if inode.dir.seqOpenDirScore >= 2 { 117 fuseLog.Debugf("%v in readdir mode", *inode.FullName()) 118 } 119 } 120 } 121 122 dh = NewDirHandle(inode) 123 return 124 } 125 126 // Dirents, sorted by name. 127 type sortedDirents []*DirHandleEntry 128 129 func (p sortedDirents) Len() int { return len(p) } 130 func (p sortedDirents) Less(i, j int) bool { return *p[i].Name < *p[j].Name } 131 func (p sortedDirents) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 132 133 func (dh *DirHandle) listObjectsSlurp(prefix string) (resp *s3.ListObjectsOutput, err error) { 134 var marker *string 135 reqPrefix := prefix 136 inode := dh.inode 137 fs := inode.fs 138 if dh.inode.Parent != nil { 139 inode = dh.inode.Parent 140 reqPrefix = *fs.key(*inode.FullName()) 141 if len(*inode.FullName()) != 0 { 142 reqPrefix += "/" 143 } 144 marker = fs.key(*dh.inode.FullName() + "/") 145 } 146 147 params := &s3.ListObjectsInput{ 148 Bucket: &fs.bucket, 149 Prefix: &reqPrefix, 150 Marker: marker, 151 } 152 153 resp, err = fs.s3.ListObjects(params) 154 if err != nil { 155 s3Log.Errorf("ListObjects %v = %v", params, err) 156 return 157 } 158 159 num := len(resp.Contents) 160 if num == 0 { 161 return 162 } 163 164 dirs := make(map[*Inode]bool) 165 for _, obj := range resp.Contents { 166 baseName := (*obj.Key)[len(reqPrefix):] 167 168 slash := strings.Index(baseName, "/") 169 if slash != -1 { 170 inode.insertSubTree(baseName, obj, dirs) 171 } 172 } 173 174 for d, sealed := range dirs { 175 if d == dh.inode { 176 // never seal the current dir because that's 177 // handled at upper layer 178 continue 179 } 180 181 if sealed || !*resp.IsTruncated { 182 d.dir.DirTime = time.Now() 183 d.Attributes.Mtime = d.findChildMaxTime() 184 } 185 } 186 187 if *resp.IsTruncated { 188 obj := resp.Contents[len(resp.Contents)-1] 189 // if we are done listing prefix, we are good 190 if strings.HasPrefix(*obj.Key, prefix) { 191 // if we are done with all the slashes, then we are good 192 baseName := (*obj.Key)[len(prefix):] 193 194 for _, c := range baseName { 195 if c <= '/' { 196 // if an entry is ex: a!b, then the 197 // next entry could be a/foo, so we 198 // are not done yet. 199 resp = nil 200 break 201 } 202 } 203 } 204 } 205 206 // we only return this response if we are totally done with listing this dir 207 if resp != nil { 208 resp.IsTruncated = aws.Bool(false) 209 resp.NextMarker = nil 210 } 211 212 return 213 } 214 215 func (dh *DirHandle) listObjects(prefix string) (resp *s3.ListObjectsOutput, err error) { 216 errSlurpChan := make(chan error, 1) 217 slurpChan := make(chan s3.ListObjectsOutput, 1) 218 errListChan := make(chan error, 1) 219 listChan := make(chan s3.ListObjectsOutput, 1) 220 221 fs := dh.inode.fs 222 223 // try to list without delimiter to see if we have slurp up 224 // multiple directories 225 if dh.Marker == nil && 226 fs.flags.TypeCacheTTL != 0 && 227 (dh.inode.Parent != nil && dh.inode.Parent.dir.seqOpenDirScore >= 2) { 228 go func() { 229 resp, err := dh.listObjectsSlurp(prefix) 230 if err != nil { 231 errSlurpChan <- err 232 } else if resp != nil { 233 slurpChan <- *resp 234 } else { 235 errSlurpChan <- fuse.EINVAL 236 } 237 }() 238 } else { 239 errSlurpChan <- fuse.EINVAL 240 } 241 242 listObjectsFlat := func() { 243 params := &s3.ListObjectsInput{ 244 Bucket: &fs.bucket, 245 Delimiter: aws.String("/"), 246 Marker: dh.Marker, 247 Prefix: &prefix, 248 } 249 250 resp, err := fs.s3.ListObjects(params) 251 if err != nil { 252 errListChan <- err 253 } else { 254 listChan <- *resp 255 } 256 } 257 258 if !fs.flags.Cheap { 259 // invoke the fallback in parallel if desired 260 go listObjectsFlat() 261 } 262 263 // first see if we get anything from the slurp 264 select { 265 case resp := <-slurpChan: 266 return &resp, nil 267 case err = <-errSlurpChan: 268 } 269 270 if fs.flags.Cheap { 271 listObjectsFlat() 272 } 273 274 // if we got an error (which may mean slurp is not applicable, 275 // wait for regular list 276 select { 277 case resp := <-listChan: 278 return &resp, nil 279 case err = <-errListChan: 280 return 281 } 282 } 283 284 func objectToDirEntry(fs *Goofys, obj *s3.Object, name string, isDir bool) (en *DirHandleEntry) { 285 if isDir { 286 en = &DirHandleEntry{ 287 Name: &name, 288 Type: fuseutil.DT_Directory, 289 Attributes: &fs.rootAttrs, 290 } 291 } else { 292 en = &DirHandleEntry{ 293 Name: &name, 294 Type: fuseutil.DT_File, 295 Attributes: &InodeAttributes{ 296 Size: uint64(*obj.Size), 297 Mtime: *obj.LastModified, 298 }, 299 ETag: obj.ETag, 300 StorageClass: obj.StorageClass, 301 } 302 } 303 304 return 305 } 306 307 // LOCKS_REQUIRED(dh.mu) 308 func (dh *DirHandle) ReadDir(offset fuseops.DirOffset) (en *DirHandleEntry, err error) { 309 // If the request is for offset zero, we assume that either this is the first 310 // call or rewinddir has been called. Reset state. 311 if offset == 0 { 312 dh.Entries = nil 313 } 314 315 en, ok := dh.inode.readDirFromCache(offset) 316 if ok { 317 return 318 } 319 320 fs := dh.inode.fs 321 322 if offset == 0 { 323 en = &DirHandleEntry{ 324 Name: aws.String("."), 325 Type: fuseutil.DT_Directory, 326 Attributes: &fs.rootAttrs, 327 Offset: 1, 328 } 329 return 330 } else if offset == 1 { 331 en = &DirHandleEntry{ 332 Name: aws.String(".."), 333 Type: fuseutil.DT_Directory, 334 Attributes: &fs.rootAttrs, 335 Offset: 2, 336 } 337 return 338 } 339 340 i := int(offset) - dh.BaseOffset - 2 341 if i < 0 { 342 panic(fmt.Sprintf("invalid offset %v, base=%v", offset, dh.BaseOffset)) 343 } 344 345 if i >= len(dh.Entries) { 346 if dh.Marker != nil { 347 // we need to fetch the next page 348 dh.Entries = nil 349 dh.BaseOffset += i 350 i = 0 351 } 352 } 353 354 if dh.Entries == nil { 355 // try not to hold the lock when we make the request 356 dh.mu.Unlock() 357 358 prefix := *fs.key(*dh.inode.FullName()) 359 if len(*dh.inode.FullName()) != 0 { 360 prefix += "/" 361 } 362 363 resp, err := dh.listObjects(prefix) 364 if err != nil { 365 dh.mu.Lock() 366 return nil, mapAwsError(err) 367 } 368 369 s3Log.Debug(resp) 370 dh.mu.Lock() 371 372 dh.Entries = make([]*DirHandleEntry, 0, len(resp.CommonPrefixes)+len(resp.Contents)) 373 374 // this is only returned for non-slurped responses 375 for _, dir := range resp.CommonPrefixes { 376 // strip trailing / 377 dirName := (*dir.Prefix)[0 : len(*dir.Prefix)-1] 378 // strip previous prefix 379 dirName = dirName[len(*resp.Prefix):] 380 if len(dirName) == 0 { 381 continue 382 } 383 en = &DirHandleEntry{ 384 Name: &dirName, 385 Type: fuseutil.DT_Directory, 386 Attributes: &fs.rootAttrs, 387 } 388 389 dh.Entries = append(dh.Entries, en) 390 } 391 392 lastDir := "" 393 for _, obj := range resp.Contents { 394 if !strings.HasPrefix(*obj.Key, prefix) { 395 // other slurped objects that we cached 396 continue 397 } 398 399 baseName := (*obj.Key)[len(prefix):] 400 401 slash := strings.Index(baseName, "/") 402 if slash == -1 { 403 if len(baseName) == 0 { 404 // shouldn't happen 405 continue 406 } 407 dh.Entries = append(dh.Entries, 408 objectToDirEntry(fs, obj, baseName, false)) 409 } else { 410 // this is a slurped up object which 411 // was already cached, unless it's a 412 // directory right under this dir that 413 // we need to return 414 dirName := baseName[:slash] 415 if dirName != lastDir && lastDir != "" { 416 // make a copy so we can take the address 417 dir := lastDir 418 en := &DirHandleEntry{ 419 Name: &dir, 420 Type: fuseutil.DT_Directory, 421 Attributes: &fs.rootAttrs, 422 } 423 dh.Entries = append(dh.Entries, en) 424 } 425 lastDir = dirName 426 } 427 } 428 if lastDir != "" { 429 en := &DirHandleEntry{ 430 Name: &lastDir, 431 Type: fuseutil.DT_Directory, 432 Attributes: &fs.rootAttrs, 433 } 434 dh.Entries = append(dh.Entries, en) 435 } 436 437 sort.Sort(sortedDirents(dh.Entries)) 438 439 // Fix up offset fields. 440 for i := 0; i < len(dh.Entries); i++ { 441 en := dh.Entries[i] 442 // offset is 1 based, also need to account for "." and ".." 443 en.Offset = fuseops.DirOffset(i+dh.BaseOffset) + 1 + 2 444 } 445 446 if *resp.IsTruncated { 447 dh.Marker = resp.NextMarker 448 } else { 449 dh.Marker = nil 450 } 451 } 452 453 if i == len(dh.Entries) { 454 // we've reached the end 455 return nil, nil 456 } else if i > len(dh.Entries) { 457 return nil, fuse.EINVAL 458 } 459 460 return dh.Entries[i], nil 461 } 462 463 func (dh *DirHandle) CloseDir() error { 464 return nil 465 }