github.com/docker/engine@v22.0.0-20211208180946-d456264580cf+incompatible/pkg/fileutils/fileutils.go (about) 1 package fileutils // import "github.com/docker/docker/pkg/fileutils" 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "regexp" 10 "strings" 11 "text/scanner" 12 "unicode/utf8" 13 ) 14 15 // escapeBytes is a bitmap used to check whether a character should be escaped when creating the regex. 16 var escapeBytes [8]byte 17 18 // shouldEscape reports whether a rune should be escaped as part of the regex. 19 // 20 // This only includes characters that require escaping in regex but are also NOT valid filepath pattern characters. 21 // Additionally, '\' is not excluded because there is specific logic to properly handle this, as it's a path separator 22 // on Windows. 23 // 24 // Adapted from regexp::QuoteMeta in go stdlib. 25 // See https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/regexp/regexp.go;l=703-715;drc=refs%2Ftags%2Fgo1.17.2 26 func shouldEscape(b rune) bool { 27 return b < utf8.RuneSelf && escapeBytes[b%8]&(1<<(b/8)) != 0 28 } 29 30 func init() { 31 for _, b := range []byte(`.+()|{}$`) { 32 escapeBytes[b%8] |= 1 << (b / 8) 33 } 34 } 35 36 // PatternMatcher allows checking paths against a list of patterns 37 type PatternMatcher struct { 38 patterns []*Pattern 39 exclusions bool 40 } 41 42 // NewPatternMatcher creates a new matcher object for specific patterns that can 43 // be used later to match against patterns against paths 44 func NewPatternMatcher(patterns []string) (*PatternMatcher, error) { 45 pm := &PatternMatcher{ 46 patterns: make([]*Pattern, 0, len(patterns)), 47 } 48 for _, p := range patterns { 49 // Eliminate leading and trailing whitespace. 50 p = strings.TrimSpace(p) 51 if p == "" { 52 continue 53 } 54 p = filepath.Clean(p) 55 newp := &Pattern{} 56 if p[0] == '!' { 57 if len(p) == 1 { 58 return nil, errors.New("illegal exclusion pattern: \"!\"") 59 } 60 newp.exclusion = true 61 p = p[1:] 62 pm.exclusions = true 63 } 64 // Do some syntax checking on the pattern. 65 // filepath's Match() has some really weird rules that are inconsistent 66 // so instead of trying to dup their logic, just call Match() for its 67 // error state and if there is an error in the pattern return it. 68 // If this becomes an issue we can remove this since its really only 69 // needed in the error (syntax) case - which isn't really critical. 70 if _, err := filepath.Match(p, "."); err != nil { 71 return nil, err 72 } 73 newp.cleanedPattern = p 74 newp.dirs = strings.Split(p, string(os.PathSeparator)) 75 pm.patterns = append(pm.patterns, newp) 76 } 77 return pm, nil 78 } 79 80 // Matches returns true if "file" matches any of the patterns 81 // and isn't excluded by any of the subsequent patterns. 82 // 83 // The "file" argument should be a slash-delimited path. 84 // 85 // Matches is not safe to call concurrently. 86 // 87 // Deprecated: This implementation is buggy (it only checks a single parent dir 88 // against the pattern) and will be removed soon. Use either 89 // MatchesOrParentMatches or MatchesUsingParentResults instead. 90 func (pm *PatternMatcher) Matches(file string) (bool, error) { 91 matched := false 92 file = filepath.FromSlash(file) 93 parentPath := filepath.Dir(file) 94 parentPathDirs := strings.Split(parentPath, string(os.PathSeparator)) 95 96 for _, pattern := range pm.patterns { 97 // Skip evaluation if this is an inclusion and the filename 98 // already matched the pattern, or it's an exclusion and it has 99 // not matched the pattern yet. 100 if pattern.exclusion != matched { 101 continue 102 } 103 104 match, err := pattern.match(file) 105 if err != nil { 106 return false, err 107 } 108 109 if !match && parentPath != "." { 110 // Check to see if the pattern matches one of our parent dirs. 111 if len(pattern.dirs) <= len(parentPathDirs) { 112 match, _ = pattern.match(strings.Join(parentPathDirs[:len(pattern.dirs)], string(os.PathSeparator))) 113 } 114 } 115 116 if match { 117 matched = !pattern.exclusion 118 } 119 } 120 121 return matched, nil 122 } 123 124 // MatchesOrParentMatches returns true if "file" matches any of the patterns 125 // and isn't excluded by any of the subsequent patterns. 126 // 127 // The "file" argument should be a slash-delimited path. 128 // 129 // Matches is not safe to call concurrently. 130 func (pm *PatternMatcher) MatchesOrParentMatches(file string) (bool, error) { 131 matched := false 132 file = filepath.FromSlash(file) 133 parentPath := filepath.Dir(file) 134 parentPathDirs := strings.Split(parentPath, string(os.PathSeparator)) 135 136 for _, pattern := range pm.patterns { 137 // Skip evaluation if this is an inclusion and the filename 138 // already matched the pattern, or it's an exclusion and it has 139 // not matched the pattern yet. 140 if pattern.exclusion != matched { 141 continue 142 } 143 144 match, err := pattern.match(file) 145 if err != nil { 146 return false, err 147 } 148 149 if !match && parentPath != "." { 150 // Check to see if the pattern matches one of our parent dirs. 151 for i := range parentPathDirs { 152 match, _ = pattern.match(strings.Join(parentPathDirs[:i+1], string(os.PathSeparator))) 153 if match { 154 break 155 } 156 } 157 } 158 159 if match { 160 matched = !pattern.exclusion 161 } 162 } 163 164 return matched, nil 165 } 166 167 // MatchesUsingParentResult returns true if "file" matches any of the patterns 168 // and isn't excluded by any of the subsequent patterns. The functionality is 169 // the same as Matches, but as an optimization, the caller keeps track of 170 // whether the parent directory matched. 171 // 172 // The "file" argument should be a slash-delimited path. 173 // 174 // MatchesUsingParentResult is not safe to call concurrently. 175 // 176 // Deprecated: this function does behave correctly in some cases (see 177 // https://github.com/docker/buildx/issues/850). 178 // 179 // Use MatchesUsingParentResults instead. 180 func (pm *PatternMatcher) MatchesUsingParentResult(file string, parentMatched bool) (bool, error) { 181 matched := parentMatched 182 file = filepath.FromSlash(file) 183 184 for _, pattern := range pm.patterns { 185 // Skip evaluation if this is an inclusion and the filename 186 // already matched the pattern, or it's an exclusion and it has 187 // not matched the pattern yet. 188 if pattern.exclusion != matched { 189 continue 190 } 191 192 match, err := pattern.match(file) 193 if err != nil { 194 return false, err 195 } 196 197 if match { 198 matched = !pattern.exclusion 199 } 200 } 201 return matched, nil 202 } 203 204 // MatchInfo tracks information about parent dir matches while traversing a 205 // filesystem. 206 type MatchInfo struct { 207 parentMatched []bool 208 } 209 210 // MatchesUsingParentResults returns true if "file" matches any of the patterns 211 // and isn't excluded by any of the subsequent patterns. The functionality is 212 // the same as Matches, but as an optimization, the caller passes in 213 // intermediate results from matching the parent directory. 214 // 215 // The "file" argument should be a slash-delimited path. 216 // 217 // MatchesUsingParentResults is not safe to call concurrently. 218 func (pm *PatternMatcher) MatchesUsingParentResults(file string, parentMatchInfo MatchInfo) (bool, MatchInfo, error) { 219 parentMatched := parentMatchInfo.parentMatched 220 if len(parentMatched) != 0 && len(parentMatched) != len(pm.patterns) { 221 return false, MatchInfo{}, errors.New("wrong number of values in parentMatched") 222 } 223 224 file = filepath.FromSlash(file) 225 matched := false 226 227 matchInfo := MatchInfo{ 228 parentMatched: make([]bool, len(pm.patterns)), 229 } 230 for i, pattern := range pm.patterns { 231 match := false 232 // If the parent matched this pattern, we don't need to recheck. 233 if len(parentMatched) != 0 { 234 match = parentMatched[i] 235 } 236 237 if !match { 238 // Skip evaluation if this is an inclusion and the filename 239 // already matched the pattern, or it's an exclusion and it has 240 // not matched the pattern yet. 241 if pattern.exclusion != matched { 242 continue 243 } 244 245 var err error 246 match, err = pattern.match(file) 247 if err != nil { 248 return false, matchInfo, err 249 } 250 251 // If the zero value of MatchInfo was passed in, we don't have 252 // any information about the parent dir's match results, and we 253 // apply the same logic as MatchesOrParentMatches. 254 if !match && len(parentMatched) == 0 { 255 if parentPath := filepath.Dir(file); parentPath != "." { 256 parentPathDirs := strings.Split(parentPath, string(os.PathSeparator)) 257 // Check to see if the pattern matches one of our parent dirs. 258 for i := range parentPathDirs { 259 match, _ = pattern.match(strings.Join(parentPathDirs[:i+1], string(os.PathSeparator))) 260 if match { 261 break 262 } 263 } 264 } 265 } 266 } 267 matchInfo.parentMatched[i] = match 268 269 if match { 270 matched = !pattern.exclusion 271 } 272 } 273 return matched, matchInfo, nil 274 } 275 276 // Exclusions returns true if any of the patterns define exclusions 277 func (pm *PatternMatcher) Exclusions() bool { 278 return pm.exclusions 279 } 280 281 // Patterns returns array of active patterns 282 func (pm *PatternMatcher) Patterns() []*Pattern { 283 return pm.patterns 284 } 285 286 // Pattern defines a single regexp used to filter file paths. 287 type Pattern struct { 288 cleanedPattern string 289 dirs []string 290 regexp *regexp.Regexp 291 exclusion bool 292 } 293 294 func (p *Pattern) String() string { 295 return p.cleanedPattern 296 } 297 298 // Exclusion returns true if this pattern defines exclusion 299 func (p *Pattern) Exclusion() bool { 300 return p.exclusion 301 } 302 303 func (p *Pattern) match(path string) (bool, error) { 304 if p.regexp == nil { 305 if err := p.compile(); err != nil { 306 return false, filepath.ErrBadPattern 307 } 308 } 309 310 b := p.regexp.MatchString(path) 311 312 return b, nil 313 } 314 315 func (p *Pattern) compile() error { 316 regStr := "^" 317 pattern := p.cleanedPattern 318 // Go through the pattern and convert it to a regexp. 319 // We use a scanner so we can support utf-8 chars. 320 var scan scanner.Scanner 321 scan.Init(strings.NewReader(pattern)) 322 323 sl := string(os.PathSeparator) 324 escSL := sl 325 if sl == `\` { 326 escSL += `\` 327 } 328 329 for scan.Peek() != scanner.EOF { 330 ch := scan.Next() 331 332 if ch == '*' { 333 if scan.Peek() == '*' { 334 // is some flavor of "**" 335 scan.Next() 336 337 // Treat **/ as ** so eat the "/" 338 if string(scan.Peek()) == sl { 339 scan.Next() 340 } 341 342 if scan.Peek() == scanner.EOF { 343 // is "**EOF" - to align with .gitignore just accept all 344 regStr += ".*" 345 } else { 346 // is "**" 347 // Note that this allows for any # of /'s (even 0) because 348 // the .* will eat everything, even /'s 349 regStr += "(.*" + escSL + ")?" 350 } 351 } else { 352 // is "*" so map it to anything but "/" 353 regStr += "[^" + escSL + "]*" 354 } 355 } else if ch == '?' { 356 // "?" is any char except "/" 357 regStr += "[^" + escSL + "]" 358 } else if shouldEscape(ch) { 359 // Escape some regexp special chars that have no meaning 360 // in golang's filepath.Match 361 regStr += `\` + string(ch) 362 } else if ch == '\\' { 363 // escape next char. Note that a trailing \ in the pattern 364 // will be left alone (but need to escape it) 365 if sl == `\` { 366 // On windows map "\" to "\\", meaning an escaped backslash, 367 // and then just continue because filepath.Match on 368 // Windows doesn't allow escaping at all 369 regStr += escSL 370 continue 371 } 372 if scan.Peek() != scanner.EOF { 373 regStr += `\` + string(scan.Next()) 374 } else { 375 regStr += `\` 376 } 377 } else { 378 regStr += string(ch) 379 } 380 } 381 382 regStr += "$" 383 384 re, err := regexp.Compile(regStr) 385 if err != nil { 386 return err 387 } 388 389 p.regexp = re 390 return nil 391 } 392 393 // Matches returns true if file matches any of the patterns 394 // and isn't excluded by any of the subsequent patterns. 395 // 396 // This implementation is buggy (it only checks a single parent dir against the 397 // pattern) and will be removed soon. Use MatchesOrParentMatches instead. 398 func Matches(file string, patterns []string) (bool, error) { 399 pm, err := NewPatternMatcher(patterns) 400 if err != nil { 401 return false, err 402 } 403 file = filepath.Clean(file) 404 405 if file == "." { 406 // Don't let them exclude everything, kind of silly. 407 return false, nil 408 } 409 410 return pm.Matches(file) 411 } 412 413 // MatchesOrParentMatches returns true if file matches any of the patterns 414 // and isn't excluded by any of the subsequent patterns. 415 func MatchesOrParentMatches(file string, patterns []string) (bool, error) { 416 pm, err := NewPatternMatcher(patterns) 417 if err != nil { 418 return false, err 419 } 420 file = filepath.Clean(file) 421 422 if file == "." { 423 // Don't let them exclude everything, kind of silly. 424 return false, nil 425 } 426 427 return pm.MatchesOrParentMatches(file) 428 } 429 430 // CopyFile copies from src to dst until either EOF is reached 431 // on src or an error occurs. It verifies src exists and removes 432 // the dst if it exists. 433 func CopyFile(src, dst string) (int64, error) { 434 cleanSrc := filepath.Clean(src) 435 cleanDst := filepath.Clean(dst) 436 if cleanSrc == cleanDst { 437 return 0, nil 438 } 439 sf, err := os.Open(cleanSrc) 440 if err != nil { 441 return 0, err 442 } 443 defer sf.Close() 444 if err := os.Remove(cleanDst); err != nil && !os.IsNotExist(err) { 445 return 0, err 446 } 447 df, err := os.Create(cleanDst) 448 if err != nil { 449 return 0, err 450 } 451 defer df.Close() 452 return io.Copy(df, sf) 453 } 454 455 // ReadSymlinkedDirectory returns the target directory of a symlink. 456 // The target of the symbolic link may not be a file. 457 func ReadSymlinkedDirectory(path string) (string, error) { 458 var realPath string 459 var err error 460 if realPath, err = filepath.Abs(path); err != nil { 461 return "", fmt.Errorf("unable to get absolute path for %s: %s", path, err) 462 } 463 if realPath, err = filepath.EvalSymlinks(realPath); err != nil { 464 return "", fmt.Errorf("failed to canonicalise path for %s: %s", path, err) 465 } 466 realPathInfo, err := os.Stat(realPath) 467 if err != nil { 468 return "", fmt.Errorf("failed to stat target '%s' of '%s': %s", realPath, path, err) 469 } 470 if !realPathInfo.Mode().IsDir() { 471 return "", fmt.Errorf("canonical path points to a file '%s'", realPath) 472 } 473 return realPath, nil 474 } 475 476 // CreateIfNotExists creates a file or a directory only if it does not already exist. 477 func CreateIfNotExists(path string, isDir bool) error { 478 if _, err := os.Stat(path); err != nil { 479 if os.IsNotExist(err) { 480 if isDir { 481 return os.MkdirAll(path, 0755) 482 } 483 if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 484 return err 485 } 486 f, err := os.OpenFile(path, os.O_CREATE, 0755) 487 if err != nil { 488 return err 489 } 490 f.Close() 491 } 492 } 493 return nil 494 }