github.com/westcoastroms/westcoastroms-build@v0.0.0-20190928114312-2350e5a73030/build/blueprint/pathtools/glob.go (about) 1 // Copyright 2014 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package pathtools 16 17 import ( 18 "errors" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/google/blueprint/deptools" 26 ) 27 28 var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **") 29 var GlobLastRecursiveErr = errors.New("pattern ** as last path element") 30 31 // Glob returns the list of files and directories that match the given pattern 32 // but do not match the given exclude patterns, along with the list of 33 // directories and other dependencies that were searched to construct the file 34 // list. The supported glob and exclude patterns are equivalent to 35 // filepath.Glob, with an extension that recursive glob (** matching zero or 36 // more complete path entries) is supported. Any directories in the matches 37 // list will have a '/' suffix. 38 // 39 // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 40 // should be used instead, as they will automatically set up dependencies 41 // to rerun the primary builder when the list of matching files changes. 42 func Glob(pattern string, excludes []string) (matches, deps []string, err error) { 43 return startGlob(OsFs, pattern, excludes) 44 } 45 46 func startGlob(fs FileSystem, pattern string, excludes []string) (matches, deps []string, err error) { 47 if filepath.Base(pattern) == "**" { 48 return nil, nil, GlobLastRecursiveErr 49 } else { 50 matches, deps, err = glob(fs, pattern, false) 51 } 52 53 if err != nil { 54 return nil, nil, err 55 } 56 57 matches, err = filterExcludes(matches, excludes) 58 if err != nil { 59 return nil, nil, err 60 } 61 62 // If the pattern has wildcards, we added dependencies on the 63 // containing directories to know about changes. 64 // 65 // If the pattern didn't have wildcards, and didn't find matches, the 66 // most specific found directories were added. 67 // 68 // But if it didn't have wildcards, and did find a match, no 69 // dependencies were added, so add the match itself to detect when it 70 // is removed. 71 if !isWild(pattern) { 72 deps = append(deps, matches...) 73 } 74 75 for i, match := range matches { 76 if isDir, err := fs.IsDir(match); err != nil { 77 return nil, nil, fmt.Errorf("IsDir(%s): %s", match, err.Error()) 78 } else if isDir { 79 matches[i] = match + "/" 80 } 81 } 82 83 return matches, deps, nil 84 } 85 86 // glob is a recursive helper function to handle globbing each level of the pattern individually, 87 // allowing searched directories to be tracked. Also handles the recursive glob pattern, **. 88 func glob(fs FileSystem, pattern string, hasRecursive bool) (matches, dirs []string, err error) { 89 if !isWild(pattern) { 90 // If there are no wilds in the pattern, check whether the file exists or not. 91 // Uses filepath.Glob instead of manually statting to get consistent results. 92 pattern = filepath.Clean(pattern) 93 matches, err = fs.glob(pattern) 94 if err != nil { 95 return matches, dirs, err 96 } 97 98 if len(matches) == 0 { 99 // Some part of the non-wild pattern didn't exist. Add the last existing directory 100 // as a dependency. 101 var matchDirs []string 102 for len(matchDirs) == 0 { 103 pattern, _ = saneSplit(pattern) 104 matchDirs, err = fs.glob(pattern) 105 if err != nil { 106 return matches, dirs, err 107 } 108 } 109 dirs = append(dirs, matchDirs...) 110 } 111 return matches, dirs, err 112 } 113 114 dir, file := saneSplit(pattern) 115 116 if file == "**" { 117 if hasRecursive { 118 return matches, dirs, GlobMultipleRecursiveErr 119 } 120 hasRecursive = true 121 } 122 123 dirMatches, dirs, err := glob(fs, dir, hasRecursive) 124 if err != nil { 125 return nil, nil, err 126 } 127 128 for _, m := range dirMatches { 129 if isDir, err := fs.IsDir(m); err != nil { 130 return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) 131 } else if isDir { 132 if file == "**" { 133 recurseDirs, err := fs.ListDirsRecursive(m) 134 if err != nil { 135 return nil, nil, err 136 } 137 matches = append(matches, recurseDirs...) 138 } else { 139 dirs = append(dirs, m) 140 newMatches, err := fs.glob(filepath.Join(m, file)) 141 if err != nil { 142 return nil, nil, err 143 } 144 if file[0] != '.' { 145 newMatches = filterDotFiles(newMatches) 146 } 147 matches = append(matches, newMatches...) 148 } 149 } 150 } 151 152 return matches, dirs, nil 153 } 154 155 // Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations 156 // Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is 157 // not "/". Returns ".", "" if path is "." 158 func saneSplit(path string) (dir, file string) { 159 if path == "." { 160 return ".", "" 161 } 162 dir, file = filepath.Split(path) 163 switch dir { 164 case "": 165 dir = "." 166 case "/": 167 // Nothing 168 default: 169 dir = dir[:len(dir)-1] 170 } 171 return dir, file 172 } 173 174 func isWild(pattern string) bool { 175 return strings.ContainsAny(pattern, "*?[") 176 } 177 178 // Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and 179 // recursive (**) glob patterns are supported. 180 func filterExcludes(matches []string, excludes []string) ([]string, error) { 181 if len(excludes) == 0 { 182 return matches, nil 183 } 184 185 var ret []string 186 matchLoop: 187 for _, m := range matches { 188 for _, e := range excludes { 189 exclude, err := Match(e, m) 190 if err != nil { 191 return nil, err 192 } 193 if exclude { 194 continue matchLoop 195 } 196 } 197 ret = append(ret, m) 198 } 199 200 return ret, nil 201 } 202 203 // filterDotFiles filters out files that start with '.' 204 func filterDotFiles(matches []string) []string { 205 ret := make([]string, 0, len(matches)) 206 207 for _, match := range matches { 208 _, name := filepath.Split(match) 209 if name[0] == '.' { 210 continue 211 } 212 ret = append(ret, match) 213 } 214 215 return ret 216 } 217 218 // Match returns true if name matches pattern using the same rules as filepath.Match, but supporting 219 // hierarchical patterns (a/*) and recursive globs (**). 220 func Match(pattern, name string) (bool, error) { 221 if filepath.Base(pattern) == "**" { 222 return false, GlobLastRecursiveErr 223 } 224 225 for { 226 var patternFile, nameFile string 227 pattern, patternFile = saneSplit(pattern) 228 name, nameFile = saneSplit(name) 229 230 if patternFile == "**" { 231 return matchPrefix(pattern, filepath.Join(name, nameFile)) 232 } 233 234 if nameFile == "" && patternFile == "" { 235 return true, nil 236 } else if nameFile == "" || patternFile == "" { 237 return false, nil 238 } 239 240 match, err := filepath.Match(patternFile, nameFile) 241 if err != nil || !match { 242 return match, err 243 } 244 } 245 } 246 247 // matchPrefix returns true if the beginning of name matches pattern using the same rules as 248 // filepath.Match, but supporting hierarchical patterns (a/*). Recursive globs (**) are not 249 // supported, they should have been handled in Match(). 250 func matchPrefix(pattern, name string) (bool, error) { 251 if len(pattern) > 0 && pattern[0] == '/' { 252 if len(name) > 0 && name[0] == '/' { 253 pattern = pattern[1:] 254 name = name[1:] 255 } else { 256 return false, nil 257 } 258 } 259 260 for { 261 var patternElem, nameElem string 262 patternElem, pattern = saneSplitFirst(pattern) 263 nameElem, name = saneSplitFirst(name) 264 265 if patternElem == "." { 266 patternElem = "" 267 } 268 if nameElem == "." { 269 nameElem = "" 270 } 271 272 if patternElem == "**" { 273 return false, GlobMultipleRecursiveErr 274 } 275 276 if patternElem == "" { 277 return true, nil 278 } else if nameElem == "" { 279 return false, nil 280 } 281 282 match, err := filepath.Match(patternElem, nameElem) 283 if err != nil || !match { 284 return match, err 285 } 286 } 287 } 288 289 func saneSplitFirst(path string) (string, string) { 290 i := strings.IndexRune(path, filepath.Separator) 291 if i < 0 { 292 return path, "" 293 } 294 return path[:i], path[i+1:] 295 } 296 297 func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) { 298 var ( 299 matches []string 300 deps []string 301 ) 302 303 globedList = make([]string, 0) 304 depDirs = make([]string, 0) 305 306 for _, pattern := range patterns { 307 if isWild(pattern) { 308 matches, deps, err = Glob(filepath.Join(prefix, pattern), nil) 309 if err != nil { 310 return nil, nil, err 311 } 312 globedList = append(globedList, matches...) 313 depDirs = append(depDirs, deps...) 314 } else { 315 globedList = append(globedList, filepath.Join(prefix, pattern)) 316 } 317 } 318 return globedList, depDirs, nil 319 } 320 321 // IsGlob returns true if the pattern contains any glob characters (*, ?, or [). 322 func IsGlob(pattern string) bool { 323 return strings.IndexAny(pattern, "*?[") >= 0 324 } 325 326 // HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). 327 func HasGlob(in []string) bool { 328 for _, s := range in { 329 if IsGlob(s) { 330 return true 331 } 332 } 333 334 return false 335 } 336 337 // GlobWithDepFile finds all files and directories that match glob. Directories 338 // will have a trailing '/'. It compares the list of matches against the 339 // contents of fileListFile, and rewrites fileListFile if it has changed. It 340 // also writes all of the the directories it traversed as dependencies on 341 // fileListFile to depFile. 342 // 343 // The format of glob is either path/*.ext for a single directory glob, or 344 // path/**/*.ext for a recursive glob. 345 // 346 // Returns a list of file paths, and an error. 347 // 348 // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 349 // should be used instead, as they will automatically set up dependencies 350 // to rerun the primary builder when the list of matching files changes. 351 func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { 352 files, deps, err := Glob(glob, excludes) 353 if err != nil { 354 return nil, err 355 } 356 357 fileList := strings.Join(files, "\n") + "\n" 358 359 WriteFileIfChanged(fileListFile, []byte(fileList), 0666) 360 deptools.WriteDepFile(depFile, fileListFile, deps) 361 362 return 363 } 364 365 // WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if 366 // the files does not already exist with identical contents. This can be used 367 // along with ninja restat rules to skip rebuilding downstream rules if no 368 // changes were made by a rule. 369 func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { 370 var isChanged bool 371 372 dir := filepath.Dir(filename) 373 err := os.MkdirAll(dir, 0777) 374 if err != nil { 375 return err 376 } 377 378 info, err := os.Stat(filename) 379 if err != nil { 380 if os.IsNotExist(err) { 381 // The file does not exist yet. 382 isChanged = true 383 } else { 384 return err 385 } 386 } else { 387 if info.Size() != int64(len(data)) { 388 isChanged = true 389 } else { 390 oldData, err := ioutil.ReadFile(filename) 391 if err != nil { 392 return err 393 } 394 395 if len(oldData) != len(data) { 396 isChanged = true 397 } else { 398 for i := range data { 399 if oldData[i] != data[i] { 400 isChanged = true 401 break 402 } 403 } 404 } 405 } 406 } 407 408 if isChanged { 409 err = ioutil.WriteFile(filename, data, perm) 410 if err != nil { 411 return err 412 } 413 } 414 415 return nil 416 }