github.com/mcuadros/enry@v1.7.3/cmd/enry/main.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "encoding/json" 7 "flag" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "sort" 15 "strings" 16 17 "gopkg.in/src-d/enry.v1" 18 "gopkg.in/src-d/enry.v1/data" 19 ) 20 21 var ( 22 version = "undefined" 23 build = "undefined" 24 commit = "undefined" 25 ) 26 27 func main() { 28 flag.Usage = usage 29 breakdownFlag := flag.Bool("breakdown", false, "") 30 jsonFlag := flag.Bool("json", false, "") 31 showVersion := flag.Bool("version", false, "Show the enry version information") 32 allLangs := flag.Bool("all", false, "Show all files, including those identifed as non-programming languages") 33 countMode := flag.String("mode", "byte", "the method used to count file size. Available options are: file, line and byte") 34 limitKB := flag.Int64("limit", 16*1024, "Analyse first N KB of the file (-1 means no limit)") 35 flag.Parse() 36 limit := (*limitKB) * 1024 37 38 if *showVersion { 39 fmt.Println(version) 40 return 41 } 42 43 root, err := filepath.Abs(flag.Arg(0)) 44 if err != nil { 45 log.Fatal(err) 46 } 47 48 fileInfo, err := os.Stat(root) 49 if err != nil { 50 log.Fatal(err) 51 } 52 53 if fileInfo.Mode().IsRegular() { 54 err = printFileAnalysis(root, limit, *jsonFlag) 55 if err != nil { 56 fmt.Println(err) 57 } 58 return 59 } 60 61 out := make(map[string][]string, 0) 62 err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error { 63 if err != nil { 64 log.Println(err) 65 return filepath.SkipDir 66 } 67 68 if !f.Mode().IsDir() && !f.Mode().IsRegular() { 69 return nil 70 } 71 72 relativePath, err := filepath.Rel(root, path) 73 if err != nil { 74 log.Println(err) 75 return nil 76 } 77 78 if relativePath == "." { 79 return nil 80 } 81 82 if f.IsDir() { 83 relativePath = relativePath + "/" 84 } 85 86 if enry.IsVendor(relativePath) || enry.IsDotFile(relativePath) || 87 enry.IsDocumentation(relativePath) || enry.IsConfiguration(relativePath) { 88 // TODO(bzz): skip enry.IsGeneratedPath() after https://github.com/src-d/enry/issues/213 89 if f.IsDir() { 90 return filepath.SkipDir 91 } 92 93 return nil 94 } 95 96 if f.IsDir() { 97 return nil 98 } 99 100 // TODO(bzz): provide API that mimics lingust CLI output for 101 // - running ByExtension & ByFilename 102 // - reading the file, if that did not work 103 // - GetLanguage([]Strategy) 104 content, err := readFile(path, limit) 105 if err != nil { 106 log.Println(err) 107 return nil 108 } 109 // TODO(bzz): skip enry.IsGeneratedContent() as well, after https://github.com/src-d/enry/issues/213 110 111 language := enry.GetLanguage(filepath.Base(path), content) 112 if language == enry.OtherLanguage { 113 return nil 114 } 115 116 // If we are not asked to display all, do as 117 // https://github.com/github/linguist/blob/bf95666fc15e49d556f2def4d0a85338423c25f3/lib/linguist/blob_helper.rb#L382 118 if !*allLangs && 119 enry.GetLanguageType(language) != enry.Programming && 120 enry.GetLanguageType(language) != enry.Markup { 121 return nil 122 } 123 124 out[language] = append(out[language], relativePath) 125 return nil 126 }) 127 128 if err != nil { 129 log.Fatal(err) 130 } 131 132 var buf bytes.Buffer 133 switch { 134 case *jsonFlag && !*breakdownFlag: 135 printJson(out, &buf) 136 case *jsonFlag && *breakdownFlag: 137 printBreakDown(out, &buf) 138 case *breakdownFlag: 139 printPercents(root, out, &buf, *countMode) 140 buf.WriteByte('\n') 141 printBreakDown(out, &buf) 142 default: 143 printPercents(root, out, &buf, *countMode) 144 } 145 146 fmt.Print(buf.String()) 147 } 148 149 func usage() { 150 fmt.Fprintf( 151 os.Stderr, 152 ` %[1]s %[2]s build: %[3]s commit: %[4]s, based on linguist commit: %[5]s 153 %[1]s, A simple (and faster) implementation of github/linguist 154 usage: %[1]s [-mode=(file|line|byte)] [-prog] <path> 155 %[1]s [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] <path> 156 %[1]s [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] 157 %[1]s [-version] 158 `, 159 os.Args[0], version, build, commit, data.LinguistCommit[:7], 160 ) 161 } 162 163 func printBreakDown(out map[string][]string, buff *bytes.Buffer) { 164 for name, language := range out { 165 fmt.Fprintln(buff, name) 166 for _, file := range language { 167 fmt.Fprintln(buff, file) 168 } 169 170 fmt.Fprintln(buff) 171 } 172 } 173 174 func printJson(out map[string][]string, buf *bytes.Buffer) { 175 json.NewEncoder(buf).Encode(out) 176 } 177 178 // filelistError represents a failed operation that took place across multiple files. 179 type filelistError []string 180 181 func (e filelistError) Error() string { 182 return fmt.Sprintf("Could not process the following files:\n%s", strings.Join(e, "\n")) 183 } 184 185 func printPercents(root string, fSummary map[string][]string, buff *bytes.Buffer, mode string) { 186 // Select the way we quantify 'amount' of code. 187 reducer := fileCountValues 188 switch mode { 189 case "file": 190 reducer = fileCountValues 191 case "line": 192 reducer = lineCountValues 193 case "byte": 194 reducer = byteCountValues 195 } 196 197 // Reduce the list of files to a quantity of file type. 198 var ( 199 total float64 200 keys []string 201 unreadableFiles filelistError 202 fileValues = make(map[string]float64) 203 ) 204 for fType, files := range fSummary { 205 val, err := reducer(root, files) 206 if err != nil { 207 unreadableFiles = append(unreadableFiles, err...) 208 } 209 fileValues[fType] = val 210 keys = append(keys, fType) 211 total += val 212 } 213 214 // Slice the keys by their quantity (file count, line count, byte size, etc.). 215 sort.Slice(keys, func(i, j int) bool { 216 return fileValues[keys[i]] > fileValues[keys[j]] 217 }) 218 219 // Calculate and write percentages of each file type. 220 for _, fType := range keys { 221 val := fileValues[fType] 222 percent := val / total * 100.0 223 buff.WriteString(fmt.Sprintf("%.2f%%\t%s\n", percent, fType)) 224 if unreadableFiles != nil { 225 buff.WriteString(fmt.Sprintf("\n%s", unreadableFiles.Error())) 226 } 227 } 228 } 229 230 func fileCountValues(_ string, files []string) (float64, filelistError) { 231 return float64(len(files)), nil 232 } 233 234 func lineCountValues(root string, files []string) (float64, filelistError) { 235 var filesErr filelistError 236 var t float64 237 for _, fName := range files { 238 l, _ := getLines(filepath.Join(root, fName), nil) 239 t += float64(l) 240 } 241 return t, filesErr 242 } 243 244 func byteCountValues(root string, files []string) (float64, filelistError) { 245 var filesErr filelistError 246 var t float64 247 for _, fName := range files { 248 f, err := os.Open(filepath.Join(root, fName)) 249 if err != nil { 250 filesErr = append(filesErr, fName) 251 continue 252 } 253 fi, err := f.Stat() 254 f.Close() 255 if err != nil { 256 filesErr = append(filesErr, fName) 257 continue 258 } 259 t += float64(fi.Size()) 260 } 261 return t, filesErr 262 } 263 264 func printFileAnalysis(file string, limit int64, isJSON bool) error { 265 data, err := readFile(file, limit) 266 if err != nil { 267 return err 268 } 269 270 isSample := limit > 0 && len(data) == int(limit) 271 272 full := data 273 if isSample { 274 // communicate to getLines that we don't have full contents 275 full = nil 276 } 277 278 totalLines, nonBlank := getLines(file, full) 279 280 // functions below can work on a sample 281 fileType := getFileType(file, data) 282 language := enry.GetLanguage(file, data) 283 mimeType := enry.GetMIMEType(file, language) 284 vendored := enry.IsVendor(file) 285 286 if isJSON { 287 return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ 288 "filename": filepath.Base(file), 289 "lines": nonBlank, 290 "total_lines": totalLines, 291 "type": fileType, 292 "mime": mimeType, 293 "language": language, 294 "vendored": vendored, 295 }) 296 } 297 298 fmt.Printf( 299 `%s: %d lines (%d sloc) 300 type: %s 301 mime_type: %s 302 language: %s 303 vendored: %t 304 `, 305 filepath.Base(file), totalLines, nonBlank, fileType, mimeType, language, vendored, 306 ) 307 return nil 308 } 309 310 func readFile(path string, limit int64) ([]byte, error) { 311 if limit <= 0 { 312 return ioutil.ReadFile(path) 313 } 314 f, err := os.Open(path) 315 if err != nil { 316 return nil, err 317 } 318 defer f.Close() 319 st, err := f.Stat() 320 if err != nil { 321 return nil, err 322 } 323 size := st.Size() 324 if limit > 0 && size > limit { 325 size = limit 326 } 327 buf := bytes.NewBuffer(nil) 328 buf.Grow(int(size)) 329 _, err = io.Copy(buf, io.LimitReader(f, limit)) 330 return buf.Bytes(), err 331 } 332 333 func getLines(file string, content []byte) (total, blank int) { 334 var r io.Reader 335 if content != nil { 336 r = bytes.NewReader(content) 337 } else { 338 // file not loaded to memory - stream it 339 f, err := os.Open(file) 340 if err != nil { 341 fmt.Println(err) 342 return 343 } 344 defer f.Close() 345 r = f 346 } 347 br := bufio.NewReader(r) 348 lastBlank := true 349 empty := true 350 for { 351 data, prefix, err := br.ReadLine() 352 if err == io.EOF { 353 break 354 } else if err != nil { 355 fmt.Println(err) 356 break 357 } 358 if prefix { 359 continue 360 } 361 empty = false 362 total++ 363 lastBlank = len(data) == 0 364 if lastBlank { 365 blank++ 366 } 367 } 368 if !empty && lastBlank { 369 total++ 370 blank++ 371 } 372 nonBlank := total - blank 373 return total, nonBlank 374 } 375 376 func getFileType(file string, content []byte) string { 377 switch { 378 case enry.IsImage(file): 379 return "Image" 380 case enry.IsBinary(content): 381 return "Binary" 382 default: 383 return "Text" 384 } 385 }