github.com/thrasher-corp/golangci-lint@v1.17.3/pkg/lint/runner.go (about) 1 package lint 2 3 import ( 4 "context" 5 "fmt" 6 "runtime/debug" 7 "sort" 8 "strings" 9 "sync" 10 "time" 11 12 "github.com/golangci/golangci-lint/pkg/lint/lintersdb" 13 14 "github.com/golangci/golangci-lint/pkg/fsutils" 15 16 "github.com/golangci/golangci-lint/pkg/config" 17 "github.com/golangci/golangci-lint/pkg/goutil" 18 "github.com/golangci/golangci-lint/pkg/lint/astcache" 19 "github.com/golangci/golangci-lint/pkg/lint/linter" 20 "github.com/golangci/golangci-lint/pkg/logutils" 21 "github.com/golangci/golangci-lint/pkg/packages" 22 "github.com/golangci/golangci-lint/pkg/result" 23 "github.com/golangci/golangci-lint/pkg/result/processors" 24 "github.com/golangci/golangci-lint/pkg/timeutils" 25 ) 26 27 type Runner struct { 28 Processors []processors.Processor 29 Log logutils.Log 30 } 31 32 func NewRunner(astCache *astcache.Cache, cfg *config.Config, log logutils.Log, goenv *goutil.Env, 33 lineCache *fsutils.LineCache, dbManager *lintersdb.Manager) (*Runner, error) { 34 35 icfg := cfg.Issues 36 excludePatterns := icfg.ExcludePatterns 37 if icfg.UseDefaultExcludes { 38 excludePatterns = append(excludePatterns, config.GetDefaultExcludePatternsStrings()...) 39 } 40 41 var excludeTotalPattern string 42 if len(excludePatterns) != 0 { 43 excludeTotalPattern = fmt.Sprintf("(%s)", strings.Join(excludePatterns, "|")) 44 } 45 46 skipFilesProcessor, err := processors.NewSkipFiles(cfg.Run.SkipFiles) 47 if err != nil { 48 return nil, err 49 } 50 51 skipDirs := append([]string{}, packages.StdExcludeDirRegexps...) 52 skipDirs = append(skipDirs, cfg.Run.SkipDirs...) 53 skipDirsProcessor, err := processors.NewSkipDirs(skipDirs, log.Child("skip dirs"), cfg.Run.Args) 54 if err != nil { 55 return nil, err 56 } 57 58 var excludeRules []processors.ExcludeRule 59 for _, r := range icfg.ExcludeRules { 60 excludeRules = append(excludeRules, processors.ExcludeRule{ 61 Text: r.Text, 62 Source: r.Source, 63 Path: r.Path, 64 Linters: r.Linters, 65 }) 66 } 67 68 return &Runner{ 69 Processors: []processors.Processor{ 70 processors.NewCgo(goenv), 71 processors.NewFilenameUnadjuster(astCache, log.Child("filename_unadjuster")), // must go after Cgo 72 processors.NewPathPrettifier(), // must be before diff, nolint and exclude autogenerated processor at least 73 skipFilesProcessor, 74 skipDirsProcessor, // must be after path prettifier 75 76 processors.NewAutogeneratedExclude(astCache), 77 processors.NewIdentifierMarker(), // must be befor exclude 78 processors.NewExclude(excludeTotalPattern), 79 processors.NewExcludeRules(excludeRules, lineCache, log.Child("exclude_rules")), 80 processors.NewNolint(astCache, log.Child("nolint"), dbManager), 81 82 processors.NewUniqByLine(cfg), 83 processors.NewDiff(icfg.Diff, icfg.DiffFromRevision, icfg.DiffPatchFilePath), 84 processors.NewMaxPerFileFromLinter(cfg), 85 processors.NewMaxSameIssues(icfg.MaxSameIssues, log.Child("max_same_issues"), cfg), 86 processors.NewMaxFromLinter(icfg.MaxIssuesPerLinter, log.Child("max_from_linter"), cfg), 87 processors.NewSourceCode(lineCache, log.Child("source_code")), 88 processors.NewPathShortener(), 89 }, 90 Log: log, 91 }, nil 92 } 93 94 type lintRes struct { 95 linter *linter.Config 96 err error 97 issues []result.Issue 98 } 99 100 func (r *Runner) runLinterSafe(ctx context.Context, lintCtx *linter.Context, 101 lc *linter.Config) (ret []result.Issue, err error) { 102 103 defer func() { 104 if panicData := recover(); panicData != nil { 105 err = fmt.Errorf("panic occurred: %s", panicData) 106 r.Log.Warnf("Panic stack trace: %s", debug.Stack()) 107 } 108 }() 109 110 specificLintCtx := *lintCtx 111 specificLintCtx.Log = r.Log.Child(lc.Name()) 112 issues, err := lc.Linter.Run(ctx, &specificLintCtx) 113 if err != nil { 114 return nil, err 115 } 116 117 for _, i := range issues { 118 i.FromLinter = lc.Name() 119 } 120 121 return issues, nil 122 } 123 124 func (r Runner) runWorker(ctx context.Context, lintCtx *linter.Context, 125 tasksCh <-chan *linter.Config, lintResultsCh chan<- lintRes, name string) { 126 127 sw := timeutils.NewStopwatch(name, r.Log) 128 defer sw.Print() 129 130 for { 131 select { 132 case <-ctx.Done(): 133 return 134 case lc, ok := <-tasksCh: 135 if !ok { 136 return 137 } 138 if ctx.Err() != nil { 139 // XXX: if check it in only int a select 140 // it's possible to not enter to this case until tasksCh is empty. 141 return 142 } 143 var issues []result.Issue 144 var err error 145 sw.TrackStage(lc.Name(), func() { 146 issues, err = r.runLinterSafe(ctx, lintCtx, lc) 147 }) 148 lintResultsCh <- lintRes{ 149 linter: lc, 150 err: err, 151 issues: issues, 152 } 153 } 154 } 155 } 156 157 func (r Runner) logWorkersStat(workersFinishTimes []time.Time) { 158 lastFinishTime := workersFinishTimes[0] 159 for _, t := range workersFinishTimes { 160 if t.After(lastFinishTime) { 161 lastFinishTime = t 162 } 163 } 164 165 logStrings := []string{} 166 for i, t := range workersFinishTimes { 167 if t.Equal(lastFinishTime) { 168 continue 169 } 170 171 logStrings = append(logStrings, fmt.Sprintf("#%d: %s", i+1, lastFinishTime.Sub(t))) 172 } 173 174 r.Log.Infof("Workers idle times: %s", strings.Join(logStrings, ", ")) 175 } 176 177 func getSortedLintersConfigs(linters []*linter.Config) []*linter.Config { 178 ret := make([]*linter.Config, len(linters)) 179 copy(ret, linters) 180 181 sort.Slice(ret, func(i, j int) bool { 182 return ret[i].GetSpeed() < ret[j].GetSpeed() 183 }) 184 185 return ret 186 } 187 188 func (r *Runner) runWorkers(ctx context.Context, lintCtx *linter.Context, linters []*linter.Config) <-chan lintRes { 189 tasksCh := make(chan *linter.Config, len(linters)) 190 lintResultsCh := make(chan lintRes, len(linters)) 191 var wg sync.WaitGroup 192 193 workersFinishTimes := make([]time.Time, lintCtx.Cfg.Run.Concurrency) 194 195 for i := 0; i < lintCtx.Cfg.Run.Concurrency; i++ { 196 wg.Add(1) 197 go func(i int) { 198 defer wg.Done() 199 name := fmt.Sprintf("worker.%d", i+1) 200 r.runWorker(ctx, lintCtx, tasksCh, lintResultsCh, name) 201 workersFinishTimes[i] = time.Now() 202 }(i) 203 } 204 205 lcs := getSortedLintersConfigs(linters) 206 for _, lc := range lcs { 207 tasksCh <- lc 208 } 209 close(tasksCh) 210 211 go func() { 212 wg.Wait() 213 close(lintResultsCh) 214 215 r.logWorkersStat(workersFinishTimes) 216 }() 217 218 return lintResultsCh 219 } 220 221 func (r Runner) processLintResults(inCh <-chan lintRes) <-chan lintRes { 222 outCh := make(chan lintRes, 64) 223 224 go func() { 225 sw := timeutils.NewStopwatch("processing", r.Log) 226 227 var issuesBefore, issuesAfter int 228 defer close(outCh) 229 230 for res := range inCh { 231 if res.err != nil { 232 r.Log.Warnf("Can't run linter %s: %s", res.linter.Name(), res.err) 233 continue 234 } 235 236 if len(res.issues) != 0 { 237 issuesBefore += len(res.issues) 238 res.issues = r.processIssues(res.issues, sw) 239 issuesAfter += len(res.issues) 240 outCh <- res 241 } 242 } 243 244 // finalize processors: logging, clearing, no heavy work here 245 246 for _, p := range r.Processors { 247 p := p 248 sw.TrackStage(p.Name(), func() { 249 p.Finish() 250 }) 251 } 252 253 if issuesBefore != issuesAfter { 254 r.Log.Infof("Issues before processing: %d, after processing: %d", issuesBefore, issuesAfter) 255 } 256 sw.PrintStages() 257 }() 258 259 return outCh 260 } 261 262 func collectIssues(resCh <-chan lintRes) <-chan result.Issue { 263 retIssues := make(chan result.Issue, 1024) 264 go func() { 265 defer close(retIssues) 266 267 for res := range resCh { 268 if len(res.issues) == 0 { 269 continue 270 } 271 272 for _, i := range res.issues { 273 retIssues <- i 274 } 275 } 276 }() 277 278 return retIssues 279 } 280 281 func (r Runner) Run(ctx context.Context, linters []*linter.Config, lintCtx *linter.Context) <-chan result.Issue { 282 lintResultsCh := r.runWorkers(ctx, lintCtx, linters) 283 processedLintResultsCh := r.processLintResults(lintResultsCh) 284 if ctx.Err() != nil { 285 // XXX: always process issues, even if timeout occurred 286 finishedLintersN := 0 287 for range processedLintResultsCh { 288 finishedLintersN++ 289 } 290 291 r.Log.Errorf("%d/%d linters finished: deadline exceeded", 292 finishedLintersN, len(linters)) 293 } 294 295 return collectIssues(processedLintResultsCh) 296 } 297 298 func (r *Runner) processIssues(issues []result.Issue, sw *timeutils.Stopwatch) []result.Issue { 299 for _, p := range r.Processors { 300 var newIssues []result.Issue 301 var err error 302 p := p 303 sw.TrackStage(p.Name(), func() { 304 newIssues, err = p.Process(issues) 305 }) 306 307 if err != nil { 308 r.Log.Warnf("Can't process result by %s processor: %s", p.Name(), err) 309 } else { 310 issues = newIssues 311 } 312 313 if issues == nil { 314 issues = []result.Issue{} 315 } 316 } 317 318 return issues 319 }