github.com/chenfeining/golangci-lint@v1.0.2-0.20230730162517-14c6c67868df/internal/pkgcache/pkgcache.go (about) 1 package pkgcache 2 3 import ( 4 "bytes" 5 "encoding/gob" 6 "encoding/hex" 7 "errors" 8 "fmt" 9 "runtime" 10 "sort" 11 "sync" 12 13 "golang.org/x/tools/go/packages" 14 15 "github.com/chenfeining/golangci-lint/internal/cache" 16 "github.com/chenfeining/golangci-lint/pkg/logutils" 17 "github.com/chenfeining/golangci-lint/pkg/timeutils" 18 ) 19 20 type HashMode int 21 22 const ( 23 HashModeNeedOnlySelf HashMode = iota 24 HashModeNeedDirectDeps 25 HashModeNeedAllDeps 26 ) 27 28 // Cache is a per-package data cache. A cached data is invalidated when 29 // package, or it's dependencies change. 30 type Cache struct { 31 lowLevelCache *cache.Cache 32 pkgHashes sync.Map 33 sw *timeutils.Stopwatch 34 log logutils.Log // not used now, but may be needed for future debugging purposes 35 ioSem chan struct{} // semaphore limiting parallel IO 36 } 37 38 func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) { 39 c, err := cache.Default() 40 if err != nil { 41 return nil, err 42 } 43 return &Cache{ 44 lowLevelCache: c, 45 sw: sw, 46 log: log, 47 ioSem: make(chan struct{}, runtime.GOMAXPROCS(-1)), 48 }, nil 49 } 50 51 func (c *Cache) Trim() { 52 c.sw.TrackStage("trim", func() { 53 c.lowLevelCache.Trim() 54 }) 55 } 56 57 func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) error { 58 var err error 59 buf := &bytes.Buffer{} 60 c.sw.TrackStage("gob", func() { 61 err = gob.NewEncoder(buf).Encode(data) 62 }) 63 if err != nil { 64 return fmt.Errorf("failed to gob encode: %w", err) 65 } 66 67 var aID cache.ActionID 68 69 c.sw.TrackStage("key build", func() { 70 aID, err = c.pkgActionID(pkg, mode) 71 if err == nil { 72 subkey, subkeyErr := cache.Subkey(aID, key) 73 if subkeyErr != nil { 74 err = fmt.Errorf("failed to build subkey: %w", subkeyErr) 75 } 76 aID = subkey 77 } 78 }) 79 if err != nil { 80 return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) 81 } 82 c.ioSem <- struct{}{} 83 c.sw.TrackStage("cache io", func() { 84 err = c.lowLevelCache.PutBytes(aID, buf.Bytes()) 85 }) 86 <-c.ioSem 87 if err != nil { 88 return fmt.Errorf("failed to save data to low-level cache by key %s for package %s: %w", key, pkg.Name, err) 89 } 90 91 return nil 92 } 93 94 var ErrMissing = errors.New("missing data") 95 96 func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) error { 97 var aID cache.ActionID 98 var err error 99 c.sw.TrackStage("key build", func() { 100 aID, err = c.pkgActionID(pkg, mode) 101 if err == nil { 102 subkey, subkeyErr := cache.Subkey(aID, key) 103 if subkeyErr != nil { 104 err = fmt.Errorf("failed to build subkey: %w", subkeyErr) 105 } 106 aID = subkey 107 } 108 }) 109 if err != nil { 110 return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) 111 } 112 113 var b []byte 114 c.ioSem <- struct{}{} 115 c.sw.TrackStage("cache io", func() { 116 b, _, err = c.lowLevelCache.GetBytes(aID) 117 }) 118 <-c.ioSem 119 if err != nil { 120 if cache.IsErrMissing(err) { 121 return ErrMissing 122 } 123 return fmt.Errorf("failed to get data from low-level cache by key %s for package %s: %w", key, pkg.Name, err) 124 } 125 126 c.sw.TrackStage("gob", func() { 127 err = gob.NewDecoder(bytes.NewReader(b)).Decode(data) 128 }) 129 if err != nil { 130 return fmt.Errorf("failed to gob decode: %w", err) 131 } 132 133 return nil 134 } 135 136 func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionID, error) { 137 hash, err := c.packageHash(pkg, mode) 138 if err != nil { 139 return cache.ActionID{}, fmt.Errorf("failed to get package hash: %w", err) 140 } 141 142 key, err := cache.NewHash("action ID") 143 if err != nil { 144 return cache.ActionID{}, fmt.Errorf("failed to make a hash: %w", err) 145 } 146 fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) 147 fmt.Fprintf(key, "pkghash %s\n", hash) 148 149 return key.Sum(), nil 150 } 151 152 // packageHash computes a package's hash. The hash is based on all Go 153 // files that make up the package, as well as the hashes of imported 154 // packages. 155 func (c *Cache) packageHash(pkg *packages.Package, mode HashMode) (string, error) { 156 type hashResults map[HashMode]string 157 hashResI, ok := c.pkgHashes.Load(pkg) 158 if ok { 159 hashRes := hashResI.(hashResults) 160 if _, ok := hashRes[mode]; !ok { 161 return "", fmt.Errorf("no mode %d in hash result", mode) 162 } 163 return hashRes[mode], nil 164 } 165 166 hashRes := hashResults{} 167 168 key, err := cache.NewHash("package hash") 169 if err != nil { 170 return "", fmt.Errorf("failed to make a hash: %w", err) 171 } 172 173 fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) 174 for _, f := range pkg.CompiledGoFiles { 175 c.ioSem <- struct{}{} 176 h, fErr := cache.FileHash(f) 177 <-c.ioSem 178 if fErr != nil { 179 return "", fmt.Errorf("failed to calculate file %s hash: %w", f, fErr) 180 } 181 fmt.Fprintf(key, "file %s %x\n", f, h) 182 } 183 curSum := key.Sum() 184 hashRes[HashModeNeedOnlySelf] = hex.EncodeToString(curSum[:]) 185 186 imps := make([]*packages.Package, 0, len(pkg.Imports)) 187 for _, imp := range pkg.Imports { 188 imps = append(imps, imp) 189 } 190 sort.Slice(imps, func(i, j int) bool { 191 return imps[i].PkgPath < imps[j].PkgPath 192 }) 193 194 calcDepsHash := func(depMode HashMode) error { 195 for _, dep := range imps { 196 if dep.PkgPath == "unsafe" { 197 continue 198 } 199 200 depHash, depErr := c.packageHash(dep, depMode) 201 if depErr != nil { 202 return fmt.Errorf("failed to calculate hash for dependency %s with mode %d: %w", dep.Name, depMode, depErr) 203 } 204 205 fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash) 206 } 207 return nil 208 } 209 210 if err := calcDepsHash(HashModeNeedOnlySelf); err != nil { 211 return "", err 212 } 213 214 curSum = key.Sum() 215 hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:]) 216 217 if err := calcDepsHash(HashModeNeedAllDeps); err != nil { 218 return "", err 219 } 220 curSum = key.Sum() 221 hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:]) 222 223 if _, ok := hashRes[mode]; !ok { 224 return "", fmt.Errorf("invalid mode %d", mode) 225 } 226 227 c.pkgHashes.Store(pkg, hashRes) 228 return hashRes[mode], nil 229 }