github.com/alex123012/deckhouse-controller-tools@v0.0.0-20230510090815-d594daf1af8c/pkg/markers/collect.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package markers 18 19 import ( 20 "go/ast" 21 "go/token" 22 "strings" 23 "sync" 24 25 "sigs.k8s.io/controller-tools/pkg/loader" 26 ) 27 28 // Collector collects and parses marker comments defined in the registry 29 // from package source code. If no registry is provided, an empty one will 30 // be initialized on the first call to MarkersInPackage. 31 type Collector struct { 32 *Registry 33 34 byPackage map[string]map[ast.Node]MarkerValues 35 mu sync.Mutex 36 } 37 38 // MarkerValues are all the values for some set of markers. 39 type MarkerValues map[string][]interface{} 40 41 // Get fetches the first value that for the given marker, returning 42 // nil if no values are available. 43 func (v MarkerValues) Get(name string) interface{} { 44 vals := v[name] 45 if len(vals) == 0 { 46 return nil 47 } 48 return vals[0] 49 } 50 51 func (c *Collector) init() { 52 if c.Registry == nil { 53 c.Registry = &Registry{} 54 } 55 if c.byPackage == nil { 56 c.byPackage = make(map[string]map[ast.Node]MarkerValues) 57 } 58 } 59 60 // MarkersInPackage computes the marker values by node for the given package. Results 61 // are cached by package ID, so this is safe to call repeatedly from different functions. 62 // Each file in the package is treated as a distinct node. 63 // 64 // We consider a marker to be associated with a given AST node if either of the following are true: 65 // 66 // - it's in the Godoc for that AST node 67 // 68 // - it's in the closest non-godoc comment group above that node, 69 // *and* that node is a type or field node, *and* [it's either 70 // registered as type-level *or* it's not registered as being 71 // package-level] 72 // 73 // - it's not in the Godoc of a node, doesn't meet the above criteria, and 74 // isn't in a struct definition (in which case it's package-level) 75 func (c *Collector) MarkersInPackage(pkg *loader.Package) (map[ast.Node]MarkerValues, error) { 76 c.mu.Lock() 77 c.init() 78 if markers, exist := c.byPackage[pkg.ID]; exist { 79 c.mu.Unlock() 80 return markers, nil 81 } 82 // unlock early, it's ok if we do a bit extra work rather than locking while we're working 83 c.mu.Unlock() 84 85 pkg.NeedSyntax() 86 nodeMarkersRaw := c.associatePkgMarkers(pkg) 87 markers, err := c.parseMarkersInPackage(nodeMarkersRaw) 88 if err != nil { 89 return nil, err 90 } 91 92 c.mu.Lock() 93 defer c.mu.Unlock() 94 c.byPackage[pkg.ID] = markers 95 96 return markers, nil 97 } 98 99 // parseMarkersInPackage parses the given raw marker comments into output values using the registry. 100 func (c *Collector) parseMarkersInPackage(nodeMarkersRaw map[ast.Node][]markerComment) (map[ast.Node]MarkerValues, error) { 101 var errors []error 102 nodeMarkerValues := make(map[ast.Node]MarkerValues) 103 for node, markersRaw := range nodeMarkersRaw { 104 var target TargetType 105 switch node.(type) { 106 case *ast.File: 107 target = DescribesPackage 108 case *ast.Field: 109 target = DescribesField 110 default: 111 target = DescribesType 112 } 113 markerVals := make(map[string][]interface{}) 114 for _, markerRaw := range markersRaw { 115 markerText := markerRaw.Text() 116 def := c.Registry.Lookup(markerText, target) 117 if def == nil { 118 continue 119 } 120 val, err := def.Parse(markerText) 121 if err != nil { 122 errors = append(errors, loader.ErrFromNode(err, markerRaw)) 123 continue 124 } 125 markerVals[def.Name] = append(markerVals[def.Name], val) 126 } 127 nodeMarkerValues[node] = markerVals 128 } 129 130 return nodeMarkerValues, loader.MaybeErrList(errors) 131 } 132 133 // associatePkgMarkers associates markers with AST nodes in the given package. 134 func (c *Collector) associatePkgMarkers(pkg *loader.Package) map[ast.Node][]markerComment { 135 nodeMarkers := make(map[ast.Node][]markerComment) 136 for _, file := range pkg.Syntax { 137 fileNodeMarkers := c.associateFileMarkers(file) 138 for node, markers := range fileNodeMarkers { 139 nodeMarkers[node] = append(nodeMarkers[node], markers...) 140 } 141 } 142 143 return nodeMarkers 144 } 145 146 // associateFileMarkers associates markers with AST nodes in the given file. 147 func (c *Collector) associateFileMarkers(file *ast.File) map[ast.Node][]markerComment { 148 // grab all the raw marker comments by node 149 visitor := markerSubVisitor{ 150 collectPackageLevel: true, 151 markerVisitor: &markerVisitor{ 152 nodeMarkers: make(map[ast.Node][]markerComment), 153 allComments: file.Comments, 154 }, 155 } 156 ast.Walk(visitor, file) 157 158 // grab the last package-level comments at the end of the file (if any) 159 lastFileMarkers := visitor.markersBetween(false, visitor.commentInd, len(visitor.allComments)) 160 visitor.pkgMarkers = append(visitor.pkgMarkers, lastFileMarkers...) 161 162 // figure out if any type-level markers are actually package-level markers 163 for node, markers := range visitor.nodeMarkers { 164 _, isType := node.(*ast.TypeSpec) 165 if !isType { 166 continue 167 } 168 endOfMarkers := 0 169 for _, marker := range markers { 170 if marker.fromGodoc { 171 // markers from godoc are never package level 172 markers[endOfMarkers] = marker 173 endOfMarkers++ 174 continue 175 } 176 markerText := marker.Text() 177 typeDef := c.Registry.Lookup(markerText, DescribesType) 178 if typeDef != nil { 179 // prefer assuming type-level markers 180 markers[endOfMarkers] = marker 181 endOfMarkers++ 182 continue 183 } 184 def := c.Registry.Lookup(markerText, DescribesPackage) 185 if def == nil { 186 // assume type-level unless proven otherwise 187 markers[endOfMarkers] = marker 188 endOfMarkers++ 189 continue 190 } 191 // it's package-level, since a package-level definition exists 192 visitor.pkgMarkers = append(visitor.pkgMarkers, marker) 193 } 194 visitor.nodeMarkers[node] = markers[:endOfMarkers] // re-set after trimming the package markers 195 } 196 visitor.nodeMarkers[file] = visitor.pkgMarkers 197 198 return visitor.nodeMarkers 199 } 200 201 // markerComment is an AST comment that contains a marker. 202 // It may or may not be from a Godoc comment, which affects 203 // marker re-associated (from type-level to package-level) 204 type markerComment struct { 205 *ast.Comment 206 fromGodoc bool 207 } 208 209 // Text returns the text of the marker, stripped of the comment 210 // marker and leading spaces, as should be passed to Registry.Lookup 211 // and Registry.Parse. 212 func (c markerComment) Text() string { 213 return strings.TrimSpace(c.Comment.Text[2:]) 214 } 215 216 // markerVisistor visits AST nodes, recording markers associated with each node. 217 type markerVisitor struct { 218 allComments []*ast.CommentGroup 219 commentInd int 220 221 declComments []markerComment 222 lastLineCommentGroup *ast.CommentGroup 223 224 pkgMarkers []markerComment 225 nodeMarkers map[ast.Node][]markerComment 226 } 227 228 // isMarkerComment checks that the given comment is a single-line (`//`) 229 // comment and it's first non-space content is `+`. 230 func isMarkerComment(comment string) bool { 231 if comment[0:2] != "//" { 232 return false 233 } 234 stripped := strings.TrimSpace(comment[2:]) 235 if len(stripped) < 1 || stripped[0] != '+' { 236 return false 237 } 238 return true 239 } 240 241 // markersBetween grabs the markers between the given indicies in the list of all comments. 242 func (v *markerVisitor) markersBetween(fromGodoc bool, start, end int) []markerComment { 243 if start < 0 || end < 0 { 244 return nil 245 } 246 var res []markerComment 247 for i := start; i < end; i++ { 248 commentGroup := v.allComments[i] 249 for _, comment := range commentGroup.List { 250 if !isMarkerComment(comment.Text) { 251 continue 252 } 253 res = append(res, markerComment{Comment: comment, fromGodoc: fromGodoc}) 254 } 255 } 256 return res 257 } 258 259 type markerSubVisitor struct { 260 *markerVisitor 261 node ast.Node 262 collectPackageLevel bool 263 } 264 265 // Visit collects markers for each node in the AST, optionally 266 // collecting unassociated markers as package-level. 267 func (v markerSubVisitor) Visit(node ast.Node) ast.Visitor { 268 if node == nil { 269 // end of the node, so we might need to advance comments beyond the end 270 // of the block if we don't want to collect package-level markers in 271 // this block. 272 273 if !v.collectPackageLevel { 274 if v.commentInd < len(v.allComments) { 275 lastCommentInd := v.commentInd 276 nextGroup := v.allComments[lastCommentInd] 277 for nextGroup.Pos() < v.node.End() { 278 lastCommentInd++ 279 if lastCommentInd >= len(v.allComments) { 280 // after the increment so our decrement below still makes sense 281 break 282 } 283 nextGroup = v.allComments[lastCommentInd] 284 } 285 v.commentInd = lastCommentInd 286 } 287 } 288 289 return nil 290 } 291 292 // skip comments on the same line as the previous node 293 // making sure to double-check for the case where we've gone past the end of the comments 294 // but still have to finish up typespec-gendecl association (see below). 295 if v.lastLineCommentGroup != nil && v.commentInd < len(v.allComments) && v.lastLineCommentGroup.Pos() == v.allComments[v.commentInd].Pos() { 296 v.commentInd++ 297 } 298 299 // stop visiting if there are no more comments in the file 300 // NB(directxman12): we can't just stop immediately, because we 301 // still need to check if there are typespecs associated with gendecls. 302 var markerCommentBlock []markerComment 303 var docCommentBlock []markerComment 304 lastCommentInd := v.commentInd 305 if v.commentInd < len(v.allComments) { 306 // figure out the first comment after the node in question... 307 nextGroup := v.allComments[lastCommentInd] 308 for nextGroup.Pos() < node.Pos() { 309 lastCommentInd++ 310 if lastCommentInd >= len(v.allComments) { 311 // after the increment so our decrement below still makes sense 312 break 313 } 314 nextGroup = v.allComments[lastCommentInd] 315 } 316 lastCommentInd-- // ...then decrement to get the last comment before the node in question 317 318 // figure out the godoc comment so we can deal with it separately 319 var docGroup *ast.CommentGroup 320 docGroup, v.lastLineCommentGroup = associatedCommentsFor(node) 321 322 // find the last comment group that's not godoc 323 markerCommentInd := lastCommentInd 324 if docGroup != nil && v.allComments[markerCommentInd].Pos() == docGroup.Pos() { 325 markerCommentInd-- 326 } 327 328 // check if we have freestanding package markers, 329 // and find the markers in our "closest non-godoc" comment block, 330 // plus our godoc comment block 331 if markerCommentInd >= v.commentInd { 332 if v.collectPackageLevel { 333 // assume anything between the comment ind and the marker ind (not including it) 334 // are package-level 335 v.pkgMarkers = append(v.pkgMarkers, v.markersBetween(false, v.commentInd, markerCommentInd)...) 336 } 337 markerCommentBlock = v.markersBetween(false, markerCommentInd, markerCommentInd+1) 338 docCommentBlock = v.markersBetween(true, markerCommentInd+1, lastCommentInd+1) 339 } else { 340 docCommentBlock = v.markersBetween(true, markerCommentInd+1, lastCommentInd+1) 341 } 342 } 343 344 resVisitor := markerSubVisitor{ 345 collectPackageLevel: false, // don't collect package level by default 346 markerVisitor: v.markerVisitor, 347 node: node, 348 } 349 350 // associate those markers with a node 351 switch typedNode := node.(type) { 352 case *ast.GenDecl: 353 // save the comments associated with the gen-decl if it's a single-line type decl 354 if typedNode.Lparen != token.NoPos || typedNode.Tok != token.TYPE { 355 // not a single-line type spec, treat them as free comments 356 v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) 357 break 358 } 359 // save these, we'll need them when we encounter the actual type spec 360 v.declComments = append(v.declComments, markerCommentBlock...) 361 v.declComments = append(v.declComments, docCommentBlock...) 362 case *ast.TypeSpec: 363 // add in comments attributed to the gen-decl, if any, 364 // as well as comments associated with the actual type 365 v.nodeMarkers[node] = append(v.nodeMarkers[node], v.declComments...) 366 v.nodeMarkers[node] = append(v.nodeMarkers[node], markerCommentBlock...) 367 v.nodeMarkers[node] = append(v.nodeMarkers[node], docCommentBlock...) 368 369 v.declComments = nil 370 v.collectPackageLevel = false // don't collect package-level inside type structs 371 case *ast.Field: 372 v.nodeMarkers[node] = append(v.nodeMarkers[node], markerCommentBlock...) 373 v.nodeMarkers[node] = append(v.nodeMarkers[node], docCommentBlock...) 374 case *ast.File: 375 v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) 376 v.pkgMarkers = append(v.pkgMarkers, docCommentBlock...) 377 378 // collect markers in root file scope 379 resVisitor.collectPackageLevel = true 380 default: 381 // assume markers before anything else are package-level markers, 382 // *but* don't include any markers in godoc 383 if v.collectPackageLevel { 384 v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) 385 } 386 } 387 388 // increment the comment ind so that we start at the right place for the next node 389 v.commentInd = lastCommentInd + 1 390 391 return resVisitor 392 393 } 394 395 // associatedCommentsFor returns the doc comment group (if relevant and present) and end-of-line comment 396 // (again if relevant and present) for the given AST node. 397 func associatedCommentsFor(node ast.Node) (docGroup *ast.CommentGroup, lastLineCommentGroup *ast.CommentGroup) { 398 switch typedNode := node.(type) { 399 case *ast.Field: 400 docGroup = typedNode.Doc 401 lastLineCommentGroup = typedNode.Comment 402 case *ast.File: 403 docGroup = typedNode.Doc 404 case *ast.FuncDecl: 405 docGroup = typedNode.Doc 406 case *ast.GenDecl: 407 docGroup = typedNode.Doc 408 case *ast.ImportSpec: 409 docGroup = typedNode.Doc 410 lastLineCommentGroup = typedNode.Comment 411 case *ast.TypeSpec: 412 docGroup = typedNode.Doc 413 lastLineCommentGroup = typedNode.Comment 414 case *ast.ValueSpec: 415 docGroup = typedNode.Doc 416 lastLineCommentGroup = typedNode.Comment 417 default: 418 lastLineCommentGroup = nil 419 } 420 421 return docGroup, lastLineCommentGroup 422 }