github.com/jhump/golang-x-tools@v0.0.0-20220218190644-4958d6d39439/internal/lsp/template/parse.go (about) 1 // Copyright 2021 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package template contains code for dealing with templates 6 package template 7 8 // template files are small enough that the code reprocesses them each time 9 // this may be a bad choice for projects with lots of template files. 10 11 // This file contains the parsing code, some debugging printing, and 12 // implementations for Diagnose, Definition, HJover, References 13 14 import ( 15 "bytes" 16 "context" 17 "fmt" 18 "io" 19 "log" 20 "regexp" 21 "runtime" 22 "sort" 23 "text/template" 24 "text/template/parse" 25 "unicode/utf8" 26 27 "github.com/jhump/golang-x-tools/internal/event" 28 "github.com/jhump/golang-x-tools/internal/lsp/protocol" 29 "github.com/jhump/golang-x-tools/internal/lsp/source" 30 "github.com/jhump/golang-x-tools/internal/span" 31 ) 32 33 var ( 34 Left = []byte("{{") 35 Right = []byte("}}") 36 ) 37 38 type Parsed struct { 39 buf []byte //contents 40 lines [][]byte // needed?, other than for debugging? 41 elided []int // offsets where Left was replaced by blanks 42 43 // tokens are matched Left-Right pairs, computed before trying to parse 44 tokens []Token 45 46 // result of parsing 47 named []*template.Template // the template and embedded templates 48 ParseErr error 49 symbols []symbol 50 stack []parse.Node // used while computing symbols 51 52 // for mapping from offsets in buf to LSP coordinates 53 // See FromPosition() and LineCol() 54 nls []int // offset of newlines before each line (nls[0]==-1) 55 lastnl int // last line seen 56 check int // used to decide whether to use lastnl or search through nls 57 nonASCII bool // are there any non-ascii runes in buf? 58 } 59 60 // Token is a single {{...}}. More precisely, Left...Right 61 type Token struct { 62 Start, End int // offset from start of template 63 Multiline bool 64 } 65 66 // All contains the Parse of all the template files 67 type All struct { 68 files map[span.URI]*Parsed 69 } 70 71 // New returns the Parses of the snapshot's tmpl files 72 // (maybe cache these, but then avoiding import cycles needs code rearrangements) 73 func New(tmpls map[span.URI]source.VersionedFileHandle) *All { 74 all := make(map[span.URI]*Parsed) 75 for k, v := range tmpls { 76 buf, err := v.Read() 77 if err != nil { // PJW: decide what to do with these errors 78 log.Printf("failed to read %s (%v)", v.URI().Filename(), err) 79 continue 80 } 81 all[k] = parseBuffer(buf) 82 } 83 return &All{files: all} 84 } 85 86 func parseBuffer(buf []byte) *Parsed { 87 ans := &Parsed{ 88 buf: buf, 89 check: -1, 90 nls: []int{-1}, 91 } 92 if len(buf) == 0 { 93 return ans 94 } 95 // how to compute allAscii... 96 for _, b := range buf { 97 if b >= utf8.RuneSelf { 98 ans.nonASCII = true 99 break 100 } 101 } 102 if buf[len(buf)-1] != '\n' { 103 ans.buf = append(buf, '\n') 104 } 105 for i, p := range ans.buf { 106 if p == '\n' { 107 ans.nls = append(ans.nls, i) 108 } 109 } 110 ans.setTokens() // ans.buf may be a new []byte 111 ans.lines = bytes.Split(ans.buf, []byte{'\n'}) 112 t, err := template.New("").Parse(string(ans.buf)) 113 if err != nil { 114 funcs := make(template.FuncMap) 115 for t == nil && ans.ParseErr == nil { 116 // in 1.17 it may be possible to avoid getting this error 117 // template: :2: function "foo" not defined 118 matches := parseErrR.FindStringSubmatch(err.Error()) 119 if len(matches) == 2 { 120 // suppress the error by giving it a function with the right name 121 funcs[matches[1]] = func() interface{} { return nil } 122 t, err = template.New("").Funcs(funcs).Parse(string(ans.buf)) 123 continue 124 } 125 ans.ParseErr = err // unfixed error 126 return ans 127 } 128 } 129 ans.named = t.Templates() 130 // set the symbols 131 for _, t := range ans.named { 132 ans.stack = append(ans.stack, t.Root) 133 ans.findSymbols() 134 if t.Name() != "" { 135 // defining a template. The pos is just after {{define...}} (or {{block...}}?) 136 at, sz := ans.FindLiteralBefore(int(t.Root.Pos)) 137 s := symbol{start: at, length: sz, name: t.Name(), kind: protocol.Namespace, vardef: true} 138 ans.symbols = append(ans.symbols, s) 139 } 140 } 141 142 sort.Slice(ans.symbols, func(i, j int) bool { 143 left, right := ans.symbols[i], ans.symbols[j] 144 if left.start != right.start { 145 return left.start < right.start 146 } 147 if left.vardef != right.vardef { 148 return left.vardef 149 } 150 return left.kind < right.kind 151 }) 152 return ans 153 } 154 155 // FindLiteralBefore locates the first preceding string literal 156 // returning its position and length in buf 157 // or returns -1 if there is none. Assume "", rather than ``, for now 158 func (p *Parsed) FindLiteralBefore(pos int) (int, int) { 159 left, right := -1, -1 160 for i := pos - 1; i >= 0; i-- { 161 if p.buf[i] != '"' { 162 continue 163 } 164 if right == -1 { 165 right = i 166 continue 167 } 168 left = i 169 break 170 } 171 if left == -1 { 172 return -1, 0 173 } 174 return left + 1, right - left - 1 175 } 176 177 var ( 178 parseErrR = regexp.MustCompile(`template:.*function "([^"]+)" not defined`) 179 ) 180 181 func (p *Parsed) setTokens() { 182 const ( 183 // InRaw and InString only occur inside an action (SeenLeft) 184 Start = iota 185 InRaw 186 InString 187 SeenLeft 188 ) 189 state := Start 190 var left, oldState int 191 for n := 0; n < len(p.buf); n++ { 192 c := p.buf[n] 193 switch state { 194 case InRaw: 195 if c == '`' { 196 state = oldState 197 } 198 case InString: 199 if c == '"' && !isEscaped(p.buf[:n]) { 200 state = oldState 201 } 202 case SeenLeft: 203 if c == '`' { 204 oldState = state // it's SeenLeft, but a little clearer this way 205 state = InRaw 206 continue 207 } 208 if c == '"' { 209 oldState = state 210 state = InString 211 continue 212 } 213 if bytes.HasPrefix(p.buf[n:], Right) { 214 right := n + len(Right) 215 tok := Token{Start: left, 216 End: right, 217 Multiline: bytes.Contains(p.buf[left:right], []byte{'\n'}), 218 } 219 p.tokens = append(p.tokens, tok) 220 state = Start 221 } 222 // If we see (unquoted) Left then the original left is probably the user 223 // typing. Suppress the original left 224 if bytes.HasPrefix(p.buf[n:], Left) { 225 p.elideAt(left) 226 left = n 227 n += len(Left) - 1 // skip the rest 228 } 229 case Start: 230 if bytes.HasPrefix(p.buf[n:], Left) { 231 left = n 232 state = SeenLeft 233 n += len(Left) - 1 // skip the rest (avoids {{{ bug) 234 } 235 } 236 } 237 // this error occurs after typing {{ at the end of the file 238 if state != Start { 239 // Unclosed Left. remove the Left at left 240 p.elideAt(left) 241 } 242 } 243 244 func (p *Parsed) elideAt(left int) { 245 if p.elided == nil { 246 // p.buf is the same buffer that v.Read() returns, so copy it. 247 // (otherwise the next time it's parsed, elided information is lost) 248 b := make([]byte, len(p.buf)) 249 copy(b, p.buf) 250 p.buf = b 251 } 252 for i := 0; i < len(Left); i++ { 253 p.buf[left+i] = ' ' 254 } 255 p.elided = append(p.elided, left) 256 } 257 258 // isEscaped reports whether the byte after buf is escaped 259 func isEscaped(buf []byte) bool { 260 backSlashes := 0 261 for j := len(buf) - 1; j >= 0 && buf[j] == '\\'; j-- { 262 backSlashes++ 263 } 264 return backSlashes%2 == 1 265 } 266 267 func (p *Parsed) Tokens() []Token { 268 return p.tokens 269 } 270 271 func (p *Parsed) utf16len(buf []byte) int { 272 cnt := 0 273 if !p.nonASCII { 274 return len(buf) 275 } 276 // we need a utf16len(rune), but we don't have it 277 for _, r := range string(buf) { 278 cnt++ 279 if r >= 1<<16 { 280 cnt++ 281 } 282 } 283 return cnt 284 } 285 286 func (p *Parsed) TokenSize(t Token) (int, error) { 287 if t.Multiline { 288 return -1, fmt.Errorf("TokenSize called with Multiline token %#v", t) 289 } 290 ans := p.utf16len(p.buf[t.Start:t.End]) 291 return ans, nil 292 } 293 294 // RuneCount counts runes in a line 295 func (p *Parsed) RuneCount(l, s, e uint32) uint32 { 296 start := p.nls[l] + 1 + int(s) 297 end := int(e) 298 if e == 0 || int(e) >= p.nls[l+1] { 299 end = p.nls[l+1] 300 } 301 return uint32(utf8.RuneCount(p.buf[start:end])) 302 } 303 304 // LineCol converts from a 0-based byte offset to 0-based line, col. col in runes 305 func (p *Parsed) LineCol(x int) (uint32, uint32) { 306 if x < p.check { 307 p.lastnl = 0 308 } 309 p.check = x 310 for i := p.lastnl; i < len(p.nls); i++ { 311 if p.nls[i] <= x { 312 continue 313 } 314 p.lastnl = i 315 var count int 316 if i > 0 && x == p.nls[i-1] { // \n 317 count = 0 318 } else { 319 count = p.utf16len(p.buf[p.nls[i-1]+1 : x]) 320 } 321 return uint32(i - 1), uint32(count) 322 } 323 if x == len(p.buf)-1 { // trailing \n 324 return uint32(len(p.nls) - 1), 0 325 } 326 // shouldn't happen 327 for i := 1; i < 4; i++ { 328 _, f, l, ok := runtime.Caller(i) 329 if !ok { 330 break 331 } 332 log.Printf("%d: %s:%d", i, f, l) 333 } 334 335 msg := fmt.Errorf("LineCol off the end, %d of %d, nls=%v, %q", x, len(p.buf), p.nls, p.buf[x:]) 336 event.Error(context.Background(), "internal error", msg) 337 return 0, 0 338 } 339 340 // Position produces a protocol.Position from an offset in the template 341 func (p *Parsed) Position(pos int) protocol.Position { 342 line, col := p.LineCol(pos) 343 return protocol.Position{Line: line, Character: col} 344 } 345 346 func (p *Parsed) Range(x, length int) protocol.Range { 347 line, col := p.LineCol(x) 348 ans := protocol.Range{ 349 Start: protocol.Position{Line: line, Character: col}, 350 End: protocol.Position{Line: line, Character: col + uint32(length)}, 351 } 352 return ans 353 } 354 355 // FromPosition translates a protocol.Position into an offset into the template 356 func (p *Parsed) FromPosition(x protocol.Position) int { 357 l, c := int(x.Line), int(x.Character) 358 if l >= len(p.nls) || p.nls[l]+1 >= len(p.buf) { 359 // paranoia to avoid panic. return the largest offset 360 return len(p.buf) 361 } 362 line := p.buf[p.nls[l]+1:] 363 cnt := 0 364 for w := range string(line) { 365 if cnt >= c { 366 return w + p.nls[l] + 1 367 } 368 cnt++ 369 } 370 // do we get here? NO 371 pos := int(x.Character) + p.nls[int(x.Line)] + 1 372 event.Error(context.Background(), "internal error", fmt.Errorf("surprise %#v", x)) 373 return pos 374 } 375 376 func symAtPosition(fh source.FileHandle, loc protocol.Position) (*symbol, *Parsed, error) { 377 buf, err := fh.Read() 378 if err != nil { 379 return nil, nil, err 380 } 381 p := parseBuffer(buf) 382 pos := p.FromPosition(loc) 383 syms := p.SymsAtPos(pos) 384 if len(syms) == 0 { 385 return nil, p, fmt.Errorf("no symbol found") 386 } 387 if len(syms) > 1 { 388 log.Printf("Hover: %d syms, not 1 %v", len(syms), syms) 389 } 390 sym := syms[0] 391 return &sym, p, nil 392 } 393 394 func (p *Parsed) SymsAtPos(pos int) []symbol { 395 ans := []symbol{} 396 for _, s := range p.symbols { 397 if s.start <= pos && pos < s.start+s.length { 398 ans = append(ans, s) 399 } 400 } 401 return ans 402 } 403 404 type wrNode struct { 405 p *Parsed 406 w io.Writer 407 } 408 409 // WriteNode is for debugging 410 func (p *Parsed) WriteNode(w io.Writer, n parse.Node) { 411 wr := wrNode{p: p, w: w} 412 wr.writeNode(n, "") 413 } 414 415 func (wr wrNode) writeNode(n parse.Node, indent string) { 416 if n == nil { 417 return 418 } 419 at := func(pos parse.Pos) string { 420 line, col := wr.p.LineCol(int(pos)) 421 return fmt.Sprintf("(%d)%v:%v", pos, line, col) 422 } 423 switch x := n.(type) { 424 case *parse.ActionNode: 425 fmt.Fprintf(wr.w, "%sActionNode at %s\n", indent, at(x.Pos)) 426 wr.writeNode(x.Pipe, indent+". ") 427 case *parse.BoolNode: 428 fmt.Fprintf(wr.w, "%sBoolNode at %s, %v\n", indent, at(x.Pos), x.True) 429 case *parse.BranchNode: 430 fmt.Fprintf(wr.w, "%sBranchNode at %s\n", indent, at(x.Pos)) 431 wr.writeNode(x.Pipe, indent+"Pipe. ") 432 wr.writeNode(x.List, indent+"List. ") 433 wr.writeNode(x.ElseList, indent+"Else. ") 434 case *parse.ChainNode: 435 fmt.Fprintf(wr.w, "%sChainNode at %s, %v\n", indent, at(x.Pos), x.Field) 436 case *parse.CommandNode: 437 fmt.Fprintf(wr.w, "%sCommandNode at %s, %d children\n", indent, at(x.Pos), len(x.Args)) 438 for _, a := range x.Args { 439 wr.writeNode(a, indent+". ") 440 } 441 //case *parse.CommentNode: // 1.16 442 case *parse.DotNode: 443 fmt.Fprintf(wr.w, "%sDotNode at %s\n", indent, at(x.Pos)) 444 case *parse.FieldNode: 445 fmt.Fprintf(wr.w, "%sFieldNode at %s, %v\n", indent, at(x.Pos), x.Ident) 446 case *parse.IdentifierNode: 447 fmt.Fprintf(wr.w, "%sIdentifierNode at %s, %v\n", indent, at(x.Pos), x.Ident) 448 case *parse.IfNode: 449 fmt.Fprintf(wr.w, "%sIfNode at %s\n", indent, at(x.Pos)) 450 wr.writeNode(&x.BranchNode, indent+". ") 451 case *parse.ListNode: 452 if x == nil { 453 return // nil BranchNode.ElseList 454 } 455 fmt.Fprintf(wr.w, "%sListNode at %s, %d children\n", indent, at(x.Pos), len(x.Nodes)) 456 for _, n := range x.Nodes { 457 wr.writeNode(n, indent+". ") 458 } 459 case *parse.NilNode: 460 fmt.Fprintf(wr.w, "%sNilNode at %s\n", indent, at(x.Pos)) 461 case *parse.NumberNode: 462 fmt.Fprintf(wr.w, "%sNumberNode at %s, %s\n", indent, at(x.Pos), x.Text) 463 case *parse.PipeNode: 464 if x == nil { 465 return // {{template "xxx"}} 466 } 467 fmt.Fprintf(wr.w, "%sPipeNode at %s, %d vars, %d cmds, IsAssign:%v\n", 468 indent, at(x.Pos), len(x.Decl), len(x.Cmds), x.IsAssign) 469 for _, d := range x.Decl { 470 wr.writeNode(d, indent+"Decl. ") 471 } 472 for _, c := range x.Cmds { 473 wr.writeNode(c, indent+"Cmd. ") 474 } 475 case *parse.RangeNode: 476 fmt.Fprintf(wr.w, "%sRangeNode at %s\n", indent, at(x.Pos)) 477 wr.writeNode(&x.BranchNode, indent+". ") 478 case *parse.StringNode: 479 fmt.Fprintf(wr.w, "%sStringNode at %s, %s\n", indent, at(x.Pos), x.Quoted) 480 case *parse.TemplateNode: 481 fmt.Fprintf(wr.w, "%sTemplateNode at %s, %s\n", indent, at(x.Pos), x.Name) 482 wr.writeNode(x.Pipe, indent+". ") 483 case *parse.TextNode: 484 fmt.Fprintf(wr.w, "%sTextNode at %s, len %d\n", indent, at(x.Pos), len(x.Text)) 485 case *parse.VariableNode: 486 fmt.Fprintf(wr.w, "%sVariableNode at %s, %v\n", indent, at(x.Pos), x.Ident) 487 case *parse.WithNode: 488 fmt.Fprintf(wr.w, "%sWithNode at %s\n", indent, at(x.Pos)) 489 wr.writeNode(&x.BranchNode, indent+". ") 490 } 491 } 492 493 // short prints at most 40 bytes of node.String(), for debugging 494 func short(n parse.Node) (ret string) { 495 defer func() { 496 if x := recover(); x != nil { 497 // all because of typed nils 498 ret = "NIL" 499 } 500 }() 501 s := n.String() 502 if len(s) > 40 { 503 return s[:40] + "..." 504 } 505 return s 506 } 507 508 var kindNames = []string{"", "File", "Module", "Namespace", "Package", "Class", "Method", "Property", 509 "Field", "Constructor", "Enum", "Interface", "Function", "Variable", "Constant", "String", 510 "Number", "Boolean", "Array", "Object", "Key", "Null", "EnumMember", "Struct", "Event", 511 "Operator", "TypeParameter"} 512 513 func kindStr(k protocol.SymbolKind) string { 514 n := int(k) 515 if n < 1 || n >= len(kindNames) { 516 return fmt.Sprintf("?SymbolKind %d?", n) 517 } 518 return kindNames[n] 519 }