github.com/jd-ly/tools@v0.5.7/internal/lsp/cmd/semantictokens.go (about) 1 // Copyright 2020 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package cmd 6 7 import ( 8 "bytes" 9 "context" 10 "flag" 11 "fmt" 12 "go/parser" 13 "go/token" 14 "io/ioutil" 15 "log" 16 "os" 17 "runtime" 18 "unicode/utf8" 19 20 "github.com/jd-ly/tools/internal/lsp" 21 "github.com/jd-ly/tools/internal/lsp/protocol" 22 "github.com/jd-ly/tools/internal/lsp/source" 23 "github.com/jd-ly/tools/internal/span" 24 ) 25 26 // generate semantic tokens and interpolate them in the file 27 28 // The output is the input file decorated with comments showing the 29 // syntactic tokens. The comments are stylized: 30 // /*<arrow><length>,<token type>,[<modifiers]*/ 31 // For most occurrences, the comment comes just before the token it 32 // describes, and arrow is a right arrow. If the token is inside a string 33 // the comment comes just after the string, and the arrow is a left arrow. 34 // <length> is the length of the token in runes, <token type> is one 35 // of the supported semantic token types, and <modifiers. is a 36 // (possibly empty) list of token type modifiers. 37 38 // There are 3 coordinate systems for lines and character offsets in lines 39 // LSP (what's returned from semanticTokens()): 40 // 0-based: the first line is line 0, the first character of a line 41 // is character 0, and characters are counted as UTF-16 code points 42 // gopls (and Go error messages): 43 // 1-based: the first line is line1, the first chararcter of a line 44 // is character 0, and characters are counted as bytes 45 // internal (as used in marks, and lines:=bytes.Split(buf, '\n')) 46 // 0-based: lines and character positions are 1 less than in 47 // the gopls coordinate system 48 49 type semtok struct { 50 app *Application 51 } 52 53 var colmap *protocol.ColumnMapper 54 55 func (c *semtok) Name() string { return "semtok" } 56 func (c *semtok) Usage() string { return "<filename>" } 57 func (c *semtok) ShortHelp() string { return "show semantic tokens for the specified file" } 58 func (c *semtok) DetailedHelp(f *flag.FlagSet) { 59 for i := 1; ; i++ { 60 _, f, l, ok := runtime.Caller(i) 61 if !ok { 62 break 63 } 64 log.Printf("%d: %s:%d", i, f, l) 65 } 66 fmt.Fprint(f.Output(), ` 67 Example: show the semantic tokens for this file: 68 69 $ gopls semtok internal/lsp/cmd/semtok.go 70 71 gopls semtok flags are: 72 `) 73 f.PrintDefaults() 74 } 75 76 // Run performs the semtok on the files specified by args and prints the 77 // results to stdout in the format described above. 78 func (c *semtok) Run(ctx context.Context, args ...string) error { 79 if len(args) != 1 { 80 return fmt.Errorf("expected one file name, got %d", len(args)) 81 } 82 // perhaps simpler if app had just had a FlagSet member 83 origOptions := c.app.options 84 c.app.options = func(opts *source.Options) { 85 origOptions(opts) 86 opts.SemanticTokens = true 87 } 88 conn, err := c.app.connect(ctx) 89 if err != nil { 90 return err 91 } 92 defer conn.terminate(ctx) 93 uri := span.URIFromPath(args[0]) 94 file := conn.AddFile(ctx, uri) 95 if file.err != nil { 96 return file.err 97 } 98 99 resp, err := conn.semanticTokens(ctx, uri) 100 if err != nil { 101 return err 102 } 103 buf, err := ioutil.ReadFile(args[0]) 104 if err != nil { 105 log.Fatal(err) 106 } 107 fset := token.NewFileSet() 108 f, err := parser.ParseFile(fset, args[0], buf, 0) 109 if err != nil { 110 log.Printf("parsing %s failed %v", args[0], err) 111 return err 112 } 113 tok := fset.File(f.Pos()) 114 if tok == nil { 115 // can't happen; just parsed this file 116 return fmt.Errorf("can't find %s in fset", args[0]) 117 } 118 tc := span.NewContentConverter(args[0], buf) 119 colmap = &protocol.ColumnMapper{ 120 URI: span.URI(args[0]), 121 Content: buf, 122 Converter: tc, 123 } 124 err = decorate(file.uri.Filename(), resp.Data) 125 if err != nil { 126 return err 127 } 128 return nil 129 } 130 131 type mark struct { 132 line, offset int // 1-based, from RangeSpan 133 len int // bytes, not runes 134 typ string 135 mods []string 136 } 137 138 // prefixes for semantic token comments 139 const ( 140 SemanticLeft = "/*⇐" 141 SemanticRight = "/*⇒" 142 ) 143 144 func markLine(m mark, lines [][]byte) { 145 l := lines[m.line-1] // mx is 1-based 146 length := utf8.RuneCount(l[m.offset-1 : m.offset-1+m.len]) 147 splitAt := m.offset - 1 148 insert := "" 149 if m.typ == "namespace" && m.offset-1+m.len < len(l) && l[m.offset-1+m.len] == '"' { 150 // it is the last component of an import spec 151 // cannot put a comment inside a string 152 insert = fmt.Sprintf("%s%d,namespace,[]*/", SemanticLeft, length) 153 splitAt = m.offset + m.len 154 } else { 155 // be careful not to generate //* 156 spacer := "" 157 if splitAt-1 >= 0 && l[splitAt-1] == '/' { 158 spacer = " " 159 } 160 insert = fmt.Sprintf("%s%s%d,%s,%v*/", spacer, SemanticRight, length, m.typ, m.mods) 161 } 162 x := append([]byte(insert), l[splitAt:]...) 163 l = append(l[:splitAt], x...) 164 lines[m.line-1] = l 165 } 166 167 func decorate(file string, result []float64) error { 168 buf, err := ioutil.ReadFile(file) 169 if err != nil { 170 return err 171 } 172 marks := newMarks(result) 173 if len(marks) == 0 { 174 return nil 175 } 176 lines := bytes.Split(buf, []byte{'\n'}) 177 for i := len(marks) - 1; i >= 0; i-- { 178 mx := marks[i] 179 markLine(mx, lines) 180 } 181 os.Stdout.Write(bytes.Join(lines, []byte{'\n'})) 182 return nil 183 } 184 185 func newMarks(d []float64) []mark { 186 ans := []mark{} 187 // the following two loops could be merged, at the cost 188 // of making the logic slightly more complicated to understand 189 // first, convert from deltas to absolute, in LSP coordinates 190 lspLine := make([]float64, len(d)/5) 191 lspChar := make([]float64, len(d)/5) 192 line, char := 0.0, 0.0 193 for i := 0; 5*i < len(d); i++ { 194 lspLine[i] = line + d[5*i+0] 195 if d[5*i+0] > 0 { 196 char = 0 197 } 198 lspChar[i] = char + d[5*i+1] 199 char = lspChar[i] 200 line = lspLine[i] 201 } 202 // second, convert to gopls coordinates 203 for i := 0; 5*i < len(d); i++ { 204 pr := protocol.Range{ 205 Start: protocol.Position{ 206 Line: lspLine[i], 207 Character: lspChar[i], 208 }, 209 End: protocol.Position{ 210 Line: lspLine[i], 211 Character: lspChar[i] + d[5*i+2], 212 }, 213 } 214 spn, err := colmap.RangeSpan(pr) 215 if err != nil { 216 log.Fatal(err) 217 } 218 m := mark{ 219 line: spn.Start().Line(), 220 offset: spn.Start().Column(), 221 len: spn.End().Column() - spn.Start().Column(), 222 typ: lsp.SemType(int(d[5*i+3])), 223 mods: lsp.SemMods(int(d[5*i+4])), 224 } 225 ans = append(ans, m) 226 } 227 return ans 228 }