github.com/jd-ly/tools@v0.5.7/internal/lsp/cmd/semantictokens.go (about)

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package cmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"flag"
    11  	"fmt"
    12  	"go/parser"
    13  	"go/token"
    14  	"io/ioutil"
    15  	"log"
    16  	"os"
    17  	"runtime"
    18  	"unicode/utf8"
    19  
    20  	"github.com/jd-ly/tools/internal/lsp"
    21  	"github.com/jd-ly/tools/internal/lsp/protocol"
    22  	"github.com/jd-ly/tools/internal/lsp/source"
    23  	"github.com/jd-ly/tools/internal/span"
    24  )
    25  
    26  // generate semantic tokens and interpolate them in the file
    27  
    28  // The output is the input file decorated with comments showing the
    29  // syntactic tokens. The comments are stylized:
    30  //   /*<arrow><length>,<token type>,[<modifiers]*/
    31  // For most occurrences, the comment comes just before the token it
    32  // describes, and arrow is a right arrow. If the token is inside a string
    33  // the comment comes just after the string, and the arrow is a left arrow.
    34  // <length> is the length of the token in runes, <token type> is one
    35  // of the supported semantic token types, and <modifiers. is a
    36  // (possibly empty) list of token type modifiers.
    37  
    38  // There are 3 coordinate systems for lines and character offsets in lines
    39  // LSP (what's returned from semanticTokens()):
    40  //    0-based: the first line is line 0, the first character of a line
    41  //      is character 0, and characters are counted as UTF-16 code points
    42  // gopls (and Go error messages):
    43  //    1-based: the first line is line1, the first chararcter of a line
    44  //      is character 0, and characters are counted as bytes
    45  // internal (as used in marks, and lines:=bytes.Split(buf, '\n'))
    46  //    0-based: lines and character positions are 1 less than in
    47  //      the gopls coordinate system
    48  
    49  type semtok struct {
    50  	app *Application
    51  }
    52  
    53  var colmap *protocol.ColumnMapper
    54  
    55  func (c *semtok) Name() string      { return "semtok" }
    56  func (c *semtok) Usage() string     { return "<filename>" }
    57  func (c *semtok) ShortHelp() string { return "show semantic tokens for the specified file" }
    58  func (c *semtok) DetailedHelp(f *flag.FlagSet) {
    59  	for i := 1; ; i++ {
    60  		_, f, l, ok := runtime.Caller(i)
    61  		if !ok {
    62  			break
    63  		}
    64  		log.Printf("%d: %s:%d", i, f, l)
    65  	}
    66  	fmt.Fprint(f.Output(), `
    67  Example: show the semantic tokens for this file:
    68  
    69    $ gopls semtok internal/lsp/cmd/semtok.go
    70  
    71  	gopls semtok flags are:
    72  `)
    73  	f.PrintDefaults()
    74  }
    75  
    76  // Run performs the semtok on the files specified by args and prints the
    77  // results to stdout in the format described above.
    78  func (c *semtok) Run(ctx context.Context, args ...string) error {
    79  	if len(args) != 1 {
    80  		return fmt.Errorf("expected one file name, got %d", len(args))
    81  	}
    82  	// perhaps simpler if app had just had a FlagSet member
    83  	origOptions := c.app.options
    84  	c.app.options = func(opts *source.Options) {
    85  		origOptions(opts)
    86  		opts.SemanticTokens = true
    87  	}
    88  	conn, err := c.app.connect(ctx)
    89  	if err != nil {
    90  		return err
    91  	}
    92  	defer conn.terminate(ctx)
    93  	uri := span.URIFromPath(args[0])
    94  	file := conn.AddFile(ctx, uri)
    95  	if file.err != nil {
    96  		return file.err
    97  	}
    98  
    99  	resp, err := conn.semanticTokens(ctx, uri)
   100  	if err != nil {
   101  		return err
   102  	}
   103  	buf, err := ioutil.ReadFile(args[0])
   104  	if err != nil {
   105  		log.Fatal(err)
   106  	}
   107  	fset := token.NewFileSet()
   108  	f, err := parser.ParseFile(fset, args[0], buf, 0)
   109  	if err != nil {
   110  		log.Printf("parsing %s failed %v", args[0], err)
   111  		return err
   112  	}
   113  	tok := fset.File(f.Pos())
   114  	if tok == nil {
   115  		// can't happen; just parsed this file
   116  		return fmt.Errorf("can't find %s in fset", args[0])
   117  	}
   118  	tc := span.NewContentConverter(args[0], buf)
   119  	colmap = &protocol.ColumnMapper{
   120  		URI:       span.URI(args[0]),
   121  		Content:   buf,
   122  		Converter: tc,
   123  	}
   124  	err = decorate(file.uri.Filename(), resp.Data)
   125  	if err != nil {
   126  		return err
   127  	}
   128  	return nil
   129  }
   130  
   131  type mark struct {
   132  	line, offset int // 1-based, from RangeSpan
   133  	len          int // bytes, not runes
   134  	typ          string
   135  	mods         []string
   136  }
   137  
   138  // prefixes for semantic token comments
   139  const (
   140  	SemanticLeft  = "/*⇐"
   141  	SemanticRight = "/*⇒"
   142  )
   143  
   144  func markLine(m mark, lines [][]byte) {
   145  	l := lines[m.line-1] // mx is 1-based
   146  	length := utf8.RuneCount(l[m.offset-1 : m.offset-1+m.len])
   147  	splitAt := m.offset - 1
   148  	insert := ""
   149  	if m.typ == "namespace" && m.offset-1+m.len < len(l) && l[m.offset-1+m.len] == '"' {
   150  		// it is the last component of an import spec
   151  		// cannot put a comment inside a string
   152  		insert = fmt.Sprintf("%s%d,namespace,[]*/", SemanticLeft, length)
   153  		splitAt = m.offset + m.len
   154  	} else {
   155  		// be careful not to generate //*
   156  		spacer := ""
   157  		if splitAt-1 >= 0 && l[splitAt-1] == '/' {
   158  			spacer = " "
   159  		}
   160  		insert = fmt.Sprintf("%s%s%d,%s,%v*/", spacer, SemanticRight, length, m.typ, m.mods)
   161  	}
   162  	x := append([]byte(insert), l[splitAt:]...)
   163  	l = append(l[:splitAt], x...)
   164  	lines[m.line-1] = l
   165  }
   166  
   167  func decorate(file string, result []float64) error {
   168  	buf, err := ioutil.ReadFile(file)
   169  	if err != nil {
   170  		return err
   171  	}
   172  	marks := newMarks(result)
   173  	if len(marks) == 0 {
   174  		return nil
   175  	}
   176  	lines := bytes.Split(buf, []byte{'\n'})
   177  	for i := len(marks) - 1; i >= 0; i-- {
   178  		mx := marks[i]
   179  		markLine(mx, lines)
   180  	}
   181  	os.Stdout.Write(bytes.Join(lines, []byte{'\n'}))
   182  	return nil
   183  }
   184  
   185  func newMarks(d []float64) []mark {
   186  	ans := []mark{}
   187  	// the following two loops could be merged, at the cost
   188  	// of making the logic slightly more complicated to understand
   189  	// first, convert from deltas to absolute, in LSP coordinates
   190  	lspLine := make([]float64, len(d)/5)
   191  	lspChar := make([]float64, len(d)/5)
   192  	line, char := 0.0, 0.0
   193  	for i := 0; 5*i < len(d); i++ {
   194  		lspLine[i] = line + d[5*i+0]
   195  		if d[5*i+0] > 0 {
   196  			char = 0
   197  		}
   198  		lspChar[i] = char + d[5*i+1]
   199  		char = lspChar[i]
   200  		line = lspLine[i]
   201  	}
   202  	// second, convert to gopls coordinates
   203  	for i := 0; 5*i < len(d); i++ {
   204  		pr := protocol.Range{
   205  			Start: protocol.Position{
   206  				Line:      lspLine[i],
   207  				Character: lspChar[i],
   208  			},
   209  			End: protocol.Position{
   210  				Line:      lspLine[i],
   211  				Character: lspChar[i] + d[5*i+2],
   212  			},
   213  		}
   214  		spn, err := colmap.RangeSpan(pr)
   215  		if err != nil {
   216  			log.Fatal(err)
   217  		}
   218  		m := mark{
   219  			line:   spn.Start().Line(),
   220  			offset: spn.Start().Column(),
   221  			len:    spn.End().Column() - spn.Start().Column(),
   222  			typ:    lsp.SemType(int(d[5*i+3])),
   223  			mods:   lsp.SemMods(int(d[5*i+4])),
   224  		}
   225  		ans = append(ans, m)
   226  	}
   227  	return ans
   228  }