github.com/aretext/aretext@v1.3.0/syntax/parser/token.go (about)

     1  package parser
     2  
     3  // TokenRole represents the role a token plays in a document,
     4  // as interpreted in a particular syntax language.
     5  type TokenRole int
     6  
     7  const (
     8  	TokenRoleNone = TokenRole(iota)
     9  	TokenRoleOperator
    10  	TokenRoleKeyword
    11  	TokenRoleNumber
    12  	TokenRoleString
    13  	TokenRoleComment
    14  )
    15  
    16  const (
    17  	TokenRoleCustom1 = TokenRole((1 << 16) + iota)
    18  	TokenRoleCustom2
    19  	TokenRoleCustom3
    20  	TokenRoleCustom4
    21  	TokenRoleCustom5
    22  	TokenRoleCustom6
    23  	TokenRoleCustom7
    24  	TokenRoleCustom8
    25  	TokenRoleCustom9
    26  	TokenRoleCustom10
    27  	TokenRoleCustom11
    28  	TokenRoleCustom12
    29  	TokenRoleCustom13
    30  	TokenRoleCustom14
    31  	TokenRoleCustom15
    32  	TokenRoleCustom16
    33  )
    34  
    35  // Token represents a distinct element in a document.
    36  type Token struct {
    37  	Role     TokenRole
    38  	StartPos uint64
    39  	EndPos   uint64
    40  }