github.com/johnnyeven/libtools@v0.0.0-20191126065708-61829c1adf46/third_party/mlir/lib/Parser/Token.cpp (about)

     1  //===- Token.cpp - MLIR Token Implementation ------------------------------===//
     2  //
     3  // Copyright 2019 The MLIR Authors.
     4  //
     5  // Licensed under the Apache License, Version 2.0 (the "License");
     6  // you may not use this file except in compliance with the License.
     7  // You may obtain a copy of the License at
     8  //
     9  //   http://www.apache.org/licenses/LICENSE-2.0
    10  //
    11  // Unless required by applicable law or agreed to in writing, software
    12  // distributed under the License is distributed on an "AS IS" BASIS,
    13  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  // See the License for the specific language governing permissions and
    15  // limitations under the License.
    16  // =============================================================================
    17  //
    18  // This file implements the Token class for the MLIR textual form.
    19  //
    20  //===----------------------------------------------------------------------===//
    21  
    22  #include "Token.h"
    23  #include "llvm/ADT/StringExtras.h"
    24  using namespace mlir;
    25  using llvm::SMLoc;
    26  using llvm::SMRange;
    27  
    28  SMLoc Token::getLoc() const { return SMLoc::getFromPointer(spelling.data()); }
    29  
    30  SMLoc Token::getEndLoc() const {
    31    return SMLoc::getFromPointer(spelling.data() + spelling.size());
    32  }
    33  
    34  SMRange Token::getLocRange() const { return SMRange(getLoc(), getEndLoc()); }
    35  
    36  /// For an integer token, return its value as an unsigned.  If it doesn't fit,
    37  /// return None.
    38  Optional<unsigned> Token::getUnsignedIntegerValue() const {
    39    bool isHex = spelling.size() > 1 && spelling[1] == 'x';
    40  
    41    unsigned result = 0;
    42    if (spelling.getAsInteger(isHex ? 0 : 10, result))
    43      return None;
    44    return result;
    45  }
    46  
    47  /// For an integer token, return its value as a uint64_t.  If it doesn't fit,
    48  /// return None.
    49  Optional<uint64_t> Token::getUInt64IntegerValue() const {
    50    bool isHex = spelling.size() > 1 && spelling[1] == 'x';
    51  
    52    uint64_t result = 0;
    53    if (spelling.getAsInteger(isHex ? 0 : 10, result))
    54      return None;
    55    return result;
    56  }
    57  
    58  /// For a floatliteral, return its value as a double. Return None if the value
    59  /// underflows or overflows.
    60  Optional<double> Token::getFloatingPointValue() const {
    61    double result = 0;
    62    if (spelling.getAsDouble(result))
    63      return None;
    64    return result;
    65  }
    66  
    67  /// For an inttype token, return its bitwidth.
    68  Optional<unsigned> Token::getIntTypeBitwidth() const {
    69    unsigned result = 0;
    70    if (spelling[1] == '0' || spelling.drop_front().getAsInteger(10, result) ||
    71        result == 0)
    72      return None;
    73    return result;
    74  }
    75  
    76  /// Given a 'string' token, return its value, including removing the quote
    77  /// characters and unescaping the contents of the string.  The lexer has already
    78  /// verified that this token is valid.
    79  std::string Token::getStringValue() const {
    80    assert(getKind() == string);
    81    // Start by dropping the quotes.
    82    StringRef bytes = getSpelling().drop_front().drop_back();
    83  
    84    std::string result;
    85    result.reserve(bytes.size());
    86    for (unsigned i = 0, e = bytes.size(); i != e;) {
    87      auto c = bytes[i++];
    88      if (c != '\\') {
    89        result.push_back(c);
    90        continue;
    91      }
    92  
    93      assert(i + 1 < e && "invalid string should be caught by lexer");
    94      auto c1 = bytes[i++];
    95      switch (c1) {
    96      case '"':
    97      case '\\':
    98        result.push_back(c1);
    99        continue;
   100      case 'n':
   101        result.push_back('\n');
   102        continue;
   103      case 't':
   104        result.push_back('\t');
   105        continue;
   106      default:
   107        break;
   108      }
   109  
   110      assert(i + 1 <= e && "invalid string should be caught by lexer");
   111      auto c2 = bytes[i++];
   112  
   113      assert(llvm::isHexDigit(c1) && llvm::isHexDigit(c2) && "invalid escape");
   114      result.push_back((llvm::hexDigitValue(c1) << 4) | llvm::hexDigitValue(c2));
   115    }
   116  
   117    return result;
   118  }
   119  
   120  /// Given a hash_identifier token like #123, try to parse the number out of
   121  /// the identifier, returning None if it is a named identifier like #x or
   122  /// if the integer doesn't fit.
   123  Optional<unsigned> Token::getHashIdentifierNumber() const {
   124    assert(getKind() == hash_identifier);
   125    unsigned result = 0;
   126    if (spelling.drop_front().getAsInteger(10, result))
   127      return None;
   128    return result;
   129  }
   130  
   131  /// Given a punctuation or keyword token kind, return the spelling of the
   132  /// token as a string.  Warning: This will abort on markers, identifiers and
   133  /// literal tokens since they have no fixed spelling.
   134  StringRef Token::getTokenSpelling(Kind kind) {
   135    switch (kind) {
   136    default:
   137      llvm_unreachable("This token kind has no fixed spelling");
   138  #define TOK_PUNCTUATION(NAME, SPELLING)                                        \
   139    case NAME:                                                                   \
   140      return SPELLING;
   141  #define TOK_OPERATOR(NAME, SPELLING)                                           \
   142    case NAME:                                                                   \
   143      return SPELLING;
   144  #define TOK_KEYWORD(SPELLING)                                                  \
   145    case kw_##SPELLING:                                                          \
   146      return #SPELLING;
   147  #include "TokenKinds.def"
   148    }
   149  }
   150  
   151  /// Return true if this is one of the keyword token kinds (e.g. kw_if).
   152  bool Token::isKeyword() const {
   153    switch (kind) {
   154    default:
   155      return false;
   156  #define TOK_KEYWORD(SPELLING)                                                  \
   157    case kw_##SPELLING:                                                          \
   158      return true;
   159  #include "TokenKinds.def"
   160    }
   161  }