From 86f1a935dc9efe7d7a7856bc342a7881ae337e67 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 7 Jan 2013 19:16:18 +0000 Subject: [PATCH] Pull the bulk of Lexer::MeasureTokenLength() out into a new function, Lexer::getRawToken(). No functionality change. llvm-svn: 171771 --- clang/include/clang/Lex/Lexer.h | 6 ++++++ clang/include/clang/Lex/Preprocessor.h | 6 ++++++ clang/lib/Lex/Lexer.cpp | 20 +++++++++++++++----- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index b57be3a0ebb2..d36189fccd41 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -273,6 +273,12 @@ public: const SourceManager &SM, const LangOptions &LangOpts); + /// \brief Relex the token at the specified location. + /// \returns true if there was a failure, false on success. + static bool getRawToken(SourceLocation Loc, Token &Result, + const SourceManager &SM, + const LangOptions &LangOpts); + /// \brief Given a location any where in a source buffer, find the location /// that corresponds to the beginning of the token in which the original /// source location lands. diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index fb915bb64dea..9d7db5d70930 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -958,6 +958,12 @@ public: SmallVectorImpl &Buffer, bool *Invalid = 0) const; + /// \brief Relex the token at the specified location. + /// \returns true if there was a failure, false on success. + bool getRawToken(SourceLocation Loc, Token &Result) { + return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts); + } + /// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant /// with length 1, return the character. char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 1bcff98cf48d..15b1061d0571 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -414,6 +414,17 @@ static bool isWhitespace(unsigned char c); unsigned Lexer::MeasureTokenLength(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { + Token TheTok; + if (getRawToken(Loc, TheTok, SM, LangOpts)) + return 0; + return TheTok.getLength(); +} + +/// \brief Relex the token at the specified location. +/// \returns true if there was a failure, false on success. +bool Lexer::getRawToken(SourceLocation Loc, Token &Result, + const SourceManager &SM, + const LangOptions &LangOpts) { // TODO: this could be special cased for common tokens like identifiers, ')', // etc to make this faster, if it mattered. Just look at StrData[0] to handle // all obviously single-char tokens. This could use @@ -427,20 +438,19 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc, bool Invalid = false; StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); if (Invalid) - return 0; + return true; const char *StrData = Buffer.data()+LocInfo.second; if (isWhitespace(StrData[0])) - return 0; + return true; // Create a lexer starting at the beginning of this token. Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, Buffer.begin(), StrData, Buffer.end()); TheLexer.SetCommentRetentionState(true); - Token TheTok; - TheLexer.LexFromRawLexer(TheTok); - return TheTok.getLength(); + TheLexer.LexFromRawLexer(Result); + return false; } static SourceLocation getBeginningOfFileToken(SourceLocation Loc,