[clang-format][tests] Fix MacroExpander lexer not parsing C++ keywords

While debugging a different clang-format failure, I tried to reuse the
MacroExpander lexer, but was surprised to see that it marks all C++
keywords (e.g. const, decltype) as being of type identifier. After stepping
through the ::format() code, I noticed that the difference between these
two is that the identifier table was not being initialized based on the
FormatStyle, so only basic tokens such as tok::semi, tok::plus, etc. were
being handled.

Reviewed By: klimek

Differential Revision: https://reviews.llvm.org/D88952
This commit is contained in:
Alex Richardson 2020-10-07 16:29:22 +01:00
parent 0a3c82e85b
commit ff6e4441b9
2 changed files with 20 additions and 2 deletions

View File

@ -182,6 +182,22 @@ TEST_F(MacroExpanderTest, SingleExpansion) {
EXPECT_ATTRIBUTES(Result, Attributes);
}
TEST_F(MacroExpanderTest, UnderstandsCppTokens) {
auto Macros = create({"A(T,name)=T name = 0;"});
auto *A = Lex.id("A");
auto Args = lexArgs({"const int", "x"});
auto Result = uneof(Macros->expand(A, Args));
std::vector<MacroAttributes> Attributes = {
{tok::kw_const, MR_ExpandedArg, 1, 0, {A}},
{tok::kw_int, MR_ExpandedArg, 0, 0, {A}},
{tok::identifier, MR_ExpandedArg, 0, 0, {A}},
{tok::equal, MR_Hidden, 0, 0, {A}},
{tok::numeric_constant, MR_Hidden, 0, 0, {A}},
{tok::semi, MR_Hidden, 0, 1, {A}},
};
EXPECT_ATTRIBUTES(Result, Attributes);
}
} // namespace
} // namespace format
} // namespace clang

View File

@ -55,7 +55,9 @@ inline std::string text(llvm::ArrayRef<FormatToken *> Tokens) {
class TestLexer {
public:
TestLexer() : SourceMgr("test.cpp", "") {}
TestLexer(FormatStyle Style = getLLVMStyle())
: Style(Style), SourceMgr("test.cpp", ""),
IdentTable(getFormattingLangOpts(Style)) {}
TokenList lex(llvm::StringRef Code) {
Buffers.push_back(
@ -74,7 +76,7 @@ public:
return Result[0];
}
FormatStyle Style = getLLVMStyle();
FormatStyle Style;
encoding::Encoding Encoding = encoding::Encoding_UTF8;
std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
clang::SourceManagerForFile SourceMgr;