maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[clang-format] Further improve support for requires expressions 

Detect requires expressions in more unusable contexts. This is far from
perfect, but currently we have no good metric to decide between a
requires expression and a trailing requires clause.

Differential Revision: https://reviews.llvm.org/D119138
This commit is contained in:
Björn Schäpers 2022-02-07 15:46:46 +01:00
parent 2e50760775
commit bcd1e4612f
3 changed files with 320 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

214
clang/lib/Format/UnwrappedLineParser.cpp
View File

@ -40,6 +40,12 @@ public:
// getNextToken().
virtual FormatToken *peekNextToken() = 0;
// Returns the token that would be returned after the next N calls to
// getNextToken(). N needs to be greater than zero, and small enough that
// there are still tokens. Check for tok::eof with N-1 before calling it with
// N.
virtual FormatToken *peekNextToken(int N) = 0;
// Returns whether we are at the end of the file.
// This can be different from whether getNextToken() returned an eof token
// when the FormatTokenSource is a view on a part of the token stream.
@ -137,6 +143,13 @@ public:
return PreviousTokenSource->peekNextToken();
}
FormatToken *peekNextToken(int N) override {
assert(N > 0);
if (eof())
return &FakeEOF;
return PreviousTokenSource->peekNextToken(N);
}
bool isEOF() override { return PreviousTokenSource->isEOF(); }
unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
@ -257,6 +270,16 @@ public:
return Tokens[Next];
}
FormatToken *peekNextToken(int N) override {
assert(N > 0);
int Next = Position + N;
LLVM_DEBUG({
llvm::dbgs() << "Peeking (+" << (N - 1) << ") ";
dbgToken(Next);
});
return Tokens[Next];
}
bool isEOF() override { return Tokens[Position]->is(tok::eof); }
unsigned getPosition() override {
@ -1537,9 +1560,12 @@ void UnwrappedLineParser::parseStructuralElement(IfStmtKind *IfKind,
case tok::kw_concept:
parseConcept();
return;
case tok::kw_requires:
parseRequiresClause();
return;
case tok::kw_requires: {
bool ParsedClause = parseRequires();
if (ParsedClause)
return;
break;
}
case tok::kw_enum:
// Ignore if this is part of "template <enum ...".
if (Previous && Previous->is(tok::less)) {
@ -2206,9 +2232,12 @@ void UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
else
nextToken();
break;
case tok::kw_requires:
parseRequiresExpression();
case tok::kw_requires: {
auto RequiresToken = FormatTok;
nextToken();
parseRequiresExpression(RequiresToken);
break;
}
case tok::ampamp:
if (AmpAmpTokenType != TT_Unknown)
FormatTok->setType(AmpAmpTokenType);
@ -2783,28 +2812,163 @@ void UnwrappedLineParser::parseConcept() {
addUnwrappedLine();
}
/// \brief Parses a requires, decides if it is a clause or an expression.
/// \pre The current token has to be the requires keyword.
/// \returns true if it parsed a clause.
bool clang::format::UnwrappedLineParser::parseRequires() {
assert(FormatTok->Tok.is(tok::kw_requires) && "'requires' expected");
auto RequiresToken = FormatTok;
// We try to guess if it is a requires clause, or a requires expression. For
// that we first consume the keyword and check the next token.
nextToken();
switch (FormatTok->Tok.getKind()) {
case tok::l_brace:
// This can only be an expression, never a clause.
parseRequiresExpression(RequiresToken);
return false;
case tok::l_paren:
// Clauses and expression can start with a paren, it's unclear what we have.
break;
default:
// All other tokens can only be a clause.
parseRequiresClause(RequiresToken);
return true;
}
// Looking forward we would have to decide if there are function declaration
// like arguments to the requires expression:
// requires (T t) {
// Or there is a constraint expression for the requires clause:
// requires (C<T> && ...
// But first let's look behind.
auto *PreviousNonComment = RequiresToken->getPreviousNonComment();
if (!PreviousNonComment ||
PreviousNonComment->is(TT_RequiresExpressionLBrace)) {
// If there is no token, or an expression left brace, we are a requires
// clause within a requires expression.
parseRequiresClause(RequiresToken);
return true;
}
switch (PreviousNonComment->Tok.getKind()) {
case tok::greater:
case tok::r_paren:
case tok::kw_noexcept:
case tok::kw_const:
// This is a requires clause.
parseRequiresClause(RequiresToken);
return true;
case tok::amp:
case tok::ampamp: {
// This can be either:
// if (... && requires (T t) ...)
// Or
// void member(...) && requires (C<T> ...
// We check the one token before that for a const:
// void member(...) const && requires (C<T> ...
auto PrevPrev = PreviousNonComment->getPreviousNonComment();
if (PrevPrev && PrevPrev->is(tok::kw_const)) {
parseRequiresClause(RequiresToken);
return true;
}
break;
}
default:
// It's an expression.
parseRequiresExpression(RequiresToken);
return false;
}
// Now we look forward and try to check if the paren content is a parameter
// list. The parameters can be cv-qualified and contain references or
// pointers.
// So we want basically to check for TYPE NAME, but TYPE can contain all kinds
// of stuff: typename, const, *, &, &&, ::, identifiers.
int NextTokenOffset = 1;
auto NextToken = Tokens->peekNextToken(NextTokenOffset);
auto PeekNext = [&NextTokenOffset, &NextToken, this] {
++NextTokenOffset;
NextToken = Tokens->peekNextToken(NextTokenOffset);
};
bool FoundType = false;
bool LastWasColonColon = false;
int OpenAngles = 0;
for (; NextTokenOffset < 50; PeekNext()) {
switch (NextToken->Tok.getKind()) {
case tok::kw_volatile:
case tok::kw_const:
case tok::comma:
parseRequiresExpression(RequiresToken);
return false;
case tok::r_paren:
case tok::pipepipe:
parseRequiresClause(RequiresToken);
return true;
case tok::eof:
// Break out of the loop.
NextTokenOffset = 50;
break;
case tok::coloncolon:
LastWasColonColon = true;
break;
case tok::identifier:
if (FoundType && !LastWasColonColon && OpenAngles == 0) {
parseRequiresExpression(RequiresToken);
return false;
}
FoundType = true;
LastWasColonColon = false;
break;
case tok::less:
++OpenAngles;
break;
case tok::greater:
--OpenAngles;
break;
default:
if (NextToken->isSimpleTypeSpecifier()) {
parseRequiresExpression(RequiresToken);
return false;
}
break;
}
}
// This seems to be a complicated expression, just assume it's a clause.
parseRequiresClause(RequiresToken);
return true;
}
/// \brief Parses a requires clause.
/// \pre The current token needs to be the requires keyword.
/// \param RequiresToken The requires keyword token, which starts this clause.
/// \pre We need to be on the next token after the requires keyword.
/// \sa parseRequiresExpression
///
/// Returns if it either has finished parsing the clause, or it detects, that
/// the clause is incorrect.
void UnwrappedLineParser::parseRequiresClause() {
assert(FormatTok->Tok.is(tok::kw_requires) && "'requires' expected");
assert(FormatTok->getType() == TT_Unknown);
void UnwrappedLineParser::parseRequiresClause(FormatToken *RequiresToken) {
assert(FormatTok->getPreviousNonComment() == RequiresToken);
assert(RequiresToken->Tok.is(tok::kw_requires) && "'requires' expected");
assert(RequiresToken->getType() == TT_Unknown);
// If there is no previous token, we are within a requires expression,
// otherwise we will always have the template or function declaration in front
// of it.
bool InRequiresExpression =
!FormatTok->Previous ||
FormatTok->Previous->is(TT_RequiresExpressionLBrace);
!RequiresToken->Previous ||
RequiresToken->Previous->is(TT_RequiresExpressionLBrace);
FormatTok->setType(InRequiresExpression
? TT_RequiresClauseInARequiresExpression
: TT_RequiresClause);
RequiresToken->setType(InRequiresExpression
? TT_RequiresClauseInARequiresExpression
: TT_RequiresClause);
nextToken();
parseConstraintExpression();
if (!InRequiresExpression)
@ -2812,17 +2976,18 @@ void UnwrappedLineParser::parseRequiresClause() {
}
/// \brief Parses a requires expression.
/// \pre The current token needs to be the requires keyword.
/// \param RequiresToken The requires keyword token, which starts this clause.
/// \pre We need to be on the next token after the requires keyword.
/// \sa parseRequiresClause
///
/// Returns if it either has finished parsing the expression, or it detects,
/// that the expression is incorrect.
void UnwrappedLineParser::parseRequiresExpression() {
assert(FormatTok->Tok.is(tok::kw_requires) && "'requires' expected");
assert(FormatTok->getType() == TT_Unknown);
void UnwrappedLineParser::parseRequiresExpression(FormatToken *RequiresToken) {
assert(FormatTok->getPreviousNonComment() == RequiresToken);
assert(RequiresToken->Tok.is(tok::kw_requires) && "'requires' expected");
assert(RequiresToken->getType() == TT_Unknown);
FormatTok->setType(TT_RequiresExpression);
nextToken();
RequiresToken->setType(TT_RequiresExpression);
if (FormatTok->is(tok::l_paren)) {
FormatTok->setType(TT_RequiresExpressionLParen);
@ -2844,9 +3009,12 @@ void UnwrappedLineParser::parseRequiresExpression() {
void UnwrappedLineParser::parseConstraintExpression() {
do {
switch (FormatTok->Tok.getKind()) {
case tok::kw_requires:
parseRequiresExpression();
case tok::kw_requires: {
auto RequiresToken = FormatTok;
nextToken();
parseRequiresExpression(RequiresToken);
break;
}
case tok::l_paren:
parseParens(/*AmpAmpTokenType=*/TT_BinaryOperator);

5
clang/lib/Format/UnwrappedLineParser.h
View File

@ -133,8 +133,9 @@ private:
bool parseEnum();
bool parseStructLike();
void parseConcept();
void parseRequiresClause();
void parseRequiresExpression();
bool parseRequires();
void parseRequiresClause(FormatToken *RequiresToken);
void parseRequiresExpression(FormatToken *RequiresToken);
void parseConstraintExpression();
void parseJavaEnumBody();
// Parses a record (aka class) as a top level element. If ParseAsExpr is true,

126
clang/unittests/Format/TokenAnnotatorTest.cpp
View File

@ -141,6 +141,9 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
" { t.foo() };\n"
"} && Bar<T> && Baz<T>;");
ASSERT_EQ(Tokens.size(), 35u) << Tokens;
EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
@ -148,6 +151,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
"requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
"struct Foo;");
ASSERT_EQ(Tokens.size(), 36u) << Tokens;
EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
@ -163,6 +167,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
"requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
"struct Foo;");
ASSERT_EQ(Tokens.size(), 38u) << Tokens;
EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
@ -173,6 +178,127 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
Tokens = annotate("template <typename T>\n"
"void foo(T) noexcept requires Bar<T>;");
ASSERT_EQ(Tokens.size(), 18u) << Tokens;
EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
Tokens = annotate("template <typename T>\n"
"struct S {\n"
" void foo() const requires Bar<T>;\n"
" void bar() const & requires Baz<T>;\n"
" void bar() && requires Baz2<T>;\n"
" void baz() const & noexcept requires Baz<T>;\n"
" void baz() && noexcept requires Baz2<T>;\n"
"};\n"
"\n"
"void S::bar() const & requires Baz<T> { }");
ASSERT_EQ(Tokens.size(), 85u) << Tokens;
EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
Tokens = annotate("void Class::member() && requires(Constant) {}");
ASSERT_EQ(Tokens.size(), 14u) << Tokens;
EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
Tokens = annotate("void Class::member() && requires(Constant<T>) {}");
ASSERT_EQ(Tokens.size(), 17u) << Tokens;
EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
Tokens =
annotate("void Class::member() && requires(Namespace::Constant<T>) {}");
ASSERT_EQ(Tokens.size(), 19u) << Tokens;
EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
Tokens = annotate("void Class::member() && requires(typename "
"Namespace::Outer<T>::Inner::Constant) {}");
ASSERT_EQ(Tokens.size(), 24u) << Tokens;
EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
}
TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
auto Tokens = annotate("bool b = requires(int i) { i + 5; };");
ASSERT_EQ(Tokens.size(), 16u) << Tokens;
EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("if (requires(int i) { i + 5; }) return;");
ASSERT_EQ(Tokens.size(), 17u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");
ASSERT_EQ(Tokens.size(), 21u) << Tokens;
EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("foo(requires(const T t) {});");
ASSERT_EQ(Tokens.size(), 13u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("foo(requires(const int t) {});");
ASSERT_EQ(Tokens.size(), 13u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("foo(requires(const T t) {});");
ASSERT_EQ(Tokens.size(), 13u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("foo(requires(int const* volatile t) {});");
ASSERT_EQ(Tokens.size(), 15u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("foo(requires(T const* volatile t) {});");
ASSERT_EQ(Tokens.size(), 15u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens =
annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");
ASSERT_EQ(Tokens.size(), 21u) << Tokens;
EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
Tokens = annotate("template <typename T>\n"
"concept C = requires(T T) {\n"
" requires Bar<T> && Foo<T>;\n"
"};");
ASSERT_EQ(Tokens.size(), 28u) << Tokens;
EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
EXPECT_TOKEN(Tokens[14], tok::kw_requires,
TT_RequiresClauseInARequiresExpression);
Tokens = annotate("template <typename T>\n"
"concept C = requires(T T) {\n"
" { t.func() } -> std::same_as<int>;"
" requires Bar<T> && Foo<T>;\n"
"};");
ASSERT_EQ(Tokens.size(), 43u) << Tokens;
EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
EXPECT_TOKEN(Tokens[29], tok::kw_requires,
TT_RequiresClauseInARequiresExpression);
}
TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {