Migrate Declarators to use the List API

After this change all nodes that have a delimited-list are using the
`List` API.

Implementation details:
Let's look at a declaration with multiple declarators:
`int a, b;`
To generate a declarator list node we need to have the range of
declarators: `a, b`:
However, the `ClangAST` actually stores them as separate declarations:
`int a   ;`
`int    b;`
We solve that by appropriately marking the declarators on each separate
declaration in the `ClangAST` and then for the final declarator `int
b`, shrinking its range to fit to the already marked declarators.

Differential Revision: https://reviews.llvm.org/D88403

clang/include/clang/Tooling/Syntax/Nodes.h

@@ -99,10 +99,14 @@ enum class NodeKind : uint16_t {
   ParametersAndQualifiers,
   MemberPointer,
   UnqualifiedId,
+
+  // Lists
+  DeclaratorList,
   ParameterDeclarationList,
   CallArguments,
-  // Nested Name Specifiers.
   NestedNameSpecifier,
+
+  // Name Specifiers.
   GlobalNameSpecifier,
   DecltypeNameSpecifier,
   IdentifierNameSpecifier,
@@ -179,6 +183,7 @@ enum class NodeRole : uint8_t {
   Member,
   Callee,
   Arguments,
+  Declarators
 };
 /// For debugging purposes.
 raw_ostream &operator<<(raw_ostream &OS, NodeRole R);
@@ -823,6 +828,17 @@ public:
   }
 };
 
+class DeclaratorList final : public List {
+public:
+  DeclaratorList() : List(NodeKind::DeclaratorList) {}
+  static bool classof(const Node *N) {
+    return N->getKind() == NodeKind::DeclaratorList;
+  }
+  std::vector<SimpleDeclarator *> getDeclarators();
+  std::vector<List::ElementAndDelimiter<syntax::SimpleDeclarator>>
+  getDeclaratorsAndCommas();
+};
+
 /// Groups multiple declarators (e.g. variables, typedefs, etc.) together. All
 /// grouped declarators share the same declaration specifiers (e.g. 'int' or
 /// 'typedef').

clang/lib/Tooling/Syntax/BuildTree.cpp

@@ -397,6 +397,17 @@ public:
       Mapping.add(From, New);
   }
 
+  /// Populate children for \p New list, assuming it covers tokens from a
+  /// subrange of \p SuperRange.
+  void foldList(ArrayRef<syntax::Token> SuperRange, syntax::List *New,
+                ASTPtr From) {
+    assert(New);
+    auto ListRange = Pending.shrinkToFitList(SuperRange);
+    Pending.foldChildren(Arena, ListRange, New);
+    if (From)
+      Mapping.add(From, New);
+  }
+
   /// Notifies that we should not consume trailing semicolon when computing
   /// token range of \p D.
   void noticeDeclWithoutSemicolon(Decl *D);
@@ -579,6 +590,35 @@ private:
       It->second->setRole(Role);
     }
 
+    /// Shrink \p Range to a subrange that only contains tokens of a list.
+    /// List elements and delimiters should already have correct roles.
+    ArrayRef<syntax::Token> shrinkToFitList(ArrayRef<syntax::Token> Range) {
+      auto BeginChildren = Trees.lower_bound(Range.begin());
+      assert((BeginChildren == Trees.end() ||
+              BeginChildren->first == Range.begin()) &&
+             "Range crosses boundaries of existing subtrees");
+
+      auto EndChildren = Trees.lower_bound(Range.end());
+      assert(
+          (EndChildren == Trees.end() || EndChildren->first == Range.end()) &&
+          "Range crosses boundaries of existing subtrees");
+
+      auto BelongsToList = [](decltype(Trees)::value_type KV) {
+        auto Role = KV.second->getRole();
+        return Role == syntax::NodeRole::ListElement ||
+               Role == syntax::NodeRole::ListDelimiter;
+      };
+
+      auto BeginListChildren =
+          std::find_if(BeginChildren, EndChildren, BelongsToList);
+
+      auto EndListChildren =
+          std::find_if_not(BeginListChildren, EndChildren, BelongsToList);
+
+      return ArrayRef<syntax::Token>(BeginListChildren->first,
+                                     EndListChildren->first);
+    }
+
     /// Add \p Node to the forest and attach child nodes based on \p Tokens.
     void foldChildren(const syntax::Arena &A, ArrayRef<syntax::Token> Tokens,
                       syntax::Tree *Node) {
@@ -1513,14 +1553,31 @@ private:
 
     // There doesn't have to be a declarator (e.g. `void foo(int)` only has
     // declaration, but no declarator).
-    if (Range.getBegin().isValid()) {
-      auto *N = new (allocator()) syntax::SimpleDeclarator;
-      Builder.foldNode(Builder.getRange(Range), N, nullptr);
-      Builder.markChild(N, syntax::NodeRole::Declarator);
+    if (!Range.getBegin().isValid()) {
+      Builder.markChild(new (allocator()) syntax::DeclaratorList,
+                        syntax::NodeRole::Declarators);
+      Builder.foldNode(Builder.getDeclarationRange(D),
+                       new (allocator()) syntax::SimpleDeclaration, D);
+      return true;
     }
 
-    if (Builder.isResponsibleForCreatingDeclaration(D)) {
-      Builder.foldNode(Builder.getDeclarationRange(D),
+    auto *N = new (allocator()) syntax::SimpleDeclarator;
+    Builder.foldNode(Builder.getRange(Range), N, nullptr);
+    Builder.markChild(N, syntax::NodeRole::ListElement);
+
+    if (!Builder.isResponsibleForCreatingDeclaration(D)) {
+      // If this is not the last declarator in the declaration we expect a
+      // delimiter after it.
+      const auto *DelimiterToken = std::next(Builder.findToken(Range.getEnd()));
+      if (DelimiterToken->kind() == clang::tok::TokenKind::comma)
+        Builder.markChildToken(DelimiterToken, syntax::NodeRole::ListDelimiter);
+    } else {
+      auto *DL = new (allocator()) syntax::DeclaratorList;
+      auto DeclarationRange = Builder.getDeclarationRange(D);
+      Builder.foldList(DeclarationRange, DL, nullptr);
+
+      Builder.markChild(DL, syntax::NodeRole::Declarators);
+      Builder.foldNode(DeclarationRange,
                        new (allocator()) syntax::SimpleDeclaration, D);
     }
     return true;

clang/lib/Tooling/Syntax/Nodes.cpp

@@ -136,6 +136,8 @@ raw_ostream &syntax::operator<<(raw_ostream &OS, NodeKind K) {
     return OS << "CallArguments";
   case NodeKind::ParameterDeclarationList:
     return OS << "ParameterDeclarationList";
+  case NodeKind::DeclaratorList:
+    return OS << "DeclaratorList";
   }
   llvm_unreachable("unknown node kind");
 }
@@ -218,6 +220,8 @@ raw_ostream &syntax::operator<<(raw_ostream &OS, NodeRole R) {
     return OS << "Callee";
   case syntax::NodeRole::Arguments:
     return OS << "Arguments";
+  case syntax::NodeRole::Declarators:
+    return OS << "Declarators";
   }
   llvm_unreachable("invalid role");
 }
@@ -291,6 +295,29 @@ syntax::ParameterDeclarationList::getParametersAndCommas() {
   return Children;
 }
 
+std::vector<syntax::SimpleDeclarator *>
+syntax::DeclaratorList::getDeclarators() {
+  auto DeclaratorsAsNodes = getElementsAsNodes();
+  std::vector<syntax::SimpleDeclarator *> Children;
+  for (const auto &DeclaratorAsNode : DeclaratorsAsNodes) {
+    Children.push_back(llvm::cast<syntax::SimpleDeclarator>(DeclaratorAsNode));
+  }
+  return Children;
+}
+
+std::vector<syntax::List::ElementAndDelimiter<syntax::SimpleDeclarator>>
+syntax::DeclaratorList::getDeclaratorsAndCommas() {
+  auto DeclaratorsAsNodesAndCommas = getElementsAsNodesAndDelimiters();
+  std::vector<syntax::List::ElementAndDelimiter<syntax::SimpleDeclarator>>
+      Children;
+  for (const auto &DeclaratorAsNodeAndComma : DeclaratorsAsNodesAndCommas) {
+    Children.push_back(
+        {llvm::cast<syntax::SimpleDeclarator>(DeclaratorAsNodeAndComma.element),
+         DeclaratorAsNodeAndComma.delimiter});
+  }
+  return Children;
+}
+
 syntax::Expression *syntax::MemberExpression::getObject() {
   return cast_or_null<syntax::Expression>(findChild(syntax::NodeRole::Object));
 }

clang/lib/Tooling/Syntax/Synthesis.cpp

@@ -183,6 +183,8 @@ syntax::Tree *allocateTree(syntax::Arena &A, syntax::NodeKind Kind) {
     return new (A.getAllocator()) syntax::CallArguments;
   case syntax::NodeKind::ParameterDeclarationList:
     return new (A.getAllocator()) syntax::ParameterDeclarationList;
+  case syntax::NodeKind::DeclaratorList:
+    return new (A.getAllocator()) syntax::DeclaratorList;
   }
   llvm_unreachable("unknown node kind");
 }

clang/unittests/Tooling/Syntax/SynthesisTest.cpp

@@ -188,8 +188,9 @@ TEST_P(SynthesisTest, DeepCopy_Original) {
 TranslationUnit Detached synthesized
 `-SimpleDeclaration synthesized
   |-'int' synthesized
-  |-SimpleDeclarator Declarator synthesized
-  | `-'a' synthesized
+  |-DeclaratorList Declarators synthesized
+  | `-SimpleDeclarator ListElement synthesized
+  |   `-'a' synthesized
   `-';' synthesized
   )txt"));
 }
@@ -201,8 +202,9 @@ TEST_P(SynthesisTest, DeepCopy_Child) {
   EXPECT_TRUE(treeDumpEqual(Copy, R"txt(
 SimpleDeclaration Detached synthesized
 |-'int' synthesized
-|-SimpleDeclarator Declarator synthesized
-| `-'a' synthesized
+|-DeclaratorList Declarators synthesized
+| `-SimpleDeclarator ListElement synthesized
+|   `-'a' synthesized
 `-';' synthesized
   )txt"));
 }
@@ -225,11 +227,12 @@ void test() {
 TranslationUnit Detached synthesized
 `-SimpleDeclaration synthesized
   |-'void' synthesized
-  |-SimpleDeclarator Declarator synthesized
-  | |-'test' synthesized
-  | `-ParametersAndQualifiers synthesized
-  |   |-'(' OpenParen synthesized
-  |   `-')' CloseParen synthesized
+  |-DeclaratorList Declarators synthesized
+  | `-SimpleDeclarator ListElement synthesized
+  |   |-'test' synthesized
+  |   `-ParametersAndQualifiers synthesized
+  |     |-'(' OpenParen synthesized
+  |     `-')' CloseParen synthesized
   `-CompoundStatement synthesized
     |-'{' OpenParen synthesized
     |-IfStatement Statement synthesized