[SyntaxTree] Extend the syntax tree dump to also cover `NodeRole`

We should see `NodeRole` information in the dump because that exposes how the
accessors will behave.

Functional changes in the dump:
* Surround Leaf tokens with `'`
* Append `Node` dumps with `NodeRole` information, except for unknown roles
* Append marks to `Node` dumps, instead of prepending

Non-functional changes:
* `::dumpTokens(llvm::raw_ostream, ArrayRef<syntax::Token>, const
SourceManager &SM)` always received as parameter a `syntax::Token *`
pointing to `Leaf::token()`. Changed the function to
`dumpLeaf(llvm::raw_ostream, syntax::Leaf *, const SourceManager&)`
* `dumpTree` acted on a Node, rename to `dumpNode`

Differential Revision: https://reviews.llvm.org/D85330

clang/include/clang/Tooling/Syntax/Tree.h

@@ -106,9 +106,9 @@ public:
   Node *nextSibling() { return NextSibling; }
 
   /// Dumps the structure of a subtree. For debugging and testing purposes.
-  std::string dump(const Arena &A) const;
+  std::string dump(const SourceManager &SM) const;
   /// Dumps the tokens forming this subtree.
-  std::string dumpTokens(const Arena &A) const;
+  std::string dumpTokens(const SourceManager &SM) const;
 
   /// Asserts invariants on this node of the tree and its immediate children.
   /// Will not recurse into the subtree. No-op if NDEBUG is set.

clang/lib/Tooling/Syntax/BuildTree.cpp

@@ -546,7 +546,7 @@ private:
         R += std::string(
             formatv("- '{0}' covers '{1}'+{2} tokens\n", It->second->kind(),
                     It->first->text(A.sourceManager()), CoveredTokens));
-        R += It->second->dump(A);
+        R += It->second->dump(A.sourceManager());
       }
       return R;
     }

clang/lib/Tooling/Syntax/Tree.cpp

@@ -133,46 +133,45 @@ void syntax::Tree::replaceChildRangeLowLevel(Node *BeforeBegin, Node *End,
 }
 
 namespace {
-static void dumpTokens(raw_ostream &OS, ArrayRef<syntax::Token> Tokens,
-                       const SourceManager &SM) {
-  assert(!Tokens.empty());
-  bool First = true;
-  for (const auto &T : Tokens) {
-    if (!First)
-      OS << " ";
-    else
-      First = false;
-    // Handle 'eof' separately, calling text() on it produces an empty string.
-    if (T.kind() == tok::eof) {
-      OS << "<eof>";
-      continue;
-    }
-    OS << T.text(SM);
-  }
+static void dumpLeaf(raw_ostream &OS, const syntax::Leaf *L,
+                     const SourceManager &SM) {
+  assert(L);
+  const auto *Token = L->token();
+  assert(Token);
+  // Handle 'eof' separately, calling text() on it produces an empty string.
+  if (Token->kind() == tok::eof)
+    OS << "<eof>";
+  else
+    OS << Token->text(SM);
 }
 
-static void dumpTree(raw_ostream &OS, const syntax::Node *N,
-                     const syntax::Arena &A, std::vector<bool> IndentMask) {
-  std::string Marks;
-  if (!N->isOriginal())
-    Marks += "M";
-  if (N->role() == syntax::NodeRole::Detached)
-    Marks += "*"; // FIXME: find a nice way to print other roles.
-  if (!N->canModify())
-    Marks += "I";
-  if (!Marks.empty())
-    OS << Marks << ": ";
+static void dumpNode(raw_ostream &OS, const syntax::Node *N,
+                     const SourceManager &SM, std::vector<bool> IndentMask) {
+  auto dumpExtraInfo = [&OS](const syntax::Node *N) {
+    if (N->role() != syntax::NodeRole::Unknown)
+      OS << " " << N->role();
+    if (!N->isOriginal())
+      OS << " synthesized";
+    if (!N->canModify())
+      OS << " unmodifiable";
+  };
 
-  if (auto *L = dyn_cast<syntax::Leaf>(N)) {
-    dumpTokens(OS, *L->token(), A.sourceManager());
+  assert(N);
+  if (const auto *L = dyn_cast<syntax::Leaf>(N)) {
+    OS << "'";
+    dumpLeaf(OS, L, SM);
+    OS << "'";
+    dumpExtraInfo(N);
     OS << "\n";
     return;
   }
 
-  auto *T = cast<syntax::Tree>(N);
-  OS << T->kind() << "\n";
+  const auto *T = cast<syntax::Tree>(N);
+  OS << T->kind();
+  dumpExtraInfo(N);
+  OS << "\n";
 
-  for (auto It = T->firstChild(); It != nullptr; It = It->nextSibling()) {
+  for (const auto *It = T->firstChild(); It; It = It->nextSibling()) {
     for (bool Filled : IndentMask) {
       if (Filled)
         OS << "| ";
@@ -186,28 +185,27 @@ static void dumpTree(raw_ostream &OS, const syntax::Node *N,
       OS << "|-";
       IndentMask.push_back(true);
     }
-    dumpTree(OS, It, A, IndentMask);
+    dumpNode(OS, It, SM, IndentMask);
     IndentMask.pop_back();
   }
 }
 } // namespace
 
-std::string syntax::Node::dump(const Arena &A) const {
+std::string syntax::Node::dump(const SourceManager &SM) const {
   std::string Str;
   llvm::raw_string_ostream OS(Str);
-  dumpTree(OS, this, A, /*IndentMask=*/{});
+  dumpNode(OS, this, SM, /*IndentMask=*/{});
   return std::move(OS.str());
 }
 
-std::string syntax::Node::dumpTokens(const Arena &A) const {
+std::string syntax::Node::dumpTokens(const SourceManager &SM) const {
   std::string Storage;
   llvm::raw_string_ostream OS(Storage);
   traverse(this, [&](const syntax::Node *N) {
-    auto *L = dyn_cast<syntax::Leaf>(N);
-    if (!L)
-      return;
-    ::dumpTokens(OS, *L->token(), A.sourceManager());
-    OS << " ";
+    if (const auto *L = dyn_cast<syntax::Leaf>(N)) {
+      dumpLeaf(OS, L, SM);
+      OS << " ";
+    }
   });
   return OS.str();
 }

clang/unittests/Tooling/Syntax/TreeTestBase.cpp

@@ -171,7 +171,7 @@ SyntaxTreeTest::buildTree(StringRef Code, const TestClangConfig &ClangConfig) {
            << "Source file has syntax errors, they were printed to the test "
               "log";
   }
-  auto Actual = StringRef(Root->dump(*Arena)).trim().str();
+  auto Actual = StringRef(Root->dump(Arena->sourceManager())).trim().str();
   // EXPECT_EQ shows the diff between the two strings if they are different.
   EXPECT_EQ(Tree.trim().str(), Actual);
   if (Actual != Tree.trim().str()) {
@@ -205,7 +205,7 @@ SyntaxTreeTest::treeDumpEqualOnAnnotations(StringRef CodeWithAnnotations,
     auto *AnnotatedNode = nodeByRange(AnnotatedRanges[i], Root);
     assert(AnnotatedNode);
     auto AnnotatedNodeDump =
-        StringRef(AnnotatedNode->dump(*Arena)).trim().str();
+        StringRef(AnnotatedNode->dump(Arena->sourceManager())).trim().str();
     // EXPECT_EQ shows the diff between the two strings if they are different.
     EXPECT_EQ(TreeDumps[i].trim().str(), AnnotatedNodeDump)
         << "Dumps diverged for the code:\n"