[clang-format] Add a MacroExpander.

Summary:
The MacroExpander allows to expand simple (non-resursive) macro
definitions from a macro identifier token and macro arguments. It
annotates the tokens with a newly introduced MacroContext that keeps
track of the role a token played in expanding the macro in order to
be able to reconstruct the macro expansion from an expanded (formatted)
token stream.

Made Token explicitly copy-able to enable copying tokens from the parsed
macro definition.

Reviewers: sammccall

Subscribers: mgorny, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83296

clang/lib/Format/CMakeLists.txt

@@ -7,6 +7,7 @@ add_clang_library(clangFormat
   Format.cpp
   FormatToken.cpp
   FormatTokenLexer.cpp
+  MacroExpander.cpp
   NamespaceEndCommentsFixer.cpp
   SortJavaScriptImports.cpp
   TokenAnalyzer.cpp

clang/lib/Format/FormatToken.h

@@ -136,6 +136,68 @@ enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive };
 
 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break };
 
+/// Roles a token can take in a configured macro expansion.
+enum MacroRole {
+  /// The token was expanded from a macro argument when formatting the expanded
+  /// token sequence.
+  MR_ExpandedArg,
+  /// The token is part of a macro argument that was previously formatted as
+  /// expansion when formatting the unexpanded macro call.
+  MR_UnexpandedArg,
+  /// The token was expanded from a macro definition, and is not visible as part
+  /// of the macro call.
+  MR_Hidden,
+};
+
+struct FormatToken;
+
+/// Contains information on the token's role in a macro expansion.
+///
+/// Given the following definitions:
+/// A(X) = [ X ]
+/// B(X) = < X >
+/// C(X) = X
+///
+/// Consider the macro call:
+/// A({B(C(C(x)))}) -> [{<x>}]
+///
+/// In this case, the tokens of the unexpanded macro call will have the
+/// following relevant entries in their macro context (note that formatting
+/// the unexpanded macro call happens *after* formatting the expanded macro
+/// call):
+///                   A( { B( C( C(x) ) ) } )
+/// Role:             NN U NN NN NNUN N N U N  (N=None, U=UnexpandedArg)
+///
+///                   [  { <       x    > } ]
+/// Role:             H  E H       E    H E H  (H=Hidden, E=ExpandedArg)
+/// ExpandedFrom[0]:  A  A A       A    A A A
+/// ExpandedFrom[1]:       B       B    B
+/// ExpandedFrom[2]:               C
+/// ExpandedFrom[3]:               C
+/// StartOfExpansion: 1  0 1       2    0 0 0
+/// EndOfExpansion:   0  0 0       2    1 0 1
+struct MacroExpansion {
+  MacroExpansion(MacroRole Role) : Role(Role) {}
+
+  /// The token's role in the macro expansion.
+  /// When formatting an expanded macro, all tokens that are part of macro
+  /// arguments will be MR_ExpandedArg, while all tokens that are not visible in
+  /// the macro call will be MR_Hidden.
+  /// When formatting an unexpanded macro call, all tokens that are part of
+  /// macro arguments will be MR_UnexpandedArg.
+  MacroRole Role;
+
+  /// The stack of macro call identifier tokens this token was expanded from.
+  llvm::SmallVector<FormatToken *, 1> ExpandedFrom;
+
+  /// The number of expansions of which this macro is the first entry.
+  unsigned StartOfExpansion = 0;
+
+  /// The number of currently open expansions in \c ExpandedFrom this macro is
+  /// the last token in.
+  unsigned EndOfExpansion = 0;
+};
+
 class TokenRole;
 class AnnotatedLine;
 
@@ -163,7 +225,9 @@ struct FormatToken {
 
   /// A token can have a special role that can carry extra information
   /// about the token's formatting.
-  std::unique_ptr<TokenRole> Role;
+  /// FIXME: Make FormatToken for parsing and AnnotatedToken two different
+  /// classes and make this a unique_ptr in the AnnotatedToken class.
+  std::shared_ptr<TokenRole> Role;
 
   /// The range of the whitespace immediately preceding the \c Token.
   SourceRange WhitespaceRange;
@@ -378,6 +442,10 @@ public:
   /// in it.
   SmallVector<AnnotatedLine *, 1> Children;
 
+  // Contains all attributes related to how this token takes part
+  // in a configured macro expansion.
+  llvm::Optional<MacroExpansion> MacroCtx;
+
   bool is(tok::TokenKind Kind) const { return Tok.is(Kind); }
   bool is(TokenType TT) const { return getType() == TT; }
   bool is(const IdentifierInfo *II) const {
@@ -631,10 +699,12 @@ public:
                : nullptr;
   }
 
+  void copyFrom(const FormatToken &Tok) { *this = Tok; }
+
 private:
-  // Disallow copying.
+  // Only allow copying via the explicit copyFrom method.
   FormatToken(const FormatToken &) = delete;
-  void operator=(const FormatToken &) = delete;
+  FormatToken &operator=(const FormatToken &) = default;
 
   template <typename A, typename... Ts>
   bool startsSequenceInternal(A K1, Ts... Tokens) const {

clang/lib/Format/MacroExpander.cpp

@@ -0,0 +1,225 @@
+//===--- MacroExpander.cpp - Format C++ code --------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the implementation of MacroExpander, which handles macro
+/// configuration and expansion while formatting.
+///
+//===----------------------------------------------------------------------===//
+
+#include "Macros.h"
+
+#include "Encoding.h"
+#include "FormatToken.h"
+#include "FormatTokenLexer.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Format/Format.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace clang {
+namespace format {
+
+struct MacroExpander::Definition {
+  StringRef Name;
+  SmallVector<FormatToken *, 8> Params;
+  SmallVector<FormatToken *, 8> Body;
+
+  // Map from each argument's name to its position in the argument list.
+  // With "M(x, y) x + y":
+  //   x -> 0
+  //   y -> 1
+  llvm::StringMap<size_t> ArgMap;
+
+  bool ObjectLike = true;
+};
+
+class MacroExpander::DefinitionParser {
+public:
+  DefinitionParser(ArrayRef<FormatToken *> Tokens) : Tokens(Tokens) {
+    assert(!Tokens.empty());
+    Current = Tokens[0];
+  }
+
+  // Parse the token stream and return the corresonding Definition object.
+  // Returns an empty definition object with a null-Name on error.
+  MacroExpander::Definition parse() {
+    if (!Current->is(tok::identifier))
+      return {};
+    Def.Name = Current->TokenText;
+    nextToken();
+    if (Current->is(tok::l_paren)) {
+      Def.ObjectLike = false;
+      if (!parseParams())
+        return {};
+    }
+    if (!parseExpansion())
+      return {};
+
+    return Def;
+  }
+
+private:
+  bool parseParams() {
+    assert(Current->is(tok::l_paren));
+    nextToken();
+    while (Current->is(tok::identifier)) {
+      Def.Params.push_back(Current);
+      Def.ArgMap[Def.Params.back()->TokenText] = Def.Params.size() - 1;
+      nextToken();
+      if (Current->isNot(tok::comma))
+        break;
+      nextToken();
+    }
+    if (Current->isNot(tok::r_paren))
+      return false;
+    nextToken();
+    return true;
+  }
+
+  bool parseExpansion() {
+    if (!Current->isOneOf(tok::equal, tok::eof))
+      return false;
+    if (Current->is(tok::equal))
+      nextToken();
+    parseTail();
+    return true;
+  }
+
+  void parseTail() {
+    while (Current->isNot(tok::eof)) {
+      Def.Body.push_back(Current);
+      nextToken();
+    }
+    Def.Body.push_back(Current);
+  }
+
+  void nextToken() {
+    if (Pos + 1 < Tokens.size())
+      ++Pos;
+    Current = Tokens[Pos];
+    Current->Finalized = true;
+  }
+
+  size_t Pos = 0;
+  FormatToken *Current = nullptr;
+  Definition Def;
+  ArrayRef<FormatToken *> Tokens;
+};
+
+MacroExpander::MacroExpander(
+    const std::vector<std::string> &Macros, clang::SourceManager &SourceMgr,
+    const FormatStyle &Style,
+    llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
+    IdentifierTable &IdentTable)
+    : SourceMgr(SourceMgr), Style(Style), Allocator(Allocator),
+      IdentTable(IdentTable) {
+  for (const std::string &Macro : Macros) {
+    parseDefinition(Macro);
+  }
+}
+
+MacroExpander::~MacroExpander() = default;
+
+void MacroExpander::parseDefinition(const std::string &Macro) {
+  Buffers.push_back(
+      llvm::MemoryBuffer::getMemBufferCopy(Macro, "<scratch space>"));
+  clang::FileID FID =
+      SourceMgr.createFileID(SourceManager::Unowned, Buffers.back().get());
+  FormatTokenLexer Lex(SourceMgr, FID, 0, Style, encoding::Encoding_UTF8,
+                       Allocator, IdentTable);
+  const auto Tokens = Lex.lex();
+  if (!Tokens.empty()) {
+    DefinitionParser Parser(Tokens);
+    auto Definition = Parser.parse();
+    Definitions[Definition.Name] = std::move(Definition);
+  }
+}
+
+bool MacroExpander::defined(llvm::StringRef Name) const {
+  return Definitions.find(Name) != Definitions.end();
+}
+
+bool MacroExpander::objectLike(llvm::StringRef Name) const {
+  return Definitions.find(Name)->second.ObjectLike;
+}
+
+llvm::SmallVector<FormatToken *, 8> MacroExpander::expand(FormatToken *ID,
+                                                          ArgsList Args) const {
+  assert(defined(ID->TokenText));
+  SmallVector<FormatToken *, 8> Result;
+  const Definition &Def = Definitions.find(ID->TokenText)->second;
+
+  // Expand each argument at most once.
+  llvm::StringSet<> ExpandedArgs;
+
+  // Adds the given token to Result.
+  auto pushToken = [&](FormatToken *Tok) {
+    Tok->MacroCtx->ExpandedFrom.push_back(ID);
+    Result.push_back(Tok);
+  };
+
+  // If Tok references a parameter, adds the corresponding argument to Result.
+  // Returns false if Tok does not reference a parameter.
+  auto expandArgument = [&](FormatToken *Tok) -> bool {
+    // If the current token references a parameter, expand the corresponding
+    // argument.
+    if (!Tok->is(tok::identifier) || ExpandedArgs.contains(Tok->TokenText))
+      return false;
+    ExpandedArgs.insert(Tok->TokenText);
+    auto I = Def.ArgMap.find(Tok->TokenText);
+    if (I == Def.ArgMap.end())
+      return false;
+    // If there are fewer arguments than referenced parameters, treat the
+    // parameter as empty.
+    // FIXME: Potentially fully abort the expansion instead.
+    if (I->getValue() >= Args.size())
+      return true;
+    for (FormatToken *Arg : Args[I->getValue()]) {
+      // A token can be part of a macro argument at multiple levels.
+      // For example, with "ID(x) x":
+      // in ID(ID(x)), 'x' is expanded first as argument to the inner
+      // ID, then again as argument to the outer ID. We keep the macro
+      // role the token had from the inner expansion.
+      if (!Arg->MacroCtx)
+        Arg->MacroCtx = MacroExpansion(MR_ExpandedArg);
+      pushToken(Arg);
+    }
+    return true;
+  };
+
+  // Expand the definition into Result.
+  for (FormatToken *Tok : Def.Body) {
+    if (expandArgument(Tok))
+      continue;
+    // Create a copy of the tokens from the macro body, i.e. were not provided
+    // by user code.
+    FormatToken *New = new (Allocator.Allocate()) FormatToken;
+    New->copyFrom(*Tok);
+    assert(!New->MacroCtx);
+    // Tokens that are not part of the user code are not formatted.
+    New->MacroCtx = MacroExpansion(MR_Hidden);
+    pushToken(New);
+  }
+  assert(Result.size() >= 1 && Result.back()->is(tok::eof));
+  if (Result.size() > 1) {
+    ++Result[0]->MacroCtx->StartOfExpansion;
+    ++Result[Result.size() - 2]->MacroCtx->EndOfExpansion;
+  }
+  return Result;
+}
+
+} // namespace format
+} // namespace clang

clang/lib/Format/Macros.h

@@ -0,0 +1,141 @@
+//===--- MacroExpander.h - Format C++ code ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the main building blocks of macro support in
+/// clang-format.
+///
+/// In order to not violate the requirement that clang-format can format files
+/// in isolation, clang-format's macro support uses expansions users provide
+/// as part of clang-format's style configuration.
+///
+/// Macro definitions are of the form "MACRO(p1, p2)=p1 + p2", but only support
+/// one level of expansion (\see MacroExpander for a full description of what
+/// is supported).
+///
+/// As part of parsing, clang-format uses the MacroExpander to expand the
+/// spelled token streams into expanded token streams when it encounters a
+/// macro call. The UnwrappedLineParser continues to parse UnwrappedLines
+/// from the expanded token stream.
+/// After the expanded unwrapped lines are parsed, the MacroUnexpander matches
+/// the spelled token stream into unwrapped lines that best resemble the
+/// structure of the expanded unwrapped lines.
+///
+/// When formatting, clang-format formats the expanded unwrapped lines first,
+/// determining the token types. Next, it formats the spelled unwrapped lines,
+/// keeping the token types fixed, while allowing other formatting decisions
+/// to change.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_LIB_FORMAT_MACROS_H
+#define CLANG_LIB_FORMAT_MACROS_H
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "Encoding.h"
+#include "FormatToken.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class MemoryBuffer;
+} // namespace llvm
+
+namespace clang {
+class IdentifierTable;
+class SourceManager;
+
+namespace format {
+struct FormatStyle;
+
+/// Takes a set of macro definitions as strings and allows expanding calls to
+/// those macros.
+///
+/// For example:
+/// Definition: A(x, y)=x + y
+/// Call      : A(int a = 1, 2)
+/// Expansion : int a = 1 + 2
+///
+/// Expansion does not check arity of the definition.
+/// If fewer arguments than expected are provided, the remaining parameters
+/// are considered empty:
+/// Call     : A(a)
+/// Expansion: a +
+/// If more arguments than expected are provided, they will be discarded.
+///
+/// The expander does not support:
+/// - recursive expansion
+/// - stringification
+/// - concatenation
+/// - variadic macros
+///
+/// Furthermore, only a single expansion of each macro argument is supported,
+/// so that we cannot get conflicting formatting decisions from different
+/// expansions.
+/// Definition: A(x)=x+x
+/// Call      : A(id)
+/// Expansion : id+x
+///
+class MacroExpander {
+public:
+  using ArgsList = llvm::ArrayRef<llvm::SmallVector<FormatToken *, 8>>;
+
+  /// Construct a macro expander from a set of macro definitions.
+  /// Macro definitions must be encoded as UTF-8.
+  ///
+  /// Each entry in \p Macros must conform to the following simple
+  /// macro-definition language:
+  /// <definition> ::= <id> <expansion> | <id> "(" <params> ")" <expansion>
+  /// <params>     ::= <id-list> | ""
+  /// <id-list>    ::= <id> | <id> "," <params>
+  /// <expansion>  ::= "=" <tail> | <eof>
+  /// <tail>       ::= <tok> <tail> | <eof>
+  ///
+  /// Macros that cannot be parsed will be silently discarded.
+  ///
+  MacroExpander(const std::vector<std::string> &Macros,
+                clang::SourceManager &SourceMgr, const FormatStyle &Style,
+                llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
+                IdentifierTable &IdentTable);
+  ~MacroExpander();
+
+  /// Returns whether a macro \p Name is defined.
+  bool defined(llvm::StringRef Name) const;
+
+  /// Returns whether the macro has no arguments and should not consume
+  /// subsequent parentheses.
+  bool objectLike(llvm::StringRef Name) const;
+
+  /// Returns the expanded stream of format tokens for \p ID, where
+  /// each element in \p Args is a positional argument to the macro call.
+  llvm::SmallVector<FormatToken *, 8> expand(FormatToken *ID,
+                                             ArgsList Args) const;
+
+private:
+  struct Definition;
+  class DefinitionParser;
+
+  void parseDefinition(const std::string &Macro);
+
+  clang::SourceManager &SourceMgr;
+  const FormatStyle &Style;
+  llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator;
+  IdentifierTable &IdentTable;
+  std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
+  llvm::StringMap<Definition> Definitions;
+};
+
+} // namespace format
+} // namespace clang
+
+#endif

clang/unittests/Format/CMakeLists.txt

@@ -15,6 +15,7 @@ add_clang_unittest(FormatTests
   FormatTestSelective.cpp
   FormatTestTableGen.cpp
   FormatTestTextProto.cpp
+  MacroExpanderTest.cpp
   NamespaceEndCommentsFixerTest.cpp
   SortImportsTestJS.cpp
   SortImportsTestJava.cpp

clang/unittests/Format/MacroExpanderTest.cpp

@@ -0,0 +1,187 @@
+#include "../../lib/Format/Macros.h"
+#include "TestLexer.h"
+#include "clang/Basic/FileManager.h"
+
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace format {
+
+namespace {
+
+class MacroExpanderTest : public ::testing::Test {
+public:
+  std::unique_ptr<MacroExpander>
+  create(const std::vector<std::string> &MacroDefinitions) {
+    return std::make_unique<MacroExpander>(MacroDefinitions,
+                                           Lex.SourceMgr.get(), Lex.Style,
+                                           Lex.Allocator, Lex.IdentTable);
+  }
+
+  std::string expand(MacroExpander &Macros, llvm::StringRef Name,
+                     const std::vector<std::string> &Args = {}) {
+    EXPECT_TRUE(Macros.defined(Name));
+    return text(Macros.expand(Lex.id(Name), lexArgs(Args)));
+  }
+
+  llvm::SmallVector<TokenList, 1>
+  lexArgs(const std::vector<std::string> &Args) {
+    llvm::SmallVector<TokenList, 1> Result;
+    for (const auto &Arg : Args) {
+      Result.push_back(uneof(Lex.lex(Arg)));
+    }
+    return Result;
+  }
+
+  struct MacroAttributes {
+    clang::tok::TokenKind Kind;
+    MacroRole Role;
+    unsigned Start;
+    unsigned End;
+    llvm::SmallVector<FormatToken *, 1> ExpandedFrom;
+  };
+
+  void expectAttributes(const TokenList &Tokens,
+                        const std::vector<MacroAttributes> &Attributes,
+                        const std::string &File, unsigned Line) {
+    EXPECT_EQ(Tokens.size(), Attributes.size()) << text(Tokens);
+    for (size_t I = 0, E = Tokens.size(); I != E; ++I) {
+      if (I >= Attributes.size())
+        continue;
+      std::string Context =
+          ("for token " + llvm::Twine(I) + ": " + Tokens[I]->Tok.getName() +
+           " / " + Tokens[I]->TokenText)
+              .str();
+      EXPECT_TRUE(Tokens[I]->is(Attributes[I].Kind))
+          << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
+      EXPECT_EQ(Tokens[I]->MacroCtx->Role, Attributes[I].Role)
+          << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
+      EXPECT_EQ(Tokens[I]->MacroCtx->StartOfExpansion, Attributes[I].Start)
+          << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
+      EXPECT_EQ(Tokens[I]->MacroCtx->EndOfExpansion, Attributes[I].End)
+          << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
+      EXPECT_EQ(Tokens[I]->MacroCtx->ExpandedFrom, Attributes[I].ExpandedFrom)
+          << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
+    }
+  }
+
+  TestLexer Lex;
+};
+
+#define EXPECT_ATTRIBUTES(Tokens, Attributes)                                  \
+  expectAttributes(Tokens, Attributes, __FILE__, __LINE__)
+
+TEST_F(MacroExpanderTest, SkipsDefinitionOnError) {
+  auto Macros =
+      create({"A(", "B(,", "C(a,", "D(a a", "E(a, a", "F(,)", "G(a;"});
+  for (const auto *Name : {"A", "B", "C", "D", "E", "F", "G"}) {
+    EXPECT_FALSE(Macros->defined(Name)) << "for Name " << Name;
+  }
+}
+
+TEST_F(MacroExpanderTest, ExpandsWithoutArguments) {
+  auto Macros = create({
+      "A",
+      "B=b",
+      "C=c + c",
+      "D()",
+  });
+  EXPECT_TRUE(Macros->objectLike("A"));
+  EXPECT_TRUE(Macros->objectLike("B"));
+  EXPECT_TRUE(Macros->objectLike("C"));
+  EXPECT_TRUE(!Macros->objectLike("D"));
+  EXPECT_EQ("", expand(*Macros, "A"));
+  EXPECT_EQ("b", expand(*Macros, "B"));
+  EXPECT_EQ("c+c", expand(*Macros, "C"));
+  EXPECT_EQ("", expand(*Macros, "D"));
+}
+
+TEST_F(MacroExpanderTest, ExpandsWithArguments) {
+  auto Macros = create({
+      "A(x)",
+      "B(x, y)=x + y",
+  });
+  EXPECT_EQ("", expand(*Macros, "A", {"a"}));
+  EXPECT_EQ("b1+b2+b3", expand(*Macros, "B", {"b1", "b2 + b3"}));
+  EXPECT_EQ("x+", expand(*Macros, "B", {"x"}));
+}
+
+TEST_F(MacroExpanderTest, AttributizesTokens) {
+  auto Macros = create({
+      "A(x, y)={ x + y; }",
+      "B(x, y)=x + 3 + y",
+  });
+  auto *A = Lex.id("A");
+  auto AArgs = lexArgs({"a1 * a2", "a3 * a4"});
+  auto Result = Macros->expand(A, AArgs);
+  EXPECT_EQ(11U, Result.size()) << text(Result) << " / " << Result;
+  EXPECT_EQ("{a1*a2+a3*a4;}", text(Result));
+  std::vector<MacroAttributes> Attributes = {
+      {tok::l_brace, MR_Hidden, 1, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::star, MR_ExpandedArg, 0, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::plus, MR_Hidden, 0, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::star, MR_ExpandedArg, 0, 0, {A}},
+      {tok::identifier, MR_ExpandedArg, 0, 0, {A}},
+      {tok::semi, MR_Hidden, 0, 0, {A}},
+      {tok::r_brace, MR_Hidden, 0, 1, {A}},
+      {tok::eof, MR_Hidden, 0, 0, {A}},
+  };
+  EXPECT_ATTRIBUTES(Result, Attributes);
+
+  auto *B = Lex.id("B");
+  auto BArgs = lexArgs({"b1", "b2"});
+  Result = Macros->expand(B, BArgs);
+  EXPECT_EQ(6U, Result.size()) << text(Result) << " / " << Result;
+  EXPECT_EQ("b1+3+b2", text(Result));
+  Attributes = {
+      {tok::identifier, MR_ExpandedArg, 1, 0, {B}},
+      {tok::plus, MR_Hidden, 0, 0, {B}},
+      {tok::numeric_constant, MR_Hidden, 0, 0, {B}},
+      {tok::plus, MR_Hidden, 0, 0, {B}},
+      {tok::identifier, MR_ExpandedArg, 0, 1, {B}},
+      {tok::eof, MR_Hidden, 0, 0, {B}},
+  };
+  EXPECT_ATTRIBUTES(Result, Attributes);
+}
+
+TEST_F(MacroExpanderTest, RecursiveExpansion) {
+  auto Macros = create({
+      "A(x)=x",
+      "B(x)=x",
+      "C(x)=x",
+  });
+
+  auto *A = Lex.id("A");
+  auto *B = Lex.id("B");
+  auto *C = Lex.id("C");
+
+  auto Args = lexArgs({"id"});
+  auto CResult = uneof(Macros->expand(C, Args));
+  auto BResult = uneof(Macros->expand(B, CResult));
+  auto AResult = uneof(Macros->expand(A, BResult));
+
+  std::vector<MacroAttributes> Attributes = {
+      {tok::identifier, MR_ExpandedArg, 3, 3, {C, B, A}},
+  };
+  EXPECT_ATTRIBUTES(AResult, Attributes);
+}
+
+TEST_F(MacroExpanderTest, SingleExpansion) {
+  auto Macros = create({"A(x)=x+x"});
+  auto *A = Lex.id("A");
+  auto Args = lexArgs({"id"});
+  auto Result = uneof(Macros->expand(A, Args));
+  std::vector<MacroAttributes> Attributes = {
+      {tok::identifier, MR_ExpandedArg, 1, 0, {A}},
+      {tok::plus, MR_Hidden, 0, 0, {A}},
+      {tok::identifier, MR_Hidden, 0, 1, {A}},
+  };
+  EXPECT_ATTRIBUTES(Result, Attributes);
+}
+
+} // namespace
+} // namespace format
+} // namespace clang

clang/unittests/Format/TestLexer.h

@@ -0,0 +1,88 @@
+//===--- TestLexer.h - Format C++ code --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains a TestLexer to create FormatTokens from strings.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_UNITTESTS_FORMAT_TESTLEXER_H
+#define CLANG_UNITTESTS_FORMAT_TESTLEXER_H
+
+#include "../../lib/Format/FormatTokenLexer.h"
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+
+#include <numeric>
+#include <ostream>
+
+namespace clang {
+namespace format {
+
+typedef llvm::SmallVector<FormatToken *, 8> TokenList;
+
+inline std::ostream &operator<<(std::ostream &Stream, const FormatToken &Tok) {
+  Stream << "(" << Tok.Tok.getName() << ", \"" << Tok.TokenText.str() << "\")";
+  return Stream;
+}
+inline std::ostream &operator<<(std::ostream &Stream, const TokenList &Tokens) {
+  Stream << "{";
+  for (size_t I = 0, E = Tokens.size(); I != E; ++I) {
+    Stream << (I > 0 ? ", " : "") << *Tokens[I];
+  }
+  Stream << "}";
+  return Stream;
+}
+
+inline TokenList uneof(const TokenList &Tokens) {
+  assert(!Tokens.empty() && Tokens.back()->is(tok::eof));
+  return TokenList(Tokens.begin(), std::prev(Tokens.end()));
+}
+
+inline std::string text(llvm::ArrayRef<FormatToken *> Tokens) {
+  return std::accumulate(Tokens.begin(), Tokens.end(), std::string(),
+                         [](const std::string &R, FormatToken *Tok) {
+                           return (R + Tok->TokenText).str();
+                         });
+}
+
+class TestLexer {
+public:
+  TestLexer() : SourceMgr("test.cpp", "") {}
+
+  TokenList lex(llvm::StringRef Code) {
+    Buffers.push_back(
+        llvm::MemoryBuffer::getMemBufferCopy(Code, "<scratch space>"));
+    clang::FileID FID = SourceMgr.get().createFileID(SourceManager::Unowned,
+                                                     Buffers.back().get());
+    FormatTokenLexer Lex(SourceMgr.get(), FID, 0, Style, Encoding, Allocator,
+                         IdentTable);
+    auto Result = Lex.lex();
+    return TokenList(Result.begin(), Result.end());
+  }
+
+  FormatToken *id(llvm::StringRef Code) {
+    auto Result = uneof(lex(Code));
+    assert(Result.size() == 1U && "Code must expand to 1 token.");
+    return Result[0];
+  }
+
+  FormatStyle Style = getLLVMStyle();
+  encoding::Encoding Encoding = encoding::Encoding_UTF8;
+  std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
+  clang::SourceManagerForFile SourceMgr;
+  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
+  IdentifierTable IdentTable;
+};
+
+} // namespace format
+} // namespace clang
+
+#endif // LLVM_CLANG_UNITTESTS_FORMAT_TEST_LEXER_H