Comment parsing: add CommentOptions to allow specifying custom comment block commands

Add an ability to specify custom documentation block comment commands via a new
class CommentOptions.  The intention is that this class will hold future
customizations for comment parsing, including defining documentation comments
with specific numbers of parameters, etc.

CommentOptions instance is a member of LangOptions.

CommentOptions is controlled by a new command-line parameter
-fcomment-block-commands=Foo,Bar,Baz.

llvm-svn: 175892

clang/include/clang/AST/CommentCommandTraits.h

@@ -16,6 +16,7 @@
 #ifndef LLVM_CLANG_AST_COMMENT_COMMAND_TRAITS_H
 #define LLVM_CLANG_AST_COMMENT_COMMAND_TRAITS_H
 
+#include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -116,7 +117,10 @@ public:
     KCI_Last
   };
 
-  CommandTraits(llvm::BumpPtrAllocator &Allocator);
+  CommandTraits(llvm::BumpPtrAllocator &Allocator,
+                const CommentOptions &CommentOptions);
+
+  void registerCommentOptions(const CommentOptions &CommentOptions);
 
   /// \returns a CommandInfo object for a given command name or
   /// NULL if no CommandInfo object exists for this command.
@@ -132,6 +136,8 @@ public:
 
   const CommandInfo *registerUnknownCommand(StringRef CommandName);
 
+  const CommandInfo *registerBlockCommand(StringRef CommandName);
+
   /// \returns a CommandInfo object for a given command name or
   /// NULL if \c Name is not a builtin command.
   static const CommandInfo *getBuiltinCommandInfo(StringRef Name);
@@ -147,6 +153,8 @@ private:
   const CommandInfo *getRegisteredCommandInfo(StringRef Name) const;
   const CommandInfo *getRegisteredCommandInfo(unsigned CommandID) const;
 
+  CommandInfo *createCommandInfoWithName(StringRef CommandName);
+
   unsigned NextID;
 
   /// Allocator for CommandInfo objects.

clang/include/clang/Basic/CommentOptions.h

@@ -0,0 +1,34 @@
+//===--- CommentOptions.h - Options for parsing comments -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Defines the clang::CommentOptions interface.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_COMMENTOPTIONS_H
+#define LLVM_CLANG_COMMENTOPTIONS_H
+
+#include <string>
+#include <vector>
+
+namespace clang {
+
+/// \brief Options for controlling comment parsing.
+struct CommentOptions {
+  typedef std::vector<std::string> BlockCommandNamesTy;
+
+  /// \brief Command names to treat as block commands in comments.
+  /// Should not include the leading backslash.
+  BlockCommandNamesTy BlockCommandNames;
+};
+
+}  // end namespace clang
+
+#endif

clang/include/clang/Basic/LangOptions.h

@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_LANGOPTIONS_H
 #define LLVM_CLANG_LANGOPTIONS_H
 
+#include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/Visibility.h"
@@ -78,6 +79,9 @@ public:
 
   /// \brief The name of the current module.
   std::string CurrentModule;
+
+  /// \brief Options for parsing comments.
+  CommentOptions CommentOpts;
   
   LangOptions();
 

clang/include/clang/Driver/Options.td

@@ -327,6 +327,9 @@ def fcatch_undefined_behavior : Flag<["-"], "fcatch-undefined-behavior">, Group<
 def fclasspath_EQ : Joined<["-"], "fclasspath=">, Group<f_Group>;
 def fcolor_diagnostics : Flag<["-"], "fcolor-diagnostics">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Use colors in diagnostics">;
+def fcomment_block_commands : CommaJoined<["-"], "fcomment-block-commands=">, Group<f_clang_Group>, Flags<[CC1Option]>,
+  HelpText<"Treat each comma separated argument in <arg> as a documentation comment block command">,
+  MetaVarName<"<arg>">;
 def fcommon : Flag<["-"], "fcommon">, Group<f_Group>;
 def fcompile_resource_EQ : Joined<["-"], "fcompile-resource=">, Group<f_Group>;
 def fconstant_cfstrings : Flag<["-"], "fconstant-cfstrings">, Group<f_Group>;

clang/lib/AST/ASTContext.cpp

@@ -654,7 +654,7 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     DeclarationNames(*this),
     ExternalSource(0), Listener(0),
     Comments(SM), CommentsLoaded(false),
-    CommentCommandTraits(BumpAlloc),
+    CommentCommandTraits(BumpAlloc, LOpts.CommentOpts),
     LastSDM(0, 0),
     UniqueBlockByRefTypeID(0)
 {

clang/lib/AST/CommentCommandTraits.cpp

@@ -15,9 +15,21 @@ namespace comments {
 
 #include "clang/AST/CommentCommandInfo.inc"
 
-CommandTraits::CommandTraits(llvm::BumpPtrAllocator &Allocator) :
-    NextID(llvm::array_lengthof(Commands)), Allocator(Allocator)
-{ }
+CommandTraits::CommandTraits(llvm::BumpPtrAllocator &Allocator,
+                             const CommentOptions &CommentOptions) :
+    NextID(llvm::array_lengthof(Commands)), Allocator(Allocator) {
+  registerCommentOptions(CommentOptions);
+}
+
+void CommandTraits::registerCommentOptions(
+    const CommentOptions &CommentOptions) {
+  for (CommentOptions::BlockCommandNamesTy::const_iterator
+           I = CommentOptions.BlockCommandNames.begin(),
+           E = CommentOptions.BlockCommandNames.end();
+       I != E; I++) {
+    registerBlockCommand(*I);
+  }
+}
 
 const CommandInfo *CommandTraits::getCommandInfoOrNULL(StringRef Name) const {
   if (const CommandInfo *Info = getBuiltinCommandInfo(Name))
@@ -31,7 +43,7 @@ const CommandInfo *CommandTraits::getCommandInfo(unsigned CommandID) const {
   return getRegisteredCommandInfo(CommandID);
 }
 
-const CommandInfo *CommandTraits::registerUnknownCommand(StringRef CommandName) {
+CommandInfo *CommandTraits::createCommandInfoWithName(StringRef CommandName) {
   char *Name = Allocator.Allocate<char>(CommandName.size() + 1);
   memcpy(Name, CommandName.data(), CommandName.size());
   Name[CommandName.size()] = '\0';
@@ -40,13 +52,25 @@ const CommandInfo *CommandTraits::registerUnknownCommand(StringRef CommandName)
   CommandInfo *Info = new (Allocator) CommandInfo();
   Info->Name = Name;
   Info->ID = NextID++;
-  Info->IsUnknownCommand = true;
 
   RegisteredCommands.push_back(Info);
 
   return Info;
 }
 
+const CommandInfo *CommandTraits::registerUnknownCommand(
+                                                  StringRef CommandName) {
+  CommandInfo *Info = createCommandInfoWithName(CommandName);
+  Info->IsUnknownCommand = true;
+  return Info;
+}
+
+const CommandInfo *CommandTraits::registerBlockCommand(StringRef CommandName) {
+  CommandInfo *Info = createCommandInfoWithName(CommandName);
+  Info->IsBlockCommand = true;
+  return Info;
+}
+
 const CommandInfo *CommandTraits::getBuiltinCommandInfo(
                                                   unsigned CommandID) {
   if (CommandID < llvm::array_lengthof(Commands))

clang/lib/Driver/Tools.cpp

@@ -3201,6 +3201,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_fretain_comments_from_system_headers))
     CmdArgs.push_back("-fretain-comments-from-system-headers");
 
+  // Forward -fcomment-block-commands to -cc1.
+  Args.AddAllArgs(CmdArgs, options::OPT_fcomment_block_commands);
+
   // Forward -Xclang arguments to -cc1, and -mllvm arguments to the LLVM option
   // parser.
   Args.AddAllArgValues(CmdArgs, options::OPT_Xclang);

clang/lib/Frontend/ASTUnit.cpp

@@ -573,6 +573,11 @@ private:
 
     // Initialize the ASTContext
     Context.InitBuiltinTypes(*Target);
+
+    // We didn't have access to the comment options when the ASTContext was
+    // constructed, so register them now.
+    Context.getCommentCommandTraits().registerCommentOptions(
+        LangOpt.CommentOpts);
   }
 };
 

clang/lib/Frontend/CompilerInvocation.cpp

@@ -279,6 +279,10 @@ static bool ParseMigratorArgs(MigratorOptions &Opts, ArgList &Args) {
   return true;
 }
 
+static void ParseCommentArgs(CommentOptions &Opts, ArgList &Args) {
+  Opts.BlockCommandNames = Args.getAllArgValues(OPT_fcomment_block_commands);
+}
+
 static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
                              DiagnosticsEngine &Diags) {
   using namespace options;
@@ -1522,6 +1526,7 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), *Args);
   Success = ParseDiagnosticArgs(Res.getDiagnosticOpts(), *Args, &Diags)
             && Success;
+  ParseCommentArgs(Res.getLangOpts()->CommentOpts, *Args);
   ParseFileSystemArgs(Res.getFileSystemOpts(), *Args);
   // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = ParseFrontendArgs(Res.getFrontendOpts(), *Args, Diags);

clang/lib/Serialization/ASTReader.cpp

@@ -108,6 +108,14 @@ static bool checkLanguageOptions(const LangOptions &LangOpts,
     return true;
   }
 
+  if (ExistingLangOpts.CommentOpts.BlockCommandNames !=
+      LangOpts.CommentOpts.BlockCommandNames) {
+    if (Diags)
+      Diags->Report(diag::err_pch_langopt_value_mismatch)
+        << "block command names";
+    return true;
+  }
+
   return false;
 }
 
@@ -3671,6 +3679,15 @@ bool ASTReader::ParseLanguageOptions(const RecordData &Record,
   unsigned Length = Record[Idx++];
   LangOpts.CurrentModule.assign(Record.begin() + Idx, 
                                 Record.begin() + Idx + Length);
+
+  Idx += Length;
+
+  // Comment options.
+  for (unsigned N = Record[Idx++]; N; --N) {
+    LangOpts.CommentOpts.BlockCommandNames.push_back(
+      ReadString(Record, Idx));
+  }
+
   return Listener.ReadLanguageOptions(LangOpts, Complain);
 }
 

clang/lib/Serialization/ASTWriter.cpp

@@ -1057,6 +1057,16 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
   
   Record.push_back(LangOpts.CurrentModule.size());
   Record.append(LangOpts.CurrentModule.begin(), LangOpts.CurrentModule.end());
+
+  // Comment options.
+  Record.push_back(LangOpts.CommentOpts.BlockCommandNames.size());
+  for (CommentOptions::BlockCommandNamesTy::const_iterator
+           I = LangOpts.CommentOpts.BlockCommandNames.begin(),
+           IEnd = LangOpts.CommentOpts.BlockCommandNames.end();
+       I != IEnd; ++I) {
+    AddString(*I, Record);
+  }
+
   Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
 
   // Target options.

clang/test/Driver/fcomment-block-commands.c

@@ -0,0 +1,8 @@
+// Check that we pass -fcomment-block-commands to frontend.
+//
+// RUN: %clang -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ARG
+// RUN: %clang -c %s -fcomment-block-commands=Foo -### 2>&1 | FileCheck %s --check-prefix=CHECK-ARG
+//
+// CHECK-ARG: -fcomment-block-commands=Foo
+//
+// CHECK-NO-ARG-NOT: -fcomment-block-commands=

clang/test/Index/comment-custom-block-command.cpp

@@ -0,0 +1,38 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+
+// Check that custom block commands are defined correctly.
+// RUN: %clang_cc1 -fcomment-block-commands=CustomCommand -x c++ -std=c++11 -emit-pch -o %t/out.pch %s
+// RUN: %clang_cc1 -x c++ -std=c++11 -fcomment-block-commands=CustomCommand -include-pch %t/out.pch -fsyntax-only %s
+
+// RUN: c-index-test -test-load-source all -comments-xml-schema=%S/../../bindings/xml/comment-xml-schema.rng %s -std=c++11 -fcomment-block-commands=CustomCommand > %t/out.c-index-direct
+// RUN: c-index-test -test-load-tu %t/out.pch all > %t/out.c-index-pch
+
+// RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-direct
+// RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-pch
+
+// Ensure that XML is not invalid
+// WRONG-NOT: CommentXMLInvalid
+
+// RUN: FileCheck %s < %t/out.c-index-direct
+// RUN: FileCheck %s < %t/out.c-index-pch
+
+// XFAIL: valgrind
+
+#ifndef HEADER
+#define HEADER
+
+/// \CustomCommand Aaa.
+void comment_custom_block_command_1();
+
+// CHECK: comment-custom-block-command.cpp:[[@LINE-2]]:6: FunctionDecl=comment_custom_block_command_1:{{.*}} FullCommentAsHTML=[<p> Aaa.</p>] FullCommentAsXML=[<Function file="{{[^"]+}}comment-custom-block-command.cpp" line="[[@LINE-2]]" column="6"><Name>comment_custom_block_command_1</Name><USR>c:@F@comment_custom_block_command_1#</USR><Declaration>void comment_custom_block_command_1()</Declaration><Discussion><Para> Aaa.</Para></Discussion></Function>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[CustomCommand]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+
+#endif
+

clang/unittests/AST/CommentLexer.cpp

@@ -9,6 +9,7 @@
 
 #include "clang/AST/CommentLexer.h"
 #include "clang/AST/CommentCommandTraits.h"
+#include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
@@ -31,7 +32,7 @@ protected:
       DiagID(new DiagnosticIDs()),
       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
       SourceMgr(Diags, FileMgr),
-      Traits(Allocator) {
+      Traits(Allocator, CommentOptions()) {
   }
 
   FileSystemOptions FileMgrOpts;
@@ -451,6 +452,76 @@ TEST_F(CommentLexerTest, DoxygenCommand9) {
   ASSERT_EQ(tok::newline,   Toks[2].getKind());
 }
 
+TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {
+  const char *Source = "/// \\NewBlockCommand Aaa.\n";
+
+  Traits.registerBlockCommand(StringRef("NewBlockCommand"));
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(4U, Toks.size());
+
+  ASSERT_EQ(tok::text,      Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+  ASSERT_EQ(tok::command,                 Toks[1].getKind());
+  ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[1]));
+
+  ASSERT_EQ(tok::text,          Toks[2].getKind());
+  ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
+
+  ASSERT_EQ(tok::newline,        Toks[3].getKind());
+}
+
+TEST_F(CommentLexerTest, RegisterMultipleBlockCommands) {
+  const char *Source =
+    "/// \\Foo\n"
+    "/// \\Bar Baz\n"
+    "/// \\Blech quux=corge\n";
+
+  Traits.registerBlockCommand(StringRef("Foo"));
+  Traits.registerBlockCommand(StringRef("Bar"));
+  Traits.registerBlockCommand(StringRef("Blech"));
+
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(11U, Toks.size());
+
+  ASSERT_EQ(tok::text,      Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+  ASSERT_EQ(tok::command,     Toks[1].getKind());
+  ASSERT_EQ(StringRef("Foo"), getCommandName(Toks[1]));
+
+  ASSERT_EQ(tok::newline,     Toks[2].getKind());
+
+  ASSERT_EQ(tok::text,      Toks[3].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[3].getText());
+
+  ASSERT_EQ(tok::command,     Toks[4].getKind());
+  ASSERT_EQ(StringRef("Bar"), getCommandName(Toks[4]));
+
+  ASSERT_EQ(tok::text,         Toks[5].getKind());
+  ASSERT_EQ(StringRef(" Baz"), Toks[5].getText());
+
+  ASSERT_EQ(tok::newline,     Toks[6].getKind());
+
+  ASSERT_EQ(tok::text,      Toks[7].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[7].getText());
+
+  ASSERT_EQ(tok::command,       Toks[8].getKind());
+  ASSERT_EQ(StringRef("Blech"), getCommandName(Toks[8]));
+
+  ASSERT_EQ(tok::text,                Toks[9].getKind());
+  ASSERT_EQ(StringRef(" quux=corge"), Toks[9].getText());
+
+  ASSERT_EQ(tok::newline,     Toks[10].getKind());
+}
+
 // Empty verbatim block.
 TEST_F(CommentLexerTest, VerbatimBlock1) {
   const char *Sources[] = {

clang/unittests/AST/CommentParser.cpp

@@ -12,6 +12,7 @@
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/AST/CommentLexer.h"
 #include "clang/AST/CommentSema.h"
+#include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
@@ -38,7 +39,7 @@ protected:
       DiagID(new DiagnosticIDs()),
       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
       SourceMgr(Diags, FileMgr),
-      Traits(Allocator) {
+      Traits(Allocator, CommentOptions()) {
   }
 
   FileSystemOptions FileMgrOpts;