Introducing new tool clang-replace

Introducing new tool 'clang-replace' that finds files containing
serialized Replacements and applies those changes after deduplication
and detecting conflicts.

Currently the tool does not apply changes. It stops just after the
deduplication and conflict report phase. Forthcoming patches will
complete functionality.

Both build systems updated for new tool.

Includes a conflict test case.

clang-replace added to Doxygen build.

Differential Revision: http://llvm-reviews.chandlerc.com/D1424

llvm-svn: 189008

clang-tools-extra/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(remove-cstr-calls)
 add_subdirectory(tool-template)
 add_subdirectory(cpp11-migrate)
+add_subdirectory(clang-replace)
 add_subdirectory(modularize)
 add_subdirectory(clang-tidy)
 

clang-tools-extra/Makefile

@@ -12,7 +12,7 @@ CLANG_LEVEL := ../..
 include $(CLANG_LEVEL)/../../Makefile.config
 
 PARALLEL_DIRS := remove-cstr-calls tool-template modularize
-DIRS := cpp11-migrate clang-tidy unittests
+DIRS := cpp11-migrate clang-tidy clang-replace unittests
 
 include $(CLANG_LEVEL)/Makefile
 

clang-tools-extra/clang-replace/ApplyReplacements.cpp

@@ -0,0 +1,193 @@
+//===-- Core/ApplyChangeDescriptions.cpp ----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides the implementation for finding and applying change
+/// description files.
+///
+//===----------------------------------------------------------------------===//
+#include "ApplyReplacements.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace clang;
+
+
+static void eatDiagnostics(const SMDiagnostic &, void *) {}
+
+namespace clang {
+namespace replace {
+
+llvm::error_code
+collectReplacementsFromDirectory(const llvm::StringRef Directory,
+                                 TUReplacements &TUs,
+                                 clang::DiagnosticsEngine &Diagnostics) {
+  using namespace llvm::sys::fs;
+  using namespace llvm::sys::path;
+
+  error_code ErrorCode;
+
+  for (recursive_directory_iterator I(Directory, ErrorCode), E;
+       I != E && !ErrorCode; I.increment(ErrorCode)) {
+    if (filename(I->path())[0] == '.') {
+      // Indicate not to descend into directories beginning with '.'
+      I.no_push();
+      continue;
+    }
+
+    if (extension(I->path()) != ".yaml")
+      continue;
+
+    OwningPtr<MemoryBuffer> Out;
+    error_code BufferError = MemoryBuffer::getFile(I->path(), Out);
+    if (BufferError) {
+      errs() << "Error reading " << I->path() << ": " << BufferError.message()
+             << "\n";
+      continue;
+    }
+
+    yaml::Input YIn(Out->getBuffer());
+    YIn.setDiagHandler(&eatDiagnostics);
+    tooling::TranslationUnitReplacements TU;
+    YIn >> TU;
+    if (YIn.error()) {
+      // File doesn't appear to be a header change description. Ignore it.
+      continue;
+    }
+
+    // Only keep files that properly parse.
+    TUs.push_back(TU);
+  }
+
+  return ErrorCode;
+}
+
+/// \brief Dumps information for a sequence of conflicting Replacements.
+///
+/// \param[in] File FileEntry for the file the conflicting Replacements are
+/// for.
+/// \param[in] ConflictingReplacements List of conflicting Replacements.
+/// \param[in] SM SourceManager used for reporting.
+static void reportConflict(
+    const FileEntry *File,
+    const llvm::ArrayRef<clang::tooling::Replacement> ConflictingReplacements,
+    SourceManager &SM) {
+  FileID FID = SM.translateFile(File);
+  if (FID.isInvalid())
+    FID = SM.createFileID(File, SourceLocation(), SrcMgr::C_User);
+
+  // FIXME: Output something a little more user-friendly (e.g. unified diff?)
+  errs() << "The following changes conflict:\n";
+  for (const tooling::Replacement *I = ConflictingReplacements.begin(),
+                                  *E = ConflictingReplacements.end();
+       I != E; ++I) {
+    if (I->getLength() == 0) {
+      errs() << "  Insert at " << SM.getLineNumber(FID, I->getOffset()) << ":"
+             << SM.getColumnNumber(FID, I->getOffset()) << " "
+             << I->getReplacementText() << "\n";
+    } else {
+      if (I->getReplacementText().empty())
+        errs() << "  Remove ";
+      else
+        errs() << "  Replace ";
+
+      errs() << SM.getLineNumber(FID, I->getOffset()) << ":"
+             << SM.getColumnNumber(FID, I->getOffset()) << "-"
+             << SM.getLineNumber(FID, I->getOffset() + I->getLength() - 1)
+             << ":"
+             << SM.getColumnNumber(FID, I->getOffset() + I->getLength() - 1);
+
+      if (I->getReplacementText().empty())
+        errs() << "\n";
+      else
+        errs() << " with \"" << I->getReplacementText() << "\"\n";
+    }
+  }
+}
+
+/// \brief Deduplicates and tests for conflicts among the replacements for each
+/// file in \c Replacements. Any conflicts found are reported.
+///
+/// \param[in,out] Replacements Container of all replacements grouped by file
+/// to be deduplicated and checked for conflicts.
+/// \param[in] SM SourceManager required for conflict reporting
+///
+/// \returns \li true if conflicts were detected
+///          \li false if no conflicts were detected
+static bool deduplicateAndDetectConflicts(FileToReplacementsMap &Replacements,
+                                          SourceManager &SM) {
+  bool conflictsFound = false;
+
+  for (FileToReplacementsMap::iterator I = Replacements.begin(),
+                                       E = Replacements.end();
+       I != E; ++I) {
+
+    const FileEntry *Entry = SM.getFileManager().getFile(I->getKey());
+    if (!Entry) {
+      errs() << "Described file '" << I->getKey()
+             << "' doesn't exist. Ignoring...\n";
+      continue;
+    }
+
+    std::vector<tooling::Range> Conflicts;
+    tooling::deduplicate(I->getValue(), Conflicts);
+
+    if (Conflicts.empty())
+      continue;
+
+    conflictsFound = true;
+
+    errs() << "There are conflicting changes to " << I->getKey() << ":\n";
+
+    for (std::vector<tooling::Range>::const_iterator
+             ConflictI = Conflicts.begin(),
+             ConflictE = Conflicts.end();
+         ConflictI != ConflictE; ++ConflictI) {
+      ArrayRef<tooling::Replacement> ConflictingReplacements(
+          &I->getValue()[ConflictI->getOffset()], ConflictI->getLength());
+      reportConflict(Entry, ConflictingReplacements, SM);
+    }
+  }
+
+  return conflictsFound;
+}
+
+bool mergeAndDeduplicate(const TUReplacements &TUs,
+                         FileToReplacementsMap &GroupedReplacements,
+                         clang::DiagnosticsEngine &Diagnostics) {
+
+  // FIXME: Use Diagnostics for output
+
+  // Group all replacements by target file.
+  for (TUReplacements::const_iterator TUI = TUs.begin(), TUE = TUs.end();
+       TUI != TUE; ++TUI)
+    for (std::vector<tooling::Replacement>::const_iterator
+             RI = TUI->Replacements.begin(),
+             RE = TUI->Replacements.end();
+         RI != RE; ++RI)
+      GroupedReplacements[RI->getFilePath()].push_back(*RI);
+
+  FileManager Files((FileSystemOptions()));
+  SourceManager SM(Diagnostics, Files);
+
+  // Ask clang to deduplicate and report conflicts.
+  if (deduplicateAndDetectConflicts(GroupedReplacements, SM))
+    return false;
+
+  return true;
+}
+
+} // end namespace replace
+} // end namespace clang

clang-tools-extra/clang-replace/ApplyReplacements.h

@@ -0,0 +1,77 @@
+//===-- Core/ApplyChangeDescriptions.h --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides the interface for finding and applying change
+/// description files.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H
+#define CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H
+
+#include "clang/Tooling/ReplacementsYaml.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/system_error.h"
+#include <vector>
+
+namespace clang {
+
+class DiagnosticsEngine;
+
+namespace replace {
+
+/// \brief Collection of TranslationUnitReplacements.
+typedef std::vector<clang::tooling::TranslationUnitReplacements>
+TUReplacements;
+
+/// \brief Map mapping file name to Replacements targeting that file.
+typedef llvm::StringMap<std::vector<clang::tooling::Replacement> >
+FileToReplacementsMap;
+
+/// \brief Recursively descends through a directory structure rooted at \p
+/// Directory and attempts to deserialize *.yaml files as
+/// TranslationUnitReplacements. All docs that successfully deserialize are
+/// added to \p TUs.
+///
+/// Directories starting with '.' are ignored during traversal.
+///
+/// \param[in] Directory Directory to begin search for serialized
+/// TranslationUnitReplacements.
+/// \param[out] TUs Collection of all found and deserialized
+/// TranslationUnitReplacements.
+/// \param[in] Diagnostics DiagnosticsEngine used for error output.
+///
+/// \returns An error_code indicating success or failure in navigating the
+/// directory structure.
+llvm::error_code
+collectReplacementsFromDirectory(const llvm::StringRef Directory,
+                                 TUReplacements &TUs,
+                                 clang::DiagnosticsEngine &Diagnostics);
+
+/// \brief Deduplicate, check for conflicts, and apply all Replacements stored
+/// in \c TUs. If conflicts occur, no Replacements are applied.
+///
+/// \param[in] TUs Collection of TranslationUnitReplacements to merge,
+/// deduplicate, and test for conflicts.
+/// \param[out] GroupedReplacements Container grouping all Replacements by the
+/// file they target.
+/// \param[in] Diagnostics DiagnosticsEngine used for error/warning output.
+///
+/// \returns \li true If all changes were applied successfully.
+///          \li false If there were conflicts.
+bool mergeAndDeduplicate(const TUReplacements &TUs,
+                         FileToReplacementsMap &GroupedReplacements,
+                         clang::DiagnosticsEngine &Diagnostics);
+
+} // end namespace replace
+} // end namespace clang
+
+#endif // CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H

clang-tools-extra/clang-replace/CMakeLists.txt

@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  asmparser
+  bitreader
+  support
+  mc
+  )
+
+add_clang_library(clangReplace
+  ApplyReplacements.cpp
+  )
+target_link_libraries(clangReplace
+  clangTooling
+  clangBasic
+  clangRewriteFrontend
+  )
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+add_subdirectory(tool)

clang-tools-extra/clang-replace/Makefile

@@ -0,0 +1,16 @@
+##===- clang-replace/Makefile ------------------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangReplace
+include $(CLANG_LEVEL)/../../Makefile.config
+
+DIRS = tool
+
+include $(CLANG_LEVEL)/Makefile

clang-tools-extra/clang-replace/tool/CMakeLists.txt

@@ -0,0 +1,17 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  asmparser
+  bitreader
+  support
+  mc
+  )
+
+add_clang_executable(clang-replace
+  ClangReplaceMain.cpp
+  )
+target_link_libraries(clang-replace
+  clangReplace
+  )
+
+install(TARGETS clang-replace
+  RUNTIME DESTINATION bin)

clang-tools-extra/clang-replace/tool/ClangReplaceMain.cpp

@@ -0,0 +1,50 @@
+//===-- ClangReplaceMain.cpp - Main file for clang-replace tool -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides the main function for the clang-replace tool.
+///
+//===----------------------------------------------------------------------===//
+
+#include "ApplyReplacements.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::replace;
+
+static cl::opt<std::string> Directory(cl::Positional, cl::Required,
+                                      cl::desc("<Search Root Directory>"));
+
+int main(int argc, char **argv) {
+  cl::ParseCommandLineOptions(argc, argv);
+
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts(new DiagnosticOptions());
+  DiagnosticsEngine Diagnostics(
+      IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()),
+      DiagOpts.getPtr());
+
+  TUReplacements TUs;
+
+  error_code ErrorCode =
+      collectReplacementsFromDirectory(Directory, TUs, Diagnostics);
+
+  if (ErrorCode) {
+    errs() << "Trouble iterating over directory '" << Directory
+           << "': " << ErrorCode.message() << "\n";
+    return false;
+  }
+
+  FileToReplacementsMap GroupedReplacements;
+  if (mergeAndDeduplicate(TUs, GroupedReplacements, Diagnostics))
+    return 0;
+  return 1;
+}

clang-tools-extra/clang-replace/tool/Makefile

@@ -0,0 +1,28 @@
+##===- clang-replace/tool/Makefile -------------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../../..
+include $(CLANG_LEVEL)/../../Makefile.config
+
+TOOLNAME = clang-replace
+
+# No plugins, optimize startup time.
+TOOL_NO_EXPORTS = 1
+
+SOURCES = ClangReplaceMain.cpp
+
+LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc mcparser option
+USEDLIBS = clangReplace.a clangFormat.a clangTooling.a clangFrontend.a \
+	   clangSerialization.a clangDriver.a clangRewriteFrontend.a \
+	   clangRewriteCore.a clangParse.a clangSema.a clangAnalysis.a \
+	   clangAST.a clangASTMatchers.a clangEdit.a clangLex.a clangBasic.a
+
+include $(CLANG_LEVEL)/Makefile
+
+CPP.Flags += -I$(PROJ_SRC_DIR)/..

clang-tools-extra/docs/Doxyfile

@@ -648,7 +648,7 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = ../cpp11-migrate
+INPUT                  = ../cpp11-migrate ../clang-replace
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is

clang-tools-extra/test/CMakeLists.txt

@@ -27,7 +27,7 @@ set(CLANG_TOOLS_TEST_DEPS
   clang clang-headers FileCheck count not
 
   # Individual tools we test.
-  remove-cstr-calls cpp11-migrate modularize clang-tidy
+  remove-cstr-calls clang-replace cpp11-migrate modularize clang-tidy
 
   # Unit tests
   ExtraToolsUnitTests

clang-tools-extra/test/clang-replace/conflict.cpp

@@ -0,0 +1,7 @@
+// RUN: mkdir -p %T/conflict
+// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file1.yaml > %T/conflict/file1.yaml
+// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file2.yaml > %T/conflict/file2.yaml
+// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file3.yaml > %T/conflict/file3.yaml
+// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/expected.txt > %T/conflict/expected.txt
+// RUN: not clang-replace %T/conflict > %T/conflict/output.txt 2>&1
+// RUN: diff -b %T/conflict/output.txt %T/conflict/expected.txt

clang-tools-extra/test/clang-replace/conflict/common.h

@@ -0,0 +1,17 @@
+#ifndef COMMON_H
+#define COMMON_H
+
+extern void ext(int (&)[5]);
+
+void func(int t) {
+  int ints[5];
+  for (unsigned i = 0; i < 5; ++i) {
+    ints[i] = t;
+  }
+
+  int *i = 0;
+
+  ext(ints);
+}
+
+#endif // COMMON_H

clang-tools-extra/test/clang-replace/conflict/expected.txt

@@ -0,0 +1,11 @@
+There are conflicting changes to $(path)/common.h:
+The following changes conflict:
+  Replace 8:8-8:33 with "auto & i : ints"
+  Replace 8:8-8:33 with "int & elem : ints"
+The following changes conflict:
+  Replace 9:5-9:11 with "elem"
+  Replace 9:5-9:11 with "i"
+The following changes conflict:
+  Remove 12:3-12:14
+  Insert at 12:12 (int*)
+  Replace 12:12-12:12 with "nullptr"

clang-tools-extra/test/clang-replace/conflict/file1.yaml

@@ -0,0 +1,16 @@
+---
+MainSourceFile: "source1.cpp"
+Replacements:
+  - FilePath:        "$(path)/common.h"
+    Offset:          106
+    Length:          26
+    ReplacementText: "auto & i : ints"
+  - FilePath:        "$(path)/common.h"
+    Offset:          140
+    Length:          7
+    ReplacementText: "i"
+  - FilePath:        "$(path)/common.h"
+    Offset:          160
+    Length:          12
+    ReplacementText: ""
+...

clang-tools-extra/test/clang-replace/conflict/file2.yaml

@@ -0,0 +1,16 @@
+---
+MainSourceFile: "source2.cpp"
+Replacements:
+  - FilePath:        "$(path)/common.h"
+    Offset:          106
+    Length:          26
+    ReplacementText: "int & elem : ints"
+  - FilePath:        "$(path)/common.h"
+    Offset:          140
+    Length:          7
+    ReplacementText: "elem"
+  - FilePath:        "$(path)/common.h"
+    Offset:          169
+    Length:          1
+    ReplacementText: "nullptr"
+...

clang-tools-extra/test/clang-replace/conflict/file3.yaml

@@ -0,0 +1,8 @@
+---
+MainSourceFile: "source1.cpp"
+Replacements:
+  - FilePath:        "$(path)/common.h"
+    Offset:          169
+    Length:          0
+    ReplacementText: "(int*)"
+...