add a simplified lexer ctor that sets up the lexer to raw-lex an

entire file.

llvm-svn: 62414

clang/Driver/CacheTokens.cpp

@@ -486,8 +486,7 @@ void PTHWriter::GeneratePTH() {
     if (!B) continue;
 
     FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
-    Lexer L(SM.getLocForStartOfFile(FID), LOpts,
-            B->getBufferStart(), B->getBufferEnd(), B);
+    Lexer L(FID, SM, LOpts);
     PM[FE] = LexTokens(L);
   }
 

clang/Driver/clang.cpp

@@ -1332,15 +1332,11 @@ static void ProcessInputFile(Preprocessor &PP, PreprocessorFactory &PPF,
       
   case DumpRawTokens: {
     SourceManager &SM = PP.getSourceManager();
-    std::pair<const char*,const char*> File =
-      SM.getBufferData(SM.getMainFileID());
     // Start lexing the specified input file.
-    Lexer RawLex(SM.getLocForStartOfFile(SM.getMainFileID()),
-                 PP.getLangOptions(), File.first, File.second);
+    Lexer RawLex(SM.getMainFileID(), SM, PP.getLangOptions());
     RawLex.SetKeepWhitespaceMode(true);
 
     Token RawTok;
-
     RawLex.LexFromRawLexer(RawTok);
     while (RawTok.isNot(tok::eof)) {
       PP.DumpToken(RawTok, true);

clang/include/clang/Lex/Lexer.h

@@ -87,6 +87,11 @@ public:
         const char *BufStart, const char *BufEnd,
         const llvm::MemoryBuffer *FromFile = 0);
   
+  /// Lexer constructor - Create a new raw lexer object.  This object is only
+  /// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+  /// range will outlive it, so it doesn't take ownership of it.
+  Lexer(FileID FID, const SourceManager &SM, const LangOptions &Features);
+  
   /// getFeatures - Return the language features currently enabled.  NOTE: this
   /// lexer modifies features as a file is parsed!
   const LangOptions &getFeatures() const { return Features; }
@@ -166,6 +171,7 @@ public:
     ExtendedTokenMode = Mode ? 1 : 0;
   }
   
+  const char *getBufferStart() const { return BufferStart; }
   
   /// ReadToEndOfLine - Read the rest of the current preprocessor line as an
   /// uninterpreted string.  This switches the lexer out of directive mode.

clang/lib/Lex/Lexer.cpp

@@ -127,7 +127,6 @@ Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
              const char *BufPtr, const char *BufEnd,
              const llvm::MemoryBuffer *FromFile)
   : FileLoc(fileloc), Features(features) {
-      
 
   // If a MemoryBuffer was specified, use its start as BufferStart. This affects
   // the source location objects produced by this lexer.
@@ -140,6 +139,20 @@ Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
   LexingRawMode = true;
 }
 
+/// Lexer constructor - Create a new raw lexer object.  This object is only
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
+Lexer::Lexer(FileID FID, const SourceManager &SM, const LangOptions &features)
+  : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) {
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+
+  InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 
+            FromFile->getBufferEnd());
+  
+  // We *are* in raw mode.
+  LexingRawMode = true;
+}
+
 
 /// Stringify - Convert the specified string into a C string, with surrounding
 /// ""'s, and with escaped \ and " characters.

clang/lib/Rewrite/HTMLRewrite.cpp

@@ -344,11 +344,8 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, Preprocessor &PP) {
   RewriteBuffer &RB = R.getEditBuffer(FID);
 
   const SourceManager &SourceMgr = PP.getSourceManager();
-  std::pair<const char*, const char*> File = SourceMgr.getBufferData(FID);
-  const char *BufferStart = File.first;
-  
-  Lexer L(SourceMgr.getLocForStartOfFile(FID),
-          PP.getLangOptions(), File.first, File.second);
+  Lexer L(FID, SourceMgr, PP.getLangOptions());
+  const char *BufferStart = L.getBufferStart();
   
   // Inform the preprocessor that we want to retain comments as tokens, so we 
   // can highlight them.

clang/lib/Rewrite/TokenRewriter.cpp

@@ -22,11 +22,8 @@ TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
                              const LangOptions &LangOpts) {
   ScratchBuf.reset(new ScratchBuffer(SM));
   
-  std::pair<const char*,const char*> File = SM.getBufferData(FID);
-  
   // Create a lexer to lex all the tokens of the main file in raw mode.
-  Lexer RawLex(SM.getLocForStartOfFile(FID),
-               LangOpts, File.first, File.second);
+  Lexer RawLex(FID, SM, LangOpts);
   
   // Return all comments and whitespace as tokens.
   RawLex.SetKeepWhitespaceMode(true);