From b5f8171a1b955e07c13226d353f4ca53f75a0e8a Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 30 Apr 2018 05:25:48 +0000 Subject: [PATCH] PR37189 Fix incorrect end source location and spelling for a split '>>' token. When a '>>' token is split into two '>' tokens (in C++11 onwards), or (as an extension) when we do the same for other tokens starting with a '>', we can't just use a location pointing to the first '>' as the location of the split token, because that would result in our miscomputing the length and spelling for the token. As a consequence, for example, a refactoring replacing 'A' with something else would sometimes replace one character too many, and similarly diagnostics highlighting a template-id source range would highlight one character too many. Fix this by creating an expansion range covering the first character of the '>>' token, whose spelling is '>'. For this to work, we generalize the expansion range of a macro FileID to be either a token range (the common case) or a character range (used in this new case). llvm-svn: 331155 --- clang/include/clang/Basic/SourceLocation.h | 4 +- clang/include/clang/Basic/SourceManager.h | 65 +++++++-- clang/include/clang/Lex/Lexer.h | 14 +- clang/include/clang/Lex/Preprocessor.h | 5 + .../include/clang/Rewrite/Core/HTMLRewrite.h | 3 +- clang/lib/ARCMigrate/PlistReporter.cpp | 3 +- clang/lib/ARCMigrate/TransGCAttrs.cpp | 5 +- clang/lib/ARCMigrate/TransUnbridgedCasts.cpp | 5 +- clang/lib/ARCMigrate/TransformActions.cpp | 8 +- clang/lib/Basic/SourceLocation.cpp | 18 --- clang/lib/Basic/SourceManager.cpp | 39 +++-- clang/lib/CodeGen/CoverageMappingGen.cpp | 6 +- clang/lib/Edit/EditedSource.cpp | 8 +- clang/lib/Frontend/DiagnosticRenderer.cpp | 77 ++++++---- clang/lib/Frontend/TextDiagnostic.cpp | 34 ++--- clang/lib/Lex/Lexer.cpp | 26 ++-- clang/lib/Lex/PPDirectives.cpp | 2 +- clang/lib/Lex/PPMacroExpansion.cpp | 2 +- clang/lib/Lex/Preprocessor.cpp | 16 +++ clang/lib/Lex/TokenLexer.cpp | 4 +- clang/lib/Parse/ParseTemplate.cpp | 134 +++++++++++------- clang/lib/Rewrite/HTMLRewrite.cpp | 19 +-- clang/lib/Sema/SemaChecking.cpp | 2 +- clang/lib/Sema/SemaDeclAttr.cpp | 2 +- clang/lib/Sema/SemaDeclCXX.cpp | 2 +- clang/lib/Sema/SemaInit.cpp | 2 +- clang/lib/Sema/SemaType.cpp | 6 +- clang/lib/Serialization/ASTReader.cpp | 1 + clang/lib/Serialization/ASTWriter.cpp | 2 + clang/lib/StaticAnalyzer/Core/BugReporter.cpp | 2 +- .../Core/BugReporterVisitors.cpp | 2 +- clang/test/Misc/diag-greatergreater.cpp | 43 ++++++ clang/test/Parser/cxx0x-decl.cpp | 9 +- clang/tools/libclang/CIndex.cpp | 10 +- clang/unittests/Lex/LexerTest.cpp | 9 +- 35 files changed, 376 insertions(+), 213 deletions(-) create mode 100644 clang/test/Misc/diag-greatergreater.cpp diff --git a/clang/include/clang/Basic/SourceLocation.h b/clang/include/clang/Basic/SourceLocation.h index a83f95e7ae83..4ca9340e80b7 100644 --- a/clang/include/clang/Basic/SourceLocation.h +++ b/clang/include/clang/Basic/SourceLocation.h @@ -265,6 +265,7 @@ public: void setBegin(SourceLocation b) { Range.setBegin(b); } void setEnd(SourceLocation e) { Range.setEnd(e); } + void setTokenRange(bool TR) { IsTokenRange = TR; } bool isValid() const { return Range.isValid(); } bool isInvalid() const { return !isValid(); } @@ -359,7 +360,6 @@ public: FullSourceLoc getExpansionLoc() const; FullSourceLoc getSpellingLoc() const; FullSourceLoc getFileLoc() const; - std::pair getImmediateExpansionRange() const; PresumedLoc getPresumedLoc(bool UseLineDirectives = true) const; bool isMacroArgExpansion(FullSourceLoc *StartLoc = nullptr) const; FullSourceLoc getImmediateMacroCallerLoc() const; @@ -377,8 +377,6 @@ public: unsigned getLineNumber(bool *Invalid = nullptr) const; unsigned getColumnNumber(bool *Invalid = nullptr) const; - std::pair getExpansionRange() const; - const FileEntry *getFileEntry() const; /// \brief Return a StringRef to the source buffer data for the diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index d6e9a9972cfd..55c138410d1d 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -317,9 +317,13 @@ namespace SrcMgr { /// invalid location. unsigned ExpansionLocStart, ExpansionLocEnd; + /// Whether the expansion range is a token range. + bool ExpansionIsTokenRange; + public: SourceLocation getSpellingLoc() const { - return SourceLocation::getFromRawEncoding(SpellingLoc); + SourceLocation SpellLoc = SourceLocation::getFromRawEncoding(SpellingLoc); + return SpellLoc.isInvalid() ? getExpansionLocStart() : SpellLoc; } SourceLocation getExpansionLocStart() const { @@ -332,8 +336,14 @@ namespace SrcMgr { return EndLoc.isInvalid() ? getExpansionLocStart() : EndLoc; } - std::pair getExpansionLocRange() const { - return std::make_pair(getExpansionLocStart(), getExpansionLocEnd()); + bool isExpansionTokenRange() const { + return ExpansionIsTokenRange; + } + + CharSourceRange getExpansionLocRange() const { + return CharSourceRange( + SourceRange(getExpansionLocStart(), getExpansionLocEnd()), + isExpansionTokenRange()); } bool isMacroArgExpansion() const { @@ -359,11 +369,13 @@ namespace SrcMgr { /// the characters from the token come from). All three can refer to /// normal File SLocs or expansion locations. static ExpansionInfo create(SourceLocation SpellingLoc, - SourceLocation Start, SourceLocation End) { + SourceLocation Start, SourceLocation End, + bool ExpansionIsTokenRange = true) { ExpansionInfo X; X.SpellingLoc = SpellingLoc.getRawEncoding(); X.ExpansionLocStart = Start.getRawEncoding(); X.ExpansionLocEnd = End.getRawEncoding(); + X.ExpansionIsTokenRange = ExpansionIsTokenRange; return X; } @@ -393,6 +405,17 @@ namespace SrcMgr { // than a normal one. return create(SpellingLoc, ExpansionLoc, SourceLocation()); } + + /// \brief Return a special ExpansionInfo representing a token that ends + /// prematurely. This is used to model a '>>' token that has been split + /// into '>' tokens and similar cases. Unlike for the other forms of + /// expansion, the expansion range in this case is a character range, not + /// a token range. + static ExpansionInfo createForTokenSplit(SourceLocation SpellingLoc, + SourceLocation Start, + SourceLocation End) { + return create(SpellingLoc, Start, End, false); + } }; /// \brief This is a discriminated union of FileInfo and ExpansionInfo. @@ -851,9 +874,16 @@ public: SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, + bool ExpansionIsTokenRange = true, int LoadedID = 0, unsigned LoadedOffset = 0); + /// \brief Return a new SourceLocation that encodes that the token starting + /// at \p TokenStart ends prematurely at \p TokenEnd. + SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, + SourceLocation TokenStart, + SourceLocation TokenEnd); + /// \brief Retrieve the memory buffer associated with the given file. /// /// \param Invalid If non-NULL, will be set \c true if an error @@ -1102,19 +1132,28 @@ public: /// expansion location. /// /// \pre \p Loc is required to be an expansion location. - std::pair - getImmediateExpansionRange(SourceLocation Loc) const; + CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const; /// \brief Given a SourceLocation object, return the range of /// tokens covered by the expansion in the ultimate file. - std::pair - getExpansionRange(SourceLocation Loc) const; + CharSourceRange getExpansionRange(SourceLocation Loc) const; /// \brief Given a SourceRange object, return the range of - /// tokens covered by the expansion in the ultimate file. - SourceRange getExpansionRange(SourceRange Range) const { - return SourceRange(getExpansionRange(Range.getBegin()).first, - getExpansionRange(Range.getEnd()).second); + /// tokens or characters covered by the expansion in the ultimate file. + CharSourceRange getExpansionRange(SourceRange Range) const { + SourceLocation Begin = getExpansionRange(Range.getBegin()).getBegin(); + CharSourceRange End = getExpansionRange(Range.getEnd()); + return CharSourceRange(SourceRange(Begin, End.getEnd()), + End.isTokenRange()); + } + + /// \brief Given a CharSourceRange object, return the range of + /// tokens or characters covered by the expansion in the ultimate file. + CharSourceRange getExpansionRange(CharSourceRange Range) const { + CharSourceRange Expansion = getExpansionRange(Range.getAsRange()); + if (Expansion.getEnd() == Range.getEnd()) + Expansion.setTokenRange(Range.isTokenRange()); + return Expansion; } /// \brief Given a SourceLocation object, return the spelling @@ -1643,7 +1682,7 @@ public: // Otherwise, the caller of the macro is located where this macro is // expanded (while the spelling is part of the macro definition). - return getImmediateExpansionRange(Loc).first; + return getImmediateExpansionRange(Loc).getBegin(); } /// \return Location of the top-level macro caller. diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index d58849654cb8..15ef8300d0aa 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -336,13 +336,23 @@ public: const SourceManager &SM, const LangOptions &LangOpts); + /// Get the physical length (including trigraphs and escaped newlines) of the + /// first \p Characters characters of the token starting at TokStart. + static unsigned getTokenPrefixLength(SourceLocation TokStart, + unsigned Characters, + const SourceManager &SM, + const LangOptions &LangOpts); + /// AdvanceToTokenCharacter - If the current SourceLocation specifies a /// location at the start of a token, return a new location that specifies a /// character within the token. This handles trigraphs and escaped newlines. static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, - unsigned Character, + unsigned Characters, const SourceManager &SM, - const LangOptions &LangOpts); + const LangOptions &LangOpts) { + return TokStart.getLocWithOffset( + getTokenPrefixLength(TokStart, Characters, SM, LangOpts)); + } /// \brief Computes the source location just past the end of the /// token at this source location. diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 630b87ced358..49d5da731a15 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -1616,6 +1616,11 @@ public: SourceLocation ExpansionLocStart = SourceLocation(), SourceLocation ExpansionLocEnd = SourceLocation()); + /// Split the first Length characters out of the token starting at TokLoc + /// and return a location pointing to the split token. Re-lexing from the + /// split token will return the split token rather than the original. + SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length); + /// \brief Computes the source location just past the end of the /// token at this source location. /// diff --git a/clang/include/clang/Rewrite/Core/HTMLRewrite.h b/clang/include/clang/Rewrite/Core/HTMLRewrite.h index 1fd7c7a3f84e..0f1f490d8305 100644 --- a/clang/include/clang/Rewrite/Core/HTMLRewrite.h +++ b/clang/include/clang/Rewrite/Core/HTMLRewrite.h @@ -31,7 +31,8 @@ namespace html { /// start/end tags are placed at the start/end of each line if the range is /// multiline. void HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E, - const char *StartTag, const char *EndTag); + const char *StartTag, const char *EndTag, + bool IsTokenRange = true); /// HighlightRange - Highlight a range in the source code with the specified /// start/end tags. The Start/end of the range must be in the same file. diff --git a/clang/lib/ARCMigrate/PlistReporter.cpp b/clang/lib/ARCMigrate/PlistReporter.cpp index 9a51690c0ce7..2ad1c8591a9e 100644 --- a/clang/lib/ARCMigrate/PlistReporter.cpp +++ b/clang/lib/ARCMigrate/PlistReporter.cpp @@ -107,8 +107,7 @@ void arcmt::writeARCDiagsToPlist(const std::string &outPath, o << " ranges\n"; o << " \n"; for (auto &R : D.getRanges()) { - CharSourceRange ExpansionRange(SM.getExpansionRange(R.getAsRange()), - R.isTokenRange()); + CharSourceRange ExpansionRange = SM.getExpansionRange(R); EmitRange(o, SM, Lexer::getAsCharRange(ExpansionRange, SM, LangOpts), FM, 4); } diff --git a/clang/lib/ARCMigrate/TransGCAttrs.cpp b/clang/lib/ARCMigrate/TransGCAttrs.cpp index 2ae6b78a4634..a8ca14b19bb8 100644 --- a/clang/lib/ARCMigrate/TransGCAttrs.cpp +++ b/clang/lib/ARCMigrate/TransGCAttrs.cpp @@ -92,7 +92,7 @@ public: ASTContext &Ctx = MigrateCtx.Pass.Ctx; SourceManager &SM = Ctx.getSourceManager(); if (Loc.isMacroID()) - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); SmallString<32> Buf; bool Invalid = false; StringRef Spell = Lexer::getSpelling( @@ -287,7 +287,8 @@ static void checkAllAtProps(MigrationContext &MigrateCtx, SourceLocation Loc = ATLs[i].first.getAttrNameLoc(); if (Loc.isMacroID()) Loc = MigrateCtx.Pass.Ctx.getSourceManager() - .getImmediateExpansionRange(Loc).first; + .getImmediateExpansionRange(Loc) + .getBegin(); TA.remove(Loc); TA.clearDiagnostic(diag::err_objc_property_attr_mutually_exclusive, AtLoc); TA.clearDiagnostic(diag::err_arc_inconsistent_property_ownership, diff --git a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp b/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp index 7ca49558a7f0..de52bef4d206 100644 --- a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp +++ b/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp @@ -283,13 +283,12 @@ private: SourceManager &SM = Pass.Ctx.getSourceManager(); SourceLocation Loc = E->getExprLoc(); assert(Loc.isMacroID()); - SourceLocation MacroBegin, MacroEnd; - std::tie(MacroBegin, MacroEnd) = SM.getImmediateExpansionRange(Loc); + CharSourceRange MacroRange = SM.getImmediateExpansionRange(Loc); SourceRange SubRange = E->getSubExpr()->IgnoreParenImpCasts()->getSourceRange(); SourceLocation InnerBegin = SM.getImmediateMacroCallerLoc(SubRange.getBegin()); SourceLocation InnerEnd = SM.getImmediateMacroCallerLoc(SubRange.getEnd()); - Outer = SourceRange(MacroBegin, MacroEnd); + Outer = MacroRange.getAsRange(); Inner = SourceRange(InnerBegin, InnerEnd); } diff --git a/clang/lib/ARCMigrate/TransformActions.cpp b/clang/lib/ARCMigrate/TransformActions.cpp index 4f3fb5845925..7439eb285943 100644 --- a/clang/lib/ARCMigrate/TransformActions.cpp +++ b/clang/lib/ARCMigrate/TransformActions.cpp @@ -590,8 +590,12 @@ StringRef TransformActionsImpl::getUniqueText(StringRef text) { SourceLocation TransformActionsImpl::getLocForEndOfToken(SourceLocation loc, SourceManager &SM, Preprocessor &PP) { - if (loc.isMacroID()) - loc = SM.getExpansionRange(loc).second; + if (loc.isMacroID()) { + CharSourceRange Exp = SM.getExpansionRange(loc); + if (Exp.isCharRange()) + return Exp.getEnd(); + loc = Exp.getEnd(); + } return PP.getLocForEndOfToken(loc); } diff --git a/clang/lib/Basic/SourceLocation.cpp b/clang/lib/Basic/SourceLocation.cpp index 71f74e4d1aee..fef1f44fc8a5 100644 --- a/clang/lib/Basic/SourceLocation.cpp +++ b/clang/lib/Basic/SourceLocation.cpp @@ -103,15 +103,6 @@ FullSourceLoc FullSourceLoc::getFileLoc() const { return FullSourceLoc(SrcMgr->getFileLoc(*this), *SrcMgr); } -std::pair -FullSourceLoc::getImmediateExpansionRange() const { - assert(isValid()); - std::pair Range = - SrcMgr->getImmediateExpansionRange(*this); - return std::make_pair(FullSourceLoc(Range.first, *SrcMgr), - FullSourceLoc(Range.second, *SrcMgr)); -} - PresumedLoc FullSourceLoc::getPresumedLoc(bool UseLineDirectives) const { if (!isValid()) return PresumedLoc(); @@ -154,15 +145,6 @@ unsigned FullSourceLoc::getColumnNumber(bool *Invalid) const { return SrcMgr->getColumnNumber(getFileID(), getFileOffset(), Invalid); } -std::pair -FullSourceLoc::getExpansionRange() const { - assert(isValid()); - std::pair Range = - SrcMgr->getExpansionRange(*this); - return std::make_pair(FullSourceLoc(Range.first, *SrcMgr), - FullSourceLoc(Range.second, *SrcMgr)); -} - const FileEntry *FullSourceLoc::getFileEntry() const { assert(isValid()); return SrcMgr->getFileEntryForID(getFileID()); diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp index 3df2015bfe54..d5e71e9363d6 100644 --- a/clang/lib/Basic/SourceManager.cpp +++ b/clang/lib/Basic/SourceManager.cpp @@ -579,13 +579,24 @@ SourceManager::createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, + bool ExpansionIsTokenRange, int LoadedID, unsigned LoadedOffset) { - ExpansionInfo Info = ExpansionInfo::create(SpellingLoc, ExpansionLocStart, - ExpansionLocEnd); + ExpansionInfo Info = ExpansionInfo::create( + SpellingLoc, ExpansionLocStart, ExpansionLocEnd, ExpansionIsTokenRange); return createExpansionLocImpl(Info, TokLength, LoadedID, LoadedOffset); } +SourceLocation SourceManager::createTokenSplitLoc(SourceLocation Spelling, + SourceLocation TokenStart, + SourceLocation TokenEnd) { + assert(getFileID(TokenStart) == getFileID(TokenEnd) && + "token spans multiple files"); + return createExpansionLocImpl( + ExpansionInfo::createForTokenSplit(Spelling, TokenStart, TokenEnd), + TokenEnd.getOffset() - TokenStart.getOffset()); +} + SourceLocation SourceManager::createExpansionLocImpl(const ExpansionInfo &Info, unsigned TokLength, @@ -895,7 +906,7 @@ SourceLocation SourceManager::getFileLocSlowCase(SourceLocation Loc) const { if (isMacroArgExpansion(Loc)) Loc = getImmediateSpellingLoc(Loc); else - Loc = getImmediateExpansionRange(Loc).first; + Loc = getImmediateExpansionRange(Loc).getBegin(); } while (!Loc.isFileID()); return Loc; } @@ -950,7 +961,7 @@ SourceLocation SourceManager::getImmediateSpellingLoc(SourceLocation Loc) const{ /// getImmediateExpansionRange - Loc is required to be an expansion location. /// Return the start/end of the expansion information. -std::pair +CharSourceRange SourceManager::getImmediateExpansionRange(SourceLocation Loc) const { assert(Loc.isMacroID() && "Not a macro expansion loc!"); const ExpansionInfo &Expansion = getSLocEntry(getFileID(Loc)).getExpansion(); @@ -965,19 +976,21 @@ SourceLocation SourceManager::getTopMacroCallerLoc(SourceLocation Loc) const { /// getExpansionRange - Given a SourceLocation object, return the range of /// tokens covered by the expansion in the ultimate file. -std::pair -SourceManager::getExpansionRange(SourceLocation Loc) const { - if (Loc.isFileID()) return std::make_pair(Loc, Loc); +CharSourceRange SourceManager::getExpansionRange(SourceLocation Loc) const { + if (Loc.isFileID()) + return CharSourceRange(SourceRange(Loc, Loc), true); - std::pair Res = - getImmediateExpansionRange(Loc); + CharSourceRange Res = getImmediateExpansionRange(Loc); // Fully resolve the start and end locations to their ultimate expansion // points. - while (!Res.first.isFileID()) - Res.first = getImmediateExpansionRange(Res.first).first; - while (!Res.second.isFileID()) - Res.second = getImmediateExpansionRange(Res.second).second; + while (!Res.getBegin().isFileID()) + Res.setBegin(getImmediateExpansionRange(Res.getBegin()).getBegin()); + while (!Res.getEnd().isFileID()) { + CharSourceRange EndRange = getImmediateExpansionRange(Res.getEnd()); + Res.setEnd(EndRange.getEnd()); + Res.setTokenRange(EndRange.isTokenRange()); + } return Res; } diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index c8a32d4f28e1..97139ef69bcf 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -183,7 +183,7 @@ public: /// \brief Find out where the current file is included or macro is expanded. SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc) { - return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).first + return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).getBegin() : SM.getIncludeLoc(SM.getFileID(Loc)); } @@ -206,7 +206,7 @@ public: SourceLocation getStart(const Stmt *S) { SourceLocation Loc = S->getLocStart(); while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); return Loc; } @@ -214,7 +214,7 @@ public: SourceLocation getEnd(const Stmt *S) { SourceLocation Loc = S->getLocEnd(); while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); return getPreciseTokenLocEnd(Loc); } diff --git a/clang/lib/Edit/EditedSource.cpp b/clang/lib/Edit/EditedSource.cpp index 0ff2447f77a5..89a3eb40d083 100644 --- a/clang/lib/Edit/EditedSource.cpp +++ b/clang/lib/Edit/EditedSource.cpp @@ -36,12 +36,14 @@ void EditedSource::deconstructMacroArgLoc(SourceLocation Loc, SourceLocation &ExpansionLoc, MacroArgUse &ArgUse) { assert(SourceMgr.isMacroArgExpansion(Loc)); - SourceLocation DefArgLoc = SourceMgr.getImmediateExpansionRange(Loc).first; + SourceLocation DefArgLoc = + SourceMgr.getImmediateExpansionRange(Loc).getBegin(); SourceLocation ImmediateExpansionLoc = - SourceMgr.getImmediateExpansionRange(DefArgLoc).first; + SourceMgr.getImmediateExpansionRange(DefArgLoc).getBegin(); ExpansionLoc = ImmediateExpansionLoc; while (SourceMgr.isMacroBodyExpansion(ExpansionLoc)) - ExpansionLoc = SourceMgr.getImmediateExpansionRange(ExpansionLoc).first; + ExpansionLoc = + SourceMgr.getImmediateExpansionRange(ExpansionLoc).getBegin(); SmallString<20> Buf; StringRef ArgName = Lexer::getSpelling(SourceMgr.getSpellingLoc(DefArgLoc), Buf, SourceMgr, LangOpts); diff --git a/clang/lib/Frontend/DiagnosticRenderer.cpp b/clang/lib/Frontend/DiagnosticRenderer.cpp index 8b607a631984..a3bcaf47fa63 100644 --- a/clang/lib/Frontend/DiagnosticRenderer.cpp +++ b/clang/lib/Frontend/DiagnosticRenderer.cpp @@ -262,41 +262,54 @@ static SourceLocation retrieveMacroLocation(SourceLocation Loc, FileID MacroFileID, FileID CaretFileID, const SmallVectorImpl &CommonArgExpansions, - bool IsBegin, const SourceManager *SM) { + bool IsBegin, const SourceManager *SM, + bool &IsTokenRange) { assert(SM->getFileID(Loc) == MacroFileID); if (MacroFileID == CaretFileID) return Loc; if (!Loc.isMacroID()) return {}; - SourceLocation MacroLocation, MacroArgLocation; + CharSourceRange MacroRange, MacroArgRange; if (SM->isMacroArgExpansion(Loc)) { // Only look at the immediate spelling location of this macro argument if // the other location in the source range is also present in that expansion. if (std::binary_search(CommonArgExpansions.begin(), CommonArgExpansions.end(), MacroFileID)) - MacroLocation = SM->getImmediateSpellingLoc(Loc); - MacroArgLocation = IsBegin ? SM->getImmediateExpansionRange(Loc).first - : SM->getImmediateExpansionRange(Loc).second; + MacroRange = + CharSourceRange(SM->getImmediateSpellingLoc(Loc), IsTokenRange); + MacroArgRange = SM->getImmediateExpansionRange(Loc); } else { - MacroLocation = IsBegin ? SM->getImmediateExpansionRange(Loc).first - : SM->getImmediateExpansionRange(Loc).second; - MacroArgLocation = SM->getImmediateSpellingLoc(Loc); + MacroRange = SM->getImmediateExpansionRange(Loc); + MacroArgRange = + CharSourceRange(SM->getImmediateSpellingLoc(Loc), IsTokenRange); } + SourceLocation MacroLocation = + IsBegin ? MacroRange.getBegin() : MacroRange.getEnd(); if (MacroLocation.isValid()) { MacroFileID = SM->getFileID(MacroLocation); + bool TokenRange = IsBegin ? IsTokenRange : MacroRange.isTokenRange(); MacroLocation = retrieveMacroLocation(MacroLocation, MacroFileID, CaretFileID, - CommonArgExpansions, IsBegin, SM); - if (MacroLocation.isValid()) + CommonArgExpansions, IsBegin, SM, TokenRange); + if (MacroLocation.isValid()) { + IsTokenRange = TokenRange; return MacroLocation; + } } + // If we moved the end of the range to an expansion location, we now have + // a range of the same kind as the expansion range. + if (!IsBegin) + IsTokenRange = MacroArgRange.isTokenRange(); + + SourceLocation MacroArgLocation = + IsBegin ? MacroArgRange.getBegin() : MacroArgRange.getEnd(); MacroFileID = SM->getFileID(MacroArgLocation); return retrieveMacroLocation(MacroArgLocation, MacroFileID, CaretFileID, - CommonArgExpansions, IsBegin, SM); + CommonArgExpansions, IsBegin, SM, IsTokenRange); } /// Walk up the chain of macro expansions and collect the FileIDs identifying the @@ -310,7 +323,7 @@ static void getMacroArgExpansionFileIDs(SourceLocation Loc, Loc = SM->getImmediateSpellingLoc(Loc); } else { auto ExpRange = SM->getImmediateExpansionRange(Loc); - Loc = IsBegin ? ExpRange.first : ExpRange.second; + Loc = IsBegin ? ExpRange.getBegin() : ExpRange.getEnd(); } } } @@ -364,14 +377,16 @@ mapDiagnosticRanges(FullSourceLoc CaretLoc, ArrayRef Ranges, llvm::SmallDenseMap BeginLocsMap; while (Begin.isMacroID() && BeginFileID != EndFileID) { BeginLocsMap[BeginFileID] = Begin; - Begin = SM->getImmediateExpansionRange(Begin).first; + Begin = SM->getImmediateExpansionRange(Begin).getBegin(); BeginFileID = SM->getFileID(Begin); } // Then, crawl the expansion chain for the end of the range. if (BeginFileID != EndFileID) { while (End.isMacroID() && !BeginLocsMap.count(EndFileID)) { - End = SM->getImmediateExpansionRange(End).second; + auto Exp = SM->getImmediateExpansionRange(End); + IsTokenRange = Exp.isTokenRange(); + End = Exp.getEnd(); EndFileID = SM->getFileID(End); } if (End.isMacroID()) { @@ -384,9 +399,11 @@ mapDiagnosticRanges(FullSourceLoc CaretLoc, ArrayRef Ranges, SmallVector CommonArgExpansions; computeCommonMacroArgExpansionFileIDs(Begin, End, SM, CommonArgExpansions); Begin = retrieveMacroLocation(Begin, BeginFileID, CaretLocFileID, - CommonArgExpansions, /*IsBegin=*/true, SM); + CommonArgExpansions, /*IsBegin=*/true, SM, + IsTokenRange); End = retrieveMacroLocation(End, BeginFileID, CaretLocFileID, - CommonArgExpansions, /*IsBegin=*/false, SM); + CommonArgExpansions, /*IsBegin=*/false, SM, + IsTokenRange); if (Begin.isInvalid() || End.isInvalid()) continue; // Return the spelling location of the beginning and end of the range. @@ -511,29 +528,31 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc, ArrayRef Ranges, ArrayRef Hints) { assert(Loc.isValid() && "must have a valid source location here"); + const SourceManager &SM = Loc.getManager(); + SourceLocation L = Loc; // Produce a stack of macro backtraces. - SmallVector LocationStack; + SmallVector LocationStack; unsigned IgnoredEnd = 0; - while (Loc.isMacroID()) { + while (L.isMacroID()) { // If this is the expansion of a macro argument, point the caret at the // use of the argument in the definition of the macro, not the expansion. - if (Loc.isMacroArgExpansion()) - LocationStack.push_back(Loc.getImmediateExpansionRange().first); + if (SM.isMacroArgExpansion(L)) + LocationStack.push_back(SM.getImmediateExpansionRange(L).getBegin()); else - LocationStack.push_back(Loc); + LocationStack.push_back(L); - if (checkRangesForMacroArgExpansion(Loc, Ranges)) + if (checkRangesForMacroArgExpansion(FullSourceLoc(L, SM), Ranges)) IgnoredEnd = LocationStack.size(); - Loc = Loc.getImmediateMacroCallerLoc(); + L = SM.getImmediateMacroCallerLoc(L); // Once the location no longer points into a macro, try stepping through // the last found location. This sometimes produces additional useful // backtraces. - if (Loc.isFileID()) - Loc = LocationStack.back().getImmediateMacroCallerLoc(); - assert(Loc.isValid() && "must have a valid source location here"); + if (L.isFileID()) + L = SM.getImmediateMacroCallerLoc(LocationStack.back()); + assert(L.isValid() && "must have a valid source location here"); } LocationStack.erase(LocationStack.begin(), @@ -544,7 +563,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc, if (MacroDepth <= MacroLimit || MacroLimit == 0) { for (auto I = LocationStack.rbegin(), E = LocationStack.rend(); I != E; ++I) - emitSingleMacroExpansion(*I, Level, Ranges); + emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges); return; } @@ -554,7 +573,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc, for (auto I = LocationStack.rbegin(), E = LocationStack.rbegin() + MacroStartMessages; I != E; ++I) - emitSingleMacroExpansion(*I, Level, Ranges); + emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges); SmallString<200> MessageStorage; llvm::raw_svector_ostream Message(MessageStorage); @@ -566,7 +585,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc, for (auto I = LocationStack.rend() - MacroEndMessages, E = LocationStack.rend(); I != E; ++I) - emitSingleMacroExpansion(*I, Level, Ranges); + emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges); } DiagnosticNoteRenderer::~DiagnosticNoteRenderer() = default; diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp index 6a72b00c602b..9bda79a276f1 100644 --- a/clang/lib/Frontend/TextDiagnostic.cpp +++ b/clang/lib/Frontend/TextDiagnostic.cpp @@ -852,23 +852,14 @@ void TextDiagnostic::emitDiagnosticLoc(FullSourceLoc Loc, PresumedLoc PLoc, // Ignore invalid ranges. if (!RI->isValid()) continue; - FullSourceLoc B = - FullSourceLoc(RI->getBegin(), Loc.getManager()).getExpansionLoc(); - FullSourceLoc E = - FullSourceLoc(RI->getEnd(), Loc.getManager()).getExpansionLoc(); + auto &SM = Loc.getManager(); + SourceLocation B = SM.getExpansionLoc(RI->getBegin()); + CharSourceRange ERange = SM.getExpansionRange(RI->getEnd()); + SourceLocation E = ERange.getEnd(); + bool IsTokenRange = ERange.isTokenRange(); - // If the End location and the start location are the same and are a - // macro location, then the range was something that came from a - // macro expansion or _Pragma. If this is an object-like macro, the - // best we can do is to highlight the range. If this is a - // function-like macro, we'd also like to highlight the arguments. - if (B == E && RI->getEnd().isMacroID()) - E = FullSourceLoc(RI->getEnd(), Loc.getManager()) - .getExpansionRange() - .second; - - std::pair BInfo = B.getDecomposedLoc(); - std::pair EInfo = E.getDecomposedLoc(); + std::pair BInfo = SM.getDecomposedLoc(B); + std::pair EInfo = SM.getDecomposedLoc(E); // If the start or end of the range is in another file, just discard // it. @@ -878,11 +869,14 @@ void TextDiagnostic::emitDiagnosticLoc(FullSourceLoc Loc, PresumedLoc PLoc, // Add in the length of the token, so that we cover multi-char // tokens. unsigned TokSize = 0; - if (RI->isTokenRange()) - TokSize = Lexer::MeasureTokenLength(E, E.getManager(), LangOpts); + if (IsTokenRange) + TokSize = Lexer::MeasureTokenLength(E, SM, LangOpts); - OS << '{' << B.getLineNumber() << ':' << B.getColumnNumber() << '-' - << E.getLineNumber() << ':' << (E.getColumnNumber() + TokSize) << '}'; + FullSourceLoc BF(B, SM), EF(E, SM); + OS << '{' + << BF.getLineNumber() << ':' << BF.getColumnNumber() << '-' + << EF.getLineNumber() << ':' << (EF.getColumnNumber() + TokSize) + << '}'; PrintedRange = true; } diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 9179650187e3..e9478d603a52 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -708,12 +708,9 @@ PreambleBounds Lexer::ComputePreamble(StringRef Buffer, TheTok.isAtStartOfLine()); } -/// AdvanceToTokenCharacter - Given a location that specifies the start of a -/// token, return a new location that specifies a character within the token. -SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, - unsigned CharNo, - const SourceManager &SM, - const LangOptions &LangOpts) { +unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo, + const SourceManager &SM, + const LangOptions &LangOpts) { // Figure out how many physical characters away the specified expansion // character is. This needs to take into consideration newlines and // trigraphs. @@ -722,7 +719,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, // If they request the first char of the token, we're trivially done. if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) - return TokStart; + return 0; unsigned PhysOffset = 0; @@ -731,7 +728,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, // chars, this method is extremely fast. while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { if (CharNo == 0) - return TokStart.getLocWithOffset(PhysOffset); + return PhysOffset; ++TokPtr; --CharNo; ++PhysOffset; @@ -753,7 +750,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; - return TokStart.getLocWithOffset(PhysOffset); + return PhysOffset; } /// \brief Computes the source location just past the end of the @@ -987,7 +984,7 @@ StringRef Lexer::getImmediateMacroName(SourceLocation Loc, // Loc points to the argument id of the macro definition, move to the // macro expansion. - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); SourceLocation SpellLoc = Expansion.getSpellingLoc(); if (SpellLoc.isFileID()) break; // No inner macro. @@ -1020,7 +1017,7 @@ StringRef Lexer::getImmediateMacroNameForDiagnostics( assert(Loc.isMacroID() && "Only reasonable to call this on macros"); // Walk past macro argument expanions. while (SM.isMacroArgExpansion(Loc)) - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); // If the macro's spelling has no FileID, then it's actually a token paste // or stringization (or similar) and not a macro at all. @@ -1030,7 +1027,7 @@ StringRef Lexer::getImmediateMacroNameForDiagnostics( // Find the spelling location of the start of the non-argument expansion // range. This is where the macro name was spelled in order to begin // expanding this macro. - Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).first); + Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin()); // Dig out the buffer where the macro name was spelled and the extents of the // name so that we can render it into the expansion note. @@ -1112,10 +1109,9 @@ static SourceLocation GetMappedTokenLoc(Preprocessor &PP, // Figure out the expansion loc range, which is the range covered by the // original _Pragma(...) sequence. - std::pair II = - SM.getImmediateExpansionRange(FileLoc); + CharSourceRange II = SM.getImmediateExpansionRange(FileLoc); - return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); + return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen); } /// getSourceLocation - Return a source location identifier for the specified diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index e1a0438d1ddd..31c8e130c8b9 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -2043,7 +2043,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, // If the filename string was the result of macro expansions, set the include // position on the file where it will be included and after the expansions. if (IncludePos.isMacroID()) - IncludePos = SourceMgr.getExpansionRange(IncludePos).second; + IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); FileID FID = SourceMgr.createFileID(File, IncludePos, FileCharacter); assert(FID.isValid() && "Expected valid file ID"); diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp index aa8414f8589b..edf508a79b3c 100644 --- a/clang/lib/Lex/PPMacroExpansion.cpp +++ b/clang/lib/Lex/PPMacroExpansion.cpp @@ -1690,7 +1690,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { // can matter for a function-like macro that expands to contain __LINE__. // Skip down through expansion points until we find a file loc for the // end of the expansion history. - Loc = SourceMgr.getExpansionRange(Loc).second; + Loc = SourceMgr.getExpansionRange(Loc).getEnd(); PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc); // __LINE__ expands to a simple numeric value. diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 4370bc10e348..94490bdf3fbf 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -487,6 +487,22 @@ void Preprocessor::CreateString(StringRef Str, Token &Tok, Tok.setLiteralData(DestPtr); } +SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) { + auto &SM = getSourceManager(); + SourceLocation SpellingLoc = SM.getSpellingLoc(Loc); + std::pair LocInfo = SM.getDecomposedLoc(SpellingLoc); + bool Invalid = false; + StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); + if (Invalid) + return SourceLocation(); + + // FIXME: We could consider re-using spelling for tokens we see repeatedly. + const char *DestPtr; + SourceLocation Spelling = + ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr); + return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length)); +} + Module *Preprocessor::getCurrentModule() { if (!getLangOpts().isCompilingModule()) return nullptr; diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp index 6bf0b84e5905..724ccb393837 100644 --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -865,9 +865,9 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, EndLoc = getExpansionLocForMacroDefLoc(EndLoc); FileID MacroFID = SM.getFileID(MacroExpansionStart); while (SM.getFileID(StartLoc) != MacroFID) - StartLoc = SM.getImmediateExpansionRange(StartLoc).first; + StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin(); while (SM.getFileID(EndLoc) != MacroFID) - EndLoc = SM.getImmediateExpansionRange(EndLoc).second; + EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd(); LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc, LHSTok.getLength())); diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp index 88a5745350d6..d089b245dffe 100644 --- a/clang/lib/Parse/ParseTemplate.cpp +++ b/clang/lib/Parse/ParseTemplate.cpp @@ -775,6 +775,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc, // What will be left once we've consumed the '>'. tok::TokenKind RemainingToken; const char *ReplacementStr = "> >"; + bool MergeWithNextToken = false; switch (Tok.getKind()) { default: @@ -800,6 +801,15 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc, case tok::greaterequal: RemainingToken = tok::equal; ReplacementStr = "> ="; + + // Join two adjacent '=' tokens into one, for cases like: + // void (*p)() = f; + // return f==p; + if (NextToken().is(tok::equal) && + areTokensAdjacent(Tok, NextToken())) { + RemainingToken = tok::equalequal; + MergeWithNextToken = true; + } break; case tok::greatergreaterequal: @@ -807,22 +817,35 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc, break; } - // This template-id is terminated by a token which starts with a '>'. Outside - // C++11, this is now error recovery, and in C++11, this is error recovery if - // the token isn't '>>' or '>>>'. - // '>>>' is for CUDA, where this sequence of characters is parsed into - // tok::greatergreatergreater, rather than two separate tokens. + // This template-id is terminated by a token that starts with a '>'. + // Outside C++11 and Objective-C, this is now error recovery. // - // We always allow this for Objective-C type parameter and type argument - // lists. - RAngleLoc = Tok.getLocation(); + // C++11 allows this when the token is '>>', and in CUDA + C++11 mode, we + // extend that treatment to also apply to the '>>>' token. + // + // Objective-C allows this in its type parameter / argument lists. + + SourceLocation TokBeforeGreaterLoc = PrevTokLocation; + SourceLocation TokLoc = Tok.getLocation(); Token Next = NextToken(); + + // Whether splitting the current token after the '>' would undesirably result + // in the remaining token pasting with the token after it. This excludes the + // MergeWithNextToken cases, which we've already handled. + bool PreventMergeWithNextToken = + (RemainingToken == tok::greater || + RemainingToken == tok::greatergreater) && + (Next.isOneOf(tok::greater, tok::greatergreater, + tok::greatergreatergreater, tok::equal, tok::greaterequal, + tok::greatergreaterequal, tok::equalequal)) && + areTokensAdjacent(Tok, Next); + + // Diagnose this situation as appropriate. if (!ObjCGenericList) { - // The source range of the '>>' or '>=' at the start of the token. - CharSourceRange ReplacementRange = - CharSourceRange::getCharRange(RAngleLoc, - Lexer::AdvanceToTokenCharacter(RAngleLoc, 2, PP.getSourceManager(), - getLangOpts())); + // The source range of the replaced token(s). + CharSourceRange ReplacementRange = CharSourceRange::getCharRange( + TokLoc, Lexer::AdvanceToTokenCharacter(TokLoc, 2, PP.getSourceManager(), + getLangOpts())); // A hint to put a space between the '>>'s. In order to make the hint as // clear as possible, we include the characters either side of the space in @@ -833,13 +856,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc, // A hint to put another space after the token, if it would otherwise be // lexed differently. FixItHint Hint2; - if ((RemainingToken == tok::greater || - RemainingToken == tok::greatergreater) && - (Next.isOneOf(tok::greater, tok::greatergreater, - tok::greatergreatergreater, tok::equal, - tok::greaterequal, tok::greatergreaterequal, - tok::equalequal)) && - areTokensAdjacent(Tok, Next)) + if (PreventMergeWithNextToken) Hint2 = FixItHint::CreateInsertion(Next.getLocation(), " "); unsigned DiagId = diag::err_two_right_angle_brackets_need_space; @@ -848,50 +865,63 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc, DiagId = diag::warn_cxx98_compat_two_right_angle_brackets; else if (Tok.is(tok::greaterequal)) DiagId = diag::err_right_angle_bracket_equal_needs_space; - Diag(Tok.getLocation(), DiagId) << Hint1 << Hint2; + Diag(TokLoc, DiagId) << Hint1 << Hint2; } + // Find the "length" of the resulting '>' token. This is not always 1, as it + // can contain escaped newlines. + unsigned GreaterLength = Lexer::getTokenPrefixLength( + TokLoc, 1, PP.getSourceManager(), getLangOpts()); + + // Annotate the source buffer to indicate that we split the token after the + // '>'. This allows us to properly find the end of, and extract the spelling + // of, the '>' token later. + RAngleLoc = PP.SplitToken(TokLoc, GreaterLength); + // Strip the initial '>' from the token. - Token PrevTok = Tok; - if (RemainingToken == tok::equal && Next.is(tok::equal) && - areTokensAdjacent(Tok, Next)) { - // Join two adjacent '=' tokens into one, for cases like: - // void (*p)() = f; - // return f==p; + bool CachingTokens = PP.IsPreviousCachedToken(Tok); + + Token Greater = Tok; + Greater.setLocation(RAngleLoc); + Greater.setKind(tok::greater); + Greater.setLength(GreaterLength); + + unsigned OldLength = Tok.getLength(); + if (MergeWithNextToken) { ConsumeToken(); - Tok.setKind(tok::equalequal); - Tok.setLength(Tok.getLength() + 1); - } else { - Tok.setKind(RemainingToken); - Tok.setLength(Tok.getLength() - 1); + OldLength += Tok.getLength(); } - Tok.setLocation(Lexer::AdvanceToTokenCharacter(RAngleLoc, 1, - PP.getSourceManager(), - getLangOpts())); - // The advance from '>>' to '>' in a ObjectiveC template argument list needs - // to be properly reflected in the token cache to allow correct interaction - // between annotation and backtracking. - if (ObjCGenericList && PrevTok.getKind() == tok::greatergreater && - RemainingToken == tok::greater && PP.IsPreviousCachedToken(PrevTok)) { - PrevTok.setKind(RemainingToken); - PrevTok.setLength(1); - // Break tok::greatergreater into two tok::greater but only add the second - // one in case the client asks to consume the last token. + Tok.setKind(RemainingToken); + Tok.setLength(OldLength - GreaterLength); + + // Split the second token if lexing it normally would lex a different token + // (eg, the fifth token in 'A>>' should re-lex as '>', not '>>'). + SourceLocation AfterGreaterLoc = TokLoc.getLocWithOffset(GreaterLength); + if (PreventMergeWithNextToken) + AfterGreaterLoc = PP.SplitToken(AfterGreaterLoc, Tok.getLength()); + Tok.setLocation(AfterGreaterLoc); + + // Update the token cache to match what we just did if necessary. + if (CachingTokens) { + // If the previous cached token is being merged, delete it. + if (MergeWithNextToken) + PP.ReplacePreviousCachedToken({}); + if (ConsumeLastToken) - PP.ReplacePreviousCachedToken({PrevTok, Tok}); + PP.ReplacePreviousCachedToken({Greater, Tok}); else - PP.ReplacePreviousCachedToken({PrevTok}); + PP.ReplacePreviousCachedToken({Greater}); } - if (!ConsumeLastToken) { - // Since we're not supposed to consume the '>' token, we need to push - // this token and revert the current token back to the '>'. + if (ConsumeLastToken) { + PrevTokLocation = RAngleLoc; + } else { + PrevTokLocation = TokBeforeGreaterLoc; PP.EnterToken(Tok); - Tok.setKind(tok::greater); - Tok.setLength(1); - Tok.setLocation(RAngleLoc); + Tok = Greater; } + return false; } diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp index 3a086537b4a9..d93961f3582e 100644 --- a/clang/lib/Rewrite/HTMLRewrite.cpp +++ b/clang/lib/Rewrite/HTMLRewrite.cpp @@ -30,7 +30,8 @@ using namespace clang; /// start/end tags are placed at the start/end of each line if the range is /// multiline. void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E, - const char *StartTag, const char *EndTag) { + const char *StartTag, const char *EndTag, + bool IsTokenRange) { SourceManager &SM = R.getSourceMgr(); B = SM.getExpansionLoc(B); E = SM.getExpansionLoc(E); @@ -41,7 +42,8 @@ void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E, unsigned EOffset = SM.getFileOffset(E); // Include the whole end token in the range. - EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts()); + if (IsTokenRange) + EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts()); bool Invalid = false; const char *BufferStart = SM.getBufferData(FID, &Invalid).data(); @@ -588,16 +590,15 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) { // Okay, we have the first token of a macro expansion: highlight the // expansion by inserting a start tag before the macro expansion and // end tag after it. - std::pair LLoc = - SM.getExpansionRange(Tok.getLocation()); + CharSourceRange LLoc = SM.getExpansionRange(Tok.getLocation()); // Ignore tokens whose instantiation location was not the main file. - if (SM.getFileID(LLoc.first) != FID) { + if (SM.getFileID(LLoc.getBegin()) != FID) { TmpPP.Lex(Tok); continue; } - assert(SM.getFileID(LLoc.second) == FID && + assert(SM.getFileID(LLoc.getEnd()) == FID && "Start and end of expansion must be in the same ultimate file!"); std::string Expansion = EscapeText(TmpPP.getSpelling(Tok)); @@ -612,7 +613,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) { // instantiation. It would be really nice to pop up a window with all the // spelling of the tokens or something. while (!Tok.is(tok::eof) && - SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) { + SM.getExpansionLoc(Tok.getLocation()) == LLoc.getBegin()) { // Insert a newline if the macro expansion is getting large. if (LineLen > 60) { Expansion += "
"; @@ -641,8 +642,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) { // highlighted. Expansion = "" + Expansion + ""; - HighlightRange(R, LLoc.first, LLoc.second, - "", Expansion.c_str()); + HighlightRange(R, LLoc.getBegin(), LLoc.getEnd(), "", + Expansion.c_str(), LLoc.isTokenRange()); } // Restore the preprocessor's old state. diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 9e7f5cae2878..7b3861224148 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -9562,7 +9562,7 @@ static void DiagnoseNullConversion(Sema &S, Expr *E, QualType T, StringRef MacroName = Lexer::getImmediateMacroNameForDiagnostics( Loc, S.SourceMgr, S.getLangOpts()); if (MacroName == "NULL") - Loc = S.SourceMgr.getImmediateExpansionRange(Loc).first; + Loc = S.SourceMgr.getImmediateExpansionRange(Loc).getBegin(); } // Only warn if the null and context location are in the same macro expansion. diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 74a79252792f..fd1cfc14d875 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -7533,7 +7533,7 @@ void DiagnoseUnguardedAvailability::DiagnoseDeclAvailability( SourceLocation StmtEndLoc = SM.getExpansionRange( (LastStmtOfUse ? LastStmtOfUse : StmtOfUse)->getLocEnd()) - .second; + .getEnd(); if (SM.getFileID(IfInsertionLoc) != SM.getFileID(StmtEndLoc)) return; diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 678f6af068ec..2cf16221f755 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -2771,7 +2771,7 @@ void Sema::DiagnoseAbsenceOfOverrideControl(NamedDecl *D) { SourceLocation Loc = MD->getLocation(); SourceLocation SpellingLoc = Loc; if (getSourceManager().isMacroArgExpansion(Loc)) - SpellingLoc = getSourceManager().getImmediateExpansionRange(Loc).first; + SpellingLoc = getSourceManager().getImmediateExpansionRange(Loc).getBegin(); SpellingLoc = getSourceManager().getSpellingLoc(SpellingLoc); if (SpellingLoc.isValid() && getSourceManager().isInSystemHeader(SpellingLoc)) return; diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index cf98805ebd8b..e44eaa5162a3 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -6487,7 +6487,7 @@ static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr, // macro only if it is at the beginning of the macro. while (ArgLoc.isMacroID() && S.getSourceManager().isAtStartOfImmediateMacroExpansion(ArgLoc)) { - ArgLoc = S.getSourceManager().getImmediateExpansionRange(ArgLoc).first; + ArgLoc = S.getSourceManager().getImmediateExpansionRange(ArgLoc).getBegin(); } if (LParen.isMacroID()) diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 52a3d75bfb9b..0e71047f00e6 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -4489,7 +4489,8 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state, // the predefined // __strong/__weak/__autoreleasing/__unsafe_unretained. if (AttrLoc.isMacroID()) - AttrLoc = S.SourceMgr.getImmediateExpansionRange(AttrLoc).first; + AttrLoc = + S.SourceMgr.getImmediateExpansionRange(AttrLoc).getBegin(); S.Diag(AttrLoc, diag::warn_arc_lifetime_result_type) << T.getQualifiers().getObjCLifetime(); @@ -5893,7 +5894,8 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state, Sema &S = state.getSema(); SourceLocation AttrLoc = attr.getLoc(); if (AttrLoc.isMacroID()) - AttrLoc = S.getSourceManager().getImmediateExpansionRange(AttrLoc).first; + AttrLoc = + S.getSourceManager().getImmediateExpansionRange(AttrLoc).getBegin(); if (!attr.isArgIdent(0)) { S.Diag(AttrLoc, diag::err_attribute_argument_type) diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 9d10b9ebfbb8..a1de22b68ecb 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1482,6 +1482,7 @@ bool ASTReader::ReadSLocEntry(int ID) { SourceMgr.createExpansionLoc(SpellingLoc, ReadSourceLocation(*F, Record[2]), ReadSourceLocation(*F, Record[3]), + Record[5], Record[4], ID, BaseOffset + Record[0]); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 7f2b1861d111..fb7276b2f8d3 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1905,6 +1905,7 @@ static unsigned CreateSLocExpansionAbbrev(llvm::BitstreamWriter &Stream) { Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Spelling location Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Start location Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // End location + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Is token range Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Token length return Stream.EmitAbbrev(std::move(Abbrev)); } @@ -2321,6 +2322,7 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, ? SourceLocation() : Expansion.getExpansionLocEnd(), Record); + Record.push_back(Expansion.isExpansionTokenRange()); // Compute the token length for this macro expansion. unsigned NextOffset = SourceMgr.getNextLocalOffset(); diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp index c8289a708842..3a909564b2d3 100644 --- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp +++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp @@ -2178,7 +2178,7 @@ static void simplifySimpleBranches(PathPieces &pieces) { static Optional getLengthOnSingleLine(SourceManager &SM, SourceRange Range) { SourceRange ExpansionRange(SM.getExpansionLoc(Range.getBegin()), - SM.getExpansionRange(Range.getEnd()).second); + SM.getExpansionRange(Range.getEnd()).getEnd()); FileID FID = SM.getFileID(ExpansionRange.getBegin()); if (FID != SM.getFileID(ExpansionRange.getEnd())) diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp index 78d6ecdbc375..e171807d36bf 100644 --- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp +++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp @@ -213,7 +213,7 @@ static bool isFunctionMacroExpansion(SourceLocation Loc, if (!Loc.isMacroID()) return false; while (SM.isMacroArgExpansion(Loc)) - Loc = SM.getImmediateExpansionRange(Loc).first; + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); std::pair TLInfo = SM.getDecomposedLoc(Loc); SrcMgr::SLocEntry SE = SM.getSLocEntry(TLInfo.first); const SrcMgr::ExpansionInfo &EInfo = SE.getExpansion(); diff --git a/clang/test/Misc/diag-greatergreater.cpp b/clang/test/Misc/diag-greatergreater.cpp new file mode 100644 index 000000000000..9110838cf3f5 --- /dev/null +++ b/clang/test/Misc/diag-greatergreater.cpp @@ -0,0 +1,43 @@ +// RUN: not %clang_cc1 %s -fdiagnostics-print-source-range-info 2>&1 | FileCheck %s --strict-whitespace + +template class C {}; +template class D {}; + +void g() { + // The range ending in the first > character does not extend to the second > + // character. + // CHECK: :{[[@LINE+3]]:5-[[@LINE+3]]:11}: error: + // CHECK-NEXT: D> a; + // CHECK-NEXT: ^~~~~~{{$}} + D> a; + + // The range ending in the second > character does not extend to the third > + // character. + // CHECK: :{[[@LINE+3]]:5-[[@LINE+3]]:14}: error: + // CHECK-NEXT: D>> b; + // CHECK-NEXT: ^~~~~~~~~{{$}} + D>> b; +} + +template int V; +// Here, we split the >>= token into a > followed by a >=. +// Then we split the >= token into a > followed by an =, +// which we merge with the other = to form an ==. +// CHECK: error: a space is required +// CHECK-NEXT: int k = V>==0; +// CHECK-NEXT: ^~{{$}} +// CHECK-NEXT: > >{{$}} +// CHECK: error: a space is required +// CHECK-NEXT: int k = V>==0; +// CHECK-NEXT: ^~{{$}} +// CHECK-NEXT: > ={{$}} +// CHECK: :{[[@LINE+3]]:11-[[@LINE+3]]:17}: error: +// CHECK-NEXT: int k = V>==0; +// CHECK-NEXT: ^~~~~~{{$}} +int k = V>==0; + +template int W; +// CHECK: :{[[@LINE+3]]:9-[[@LINE+3]]:18}{[[@LINE+3]]:20-[[@LINE+3]]:22}: error: comparison +// CHECK-NEXT: int l = W>==&k; +// CHECK-NEXT: ~~~~~~~~~^ ~~{{$}} +int l = W>==&k; diff --git a/clang/test/Parser/cxx0x-decl.cpp b/clang/test/Parser/cxx0x-decl.cpp index 1eaf84a471e2..7bd82e8220be 100644 --- a/clang/test/Parser/cxx0x-decl.cpp +++ b/clang/test/Parser/cxx0x-decl.cpp @@ -130,15 +130,18 @@ namespace AliasDeclEndLocation { using B = AliasDeclEndLocation::A // expected-error {{expected ';' after alias declaration}} +; - // FIXME: After splitting this >> into two > tokens, we incorrectly determine - // the end of the template-id to be after the *second* '>'. - // Perhaps we could synthesize an expansion FileID containing '> >' to fix this? using C = AliasDeclEndLocation::A\ > // expected-error {{expected ';' after alias declaration}} ; using D = AliasDeclEndLocation::A // expected-error {{expected ';' after alias declaration}} + // FIXME: After splitting this >> into two > tokens, we incorrectly determine + // the end of the template-id to be after the *second* '>'. + using E = AliasDeclEndLocation::A>; +#define GGG >>> + using F = AliasDeclEndLocation::A - macroPair = SourceMgr.getExpansionRange(lsqrLoc); - SourceRange macroRange = SourceRange(macroPair.first, macroPair.second); + CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc); SourceLocation Loc; EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); @@ -297,6 +295,7 @@ TEST_F(LexerTest, LexAPI) { EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); EXPECT_EQ(Loc, macroRange.getEnd()); + EXPECT_TRUE(macroRange.isTokenRange()); CharSourceRange range = Lexer::makeFileCharRange( CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); @@ -334,11 +333,11 @@ TEST_F(LexerTest, LexAPI) { EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), range.getAsRange()); - macroPair = SourceMgr.getExpansionRange(macroLsqrLoc); + macroRange = SourceMgr.getExpansionRange(macroLsqrLoc); range = Lexer::makeFileCharRange( CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), SourceMgr, LangOpts); - EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)), + EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)), range.getAsRange()); text = Lexer::getSourceText(