From edbec459fb961ea16d349d77bf49c8fb53f202b2 Mon Sep 17 00:00:00 2001 From: peter klausler Date: Thu, 15 Feb 2018 13:13:28 -0800 Subject: [PATCH] [flang] Complete provenance tracking through macro calls. Original-commit: flang-compiler/f18@8c05a6543b647299104ae07a6a248b67f13ee24e Reviewed-on: https://github.com/flang-compiler/f18/pull/9 Tree-same-pre-rewrite: false --- flang/lib/parser/basic-parsers.h | 2 +- flang/lib/parser/cooked-tokens.h | 5 +++-- flang/lib/parser/message.cc | 12 ++++++---- flang/lib/parser/message.h | 3 ++- flang/lib/parser/preprocessor.cc | 32 +++++++++++++++++--------- flang/lib/parser/provenance.cc | 36 ++++++++++++++++++++++-------- flang/lib/parser/provenance.h | 12 +++++----- flang/lib/parser/source.h | 3 +++ flang/lib/parser/token-sequence.cc | 32 ++++++++++++++++++++++++-- flang/lib/parser/token-sequence.h | 9 ++++++-- flang/tools/f18/f18.cc | 10 ++++----- 11 files changed, 114 insertions(+), 42 deletions(-) diff --git a/flang/lib/parser/basic-parsers.h b/flang/lib/parser/basic-parsers.h index 076aef81cbf6..8b655a1c904e 100644 --- a/flang/lib/parser/basic-parsers.h +++ b/flang/lib/parser/basic-parsers.h @@ -1204,7 +1204,7 @@ public: constexpr WithinCharLiteral(const WithinCharLiteral &) = default; constexpr WithinCharLiteral(const PA &parser) : parser_{parser} {} std::optional Parse(ParseState *state) const { - bool was = state->set_inCharLiteral(true); + bool was{state->inCharLiteral()}; std::optional result{parser_.Parse(state)}; state->set_inCharLiteral(was); return result; diff --git a/flang/lib/parser/cooked-tokens.h b/flang/lib/parser/cooked-tokens.h index 030e08ab79b3..b358a1572556 100644 --- a/flang/lib/parser/cooked-tokens.h +++ b/flang/lib/parser/cooked-tokens.h @@ -249,7 +249,7 @@ template struct CharLiteral { using resultType = std::string; static std::optional Parse(ParseState *state) { std::string str; - CHECK(!state->set_inCharLiteral(true)); + CHECK(!state->inCharLiteral()); static constexpr auto nextch = attempt(CharLiteralChar{}); while (std::optional ch{nextch.Parse(state)}) { if (ch->ch == quote && !ch->wasEscaped) { @@ -395,7 +395,8 @@ struct HollerithLiteral { return {}; } std::string content; - CHECK(!state->set_inCharLiteral(true)); + CHECK(!state->inCharLiteral()); + state->set_inCharLiteral(true); for (auto j = *charCount; j-- > 0;) { std::optional ch{cookedNextChar.Parse(state)}; if (!ch || !isprint(*ch)) { diff --git a/flang/lib/parser/message.cc b/flang/lib/parser/message.cc index c8a2aa0082e3..9ee48de19700 100644 --- a/flang/lib/parser/message.cc +++ b/flang/lib/parser/message.cc @@ -3,16 +3,20 @@ namespace Fortran { namespace parser { -void Message::Emit(std::ostream &o, const AllSources &sources) const { - if (context_) { - context_->Emit(o, sources); +Provenance Message::Emit( + std::ostream &o, const AllSources &sources, bool echoSourceLine) const { + if (!context_ || context_->Emit(o, sources, false) != provenance_) { + sources.Identify(o, provenance_, "", echoSourceLine); } - sources.Identify(o, provenance_, ""); o << " " << message_ << '\n'; + return provenance_; } void Messages::Emit(std::ostream &o) const { for (const auto &msg : messages_) { + if (msg.context()) { + o << "In the context "; + } msg.Emit(o, allSources_); } } diff --git a/flang/lib/parser/message.h b/flang/lib/parser/message.h index b3a3f89f320e..f2718994a595 100644 --- a/flang/lib/parser/message.h +++ b/flang/lib/parser/message.h @@ -42,7 +42,8 @@ public: std::string message() const { return message_; } MessageContext context() const { return context_; } - void Emit(std::ostream &, const AllSources &) const; + Provenance Emit( + std::ostream &, const AllSources &, bool echoSourceLine = true) const; private: Provenance provenance_; diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc index c8215556c5dc..d9185ae193b8 100644 --- a/flang/lib/parser/preprocessor.cc +++ b/flang/lib/parser/preprocessor.cc @@ -58,7 +58,7 @@ TokenSequence Definition::Tokenize(const std::vector &argNames, if (IsIdentifierFirstCharacter(tok)) { auto it = args.find(tok.ToString()); if (it != args.end()) { - result.Put(it->second, token.GetProvenance(j)); + result.Put(it->second, token.GetTokenProvenance(j)); continue; } } @@ -111,7 +111,7 @@ TokenSequence Definition::Apply( size_t argBytes{args[index][k].size()}; for (size_t n{0}; n < argBytes; ++n) { char ch{arg[n]}; - Provenance from{args[index].GetProvenance(k, n)}; + Provenance from{args[index].GetTokenProvenance(k, n)}; if (ch == '"' || ch == '\\') { result.PutNextTokenChar(ch, from); } @@ -238,14 +238,19 @@ bool Preprocessor::MacroReplacement(const TokenSequence &input, if (!repl.empty()) { ProvenanceRange insert{allSources_->AddCompilerInsertion(repl)}; ProvenanceRange call{allSources_->AddMacroCall( - insert, input.GetProvenanceRange(j), repl)}; + insert, input.GetTokenProvenanceRange(j), repl)}; result->Put(repl, call.LocalOffsetToProvenance(0)); continue; } } def.set_isDisabled(true); - result->Put(ReplaceMacros(def.replacement(), prescanner)); + TokenSequence replaced{ReplaceMacros(def.replacement(), prescanner)}; def.set_isDisabled(false); + ProvenanceRange from{def.replacement().GetProvenanceRange()}; + ProvenanceRange use{input.GetTokenProvenanceRange(j)}; + ProvenanceRange newRange{ + allSources_->AddMacroCall(from, use, replaced.ToString())}; + result->Put(replaced, newRange); continue; } // Possible function-like macro call. Skip spaces and newlines to see @@ -284,16 +289,22 @@ bool Preprocessor::MacroReplacement(const TokenSequence &input, result->Put(input, j); continue; } - j = k; // advance to the terminal ')' std::vector args; - for (k = 0; k < argStart.size(); ++k) { - size_t at{argStart[k]}; - size_t count{(k + 1 == argStart.size() ? j : argStart[k + 1] - 1) - at}; + for (size_t n{0}; n < argStart.size(); ++n) { + size_t at{argStart[n]}; + size_t count{(n + 1 == argStart.size() ? k : argStart[n + 1] - 1) - at}; args.emplace_back(TokenSequence(input, at, count)); } def.set_isDisabled(true); - result->Put(ReplaceMacros(def.Apply(args, *allSources_), prescanner)); + TokenSequence replaced{ + ReplaceMacros(def.Apply(args, *allSources_), prescanner)}; def.set_isDisabled(false); + ProvenanceRange from{def.replacement().GetProvenanceRange()}; + ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)}; + ProvenanceRange newRange{ + allSources_->AddMacroCall(from, use, replaced.ToString())}; + result->Put(replaced, newRange); + j = k; // advance to the terminal ')' } return true; } @@ -521,7 +532,6 @@ bool Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) { prescanner->Complain("#include: missing name of file to include"); return false; } - ProvenanceRange includeDirRange{dir.GetProvenanceRange(j)}; std::string include; if (dir[j].ToString() == "<") { if (dir[tokens - 1].ToString() != ">") { @@ -549,7 +559,7 @@ bool Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) { return false; } ProvenanceRange fileRange{ - allSources_->AddIncludedFile(*included, includeDirRange)}; + allSources_->AddIncludedFile(*included, dir.GetProvenanceRange())}; return Prescanner{*prescanner}.Prescan(fileRange); } prescanner->Complain("#"s + dirName + ": unknown or unimplemented directive"); diff --git a/flang/lib/parser/provenance.cc b/flang/lib/parser/provenance.cc index 40b1bd4efe05..bdd67c18d9d0 100644 --- a/flang/lib/parser/provenance.cc +++ b/flang/lib/parser/provenance.cc @@ -125,8 +125,8 @@ ProvenanceRange AllSources::AddCompilerInsertion(std::string text) { return covers; } -void AllSources::Identify( - std::ostream &o, Provenance at, const std::string &prefix) const { +void AllSources::Identify(std::ostream &o, Provenance at, + const std::string &prefix, bool echoSourceLine) const { CHECK(IsValid(at)); static const std::string indented{prefix + " "}; const Origin &origin{MapToOrigin(at)}; @@ -135,11 +135,28 @@ void AllSources::Identify( [&](const Inclusion &inc) { size_t offset{origin.covers.ProvenanceToLocalOffset(at)}; std::pair pos{inc.source.FindOffsetLineAndColumn(offset)}; - o << prefix << "at line " << pos.first << ", column " << pos.second - << " in the " << (inc.isModule ? "module " : "file ") + o << prefix << "at line " << pos.first << ", column " << pos.second; + if (echoSourceLine) { + o << ":\n" << indented << " "; + const char *text{inc.source.content() + + inc.source.GetLineStartOffset(pos.first)}; + for (const char *p{text}; *p != '\n'; ++p) { + o << *p; + } + o << '\n' << indented << " "; + for (int j{1}; j < pos.second; ++j) { + char ch{text[j - 1]}; + o << (ch == '\t' ? '\t' : ' '); + } + o << "^\n" << prefix; + } else { + o << ' '; + } + o << "in the " << (inc.isModule ? "module " : "file ") << inc.source.path() << '\n'; if (IsValid(origin.replaces)) { - o << prefix << " that was " << (inc.isModule ? "used\n" : "included\n"); + o << prefix << " that was " + << (inc.isModule ? "used\n" : "included\n"); Identify(o, origin.replaces.LocalOffsetToProvenance(0), indented); } }, @@ -206,9 +223,8 @@ Provenance AllSources::CompilerInsertionProvenance(char ch) const { AllSources::Origin::Origin(ProvenanceRange r, const SourceFile &source) : u{Inclusion{source}}, covers{r} {} -AllSources::Origin::Origin( - ProvenanceRange r, const SourceFile &included, ProvenanceRange from, - bool isModule) +AllSources::Origin::Origin(ProvenanceRange r, const SourceFile &included, + ProvenanceRange from, bool isModule) : u{Inclusion{included, isModule}}, covers{r}, replaces{from} {} AllSources::Origin::Origin(ProvenanceRange r, ProvenanceRange def, ProvenanceRange use, const std::string &expansion) @@ -284,7 +300,9 @@ void AllSources::Dump(std::ostream &o) const { m.covers.Dump(o); o << " -> "; std::visit(visitors{[&](const Inclusion &inc) { - if (inc.isModule) { o << "module "; } + if (inc.isModule) { + o << "module "; + } o << "file " << inc.source.path(); }, [&](const Macro &mac) { o << "macro " << mac.expansion; }, diff --git a/flang/lib/parser/provenance.h b/flang/lib/parser/provenance.h index c674326a448b..4949d004902b 100644 --- a/flang/lib/parser/provenance.h +++ b/flang/lib/parser/provenance.h @@ -48,8 +48,9 @@ public: } Provenance operator+(size_t n) const { return {offset_ + n}; } bool operator<(Provenance that) const { return offset_ < that.offset_; } - bool operator<=(Provenance that) const { return offset_ <= that.offset_; } + bool operator<=(Provenance that) const { return !(that < *this); } bool operator==(Provenance that) const { return offset_ == that.offset_; } + bool operator!=(Provenance that) const { return !(*this == that); } size_t offset() const { return offset_; } private: @@ -159,8 +160,8 @@ public: std::string PopSearchPathDirectory(); const SourceFile *Open(std::string path, std::stringstream *error); - ProvenanceRange AddIncludedFile(const SourceFile &, ProvenanceRange, - bool isModule = false); + ProvenanceRange AddIncludedFile( + const SourceFile &, ProvenanceRange, bool isModule = false); ProvenanceRange AddMacroCall( ProvenanceRange def, ProvenanceRange use, const std::string &expansion); ProvenanceRange AddCompilerInsertion(std::string); @@ -169,7 +170,8 @@ public: bool IsValid(ProvenanceRange range) const { return range.size() > 0 && range_.Contains(range); } - void Identify(std::ostream &, Provenance, const std::string &prefix) const; + void Identify(std::ostream &, Provenance, const std::string &prefix, + bool echoSourceLine = false) const; const SourceFile *GetSourceFile(Provenance, size_t *offset = nullptr) const; ProvenanceRange GetContiguousRangeAround(ProvenanceRange) const; std::string GetPath(Provenance) const; // __FILE__ @@ -196,7 +198,7 @@ private: struct Origin { Origin(ProvenanceRange, const SourceFile &); Origin(ProvenanceRange, const SourceFile &, ProvenanceRange, - bool isModule = false); + bool isModule = false); Origin(ProvenanceRange, ProvenanceRange def, ProvenanceRange use, const std::string &expansion); Origin(ProvenanceRange, const std::string &); diff --git a/flang/lib/parser/source.h b/flang/lib/parser/source.h index 95e8521aeb0e..d28837d58510 100644 --- a/flang/lib/parser/source.h +++ b/flang/lib/parser/source.h @@ -29,6 +29,9 @@ public: bool Open(std::string path, std::stringstream *error); void Close(); std::pair FindOffsetLineAndColumn(size_t) const; + size_t GetLineStartOffset(int lineNumber) const { + return lineStart_.at(lineNumber - 1); + } private: std::string path_; diff --git a/flang/lib/parser/token-sequence.cc b/flang/lib/parser/token-sequence.cc index 99f7cd0fadab..5c2a2f9f40ad 100644 --- a/flang/lib/parser/token-sequence.cc +++ b/flang/lib/parser/token-sequence.cc @@ -47,6 +47,16 @@ void TokenSequence::Put(const TokenSequence &that) { provenances_.Put(that.provenances_); } +void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) { + size_t offset{0}; + for (size_t j{0}; j < that.size(); ++j) { + CharPointerWithLength tok{that[j]}; + Put(tok, range.LocalOffsetToProvenance(offset)); + offset += tok.size(); + } + CHECK(offset == range.size()); +} + void TokenSequence::Put(const TokenSequence &that, size_t at, size_t tokens) { ProvenanceRange provenance; size_t offset{0}; @@ -74,6 +84,7 @@ void TokenSequence::Put(const char *s, size_t bytes, Provenance provenance) { void TokenSequence::Put(const CharPointerWithLength &t, Provenance provenance) { Put(&t[0], t.size(), provenance); } + void TokenSequence::Put(const std::string &s, Provenance provenance) { Put(s.data(), s.size(), provenance); } @@ -104,15 +115,32 @@ std::string TokenSequence::ToString() const { return {&char_[0], char_.size()}; } -Provenance TokenSequence::GetProvenance(size_t token, size_t offset) const { +Provenance TokenSequence::GetTokenProvenance( + size_t token, size_t offset) const { ProvenanceRange range{provenances_.Map(start_[token] + offset)}; return range.LocalOffsetToProvenance(0); } -ProvenanceRange TokenSequence::GetProvenanceRange( +ProvenanceRange TokenSequence::GetTokenProvenanceRange( size_t token, size_t offset) const { ProvenanceRange range{provenances_.Map(start_[token] + offset)}; return range.Prefix(TokenBytes(token) - offset); } + +ProvenanceRange TokenSequence::GetIntervalProvenanceRange( + size_t token, size_t tokens) const { + if (tokens == 0) { + return {}; + } + ProvenanceRange range{provenances_.Map(start_[token])}; + while (--tokens > 0 && + range.AnnexIfPredecessor(provenances_.Map(start_[++token]))) { + } + return range; +} + +ProvenanceRange TokenSequence::GetProvenanceRange() const { + return GetIntervalProvenanceRange(0, start_.size()); +} } // namespace parser } // namespace Fortran diff --git a/flang/lib/parser/token-sequence.h b/flang/lib/parser/token-sequence.h index 324ac0fcafe3..b3e71ea03d2b 100644 --- a/flang/lib/parser/token-sequence.h +++ b/flang/lib/parser/token-sequence.h @@ -121,6 +121,7 @@ public: } void Put(const TokenSequence &); + void Put(const TokenSequence &, ProvenanceRange); void Put(const TokenSequence &, size_t at, size_t tokens = 1); void Put(const char *, size_t, Provenance); void Put(const CharPointerWithLength &, Provenance); @@ -128,8 +129,12 @@ public: void Put(const std::stringstream &, Provenance); void EmitWithCaseConversion(CookedSource *) const; std::string ToString() const; - Provenance GetProvenance(size_t token, size_t offset = 0) const; - ProvenanceRange GetProvenanceRange(size_t token, size_t offset = 0) const; + Provenance GetTokenProvenance(size_t token, size_t offset = 0) const; + ProvenanceRange GetTokenProvenanceRange( + size_t token, size_t offset = 0) const; + ProvenanceRange GetIntervalProvenanceRange( + size_t token, size_t tokens = 1) const; + ProvenanceRange GetProvenanceRange() const; private: size_t TokenBytes(size_t token) const { diff --git a/flang/tools/f18/f18.cc b/flang/tools/f18/f18.cc index 8045b7078f73..d7c27fd5fd48 100644 --- a/flang/tools/f18/f18.cc +++ b/flang/tools/f18/f18.cc @@ -138,11 +138,11 @@ int main(int argc, char *const argv[]) { Fortran::parser::ParseState state{cooked}; Fortran::parser::UserState ustate; state.set_inFixedForm(fixedForm) - .set_enableBackslashEscapesInCharLiterals(backslashEscapes) - .set_strictConformance(standard) - .set_columns(columns) - .set_enableOldDebugLines(enableOldDebugLines) - .set_userState(&ustate); + .set_enableBackslashEscapesInCharLiterals(backslashEscapes) + .set_strictConformance(standard) + .set_columns(columns) + .set_enableOldDebugLines(enableOldDebugLines) + .set_userState(&ustate); if (dumpCookedChars) { while (std::optional och{