maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[flang] Forge provenances for compiler-inserted text. 

Original-commit: flang-compiler/f18@dbdd01aa25
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false
This commit is contained in:
peter klausler 2018-02-12 14:43:16 -08:00 committed by GitHub
parent 8e585de36b
commit 97a7c501dc
5 changed files with 73 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
flang/lib/parser/preprocessor.cc
View File

@ -132,8 +132,9 @@ Definition::Definition(const std::vector<std::string> &argNames,
argumentCount_(argNames.size()), isVariadic_{isVariadic}, argumentCount_(argNames.size()), isVariadic_{isVariadic},
replacement_{Tokenize(argNames, repl, firstToken, tokens)} {} replacement_{Tokenize(argNames, repl, firstToken, tokens)} {}
Definition::Definition(const std::string &predefined) Definition::Definition(const std::string &predefined, AllSources *sources)
: isPredefined_{true}, replacement_{predefined} {} : isPredefined_{true}, replacement_{predefined,
sources->AddCompilerInsertion(predefined).start} {}
bool Definition::set_isDisabled(bool disable) { bool Definition::set_isDisabled(bool disable) {
bool was{isDisabled_}; bool was{isDisabled_};
@ -172,7 +173,8 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
return result; return result;
} }
TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) { TokenSequence Definition::Apply(
const std::vector<TokenSequence> &args, const Prescanner &prescanner) {
TokenSequence result; TokenSequence result;
bool pasting{false}; bool pasting{false};
bool skipping{false}; bool skipping{false};
@ -208,7 +210,8 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
while (result.size() >= afterLastNonBlank) { while (result.size() >= afterLastNonBlank) {
result.pop_back(); result.pop_back();
} }
result.PutNextTokenChar('"', 0); // TODO provenance result.PutNextTokenChar(
'"', prescanner.CompilerInsertionProvenance('"'));
for (size_t k{0}; k < argTokens; ++k) { for (size_t k{0}; k < argTokens; ++k) {
const CharPointerWithLength &arg{args[index][k]}; const CharPointerWithLength &arg{args[index][k]};
size_t argBytes{args[index][k].size()}; size_t argBytes{args[index][k].size()};
@ -221,7 +224,8 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
result.PutNextTokenChar(ch, from); result.PutNextTokenChar(ch, from);
} }
} }
result.PutNextTokenChar('"', 0); // TODO provenance result.PutNextTokenChar(
'"', prescanner.CompilerInsertionProvenance('"'));
result.CloseToken(); result.CloseToken();
} else { } else {
for (size_t k{0}; k < argTokens; ++k) { for (size_t k{0}; k < argTokens; ++k) {
@ -247,7 +251,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
token.ToString() == "__VA_ARGS__") { token.ToString() == "__VA_ARGS__") {
for (size_t k{argumentCount_}; k < args.size(); ++k) { for (size_t k{argumentCount_}; k < args.size(); ++k) {
if (k > argumentCount_) { if (k > argumentCount_) {
result.Put(","s, 0); // TODO provenance result.Put(","s, prescanner.CompilerInsertionProvenance(','));
} }
result.Put(args[k]); result.Put(args[k]);
} }
@ -283,13 +287,15 @@ Preprocessor::Preprocessor(Prescanner &ps) : prescanner_{ps} {
// of __DATE__ or __TIME__ change during compilation. // of __DATE__ or __TIME__ change during compilation.
std::time_t now; std::time_t now;
std::time(&now); std::time(&now);
definitions_.emplace(SaveToken("__DATE__"s), // e.g., "Jun 16 1904" definitions_.emplace(SaveTokenAsName("__DATE__"s), // e.g., "Jun 16 1904"
Definition{FormatTime(now, "\"%h %e %Y\""), 0, 1}); Definition{FormatTime(now, "\"%h %e %Y\""), ps.allSources()});
definitions_.emplace(SaveToken("__TIME__"s), // e.g., "23:59:60" definitions_.emplace(SaveTokenAsName("__TIME__"s), // e.g., "23:59:60"
Definition{FormatTime(now, "\"%T\""), 0, 1}); Definition{FormatTime(now, "\"%T\""), ps.allSources()});
// The values of these predefined macros depend on their invocation sites. // The values of these predefined macros depend on their invocation sites.
definitions_.emplace(SaveToken("__FILE__"s), Definition{"__FILE__"s}); definitions_.emplace(
definitions_.emplace(SaveToken("__LINE__"s), Definition{"__LINE__"s}); SaveTokenAsName("__FILE__"s), Definition{"__FILE__"s, ps.allSources()});
definitions_.emplace(
SaveTokenAsName("__LINE__"s), Definition{"__LINE__"s, ps.allSources()});
} }
bool Preprocessor::MacroReplacement( bool Preprocessor::MacroReplacement(
@ -328,17 +334,21 @@ bool Preprocessor::MacroReplacement(
if (def.isPredefined()) { if (def.isPredefined()) {
std::string name{def.replacement()[0].ToString()}; std::string name{def.replacement()[0].ToString()};
if (name == "__FILE__") { if (name == "__FILE__") {
result->Put("\""s + std::string f{"\""s +
prescanner_.allSources().GetPath( prescanner_.allSources()->GetPath(
prescanner_.GetCurrentProvenance()) + prescanner_.GetCurrentProvenance()) +
'"'); '"'};
result->Put(
f, prescanner_.allSources()->AddCompilerInsertion(f).start);
continue; continue;
} }
if (name == "__LINE__") { if (name == "__LINE__") {
std::stringstream ss; std::stringstream ss;
ss << prescanner_.allSources().GetLineNumber( ss << prescanner_.allSources()->GetLineNumber(
prescanner_.GetCurrentProvenance()); prescanner_.GetCurrentProvenance());
result->Put(ss.str()); std::string s{ss.str()};
result->Put(
s, prescanner_.allSources()->AddCompilerInsertion(s).start);
continue; continue;
} }
} }
@ -391,7 +401,7 @@ bool Preprocessor::MacroReplacement(
args.emplace_back(TokenSequence(input, at, count)); args.emplace_back(TokenSequence(input, at, count));
} }
def.set_isDisabled(true); def.set_isDisabled(true);
result->Put(ReplaceMacros(def.Apply(args))); result->Put(ReplaceMacros(def.Apply(args, prescanner_)));
def.set_isDisabled(false); def.set_isDisabled(false);
} }
return true; return true;
@ -478,7 +488,7 @@ bool Preprocessor::Directive(const TokenSequence &dir) {
Complain("#define: missing or invalid name"); Complain("#define: missing or invalid name");
return false; return false;
} }
nameToken = SaveToken(nameToken); nameToken = SaveTokenAsName(nameToken);
definitions_.erase(nameToken); definitions_.erase(nameToken);
if (++j < tokens && dir[j].size() == 1 && dir[j][0] == '(') { if (++j < tokens && dir[j].size() == 1 && dir[j][0] == '(') {
j = SkipBlanks(dir, j + 1, tokens); j = SkipBlanks(dir, j + 1, tokens);
@ -615,7 +625,8 @@ bool Preprocessor::Directive(const TokenSequence &dir) {
return false; return false;
} }
CharPointerWithLength Preprocessor::SaveToken(const CharPointerWithLength &t) { CharPointerWithLength Preprocessor::SaveTokenAsName(
const CharPointerWithLength &t) {
names_.push_back(t.ToString()); names_.push_back(t.ToString());
return {names_.back().data(), names_.back().size()}; return {names_.back().data(), names_.back().size()};
} }
@ -652,7 +663,7 @@ bool Preprocessor::SkipDisabledConditionalCode(
} }
void Preprocessor::Complain(const std::string &message) { void Preprocessor::Complain(const std::string &message) {
prescanner_.messages().Put({prescanner_.GetCurrentProvenance(), message}); prescanner_.messages()->Put({prescanner_.GetCurrentProvenance(), message});
} }
// Precedence level codes used here to accommodate mixed Fortran and C: // Precedence level codes used here to accommodate mixed Fortran and C:
@ -661,7 +672,7 @@ void Preprocessor::Complain(const std::string &message) {
// 13: ** // 13: **
// 12: *, /, % (modulus) // 12: *, /, % (modulus)
// 11: + and - // 11: + and -
// 0: << and >> // 10: << and >>
// 9: bitwise & // 9: bitwise &
// 8: bitwise ^ // 8: bitwise ^
// 7: bitwise | // 7: bitwise |
@ -942,7 +953,8 @@ bool Preprocessor::IsIfPredicateTrue(
name = expr1[j++]; name = expr1[j++];
} }
if (!name.empty()) { if (!name.empty()) {
expr2.Put(IsNameDefined(name) ? "1" : "0", 1, 0); // TODO provenance char truth{IsNameDefined(name) ? '1' : '0'};
expr2.Put(&truth, 1, prescanner_.CompilerInsertionProvenance(truth));
continue; continue;
} }
} }

9
flang/lib/parser/preprocessor.h
View File

@ -91,7 +91,7 @@ public:
: start_{std::move(that.start_)}, nextStart_{that.nextStart_}, : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
char_{std::move(that.char_)}, provenances_{std::move(that.provenances_)} { char_{std::move(that.char_)}, provenances_{std::move(that.provenances_)} {
} }
TokenSequence(const std::string &s) { Put(s, 0); } // TODO predefined prov. TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
TokenSequence &operator=(const TokenSequence &that) { TokenSequence &operator=(const TokenSequence &that) {
clear(); clear();
@ -159,7 +159,7 @@ public:
Definition(const TokenSequence &, size_t firstToken, size_t tokens); Definition(const TokenSequence &, size_t firstToken, size_t tokens);
Definition(const std::vector<std::string> &argNames, const TokenSequence &, Definition(const std::vector<std::string> &argNames, const TokenSequence &,
size_t firstToken, size_t tokens, bool isVariadic = false); size_t firstToken, size_t tokens, bool isVariadic = false);
explicit Definition(const std::string &predefined); Definition(const std::string &predefined, AllSources *);
bool isFunctionLike() const { return isFunctionLike_; } bool isFunctionLike() const { return isFunctionLike_; }
size_t argumentCount() const { return argumentCount_; } size_t argumentCount() const { return argumentCount_; }
@ -170,7 +170,8 @@ public:
bool set_isDisabled(bool disable); bool set_isDisabled(bool disable);
TokenSequence Apply(const std::vector<TokenSequence> &args); TokenSequence Apply(
const std::vector<TokenSequence> &args, const Prescanner &);
private: private:
static TokenSequence Tokenize(const std::vector<std::string> &argNames, static TokenSequence Tokenize(const std::vector<std::string> &argNames,
@ -203,7 +204,7 @@ private:
enum class CanDeadElseAppear { No, Yes }; enum class CanDeadElseAppear { No, Yes };
void Complain(const std::string &); void Complain(const std::string &);
CharPointerWithLength SaveToken(const CharPointerWithLength &); CharPointerWithLength SaveTokenAsName(const CharPointerWithLength &);
bool IsNameDefined(const CharPointerWithLength &); bool IsNameDefined(const CharPointerWithLength &);
TokenSequence ReplaceMacros(const TokenSequence &); TokenSequence ReplaceMacros(const TokenSequence &);
bool SkipDisabledConditionalCode(const std::string &dirName, IsElseActive); bool SkipDisabledConditionalCode(const std::string &dirName, IsElseActive);

30
flang/lib/parser/prescan.cc
View File

@ -9,14 +9,23 @@
namespace Fortran { namespace Fortran {
namespace parser { namespace parser {
CookedSource Prescanner::Prescan(AllSources *allSources) { Prescanner::Prescanner(Messages *messages, AllSources *allSources)
: messages_{messages}, allSources_{allSources}, preprocessor_{*this} {
std::string compilerInserts{" ,\"01"};
ProvenanceRange range{allSources->AddCompilerInsertion(compilerInserts)};
for (size_t j{0}; j < compilerInserts.size(); ++j) {
compilerInsertionProvenance_[compilerInserts[j]] = range.start + j;
}
}
CookedSource Prescanner::Prescan() {
startProvenance_ = 0; startProvenance_ = 0;
start_ = &(*allSources)[0]; start_ = &(*allSources_)[0];
limit_ = start_ + allSources->size(); limit_ = start_ + allSources_->size();
lineStart_ = start_; lineStart_ = start_;
BeginSourceLine(start_); BeginSourceLine(start_);
TokenSequence tokens, preprocessed; TokenSequence tokens, preprocessed;
CookedSource cooked{allSources}; CookedSource cooked{allSources_};
while (lineStart_ < limit_) { while (lineStart_ < limit_) {
if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) { if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) {
break; break;
@ -68,6 +77,10 @@ std::optional<TokenSequence> Prescanner::NextTokenizedLine() {
return {std::move(tokens)}; return {std::move(tokens)};
} }
Provenance Prescanner::CompilerInsertionProvenance(char ch) const {
return compilerInsertionProvenance_.find(ch)->second;
}
void Prescanner::NextLine() { void Prescanner::NextLine() {
void *vstart{static_cast<void *>(const_cast<char *>(lineStart_))}; void *vstart{static_cast<void *>(const_cast<char *>(lineStart_))};
void *v{std::memchr(vstart, '\n', limit_ - lineStart_)}; void *v{std::memchr(vstart, '\n', limit_ - lineStart_)};
@ -98,8 +111,9 @@ void Prescanner::LabelField(TokenSequence *token) {
token->CloseToken(); token->CloseToken();
} }
if (outCol < 7) { if (outCol < 7) {
Provenance provenance{CompilerInsertionProvenance(' ')};
for (; outCol < 7; ++outCol) { for (; outCol < 7; ++outCol) {
EmitChar(token, ' '); token->PutNextTokenChar(' ', provenance);
} }
token->CloseToken(); token->CloseToken();
} }
@ -192,7 +206,7 @@ bool Prescanner::NextToken(TokenSequence *tokens) {
inCharLiteral_ = true; inCharLiteral_ = true;
while (n-- > 0) { while (n-- > 0) {
if (PadOutCharacterLiteral()) { if (PadOutCharacterLiteral()) {
EmitChar(tokens, ' '); tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
} else { } else {
if (*at_ == '\n') { if (*at_ == '\n') {
break; // TODO error break; // TODO error
@ -283,12 +297,12 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) {
do { do {
EmitCharAndAdvance(tokens, *at_); EmitCharAndAdvance(tokens, *at_);
while (PadOutCharacterLiteral()) { while (PadOutCharacterLiteral()) {
EmitChar(tokens, ' '); tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
} }
if (*at_ == '\\' && enableBackslashEscapesInCharLiterals_) { if (*at_ == '\\' && enableBackslashEscapesInCharLiterals_) {
EmitCharAndAdvance(tokens, '\\'); EmitCharAndAdvance(tokens, '\\');
while (PadOutCharacterLiteral()) { while (PadOutCharacterLiteral()) {
EmitChar(tokens, ' '); tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
} }
} else if (*at_ == quote) { } else if (*at_ == quote) {
// A doubled quote mark becomes a single instance of the quote character // A doubled quote mark becomes a single instance of the quote character

17
flang/lib/parser/prescan.h
View File

@ -12,6 +12,7 @@
#include "preprocessor.h" #include "preprocessor.h"
#include "provenance.h" #include "provenance.h"
#include "source.h" #include "source.h"
#include <map>
#include <optional> #include <optional>
#include <string> #include <string>
@ -20,10 +21,9 @@ namespace parser {
class Prescanner { class Prescanner {
public: public:
explicit Prescanner(Messages &messages) Prescanner(Messages *, AllSources *);
: messages_{messages}, preprocessor_{*this} {}
Messages &messages() const { return messages_; } Messages *messages() const { return messages_; }
bool anyFatalErrors() const { return anyFatalErrors_; } bool anyFatalErrors() const { return anyFatalErrors_; }
Prescanner &set_fixedForm(bool yes) { Prescanner &set_fixedForm(bool yes) {
@ -43,13 +43,12 @@ public:
return *this; return *this;
} }
const AllSources &allSources() const { return messages_.allSources(); } AllSources *allSources() const { return allSources_; }
CookedSource Prescan(AllSources *); CookedSource Prescan();
std::optional<TokenSequence> NextTokenizedLine(); std::optional<TokenSequence> NextTokenizedLine();
Provenance GetCurrentProvenance() const { return GetProvenance(at_); } Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
std::string GetCurrentPath() const; // __FILE__ Provenance CompilerInsertionProvenance(char ch) const;
int GetCurrentLineNumber() const; // __LINE__
private: private:
void BeginSourceLine(const char *at) { void BeginSourceLine(const char *at) {
@ -97,7 +96,8 @@ private:
bool FreeFormContinuation(); bool FreeFormContinuation();
void PayNewlineDebt(CookedSource *); void PayNewlineDebt(CookedSource *);
Messages &messages_; Messages *messages_;
AllSources *allSources_;
Provenance startProvenance_; Provenance startProvenance_;
const char *start_{nullptr}; // beginning of sourceFile_ content const char *start_{nullptr}; // beginning of sourceFile_ content
@ -118,6 +118,7 @@ private:
bool enableBackslashEscapesInCharLiterals_{true}; bool enableBackslashEscapesInCharLiterals_{true};
int delimiterNesting_{0}; int delimiterNesting_{0};
Preprocessor preprocessor_; Preprocessor preprocessor_;
std::map<char, Provenance> compilerInsertionProvenance_;
}; };
} // namespace parser } // namespace parser
} // namespace Fortran } // namespace Fortran

4
flang/tools/f18/f18.cc
View File

@ -106,13 +106,13 @@ int main(int argc, char *const argv[]) {
Fortran::parser::AllSources allSources{sourceFile}; Fortran::parser::AllSources allSources{sourceFile};
Fortran::parser::Messages messages{allSources}; Fortran::parser::Messages messages{allSources};
Fortran::parser::Prescanner prescanner{messages}; Fortran::parser::Prescanner prescanner{&messages, &allSources};
Fortran::parser::CookedSource cooked{ Fortran::parser::CookedSource cooked{
prescanner.set_fixedForm(fixedForm) prescanner.set_fixedForm(fixedForm)
.set_enableBackslashEscapesInCharLiterals(backslashEscapes) .set_enableBackslashEscapesInCharLiterals(backslashEscapes)
.set_fixedFormColumnLimit(columns) .set_fixedFormColumnLimit(columns)
.set_enableOldDebugLines(enableOldDebugLines) .set_enableOldDebugLines(enableOldDebugLines)
.Prescan(&allSources)}; .Prescan()};
messages.Emit(std::cerr); messages.Emit(std::cerr);
if (prescanner.anyFatalErrors()) { if (prescanner.anyFatalErrors()) {
return 1; return 1;