maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[flang] Parser instrumentation and fail-fast experiment begun. 

Original-commit: flang-compiler/f18@ab46163d6e
Reviewed-on: https://github.com/flang-compiler/f18/pull/62
Tree-same-pre-rewrite: false
This commit is contained in:
peter klausler 2018-04-19 15:46:02 -07:00
parent 2c06be3fb8
commit 56b09e08eb
12 changed files with 154 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
flang/lib/parser/char-set.h
View File

@ -20,16 +20,16 @@ struct SetOfChars {
// This is basically the old DECSIX encoding, which maps the
// 7-bit ASCII codes [32..95] to [0..63]. Only '#', '&', '?', '\', and '^'
// in that range are unused in Fortran after preprocessing outside
// character literals. We repurpose '?' and '^' for newline and unknown
// character literals. We repurpose '^' and '?' for newline and unknown
// characters (resp.), leaving the others alone in case this code might
// be useful in preprocssing.
// TODO: EBCDIC?
if (c == '\n') {
// map newline to '?'
c = '?';
} else if (c < 32 || c >= 127) {
// map other control characters, DEL, and 8-bit characters to '^'
// map newline to '^'
c = '^';
} else if (c < 32 || c >= 127) {
// map other control characters, DEL, and 8-bit characters to '?'
c = '?';
} else if (c >= 96) {
// map lower-case letters to upper-case
c -= 32;

42
flang/lib/parser/grammar.h
View File

@ -1533,7 +1533,7 @@ TYPE_PARSER(construct<CommonBlockObject>{}(name, maybe(arraySpec)))
// each part is a name, maybe a (section-subscript-list), and
// maybe an [image-selector].
// If it's a substring, it ends with (substring-range).
TYPE_PARSER(
TYPE_CONTEXT_PARSER("designator"_en_US,
construct<Designator>{}(substring) || construct<Designator>{}(dataRef))
constexpr struct OldStructureComponentName {
@ -1564,9 +1564,10 @@ constexpr auto percentOrDot = "%"_tok ||
// that are NOPASS). However, Fortran constrains the use of a variable in a
// proc-component-ref to be a data-ref without coindices (C1027).
// Some array element references will be misrecognized as function references.
TYPE_PARSER(construct<Variable>{}(
indirect(functionReference / !"("_ch) / !percentOrDot) ||
construct<Variable>{}(indirect(designator)))
TYPE_CONTEXT_PARSER("variable"_en_US,
construct<Variable>{}(
indirect(functionReference / !"("_ch) / !percentOrDot) ||
construct<Variable>{}(indirect(designator)))
// R904 logical-variable -> variable
// Appears only as part of scalar-logical-variable.
@ -1755,20 +1756,21 @@ TYPE_PARSER("STAT =" >> construct<StatOrErrmsg>{}(statVariable) ||
// literal-constant | designator | array-constructor |
// structure-constructor | function-reference | type-param-inquiry |
// type-param-name | ( expr )
constexpr auto primary =
constexpr auto primary = instrumented("primary"_en_US,
construct<Expr>{}(indirect(Parser<CharLiteralConstantSubstring>{})) ||
construct<Expr>{}(literalConstant) ||
construct<Expr>{}(construct<Expr::Parentheses>{}(parenthesized(expr))) ||
construct<Expr>{}(indirect(functionReference) / !"("_tok) ||
construct<Expr>{}(designator / !"("_tok) ||
construct<Expr>{}(Parser<StructureConstructor>{}) ||
construct<Expr>{}(Parser<ArrayConstructor>{}) ||
construct<Expr>{}(indirect(Parser<TypeParamInquiry>{})) || // occulted
// PGI/XLF extension: COMPLEX constructor (x,y)
extension(construct<Expr>{}(parenthesized(
construct<Expr::ComplexConstructor>{}(expr, "," >> expr)))) ||
extension(construct<Expr>{}("%LOC" >>
parenthesized(construct<Expr::PercentLoc>{}(indirect(variable)))));
construct<Expr>{}(literalConstant) ||
construct<Expr>{}(
construct<Expr::Parentheses>{}(parenthesized(expr))) ||
construct<Expr>{}(indirect(functionReference) / !"("_tok) ||
construct<Expr>{}(designator / !"("_tok) ||
construct<Expr>{}(Parser<StructureConstructor>{}) ||
construct<Expr>{}(Parser<ArrayConstructor>{}) ||
construct<Expr>{}(indirect(Parser<TypeParamInquiry>{})) || // occulted
// PGI/XLF extension: COMPLEX constructor (x,y)
extension(construct<Expr>{}(parenthesized(
construct<Expr::ComplexConstructor>{}(expr, "," >> expr)))) ||
extension(construct<Expr>{}("%LOC" >>
parenthesized(construct<Expr::PercentLoc>{}(indirect(variable))))));
// R1002 level-1-expr -> [defined-unary-op] primary
// TODO: Reasonable extension: permit multiple defined-unary-ops
@ -3398,8 +3400,10 @@ TYPE_PARSER("INTRINSIC" >> maybe("::"_tok) >>
construct<IntrinsicStmt>{}(nonemptyList(name)))
// R1520 function-reference -> procedure-designator ( [actual-arg-spec-list] )
TYPE_PARSER(construct<FunctionReference>{}(construct<Call>{}(
Parser<ProcedureDesignator>{}, parenthesized(optionalList(actualArgSpec)))))
TYPE_CONTEXT_PARSER("function reference"_en_US,
construct<FunctionReference>{}(
construct<Call>{}(Parser<ProcedureDesignator>{},
parenthesized(optionalList(actualArgSpec)))))
// R1521 call-stmt -> CALL procedure-designator [( [actual-arg-spec-list] )]
TYPE_PARSER(construct<CallStmt>{}(

38
flang/lib/parser/instrumented-parser.cc
View File

@ -1,5 +1,6 @@
#include "instrumented-parser.h"
#include "message.h"
#include "provenance.h"
#include <map>
#include <ostream>
@ -12,21 +13,42 @@ bool operator<(const MessageFixedText &x, const MessageFixedText &y) {
return x.str() < y.str();
}
void ParsingLog::Note(const char *at, const MessageFixedText &tag, bool pass) {
bool ParsingLog::Fails(
const char *at, const MessageFixedText &tag, Messages &messages) {
std::size_t offset = reinterpret_cast<std::size_t>(at);
if (pass) {
++perPos_[offset].perTag[tag].passes;
auto posIter = perPos_.find(offset);
if (posIter == perPos_.end()) {
return false;
}
auto tagIter = posIter->second.perTag.find(tag);
if (tagIter == posIter->second.perTag.end()) {
return false;
}
auto &entry = tagIter->second;
++entry.count;
messages.Copy(entry.messages);
return !entry.pass;
}
void ParsingLog::Note(const char *at, const MessageFixedText &tag, bool pass,
const Messages &messages) {
std::size_t offset = reinterpret_cast<std::size_t>(at);
auto &entry = perPos_[offset].perTag[tag];
if (++entry.count == 1) {
entry.pass = pass;
entry.messages.Copy(messages);
} else {
++perPos_[offset].perTag[tag].failures;
CHECK(entry.pass == pass);
}
}
void ParsingLog::Dump(std::ostream &o) const {
void ParsingLog::Dump(std::ostream &o, const CookedSource &cooked) const {
for (const auto &posLog : perPos_) {
o << "at offset " << posLog.first << ":\n";
const char *at{reinterpret_cast<const char *>(posLog.first)};
for (const auto &tagLog : posLog.second.perTag) {
o << " " << tagLog.first.ToString() << ' ' << tagLog.second.passes
<< ", " << tagLog.second.failures << '\n';
Message{at, tagLog.first}.Emit(o, cooked, true);
o << " " << (tagLog.second.pass ? "pass" : "fail") << " "
<< tagLog.second.count << '\n';
}
}
}

32
flang/lib/parser/instrumented-parser.h
View File

@ -3,6 +3,7 @@
#include "message.h"
#include "parse-state.h"
#include "provenance.h"
#include "user-state.h"
#include <cstddef>
#include <map>
@ -13,16 +14,20 @@ namespace parser {
class ParsingLog {
public:
void Note(const char *at, const MessageFixedText &tag, bool pass);
void Dump(std::ostream &) const;
bool Fails(const char *at, const MessageFixedText &tag, Messages &);
void Note(
const char *at, const MessageFixedText &tag, bool pass, const Messages &);
void Dump(std::ostream &, const CookedSource &) const;
private:
struct LogForPosition {
struct Entries {
int passes{0};
int failures{0};
struct Entry {
Entry() {}
bool pass{true};
int count{0};
Messages messages;
};
std::map<MessageFixedText, Entries> perTag;
std::map<MessageFixedText, Entry> perTag;
};
std::map<std::size_t, LogForPosition> perPos_;
};
@ -34,14 +39,21 @@ public:
constexpr InstrumentedParser(const MessageFixedText &tag, const PA &parser)
: tag_{tag}, parser_{parser} {}
std::optional<resultType> Parse(ParseState *state) const {
const char *at{state->GetLocation()};
std::optional<resultType> result{parser_.Parse(state)};
if (UserState * ustate{state->userState()}) {
if (ParsingLog * log{ustate->log()}) {
log->Note(at, tag_, result.has_value());
const char *at{state->GetLocation()};
if (log->Fails(at, tag_, state->messages())) {
return {};
}
Messages messages{std::move(state->messages())};
std::optional<resultType> result{parser_.Parse(state)};
log->Note(at, tag_, result.has_value(), state->messages());
messages.Annex(state->messages());
state->messages() = std::move(messages);
return result;
}
}
return result;
return parser_.Parse(state);
}
private:

12
flang/lib/parser/message.cc
View File

@ -114,8 +114,14 @@ void Messages::Incorporate(Messages &that) {
}
}
void Messages::Emit(
std::ostream &o, const char *prefix, bool echoSourceLines) const {
void Messages::Copy(const Messages &that) {
for (const Message &m : that) {
Put(Message{m});
}
}
void Messages::Emit(std::ostream &o, const CookedSource &cooked,
const char *prefix, bool echoSourceLines) const {
for (const auto &msg : messages_) {
if (prefix) {
o << prefix;
@ -123,7 +129,7 @@ void Messages::Emit(
if (msg.context()) {
o << "In the context ";
}
msg.Emit(o, cooked_, echoSourceLines);
msg.Emit(o, cooked, echoSourceLines);
}
}

24
flang/lib/parser/message.h
View File

@ -95,7 +95,9 @@ public:
using Context = CountedReference<Message>;
Message() {}
Message(const Message &) = default;
Message(Message &&) = default;
Message &operator=(const Message &that) = default;
Message &operator=(Message &&that) = default;
// TODO: Change these to cover ranges of provenance
@ -161,9 +163,8 @@ public:
using iterator = listType::iterator;
using const_iterator = listType::const_iterator;
explicit Messages(const CookedSource &cooked) : cooked_{cooked} {}
Messages(Messages &&that)
: cooked_{that.cooked_}, messages_{std::move(that.messages_)} {
Messages() {}
Messages(Messages &&that) : messages_{std::move(that.messages_)} {
if (!messages_.empty()) {
last_ = that.last_;
that.last_ = that.messages_.before_begin();
@ -188,18 +189,7 @@ public:
const_iterator cbegin() const { return messages_.cbegin(); }
const_iterator cend() const { return messages_.cend(); }
const CookedSource &cooked() const { return cooked_; }
bool IsValidLocation(const Message &m) {
if (auto p{m.cookedSourceLocation()}) {
return cooked_.IsValid(p);
} else {
return cooked_.IsValid(m.provenance());
}
}
Message &Put(Message &&m) {
CHECK(IsValidLocation(m));
last_ = messages_.emplace_after(last_, std::move(m));
return *last_;
}
@ -218,14 +208,14 @@ public:
}
void Incorporate(Messages &);
void Copy(const Messages &);
void Emit(std::ostream &, const char *prefix = nullptr,
bool echoSourceLines = true) const;
void Emit(std::ostream &, const CookedSource &cooked,
const char *prefix = nullptr, bool echoSourceLines = true) const;
bool AnyFatalError() const;
private:
const CookedSource &cooked_;
listType messages_;
iterator last_{messages_.before_begin()};
};

9
flang/lib/parser/parse-state.h
View File

@ -27,12 +27,11 @@ class ParseState {
public:
// TODO: Add a constructor for parsing a normalized module file.
ParseState(const CookedSource &cooked)
: p_{&cooked[0]}, limit_{p_ + cooked.size()}, messages_{cooked} {}
: p_{&cooked[0]}, limit_{p_ + cooked.size()} {}
ParseState(const ParseState &that)
: p_{that.p_}, limit_{that.limit_}, messages_{that.messages_.cooked()},
context_{that.context_}, userState_{that.userState_},
inFixedForm_{that.inFixedForm_}, encoding_{that.encoding_},
strictConformance_{that.strictConformance_},
: p_{that.p_}, limit_{that.limit_}, context_{that.context_},
userState_{that.userState_}, inFixedForm_{that.inFixedForm_},
encoding_{that.encoding_}, strictConformance_{that.strictConformance_},
warnOnNonstandardUsage_{that.warnOnNonstandardUsage_},
warnOnDeprecatedUsage_{that.warnOnDeprecatedUsage_},
anyErrorRecovery_{that.anyErrorRecovery_},

10
flang/lib/parser/parsing.cc
View File

@ -74,11 +74,13 @@ void Parsing::DumpCookedChars(std::ostream &out) const {
void Parsing::DumpProvenance(std::ostream &out) const { cooked_.Dump(out); }
void Parsing::DumpParsingLog(std::ostream &out) const { log_.Dump(out); }
void Parsing::DumpParsingLog(std::ostream &out) const {
log_.Dump(out, cooked_);
}
void Parsing::Parse() {
UserState userState;
if (options_.instrumentedParse) {
if (options_.instrumentedParse || true /*pmk*/) {
userState.set_log(&log_);
}
ParseState parseState{cooked_};
@ -98,12 +100,12 @@ void Parsing::Parse() {
bool Parsing::ForTesting(std::string path, std::ostream &err) {
Prescan(path, Options{});
if (messages_.AnyFatalError()) {
messages_.Emit(err);
messages_.Emit(err, cooked_);
err << "could not scan " << path << '\n';
return false;
}
Parse();
messages_.Emit(err);
messages_.Emit(err, cooked_);
if (!consumedWholeFile_) {
err << "f18 parser FAIL; final position: ";
Identify(err, finalRestingPlace_, " ");

3
flang/lib/parser/parsing.h
View File

@ -37,6 +37,7 @@ public:
bool consumedWholeFile() const { return consumedWholeFile_; }
const char *finalRestingPlace() const { return finalRestingPlace_; }
CookedSource &cooked() { return cooked_; }
Messages &messages() { return messages_; }
std::optional<Program> &parseTree() { return parseTree_; }
@ -58,7 +59,7 @@ private:
Options options_;
AllSources allSources_;
CookedSource cooked_{allSources_};
Messages messages_{cooked_};
Messages messages_;
bool consumedWholeFile_{false};
const char *finalRestingPlace_{nullptr};
std::optional<Program> parseTree_;

4
flang/lib/semantics/resolve-names.cc
View File

@ -1097,11 +1097,11 @@ void ResolveNamesVisitor::Post(const parser::Program &) {
void ResolveNames(
parser::Program &program, const parser::CookedSource &cookedSource) {
parser::Messages messages{cookedSource};
parser::Messages messages;
ResolveNamesVisitor visitor{messages};
parser::Walk(static_cast<const parser::Program &>(program), visitor);
if (!messages.empty()) {
messages.Emit(std::cerr);
messages.Emit(std::cerr, cookedSource);
return;
}
RewriteParseTree(program);

86
flang/tools/f18/f18.cc
View File

@ -44,8 +44,7 @@ void MeasureParseTree(const Fortran::parser::Program &program) {
MeasurementVisitor visitor;
Fortran::parser::Walk(program, visitor);
std::cout << "Parse tree comprises " << visitor.objects
<< " objects and occupies " << visitor.bytes
<< " total bytes.\n";
<< " objects and occupies " << visitor.bytes << " total bytes.\n";
}
std::vector<std::string> filesToDelete;
@ -84,8 +83,7 @@ bool ParentProcess() {
}
int childStat{0};
wait(&childStat);
if (!WIFEXITED(childStat) ||
WEXITSTATUS(childStat) != 0) {
if (!WIFEXITED(childStat) || WEXITSTATUS(childStat) != 0) {
exit(EXIT_FAILURE);
}
return true;
@ -100,8 +98,8 @@ void Exec(std::vector<char *> &argv, bool verbose = false) {
}
argv.push_back(nullptr);
execvp(argv[0], &argv[0]);
std::cerr << "execvp(" << argv[0] << ") failed: "
<< std::strerror(errno) << '\n';
std::cerr << "execvp(" << argv[0] << ") failed: " << std::strerror(errno)
<< '\n';
exit(EXIT_FAILURE);
}
@ -136,8 +134,8 @@ std::string RelocatableName(const DriverOptions &driver, std::string path) {
return relo;
}
std::string CompileFortran(std::string path, Fortran::parser::Options options,
DriverOptions &driver) {
std::string CompileFortran(
std::string path, Fortran::parser::Options options, DriverOptions &driver) {
if (!driver.forcedForm) {
auto dot = path.rfind(".");
if (dot != std::string::npos) {
@ -150,7 +148,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
if (!parsing.messages().empty() &&
(driver.warningsAreErrors || parsing.messages().AnyFatalError())) {
std::cerr << driver.prefix << "could not scan " << path << '\n';
parsing.messages().Emit(std::cerr, driver.prefix);
parsing.messages().Emit(std::cerr, parsing.cooked(), driver.prefix);
exit(EXIT_FAILURE);
}
if (driver.dumpProvenance) {
@ -166,14 +164,14 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
parsing.DumpParsingLog(std::cout);
return {};
}
parsing.messages().Emit(std::cerr, driver.prefix);
parsing.messages().Emit(std::cerr, parsing.cooked(), driver.prefix);
if (!parsing.consumedWholeFile()) {
std::cerr << "f18 parser FAIL; final position: ";
parsing.Identify(std::cerr, parsing.finalRestingPlace(), " ");
exit(EXIT_FAILURE);
}
if ((!parsing.messages().empty() &&
(driver.warningsAreErrors || parsing.messages().AnyFatalError())) ||
(driver.warningsAreErrors || parsing.messages().AnyFatalError())) ||
!parsing.parseTree().has_value()) {
std::cerr << driver.prefix << "could not parse " << path << '\n';
exit(EXIT_FAILURE);
@ -182,8 +180,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
MeasureParseTree(*parsing.parseTree());
}
if (driver.debugResolveNames || driver.dumpSymbols) {
Fortran::semantics::ResolveNames(
*parsing.parseTree(), parsing.messages().cooked());
Fortran::semantics::ResolveNames(*parsing.parseTree(), parsing.cooked());
if (driver.dumpSymbols) {
Fortran::semantics::DumpSymbols(std::cout);
}
@ -192,8 +189,8 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
Fortran::parser::DumpTree(*parsing.parseTree());
}
if (driver.dumpUnparse) {
Unparse(std::cout, *parsing.parseTree(), driver.encoding,
true /*capitalize*/);
Unparse(
std::cout, *parsing.parseTree(), driver.encoding, true /*capitalize*/);
return {};
}
if (driver.parseOnly) {
@ -204,8 +201,9 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
char tmpSourcePath[32];
std::snprintf(tmpSourcePath, sizeof tmpSourcePath, "/tmp/f18-%lx.f90",
static_cast<unsigned long>(getpid()));
{ std::ofstream tmpSource;
static_cast<unsigned long>(getpid()));
{
std::ofstream tmpSource;
tmpSource.open(tmpSourcePath);
Unparse(tmpSource, *parsing.parseTree(), driver.encoding);
}
@ -284,9 +282,8 @@ int main(int argc, char *const argv[]) {
driver.pgf90Args.push_back(arg);
} else {
std::string suffix{arg.substr(dot + 1)};
if (suffix == "f" || suffix == "F" ||
suffix == "f90" || suffix == "F90" ||
suffix == "cuf" || suffix == "CUF" ||
if (suffix == "f" || suffix == "F" || suffix == "f90" ||
suffix == "F90" || suffix == "cuf" || suffix == "CUF" ||
suffix == "f18" || suffix == "F18") {
fortranSources.push_back(arg);
} else if (suffix == "o" || suffix == "a") {
@ -349,31 +346,34 @@ int main(int argc, char *const argv[]) {
if (eq == std::string::npos) {
options.predefinitions.emplace_back(arg.substr(2), "1");
} else {
options.predefinitions.emplace_back(arg.substr(2, eq-2),
arg.substr(eq+1));
options.predefinitions.emplace_back(
arg.substr(2, eq - 2), arg.substr(eq + 1));
}
} else if (arg.substr(0, 2) == "-U") {
options.predefinitions.emplace_back(arg.substr(2), std::optional<std::string>{});
} else if (arg.substr(0, 2) == "-U") {
options.predefinitions.emplace_back(
arg.substr(2), std::optional<std::string>{});
} else if (arg == "-help" || arg == "--help" || arg == "-?") {
std::cerr << "f18 options:\n"
<< " -Mfixed | -Mfree force the source form\n"
<< " -Mextend 132-column fixed form\n"
<< " -M[no]backslash disable[enable] \\escapes in literals\n"
<< " -Mstandard enable conformance warnings\n"
<< " -Mx,125,4 set bit 2 in xflag[125] (all Kanji mode)\n"
<< " -Werror treat warnings as errors\n"
<< " -ed enable fixed form D lines\n"
<< " -E prescan & preprocess only\n"
<< " -fparse-only parse only, no output except messages\n"
<< " -funparse parse & reformat only, no code generation\n"
<< " -fdebug-measure-parse-tree\n"
<< " -fdebug-dump-provenance\n"
<< " -fdebug-dump-parse-tree\n"
<< " -fdebug-resolve-names\n"
<< " -fdebug-instrumented-parse\n"
<< " -v -c -o -I -D -U have their usual meanings\n"
<< " -help print this again\n"
<< "Other options are passed through to the compiler.\n";
std::cerr
<< "f18 options:\n"
<< " -Mfixed | -Mfree force the source form\n"
<< " -Mextend 132-column fixed form\n"
<< " -M[no]backslash disable[enable] \\escapes in literals\n"
<< " -Mstandard enable conformance warnings\n"
<< " -Mx,125,4 set bit 2 in xflag[125] (all Kanji mode)\n"
<< " -Werror treat warnings as errors\n"
<< " -ed enable fixed form D lines\n"
<< " -E prescan & preprocess only\n"
<< " -fparse-only parse only, no output except messages\n"
<< " -funparse parse & reformat only, no code "
"generation\n"
<< " -fdebug-measure-parse-tree\n"
<< " -fdebug-dump-provenance\n"
<< " -fdebug-dump-parse-tree\n"
<< " -fdebug-resolve-names\n"
<< " -fdebug-instrumented-parse\n"
<< " -v -c -o -I -D -U have their usual meanings\n"
<< " -help print this again\n"
<< "Other options are passed through to the compiler.\n";
return EXIT_SUCCESS;
} else if (arg == "-V") {
std::cerr << "\nf18 compiler (under development)\n";

2
flang/tools/f18/test-sema.cc
View File

@ -24,7 +24,7 @@ int main(int argc, char *const argv[]) {
std::string path{argv[1]};
Parsing parsing;
if (parsing.ForTesting(path, std::cerr)) {
DoSemanticAnalysis(parsing.messages().cooked(), *parsing.parseTree());
DoSemanticAnalysis(parsing.cooked(), *parsing.parseTree());
return EXIT_SUCCESS;
}
return EXIT_FAILURE;